diff --git a/.coveragerc b/.coveragerc index 661a219f10b..fda7d19fdba 100644 --- a/.coveragerc +++ b/.coveragerc @@ -29,21 +29,14 @@ debug= #include note= omit= - */lib/cherrypy/* - */lib/cylc/gui/* */lib/cylc/tests/* - */lib/cylc/cylc_xdot.py */lib/cylc/profiler.py - */lib/cylc/cfgspec/gcylc.py - */lib/cylc/cfgspec/gscan.py */lib/cylc/profiling/* - */lib/cylc/mkdir_p.py */lib/parsec/OrderedDictCompat.py */lib/isodatetime/* */lib/jinja2/* */lib/markupsafe/* */lib/parsec/tests/* - */lib/xdot.py parallel = True plugins= source= @@ -64,21 +57,14 @@ fail_under=0 ignore_errors = False include= omit= - */lib/cherrypy/* - */lib/cylc/gui/* */lib/cylc/tests/* - */lib/cylc/cylc_xdot.py */lib/cylc/profiler.py - */lib/cylc/cfgspec/gcylc.py - */lib/cylc/cfgspec/gscan.py */lib/cylc/profiling/* - */lib/cylc/mkdir_p.py */lib/parsec/OrderedDictCompat.py */lib/isodatetime/* */lib/jinja2/* */lib/markupsafe/* */lib/parsec/tests/* - */lib/xdot.py partial_branches= precision=2 show_missing=False diff --git a/.gitignore b/.gitignore index 69006591523..b57bd80f9bc 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ # python bytecode -*.pyc +*.py[cod] +__pycache__/ # Generated documentation files (from 'cylc make-docs') # - Command reference from called script 'custom/make-commands.sh'. diff --git a/.travis.yml b/.travis.yml index 67ff2fcd1ec..ee55aeb55ab 100644 --- a/.travis.yml +++ b/.travis.yml @@ -25,63 +25,93 @@ # More information for Travis can be found at http://docs.travis-ci.com/user/getting-started/ --- -language: python cache: pip -virtualenv: - # we need system packages in Travis-CI's virtualenv in order to access pygtk, installed via apt-get - system_site_packages: true dist: xenial addons: apt: - # instruct travis-ci to always run apt-get before each build - update: true + update: true # instruct travis-ci to always run apt-get before each build + +language: python stages: - unit-test -- test +- functional-test +- documentation env: global: - - PATH="${TRAVIS_BUILD_DIR}/bin:$PATH" - # Only run the generic tests on Travis CI. - - CYLC_TEST_RUN_PLATFORM=false - # Custom diff command to ignore Xlib errors (xvfb has not RANDR extension). - - CYLC_TEST_DIFF_CMD="diff -I Xlib -u" - # This coverage RC file is created under the script task - - COVERAGE_PROCESS_START="/tmp/.coveragerc" - matrix: - - CHUNK="1/4" - - CHUNK="2/4" - - CHUNK="3/4" - - CHUNK="4/4" + - PATH="${TRAVIS_BUILD_DIR}/bin:$PATH" + # Only run the generic tests on Travis CI. + - CYLC_TEST_RUN_PLATFORM=false + # Custom diff command to ignore Xlib errors (xvfb has not RANDR extension). + - CYLC_TEST_DIFF_CMD="diff -I Xlib -u" + # This coverage RC file is created under the script task + - COVERAGE_PROCESS_START="/tmp/.coveragerc" + +unittest: &unittest + stage: unit-test + install: + - .travis/install.sh unit-tests + script: + - export PYTHONPATH=$(pwd -P)/lib/ + - pycodestyle + - pytest --cov-append --cov=lib/cylc --cov=lib/parsec + after_success: + # Report metrics, such as coverage + - coverage combine --append + - coverage xml --ignore-errors + - bash <(curl -s https://codecov.io/bash) -# this is the default test stage, which is used for functional-tests -install: .travis/install.sh functional-tests docs -script: -- export PYTHONPATH="${TRAVIS_BUILD_DIR}/.travis" -# When we run cylc commands, there are processes being forked, that get a -# new working directory. As .coveragerc contains relatives paths, it fails -# to produce the correct coverage, unless we use absolute paths. The `sed` -# call below tries to define the data_file, and sources locations for Travis. -- sed -e "s|data_file=.coverage|data_file=${TRAVIS_BUILD_DIR}/.coverage|g; s|./bin|${TRAVIS_BUILD_DIR}/bin|g; s|./lib|${TRAVIS_BUILD_DIR}/lib|g" .coveragerc > /tmp/.coveragerc -# And some tests fail if we touch files in the git working directory, due -# to Cylc's version appearing with the "dirty" suffix. To avoid this, we -# are using a new coveragerc created under the temporary directory. -- coverage run .travis/cover.py -- unset PYTHONPATH -after_script: .travis/after_script.sh -after_success: -# Report metrics, such as coverage -- coverage combine --append -- coverage xml --ignore-errors -- bash <(curl -s https://codecov.io/bash) +regressiontest: ®ressiontest + stage: functional-test + python: + - "3.7" + install: + - .travis/install.sh functional-tests docs + script: + - export PYTHONPATH="${TRAVIS_BUILD_DIR}/.travis" + - .travis/coverage-data.sh + - coverage run .travis/cover.py + - unset PYTHONPATH + after_failure: + - .travis/after_failure.sh + after_success: + # Report metrics, such as coverage + - coverage combine --append + - coverage xml --ignore-errors + - bash <(curl -s https://codecov.io/bash) jobs: include: - - stage: unit-test + + # run unittests at each supported Python version + - <<: *unittest + python: "3.6" + - <<: *unittest + python: "3.7" + + # run regression tests in chunks with the most contemporary + # supported Python version + - <<: *regressiontest + python: "3.7" + env: CHUNK="1/4" + - <<: *regressiontest + python: "3.7" + env: CHUNK="2/4" + - <<: *regressiontest + python: "3.7" + env: CHUNK="3/4" + - <<: *regressiontest + python: "3.7" + env: CHUNK="4/4" + + # TODO: run shortlist of regression tests with the + # least contemporary Python version + + # build documentation + - stage: documentation + python: "3.7" install: - - .travis/install.sh unit-tests functional-tests + - .travis/install.sh docs script: - - pycodestyle --ignore=E402,W503,W504 lib/cylc lib/Jinja2Filters/*.py lib/parsec/*.py $(grep -l '#!.*\' bin/*) - - PYTHONPATH=$(pwd -P)/lib/ pytest --cov-append --cov=lib/cylc --cov=lib/parsec - after_script: true + - cylc make-docs diff --git a/.travis/after_script.sh b/.travis/after_failure.sh similarity index 97% rename from .travis/after_script.sh rename to .travis/after_failure.sh index 42f25bed20d..80685aadbb5 100755 --- a/.travis/after_script.sh +++ b/.travis/after_failure.sh @@ -20,5 +20,7 @@ # Check output (more useful if you narrow down what tests get run) +cylc scan -f --color=never + find $HOME/cylc-run -name '*.err' -type f -exec echo '==== {} ====' \; -exec cat '{}' \; find /tmp/${USER}/cylctb-* -type f -exec echo '==== {} ====' \; -exec cat '{}' \; diff --git a/.travis/cover.py b/.travis/cover.py index dcaf6969f03..bf7c351584a 100644 --- a/.travis/cover.py +++ b/.travis/cover.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 # THIS FILE IS PART OF THE CYLC SUITE ENGINE. # Copyright (C) 2008-2019 NIWA & British Crown (Met Office) & Contributors. diff --git a/bin/gcontrol b/.travis/coverage-data.sh similarity index 52% rename from bin/gcontrol rename to .travis/coverage-data.sh index 8cb501acf87..a3db37d7b44 100755 --- a/bin/gcontrol +++ b/.travis/coverage-data.sh @@ -16,6 +16,11 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -echo "WARNING: \"gcontrol\" is deprecated - it is now called \"gcylc\"" -echo "WARNING: This command alias will eventually be deleted." -$(dirname $0)/cylc gui "$@" +# When we run cylc commands, there are processes being forked, that get a +# new working directory. As .coveragerc contains relatives paths, it fails +# to produce the correct coverage, unless we use absolute paths. The `sed` +# call below tries to define the data_file, and sources locations for Travis. +sed -e "s|data_file=.coverage|data_file=${TRAVIS_BUILD_DIR}/.coverage|g; s|./bin|${TRAVIS_BUILD_DIR}/bin|g; s|./lib|${TRAVIS_BUILD_DIR}/lib|g" .coveragerc > /tmp/.coveragerc +# And some tests fail if we touch files in the git working directory, due +# to Cylc's version appearing with the "dirty" suffix. To avoid this, we +# are using a new coveragerc created under the temporary directory. diff --git a/.travis/install.sh b/.travis/install.sh index c333e5f3d38..9e633aecd74 100755 --- a/.travis/install.sh +++ b/.travis/install.sh @@ -24,27 +24,23 @@ shopt -s extglob args=("$@") -# pygtk via apt-get, necessary for both unit and functional tests -sudo apt-get install graphviz libgraphviz-dev python-gtk2-dev heirloom-mailx -# coverage dependencies -pip install coverage pytest-cov mock - -# install dependencies required for running unit tests -if grep 'unit-tests' <<< "${args[@]}"; then - pip install EmPy pyopenssl pycodestyle pytest mock +if grep -E '(unit-tests|functional-tests)' <<< "${args[@]}"; then + sudo apt-get install heirloom-mailx + # coverage dependencies + pip install coverage pytest-cov + # common Cylc reqirements + pip install colorama python-jose zmq fi -# install dependencies required for running functional tests -if grep 'functional-tests' <<< "${args[@]}"; then - # pygraphviz needs special treatment to avoid an error from "from . import release" - pip install EmPy pyopenssl - pip install pygraphviz \ - --install-option="--include-path=/usr/include/graphviz" \ - --install-option="--library-path=/usr/lib/graphviz/" +if grep 'unit-tests' <<< "${args[@]}"; then + pip install pycodestyle pytest + # TODO: EmPy removed from testing, see: #2958 fi -# install dependencies required for building documentation, only when instructed to do so +# install dependencies required for building documentation if grep 'docs' <<< "${args[@]}$"; then + pip install sphinx + # for PDF output via LaTeX builder sudo apt-get install texlive-latex-base fi diff --git a/.travis/sitecustomize.py b/.travis/sitecustomize.py index 1f0b29abcd6..350e7ac0022 100644 --- a/.travis/sitecustomize.py +++ b/.travis/sitecustomize.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 # THIS FILE IS PART OF THE CYLC SUITE ENGINE. # Copyright (C) 2008-2019 NIWA & British Crown (Met Office) & Contributors. @@ -16,8 +16,9 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -# This file is used by Travis-CI to start the coverage process. In order to make -# Cylc and Python aware of it, we export PYTHONPATH when running the tests +# This file is used by Travis-CI to start the coverage process. +# In order to make Cylc and Python aware of it, we export PYTHONPATH when +# running the tests. import coverage coverage.process_startup() diff --git a/ACKNOWLEDGEMENT.md b/ACKNOWLEDGEMENT.md index ab8be5ddfb2..d3604a4a88b 100644 --- a/ACKNOWLEDGEMENT.md +++ b/ACKNOWLEDGEMENT.md @@ -2,10 +2,6 @@ Licences for non-cylc work included in this distribution can be found in the `licences/` directory. - * `lib/cherrypy/`: - External software library released under a BSD license. - Minor modification to ignore an import warning. - See [cherrypy](http://www.cherrypy.org/). * `lib/isodatetime/`: Unmodified external software library released under the LGPL license. See [metomi/isodatetime](https://github.com/metomi/isodatetime/). @@ -15,10 +11,6 @@ Licences for non-cylc work included in this distribution can be found in the * `lib/markupsafe/`: External software library released under a BSD license, used by Jinja2. See [MarkupSafe](http://www.pocoo.org/projects/markupsafe/). - * `lib/xdot.py`: - External software released under the LGPL license. - Modifications based on version 0.6. See - [xdot](https://github.com/jrfonseca/xdot.py) * `lib/cylc/cylc-review/static/css/bootstrap-???`, `lib/cylc/cylc-review/static/images/glyphicons-halflings-???` & `lib/cylc/cylc-review/static/js/bootstrap.min.js`: diff --git a/CHANGES.md b/CHANGES.md index 1c2dee3d48b..78f7f9d17d2 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -229,7 +229,7 @@ warnings emitted by `urllib3` under old Python versions (2.6). *We are aware of the security issues, but these warnings serve no purpose on affected platforms except to confuse and annoy users.* -[#2676](https://github.com/cylc/cylc/pull/2676) - use `#!/usr/bin/env python2` +[#2676](https://github.com/cylc/cylc/pull/2676) - use `#!/usr/bin/env python3` (i.e. Python-2 specific) in Cylc source files, to avoid issues with default Python 3 installations (note Cylc is going to Python 3 next year) diff --git a/INSTALL.md b/INSTALL.md index 52634d17929..4b411820259 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -63,39 +63,27 @@ $ cylc check-software Checking your software... Individual results: -=============================================================================== -Package (version requirements) Outcome (version found) -=============================================================================== - *REQUIRED SOFTWARE* -Python (2.6+, <3).....................FOUND & min. version MET (2.7.12.final.0) - - *OPTIONAL SOFTWARE for the GUI & dependency graph visualisation* -Python:pygtk (2.0+)...........................FOUND & min. version MET (2.24.0) -graphviz (any)...................................................FOUND (2.38.0) -Python:pygraphviz (any)...........................................FOUND (1.3.1) - - *OPTIONAL SOFTWARE for the HTML User Guide* -ImageMagick (any)...............................................FOUND (6.8.9-9) - - *OPTIONAL SOFTWARE for the HTTPS communications layer* -Python:urllib3 (any).............................................FOUND (1.13.1) -Python:OpenSSL (any).............................................FOUND (17.2.0) -Python:requests (2.4.2+).......................FOUND & min. version MET (2.9.1) - - *OPTIONAL SOFTWARE for the LaTeX User Guide* -TeX:framed (any)....................................................FOUND (n/a) -TeX (3.0+)................................FOUND & min. version MET (3.14159265) -TeX:preprint (any)..................................................FOUND (n/a) -TeX:tex4ht (any)....................................................FOUND (n/a) -TeX:tocloft (any)...................................................FOUND (n/a) -TeX:texlive (any)...................................................FOUND (n/a) -=============================================================================== +================================================================================ +Package (version requirements) Outcome (version found) +================================================================================ + *REQUIRED SOFTWARE* +Python (3+).............................FOUND & min. version MET (3.7.2.final.0) +Python:zmq (any)..................................................FOUND (17.1.2) +Python:jose (any)..................................................FOUND (2.0.2) +Python:colorama (any)..............................................FOUND (0.4.1) + + *OPTIONAL SOFTWARE for the configuration templating* +Python:EmPy (any)..................................................FOUND (3.3.2) + + *OPTIONAL SOFTWARE for the HTML documentation* +Python:sphinx (1.5.3+)..........................FOUND & min. version MET (1.8.4) +================================================================================ Summary: - **************************** - Core requirements: ok - Full-functionality: ok - **************************** + **************************** + Core requirements: ok + Full-functionality: ok + **************************** ``` ### Installing The Documentation diff --git a/bin/cylc b/bin/cylc index d756fc18d4f..970efe9694d 100755 --- a/bin/cylc +++ b/bin/cylc @@ -140,7 +140,7 @@ __HELP__ } category_help_command() { -# Special case for printing categories and the gcylc gui +# Special case for printing categories local COMMAND="${CYLC_HOME_BIN}/cylc-help" exec "${COMMAND}" "categories" } diff --git a/bin/cylc-broadcast b/bin/cylc-broadcast index a4d42050216..b1bf638a743 100755 --- a/bin/cylc-broadcast +++ b/bin/cylc-broadcast @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 # THIS FILE IS PART OF THE CYLC SUITE ENGINE. # Copyright (C) 2008-2019 NIWA & British Crown (Met Office) & Contributors. @@ -86,7 +86,7 @@ from cylc.broadcast_report import ( from cylc.cfgspec.suite import SPEC, upg from cylc.cfgvalidate import cylc_config_validate import cylc.flags -from cylc.network.httpclient import SuiteRuntimeServiceClient +from cylc.network.client import SuiteRuntimeClient from cylc.option_parsers import CylcOptionParser as COP from cylc.print_tree import print_tree from cylc.task_id import TaskID @@ -146,7 +146,7 @@ def files_to_settings(settings, setting_files, cancel_mode=False): for setting_file in setting_files: if setting_file == '-': with NamedTemporaryFile() as handle: - handle.write(sys.stdin.read()) + handle.write(sys.stdin.read().encode()) handle.seek(0, 0) cfg.loadcfg(handle.name) else: @@ -248,10 +248,9 @@ def main(): options, args = parser.parse_args() suite = args[0] - pclient = SuiteRuntimeServiceClient( + pclient = SuiteRuntimeClient( suite, options.owner, options.host, options.port, - options.comms_timeout, my_uuid=options.set_uuid, - print_uuid=options.print_uuid) + options.comms_timeout) if options.show or options.showtask: if options.showtask: @@ -259,29 +258,33 @@ def main(): TaskID.split(options.showtask) except ValueError: parser.error("TASKID must be " + TaskID.SYNTAX) - settings = pclient.get_broadcast(task_id=options.showtask) + settings = pclient('get_broadcast', {'task_id': options.showtask}) padding = get_padding(settings) * ' ' if options.raw: - print str(settings) + print(str(settings)) else: print_tree(settings, padding, options.unicode) sys.exit(0) if options.clear: - modified_settings, bad_options = pclient.clear_broadcast({ - "point_strings": options.point_strings, - "namespaces": options.namespaces}) + modified_settings, bad_options = pclient( + 'clear_broadcast', + {'point_strings': options.point_strings, + 'namespaces': options.namespaces} + ) if modified_settings: - print get_broadcast_change_report( - modified_settings, is_cancel=True) + print(get_broadcast_change_report( + modified_settings, is_cancel=True)) sys.exit(get_broadcast_bad_options_report(bad_options)) if options.expire: - modified_settings, bad_options = pclient.expire_broadcast( - cutoff=options.expire) + modified_settings, bad_options = pclient( + 'expire_broadcast', + {'cutoff': options.expire} + ) if modified_settings: - print get_broadcast_change_report( - modified_settings, is_cancel=True) + print(get_broadcast_change_report( + modified_settings, is_cancel=True)) sys.exit(get_broadcast_bad_options_report(bad_options)) # implement namespace and cycle point defaults here @@ -302,12 +305,15 @@ def main(): setting = get_rdict(option_item) settings.append(setting) files_to_settings(settings, options.cancel_files, options.cancel) - modified_settings, bad_options = pclient.clear_broadcast({ - "point_strings": point_strings, "namespaces": namespaces, - "cancel_settings": settings}) + modified_settings, bad_options = pclient( + 'clear_broadcast', + {'point_strings': point_strings, + 'namespaces': namespaces, + 'cancel_settings': settings} + ) if modified_settings: - print get_broadcast_change_report( - modified_settings, is_cancel=True) + print(get_broadcast_change_report( + modified_settings, is_cancel=True)) sys.exit(get_broadcast_bad_options_report(bad_options)) if options.settings or options.setting_files: @@ -320,10 +326,14 @@ def main(): setting = get_rdict(lhs, rhs) settings.append(setting) files_to_settings(settings, options.setting_files) - modified_settings, bad_options = pclient.put_broadcast({ - "point_strings": point_strings, "namespaces": namespaces, - "settings": settings}) - print get_broadcast_change_report(modified_settings) + modified_settings, bad_options = pclient( + 'put_broadcast', + {'point_strings': point_strings, + 'namespaces': namespaces, + 'settings': settings + } + ) + print(get_broadcast_change_report(modified_settings)) sys.exit(get_broadcast_bad_options_report(bad_options, is_set=True)) diff --git a/bin/cylc-cat-log b/bin/cylc-cat-log index 5fa74f0bb07..20aacc8db25 100755 --- a/bin/cylc-cat-log +++ b/bin/cylc-cat-log @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 # THIS FILE IS PART OF THE CYLC SUITE ENGINE. # Copyright (C) 2008-2019 NIWA & British Crown (Met Office) & Contributors. @@ -32,8 +32,8 @@ If remote job logs are retrieved to the suite host on completion (global config '[JOB-HOST]retrieve job logs = True') and the job is not currently running, the local (retrieved) log will be accessed unless '-o/--force-remote' is used. -Custom job logs (written to $CYLC_TASK_LOG_DIR on the job host) are available -from the GUI if listed in 'extra log files' in the suite definition. The file +Custom job logs (written to $CYLC_TASK_LOG_DIR on the job host) can be +listed in 'extra log files' in the suite definition. The file name must be given here, but can be discovered with '--mode=l' (list-dir). The correct cycle point format of the suite must be for task job logs. @@ -51,7 +51,7 @@ if remrun(abort_if='edit', forward_x11=True): import os import shlex from glob import glob -from pipes import quote +from shlex import quote from stat import S_IRUSR from tempfile import mkstemp from subprocess import Popen, PIPE @@ -81,7 +81,7 @@ from cylc.task_job_logs import ( # => on host C: cat-log spawns subprocess # tail -f .out # -# Then Ctrl-C (or exit GUI log viewer) on host-A: +# Then Ctrl-C (or exit log viewer) on host-A: # => ssh from A to B dies # => on B: cat-log detects the previous, # and kills its ssh subprocess to C @@ -130,7 +130,7 @@ def view_log(logpath, mode, tailer_tmpl, batchview_cmd=None, remote=False): logpath = os.path.expandvars(logpath) if mode == 'print': # Print location even if the suite does not exist yet. - print logpath + print(logpath) return 0 elif not os.path.exists(logpath) and batchview_cmd is None: # Note: batchview_cmd may not need to have access to logpath, so don't @@ -138,11 +138,11 @@ def view_log(logpath, mode, tailer_tmpl, batchview_cmd=None, remote=False): sys.stderr.write('ERROR: file not found: %s\n' % logpath) return 1 elif mode == 'print-dir': - print os.path.dirname(logpath) + print(os.path.dirname(logpath)) return 0 elif mode == 'list-dir': for entry in sorted(os.listdir(os.path.dirname(logpath))): - print entry + print(entry) return 0 elif not remote and mode == 'edit': # Copy the log to a temporary read-only file for viewing in editor. @@ -192,7 +192,7 @@ def get_option_parser(): "-m", "--mode", help="Mode: %s. Default c(cat)." % ( ', '.join(['%s(%s)' % (i, j) for i, j in MODES.items()])), - action="store", choices=MODES.keys() + MODES.values(), default='c', + action="store", choices=list(MODES.keys()) + list(MODES.values()), default='c', dest="mode") parser.add_option( @@ -266,7 +266,7 @@ def tmpfile_edit(tmpfile, geditor=False): cmd = shlex.split(editor) cmd.append(tmpfile) proc = Popen(cmd, stderr=PIPE) - err = proc.communicate()[1] + err = proc.communicate()[1].decode() ret_code = proc.wait() if ret_code == 0: if os.stat(tmpfile).st_mtime > modtime1: diff --git a/bin/cylc-cat-state b/bin/cylc-cat-state index 808a3177cc0..1a1d296e07f 100755 --- a/bin/cylc-cat-state +++ b/bin/cylc-cat-state @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 # THIS FILE IS PART OF THE CYLC SUITE ENGINE. # Copyright (C) 2008-2019 NIWA & British Crown (Met Office) & Contributors. @@ -27,16 +27,17 @@ if remrun(): sys.exit(0) import os -import pickle +import json import re import sqlite3 import traceback from cylc.cfgspec.glbl_cfg import glbl_cfg -from cylc.dump import dump_to_stdout, get_stop_state_summary +from cylc.dump import dump_to_stdout import cylc.flags from cylc.option_parsers import CylcOptionParser as COP from cylc.rundb import CylcSuiteDAO +from cylc.task_id import TaskID from cylc.wallclock import get_unix_time_from_time_string @@ -56,26 +57,21 @@ def main(): suite = args[0] try: - lines = _get_state_lines(suite) + state = get_state(suite) except sqlite3.Error: if cylc.flags.debug: traceback.print_exc() - try: - lines = _get_state_lines_compat(suite) - except IOError: - if cylc.flags.debug: - traceback.print_exc() - sys.exit("ERROR: cannot print suite state for %s" % suite) + sys.exit("ERROR: cannot print suite state for %s" % suite) if options.dumpform: - dump_to_stdout(get_stop_state_summary(lines)[1]) + dump_to_stdout(extract_json(state)) else: - for line in lines: - print line + for line in extract_lines(state): + print(line) -def _get_state_lines(suite): - """Get state lines from suite runtime DB.""" +def get_state(suite): + """Extract suite state and task_pool from the DB""" dao = CylcSuiteDAO( os.path.join( glbl_cfg().get_derived_host_item(suite, 'suite run directory'), @@ -99,18 +95,37 @@ def _get_state_lines(suite): dao.select_task_pool( lambda row_idx, row: _callback_task_pool(data, row)) - lines = [ - r"run mode : %(run_mode)s" % data, - r"time : %(time_str)s (%(time_since_epoch)s)" % data, - r"initial cycle : %(initial_point)s" % data, - r"final cycle : %(final_point)s" % data] - lines += pickle.dumps(data["broadcast_states"]).splitlines() - lines.append(r"Begin task states") - for item in data["task_pool"]: - lines.append( + return data + + +def extract_json(state): + """Map task pool DB fields onto cat-state output fields.""" + return { + TaskID.get(task['name'], task['cycle']): { + out_key: task[db_key] for out_key, db_key in ( + ('name', 'name'), + ('point', 'cycle'), + ('label', 'cycle'), + ('state', 'status'), + ('spawned', 'spawned') + ) + } for task in state['task_pool'] + } + + +def extract_lines(state): + """Extract textoral representation of the state.""" + yield r"run mode : %(run_mode)s" % state + yield r"time : %(time_str)s (%(time_since_epoch)s)" % state + yield r"initial cycle : %(initial_point)s" % state + yield r"final cycle : %(final_point)s" % state + yield json.dumps(state["broadcast_states"]).splitlines() + yield "Begin task states" + for item in state["task_pool"]: + yield ( r"%(name)s.%(cycle)s : status=%(status)s, spawned=%(spawned)s" % - item) - return lines + item + ) def _callback_checkpoint_id(data, row): @@ -146,16 +161,5 @@ def _callback_task_pool(data, row): "spawned": bool(spawned)}) -def _get_state_lines_compat(suite): - """Read old state file from normal location, for backward compat.""" - state_file_path = os.path.join( - glbl_cfg().get_derived_host_item(suite, 'suite run directory'), - "state", "state") - lines = [] - for line in open(state_file_path, 'rb').readlines(): - lines.append(line.rstrip()) - return lines - - if __name__ == "__main__": main() diff --git a/bin/cylc-check-software b/bin/cylc-check-software index 39e365649eb..48f46bf2988 100755 --- a/bin/cylc-check-software +++ b/bin/cylc-check-software @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 # THIS FILE IS PART OF THE CYLC SUITE ENGINE. # Copyright (C) 2008-2019 NIWA & British Crown (Met Office) & Contributors. @@ -49,29 +49,20 @@ latter grouped as Python, TeX or 'other' (neither). 'opt_spec' item format: : [, , , <'OTHER' TUPLE>] with <'OTHER' TUPLE> = ([], , , ). """ -req_py_ver_range = (2, 6), (3,) +req_py_ver_range = ((3,6),) opt_spec = { 'EmPy': [None, 'TEMPLATING', 'PY'], - 'requests': [(2, 4, 2), 'HTTPSCOMMS', 'PY'], - 'OpenSSL': [None, 'HTTPSCOMMS', 'PY'], - 'urllib3': [None, 'HTTPSCOMMS', 'PY'], 'sphinx': [(1, 5, 3), 'HTMLDOCS', 'PY'], - 'pygtk': [(2, 0), 'GUIORGRAPH', 'PY'], - 'pygraphviz': [None, 'GUIORGRAPH', 'PY'], - 'graphviz': [None, 'GUIORGRAPH', 'OTHER', - (['dot'], '-V', r'graphviz version ([^\s]+)', 2)], } # All possible module reqs to accept as arguments, as above or all lower case. -module_args = ['Python'] + opt_spec.keys() +module_args = ['Python'] + list(opt_spec) upper_case_conv = dict( (upper.lower(), upper) for upper in module_args if upper.lower() != upper) # Package-dep. functionality dict; item format : func_tags_and_text = { 'TEMPLATING': 'configuration templating', - 'HTTPSCOMMS': 'HTTPS communications layer', - 'GUIORGRAPH': 'GUI & dependency graph visualisation', 'HTMLDOCS': 'HTML documentation', } @@ -154,10 +145,7 @@ def check_py_module_ver(module, min_ver, write=True): msg = 'Python:%s (%s)' % (module, string_ver(min_ver) + '+' if min_ver is not None else 'any') try: - if module == 'pygtk': - import gtk - module_version = '.'.join(map(str, gtk.pygtk_version)) - elif module == 'EmPy': + if module == 'EmPy': # we want the 'em' module, but there is more than one out there # we want the one that provides 'Interpreter' from em import Interpreter @@ -217,8 +205,8 @@ def cmd_find_ver( else: try: output = Popen([cmd, ver_opt], stdin=open(os.devnull), - stdout=PIPE, - stderr=PIPE).communicate()[outfile - 1].strip() + stdout=PIPE, stderr=PIPE + ).communicate()[outfile - 1].decode().strip() version = re.search(ver_extr, output).groups()[0] try_next_cmd = False if min_ver is None: @@ -301,7 +289,12 @@ def main(): # Individual results section in mock-table format. shell_centre_write(False, '*REQUIRED SOFTWARE*') - req_result = check_py_ver(*req_py_ver_range) + req_result = ( + check_py_ver(*req_py_ver_range) + and check_py_module_ver('zmq', None) + and check_py_module_ver('jose', None) + and check_py_module_ver('colorama', None) + ) for tag, text in func_tags_and_text.items(): shell_centre_write(True, '*OPTIONAL SOFTWARE for the ' + text + '*') functionality_print(tag) @@ -320,6 +313,6 @@ def main(): if __name__ == '__main__': if 'help' in sys.argv or '--help' in sys.argv: - print __doc__ + print(__doc__) else: main() diff --git a/bin/cylc-check-triggering b/bin/cylc-check-triggering index 46704147402..43c47b7039f 100755 --- a/bin/cylc-check-triggering +++ b/bin/cylc-check-triggering @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 # THIS FILE IS PART OF THE CYLC SUITE ENGINE. # Copyright (C) 2008-2019 NIWA & British Crown (Met Office) & Contributors. @@ -51,10 +51,10 @@ import sys from cylc.log_diagnosis import LogAnalyser if len(sys.argv) == 2 and sys.argv[1] == '--help': - print __doc__ + print(__doc__) sys.exit(0) -print "\nThis is the cylc check-triggering shutdown event handler" +print("\nThis is the cylc check-triggering shutdown event handler") event, suite = sys.argv[1], sys.argv[2] @@ -64,8 +64,8 @@ if event != 'shutdown': try: log_dir = os.path.expandvars(os.environ['CYLC_SUITE_LOG_DIR']) suite_dir = os.path.expandvars(os.environ['CYLC_SUITE_DEF_PATH']) -except KeyError, x: - raise SystemExit(x) +except KeyError as exc: + raise SystemExit(exc) new_log = os.path.join(log_dir, 'log') ref_log = os.path.join(suite_dir, 'reference.log') @@ -73,8 +73,8 @@ ref_log = os.path.join(suite_dir, 'reference.log') try: lanal = LogAnalyser(new_log, ref_log) lanal.verify_triggering() -except Exception, x: - print >> sys.stderr, x +except Exception as exc: + print(exc, file=sys.stderr) raise SystemExit("ERROR: Triggering check FAILED") else: - print "Triggering check passed" + print("Triggering check passed") diff --git a/bin/cylc-check-versions b/bin/cylc-check-versions index 6b93d0a1de8..9c408755c09 100755 --- a/bin/cylc-check-versions +++ b/bin/cylc-check-versions @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 # THIS FILE IS PART OF THE CYLC SUITE ENGINE. # Copyright (C) 2008-2019 NIWA & British Crown (Met Office) & Contributors. @@ -111,30 +111,32 @@ def main(): argv += ["--host=%s" % host] user_at_host = host if verbose: - print "%s: %s" % (user_at_host, ' '.join(argv)) + print("%s: %s" % (user_at_host, ' '.join(argv))) proc = Popen(argv, stdin=open(os.devnull), stdout=PIPE, stderr=PIPE) out, err = proc.communicate() + out = out.decode() + err = err.decode() if proc.wait() == 0: if verbose: - print " %s" % out + print(" %s" % out) contacted += 1 out = out.strip() if out != CYLC_VERSION: warn[user_at_host] = out else: - print >> sys.stderr, 'ERROR ' + user_at_host + ':' - print >> sys.stderr, err + print('ERROR ' + user_at_host + ':', file=sys.stderr) + print(err, file=sys.stderr) # report results if not warn: if contacted: - print "All", contacted, "accounts have cylc-" + CYLC_VERSION + print("All", contacted, "accounts have cylc-" + CYLC_VERSION) else: - print "WARNING: failed to invoke cylc-%s on %d accounts:" % ( - CYLC_VERSION, len(warn.keys())) + print("WARNING: failed to invoke cylc-%s on %d accounts:" % ( + CYLC_VERSION, len(warn))) m = max(len(ac) for ac in warn) for ac, warning in warn.items(): - print ' ', ac.ljust(m), warning + print(' ', ac.ljust(m), warning) if options.error: sys.exit(1) diff --git a/bin/cylc-checkpoint b/bin/cylc-checkpoint index dcfcd0cd2fb..427565badf4 100755 --- a/bin/cylc-checkpoint +++ b/bin/cylc-checkpoint @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 # THIS FILE IS PART OF THE CYLC SUITE ENGINE. # Copyright (C) 2008-2019 NIWA & British Crown (Met Office) & Contributors. @@ -30,7 +30,7 @@ if "--use-ssh" in sys.argv[1:]: import cylc.flags from cylc.option_parsers import CylcOptionParser as COP -from cylc.network.httpclient import SuiteRuntimeServiceClient +from cylc.network.client import SuiteRuntimeClient def main(): @@ -41,11 +41,10 @@ def main(): options, args = parser.parse_args() suite, name = args - pclient = SuiteRuntimeServiceClient( + pclient = SuiteRuntimeClient( suite, options.owner, options.host, options.port, - options.comms_timeout, my_uuid=options.set_uuid, - print_uuid=options.print_uuid) - pclient.put_command("take_checkpoints", items=[name]) + options.comms_timeout) + pclient('take_checkpoints', {'items': [name]}) if __name__ == "__main__": diff --git a/bin/cylc-client b/bin/cylc-client index deecd358e8a..b1a8acfc0ba 100755 --- a/bin/cylc-client +++ b/bin/cylc-client @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 # THIS FILE IS PART OF THE CYLC SUITE ENGINE. # Copyright (C) 2008-2019 NIWA & British Crown (Met Office) & Contributors. @@ -19,7 +19,7 @@ """cylc client [OPTIONS] ARGS (This command is for internal use.) -Invoke HTTP(S) client, expect JSON from STDIN for keyword arguments. +Invoke suite runtime client, expect JSON from STDIN for keyword arguments. Use the -n option if client function requires no keyword arguments. """ @@ -32,38 +32,31 @@ if '--use-ssh' in sys.argv[1:]: sys.exit(0) from cylc.option_parsers import CylcOptionParser as COP -from cylc.network.httpclient import SuiteRuntimeServiceClient +from cylc.network.client import SuiteRuntimeClient def main(): parser = COP(__doc__, comms=True, argdoc=[ - ('METHOD', 'Network API function name'), - ('[REG]', 'Suite name')]) + ('REG', 'Suite name'), + ('METHOD', 'Network API function name')]) + parser.add_option( '-n', '--no-input', help='Do not read from STDIN, assume null input', action='store_true', dest='no_input') options, args = parser.parse_args() - # Client API function name - func = args[0] - if len(args) == 2: - # 2nd argument is suite name, assume authentication required - suite = args[1] - auth = None - else: - # No 2nd argument, assume anonymous authentication - suite = None - auth = SuiteRuntimeServiceClient.ANON_AUTH - pclient = SuiteRuntimeServiceClient( - suite, options.owner, options.host, options.port, - options.comms_timeout, my_uuid=options.set_uuid, - print_uuid=options.print_uuid, auth=auth) + suite = args[0] + func = args[1] + pclient = SuiteRuntimeClient( + suite, options.owner, options.host, options.port) if options.no_input: kwargs = {} else: kwargs = json.load(sys.stdin) sys.stdin.close() - sys.stdout.write(json.dumps(getattr(pclient, func)(**kwargs), indent=4)) + sys.stdout.write(json.dumps( + pclient(func, kwargs, timeout=options.comms_timeout), indent=4 + ) + '\n') if __name__ == '__main__': diff --git a/bin/cylc-cycle-point b/bin/cylc-cycle-point index f4357d32868..a294ad0f335 100755 --- a/bin/cylc-cycle-point +++ b/bin/cylc-cycle-point @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 # -*- coding: utf-8 -*- # THIS FILE IS PART OF THE CYLC SUITE ENGINE. @@ -165,7 +165,7 @@ def main(): if options.print_year: n_chosen += 1 if options.num_expanded_year_digits: - template = u"±XCCYY" + template = "±XCCYY" else: template = "CCYY" @@ -252,15 +252,15 @@ def main(): parser.error('ERROR: offset not valid: %s' % exc) cycle_point += offset if template is None: - print cycle_point + print(cycle_point) else: dump_string = iso_point_dumper.dump(cycle_point, template) if (dump_string == template and not any(_ in template for _ in ["CCYY", "MM", "DD", "%"])): # A pure time string, no date - not well handled in isodatetime. - print iso_point_dumper.dump(cycle_point, "T" + template)[1:] + print(iso_point_dumper.dump(cycle_point, "T" + template)[1:]) else: - print dump_string + print(dump_string) if __name__ == "__main__": diff --git a/bin/cylc-diff b/bin/cylc-diff index a6a41da654f..194bf2322f4 100755 --- a/bin/cylc-diff +++ b/bin/cylc-diff @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 # THIS FILE IS PART OF THE CYLC SUITE ENGINE. # Copyright (C) 2008-2019 NIWA & British Crown (Met Office) & Contributors. @@ -100,16 +100,16 @@ def prdict(dct, arrow='<', section='', level=0, diff=False, nested=False): else: if not foo: if nested: - print ' ', sctn + print(' ', sctn) else: - print '\n ', sctn + print('\n ', sctn) foo = True if diff: - print ' < ', key, '=', dct[key][0] - print ' > ', key, '=', dct[key][1] + print(' < ', key, '=', dct[key][0]) + print(' > ', key, '=', dct[key][1]) else: - print ' ' + arrow + ' ', key, '=', dct[key] + print(' ' + arrow + ' ', key, '=', dct[key]) def main(): @@ -130,22 +130,22 @@ def main(): suite2, suite2rc = srv_files_manager.parse_suite_arg(options, args[1]) if suite1 == suite2: parser.error("You can't diff a single suite.") - print "Parsing %s (%s)" % (suite1, suite1rc) + print("Parsing %s (%s)" % (suite1, suite1rc)) template_vars = load_template_vars( options.templatevars, options.templatevars_file) config1 = SuiteConfig( suite1, suite1rc, template_vars, cli_initial_point_string=options.icp).cfg - print "Parsing %s (%s)" % (suite2, suite2rc) + print("Parsing %s (%s)" % (suite2, suite2rc)) config2 = SuiteConfig( suite2, suite2rc, template_vars, cli_initial_point_string=options.icp, is_reload=True).cfg if config1 == config2: - print "Suite definitions %s and %s are identical" % (suite1, suite2) + print("Suite definitions %s and %s are identical" % (suite1, suite2)) sys.exit(0) - print "Suite definitions %s and %s differ" % (suite1, suite2) + print("Suite definitions %s and %s differ" % (suite1, suite2)) suite1_only = {} suite2_only = {} @@ -154,22 +154,22 @@ def main(): diffdict(config1, config2, suite1_only, suite2_only, diff_1_2) if n_oone > 0: - print + print() msg = str(n_oone) + ' items only in ' + suite1 + ' (<)' - print msg + print(msg) prdict(suite1_only, '<', nested=options.nested) if n_otwo > 0: - print + print() msg = str(n_otwo) + ' items only in ' + suite2 + ' (>)' - print msg + print(msg) prdict(suite2_only, '>', nested=options.nested) if n_diff > 0: - print + print() msg = (str(n_diff) + ' common items differ ' + suite1 + '(<) ' + suite2 + '(>)') - print msg + print(msg) prdict(diff_1_2, '', diff=True, nested=options.nested) diff --git a/bin/cylc-documentation b/bin/cylc-documentation index a9f4785dad4..9ebc3727d30 100755 --- a/bin/cylc-documentation +++ b/bin/cylc-documentation @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 # THIS FILE IS PART OF THE CYLC SUITE ENGINE. # Copyright (C) 2008-2019 NIWA & British Crown (Met Office) & Contributors. @@ -24,8 +24,7 @@ View documentation in the browser, as per Cylc global config. Open the Cylc documentation. % cylc doc [-t TASK] SUITE View suite or task documentation, if URLs are specified in the suite. This - parses the suite definition to extract the requested URL. Note that suite - server programs also hold suite URLs for access from the Cylc GUI.""" + parses the suite definition to extract the requested URL.""" import sys @@ -118,15 +117,15 @@ def main(): parser.error("Too many arguments.") if options.stdout: - print target + print(target) sys.exit(0) + # viewer may have spaces (e.g. 'firefox --no-remote'): + command = ('%s %s' % (viewer, target)).split(' ') try: - retcode = check_call( - # viewer may have spaces (e.g. 'firefox --no-remote'): - ('%s %s' % (viewer, target)).split(' '), stdin=open(os.devnull)) + check_call(command, stdin=open(os.devnull)) except CalledProcessError: - print >> sys.stderr, 'ERROR, failed to execute: %s' % command + print('ERROR, failed to execute: %s' % command, file=sys.stderr) raise diff --git a/bin/cylc-dump b/bin/cylc-dump index 8b7166a3826..78751ceb23b 100755 --- a/bin/cylc-dump +++ b/bin/cylc-dump @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 # THIS FILE IS PART OF THE CYLC SUITE ENGINE. # Copyright (C) 2008-2019 NIWA & British Crown (Met Office) & Contributors. @@ -41,7 +41,7 @@ if '--use-ssh' in sys.argv[1:]: import cylc.flags from cylc.option_parsers import CylcOptionParser as COP -from cylc.network.httpclient import SuiteRuntimeServiceClient +from cylc.network.client import SuiteRuntimeClient from cylc.dump import dump_to_stdout @@ -65,17 +65,17 @@ def main(): options, args = parser.parse_args() suite = args[0] - summary = SuiteRuntimeServiceClient( + pclient = SuiteRuntimeClient( suite, options.owner, options.host, options.port, - options.comms_timeout, my_uuid=options.set_uuid, - print_uuid=options.print_uuid).get_suite_state_summary() + options.comms_timeout) + summary = pclient('get_suite_state_summary') if options.disp_form == "raw": - print summary + print(summary) else: if options.disp_form != "tasks": for key, value in sorted(summary[0].items()): - print "%s=%s" % (key, value) + print("%s=%s" % (key, value)) if options.disp_form != "global": dump_to_stdout(summary[1], options.sort_by_cycle) diff --git a/bin/cylc-edit b/bin/cylc-edit index e65cf3c6fa9..3169791110d 100755 --- a/bin/cylc-edit +++ b/bin/cylc-edit @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 # THIS FILE IS PART OF THE CYLC SUITE ENGINE. # Copyright (C) 2008-2019 NIWA & British Crown (Met Office) & Contributors. @@ -118,7 +118,7 @@ def main(): # edit the suite.rc file if not os.path.isfile(suiterc): - print >> sys.stderr, 'ERROR, file not found: ', suiterc + print('ERROR, file not found: ', suiterc, file=sys.stderr) sys.exit(1) # in case editor has options, e.g. 'emacs -nw': @@ -129,7 +129,7 @@ def main(): retcode = call(command_list) if retcode != 0: # the command returned non-zero exist status - print >> sys.stderr, command, 'failed:', retcode + print(command, 'failed:', retcode, file=sys.stderr) sys.exit(1) # !!!EDITING FINISHED!!! @@ -142,11 +142,11 @@ def main(): # record original modtime modtimes[suiterc] = os.stat(suiterc).st_mtime # read the file - h = open(suiterc, 'rb') + h = open(suiterc, 'r') lines0 = h.readlines() h.close() if lines0[0].startswith('# !WARNING! CYLC EDIT INLINED'): - print 'WARNING: RECOVERING A PREVIOUSLY INLINED FILE' + print('WARNING: RECOVERING A PREVIOUSLY INLINED FILE') recovery = True lines = lines0 else: @@ -160,14 +160,14 @@ def main(): # overwrite the (now backed up) original with the inlined file: h = open(suiterc, 'wb') for line in lines: - h.write(line + '\n') + h.write((line + '\n').encode()) h.close() - print 'PRE-EDIT BACKUPS:' + print('PRE-EDIT BACKUPS:') for file in backups: src = re.sub(suitedir + '/', '', file) dst = re.sub(suitedir + '/', '', backups[file]) - print ' + ' + src + ' ---> ' + dst + print(' + ' + src + ' ---> ' + dst) # in case editor has options, e.g. 'emacs -nw': command_list = re.split(' ', editor) @@ -177,9 +177,9 @@ def main(): retcode = call(command_list) if retcode != 0: # the command returned non-zero exist status - print >> sys.stderr, command, 'failed:', retcode + print(command, 'failed:', retcode, file=sys.stderr) sys.exit(1) - print 'EDITING DONE' + print('EDITING DONE') # Now back up the inlined file in case of absolute disaster, so as the # user or his editor corrupting the inlined-include-file marker lines. @@ -190,22 +190,22 @@ def main(): copy(suiterc, inlined_suiterc_backup) # read in the edited inlined file - h = open(suiterc, 'rb') + h = open(suiterc, 'r') lines = h.readlines() h.close() # split it back into separate files split_file(suitedir, lines, suiterc, recovery) - print ' + edited:', suiterc - print ' + backup:', inlined_suiterc_backup - print 'INCLUDE-FILES WRITTEN:' + print(' + edited:', suiterc) + print(' + backup:', inlined_suiterc_backup) + print('INCLUDE-FILES WRITTEN:') for file in newfiles: f = re.sub(suitedir + '/', '', file) if re.search(r'\.EDIT\.NEW\.', f): - print ' + ' + f + ' (!!! WARNING: original changed on disk !!!)' + print(' + ' + f + ' (!!! WARNING: original changed on disk !!!)') else: - print ' + ' + f + print(' + ' + f) # DONE diff --git a/bin/cylc-ext-trigger b/bin/cylc-ext-trigger index 625a334b21d..d2292f9390e 100755 --- a/bin/cylc-ext-trigger +++ b/bin/cylc-ext-trigger @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 # THIS FILE IS PART OF THE CYLC SUITE ENGINE. # Copyright (C) 2008-2019 NIWA & British Crown (Met Office) & Contributors. @@ -41,7 +41,7 @@ from time import sleep from cylc import LOG import cylc.flags from cylc.option_parsers import CylcOptionParser as COP -from cylc.network.httpclient import ClientError, SuiteRuntimeServiceClient +from cylc.network.client import ClientError, SuiteRuntimeClient MAX_N_TRIES = 5 @@ -74,24 +74,27 @@ def main(): LOG.info('Send to suite %s: "%s" (%s)', suite, event_msg, event_id) - pclient = SuiteRuntimeServiceClient( - suite, options.owner, options.host, options.port, - options.comms_timeout, my_uuid=options.set_uuid, - print_uuid=options.print_uuid) + pclient = SuiteRuntimeClient( + suite, options.owner, options.host, options.port) max_n_tries = int(options.max_n_tries) retry_intvl_secs = float(options.retry_intvl_secs) for i_try in range(max_n_tries): try: - pclient.put_ext_trigger(event_msg, event_id) + pclient( + 'put_ext_trigger', + {'event_message': event_msg, 'event_id': event_id}, + timeout=options.comms_timeout + ) except ClientError as exc: LOG.exception(exc) LOG.info(MSG_SEND_FAILED, i_try + 1, max_n_tries) if i_try == max_n_tries - 1: # final attempt sys.exit('ERROR: send failed') else: - LOG.info(MSG_SEND_RETRY, retry_intvl_secs, pclient.timeout) + LOG.info(MSG_SEND_RETRY, retry_intvl_secs, + options.comms_timeout) sleep(retry_intvl_secs) else: if i_try > 0: diff --git a/bin/cylc-function-run b/bin/cylc-function-run index 4c65fa5e0dd..03faed0257d 100755 --- a/bin/cylc-function-run +++ b/bin/cylc-function-run @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 # THIS FILE IS PART OF THE CYLC SUITE ENGINE. # Copyright (C) 2008-2019 NIWA & British Crown (Met Office) & Contributors. @@ -32,6 +32,6 @@ from cylc.subprocpool import run_function if __name__ == "__main__": if sys.argv[1] in ['help', '--help'] or len(sys.argv) != 5: - print __doc__ + print(__doc__) sys.exit(0) run_function(sys.argv[1], sys.argv[2], sys.argv[3], sys.argv[4]) diff --git a/bin/cylc-get-directory b/bin/cylc-get-directory index 81a869f8f4b..2fa9eba9c19 100755 --- a/bin/cylc-get-directory +++ b/bin/cylc-get-directory @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 # THIS FILE IS PART OF THE CYLC SUITE ENGINE. # Copyright (C) 2008-2019 NIWA & British Crown (Met Office) & Contributors. @@ -35,8 +35,8 @@ if remrun(): def main(): """Implement "cylc get-directory".""" options, args = COP(__doc__, prep=True).parse_args() - print SuiteSrvFilesManager().get_suite_source_dir( - args[0], options.suite_owner) + print(SuiteSrvFilesManager().get_suite_source_dir( + args[0], options.suite_owner)) if __name__ == "__main__": diff --git a/bin/cylc-get-gui-config b/bin/cylc-get-gui-config deleted file mode 100755 index eb70a1afb19..00000000000 --- a/bin/cylc-get-gui-config +++ /dev/null @@ -1,58 +0,0 @@ -#!/usr/bin/env python2 - -# THIS FILE IS PART OF THE CYLC SUITE ENGINE. -# Copyright (C) 2008-2019 NIWA & British Crown (Met Office) & Contributors. -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -"""cylc [admin] get-gui-config [OPTIONS] - -Print gcylc configuration settings. - -By default all settings are printed. For specific sections or items -use -i/--item and wrap parent sections in square brackets: - cylc get-gui-config --item '[themes][default]succeeded' -Multiple items can be specified at once.""" - - -from cylc.cfgspec.gcylc import GcylcConfig -from cylc.option_parsers import CylcOptionParser as COP - - -def main(): - parser = COP(__doc__, argdoc=[]) - - parser.add_option( - "-i", "--item", metavar="[SEC...]ITEM", - help="Item or section to print (multiple use allowed).", - action="append", dest="item", default=[]) - - parser.add_option( - "--sparse", - help="Only print items explicitly set in the config files.", - action="store_true", default=False, dest="sparse") - - parser.add_option( - "-p", "--python", help="Print native Python format.", - action="store_true", default=False, dest="pnative") - - options = parser.parse_args(remove_opts=['--host', '--user'])[0] - - # Import gcfg here to avoid aborting before command help is printed. - GcylcConfig.get_inst().idump( - options.item, sparse=options.sparse, pnative=options.pnative) - - -if __name__ == "__main__": - main() diff --git a/bin/cylc-get-host-metrics b/bin/cylc-get-host-metrics index 02af62fd3d8..4df21335fe2 100755 --- a/bin/cylc-get-host-metrics +++ b/bin/cylc-get-host-metrics @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # THIS FILE IS PART OF THE CYLC SUITE ENGINE. # Copyright (C) 2008-2019 NIWA @@ -108,7 +108,7 @@ def main(): options, args = parser.parse_args() - print get_host_metric(options.load, options.memory, options.disk) + print(get_host_metric(options.load, options.memory, options.disk)) sys.exit() @@ -122,7 +122,7 @@ def run_command_string(command_string): # Note: can replace function w/ 'subprocess.check_output()' when drop v2.6. process = Popen(command_sequence, stdin=open(os.devnull), stdout=PIPE, stderr=PIPE) - stdoutput = process.communicate()[0] + stdoutput = process.communicate()[0].decode() process.wait() return_code = process.returncode if return_code: diff --git a/bin/cylc-get-site-config b/bin/cylc-get-site-config index abbf020e2b4..cf8bd0c1be8 100755 --- a/bin/cylc-get-site-config +++ b/bin/cylc-get-site-config @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 # THIS FILE IS PART OF THE CYLC SUITE ENGINE. # Copyright (C) 2008-2019 NIWA & British Crown (Met Office) & Contributors. @@ -61,9 +61,9 @@ def main(): options = parser.parse_args(remove_opts=['--host', '--user'])[0] if options.run_dir: - print glbl_cfg().get_host_item('run directory') + print(glbl_cfg().get_host_item('run directory')) elif options.site_dir: - print glbl_cfg().SITE_CONF_DIR + print(glbl_cfg().SITE_CONF_DIR) else: glbl_cfg().idump( options.item, sparse=options.sparse, pnative=options.pnative) diff --git a/bin/cylc-get-suite-config b/bin/cylc-get-suite-config index 68da7b887b4..ba07309218a 100755 --- a/bin/cylc-get-suite-config +++ b/bin/cylc-get-suite-config @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 # THIS FILE IS PART OF THE CYLC SUITE ENGINE. # Copyright (C) 2008-2019 NIWA & British Crown (Met Office) & Contributors. @@ -127,11 +127,11 @@ def main(): cli_initial_point_string=options.icp, run_mode=options.run_mode) if options.tasks: for task in config.get_task_name_list(): - print prefix + task + print(prefix + task) elif options.alltasks: for task in config.get_task_name_list(): items = ['[runtime][' + task + ']' + i for i in options.item] - print prefix + task, + print(prefix + task, end=' ') config.pcfg.idump( items, options.sparse, options.pnative, prefix, options.oneline, none_str=options.none_str) diff --git a/bin/cylc-get-suite-contact b/bin/cylc-get-suite-contact index b58dc64bd4c..45491577317 100755 --- a/bin/cylc-get-suite-contact +++ b/bin/cylc-get-suite-contact @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 # THIS FILE IS PART OF THE CYLC SUITE ENGINE. # Copyright (C) 2008-2019 NIWA & British Crown (Met Office) & Contributors. diff --git a/bin/cylc-get-suite-version b/bin/cylc-get-suite-version index 962d76ed1e7..109ee4c0df2 100755 --- a/bin/cylc-get-suite-version +++ b/bin/cylc-get-suite-version @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 # THIS FILE IS PART OF THE CYLC SUITE ENGINE. # Copyright (C) 2008-2019 NIWA & British Crown (Met Office) & Contributors. @@ -31,7 +31,7 @@ if '--use-ssh' in sys.argv[1:]: import cylc.flags from cylc.option_parsers import CylcOptionParser as COP -from cylc.network.httpclient import SuiteRuntimeServiceClient +from cylc.network.client import SuiteRuntimeClient def main(): @@ -40,11 +40,9 @@ def main(): (options, args) = parser.parse_args() suite = args[0] - pclient = SuiteRuntimeServiceClient( - suite, options.owner, options.host, options.port, - options.comms_timeout, my_uuid=options.set_uuid, - print_uuid=options.print_uuid) - print pclient.get_info('get_cylc_version') + pclient = SuiteRuntimeClient( + suite, options.owner, options.host, options.port) + print(pclient('get_cylc_version', timeout=options.comms_timeout)) if __name__ == "__main__": diff --git a/bin/cylc-gpanel b/bin/cylc-gpanel deleted file mode 100755 index 1d3c0b66a1a..00000000000 --- a/bin/cylc-gpanel +++ /dev/null @@ -1,135 +0,0 @@ -#!/usr/bin/env python2 - -# THIS FILE IS PART OF THE CYLC SUITE ENGINE. -# Copyright (C) 2008-2019 NIWA & British Crown (Met Office) & Contributors. -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -"""cylc gpanel [OPTIONS] - -This is a cylc scan panel applet for monitoring running suites on a set of -hosts in GNOME 2. - -To install this applet, run "cylc gpanel --install" and follow the instructions -that it gives you. - -This applet can be tested using the --test option. - -To customize themes, copy $CYLC_DIR/etc/gcylc.rc.eg to $HOME/.cylc/gcylc.rc and -follow the instructions in the file. - -To configure default suite hosts, edit "[suite servers]scan hosts" in your -global.rc file.""" - -from optparse import OptionParser -import os -import sys -import warnings -warnings.filterwarnings('ignore', 'use the new', Warning) - -sys.path.append( - os.path.dirname(os.path.realpath(os.path.abspath(__file__))) + '/../lib/') -cylc_dir = os.path.dirname(os.path.realpath(os.path.abspath(__file__))) -if cylc_dir != os.getenv('CYLC_DIR', ''): - os.environ['CYLC_DIR'] = cylc_dir - -parser = OptionParser(__doc__) - -parser.add_option("--compact", - help="Switch on compact mode at runtime.", - action="store_true", dest="compact") - -parser.add_option("--install", - help="Install the panel applet.", - default=False, - action="store_true", dest="install") - -parser.add_option("--test", - help="Run in a standalone window.", - default=False, - action="store_true", dest="test") - -arglist = [a for a in sys.argv[1:] if not a.startswith("--oaf")] -(options, args) = parser.parse_args(arglist) - -import cylc - -cylc_dir = os.path.dirname( - os.path.dirname(os.path.realpath(os.path.abspath(__file__)))) -if os.path.basename(cylc_dir).startswith("cylc-"): - # If using the wrapper, reference 'cylc' rather than 'cylc-6.3.1'. - cylc_alt_dir = os.path.join(os.path.dirname(cylc_dir), "cylc") - if os.path.realpath(cylc_alt_dir) == os.path.realpath(cylc_dir): - cylc_dir = cylc_alt_dir - -if cylc_dir != os.getenv('CYLC_DIR', ''): - os.environ['CYLC_DIR'] = cylc_dir - -from cylc.gui.gpanel import ScanPanelApplet, run_in_window - - -def install_panel_applet(): - home = os.path.expanduser("~") - dest_dir = os.path.join(home, ".bonobo") - if not os.path.isdir(dest_dir): - os.mkdir(dest_dir) - for server_filename in ["cylc.server", "cylc-compact.server"]: - source_file = os.path.join(cylc_dir, "etc", server_filename) - dest_file = os.path.join(dest_dir, server_filename) - f = open(source_file, "r") - text = f.read().replace("$CYLC_DIR", cylc_dir) - f.close() - open(dest_file, "w").write(text) - print dest_file, "created." - print - print """Further Instructions: - * normal users - place this line in your ~/.profile file: -export BONOBO_ACTIVATION_PATH=$HOME/.bonobo - * (sys)admin users - move the new cylc.server and cylc-compact.server -files into your /usr/lib/bonobo/servers/ directory, or equivalent. - * After logging in and out, right-click within the GNOME 2 panel and -select "Add to Panel", then choose a Cylc Applet.""" - - -def panel_applet_factory_compact(applet, _): - my_panel_app = ScanPanelApplet(is_compact=True) - applet.add(my_panel_app.get_widget()) - applet.show_all() - return True - - -def panel_applet_factory(applet, _): - my_panel_app = ScanPanelApplet() - applet.add(my_panel_app.get_widget()) - applet.show_all() - return True - - -if __name__ == "__main__": - if options.test: - run_in_window(options.compact) - elif options.install: - install_panel_applet() - elif options.compact: - import gnomeapplet - gnomeapplet.bonobo_factory( - "OAFIID:GNOME_CylcCompactMonitorFactory", - gnomeapplet.Applet.__gtype__, - "cylc gpanel compact", "0", panel_applet_factory_compact) - else: - import gnomeapplet - gnomeapplet.bonobo_factory( - "OAFIID:GNOME_CylcMonitorFactory", - gnomeapplet.Applet.__gtype__, - "cylc gpanel", "0", panel_applet_factory) diff --git a/bin/cylc-graph b/bin/cylc-graph index 4ed0d9ae415..9ce64a33ab5 100755 --- a/bin/cylc-graph +++ b/bin/cylc-graph @@ -1,5 +1,5 @@ -#!/usr/bin/env python2 - +#!/usr/bin/env python3 +# # THIS FILE IS PART OF THE CYLC SUITE ENGINE. # Copyright (C) 2008-2019 NIWA & British Crown (Met Office) & Contributors. # @@ -15,124 +15,169 @@ # # You should have received a copy of the GNU General Public License # along with this program. If not, see . +"""Usage: + cylc graph SUITE [START] [STOP] -"""1/ cylc [prep] graph [OPTIONS] SUITE [START[STOP]] - Plot the suite.rc dependency graph for SUITE. - 2/ cylc [prep] graph [OPTIONS] -f,--file FILE - Plot the specified dot-language graph file. - 3/ cylc [prep] graph [OPTIONS] --reference SUITE [START[STOP]] - Print out a reference format for the dependencies in SUITE. - 4/ cylc [prep] graph [OPTIONS] --output-file FILE SUITE - Plot SUITE dependencies to a file FILE with a extension-derived format. - If FILE endswith ".png", output in PNG format, etc. - -Plot suite dependency graphs in an interactive graph viewer. - -If START is given it overrides "[visualization] initial cycle point" to -determine the start point of the graph, which defaults to the suite initial -cycle point. If STOP is given it overrides "[visualization] final cycle point" -to determine the end point of the graph, which defaults to the graph start -point plus "[visualization] number of cycle points" (which defaults to 3). -The graph start and end points are adjusted up and down to the suite initial -and final cycle points, respectively, if necessary. - -The "Save" button generates an image of the current view, of format (e.g. png, -svg, jpg, eps) determined by the filename extension. If the chosen format is -not available a dialog box will show those that are available. - -If the optional output filename is specified, the viewer will not open and a -graph will be written directly to the file. - -GRAPH VIEWER CONTROLS: - * Center on a node: left-click. - * Pan view: left-drag. - * Zoom: +/- buttons, mouse-wheel, or ctrl-left-drag. - * Box zoom: shift-left-drag. - * "Best Fit" and "Normal Size" buttons. - * Left-to-right graphing mode toggle button. - * "Ignore suicide triggers" button. - * "Save" button: save an image of the view. - Family (namespace) grouping controls: - Toolbar: - * "group" - group all families up to root. - * "ungroup" - recursively ungroup all families. - Right-click menu: - * "group" - close this node's parent family. - * "ungroup" - open this family node. - * "recursive ungroup" - ungroup all families below this node.""" +Implement the old ``cylc graph --reference command`` for producing a textural +graph of a suite. +""" import sys -import StringIO -import cylc.flags -from cylc.remote import remrun -from cylc.suite_srv_files_mgr import SuiteSrvFilesManager -from cylc.task_id import TaskID +from cylc.config import SuiteConfig +from cylc.cycling.loader import get_point +from cylc.option_parsers import CylcOptionParser as COP +from cylc.suite_srv_files_mgr import ( + SuiteSrvFilesManager, SuiteServiceFileError) from cylc.templatevars import load_template_vars -if remrun(forward_x11=True): - sys.exit(0) -from cylc.option_parsers import CylcOptionParser as COP +def sort_integer_node(item): + """Return sort tokens for nodes with cyclepoints in integer format. -# DEVELOPER NOTE: family grouping controls via the viewer toolbar and -# right-click menu have been rather hastily stuck on to the original -# viewer, via changes to this file and to lib/cylc/cylc_xdot.py - all -# of which could stand some refactoring to streamline the code a bit. + Example: + >>> sort_integer_node('foo.11') + ('foo', 11) -# TODO - clarify what it means to choose visualization boundaries (by CLI -# or in-suite) outside of the defined suite initial and final cycle points. + """ + name, point = item.split('.') + return (name, int(point)) -def on_url_clicked(_, task_id, event, window): - """Display right click menu for a graph node.""" - if event.button != 3: - return False - # URL is node ID - name = TaskID.split(task_id)[0] +def sort_integer_edge(item): + """Return sort tokens for edges with cyclepoints in integer format. - import gtk - menu = gtk.Menu() - menu_root = gtk.MenuItem(task_id) - menu_root.set_submenu(menu) + Example: + >>> sort_integer_edge(('foo.11', 'foo.12', None)) + (('foo', 11), ('foo', 12)) + >>> sort_integer_edge(('foo.11', None , None)) + (('foo', 11), ('', 0)) - group_item = gtk.MenuItem('Group') - group_item.connect('activate', grouping, name, window, False, False) - ungroup_item = gtk.MenuItem('UnGroup') - ungroup_item.connect('activate', grouping, name, window, True, False) - ungroup_rec_item = gtk.MenuItem('Recursive UnGroup') - ungroup_rec_item.connect('activate', grouping, name, window, True, True) + """ - title_item = gtk.MenuItem(task_id) - title_item.set_sensitive(False) - menu.append(title_item) + return ( + sort_integer_node(item[0]), + sort_integer_node(item[1]) if item[1] else ('', 0) + ) - menu.append(gtk.SeparatorMenuItem()) - menu.append(group_item) - menu.append(ungroup_item) - menu.append(ungroup_rec_item) +def sort_datetime_edge(item): + """Return sort tokens for edges with cyclepoints in ISO8601 format. - menu.show_all() - menu.popup(None, None, None, event.button, event.time) + Example: + >>> sort_datetime_edge(('a', None, None)) + ('a', '') - # TODO - popup menus are not automatically destroyed and can be - # reused if saved; however, we need to reconstruct or at least - # alter ours dynamically => should destroy after each use to - # prevent a memory leak? But I'm not sure how to do this as yet.) + """ + return (item[0], item[1] or '') - return True +def get_cycling_bounds(config, start_point=None, stop_point=None): + """Determine the start and stop points for graphing a suite.""" + # default start and stop points to values in the visualization section + if not start_point: + start_point = config.cfg['visualization']['initial cycle point'] + if not stop_point: + viz_stop_point = config.cfg['visualization']['final cycle point'] + if viz_stop_point: + stop_point = viz_stop_point -def grouping(_, name, window, ungroup, recursive): - """Control graph grouping.""" - if ungroup: - window.get_graph(ungroup_nodes=[name], ungroup_recursive=recursive) + # don't allow stop_point before start_point + if stop_point is not None: + if get_point(stop_point) < get_point(start_point): + # NOTE: we need to cast with get_point for this comparison due to + # ISO8061 extended datetime formats + stop_point = start_point + else: + stop_point = stop_point else: - window.get_graph(group_nodes=[name]) + stop_point = None + return start_point, stop_point -def main(): + +def graph_workflow(config, start_point=None, stop_point=None, ungrouped=False, + show_suicide=False): + """Implement ``cylc-graph --reference``.""" + # set sort keys based on cycling mode + if config.cfg['scheduling']['cycling mode'] == 'integer': + # integer sorting + node_sort = sort_integer_node + edge_sort = sort_integer_edge + else: + # datetime sorting + node_sort = None # lexicographically sortable + edge_sort = sort_datetime_edge + + # get graph + start_point, stop_point = get_cycling_bounds( + config, start_point, stop_point) + graph = config.get_graph_raw( + start_point, stop_point, ungroup_all=ungrouped) + if not graph: + return + + edges = ( + (left, right) + for left, right, _, suicide, _ in graph + if right + if show_suicide or not suicide + ) + for left, right in sorted(set(edges), key=edge_sort): + print('edge "%s" "%s"' % (left, right)) + + print('graph') + + # print nodes + nodes = ( + node + for left, right, _, suicide, _ in graph + for node in (left, right) + if node + if show_suicide or not suicide + ) + for node in sorted(set(nodes), key=node_sort): + print('node "%s" "%s"' % (node, node.replace('.', r'\n'))) + + print('stop') + + +def graph_inheritance(config): + """Implement ``cylc-graph --reference --namespaces``.""" + edges = set() + nodes = set() + for namespace, tasks in config.get_parent_lists().items(): + for task in tasks: + edges.add((task, namespace)) + nodes.add(task) + + for namespace in config.get_parent_lists(): + nodes.add(namespace) + + for edge in sorted(edges): + print('edge %s %s' % edge) + + print('graph') + + for node in sorted(nodes): + print('node %s %s' % (node, node)) + + print('stop') + + +def get_config(suite, icp=None, template_vars=None): + """Return a SuiteConfig object for the provided reg / path.""" + try: + suiterc = SuiteSrvFilesManager().get_suite_rc(suite) + except SuiteServiceFileError: + # could not find suite, assume we have been given a path instead + suiterc = suite + suite = 'test' + return SuiteConfig(suite, suiterc, cli_initial_point_string=icp, + template_vars=template_vars) + + +def parse_args(): """CLI.""" parser = COP( __doc__, jset=True, prep=True, @@ -144,147 +189,57 @@ def main(): '(default: initial + 3 points)')]) parser.add_option( - "-u", "--ungrouped", - help="Start with task families ungrouped (the default is grouped).", - action="store_true", default=False, dest="start_ungrouped") + '-u', '--ungrouped', + help='Start with task families ungrouped (the default is grouped).', + action='store_true', default=False, dest='ungrouped') parser.add_option( - "-n", "--namespaces", - help="Plot the suite namespace inheritance hierarchy " - "(task run time properties).", - action="store_true", default=False, dest="namespaces") + '-n', '--namespaces', + help='Plot the suite namespace inheritance hierarchy ' + '(task run time properties).', + action='store_true', default=False, dest='namespaces') parser.add_option( - "-f", "--file", - help="View a specific dot-language graphfile.", - metavar="FILE", action="store", default=None, dest="filename") + '-r', '--reference', + help='Output in a sorted plain text format for comparison purposes. ' + 'If not given, assume --output-file=-.', + action='store_true', default=False, dest='reference') parser.add_option( - "--filter", help="Filter out one or many nodes.", - metavar="NODE_NAME_PATTERN", action="append", dest="filter_patterns") + '--show-suicide', + help='Show suicide triggers. They are not shown by default, unless ' + 'toggled on with the tool bar button.', + action='store_true', default=False, dest='show_suicide') parser.add_option( - "-O", "--output-file", - help="Output to a specific file, with a format given by " - "--output-format or extrapolated from the extension. " - "'-' implies stdout in plain format.", - metavar="FILE", action="store", default=None, dest="output_filename") + '--icp', action='store', default=None, metavar='CYCLE_POINT', help=( + 'Set initial cycle point. Required if not defined in suite.rc.')) - parser.add_option( - "--output-format", - help="Specify a format for writing out the graph to --output-file " - "e.g. png, svg, jpg, eps, dot. 'ref' is a special sorted plain " - "text format for comparison and reference purposes.", - metavar="FORMAT", action="store", default=None, dest="output_format") + opts, args = parser.parse_args() - parser.add_option( - "-r", "--reference", - help="Output in a sorted plain text format for comparison purposes. " - "If not given, assume --output-file=-.", - action="store_true", default=False, dest="reference") + if opts.ungrouped and opts.namespaces: + sys.exit('Cannot combine --ungrouped and --namespaces.') - parser.add_option( - "--show-suicide", - help="Show suicide triggers. They are not shown by default, unless " - "toggled on with the tool bar button.", - action="store_true", default=False, dest="show_suicide") + return opts, args + [None] * (3 - len(args)) - (options, args) = parser.parse_args() - try: - import gtk - import gobject - from xdot import DotWindow - from cylc.cylc_xdot import ( - MyDotWindow, MyDotWindow2, get_reference_from_plain_format) - except (ImportError, RuntimeError) as error: - # Allow command help generation without a graphical environment. - sys.exit('ERROR: no X environment? %s' % error) - - if options.filename: - if len(args) != 0: - parser.error( - "file graphing arguments: '-f FILE' or '--file=FILE'") - sys.exit(1) - if options.output_filename: - sys.exit("ERROR: output-file not supported for dot files." - " Use 'dot' command instead.") - window = DotWindow() - window.update(options.filename) - window.connect('destroy', gtk.main_quit) - # checking periodically for file changed - gobject.timeout_add(1000, window.update, options.filename) - gtk.main() - sys.exit(0) - - if not args: - parser.error('Argument 1 SUITE is required except for usage 2.') - suite, suiterc = SuiteSrvFilesManager().parse_suite_arg(options, args[0]) - - start_point_string = stop_point_string = None - if len(args) >= 2: - start_point_string = args[1] - if len(args) == 3: - stop_point_string = args[2] - - interactive = not (options.reference or options.output_filename) +def main(): + """Implement ``cylc graph``.""" + opts, (suite, start, stop) = parse_args() + + if not opts.reference: + sys.exit('Only the --reference use cases are supported') + template_vars = load_template_vars( - options.templatevars, options.templatevars_file) - should_hide_gtk_window = (options.output_filename is not None) - if options.namespaces: - window = MyDotWindow2(suite, suiterc, template_vars, - should_hide=should_hide_gtk_window, - interactive=interactive) + opts.templatevars, opts.templatevars_file) + + config = get_config(suite, icp=opts.icp, template_vars=template_vars) + if opts.namespaces: + graph_inheritance(config) else: - hide_suicide = not options.show_suicide - window = MyDotWindow(suite, suiterc, template_vars, - start_point_string, stop_point_string, - should_hide=should_hide_gtk_window, - ignore_suicide=hide_suicide, - interactive=interactive) - - if options.start_ungrouped: - window.ungroup_all(None) - - window.widget.connect('clicked', on_url_clicked, window) - if options.filter_patterns: - window.set_filter_graph_patterns(options.filter_patterns) - window.get_graph() - - if options.reference and options.output_filename is None: - options.output_filename = "-" - - if options.output_filename: - if (options.reference or options.output_filename.endswith(".ref") or - options.output_format == "ref"): - dest = StringIO.StringIO() - window.graph.draw(dest, format="plain", prog="dot") - output_text = get_reference_from_plain_format(dest.getvalue()) - if options.output_filename == "-": - sys.stdout.write(output_text) - else: - with open(options.output_filename, 'w') as handle: - handle.write(output_text) - else: - if options.output_filename == "-": - window.graph.draw(sys.stdout, format="plain", prog="dot") - elif options.output_format: - window.graph.draw( - options.output_filename, format=options.output_format, - prog="dot" - ) - else: - window.graph.draw(options.output_filename, prog="dot") - sys.exit(0) - - window.connect('destroy', gtk.main_quit) - gtk.main() - - -if __name__ == "__main__": - try: - main() - except Exception as exc: - if cylc.flags.debug: - raise - sys.exit(str(exc)) + graph_workflow(config, start, stop, ungrouped=opts.ungrouped, + show_suicide=opts.show_suicide) + + +if __name__ == '__main__': + main() diff --git a/bin/cylc-gscan b/bin/cylc-gscan deleted file mode 100755 index 3da53cafd49..00000000000 --- a/bin/cylc-gscan +++ /dev/null @@ -1,102 +0,0 @@ -#!/usr/bin/env python2 - -# THIS FILE IS PART OF THE CYLC SUITE ENGINE. -# Copyright (C) 2008-2019 NIWA & British Crown (Met Office) & Contributors. -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -"""cylc gscan [OPTIONS] - -This is the cylc scan gui for monitoring running suites on a set of -hosts. - -To customize themes copy $CYLC_DIR/etc/gcylc.rc.eg to ~/.cylc/gcylc.rc and -follow the instructions in the file.""" - -import sys -if "--use-ssh" in sys.argv[1:]: - sys.argv.remove("--use-ssh") - from cylc.remote import remrun - if remrun(forward_x11=True): - sys.exit(0) - -from cylc.option_parsers import CylcOptionParser as COP - - -def main(): - """Implement "cylc gscan".""" - parser = COP( - __doc__, - comms=True, - noforce=True, - argdoc=[( - "[HOSTS ...]", "Hosts to scan instead of the configured hosts.")]) - parser.add_option( - "-a", "--all", - help="Scan all port ranges in known hosts.", - action="store_true", default=False, dest="all_ports") - parser.add_option( - "-n", "--name", - metavar="PATTERN", - help="List suites with name matching PATTERN (regular expression). " - "Defaults to any name. Can be used multiple times.", - action="append", dest="patterns_name", default=[]) - parser.add_option( - "-o", "--suite-owner", - metavar="PATTERN", - help="List suites with owner matching PATTERN (regular expression). " - "Defaults to just your own suites. Can be used multiple times.", - action="append", dest="patterns_owner", default=[]) - parser.add_option( - "--comms-timeout", metavar="SEC", - help="Set a timeout for network connections " - "to each running suite. The default is 5 seconds.", - action="store", default=None, dest="comms_timeout") - parser.add_option( - "--interval", - help="Time interval (in seconds) between full updates", - type="int", metavar="SECONDS", dest="interval") - options, args = parser.parse_args() - - # Only import GTK, etc here, as the command may be called with "--help" in - # batch mode, which may be unable to import GTK. - import gtk - import warnings - warnings.filterwarnings('ignore', 'use the new', Warning) - gtk.settings_get_default().set_long_property( - "gtk-button-images", True, "main") - gtk.settings_get_default().set_long_property( - "gtk-menu-images", True, "main") - from cylc.cfgspec.glbl_cfg import glbl_cfg - from cylc.gui.gscan import ScanApp - - if options.all_ports: - args.extend( - glbl_cfg().get(["suite servers", "scan hosts"]) or ["localhost"]) - scan_app = ScanApp( - hosts=args, - patterns_name=options.patterns_name, - patterns_owner=options.patterns_owner, - comms_timeout=options.comms_timeout, - interval=options.interval) - try: - gtk.main() - except KeyboardInterrupt: - print >> sys.stderr, "Stopping..." - scan_app.window.destroy() - raise SystemExit(1) - - -if __name__ == "__main__": - main() diff --git a/bin/cylc-gsummary b/bin/cylc-gsummary deleted file mode 100755 index 66fb2d94aa8..00000000000 --- a/bin/cylc-gsummary +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/bash - -# THIS FILE IS PART OF THE CYLC SUITE ENGINE. -# Copyright (C) 2008-2019 NIWA & British Crown (Met Office) & Contributors. -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -exec "$(dirname "$0")/cylc-gscan" "$@" diff --git a/bin/cylc-gui b/bin/cylc-gui deleted file mode 100755 index 4a40f395ee2..00000000000 --- a/bin/cylc-gui +++ /dev/null @@ -1,124 +0,0 @@ -#!/usr/bin/env python2 - -# THIS FILE IS PART OF THE CYLC SUITE ENGINE. -# Copyright (C) 2008-2019 NIWA & British Crown (Met Office) & Contributors. -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -"""cylc gui [OPTIONS] [REG] [USER_AT_HOST] -gcylc [OPTIONS] [REG] [USER_AT_HOST] - -This is the cylc Graphical User Interface. - -The USER_AT_HOST argument allows suite selection by 'cylc scan' output: - cylc gui $(cylc scan | grep ) - -Local suites can be opened and switched between from within gcylc. To connect -to running remote suites (whose passphrase you have installed) you must -currently use --host and/or --user on the gcylc command line. - -Available task state color themes are shown under the View menu. To customize -themes copy /etc/gcylc.rc.eg to ~/.cylc/gcylc.rc and follow the -instructions in the file. - -To see current configuration settings use "cylc get-gui-config". - -In the graph view, View -> Options -> "Write Graph Frames" writes .dot graph -files to the suite share directory (locally, for a remote suite). These can -be processed into a movie by $CYLC_DIR/dev/bin/live-graph-movie.sh=.""" - -import sys -if '--use-ssh' in sys.argv[1:]: - sys.argv.remove('--use-ssh') - from cylc.remote import remrun - if remrun(forward_x11=True): - sys.exit(0) - -import os -from cylc.option_parsers import CylcOptionParser as COP -import cylc.flags -from cylc.templatevars import load_template_vars - - -def main(): - sys.path.append( - os.path.dirname( - os.path.realpath(os.path.abspath(__file__))) + '/../lib') - sys.path.append( - os.path.dirname(os.path.realpath(os.path.abspath(__file__))) + '/../') - - parser = COP( - __doc__, comms=True, noforce=True, jset=True, argdoc=[ - ('[REG]', 'Suite name'), - ('[USER_AT_HOST]', 'user@host:port, shorthand for --user, --host ' - '& --port.')]) - - parser.add_option( - "-r", "--restricted", - help="Restrict display to 'active' task states: submitted, " - "submit-failed, submit-retrying, running, failed, retrying; " - "and disable the graph view. This may be needed for very large " - "suites. The state summary icons in the status bar still " - "represent all task proxies.", - action="store_true", default=False, dest="restricted") - - (options, args) = parser.parse_args() - - # import modules that require gtk now, so that a display is not needed - # just to get command help (e.g. when running make on a post-commit hook - # on a remote repository). - - import gtk - import warnings - warnings.filterwarnings('ignore', 'use the new', Warning) - from cylc.gui.app_gcylc import ControlApp - - # Make current working directory be $HOME. Otherwise if the CWD gets - # removed later while gcylc is running, subprocesses spawned by gcylc will - # fail when they attempt to determine their CWD. - os.chdir(os.environ['HOME']) - - gtk.settings_get_default().set_long_property( - "gtk-toolbar-icon-size", gtk.ICON_SIZE_SMALL_TOOLBAR, "main") - gtk.settings_get_default().set_long_property( - "gtk-button-images", True, "main") - gtk.settings_get_default().set_long_property( - "gtk-menu-images", True, "main") - - suite = None - if len(args) > 0: - suite = args[0] - if len(args) == 2: - try: - user_at_host, options.port = args[1].split(':') - options.owner, options.host = user_at_host.split('@') - except ValueError: - print >> sys.stderr, ('USER_AT_HOST must take the form ' - '"user@host:port"') - sys.exit(1) - ControlApp( - suite, options.owner, options.host, - options.port, options.comms_timeout, - load_template_vars(options.templatevars, options.templatevars_file), - options.restricted) - gtk.main() - - -if __name__ == "__main__": - try: - main() - except Exception as exc: - if cylc.flags.debug: - raise - sys.exit(str(exc)) diff --git a/bin/cylc-help b/bin/cylc-help index f991f642b2d..124c102e0b6 100755 --- a/bin/cylc-help +++ b/bin/cylc-help @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 # Copyright (C) 2008-2019 NIWA & British Crown (Met Office) & Contributors. # @@ -18,7 +18,7 @@ import sys # Import cylc to initialise CYLC_DIR and the path for python (__init__.py). -import cylc +from cylc.version import CYLC_VERSION class CommandError(Exception): @@ -128,49 +128,40 @@ def pretty_print(incom, choose_dict, indent=True, numbered=False, sort=False): if item not in incom: raise SystemExit("ERROR: summary for '" + item + "' not found") - print spacer, + print(spacer, end=' ') if numbered: count += 1 if pad and count < 10: digit = ' ' + str(count) else: digit = str(count) - print digit + '/', - print "%s %s %s" % ( + print(digit + '/', end=' ') + print("%s %s %s" % ( label[item], '.' * (longest - len(label[item])) + '...', - incom[item]) + incom[item])) def category_help(category): coms = eval(category + '_commands') alts = '|'.join(categories[category]) - print 'CATEGORY: ' + alts + ' - ' + catsum[category] - print - print 'HELP: cylc [' + alts + '] COMMAND help,--help' - print ' You can abbreviate ' + alts + ' and COMMAND.' - print ' The category ' + alts + ' may be omitted.' - print - print 'COMMANDS:' + print('CATEGORY: ' + alts + ' - ' + catsum[category]) + print() + print('HELP: cylc [' + alts + '] COMMAND help,--help') + print(' You can abbreviate ' + alts + ' and COMMAND.') + print(' The category ' + alts + ' may be omitted.') + print() + print('COMMANDS:') pretty_print(comsum, coms, sort=True) -def typo(str): - corrected = str - if str == 'gcycl': - corrected = 'gcylc' - return corrected - - # BEGIN MAIN -general_usage = """Cylc ("silk") is a workflow engine for orchestrating complex +general_usage = f"""Cylc ("silk") is a workflow engine for orchestrating complex *suites* of inter-dependent distributed cycling (repeating) tasks, as well as ordinary non-cycling workflows. For detailed documentation see the Cylc User Guide (cylc doc --help). -Version __CYLC_VERSION__ - -The graphical user interface for cylc is "gcylc" (a.k.a. "cylc gui"). +Version {CYLC_VERSION} USAGE: % cylc -V,--version,version # print cylc version @@ -227,9 +218,6 @@ categories['license'] = ['license', 'GPL'] information_commands = {} -information_commands['gscan'] = ['gscan', 'gsummary'] -information_commands['gpanel'] = ['gpanel'] -information_commands['gui'] = ['gui', 'gcylc'] information_commands['list'] = ['list', 'ls'] information_commands['dump'] = ['dump'] information_commands['cat-state'] = ['cat-state'] @@ -246,11 +234,9 @@ information_commands['monitor'] = ['monitor'] information_commands['get-suite-config'] = ['get-suite-config', 'get-config'] information_commands['get-site-config'] = [ 'get-site-config', 'get-global-config'] -information_commands['get-gui-config'] = ['get-gui-config'] information_commands['get-host-metrics'] = ['get-host-metrics'] control_commands = {} -control_commands['gui'] = ['gui'] # NOTE: don't change 'run' to 'start' or the category [control] # becomes compulsory to disambiguate from 'cylc [task] started'. # Keeping 'start' as an alias however: 'cylc con start'. @@ -383,19 +369,15 @@ comsum['cat-state'] = 'Print the state of tasks from the state dump' comsum['show'] = 'Print task state (prerequisites and outputs etc.)' comsum['cat-log'] = 'Print various suite and task log files' comsum['documentation'] = 'Display cylc documentation (User Guide etc.)' -comsum['monitor'] = 'An in-terminal suite monitor (see also gcylc)' +comsum['monitor'] = 'An in-terminal suite monitor' comsum['get-suite-config'] = 'Print suite configuration items' comsum['get-site-config'] = 'Print site/user configuration items' -comsum['get-gui-config'] = 'Print gcylc configuration items' comsum['get-host-metrics'] = 'Print localhost metric data' comsum['get-suite-contact'] = ( 'Print contact information of a suite server program') comsum['get-suite-version'] = 'Print cylc version of a suite server program' -comsum['gscan'] = 'Scan GUI for monitoring multiple suites' -comsum['gpanel'] = 'Internal interface for GNOME 2 panel applet' comsum['review'] = 'Start/stop ad-hoc Cylc Review web service server.' # control -comsum['gui'] = '(a.k.a. gcylc) cylc GUI for suite control etc.' comsum['run'] = 'Start a suite at a given cycle point' comsum['stop'] = 'Shut down running suites' comsum['restart'] = 'Restart a suite from a previous state' @@ -414,7 +396,7 @@ comsum['set-verbosity'] = 'Change a running suite\'s logging verbosity' comsum['broadcast'] = 'Change suite [runtime] settings on the fly' comsum['ext-trigger'] = 'Report an external trigger event to a suite' comsum['checkpoint'] = 'Tell suite to checkpoint its current state' -comsum['client'] = '(Internal) Invoke HTTP(S) client, expect JSON input' +comsum['client'] = '(Internal) Invoke suite runtime client, expect JSON input' # discovery comsum['ping'] = 'Check that a suite is running' comsum['scan'] = 'Scan a host for running suites' @@ -444,8 +426,7 @@ comsum['check-triggering'] = 'A suite shutdown event hook for cylc testing' if __name__ == "__main__": # no arguments: print help and exit if len(sys.argv) == 1: - from cylc.version import CYLC_VERSION - print usage.replace("__CYLC_VERSION__", CYLC_VERSION) + print(usage.replace("__CYLC_VERSION__", CYLC_VERSION)) pretty_print(catsum, categories) sys.exit(0) @@ -454,40 +435,34 @@ if __name__ == "__main__": if len(args) == 1: if args[0] == 'categories': # secret argument for document processing - keys = catsum.keys() - keys.sort() - for key in keys: - print key + for key in sorted(catsum): + print(key) sys.exit(0) if args[0] == 'commands': # secret argument for document processing - keys = comsum.keys() - keys.sort() - for key in keys: - print key + for key in sorted(comsum): + print(key) sys.exit(0) if args[0].startswith('category='): # secret argument for gcylc category = args[0][9:] commands = eval(category + '_commands') for command in commands: - print command + print(command) sys.exit(0) if is_help(args[0]): # cylc help - from cylc.version import CYLC_VERSION - print usage.replace("__CYLC_VERSION__", CYLC_VERSION) + print(usage) pretty_print(catsum, categories) sys.exit(0) if (args[0] in ['--version', '-V']): - from cylc.version import CYLC_VERSION - print CYLC_VERSION + print(CYLC_VERSION) sys.exit(0) # cylc CATEGORY with no args => category help try: category = match_dict(args[0], categories, 'CATEGORY') - except CommandError, x: + except CommandError: # No matching category # (no need to print this, the exception will recur below) # Carry on in case of a no-argument command (e.g. 'cylc scan') @@ -510,13 +485,13 @@ if __name__ == "__main__": item = args[1] try: category = match_dict(item, categories, 'CATEGORY') - except CommandError, x: + except CommandError as exc: # no matching category, try command try: - command = match_command(typo(item)) - except CommandError, y: - print >> sys.stderr, x - raise SystemExit(y) + command = match_command(item) + except CommandError as exc2: + print(exc, file=sys.stderr) + raise SystemExit(exc2) else: # cylc COMMAND --help command_args = ['--help'] diff --git a/bin/cylc-hold b/bin/cylc-hold index c99a02e6d11..5c7abad6e57 100755 --- a/bin/cylc-hold +++ b/bin/cylc-hold @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 # THIS FILE IS PART OF THE CYLC SUITE ENGINE. # Copyright (C) 2008-2019 NIWA & British Crown (Met Office) & Contributors. @@ -34,9 +34,9 @@ if '--use-ssh' in sys.argv[1:]: sys.exit(0) import cylc.flags -from cylc.prompt import prompt from cylc.option_parsers import CylcOptionParser as COP -from cylc.network.httpclient import SuiteRuntimeServiceClient +from cylc.network.client import SuiteRuntimeClient +from cylc.terminal import prompt def main(): @@ -61,20 +61,28 @@ def main(): 'Hold suite after %s' % options.hold_point_string, options.force) else: prompt('Hold suite %s' % suite, options.force) - pclient = SuiteRuntimeServiceClient( - suite, options.owner, options.host, options.port, - options.comms_timeout, my_uuid=options.set_uuid, - print_uuid=options.print_uuid) + + pclient = SuiteRuntimeClient( + suite, options.owner, options.host, options.port) + if args: items = parser.parse_multitask_compat(options, args) - pclient.put_command('hold_tasks', items=items) + pclient( + 'hold_tasks', + {'items': items}, + timeout=options.comms_timeout + ) elif options.hold_point_string: - pclient.put_command( + pclient( 'hold_after_point_string', - point_string=options.hold_point_string + {'point_string': options.hold_point_string}, + timeout=options.comms_timeout ) else: - pclient.put_command('hold_suite') + pclient( + 'hold_suite', + timeout=options.comms_timeout + ) if __name__ == "__main__": diff --git a/bin/cylc-insert b/bin/cylc-insert index 01f4b87dee1..361b8b53e49 100755 --- a/bin/cylc-insert +++ b/bin/cylc-insert @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 # THIS FILE IS PART OF THE CYLC SUITE ENGINE. # Copyright (C) 2008-2019 NIWA & British Crown (Met Office) & Contributors. @@ -41,10 +41,10 @@ if '--use-ssh' in sys.argv[1:]: sys.exit(0) import cylc.flags -from cylc.prompt import prompt from cylc.option_parsers import CylcOptionParser as COP -from cylc.network.httpclient import SuiteRuntimeServiceClient +from cylc.network.client import SuiteRuntimeClient from cylc.task_id import TaskID +from cylc.terminal import prompt def main(): @@ -84,14 +84,14 @@ def main(): item, i + 1)) prompt('Insert %s in %s' % (items, suite), options.force) - pclient = SuiteRuntimeServiceClient( - suite, options.owner, options.host, options.port, - options.comms_timeout, my_uuid=options.set_uuid, - print_uuid=options.print_uuid) + pclient = SuiteRuntimeClient( + suite, options.owner, options.host, options.port) - pclient.put_command( - 'insert_tasks', items=items, no_check=options.no_check, - stop_point_string=options.stop_point_string + pclient( + 'insert_tasks', + {'items': items, 'no_check': options.no_check, + 'stop_point_string': options.stop_point_string}, + timeout=options.comms_timeout ) diff --git a/bin/cylc-jobs-kill b/bin/cylc-jobs-kill index ecbfc4dd88f..6cd32e87193 100755 --- a/bin/cylc-jobs-kill +++ b/bin/cylc-jobs-kill @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 # THIS FILE IS PART OF THE CYLC SUITE ENGINE. # Copyright (C) 2008-2019 NIWA & British Crown (Met Office) & Contributors. diff --git a/bin/cylc-jobs-poll b/bin/cylc-jobs-poll index 84898605e30..d6b23b14779 100755 --- a/bin/cylc-jobs-poll +++ b/bin/cylc-jobs-poll @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 # THIS FILE IS PART OF THE CYLC SUITE ENGINE. # Copyright (C) 2008-2019 NIWA & British Crown (Met Office) & Contributors. diff --git a/bin/cylc-jobs-submit b/bin/cylc-jobs-submit index 9e69f0a39c8..dbbfafddc5c 100755 --- a/bin/cylc-jobs-submit +++ b/bin/cylc-jobs-submit @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 # THIS FILE IS PART OF THE CYLC SUITE ENGINE. # Copyright (C) 2008-2019 NIWA & British Crown (Met Office) & Contributors. diff --git a/bin/cylc-jobscript b/bin/cylc-jobscript index e1e9f92eb6f..b2c2449221d 100755 --- a/bin/cylc-jobscript +++ b/bin/cylc-jobscript @@ -47,7 +47,7 @@ __END__ } editor () { - python2 -c " + python3 -c " import sys from cylc.cfgspec.glbl_cfg import glbl_cfg if sys.argv[1] in ['-g', '--gedit']: diff --git a/bin/cylc-kill b/bin/cylc-kill index 53aca16a18b..52146224510 100755 --- a/bin/cylc-kill +++ b/bin/cylc-kill @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 # THIS FILE IS PART OF THE CYLC SUITE ENGINE. # Copyright (C) 2008-2019 NIWA & British Crown (Met Office) & Contributors. @@ -32,9 +32,9 @@ if '--use-ssh' in sys.argv[1:]: sys.exit(0) import cylc.flags -from cylc.prompt import prompt from cylc.option_parsers import CylcOptionParser as COP -from cylc.network.httpclient import SuiteRuntimeServiceClient +from cylc.network.client import SuiteRuntimeClient +from cylc.terminal import prompt def main(): @@ -52,12 +52,13 @@ def main(): prompt('Kill task %s in %s' % (args, suite), options.force) else: prompt('Kill ALL tasks in %s' % (suite), options.force) - pclient = SuiteRuntimeServiceClient( - suite, options.owner, options.host, options.port, - options.comms_timeout, my_uuid=options.set_uuid, - print_uuid=options.print_uuid) - items = parser.parse_multitask_compat(options, args) - pclient.put_command('kill_tasks', items=items) + pclient = SuiteRuntimeClient( + suite, options.owner, options.host, options.port) + pclient( + 'kill_tasks', + {'items': parser.parse_multitask_compat(options, args)}, + timeout=options.comms_timeout + ) if __name__ == "__main__": diff --git a/bin/cylc-list b/bin/cylc-list index 9283eae9a0b..20f9329b2ba 100755 --- a/bin/cylc-list +++ b/bin/cylc-list @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 # THIS FILE IS PART OF THE CYLC SUITE ENGINE. # Copyright (C) 2008-2019 NIWA & British Crown (Met Office) & Contributors. @@ -22,8 +22,9 @@ Print runtime namespace names (tasks and families), the first-parent inheritance graph, or actual tasks for a given cycle range. The first-parent inheritance graph determines the primary task family -groupings that are collapsible in gcylc suite views and the graph -viewer tool. To visualize the full multiple inheritance hierarchy use: +groupings that are collapsible in cylc visualisation tools. + +To visualize the full multiple inheritance hierarchy use: 'cylc graph -n'.""" import os @@ -101,7 +102,7 @@ def main(): if options.tree: if os.environ['LANG'] == 'C' and options.box: - print >> sys.stderr, "WARNING, ignoring -t/--tree: $LANG=C" + print("WARNING, ignoring -t/--tree: $LANG=C", file=sys.stderr) options.tree = False if options.titles and options.mro: @@ -109,7 +110,7 @@ def main(): if options.tree and any( [options.all_tasks, options.all_namespaces, options.mro]): - print >> sys.stderr, "WARNING: -t chosen, ignoring non-tree options." + print("WARNING: -t chosen, ignoring non-tree options.", file=sys.stderr) config = SuiteConfig( suite, suiterc, load_template_vars(options.templatevars, options.templatevars_file), @@ -119,10 +120,10 @@ def main(): pretty=options.box, titles=options.titles) elif options.crange: for node in sorted(config.get_node_labels(tr_start, tr_stop)): - print node + print(node) else: result = config.get_namespace_list(which) - namespaces = result.keys() + namespaces = list(result) namespaces.sort() if (options.mro or options.titles): @@ -135,15 +136,13 @@ def main(): for ns in namespaces: if options.mro: - print ns, padding[0:len(padding) - len(ns)], - for i in config.get_mro(ns): - print i, - print + print(ns, padding[0:len(padding) - len(ns)], end=' ') + print(' '.join(config.get_mro(ns))) elif options.titles: - print ns, padding[0:len(padding) - len(ns)], - print result[ns] + print(ns, padding[0:len(padding) - len(ns)], end=' ') + print(result[ns]) else: - print ns + print(ns) if __name__ == "__main__": diff --git a/bin/cylc-ls-checkpoints b/bin/cylc-ls-checkpoints index 6657f82322e..ba7cd998232 100755 --- a/bin/cylc-ls-checkpoints +++ b/bin/cylc-ls-checkpoints @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 # THIS FILE IS PART OF THE CYLC SUITE ENGINE. # Copyright (C) 2008-2019 NIWA & British Crown (Met Office) & Contributors. diff --git a/bin/cylc-message b/bin/cylc-message index 0190f1f23d0..3cae35b1369 100755 --- a/bin/cylc-message +++ b/bin/cylc-message @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 # THIS FILE IS PART OF THE CYLC SUITE ENGINE. # Copyright (C) 2008-2019 NIWA & British Crown (Met Office) & Contributors. diff --git a/bin/cylc-monitor b/bin/cylc-monitor index e07064c4706..8d210f9d1cf 100755 --- a/bin/cylc-monitor +++ b/bin/cylc-monitor @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 # THIS FILE IS PART OF THE CYLC SUITE ENGINE. # Copyright (C) 2008-2019 NIWA & British Crown (Met Office) & Contributors. @@ -35,7 +35,7 @@ from time import sleep, time from parsec.OrderedDict import OrderedDict from cylc.option_parsers import CylcOptionParser as COP -from cylc.network.httpclient import SuiteRuntimeServiceClient, ClientError +from cylc.network.client import SuiteRuntimeClient, ClientError from cylc.cfgspec.glbl_cfg import glbl_cfg from cylc.task_state import ( TASK_STATUS_RUNAHEAD, TASK_STATUSES_ORDERED, @@ -102,7 +102,7 @@ The USER_AT_HOST argument allows suite selection by 'cylc scan' output: dest="update_interval") def reset(self, suite, owner, host, port, timeout): - self.pclient = SuiteRuntimeServiceClient( + self.pclient = SuiteRuntimeClient( suite, owner, host, port, timeout) def run(self): @@ -114,8 +114,8 @@ The USER_AT_HOST argument allows suite selection by 'cylc scan' output: user_at_host, options.port = args[1].split(':') options.owner, options.host = user_at_host.split('@') except ValueError: - print >> sys.stderr, ('USER_AT_HOST must take the form ' - '"user@host:port"') + print(('USER_AT_HOST must take the form ' + '"user@host:port"'), file=sys.stderr) sys.exit(1) client_name = os.path.basename(sys.argv[0]) @@ -142,15 +142,15 @@ The USER_AT_HOST argument allows suite selection by 'cylc scan' output: is_cont = True try: glbl, task_summaries = ( - self.pclient.get_suite_state_summary()[0:2]) + self.pclient('get_suite_state_summary')[0:2]) except ClientError as exc: - print >> sys.stderr, "\033[1;37;41mERROR\033[0m", str(exc) + print("\033[1;37;41mERROR\033[0m", str(exc), file=sys.stderr) self.reset(suite, options.owner, options.host, options.port, options.comms_timeout) else: if not glbl: - print >> sys.stderr, ( - "\033[1;37;41mWARNING\033[0m suite initialising") + print(( + "\033[1;37;41mWARNING\033[0m suite initialising"), file=sys.stderr) continue states = [t["state"] for t in task_summaries.values() if ( "state" in t)] @@ -178,7 +178,7 @@ The USER_AT_HOST argument allows suite selection by 'cylc scan' output: task_info = {} name_list = set() - task_ids = task_summaries.keys() + task_ids = list(task_summaries) for task_id in task_ids: name = task_summaries[task_id]['name'] point_string = task_summaries[task_id]['label'] @@ -210,7 +210,7 @@ The USER_AT_HOST argument allows suite selection by 'cylc scan' output: if run_mode != "live": suite_name += " (%s)" % run_mode prefix = "%s - %d tasks" % (suite_name, int(n_tasks_total)) - suffix = "%s %s" % (client_name, self.pclient.my_uuid) + suffix = "%s" % client_name title_str = ' ' * len_header title_str = prefix + title_str[len(prefix):] title_str = '\033[1;37;44m%s%s\033[0m' % ( @@ -254,8 +254,8 @@ The USER_AT_HOST argument allows suite selection by 'cylc scan' output: if not options.once: os.system("clear") - print '\n'.join(blit) - indxs = blitlines.keys() + print('\n'.join(blit)) + indxs = list(blitlines) try: int(indxs[1]) except (ValueError, IndexError): @@ -263,7 +263,7 @@ The USER_AT_HOST argument allows suite selection by 'cylc scan' output: else: indxs.sort(key=int) for ix in indxs: - print blitlines[ix] + print(blitlines[ix]) if __name__ == "__main__": diff --git a/bin/cylc-nudge b/bin/cylc-nudge index 24402a8f8ba..79ef0124467 100755 --- a/bin/cylc-nudge +++ b/bin/cylc-nudge @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 # THIS FILE IS PART OF THE CYLC SUITE ENGINE. # Copyright (C) 2008-2019 NIWA & British Crown (Met Office) & Contributors. @@ -25,8 +25,7 @@ task processing (dependency negotation etc.) is required, or if a clock-trigger task is ready to run. The main reason to use this command is to update the "estimated time till -completion" intervals shown in the tree-view suite control GUI, during -periods when nothing else is happening. +completion" intervals, during periods when nothing else is happening. """ import sys @@ -38,7 +37,7 @@ if '--use-ssh' in sys.argv[1:]: import cylc.flags from cylc.option_parsers import CylcOptionParser as COP -from cylc.network.httpclient import SuiteRuntimeServiceClient +from cylc.network.client import SuiteRuntimeClient def main(): @@ -47,14 +46,13 @@ def main(): (options, args) = parser.parse_args() suite = args[0] - pclient = SuiteRuntimeServiceClient( + pclient = SuiteRuntimeClient( suite, options.owner, options.host, options.port, - options.comms_timeout, my_uuid=options.set_uuid, - print_uuid=options.print_uuid) + options.comms_timeout) - success, msg = pclient.put_command('nudge') + success, msg = pclient('nudge') if success: - print msg + print(msg) else: sys.exit(msg) diff --git a/bin/cylc-ping b/bin/cylc-ping index 925efa0128c..6a67b88f47d 100755 --- a/bin/cylc-ping +++ b/bin/cylc-ping @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 # THIS FILE IS PART OF THE CYLC SUITE ENGINE. # Copyright (C) 2008-2019 NIWA & British Crown (Met Office) & Contributors. @@ -31,7 +31,7 @@ if '--use-ssh' in sys.argv[1:]: import cylc.flags from cylc.option_parsers import CylcOptionParser as COP from cylc.task_id import TaskID -from cylc.network.httpclient import SuiteRuntimeServiceClient +from cylc.network.client import SuiteRuntimeClient def main(): @@ -42,16 +42,16 @@ def main(): (options, args) = parser.parse_args() suite = args[0] - pclient = SuiteRuntimeServiceClient( + pclient = SuiteRuntimeClient( suite, options.owner, options.host, options.port, - options.comms_timeout, my_uuid=options.set_uuid, - print_uuid=options.print_uuid, - auth=SuiteRuntimeServiceClient.ANON_AUTH) + options.comms_timeout) # cylc ping SUITE - pclient.get_info('ping_suite') # (no need to check the result) + pclient('ping_suite') # (no need to check the result) if cylc.flags.verbose: - sys.stdout.write("Running on %s:%s\n" % (pclient.host, pclient.port)) + host, port = SuiteRuntimeClient.get_location( + suite, options.owner, options.host) + sys.stdout.write("Running on %s:%s\n" % (host, port)) if len(args) == 1: sys.exit(0) @@ -59,7 +59,7 @@ def main(): task_id = args[1] if not TaskID.is_valid_id(task_id): sys.exit("Invalid task ID: " + task_id) - success, msg = pclient.get_info('ping_task', task_id=task_id) + success, msg = pclient('ping_task', {'task_id': task_id}) if not success: sys.exit('ERROR: ' + msg) diff --git a/bin/cylc-poll b/bin/cylc-poll index 2860a171661..366545fb2c7 100755 --- a/bin/cylc-poll +++ b/bin/cylc-poll @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 # THIS FILE IS PART OF THE CYLC SUITE ENGINE. # Copyright (C) 2008-2019 NIWA & British Crown (Met Office) & Contributors. @@ -32,9 +32,9 @@ if '--use-ssh' in sys.argv[1:]: sys.exit(0) import cylc.flags -from cylc.prompt import prompt from cylc.option_parsers import CylcOptionParser as COP -from cylc.network.httpclient import SuiteRuntimeServiceClient +from cylc.network.client import SuiteRuntimeClient +from cylc.terminal import prompt def main(): @@ -55,18 +55,16 @@ def main(): prompt('Poll task %s in %s' % (args, suite), options.force) else: prompt('Poll ALL tasks in %s' % (suite), options.force) - pclient = SuiteRuntimeServiceClient( + pclient = SuiteRuntimeClient( suite, options.owner, options.host, options.port, - options.comms_timeout, my_uuid=options.set_uuid, - print_uuid=options.print_uuid) + options.comms_timeout) items = parser.parse_multitask_compat(options, args) # Back compat: back_out introduced >7.5.0 # So don't call with "poll_succ" if not necessary to avoid breakage. if options.poll_succ: - pclient.put_command( - 'poll_tasks', items=items, poll_succ=options.poll_succ) + pclient('poll_tasks', {'items': items, 'poll_succ': options.poll_succ}) else: - pclient.put_command('poll_tasks', items=items) + pclient('poll_tasks', {'items': items}) if __name__ == "__main__": diff --git a/bin/cylc-print b/bin/cylc-print index b86e8b76893..6205fe7233a 100755 --- a/bin/cylc-print +++ b/bin/cylc-print @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 # THIS FILE IS PART OF THE CYLC SUITE ENGINE. # Copyright (C) 2008-2019 NIWA & British Crown (Met Office) & Contributors. @@ -118,7 +118,7 @@ def main(): line = suite + ' | ' + dir_ else: line = suite - print line + print(line) else: tree = {} if options.align: diff --git a/bin/cylc-profile-battery b/bin/cylc-profile-battery index f1bb8fc1905..22ffc2753b2 100755 --- a/bin/cylc-profile-battery +++ b/bin/cylc-profile-battery @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 # THIS FILE IS PART OF THE CYLC SUITE ENGINE. # Copyright (C) 2008-2019 NIWA & British Crown (Met Office) & Contributors. # @@ -186,7 +186,7 @@ def get_results(): try: profile_results = json.load(profile_file) except ValueError as exc: - print exc + print(exc) sys.exit('ERROR: Could not read "%s". Check that it is valid' ' JSON or delete the file.' % profile_file_path) return profile_results @@ -196,7 +196,7 @@ def get_result_keys(): """Return a list of (version_id, experiment_id,) tuples.""" profile_results = get_results() result_keys = [] - for version_id, experiment_ids in profile_results.iteritems(): + for version_id, experiment_ids in profile_results.items(): result_keys.extend([(version_id, experiment_id) for experiment_id in experiment_ids.keys()]) return result_keys @@ -278,7 +278,7 @@ def load_experiment_config(experiment_file): run['suite dir']) run['suite dir'] = os.path.realpath(run['suite dir']) except KeyError as exc: - print exc + print(exc) sys.exit('Error: Experiment definition not complete.') # Apply defaults. @@ -377,8 +377,8 @@ def install_experiments(experiment_ids, experiments, install_dir, for sdir, append_codicil in suite_dirs: key = (sdir, append_codicil,) for run in suite_dirs[key]['runs']: - print 'installing suite "%s" => "%s"' % ( - sdir, suite_dirs[key]['install dir']) + print('installing suite "%s" => "%s"' % ( + sdir, suite_dirs[key]['install dir'])) run['suite dir'] = suite_dirs[key]['install dir'] # Global config sourcing. @@ -429,8 +429,8 @@ def get_experiments(experiment_names): file_name) if not os.path.exists(file_path): # Could not find experiment file in either path. Exit! - print 'ERROR: Could not find experiment file for "%s"' % ( - experiment_name) + print('ERROR: Could not find experiment file for "%s"' % ( + experiment_name)) experiments.append({'name': experiment_name, 'id': 'Invalid', 'file': None}) @@ -455,8 +455,8 @@ def print_manual_scheme(versions, experiments, all_versions=None): ver = ' '.join(version['id'] for version in versions) exp = ' '.join(experiment['name'] for experiment in experiments) for version in versions: - print '\t$ git checkout ' + version['id'] - print '\t$ cylc profile-battery --experiments ' + exp + print('\t$ git checkout ' + version['id']) + print('\t$ cylc profile-battery --experiments ' + exp) print('\t$ cylc profile-battery --versions {versions} --experiments ' '{experiments}'.format(versions=ver, experiments=exp)) @@ -499,13 +499,14 @@ def determine_action(schedule, versions, experiments, non_interactive=False): ' checkpointing in the main loop.\n') usr = None while usr not in ['y', 'n']: - usr = raw_input('proceed? (y/n): ') + usr = input('proceed? (y/n): ') if usr == 'n': sys.exit('Profiling aborted by user.') - print + print() elif temp: - print >> sys.stderr, ('WARNING: You are using profile-mode ' - '"cylc" with older versions of cylc.') + print(('WARNING: You are using profile-mode ' + '"cylc" with older versions of cylc.'), + file=sys.stderr) # Prompt user over using automated checkout. to_checkout = [] @@ -538,13 +539,13 @@ def determine_action(schedule, versions, experiments, non_interactive=False): ' '.join(version['name'] for version in manual_versions))) - print + print() if manual_versions and not automatic_only_versions: response = None while response not in ['y', 'n']: - response = raw_input('Do you want to checkout these versions ' - 'automatically? (y/n): ') + response = input('Do you want to checkout these versions ' + 'automatically? (y/n): ') if response == 'n': print('You can perform this profiling manually by doing ' 'something like:') @@ -556,13 +557,13 @@ def determine_action(schedule, versions, experiments, non_interactive=False): elif manual_versions and automatic_only_versions: response = None while response not in ['some', 'all', 'none']: - response = raw_input( + response = input( 'Which versions should I check out:\n\t' 'Only those which cannot be profiled otherwise (some)\n\t' 'All versions (all)\n\t' 'None (none)\n> ') if response == 'some': - print 'The remainder can be profiled by doing something like:' + print('The remainder can be profiled by doing something like:') print_manual_scheme(manual_versions, experiments, all_versions=versions) to_checkout = automatic_only_versions @@ -577,8 +578,8 @@ def determine_action(schedule, versions, experiments, non_interactive=False): elif automatic_only_versions: response = None while response not in ['y', 'n']: - response = raw_input('Do you want to checkout these versions ' - 'automatically? (y/n): ') + response = input('Do you want to checkout these versions ' + 'automatically? (y/n): ') if response == 'y': to_checkout = automatic_only_versions else: @@ -593,9 +594,9 @@ def update_nested_dictionaries(old, new): of a conflict.""" old = old.copy() new = new.copy() - for key, value in new.iteritems(): + for key, value in new.items(): if key in old: - if type(value) is dict: + if isinstance(value, dict): old[key] = update_nested_dictionaries(old[key], new[key]) else: old[key] = value @@ -629,7 +630,7 @@ def delete_results(result_keys, interactive=False): if interactive: usr = None while usr not in ['y', 'n']: - usr = raw_input('Delete these results (y/n)? ') + usr = input('Delete these results (y/n)? ') if usr != 'y': sys.exit(0) @@ -656,7 +657,7 @@ def install_profiler(): try: # Temp dir to install files to tempdir = tempfile.mkdtemp() - print 'Installing profiler to:', tempdir + print('Installing profiler to:', tempdir) shutil.copytree( os.path.join(prof.CYLC_DIR, 'lib', 'cylc', 'profiling'), @@ -668,7 +669,7 @@ def install_profiler(): sys.path.insert(0, os.path.join(tempdir, 'tempprofiling')) except Exception as exc: # Slightest hint of trouble => abort. - print exc + print(exc) sys.exit('ERROR: Problem installing profiling.') return tempdir @@ -738,8 +739,8 @@ def run_schedule(schedule, experiments, versions, exps_to_run, raise GitCheckoutError() checkout(r'@{-%d}' % checkout_count, delete_pyc=True) except GitCheckoutError: - print('ERROR: Could not checkout git repo to original location. ' - r'\n\t$ git checkout @{-%d}' % checkout_count) + print(('ERROR: Could not checkout git repo to original location. ' + r'\n\t$ git checkout @{-%d}' % checkout_count)) # Stop here if profiling was un-successfull. if not success: @@ -783,13 +784,13 @@ def run_analysis(experiments, versions, interactive=False, os.makedirs(plt_dir) # Print a table of results. - print + print() print_table( make_table(full_results, versions, experiment, quick_analysis=quick_analysis), transpose=not quick_analysis ) - print + print() # Plot results. if not plot: @@ -797,8 +798,9 @@ def run_analysis(experiments, versions, interactive=False, plot_results(full_results, versions, experiment, plt_dir, quick_analysis=quick_analysis, lobf_order=lobf_order) if plt_dir: - print('Results for experiment "{exp}" have been written out to ' - '"{dir}"'.format(exp=experiment['name'], dir=plt_dir)) + print( + f'Results for experiment "{experiment["name"]}" have been ' + f'written out to "{plt_dir}"') def ls(exp_names, ver_names, delete=False): @@ -830,7 +832,7 @@ def ls(exp_names, ver_names, delete=False): include_result(experiment_name, experiment_id, version_id) else: # List only specified experiments and versions. - version_ids = map(git.describe, ver_names) + version_ids = list(map(git.describe, ver_names)) experiment_ids = set(name for name in exp_names if '@' in name) experiment_names = set(exp_names) - experiment_ids @@ -845,7 +847,7 @@ def ls(exp_names, ver_names, delete=False): git.order_identifiers_by_date(all_versions) - experiments = get_experiments(include.keys()) + experiments = get_experiments(list(include)) current_experiment_ids = [] for experiment in experiments: current_experiment_ids.append(experiment['id']) @@ -905,31 +907,30 @@ def promote(experiment_id, yes=False): if target_versions: candidate_versions = [version for version in candidate_versions if version not in target_versions] - print + print() print('Only the results for cylc versions not already profiled in ' 'the current experiment version will be promoted.') git.order_identifiers_by_date(candidate_versions) - print - print('Promote the following results for experiment "{name}" at version ' - '"{candidate}" to the current version "{target}":'.format( - name=experiment_name, - candidate=experiment_version, - target=cur_exp_id.rsplit('@', 1)[1] - )) - print '\t', ' '.join(candidate_versions) + print() + print(('Promote the following results for experiment "{name}" at version ' + '"{candidate}" to the current version "{target}":'.format( + name=experiment_name, + candidate=experiment_version, + target=cur_exp_id.rsplit('@', 1)[1]))) + print('\t', ' '.join(candidate_versions)) if not yes: response = None while response not in ['y', 'n']: - response = raw_input('Upgrade these versions? (y/n): ') + response = input('Upgrade these versions? (y/n): ') if yes or response == 'y': # Promote results. try: for version in candidate_versions: results[version][cur_exp_id] = results[version][experiment_id] except KeyError as exc: - print exc + print(exc) sys.exit('Unexpected error.') else: append_new_results(results) @@ -944,8 +945,8 @@ def main(): opts = parse_args() if not prof.IS_GIT_REPO: - print >> sys.stderr, ('ERROR: profiling requires cylc to be a git ' - 'repository.') + print('ERROR: profiling requires cylc to be a git repository.', + file=sys.stderr) sys.exit(2) # Promote mode. diff --git a/bin/cylc-register b/bin/cylc-register index 976e4db9f3e..0e8a2174174 100755 --- a/bin/cylc-register +++ b/bin/cylc-register @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 # THIS FILE IS PART OF THE CYLC SUITE ENGINE. # Copyright (C) 2008-2019 NIWA & British Crown (Met Office) & Contributors. diff --git a/bin/cylc-release b/bin/cylc-release index 0d1ecdbf110..469e2646c21 100755 --- a/bin/cylc-release +++ b/bin/cylc-release @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 # THIS FILE IS PART OF THE CYLC SUITE ENGINE. # Copyright (C) 2008-2019 NIWA & British Crown (Met Office) & Contributors. @@ -33,9 +33,9 @@ if '--use-ssh' in sys.argv[1:]: sys.exit(0) import cylc.flags -from cylc.prompt import prompt from cylc.option_parsers import CylcOptionParser as COP -from cylc.network.httpclient import SuiteRuntimeServiceClient +from cylc.network.client import SuiteRuntimeClient +from cylc.terminal import prompt def main(): @@ -52,15 +52,14 @@ def main(): prompt('Release task(s) %s in %s' % (args, suite), options.force) else: prompt('Release suite %s' % suite, options.force) - pclient = SuiteRuntimeServiceClient( + pclient = SuiteRuntimeClient( suite, options.owner, options.host, options.port, - options.comms_timeout, my_uuid=options.set_uuid, - print_uuid=options.print_uuid) + options.comms_timeout) if args: items = parser.parse_multitask_compat(options, args) - pclient.put_command('release_tasks', items=items) + pclient('release_tasks', {'items': items}) else: - pclient.put_command('release_suite') + pclient('release_suite') if __name__ == "__main__": diff --git a/bin/cylc-reload b/bin/cylc-reload index 257b7a29aac..0f94e2999e7 100755 --- a/bin/cylc-reload +++ b/bin/cylc-reload @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 # THIS FILE IS PART OF THE CYLC SUITE ENGINE. # Copyright (C) 2008-2019 NIWA & British Crown (Met Office) & Contributors. @@ -48,8 +48,8 @@ if '--use-ssh' in sys.argv[1:]: import cylc.flags from cylc.option_parsers import CylcOptionParser as COP -from cylc.network.httpclient import SuiteRuntimeServiceClient -from cylc.prompt import prompt +from cylc.network.client import SuiteRuntimeClient +from cylc.terminal import prompt def main(): @@ -58,11 +58,10 @@ def main(): suite = args[0] prompt('Reload %s' % suite, options.force) - pclient = SuiteRuntimeServiceClient( + pclient = SuiteRuntimeClient( suite, options.owner, options.host, options.port, - options.comms_timeout, my_uuid=options.set_uuid, - print_uuid=options.print_uuid) - pclient.put_command('reload_suite') + options.comms_timeout) + pclient('reload_suite') if __name__ == "__main__": diff --git a/bin/cylc-remote-init b/bin/cylc-remote-init index 41b3b720ae1..54843273767 100755 --- a/bin/cylc-remote-init +++ b/bin/cylc-remote-init @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 # THIS FILE IS PART OF THE CYLC SUITE ENGINE. # Copyright (C) 2008-2019 NIWA & British Crown (Met Office) & Contributors. @@ -20,8 +20,7 @@ (This command is for internal use.) Install suite service files on a task remote (i.e. a [owner@]host): .service/contact: All task -> suite communication methods. - .service/passphrase: Direct task -> suite HTTP(S) communication only. - .service/ssl.cert: Direct task -> suite HTTPS communication only. + .service/passphrase: Direct task -> suite communication only. Content of items to install from a tar file read from STDIN. diff --git a/bin/cylc-remote-tidy b/bin/cylc-remote-tidy index 09df2269030..3ce0d63ebc7 100755 --- a/bin/cylc-remote-tidy +++ b/bin/cylc-remote-tidy @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 # THIS FILE IS PART OF THE CYLC SUITE ENGINE. # Copyright (C) 2008-2019 NIWA & British Crown (Met Office) & Contributors. diff --git a/bin/cylc-remove b/bin/cylc-remove index e7688661eb0..3906caa41c8 100755 --- a/bin/cylc-remove +++ b/bin/cylc-remove @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 # THIS FILE IS PART OF THE CYLC SUITE ENGINE. # Copyright (C) 2008-2019 NIWA & British Crown (Met Office) & Contributors. @@ -32,9 +32,9 @@ if '--use-ssh' in sys.argv[1:]: sys.exit(0) import cylc.flags -from cylc.prompt import prompt from cylc.option_parsers import CylcOptionParser as COP -from cylc.network.httpclient import SuiteRuntimeServiceClient +from cylc.network.client import SuiteRuntimeClient +from cylc.terminal import prompt def main(): @@ -53,13 +53,14 @@ def main(): suite = args.pop(0) prompt('remove task(s) %s in %s' % (args, suite), options.force) - pclient = SuiteRuntimeServiceClient( + pclient = SuiteRuntimeClient( suite, options.owner, options.host, options.port, - options.comms_timeout, my_uuid=options.set_uuid, - print_uuid=options.print_uuid) + options.comms_timeout) items = parser.parse_multitask_compat(options, args) - pclient.put_command('remove_tasks', items=items, - spawn=(not options.no_spawn)) + pclient( + 'remove_tasks', + {'items': items, 'spawn': (not options.no_spawn)} + ) if __name__ == "__main__": diff --git a/bin/cylc-report-timings b/bin/cylc-report-timings index 510f4362419..2feb74bb0d2 100755 --- a/bin/cylc-report-timings +++ b/bin/cylc-report-timings @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 # THIS FILE IS PART OF THE CYLC SUITE ENGINE. # Copyright (C) 2008-2019 NIWA & British Crown (Met Office) & Contributors. @@ -52,7 +52,7 @@ from cylc.remote import remrun if remrun(): sys.exit(0) -import cStringIO as StringIO +import io as StringIO import collections import contextlib import os diff --git a/bin/cylc-reset b/bin/cylc-reset index f4ebfd3a8cb..357f7f8ca40 100755 --- a/bin/cylc-reset +++ b/bin/cylc-reset @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 # THIS FILE IS PART OF THE CYLC SUITE ENGINE. # Copyright (C) 2008-2019 NIWA & British Crown (Met Office) & Contributors. @@ -41,10 +41,10 @@ if '--use-ssh' in sys.argv[1:]: sys.exit(0) import cylc.flags -from cylc.prompt import prompt from cylc.option_parsers import CylcOptionParser as COP -from cylc.network.httpclient import SuiteRuntimeServiceClient +from cylc.network.client import SuiteRuntimeClient from cylc.task_state import TASK_STATUSES_CAN_RESET_TO +from cylc.terminal import prompt def main(): @@ -93,14 +93,14 @@ def main(): options.state = '' prompt('Reset task(s) %s in %s' % (args, suite), options.force) - pclient = SuiteRuntimeServiceClient( + pclient = SuiteRuntimeClient( suite, options.owner, options.host, options.port, - options.comms_timeout, my_uuid=options.set_uuid, - print_uuid=options.print_uuid) + options.comms_timeout) items = parser.parse_multitask_compat(options, args) - pclient.put_command( - 'reset_task_states', items=items, state=options.state, - outputs=options.outputs) + pclient( + 'reset_task_states', + {'items': items, 'state': options.state, 'outputs': options.outputs} + ) if __name__ == "__main__": diff --git a/bin/cylc-restart b/bin/cylc-restart index b26fb29c79c..7d7f19a8d25 100755 --- a/bin/cylc-restart +++ b/bin/cylc-restart @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 # THIS FILE IS PART OF THE CYLC SUITE ENGINE. # Copyright (C) 2008-2019 NIWA & British Crown (Met Office) & Contributors. diff --git a/bin/cylc-review b/bin/cylc-review deleted file mode 100755 index fcebb6f4c11..00000000000 --- a/bin/cylc-review +++ /dev/null @@ -1,50 +0,0 @@ -#!/usr/bin/env python2 - -# THIS FILE IS PART OF THE CYLC SUITE ENGINE. -# Copyright (C) 2008-2019 NIWA & British Crown (Met Office) & Contributors. -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -"""cylc [info] review [OPTIONS] ARGS - -Start/stop ad-hoc Cylc Review web service server for browsing users' suite -logs via an HTTP interface. - -With no arguments, the status of the ad-hoc web service server is printed. - -For 'cylc review start', if 'PORT' is not specified, port 8080 is used.""" - - -import sys - -import cylc.flags -from cylc.review import CylcReviewService - - -def main(): - from cylc.ws import ws_cli - ws_cli(CylcReviewService, __doc__) - - -if __name__ == "__main__": - try: - main() - except Exception as exc: - if cylc.flags.debug: - raise - sys.exit(str(exc)) -elif 'doctest' not in sys.argv[0]: - # If called as a module but not by the doctest module. - from cylc.ws import wsgi_app - application = wsgi_app(CylcReviewService) diff --git a/bin/cylc-run b/bin/cylc-run index 034145b03b9..a92c5baf900 100755 --- a/bin/cylc-run +++ b/bin/cylc-run @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 # THIS FILE IS PART OF THE CYLC SUITE ENGINE. # Copyright (C) 2008-2019 NIWA & British Crown (Met Office) & Contributors. diff --git a/bin/cylc-scan b/bin/cylc-scan index a1bf3673239..83c0db328d3 100755 --- a/bin/cylc-scan +++ b/bin/cylc-scan @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 # THIS FILE IS PART OF THE CYLC SUITE ENGINE. # Copyright (C) 2008-2019 NIWA & British Crown (Met Office) & Contributors. @@ -20,17 +20,9 @@ Print information about running suites. -By default, it will obtain a listing of running suites for the current user -from the file system, before connecting to the suites to obtain information. Use the -o/--suite-owner option to get information of running suites for other users. -If a list of HOSTS is specified, it will obtain a listing of running suites by -scanning all ports in the relevant range for running suites on the specified -hosts. If the -a/--all option is specified, it will use the global -configuration "[suite servers]scan hosts" setting to determine a list of hosts -to scan. - Suite passphrases are not needed to get identity information (name and owner). Titles, descriptions, state totals, and cycle point state totals may also be revealed publicly, depending on global and suite authentication settings. Suite @@ -48,46 +40,37 @@ if "--use-ssh" in sys.argv[1:]: import json -from cylc.cfgspec.glbl_cfg import glbl_cfg -from cylc.hostuserutil import get_user -from cylc.network.port_scan import ( +from colorama import Fore, Style, init as color_init + +from cylc.network.scan import ( get_scan_items_from_fs, re_compile_filters, scan_many) from cylc.option_parsers import CylcOptionParser as COP -from cylc.suite_status import ( - KEY_DESCRIPTION, KEY_META, KEY_NAME, KEY_OWNER, KEY_STATES, - KEY_TITLE, KEY_UPDATE_TIME) +from cylc.suite_status import KEY_META, KEY_NAME, KEY_OWNER, KEY_STATES from cylc.task_state import TASK_STATUSES_ORDERED from cylc.task_state_prop import get_status_prop +from cylc import terminal -class TitleFormatter(object): - """Format title line, in bold or not if .no_bold set to False.""" - - def __init__(self, no_bold=False): - self.no_bold = no_bold - - def __call__(self, line): - """Add terminal control characters for bold text, if .no_bold=False""" - if self.no_bold: - return line - else: - return "\033[1m" + line + "\033[0m" +INDENT = " " +TITLE_STYLE = Style.BRIGHT +MISSING_STYLE = Fore.MAGENTA +ERROR_STYLE = Fore.RED +META_KEY_ORDER = ['title', 'description', 'group'] -def main(): - """Implement "cylc scan".""" +def get_parser(): + """CLI opts for "cylc scan".""" parser = COP( __doc__, comms=True, noforce=True, - argdoc=[( - "[HOSTS ...]", "Hosts to scan instead of the configured hosts.")] + argdoc=[] ) parser.add_option( - "-a", "--all", - help="Scan all port ranges in known hosts.", - action="store_true", default=False, dest="all_ports") + '--ordered', + help='Display results in order, this may take longer.', + action='store_true') parser.add_option( "-n", "--name", @@ -113,217 +96,193 @@ def main(): "-s", "--state-totals", help="Print number of tasks in each state if available " "(total, and by cycle point).", - action="store_true", default=False, dest="print_totals") + action="store_true", default=False, dest="state_totals") parser.add_option( "-f", "--full", help="Print all available information about each suite.", - action="store_true", default=False, dest="print_full") - - parser.add_option( - "-c", "--color", "--colour", - help="Print task state summaries using terminal color control codes.", - action="store_true", default=False, dest="color") - - parser.add_option( - "-b", "--no-bold", - help="Don't use any bold text in the command output.", - action="store_true", default=False, dest="no_bold") + action="store_true", default=False, dest="full") parser.add_option( - "--print-ports", - help="Print the port range from the global config file.", - action="store_true", default=False, dest="print_ports") + '--color', '--colour', action='store', dest='color', default='auto', + help='Colorize the output, can be "always", "never" or "auto".') parser.add_option( "--comms-timeout", metavar="SEC", help="Set a timeout for network connections " "to each running suite. The default is 5 seconds.", - action="store", default=None, dest="comms_timeout") + action="store", default=5.0, dest="comms_timeout") - parser.add_option( - "--old", "--old-format", - help='Legacy output format ("suite owner host port").', - action="store_true", default=False, dest="old_format") - - parser.add_option( - "-r", "--raw", "--raw-format", - help='Parsable format ("suite|owner|host|property|value").', - action="store_true", default=False, dest="raw_format") + parser.add_option('-t', '--format', default='plain', action='store', + dest='format', help=( + 'Set output format:\n' + ' * plain (default) - text format for interactive use\n' + ' * raw - parsable format (suite|owner|host|property|value)\n' + ' * json - JSON format ({suite: {owner: OWNER, host: HOST ...)')) - parser.add_option( - "-j", "--json", "--json-format", - help="JSON format.", - action="store_true", default=False, dest="json_format" - ) + return parser - options, args = parser.parse_args() - if options.print_ports: - run_ports_list = glbl_cfg().get(['suite servers', 'run ports']) - print run_ports_list[0], '<= port <=', run_ports_list[-1] - sys.exit(0) - - indent = " " - - bold = TitleFormatter(options.no_bold) - - if options.print_full: - options.describe = options.print_totals = True - - if options.color: - options.print_totals = True +def main(): + """Implement "cylc scan".""" + parser = get_parser() + options, _ = parser.parse_args() - # Check for incompatible formatting options - incompatibles = [options.raw_format, options.json_format] - format_ops = [options.old_format, options.describe, options.print_totals, - options.color] - if incompatibles.count(True) == 2: - parser.error("--raw and --json are incompatible formatting options.") - if incompatibles.count(True) == 1 and format_ops.count(True) >= 1: - parser.error("Incompatible format options: --%s forbidden with any " - "of --old-format, --describe, --state-totals or --color." - % ("raw" if incompatibles[0] else "json")) + if options.full: + options.describe = options.state_totals = True + if options.format in ['raw', 'json']: + options.color = False - state_legend = "" - if options.color: - n_states = len(TASK_STATUSES_ORDERED) - for index, state in enumerate(TASK_STATUSES_ORDERED): - state_legend += get_status_prop(state, 'ascii_ctrl') - if index == n_states / 2: - state_legend += "\n" - state_legend = state_legend.rstrip() + # color settings + if options.color in ['auto', 'always'] and terminal.supports_color(): + options.color = True + else: + options.color = False + color_init(autoreset=True, strip=not options.color) + # name and owner patterns if options.patterns_name: patterns_name = options.patterns_name else: - # Any suite name. - patterns_name = ['.*'] + patterns_name = ['.*'] # Any suite name. patterns_owner = None if options.patterns_owner: patterns_owner = options.patterns_owner - # Compile and check "name" and "owner" regular expressions - try: + try: # Compile and check "name" and "owner" regular expressions cre_owner, cre_name = re_compile_filters(patterns_owner, patterns_name) except ValueError as exc: parser.error(str(exc)) - if options.all_ports: - args.extend( - glbl_cfg().get(["suite servers", "scan hosts"]) or ["localhost"]) - if not args: - args = get_scan_items_from_fs(cre_owner) - if not args: - return - - json_filter = [] - skip_one = True - for host, port, suite_identity in scan_many(args, options.comms_timeout): - name = suite_identity[KEY_NAME] - owner = suite_identity[KEY_OWNER] - - if cre_name and not cre_name.match(name): - continue - if (cre_owner is None and owner != get_user() or - cre_owner and not cre_owner.match(owner)): - continue - if options.json_format: - json_filter.append([host, port, suite_identity]) - continue - - if options.old_format: - print name, owner, host, port - continue - - if options.raw_format or options.describe: - # Extracting required data for these options before processing - try: - meta_items = suite_identity[KEY_META] - except KeyError: - # Compat:<=7.5.0 - meta_items = { - "title": suite_identity.get(KEY_TITLE), - "description": suite_identity.get(KEY_DESCRIPTION) - } - - if options.raw_format: - suite_identity.get(KEY_UPDATE_TIME) - clean_meta_items = {} - for key, value in meta_items.items(): - if value: - clean_meta_items.update({ - key: ' '.join([x.strip() for x in - str(value).split('\n') if x])}) - totals = suite_identity.get(KEY_STATES) - print "%s|%s|%s|port|%s" % (name, owner, host, port) - for key, value in clean_meta_items.items(): - print "%s|%s|%s|%s|%s" % (name, owner, host, key, value) - if totals is None: - continue - for point, state_line in get_point_state_count_lines(*totals): - key = KEY_STATES - if point: - key = "%s:%s" % (KEY_STATES, point) - print "%s|%s|%s|%s|%s" % (name, owner, host, key, state_line) - continue - - line = '%s %s@%s:%s' % (name, owner, host, port) - if options.describe or options.print_totals: - if skip_one: - skip_one = False - if state_legend != "": - print state_legend + "\n" - else: - print - print bold(line) + + # list of endpoints to call + methods = ['identify'] + if options.describe: + methods.append('describe') + if options.state_totals: + methods.append('state_totals') + + # suite generator + suites = scan_many( + get_scan_items_from_fs(cre_owner, cre_name), + timeout=options.comms_timeout, + methods=methods, + ordered=options.ordered + ) + + # determine output format + if options.format == 'json': + print(json.dumps(list(suites), indent=4)) + elif options.format == 'raw': + formatter = format_raw + elif options.format == 'plain': + formatter = format_plain + else: + raise Exception('Unknown format: %s' % options.format) + + # output state legend if necessary + state_legend = "" + if options.color and options.state_totals: + n_states = len(TASK_STATUSES_ORDERED) + for index, state in enumerate(TASK_STATUSES_ORDERED): + state_legend += get_status_prop(state, 'ascii_ctrl') + if index == n_states / 2: + state_legend += "\n" + print(state_legend.rstrip() + "\n") + + # work through scan results one by one + for reg, host, port, info in suites: + if isinstance(info, str): + print(ERROR_STYLE + ' '.join([reg, host, port, info])) + elif info is None: + print(ERROR_STYLE + ' '.join([reg, host, port, 'Error Connecting'])) + elif info[KEY_NAME] != reg: + # TODO - should we do anything here, is this likely? + print(ERROR_STYLE + 'Warning: suite has changed name %s => %s' % ( + reg, info[KEY_NAME])) else: - print line - - if options.describe: - # Deal with title first for case of just withheld statement - if meta_items["title"] is None: - print(indent + bold("(description and state totals withheld)")) - continue - if not meta_items["title"]: - print(indent + bold("Title:\n") + indent * 2 + "(no title)") - else: - print(indent + bold("Title:\n") + indent * 2 + - '"%s"' % meta_items["title"]) - for metaitem, metavalue in meta_items.items(): - if metaitem != "title": - if metaitem == "description" or metaitem == "group": - print indent + bold(metaitem.capitalize() + ":") - else: - print indent + bold(metaitem + ":") - if not metavalue: - lines = "(no %s)" % metaitem - else: - lines = '"%s"' % metavalue - line1 = True - for line in lines.split('\n'): - line = line.lstrip() - if not line1: - # Indent under the double quote. - line = " " + line - line1 = False - print indent * 2 + line - - totals = suite_identity.get(KEY_STATES) - - if options.print_totals: - if totals is None: - print indent + bold("(state totals withheld)") - continue - print indent + bold("Task state totals:") - point_state_lines = get_point_state_count_lines( - *totals, use_color=options.color) - for point, state_line in point_state_lines: - point_prefix = "" - if point: - point_prefix = "%s " % point - print indent * 2 + "%s%s" % (point_prefix, state_line) - - if options.json_format: - print json.dumps(json_filter, indent=4) + formatter(reg, host, port, info, options) + + +def sort_meta(item): + """Sort meta items for the "plain" output format.""" + key = item[0] + if key in META_KEY_ORDER: + return str(META_KEY_ORDER.index(key)) + return key + + +def format_plain(name, host, port, info, options): + """Print a scan result, implements --format=plain""" + owner = info[KEY_OWNER] + + print(Style.BRIGHT + name + Style.NORMAL + + ' %s@%s:%s' % (owner, host, port)) + + if options.describe: + meta_items = info.get(KEY_META) + if meta_items is None: + print(INDENT + MISSING_STYLE + "(description withheld)") + return + for metaitem, metavalue in sorted(meta_items.items(), key=sort_meta): + if metaitem in META_KEY_ORDER: + metaitem = metaitem.capitalize() + print(INDENT + TITLE_STYLE + metaitem + ":") + if not metavalue: + metavalue = MISSING_STYLE + '(no %s)' % metaitem + Fore.RESET + for line in metavalue.splitlines(): + print(INDENT * 2 + line) + + if options.state_totals: + totals = info.get(KEY_STATES) + if totals is None: + print(INDENT + MISSING_STYLE + "(state totals withheld)") + return + print(INDENT + TITLE_STYLE + "Task state totals:") + for point, state_line in get_point_state_count_lines( + *totals, use_color=options.color): + point_prefix = "" + if point: + point_prefix = "%s " % point + print(INDENT * 2 + "%s%s" % (point_prefix, state_line)) + + +def format_raw(name, host, port, info, options): + """Print a scan result, implements --format=raw""" + owner = info[KEY_OWNER] + + print("%s|%s|%s|port|%s" % (name, owner, host, port)) + + if options.describe: + # Extracting required data for these options before processing + meta_items = info.get(KEY_META) + + # clean_meta_items = {} + # for key, value in meta_items.items(): + # if value: + # clean_meta_items.update({ + # key: ' '.join([x.strip() for x in + # str(value).split('\n') if x])}) + + # for key, value in meta_items.items(): + # if value: + # clean_meta_items.update({ + # key: ' '.join([x.strip() for x in + # str(value).split('\n') if x])}) + + for key, value in sorted(meta_items.items(), key=sort_meta): + value = ' '.join(line.strip() + for line in str(value).splitlines() + if line) + print("%s|%s|%s|%s|%s" % (name, owner, host, key, value)) + + if options.state_totals: + totals = info.get(KEY_STATES) + if totals is None: + return + for point, state_line in get_point_state_count_lines(*totals): + key = KEY_STATES + if point: + key = "%s:%s" % (KEY_STATES, point) + print("%s|%s|%s|%s|%s" % (name, owner, host, key, state_line)) def get_point_state_count_lines(state_count_totals, state_count_cycles, diff --git a/bin/cylc-search b/bin/cylc-search index fc456ff906f..b1df0714dc0 100755 --- a/bin/cylc-search +++ b/bin/cylc-search @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 # THIS FILE IS PART OF THE CYLC SUITE ENGINE. # Copyright (C) 2008-2019 NIWA & British Crown (Met Office) & Contributors. @@ -57,7 +57,7 @@ def section_level(heading): def print_heading(heading): - print '>>>' + '->'.join(heading) + print('>>>' + '->'.join(heading)) parser = COP( @@ -80,7 +80,7 @@ pattern = '|'.join(args[1:]) suitedir = os.path.dirname(suiterc) if os.path.isfile(suiterc): - h = open(suiterc, 'rb') + h = open(suiterc, 'r') lines = h.readlines() h.close() lines = inline(lines, suitedir, suiterc, for_grep=True) @@ -138,16 +138,16 @@ for line in lines: if curr_file != prev_file: prev_file = curr_file - print "\nFILE:", curr_file + print("\nFILE:", curr_file) # Print the nested section headings section_key = '->'.join(sections) if section_key != prev_section_key: prev_section_key = section_key - print ' SECTION:', section_key + print(' SECTION:', section_key) # Print the pattern match, with line number - print ' (' + str(line_no) + '):', line.rstrip('\n') + print(' (' + str(line_no) + '):', line.rstrip('\n')) if not options.search_bin: sys.exit(0) @@ -155,7 +155,7 @@ if not options.search_bin: # search files in suite bin directory bin_ = os.path.join(suitedir, 'bin') if not os.path.isdir(bin_): - print >> sys.stderr, "\nSuite " + suite + " has no bin directory" + print("\nSuite " + suite + " has no bin directory", file=sys.stderr) sys.exit(0) for name in os.listdir(bin_): @@ -165,11 +165,11 @@ for name in os.listdir(bin_): continue new_file = True try: - h = open(os.path.join(bin_, name), 'rb') - except IOError, x: + h = open(os.path.join(bin_, name), 'r') + except IOError as exc: # e.g. there's a sub-directory under bin; ignore it. - print >> sys.stderr, 'Unable to open file ' + os.path.join(bin_, name) - print >> sys.stderr, x + print('Unable to open file ' + os.path.join(bin_, name), file=sys.stderr) + print(exc, file=sys.stderr) continue contents = h.readlines() h.close() @@ -180,6 +180,6 @@ for name in os.listdir(bin_): count += 1 if re.search(pattern, line): if new_file: - print '\nFILE:', os.path.join(bin_, name) + print('\nFILE:', os.path.join(bin_, name)) new_file = False - print ' (' + str(count) + '): ' + line + print(' (' + str(count) + '): ' + line) diff --git a/bin/cylc-set-verbosity b/bin/cylc-set-verbosity index d75d616769c..5da5fb5320a 100755 --- a/bin/cylc-set-verbosity +++ b/bin/cylc-set-verbosity @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 # THIS FILE IS PART OF THE CYLC SUITE ENGINE. # Copyright (C) 2008-2019 NIWA & British Crown (Met Office) & Contributors. @@ -32,9 +32,9 @@ if '--use-ssh' in sys.argv[1:]: sys.exit(0) import cylc.flags -from cylc.prompt import prompt from cylc.option_parsers import CylcOptionParser as COP -from cylc.network.httpclient import SuiteRuntimeServiceClient +from cylc.network.client import SuiteRuntimeClient +from cylc.terminal import prompt LOGGING_LVL_OF = { "INFO": INFO, @@ -65,12 +65,11 @@ def main(): prompt("Set logging level to %s in %s" % (severity_str, suite), options.force) - pclient = SuiteRuntimeServiceClient( + pclient = SuiteRuntimeClient( suite, options.owner, options.host, options.port, - options.comms_timeout, my_uuid=options.set_uuid, - print_uuid=options.print_uuid) + options.comms_timeout) - pclient.put_command('set_verbosity', level=severity) + pclient('set_verbosity', {'level': severity}) if __name__ == "__main__": diff --git a/bin/cylc-show b/bin/cylc-show index aac86e8dfbf..4db439914a1 100755 --- a/bin/cylc-show +++ b/bin/cylc-show @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 # THIS FILE IS PART OF THE CYLC SUITE ENGINE. # Copyright (C) 2008-2019 NIWA & British Crown (Met Office) & Contributors. @@ -33,7 +33,7 @@ import json import cylc.flags from cylc.option_parsers import CylcOptionParser as COP -from cylc.network.httpclient import SuiteRuntimeServiceClient +from cylc.network.client import SuiteRuntimeClient from cylc.task_id import TaskID @@ -54,76 +54,73 @@ def main(): options, args = parser.parse_args() suite = args[0] task_args = args[1:] - pclient = SuiteRuntimeServiceClient( + pclient = SuiteRuntimeClient( suite, options.owner, options.host, options.port, - options.comms_timeout, my_uuid=options.set_uuid, - print_uuid=options.print_uuid - ) + options.comms_timeout) json_filter = [] if not task_args: # Print suite info. - suite_info = pclient.get_info('get_suite_info') + suite_info = pclient('get_suite_info') if options.json: json_filter.append(suite_info) else: for key, value in sorted(suite_info.items(), reverse=True): - print '%s: %s' % (key, value or "(not given)") + print('%s: %s' % (key, value or "(not given)")) task_names = [arg for arg in task_args if TaskID.is_valid_name(arg)] task_ids = [arg for arg in task_args if TaskID.is_valid_id_2(arg)] if task_names: - results = pclient.get_info('get_task_info', names=task_names) + results = pclient('get_task_info', {'names': task_names}) if options.json: json_filter.append(results) else: for task_name, result in sorted(results.items()): if len(results) > 1: - print "----\nTASK NAME: %s" % task_name + print("----\nTASK NAME: %s" % task_name) for key, value in sorted(result.items(), reverse=True): - print "%s: %s" % (key, value or "(not given)") + print("%s: %s" % (key, value or "(not given)")) if task_ids: - results, bad_items = pclient.get_info( - 'get_task_requisites', items=task_ids, - list_prereqs=options.list_prereqs) + results, bad_items = pclient('get_task_requisites', { + 'items': task_ids, 'list_prereqs': options.list_prereqs}) if options.json: json_filter.append(results) else: for task_id, result in sorted(results.items()): if len(results) > 1: - print "----\nTASK ID: %s" % task_id + print("----\nTASK ID: %s" % task_id) if options.list_prereqs: for prereq in result["prerequisites"]: - print prereq + print(prereq) else: for key, value in sorted( result["meta"].items(), reverse=True): - print "%s: %s" % (key, value or "(not given)") + print("%s: %s" % (key, value or "(not given)")) for name, done in [("prerequisites", "satisfied"), ("outputs", "completed")]: - print '\n%s (- => not %s):' % (name, done) + print('\n%s (- => not %s):' % (name, done)) if not result[name]: - print ' (None)' + print(' (None)') for msg, state in result[name]: if state: - print ' + ' + msg + print(' + ' + msg) else: - print ' - ' + msg + print(' - ' + msg) if result["extras"]: - print '\nother:' + print('\nother:') for key, value in result["extras"].items(): - print ' o %s ... %s' % (key, value) + print(' o %s ... %s' % (key, value)) for bad_item in bad_items: sys.stderr.write("No matching tasks found: %s\n" % bad_item) if bad_items: sys.exit(1) if options.json: - print json.dumps(json_filter, indent=4) + print(json.dumps(json_filter, indent=4)) if __name__ == "__main__": diff --git a/bin/cylc-spawn b/bin/cylc-spawn index 161a4a12058..58c13de06a1 100755 --- a/bin/cylc-spawn +++ b/bin/cylc-spawn @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 # THIS FILE IS PART OF THE CYLC SUITE ENGINE. # Copyright (C) 2008-2019 NIWA & British Crown (Met Office) & Contributors. @@ -31,9 +31,9 @@ if '--use-ssh' in sys.argv[1:]: sys.exit(0) import cylc.flags -from cylc.prompt import prompt from cylc.option_parsers import CylcOptionParser as COP -from cylc.network.httpclient import SuiteRuntimeServiceClient +from cylc.network.client import SuiteRuntimeClient +from cylc.terminal import prompt def main(): @@ -48,12 +48,14 @@ def main(): suite = args.pop(0) prompt('Spawn task(s) %s in %s' % (args, suite), options.force) - pclient = SuiteRuntimeServiceClient( + pclient = SuiteRuntimeClient( suite, options.owner, options.host, options.port, - options.comms_timeout, my_uuid=options.set_uuid, - print_uuid=options.print_uuid) - items = parser.parse_multitask_compat(options, args) - pclient.put_command('spawn_tasks', items=items) + options.comms_timeout) + + pclient( + 'spawn_tasks', + {'items': parser.parse_multitask_compat(options, args)} + ) if __name__ == "__main__": diff --git a/bin/cylc-stop b/bin/cylc-stop index 7c0f9c52062..9a1feef68ac 100755 --- a/bin/cylc-stop +++ b/bin/cylc-stop @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 # THIS FILE IS PART OF THE CYLC SUITE ENGINE. # Copyright (C) 2008-2019 NIWA & British Crown (Met Office) & Contributors. @@ -46,11 +46,11 @@ if '--use-ssh' in sys.argv[1:]: sys.exit(0) import cylc.flags -from cylc.prompt import prompt from cylc.task_id import TaskID from cylc.option_parsers import CylcOptionParser as COP -from cylc.network.httpclient import ClientError, SuiteRuntimeServiceClient +from cylc.network.client import ClientError, ClientTimeout, SuiteRuntimeClient from cylc.command_polling import Poller +from cylc.terminal import prompt class StopPoller(Poller): @@ -63,8 +63,8 @@ class StopPoller(Poller): def check(self): """Return True if suite has stopped (success) else False""" try: - self.pclient.get_info('ping_suite') - except ClientError: + self.pclient('ping_suite') + except (ClientError, ClientTimeout): # failed to ping - suite stopped return True else: @@ -114,10 +114,9 @@ def main(): if options.kill and options.now: parser.error("ERROR: --kill is not compatible with --now") - pclient = SuiteRuntimeServiceClient( + pclient = SuiteRuntimeClient( suite, options.owner, options.host, options.port, - options.comms_timeout, my_uuid=options.set_uuid, - print_uuid=options.print_uuid) + options.comms_timeout) if int(options.max_polls) > 0: # (test to avoid the "nothing to do" warning for # --max-polls=0) @@ -129,30 +128,29 @@ def main(): 'Set shutdown at wall clock %s for %s' % ( options.wall_clock, suite), options.force) - pclient.put_command('set_stop_after_clock_time', - datetime_string=options.wall_clock) + pclient('set_stop_after_clock_time', + {'datetime_string': options.wall_clock}) elif shutdown_at and TaskID.is_valid_id(shutdown_arg): # STOP argument detected prompt( 'Set shutdown after task %s for %s' % (shutdown_arg, suite), options.force) - pclient.put_command('set_stop_after_task', task_id=shutdown_arg) + pclient('set_stop_after_task', {'task_id': shutdown_arg}) elif shutdown_at: # not a task ID, may be a cycle point prompt( 'Set shutdown at cycle point %s for %s' % (shutdown_arg, suite), options.force) - pclient.put_command('set_stop_after_point', point_string=shutdown_arg) + pclient('set_stop_after_point', {'point_string': shutdown_arg}) elif options.now > 1: prompt('Shut down and terminate %s now' % suite, options.force) - pclient.put_command('stop_now', terminate=True) + pclient('stop_now', {'terminate': True}) elif options.now: prompt('Shut down %s now' % suite, options.force) - pclient.put_command('stop_now') + pclient('stop_now') else: prompt('Shut down %s' % suite, options.force) - pclient.put_command('set_stop_cleanly', - kill_active_tasks=options.kill) + pclient('set_stop_cleanly', {'kill_active_tasks': options.kill}) if int(options.max_polls) > 0: # (test to avoid the "nothing to do" warning for # --max-polls=0) diff --git a/bin/cylc-submit b/bin/cylc-submit index 570a1388a0e..e41793ee27c 100755 --- a/bin/cylc-submit +++ b/bin/cylc-submit @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 # THIS FILE IS PART OF THE CYLC SUITE ENGINE. # Copyright (C) 2008-2019 NIWA & British Crown (Met Office) & Contributors. @@ -137,10 +137,11 @@ def main(): for itask in itasks: if itask.local_job_file_path: - print('JOB SCRIPT=%s' % itask.local_job_file_path) + print(('JOB SCRIPT=%s' % itask.local_job_file_path)) else: - print >> sys.stderr, ( - 'Unable to prepare job file for %s' % itask.identity) + print(( + 'Unable to prepare job file for %s' % itask.identity), + file=sys.stderr) ret_code = 1 else: while waiting_tasks: @@ -153,8 +154,8 @@ def main(): task_job_mgr.proc_pool.process() for itask in itasks: if itask.summary.get('submit_method_id') is not None: - print('[%s] Job ID: %s' % ( - itask.identity, itask.summary['submit_method_id'])) + print(('[%s] Job ID: %s' % ( + itask.identity, itask.summary['submit_method_id']))) if itask.state.status == TASK_STATUS_SUBMIT_FAILED: ret_code = 1 sys.exit(ret_code) diff --git a/bin/cylc-suite-state b/bin/cylc-suite-state index a2cf55fe6f7..2a2535d809c 100755 --- a/bin/cylc-suite-state +++ b/bin/cylc-suite-state @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 # THIS FILE IS PART OF THE CYLC SUITE ENGINE. # Copyright (C) 2008-2019 NIWA & British Crown (Met Office) & Contributors. @@ -194,7 +194,7 @@ def main(): sys.exit("ERROR: You must target a cycle point to use an offset") if options.template: - print >> sys.stderr, "WARNING: ignoring --template (no longer needed)" + print("WARNING: ignoring --template (no longer needed)", file=sys.stderr) # Attempt to apply specified offset to the targeted cycle if options.offset: diff --git a/bin/cylc-test-battery b/bin/cylc-test-battery index 48e74863a18..3c4bc9704c7 100755 --- a/bin/cylc-test-battery +++ b/bin/cylc-test-battery @@ -146,13 +146,14 @@ done # Recompile *.pyc files to ensure we are running the current code. if [[ -w "$CYLC_DIR/lib" ]]; then find "$CYLC_DIR/lib" -name '*.pyc' -type 'f' -delete - python2 -mcompileall -q "$CYLC_DIR/lib" + python3 -mcompileall -q "$CYLC_DIR/lib" fi if perl -e 'use Test::Harness 3.00' 2>/dev/null; then NPROC=$(cylc get-global-config '--item=process pool size') if [[ -z "${NPROC}" ]]; then - NPROC=$(python2 -c 'import multiprocessing as mp; print mp.cpu_count()') + NPROC=$(python3 -c \ + 'import multiprocessing as mp; print(mp.cpu_count())') fi exec prove -j "$NPROC" -s -r "${@:-tests}" else diff --git a/bin/cylc-trigger b/bin/cylc-trigger index 54deb8f713b..8ada6de26a1 100755 --- a/bin/cylc-trigger +++ b/bin/cylc-trigger @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 # THIS FILE IS PART OF THE CYLC SUITE ENGINE. # Copyright (C) 2008-2019 NIWA & British Crown (Met Office) & Contributors. @@ -49,11 +49,11 @@ import difflib from subprocess import call import cylc.flags -from cylc.prompt import prompt from cylc.option_parsers import CylcOptionParser as COP -from cylc.network.httpclient import SuiteRuntimeServiceClient +from cylc.network.client import SuiteRuntimeClient from cylc.cfgspec.glbl_cfg import glbl_cfg from cylc.task_job_logs import JOB_LOG_DIFF +from cylc.terminal import prompt def main(): @@ -80,25 +80,26 @@ def main(): msg = 'Trigger task(s) %s in %s' % (args, suite) prompt(msg, options.force) - pclient = SuiteRuntimeServiceClient( + pclient = SuiteRuntimeClient( suite, options.owner, options.host, options.port, - options.comms_timeout, my_uuid=options.set_uuid, - print_uuid=options.print_uuid) + options.comms_timeout) aborted = False if options.edit_run: items = parser.parse_multitask_compat(options, args) task_id = items[0] # Check that TASK is a unique task. - success, msg = pclient.get_info( - 'ping_task', task_id=task_id, exists_only=True) + success, msg = pclient( + 'ping_task', + {'task_id': task_id, 'exists_only': True} + ) if not success: sys.exit('ERROR: %s' % msg) # Get the job filename from the suite server program - the task cycle # point may need standardising to the suite cycle point format. - jobfile_path = pclient.get_info( - 'get_task_jobfile_path', task_id=task_id) + jobfile_path = pclient( + 'get_task_jobfile_path', {'task_id': task_id}) if not jobfile_path: sys.exit('ERROR: task not found') @@ -114,8 +115,7 @@ def main(): old_mtime = None # Tell the suite server program to generate the job file. - pclient.put_command('dry_run_tasks', items=[task_id], - check_syntax=False) + pclient('dry_run_tasks', {'items': [task_id], 'check_syntax': False}) # Wait for the new job file to be written. Use mtime because the same # file could potentially exist already, left from a previous run. @@ -160,7 +160,7 @@ def main(): # Don't allow force-no-prompt in this case. if options.geditor: # Alert stdout of the dialog window, in case it's missed. - print "Editing done. I'm popping up a confirmation dialog now." + print("Editing done. I'm popping up a confirmation dialog now.") # Save a diff to record the changes made. difflog = os.path.join(os.path.dirname(jobfile_path), JOB_LOG_DIFF) @@ -170,7 +170,7 @@ def main(): open(jobfile_path).readlines(), fromfile="original", tofile="edited"): - diff_file.write(line) + diff_file.write(line.encode()) os.unlink(jobfile_copy_path) msg = "Trigger edited task %s?" % task_id @@ -194,9 +194,9 @@ def main(): # Back compat: back_out introduced >7.5.0 # So don't call with "back_out" if not necessary to avoid breakage. if aborted: - pclient.put_command('trigger_tasks', items=items, back_out=aborted) + pclient('trigger_tasks', {'items': items, 'back_out': aborted}) else: - pclient.put_command('trigger_tasks', items=items) + pclient('trigger_tasks', {'items': items}) if __name__ == "__main__": diff --git a/bin/cylc-validate b/bin/cylc-validate index 279c1d06cd0..770f0fc3d32 100755 --- a/bin/cylc-validate +++ b/bin/cylc-validate @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 # THIS FILE IS PART OF THE CYLC SUITE ENGINE. # Copyright (C) 2008-2019 NIWA & British Crown (Met Office) & Contributors. @@ -109,7 +109,7 @@ def main(): # (Taken from config.py to avoid circular import problems.) # TODO - This is not exhaustive, it only uses the initial cycle point. if cylc.flags.verbose: - print 'Instantiating tasks to check trigger expressions' + print('Instantiating tasks to check trigger expressions') for name, taskdef in cfg.taskdefs.items(): try: itask = TaskProxy(taskdef, cfg.start_point, is_startup=True) @@ -129,20 +129,21 @@ def main(): except TriggerExpressionError as exc: err = str(exc) if '@' in err: - print >> sys.stderr, ( - "ERROR, %s: xtriggers can't be in conditional" - " expressions: %s" % (name, err)) + print(f"ERROR, {name}: xtriggers can't be in conditional" + f" expressions: {err}", + file=sys.stderr) else: - print >> sys.stderr, 'ERROR, %s: bad trigger: %s' % (name, err) + print('ERROR, %s: bad trigger: %s' % (name, err), + file=sys.stderr) raise SuiteConfigError("ERROR: bad trigger") except Exception as exc: - print >> sys.stderr, str(exc) + print(str(exc), file=sys.stderr) raise SuiteConfigError( 'ERROR, %s: failed to evaluate triggers.' % name) if cylc.flags.verbose: - print ' + %s ok' % itask.identity + print(' + %s ok' % itask.identity) - print 'Valid for cylc-%s' % CYLC_VERSION + print('Valid for cylc-%s' % CYLC_VERSION) profiler.stop() diff --git a/bin/cylc-view b/bin/cylc-view index db3fa76261e..914cb85f390 100755 --- a/bin/cylc-view +++ b/bin/cylc-view @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 # THIS FILE IS PART OF THE CYLC SUITE ENGINE. # Copyright (C) 2008-2019 NIWA & British Crown (Met Office) & Contributors. @@ -136,7 +136,7 @@ def main(): if options.stdout: for line in lines: - print line + print(line) sys.exit(0) # write to a temporary file @@ -145,11 +145,11 @@ def main(): dir=cylc_tmpdir ) for line in lines: - viewfile.write(line + '\n') + viewfile.write((line + '\n').encode()) viewfile.seek(0, 0) # set the file to be read only - os.chmod(viewfile.name, 0400) + os.chmod(viewfile.name, 0o400) # capture the temp file's mod time in case the user edits it # and overrides the readonly mode. @@ -163,7 +163,7 @@ def main(): retcode = call(command_list) if retcode != 0: # the command returned non-zero exist status - print >> sys.stderr, command, 'failed:', retcode + print(command, 'failed:', retcode, file=sys.stderr) sys.exit(1) # !!!VIEWING FINISHED!!! @@ -172,13 +172,13 @@ def main(): modtime2 = os.stat(viewfile.name).st_mtime if modtime2 > modtime1: - print - print >> sys.stderr, ( - 'WARNING: YOU HAVE EDITED A TEMPORARY READ-ONLY SUITE COPY:') - print >> sys.stderr, viewfile.name - print >> sys.stderr, ( - 'In future use \'cylc [prep] edit\' to edit a suite.') - print + print() + print('WARNING: YOU HAVE EDITED A TEMPORARY READ-ONLY SUITE COPY:', + file=sys.stderr) + print(viewfile.name, file=sys.stderr) + print('In future use \'cylc [prep] edit\' to edit a suite.', + file=sys.stderr) + print() # DONE viewfile.close() diff --git a/bin/cylc-warranty b/bin/cylc-warranty index bc592e3ca5b..4e9aef93cae 100755 --- a/bin/cylc-warranty +++ b/bin/cylc-warranty @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 # THIS FILE IS PART OF THE CYLC SUITE ENGINE. # Copyright (C) 2008-2019 NIWA & British Crown (Met Office) & Contributors. @@ -29,7 +29,7 @@ import cylc.flags def main(): if len(sys.argv) == 1: - print """GNU General Public License v3.0, Section 15: + print("""GNU General Public License v3.0, Section 15: 15. Disclaimer of Warranty. @@ -40,9 +40,9 @@ OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF -ALL NECESSARY SERVICING, REPAIR OR CORRECTION.""" +ALL NECESSARY SERVICING, REPAIR OR CORRECTION.""") else: - print __doc__ + print(__doc__) if sys.argv[1] not in ['-h', 'help', '--help']: sys.exit(1) diff --git a/bin/gcycl b/bin/gcycl deleted file mode 100755 index 4b4180859aa..00000000000 --- a/bin/gcycl +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/bash - -# THIS FILE IS PART OF THE CYLC SUITE ENGINE. -# Copyright (C) 2008-2019 NIWA & British Crown (Met Office) & Contributors. -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -# Catch a common command line typo: 'gcycl': -echo >&2 -echo "TYPO ALERT: it's \"gcylc\", not \"gcycl\"!" >&2 -echo >&2 -exec "$(dirname "$0")/cylc" 'gui' "$@" diff --git a/bin/gcylc b/bin/gcylc deleted file mode 100755 index f747343a038..00000000000 --- a/bin/gcylc +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/bash - -# THIS FILE IS PART OF THE CYLC SUITE ENGINE. -# Copyright (C) 2008-2019 NIWA & British Crown (Met Office) & Contributors. -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -exec "$(dirname "$0")/cylc" 'gui' "$@" diff --git a/doc/src/appendices/appendices-master.rst b/doc/src/appendices/appendices-master.rst index 148bfd5ca76..0132fa43b4a 100644 --- a/doc/src/appendices/appendices-master.rst +++ b/doc/src/appendices/appendices-master.rst @@ -9,11 +9,8 @@ Appendices suiterc-config-ref site-user-config-ref - gcylc-config-ref - gscan-config-ref remote-job-management command-ref - gcylc-graph-view readme-file install-file dev-history-major-changes diff --git a/doc/src/appendices/dev-history-major-changes.rst b/doc/src/appendices/dev-history-major-changes.rst index ad6fc290c98..d28bebb4ad7 100644 --- a/doc/src/appendices/dev-history-major-changes.rst +++ b/doc/src/appendices/dev-history-major-changes.rst @@ -46,3 +46,9 @@ Cylc Development History - Major Changes - Replaced the Pyro communications layer with RESTful HTTPS - Removed deprecated pre cylc-6 syntax and features. + +- **cylc-8** + + - Upgraded to Python3 + - Replaced the RESTful HTTP(S) communications layer with ZMQ over TCP + - Removed deprecated pre cylc-6 syntax and features. diff --git a/doc/src/appendices/gcylc-config-ref.rst b/doc/src/appendices/gcylc-config-ref.rst deleted file mode 100644 index 8a7dae09e4a..00000000000 --- a/doc/src/appendices/gcylc-config-ref.rst +++ /dev/null @@ -1,270 +0,0 @@ -.. _GcylcRCReference: - -Gcylc GUI (cylc gui) Config File Reference -========================================== - -This section defines all legal items and values for the gcylc user config file, -which should be located in ``$HOME/.cylc/gcylc.rc``. Current settings -can be printed with the ``cylc get-gui-config`` command. - - -Top Level Items ---------------- - -dot icon size -^^^^^^^^^^^^^ - -Set the size of the task state dot icons displayed in the text and dot -views. - -- *type*: string -- *legal values*: ``small`` (10px), ``medium`` (14px), ``large`` (20px), - ``extra large`` (30px) -- *default*: ``medium`` - - -initial side-by-side views -^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Set the suite view panels initial orientation when the GUI starts. -This can be changed later using the "View" menu "Toggle views side-by-side" -option. - -- *type*: boolean (False or True) -- *default*: ``False`` - - -initial views -^^^^^^^^^^^^^ - -Set the suite view panel(s) displayed initially, when the GUI starts. -This can be changed later using the tool bar. - -- *type*: string (a list of one or two view names) -- *legal values*: ``text``, ``dot``, ``graph`` -- *default*: ``text`` -- *example*: ``initial views = graph, dot`` - - -maximum update interval -^^^^^^^^^^^^^^^^^^^^^^^ - -Set the maximum (longest) time interval between calls to the suite for data -update. - -The update frequency of the GUI is variable. It is determined by considering -the time of last update and the mean duration of the last 10 main loops of the -suite. - -In general, the GUI will use an update frequency that matches the mean duration -of the suite's main loop. In quiet time (or if the suite is not contactable), -it will gradually increase the update interval (i.e. reduce the update -frequency) to a maximum determined by this setting. - -Increasing this setting will reduce the network traffic and hits on the suite -process. However, if a quiet suite starts to pick up activity, the GUI may -initially appear out of sync with what is happening in the suite for the -duration of this interval. - -- *type*: ISO 8601 duration/interval representation (e.g. - ``PT10S``, 10 seconds, or ``PT1M``, 1 minute). -- *default*: PT15S - - -sort by definition order -^^^^^^^^^^^^^^^^^^^^^^^^ - -If this is not turned off the default sort order for task names and -families in the dot and text views will the order they appear in the -suite definition. Clicking on the task name column in the treeview will -toggle to alphanumeric sort, and a View menu item does the same for the -dot view. If turned off, the default sort order is alphanumeric and -definition order is not available at all. - -- *type*: boolean -- *default*: ``True`` - - -sort column -^^^^^^^^^^^ - -If ``text`` is in ``initial views`` then ``sort column`` sets -the column that will be sorted initially when the GUI launches. Sorting can be -changed later by clicking on the column headers. - -- *type*: string -- *legal values*: ``task``, ``state``, ``host``, ``job system``, - ``job ID``, ``T-submit``, ``T-start``, ``T-finish``, ``dT-mean``, - ``latest message``, ``none`` -- *default*: ``none`` -- *example*: ``sort column = T-start`` - - -sort column ascending -^^^^^^^^^^^^^^^^^^^^^ - -For use in combination with ``sort column``, sets whether the column will -be sorted using ascending or descending order. - -- *type*: boolean -- *default*: ``True`` -- *example*: ``sort column ascending = False`` - - -sub-graphs on -^^^^^^^^^^^^^ - -Set the sub-graphs view to be enabled by default. -This can be changed later using the toggle options for the graph view. - -- *type*: boolean (False or True) -- *default*: ``False`` - - -task filter highlight color -^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -The color used to highlight active task filters in gcylc. It must be a name -from the X11 ``rgb.txt`` file, e.g. ``SteelBlue``; or a -*quoted* hexadecimal color code, e.g. ``"#ff0000"`` for red (quotes -are required to prevent the hex code being interpreted as a comment). - -- *type*: string -- *default*: ``PowderBlue`` - - -task states to filter out -^^^^^^^^^^^^^^^^^^^^^^^^^ - -Set the initial filtering options when the GUI starts. Later this can be -changed by using the "View" menu "Task Filtering" option. - -- *type*: string list -- *legal values*: waiting, held, queued, ready, expired, submitted, - submit-failed, submit-retrying, running, succeeded, failed, retrying, - runahead -- *default*: runahead - - -transpose dot -^^^^^^^^^^^^^ - -Transposes the content in dot view so that it displays from left to right -rather than from top to bottom. Can be changed later using the options -submenu available via the view menu. - -- *type*: boolean -- *default*: ``False`` -- *example*: ``transpose dot = True`` - - -transpose graph -^^^^^^^^^^^^^^^ - -Transposes the content in graph view so that it displays from left to right -rather than from top to bottom. Can be changed later using the options submenu -via the view menu. - -- *type*: boolean -- *default*: ``False`` -- *example*: ``transpose graph = True`` - - -ungrouped views -^^^^^^^^^^^^^^^ - -List suite views, if any, that should be displayed initially in an -ungrouped state. Namespace family grouping can be changed later -using the tool bar. - -- *type*: string (a list of zero or more view names) -- *legal values*: ``text``, ``dot``, ``graph`` -- *default*: (none) -- *example*: ``ungrouped views = text, dot`` - - -use theme -^^^^^^^^^ - -Set the task state color theme, common to all views, to use initially. The -color theme can be changed later using the tool bar. See -``etc/gcylc.rc.eg`` and ``etc/gcylc-themes.rc`` in the Cylc -installation directory for how to modify existing themes or define your own. -Use ``cylc get-gui-config`` to list available themes. - -- *type*: string (theme name) -- *legal values*: ``default``, ``solid``, ``high-contrast``, - ``color-blind``, and any custom or user-modified themes. -- *default*: ``default`` - - -window size -^^^^^^^^^^^ - -Sets the size (in pixels) of the cylc GUI at startup. - -- *type*: integer list: x, y -- *legal values*: positive integers -- *default*: 800, 500 -- *example*: ``window size = 1000, 700`` - - -[themes] --------- - -This section may contain task state color theme definitions. - - -[themes] ``->`` [[THEME]] -^^^^^^^^^^^^^^^^^^^^^^^^^ - -The name of the task state color-theme to be defined in this section. - -- *type*: string - - -[themes] ``->`` [[THEME]] ``->`` inherit -"""""""""""""""""""""""""""""""""""""""" - -You can inherit from another theme in order to avoid defining all states. - -- *type*: string (parent theme name) -- *default*: ``default`` - - -[themes] ``->`` [[THEME]] ``->`` defaults -""""""""""""""""""""""""""""""""""""""""" - -Set default icon attributes for all state icons in this theme. - -- *type*: string list (icon attributes) -- *legal values*: ``"color=COLOR"``, ``"style=STYLE"``, - ``"fontcolor=FONTCOLOR"`` -- *default*: (none) - -For the attribute values, ``COLOR`` and ``FONTCOLOR`` can be color names from -the X11 ``rgb.txt`` file, e.g. ``SteelBlue``; or hexadecimal color codes, e.g. -``#ff0000`` for red; and ``STYLE`` can be ``filled`` or ``unfilled``. -See ``etc/gcylc.rc.eg`` and ``etc/gcylc-themes.rc`` in -the Cylc installation directory for examples. - - -[themes] ``->`` [[THEME]] ``->`` STATE -"""""""""""""""""""""""""""""""""""""" - -Set icon attributes for all task states in ``THEME``, or for a subset of them -if you have used theme inheritance and/or defaults. Legal values of ``STATE`` -are any of the cylc task proxy states: *waiting, runahead, held, queued, -ready, submitted, submit-failed, running, succeeded, failed, retrying, -submit-retrying*. - -- *type*: string list (icon attributes) -- *legal values*: ``"color=COLOR"``, ``"style=STYLE"``, - ``"fontcolor=FONTCOLOR"`` -- *default:* (none) - -For the attribute values, ``COLOR`` and ``FONTCOLOR`` can be color names from -the X11 ``rgb.txt`` file, e.g. ``SteelBlue``; or hexadecimal color codes, e.g. -``#ff0000`` for red; and ``STYLE`` can be ``filled`` or ``unfilled``. -See ``etc/gcylc.rc.eg`` and ``etc/gcylc-themes.rc`` in -the Cylc installation directory for examples. diff --git a/doc/src/appendices/gcylc-graph-view.rst b/doc/src/appendices/gcylc-graph-view.rst deleted file mode 100644 index f7a57144ff5..00000000000 --- a/doc/src/appendices/gcylc-graph-view.rst +++ /dev/null @@ -1,29 +0,0 @@ -.. _TheGraphBasedcontrolGUI: - -The gcylc Graph View -==================== - -The graph view in the gcylc GUI shows the structure of the suite as it -evolves. It can work well even for large suites, but be aware that the -Graphviz layout engine has to do a new global layout every time a task -proxy appears in or disappears from the task pool. The following may help -mitigate any jumping layout problems: - -- The disconnect button can be used to temporarily prevent the - graph from changing as the suite evolves. -- The greyed-out base nodes, which are only present to fill out - the graph structure, can be toggled off (but this will split the - graph into disconnected sub-trees). -- Right-click on a task and choose the "Focus" option to restrict - the graph display to that task's cycle point. Anything interesting - happening in other cycle points will show up as disconnected - rectangular nodes to the right of the graph (and you can click on - those to instantly refocus to their cycle points). -- Task filtering is the ultimate quick route to focusing on just - the tasks you're interested in, but this will destroy the graph - structure. - - -.. only:: builder_html - - .. include:: ../custom/whitespace_include diff --git a/doc/src/appendices/gscan-config-ref.rst b/doc/src/appendices/gscan-config-ref.rst deleted file mode 100644 index 3bea0c962fc..00000000000 --- a/doc/src/appendices/gscan-config-ref.rst +++ /dev/null @@ -1,105 +0,0 @@ -.. _GscanRCReference: - -Gscan GUI (cylc gscan) Config File Reference -============================================ - -This section defines all legal items and values for the gscan config -file which should be located in ``$HOME/.cylc/gscan.rc``. Some items -also affect the gpanel panel app. - -The main menubar can be hidden to maximise the display area. Its visibility -can be toggled via the mouse right-click menu, or by typing ``Alt-m``. When -visible, the main View menu allows you to change properties such as the columns -that are displayed, which hosts to scan for running suites, and the task state -icon theme. - -At startup, the task state icon theme and icon size are taken from the gcylc -config file ``$HOME/.cylc/gcylc.rc``. - - -Top Level Items ---------------- - - -activate on startup -^^^^^^^^^^^^^^^^^^^ - -Set whether ``cylc gpanel`` will activate automatically when the GUI is -loaded or not. - -- *type*: boolean (True or False) -- *legal values*: ``True``, ``False`` -- *default*: ``False`` -- *example*: ``activate on startup = True`` - - -columns -^^^^^^^ - -Set the columns to display when the ``cylc gscan`` GUI starts. This can -be changed later with the View menu. The order in which the columns are -specified here does not affect the display order. - -- *type*: string (a list of one or more view names) -- *legal values*: ``host``, ``owner``, ``status``, ``suite``, - ``title``, ``updated`` -- *default*: ``status``, ``suite`` -- *example*: ``columns = suite, title, status`` - - -suite listing update interval -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Set the time interval between refreshing the suite listing (by file system or -port range scan). - -Increasing this setting will reduce the frequency of gscan looking for running -suites. Scanning for suites by port range scan can be a hit on the network and -the running suite processes, while scanning for suites by walking the file -system can hit the file system (especially if the file system is a network file -system). Therefore, this is normally set with a lower frequency than the status -update interval. Increasing this setting will make gscan friendlier to the -network and/or the file system, but gscan may appear out of sync if there are -many start up or shut down of suites between the intervals. - -- *type*: ISO 8601 duration/interval representation (e.g. ``PT10S``, - 10 seconds, or ``PT1M``, 1 minute). -- *default*: PT1M - - -suite status update interval -^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Set the time interval between calls to known running suites (suites that are -known via the latest suite listing) for data updates. - -Increasing this setting will reduce the network traffic and hits on the suite -processes. However, gscan may appear out of sync with what may be happening -in very busy suites. - -- *type*: ISO 8601 duration/interval representation (e.g. ``PT10S``, - 10 seconds, or ``PT1M``, 1 minute). -- *default*: PT15S - - -window size -^^^^^^^^^^^ - -Sets the size in pixels of the ``cylc gscan`` GUI window at startup. - -- *type*: integer list: x, y -- *legal values*: positive integers -- *default*: 300, 200 -- *example*: ``window size = 1000, 700`` - - -hide main menubar -^^^^^^^^^^^^^^^^^ - -Hide the main menubar of the ``cylc gscan`` GUI window at startup. By -default, the menubar is not hidden. Either way, you can toggle its -visibility with ``Alt-m`` or via the right-click menu. - -- *type*: boolean (True or False) -- *default*: False -- *example*: ``hide main menubar = True`` diff --git a/doc/src/appendices/remote-job-management.rst b/doc/src/appendices/remote-job-management.rst index cc8369da905..0ec4d7c2a05 100644 --- a/doc/src/appendices/remote-job-management.rst +++ b/doc/src/appendices/remote-job-management.rst @@ -182,7 +182,6 @@ Result: |__.service # SUITE SERVICE DIRECTORY | |__contact # server address information | |__passphrase # suite passphrase - | |__ssl.cert # suite SSL certificate |__log->log.20180418T025047Z # LOG DIRECTORIES |__log.20180418T025047Z # log directory for current suite run |__suiter.rc @@ -216,7 +215,6 @@ Result: |__.service # SUITE SERVICE DIRECTORY | |__contact # server address information | |__passphrase # suite passphrase - | |__ssl.cert # suite SSL certificate |__log->log.20180418T025047Z # LOG DIRECTORIES |__log.20180418T025047Z # log directory for current suite run | |__ job # job logs (to be distinguished from log/suite/ on the suite host) @@ -263,8 +261,7 @@ Server tracks job progress User views job logs ^^^^^^^^^^^^^^^^^^^ -- command ``cylc cat-log`` via CLI or GUI, invokes itself over SSH to the - remote +- command ``cylc cat-log``, invokes itself over SSH to the remote - suites will serve job logs in future, but this will still be needed (e.g. if the suite is down) diff --git a/doc/src/appendices/site-user-config-ref.rst b/doc/src/appendices/site-user-config-ref.rst index 6bcdc856db2..6e1401336e6 100644 --- a/doc/src/appendices/site-user-config-ref.rst +++ b/doc/src/appendices/site-user-config-ref.rst @@ -176,8 +176,7 @@ Suite event logs are rolled over when they reach this file size. [documentation] --------------- -Documentation locations for the ``cylc doc`` command and gcylc -Help menus. +Documentation locations for the ``cylc doc`` command. [documentation] ``->`` [[online]] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -235,7 +234,7 @@ The editor to be invoked by the cylc command line interface. [editors] ``->`` gui ^^^^^^^^^^^^^^^^^^^^ -The editor to be invoked by the cylc GUI. +GUI Text editor to be invoked by Cylc: - *type*: string - *default*: ``gvim -f`` @@ -244,47 +243,6 @@ The editor to be invoked by the cylc GUI. - ``gui = xterm -e vim`` -[communication] ---------------- - -This section covers options for network communication between cylc -clients (suite-connecting commands and guis) servers (running suites). - -By default, the communication method is HTTPS secured with HTTP Digest -Authentication. If the system does not support SSL, you should configure -this section to use HTTP. Cylc will not automatically fall back to HTTP -if HTTPS is not available. - - -[communication] ``->`` method -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -The choice of client-server communication method - currently only HTTPS -and HTTP are supported, although others could be developed and plugged in. -Cylc defaults to HTTPS if this setting is not explicitly configured. - -- *type*: string -- *options*: - - **https** - - **http** -- *default*: https - -[communication] ``->`` options -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Option flags for the communication method. Currently only 'SHA1' is -supported for HTTPS, which alters HTTP Digest Auth to use the SHA1 hash -algorithm rather than the standard MD5. This is more secure but is also -less well supported by third party web clients including web browsers. -You may need to add the 'SHA1' option if you are running on platforms -where MD5 is discouraged (e.g. under FIPS). - -- *type*: string\_list -- *default*: ``[]`` -- *options*: - - **SHA1** - - [monitor] --------- @@ -582,7 +540,7 @@ setting. """""""""""""""""""""""""""""""""""""""""""""""""""" A command template (with ``%(filename)s`` substitution) to tail-follow -job logs on HOST, by the GUI log viewer and ``cylc cat-log``. You are +job logs on HOST, by ``cylc cat-log``. You are unlikely to need to override this. - *type*: string @@ -718,8 +676,7 @@ exceeds its execution time limit. --------------- Configure allowed suite hosts and ports for starting up (running or -restarting) suites and enabling them to be detected whilst running via -utilities such as ``cylc gscan``. Additionally configure host +restarting) suites. Additionally configure host selection settings specifying how to determine the most suitable run host at any given time from those configured. @@ -744,6 +701,12 @@ Hosts specified in ``condemned hosts`` will not be considered as suite run hosts. If suites are already running on ``condemned hosts`` they will be automatically shutdown and restarted (see :ref:`auto-stop-restart`). +.. warning:: + + Cylc will reject hosts with ambiguous names such as ``localhost`` or + ``127.0.0.1`` for this configuration as ``condemned hosts`` are evaluated + on the suite host server. + - *type*: comma-separated list of host names and/or IP addresses. - *default*: (none) @@ -759,15 +722,6 @@ a ``run`` or ``restart`` command. - *default*: ``localhost`` -[suite servers] ``->`` scan hosts -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -A list of hosts to scan for running suites. - -- *type*: comma-separated list of host names and/or IP addresses. -- *default*: ``localhost`` - - [suite servers] ``->`` run ports ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -785,17 +739,6 @@ A list of allowed ports for Cylc to use to run suites. ``43001, 43002, ... , 43099, 43100``) -[suite servers] ``->`` scan ports -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -A list of ports to scan for running suites on each host set in scan hosts. - -- *type*: string in the format ``X .. Y`` for - ``X <= Y`` where ``X`` and ``Y`` are integers. -- *default*: ``43001 .. 43100`` (equivalent to the list - ``43001, 43002, ... , 43099, 43100``) - - [suite servers] ``->`` [[run host select]] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -1155,11 +1098,20 @@ suite passphrase required. - *type*: string (must be one of the following options) - *options*: - - *identity* - only suite and owner names revealed - - *description* - identity plus suite title and description - - *state-totals* - identity, description, and task state totals - - *full-read* - full read-only access for monitor and GUI - - *shutdown* - full read access plus shutdown, but no other - control. + none + Permit no public suite access. + identity + Only suite and owner names revealed. + description + Identity plus suite title and description. + state-totals + Identity, description, and task state totals. + read + Full read-only access. + shutdown + *Not yet implemented* + Full read access plus shutdown, but no other control. + control + Permit full control (not recommended). - *default*: state-totals diff --git a/doc/src/appendices/suiterc-config-ref.rst b/doc/src/appendices/suiterc-config-ref.rst index 0f4216410c5..24e6ed30b4c 100644 --- a/doc/src/appendices/suiterc-config-ref.rst +++ b/doc/src/appendices/suiterc-config-ref.rst @@ -20,7 +20,7 @@ and description. ------ Section containing metadata items for this suite. Several items -(title, description, URL) are pre-defined and are used by the GUI. Others +(title, description, URL) are pre-defined and are used by Cylc. Others can be user-defined and passed to suite event handlers to be interpreted according to your needs. For example, the value of a "suite-priority" item could determine how an event handler responds to failure events. @@ -29,8 +29,7 @@ could determine how an event handler responds to failure events. [meta] ``->`` title ^^^^^^^^^^^^^^^^^^^ -A single line description of the suite. It is displayed in the GUI -"Open Another Suite" window and can be retrieved at run time with the +A single line description of the suite, can be retrieved at run time with the ``cylc show`` command. - *type*: single line string @@ -53,7 +52,7 @@ with the ``cylc show`` command. ^^^^^^^^^^^^^^^^^ A web URL to suite documentation. If present it can be browsed with the -``cylc doc`` command, or from the gcylc Suite menu. The string +``cylc doc`` command. The string template ``%(suite_name)s`` will be replaced with the actual suite name. See also :ref:`TaskURL`. @@ -65,8 +64,7 @@ name. See also :ref:`TaskURL`. [meta] ``->`` group ^^^^^^^^^^^^^^^^^^^ -A group name for a suite. In the gscan GUI, suites with the same group name can -be collapsed into a single state summary when the "group" column is displayed. +A group name for a suite. - *type*: single line string - *default*: (none) @@ -754,7 +752,7 @@ leap year) and 366 (always a leap year) calendars. In a cold start each cycling task (unless specifically excluded under ``[special tasks]``) will be loaded into the suite with this cycle point, or with the closest subsequent valid cycle point for the task. This item can -be overridden on the command line or in the gcylc suite start panel. +be overridden on the command line. In date-time cycling, if you do not provide time zone information for this, it will be assumed to be local time, or in UTC if :ref:`UTC-mode` is set, or in @@ -862,8 +860,7 @@ Examples of interpretation are given in Cycling tasks are held once they pass the final cycle point, if one is specified. Once all tasks have achieved this state the suite will shut -down. If this item is provided you can override it on the command line -or in the gcylc suite start panel. +down. If this item is provided you can override it on the command line. In date-time cycling, if you do not provide time zone information for this, it will be assumed to be local time, or in UTC if :ref:`UTC-mode` is set, or in @@ -909,7 +906,7 @@ constraints. Cycling tasks are held once they pass the hold after cycle point, if one is specified. Unlike the final cycle point suite will not shut down once all tasks have passed this point. If this item is provided you can override it on the -command line or in the gcylc suite start panel. +command line. .. _runahead limit: @@ -956,8 +953,8 @@ Allows tasks to spawn out to ``max active cycle points`` submitted before its successor can be spawned. *Important*: This should be used with care given the potential impact of -additional task proxies both in terms of memory and cpu for the cylc daemon as -well as overheads in rendering all the additional tasks in gcylc. Also, use +additional task proxies in terms of memory and cpu for the cylc server +program. Also, use of the setting may highlight any issues with suite design relying on the default behaviour where downstream tasks would otherwise be waiting on ones upstream submitting and the suite would have stalled e.g. a housekeeping task @@ -1264,8 +1261,7 @@ for all tasks in the suite. [runtime] ``->`` [[\_\_NAME\_\_]] ``->`` extra log files """""""""""""""""""""""""""""""""""""""""""""""""""""""" -A list of user-defined log files associated with a task. Files defined here -will appear alongside the default log files in the cylc GUI. Log files +A list of user-defined log files associated with a task. Log files must reside in the job log directory ``$CYLC_TASK_LOG_DIR`` and ideally should be named using the ``$CYLC_TASK_LOG_ROOT`` prefix (see :ref:`Task Job Script Variables`). @@ -1431,7 +1427,7 @@ large data area) by a global config setting (see :ref:`workdirectory`). Section containing metadata items for this task or family namespace. Several items (title, description, URL) are pre-defined and are used by -the GUI. Others can be user-defined and passed to task event handlers to be +Cylc. Others can be user-defined and passed to task event handlers to be interpreted according to your needs. For example, the value of an "importance" item could determine how an event handler responds to task failure events. @@ -1474,8 +1470,7 @@ A multi-line description of this namespace, retrievable from running tasks with '''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''' A web URL to task documentation for this suite. If present it can be browsed -with the ``cylc doc`` command, or by right-clicking on the task in -gcylc. The string templates ``%(suite_name)s`` and +with the ``cylc doc`` command. The string templates ``%(suite_name)s`` and ``%(task_name)s`` will be replaced with the actual suite and task names. See also :ref:`SuiteURL`. @@ -2330,11 +2325,7 @@ in simulation or dummy modes. [visualization] --------------- -Configuration of suite graphing for the ``cylc graph`` command (graph -extent, styling, and initial family-collapsed state) and the gcylc graph view -(initial family-collapsed state). See the -`Graphviz documentation of node shapes `_. - +Configure the appearance of suites when displayed in Cylc visualisation tools. [visualization] ``->`` initial cycle point ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -2376,11 +2367,9 @@ it can be overridden by an explicit *final cycle point* (above). ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ A list of family (namespace) names to be shown in the collapsed state -(i.e. the family members will be replaced by a single family node) when -the suite is first plotted in the graph viewer or the gcylc graph view. +(i.e. the family members will be replaced by a single family node) +by default. If this item is not set, the default is to collapse all families at first. -Interactive GUI controls can then be used to group and ungroup family -nodes at will. - *type*: Comma-separated list of family names. - *default*: (none) diff --git a/doc/src/conf.py b/doc/src/conf.py index eb0ce1fdc8a..332e4e6cd7b 100644 --- a/doc/src/conf.py +++ b/doc/src/conf.py @@ -44,8 +44,8 @@ master_doc = 'index' # General information about the project. -project = u'The Cylc Suite Engine' -copyright = u'2008-2019 NIWA & British Crown (Met Office) & Contributors' +project = 'The Cylc Suite Engine' +copyright = '2008-2019 NIWA & British Crown (Met Office) & Contributors' # Versioning information. Sphinx advises version strictly meaning X.Y. version = '.'.join(CYLC_VERSION.split('.')[:2]) # The short X.Y version. @@ -78,8 +78,8 @@ "sidebarbgcolor": "white", "sidebartextcolor": "black", "sidebarlinkcolor": "#0000EE;", - "headbgcolor": "white", - "headtextcolor": "#FF5966", + "headbgcolor": "white", + "headtextcolor": "#FF5966", "linkcolor": "#0000EE;", "visitedlinkcolor": "#551A8B;", "headlinkcolor": "#0000EE;", @@ -88,8 +88,8 @@ # Custom sidebar templates, maps document names to template names. html_sidebars = { - '**': ['globaltoc.html', 'searchbox.html', 'sourcelink.html'], - 'using/windows': ['windowssidebar.html', 'searchbox.html'], + '**': ['globaltoc.html', 'searchbox.html', 'sourcelink.html'], + 'using/windows': ['windowssidebar.html', 'searchbox.html'], } # Logo and favicon to display. @@ -110,16 +110,18 @@ # -- Options for LaTeX output --------------------------------------------- latex_elements = { -'papersize': 'a4paper', -'pointsize': '11pt', + 'papersize': 'a4paper', + 'pointsize': '11pt', } +# Title for the cylc documentation section +CYLC_DOC_TITLE = 'Cylc Documentation' + # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, # author, documentclass [howto, manual, or own class]). latex_documents = [ - ('index', 'cylc.tex', u'Cylc Documentation', - u'2008-2019 NIWA & British Crown (Met Office) & Contributors', 'manual'), + ('index', 'cylc.tex', CYLC_DOC_TITLE, copyright, 'manual'), ] # Image file to place at the top of the title page. @@ -134,8 +136,7 @@ # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). man_pages = [ - ('index', 'cylc', u'cylc Documentation', - u'2008-2019 NIWA & British Crown (Met Office) & Contributors', 1), + ('index', 'cylc', CYLC_DOC_TITLE, copyright, 1), ] # If true, show URL addresses after external links. @@ -148,9 +149,8 @@ # (source start file, target name, title, author, # dir menu entry, description, category) texinfo_documents = [ - ('index', 'cylc', u'Cylc Documentation', - u'2008-2019 NIWA & British Crown (Met Office) & Contributors', - 'cylc', 'The Cylc Suite Engine', 'Miscellaneous'), + ('index', 'cylc', CYLC_DOC_TITLE, copyright, 'cylc', project, + 'Miscellaneous'), ] # How to display URL addresses. diff --git a/doc/src/custom/cylc_lang.py b/doc/src/custom/cylc_lang.py index a0add0e46f6..b14f729ab11 100644 --- a/doc/src/custom/cylc_lang.py +++ b/doc/src/custom/cylc_lang.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 # THIS FILE IS PART OF THE CYLC SUITE ENGINE. # Copyright (C) 2008-2019 NIWA & British Crown (Met Office) & Contributors. diff --git a/doc/src/graphics/png/orig/QuickStartA-ControlRunning.png b/doc/src/graphics/png/orig/QuickStartA-ControlRunning.png deleted file mode 100644 index 63ef0a93340..00000000000 Binary files a/doc/src/graphics/png/orig/QuickStartA-ControlRunning.png and /dev/null differ diff --git a/doc/src/graphics/png/orig/QuickStartA-ControlStalled.png b/doc/src/graphics/png/orig/QuickStartA-ControlStalled.png deleted file mode 100644 index e9c75c23236..00000000000 Binary files a/doc/src/graphics/png/orig/QuickStartA-ControlStalled.png and /dev/null differ diff --git a/doc/src/graphics/png/orig/QuickStartA-ControlStart00.png b/doc/src/graphics/png/orig/QuickStartA-ControlStart00.png deleted file mode 100644 index fce9ce72165..00000000000 Binary files a/doc/src/graphics/png/orig/QuickStartA-ControlStart00.png and /dev/null differ diff --git a/doc/src/graphics/png/orig/QuickStartA-ControlStart06.png b/doc/src/graphics/png/orig/QuickStartA-ControlStart06.png deleted file mode 100644 index 01d21fdac57..00000000000 Binary files a/doc/src/graphics/png/orig/QuickStartA-ControlStart06.png and /dev/null differ diff --git a/doc/src/graphics/png/orig/QuickStartA-ModelState.png b/doc/src/graphics/png/orig/QuickStartA-ModelState.png deleted file mode 100644 index 5619022eb32..00000000000 Binary files a/doc/src/graphics/png/orig/QuickStartA-ModelState.png and /dev/null differ diff --git a/doc/src/graphics/png/orig/QuickStartA-graph18.png b/doc/src/graphics/png/orig/QuickStartA-graph18.png deleted file mode 100644 index ec2c1a5ff2b..00000000000 Binary files a/doc/src/graphics/png/orig/QuickStartA-graph18.png and /dev/null differ diff --git a/doc/src/graphics/png/orig/QuickStartB-graph18.png b/doc/src/graphics/png/orig/QuickStartB-graph18.png deleted file mode 100644 index 12f181d2e9b..00000000000 Binary files a/doc/src/graphics/png/orig/QuickStartB-graph18.png and /dev/null differ diff --git a/doc/src/graphics/png/orig/ecox-1.png b/doc/src/graphics/png/orig/ecox-1.png deleted file mode 100644 index 384b29b5653..00000000000 Binary files a/doc/src/graphics/png/orig/ecox-1.png and /dev/null differ diff --git a/doc/src/graphics/png/orig/gcylc-graph-and-dot-views.png b/doc/src/graphics/png/orig/gcylc-graph-and-dot-views.png deleted file mode 100644 index 13d3da7d382..00000000000 Binary files a/doc/src/graphics/png/orig/gcylc-graph-and-dot-views.png and /dev/null differ diff --git a/doc/src/graphics/png/orig/gcylc-text-view.png b/doc/src/graphics/png/orig/gcylc-text-view.png deleted file mode 100644 index f3802905d07..00000000000 Binary files a/doc/src/graphics/png/orig/gcylc-text-view.png and /dev/null differ diff --git a/doc/src/graphics/png/orig/gscan.png b/doc/src/graphics/png/orig/gscan.png deleted file mode 100644 index 25620158cb3..00000000000 Binary files a/doc/src/graphics/png/orig/gscan.png and /dev/null differ diff --git a/doc/src/graphics/png/orig/inherit-2.png b/doc/src/graphics/png/orig/inherit-2.png deleted file mode 100644 index 3c46c92ba6f..00000000000 Binary files a/doc/src/graphics/png/orig/inherit-2.png and /dev/null differ diff --git a/doc/src/graphics/png/orig/inherit-3.png b/doc/src/graphics/png/orig/inherit-3.png deleted file mode 100644 index 5d2d5e30895..00000000000 Binary files a/doc/src/graphics/png/orig/inherit-3.png and /dev/null differ diff --git a/doc/src/graphics/png/orig/inherit-4.png b/doc/src/graphics/png/orig/inherit-4.png deleted file mode 100644 index c9dc9e0d7c3..00000000000 Binary files a/doc/src/graphics/png/orig/inherit-4.png and /dev/null differ diff --git a/doc/src/graphics/png/orig/inherit-5.png b/doc/src/graphics/png/orig/inherit-5.png deleted file mode 100644 index 8036ac91bad..00000000000 Binary files a/doc/src/graphics/png/orig/inherit-5.png and /dev/null differ diff --git a/doc/src/graphics/png/orig/inherit-6.png b/doc/src/graphics/png/orig/inherit-6.png deleted file mode 100644 index cd92d0d07a3..00000000000 Binary files a/doc/src/graphics/png/orig/inherit-6.png and /dev/null differ diff --git a/doc/src/graphics/png/orig/inherit-7.png b/doc/src/graphics/png/orig/inherit-7.png deleted file mode 100644 index fd53fbebeeb..00000000000 Binary files a/doc/src/graphics/png/orig/inherit-7.png and /dev/null differ diff --git a/doc/src/graphics/png/scaled/QuickStartA-ControlRunning.png b/doc/src/graphics/png/scaled/QuickStartA-ControlRunning.png deleted file mode 100644 index f82b1768388..00000000000 Binary files a/doc/src/graphics/png/scaled/QuickStartA-ControlRunning.png and /dev/null differ diff --git a/doc/src/graphics/png/scaled/QuickStartA-ControlStalled.png b/doc/src/graphics/png/scaled/QuickStartA-ControlStalled.png deleted file mode 100644 index e3e6cd4c51b..00000000000 Binary files a/doc/src/graphics/png/scaled/QuickStartA-ControlStalled.png and /dev/null differ diff --git a/doc/src/graphics/png/scaled/QuickStartA-ControlStart00.png b/doc/src/graphics/png/scaled/QuickStartA-ControlStart00.png deleted file mode 100644 index 9647e03bb28..00000000000 Binary files a/doc/src/graphics/png/scaled/QuickStartA-ControlStart00.png and /dev/null differ diff --git a/doc/src/graphics/png/scaled/QuickStartA-ControlStart06.png b/doc/src/graphics/png/scaled/QuickStartA-ControlStart06.png deleted file mode 100644 index 2af42c8a784..00000000000 Binary files a/doc/src/graphics/png/scaled/QuickStartA-ControlStart06.png and /dev/null differ diff --git a/doc/src/graphics/png/scaled/QuickStartA-ModelState.png b/doc/src/graphics/png/scaled/QuickStartA-ModelState.png deleted file mode 100644 index 62295d7fb0b..00000000000 Binary files a/doc/src/graphics/png/scaled/QuickStartA-ModelState.png and /dev/null differ diff --git a/doc/src/graphics/png/scaled/QuickStartA-graph18.png b/doc/src/graphics/png/scaled/QuickStartA-graph18.png deleted file mode 100644 index b0b1cbe7f7f..00000000000 Binary files a/doc/src/graphics/png/scaled/QuickStartA-graph18.png and /dev/null differ diff --git a/doc/src/graphics/png/scaled/QuickStartB-graph18.png b/doc/src/graphics/png/scaled/QuickStartB-graph18.png deleted file mode 100644 index fee71792fcb..00000000000 Binary files a/doc/src/graphics/png/scaled/QuickStartB-graph18.png and /dev/null differ diff --git a/doc/src/index.rst b/doc/src/index.rst index 257f3153f4e..a98bc415426 100644 --- a/doc/src/index.rst +++ b/doc/src/index.rst @@ -29,7 +29,6 @@ indefinitely. :numbered: introduction - screenshots installation terminology workflows diff --git a/doc/src/installation.rst b/doc/src/installation.rst index 69ebe24978c..914dc29f80b 100644 --- a/doc/src/installation.rst +++ b/doc/src/installation.rst @@ -10,50 +10,25 @@ including Apple OS X, but they are not officially tested and supported. Third-Party Software Packages ----------------------------- -**Python 2** ``>=`` **2.6** is required. **Python 2** ``>=`` **2.7.9** is -recommended for the best security. `Python `_ 2 should -already be installed in your Linux system. +Requirements: -For Cylc's HTTPS communications layer: +- Python 3.6+ -- `OpenSSL `_ -- `pyOpenSSL `_ -- `python-requests `_ -- **python-urllib3** - should be bundled with python-requests + - `python-jose `_ + - `zmq `_ + - `colorama `_ -The following packages are highly recommended, but are technically optional as -you can construct and run suites without dependency graph visualisation or -the Cylc GUIs: +The following packages are necessary for running tests in Cylc: -- `PyGTK `_ - GUI toolkit. - - .. note:: - - PyGTK typically comes with your system Python. It is allegedly quite - difficult to install if you need to do so for another Python version. - -- `Graphviz `_ - graph layout engine (tested 2.36.0) -- `Pygraphviz `_ - Python Graphviz interface - (tested 1.2). To build this you may need some *devel* packages too: - - - python-devel - - graphviz-devel - -The Cylc Review service does not need any additional packages. - -The following packages are necessary for running all the tests in Cylc: - -- `mock `_ +- `pytest `_ To generate the HTML User Guide, you will need: - `Sphinx `_ of compatible version, ``>=`` **1.5.3** and ``<=`` **1.7.9**. -In most modern Linux distributions all of the software above can be installed -via the system package manager. Otherwise download packages manually and follow -their native installation instructions. To check that all packages -are installed properly: +To check that dependencies are installed and environment is configured +correctly run ``cylc check-software``: .. code-block:: none @@ -61,34 +36,28 @@ are installed properly: Checking your software... Individual results: - =============================================================================== - Package (version requirements) Outcome (version found) - =============================================================================== + ================================================================================ + Package (version requirements) Outcome (version found) + ================================================================================ *REQUIRED SOFTWARE* - Python (2.6+, <3).....................FOUND & min. version MET (2.7.12.final.0) + Python (3+).............................FOUND & min. version MET (3.7.2.final.0) + Python:zmq (any)..................................................FOUND (17.1.2) + Python:jose (any)..................................................FOUND (2.0.2) + Python:colorama (any)..............................................FOUND (0.4.1) - *OPTIONAL SOFTWARE for the GUI & dependency graph visualisation* - Python:pygraphviz (any)...........................................NOT FOUND (-) - graphviz (any)...................................................FOUND (2.26.0) - Python:pygtk (2.0+)...............................................NOT FOUND (-) + *OPTIONAL SOFTWARE for the configuration templating* + Python:EmPy (any)..................................................FOUND (3.3.2) - *OPTIONAL SOFTWARE for the HTTPS communications layer* - Python:requests (2.4.2+)......................FOUND & min. version MET (2.11.1) - Python:urllib3 (any)..............................................NOT FOUND (-) - Python:OpenSSL (any)..............................................NOT FOUND (-) - - *OPTIONAL SOFTWARE for the configuration templating* - Python:EmPy (any).................................................NOT FOUND (-) - - *OPTIONAL SOFTWARE for the HTML documentation* - Python:sphinx (1.5.3+).........................FOUND & min. version MET (1.7.0) - =============================================================================== + *OPTIONAL SOFTWARE for the HTML documentation* + Python:sphinx (1.5.3+)..........................FOUND & min. version MET (1.8.4) + ================================================================================ Summary: - **************************** + **************************** Core requirements: ok - Full-functionality: not ok - **************************** + Full-functionality: ok + **************************** + If errors are reported then the packages concerned are either not installed or not in your Python search path. @@ -120,23 +89,11 @@ Software Bundled With Cylc Cylc bundles several third party packages which do not need to be installed separately. -- `cherrypy `_ **6.0.2** (slightly modified): a pure - Python HTTP framework that we use as a web server for communication between - server processes (suite server programs) and client programs (running tasks, - GUIs, CLI commands). - - - Client communication is via the Python - `requests `_ library if available - (recommended) or else pure Python via **urllib2**. - - `Jinja2 `_ **2.10**: a full featured template engine for Python, and its dependency `MarkupSafe `_ **0.23**; both BSD licensed. -- the `xdot `_ graph viewer (modified), - LGPL licensed. - .. _InstallCylc: diff --git a/doc/src/running-suites.rst b/doc/src/running-suites.rst index 2eb7d5df717..2f3c4f5b002 100644 --- a/doc/src/running-suites.rst +++ b/doc/src/running-suites.rst @@ -91,8 +91,7 @@ Restart From Latest Checkpoint """""""""""""""""""""""""""""" To restart from the latest checkpoint simply invoke the ``cylc restart`` -command with the suite name (or select "restart" in the GUI suite start dialog -window): +command with the suite name. .. code-block:: bash @@ -163,8 +162,6 @@ Once you have identified the right checkpoint, restart the suite like this: $ cylc restart --checkpoint=CHECKPOINT-ID SUITE -or enter the checkpoint ID in the space provided in the GUI restart window. - Checkpointing With A Task """"""""""""""""""""""""" @@ -289,7 +286,7 @@ Task Job Polling At any point after job submission task jobs can be *polled* to check that their true state conforms to what is currently recorded by the suite server program. See ``cylc poll --help`` for how to poll one or more tasks -manually, or right-click poll a task or family in GUI. +manually. Polling may be necessary if, for example, a task job gets killed by the untrappable SIGKILL signal (e.g. ``kill -9 PID``), or if a network @@ -313,7 +310,7 @@ to find out what happened to them while the suite was down. Finally, in necessary routine polling can be configured as a way to track job status on job hosts that do not allow networking routing back to the suite host -for task messaging by HTTPS or ssh. See :ref:`Polling To Track Job Status`. +for task messaging by TCP or SSH. See :ref:`Polling To Track Job Status`. .. _TaskComms: @@ -323,37 +320,36 @@ Tracking Task State Cylc supports three ways of tracking task state on job hosts: -- task-to-suite messaging via HTTPS -- task-to-suite messaging via non-interactive ssh to the suite host, - then local HTTPS +- task-to-suite messaging via TCP (using ZMQ protocol) +- task-to-suite messaging via non-interactive SSH to the suite host, + then local TCP - regular polling by the suite server program These can be configured per job host in the Cylc global config file - see :ref:`SiteRCReference`. -If your site prohibits HTTPS and ssh back from job hosts to +If your site prohibits TCP and SSH back from job hosts to suite hosts, before resorting to the polling method you should consider installing dedicated Cylc servers or -VMs inside the HPC trust zone (where HTTPS and ssh should be allowed). +VMs inside the HPC trust zone (where TCP and SSH should be allowed). It is also possible to run Cylc suite server programs on HPC login -nodes, but this is not recommended for load, run duration, -and GUI reasons. +nodes, but this is not recommended for load and run duration, Finally, it has been suggested that *port forwarding* may provide another solution - but that is beyond the scope of this document. -HTTPS Task Messaging -^^^^^^^^^^^^^^^^^^^^ +TCP Task Messaging +^^^^^^^^^^^^^^^^^^ Task job wrappers automatically invoke ``cylc message`` to report progress back to the suite server program when they begin executing, at normal exit (success) and abnormal exit (failure). -By default the messaging occurs via an authenticated, HTTPS connection to the -suite server program. This is the preferred task communications -method - it is efficient and direct. +By default the messaging occurs via an authenticated, TCP connection to the +suite server program using the ZMQ protocol. +This is the preferred task communications method - it is efficient and direct. Suite server programs automatically install suite contact information and credentials on job hosts. Users only need to do this manually @@ -365,14 +361,14 @@ Ssh Task Messaging ^^^^^^^^^^^^^^^^^^ Cylc can be configured to re-invoke task messaging commands on the -suite host via non-interactive ssh (from job host to suite host). -Then a local HTTPS connection is made to the suite server program. +suite host via non-interactive SSH (from job host to suite host). +Then a local TCP connection is made to the suite server program. -(User-invoked client commands (aside from the GUI, which requires HTTPS) +(User-invoked client commands can do the same thing with the ``--use-ssh`` command option). -This is less efficient than direct HTTPS messaging, but it may be useful at -sites where the HTTPS ports are blocked but non-interactive ssh is allowed. +This is less efficient than direct TCP messaging, but it may be useful at +sites where the TCP ports are blocked but non-interactive SSH is allowed. .. _Polling To Track Job Status: @@ -381,15 +377,15 @@ Polling to Track Job Status ^^^^^^^^^^^^^^^^^^^^^^^^^^^ Finally, suite server programs can actively poll task jobs at -configurable intervals, via non-interactive ssh to the job host. +configurable intervals, via non-interactive SSH to the job host. Polling is the least efficient task communications method because task state is updated only at intervals, not when task events actually occur. However, it -may be needed at sites that do not allow HTTPS or non-interactive ssh from job +may be needed at sites that do not allow TCP or non-interactive SSH from job host to suite host. Be careful to avoid spamming task hosts with polling commands. Each poll -opens (and then closes) a new ssh connection. +opens (and then closes) a new SSH connection. Polling intervals are configurable under ``[runtime]`` because they should may depend on the expected execution time. For instance, a @@ -444,7 +440,7 @@ information from the contact file, if they have access to it. File-Reading Commands --------------------- -Some Cylc commands and GUI actions parse suite configurations or read +Some Cylc commands parse suite configurations or read other files from the suite host account, rather than communicate with a suite server program over the network. In future we plan to have suite server program serve @@ -467,7 +463,7 @@ Remote Host, Different Home Directory ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ If you are logged into another host with no shared home directory, file-reading -commands require non-interactive ssh to the suite host account, and use of the +commands require non-interactive SSH to the suite host account, and use of the ``--host`` and ``--user`` options to re-invoke the command on the suite account. @@ -484,8 +480,7 @@ Client-Server Interaction ------------------------- Cylc server programs listen on dedicated network ports for -HTTPS communications from Cylc clients (task jobs, and user-invoked commands -and GUIs). +TCP communications from Cylc clients (task jobs and user-invoked commands) Use ``cylc scan`` to see which suites are listening on which ports on scanned hosts (this lists your own suites by default, but it can show others @@ -509,15 +504,25 @@ server program is determined by the public access privilege level set in global site/user config (:ref:`GlobalAuth`) and optionally overidden in suites (:ref:`SuiteAuth`): -- *identity* - only suite and owner names revealed -- *description* - identity plus suite title and description -- *state-totals* - identity, description, and task state totals -- *full-read* - full read-only access for monitor and GUI -- *shutdown* - full read access plus shutdown, but no other control. +none + Permit no public suite access. +identity + Only suite and owner names revealed. +description + Identity plus suite title and description. +state-totals + Identity, description, and task state totals. +read + Full read-only access. +shutdown + *Not yet implemented* + Full read access plus shutdown, but no other control. +control + Permit full control (not recommended). The default public access level is *state-totals*. -The ``cylc scan`` command and the ``cylc gscan`` GUI can print +The ``cylc scan`` command can print descriptions and task state totals in addition to basic suite identity, if the that information is revealed publicly. @@ -538,7 +543,7 @@ possess the passphrase file for that suite. Fine-grained access to a single suite server program via distinct user accounts is not currently supported. Suite server programs automatically install their auth and contact files to job -hosts via ssh, to enable task jobs to connect back to the suite server program +hosts via SSH, to enable task jobs to connect back to the suite server program for task messaging. Client programs invoked by the suite owner automatically load the passphrase, @@ -548,34 +553,18 @@ SSL certificate, and contact file too, for automatic connection to suites. if you do not have a shared filesystem - see below.* -.. _GUI-to-Suite Interaction: - -GUI-to-Suite Interaction ------------------------- - -The gcylc GUI is mainly a network client to retrieve and display suite status -information from the suite server program, but it can also invoke file-reading -commands to view and graph the suite configuration and so on. This is entirely -transparent if the GUI is running on the suite host account, but full -functionality for remote suites requires either a shared filesystem, or -(see :ref:`RemoteControl`) auth file installation *and* non-interactive ssh -access to the suite host. Without the auth files you will not be able -to connect to the suite, and without ssh you will see "permission denied" -errors on attempting file access. - - .. _RemoteControl: Remote Control -------------- -Cylc client programs - command line and GUI - can interact with suite server +Cylc client programs can interact with suite server programs running on other accounts or hosts. How this works depends on whether or not you have: - a *shared filesystem* such that you see the same home directory on both hosts. -- *non-interactive ssh* from the client account to the server +- *non-interactive SSH* from the client account to the server account. With a shared filesystem, a suite registered on the remote (server) host is @@ -591,7 +580,6 @@ must be installed under your ``$HOME/.cylc/`` directory: .. code-block:: bash $HOME/.cylc/auth/OWNER@HOST/SUITE/ - ssl.cert passphrase contact # (optional - see below) @@ -601,16 +589,16 @@ is the suite name. Client commands should then be invoked with the .. code-block:: bash - $ cylc gui --user=OWNER --host=HOST SUITE + $ cylc edit --user=OWNER --host=HOST SUITE .. note:: Remote suite auth files do not need to be installed for read-only - access - see :ref:`PublicAccess` - via the GUI or monitor. + access - see :ref:`PublicAccess`. The suite contact file (see :ref:`The Suite Contact File`) is not needed if you have read-access to the remote suite run directory via the local -filesystem or non-interactive ssh to the suite host account - client commands +filesystem or non-interactive SSH to the suite host account - client commands will automatically read it. If you do install the contact file in your auth directory note that the port number will need to be updated if the suite gets restarted on a different port. Otherwise use ``cylc scan`` to determine @@ -621,23 +609,17 @@ the suite port number and use the ``--port`` client command option. Possession of a suite passphrase gives full control over the target suite, including edit run functionality - which lets you run arbitrary scripting on job hosts as the suite owner. Further, - non-interactive ssh gives full access to the target user account, so we + non-interactive SSH gives full access to the target user account, so we recommended that this is only used to interact with suites running on accounts to which you already have full access. -.. _Scan And Gscan: - -Scan And Gscan --------------- +Scan +---- -Both ``cylc scan`` and the ``cylc gscan`` GUI can display +``cylc scan`` can display suites owned by other users on other hosts, including task state totals if the -public access level permits that (see :ref:`PublicAccess`). Clicking on a -remote suite in ``gscan`` will open a ``cylc gui`` to connect to that -suite. This will give you full control, if you have the suite auth files -installed; or it will display full read only information if the public access -level allows that. +public access level permits that (see :ref:`PublicAccess`). Task States Explained @@ -679,25 +661,6 @@ As a suite runs, its task proxies may pass through the following states: see :ref:`ClockExpireTasks`. -What The Suite Control GUI Shows --------------------------------- - -The GUI Text-tree and Dot Views display the state of every task proxy present -in the task pool. Once a task has succeeded and Cylc has determined that it can -no longer be needed to satisfy the prerequisites of other tasks, its proxy will -be cleaned up (removed from the pool) and it will disappear from the GUI. To -rerun a task that has disappeared from the pool, you need to re-insert its task -proxy and then re-trigger it. - -The Graph View is slightly different: it displays the complete dependency graph -over the range of cycle points currently present in the task pool. This often -includes some greyed-out *base* or *ghost nodes* that are empty - i.e. -there are no corresponding task proxies currently present in the pool. Base -nodes just flesh out the graph structure. Groups of them may be cut out and -replaced by single *scissor nodes* in sections of the graph that are -currently inactive. - - Network Connection Timeouts --------------------------- @@ -777,7 +740,7 @@ members: Any tasks not assigned to a particular queue will remain in the default queue. The *queues* example suite illustrates how queues work by -running two task trees side by side (as seen in the graph GUI) each +running two task trees side by side each limited to 2 and 3 tasks respectively: .. literalinclude:: ../../etc/examples/queues/suite.rc @@ -796,8 +759,8 @@ ISO 8601 durations. If the task job fails it will go into the *retrying* state and resubmit after the next configured delay interval. An example is shown in the suite listed below under :ref:`EventHandling`. -If a task with configured retries is *killed* (by ``cylc kill`` or -via the GUI) it goes to the *held* state so that the operator can decide +If a task with configured retries is *killed* (by ``cylc kill`` +it goes to the *held* state so that the operator can decide whether to release it and continue the retry sequence or to abort the retry sequence by manually resetting it to the *failed* state. @@ -1042,12 +1005,12 @@ Manual Task Triggering and Edit-Run ----------------------------------- Any task proxy currently present in the suite can be manually triggered at any -time using the ``cylc trigger`` command, or from the right-click task -menu in gcylc. If the task belongs to a limited internal queue +time using the ``cylc trigger`` command. +If the task belongs to a limited internal queue (see :ref:`InternalQueues`), this will queue it; if not, or if it is already queued, it will submit immediately. -With ``cylc trigger --edit`` (also in the gcylc right-click task menu) +With ``cylc trigger --edit`` you can edit the generated task job script to make one-off changes before the task submits. @@ -1139,8 +1102,7 @@ running the suite's real jobs - which may be long-running and resource-hungry: - simulates scheduling without generating any job files. -Set the run mode (default *live*) in the GUI suite start dialog box, or on -the command line: +Set the run mode (default *live*) on the command line: .. code-block:: bash @@ -1429,7 +1391,7 @@ suite to finish and shut down. Here's the complete suite log for this run: 2017-03-30T09:46:42Z WARNING - * foo.2017-01-01T00Z succeeded 2017-03-30T09:47:58Z INFO - [client-command] reset_task_states vagrant@cylon:cylc-reset 1e0d8e9f-2833-4dc9-a0c8-9cf263c4c8c3 2017-03-30T09:47:58Z INFO - [foo.2017-01-01T00Z] -resetting state to succeeded - 2017-03-30T09:47:58Z INFO - Command succeeded: reset_task_states([u'foo.2017'], state=succeeded) + 2017-03-30T09:47:58Z INFO - Command succeeded: reset_task_states(['foo.2017'], state=succeeded) 2017-03-30T09:47:59Z INFO - [bar.2017-01-01T00Z] -submit_method_id=3565 2017-03-30T09:47:59Z INFO - [bar.2017-01-01T00Z] -submission succeeded 2017-03-30T09:47:59Z INFO - [bar.2017-01-01T00Z] -(current:submitted)> started at 2017-03-30T09:47:58Z @@ -1611,6 +1573,12 @@ running on ``bar`` will stop immediately, making no attempt to restart. run hosts = pub condemned hosts = foo, bar! +.. warning:: + + Cylc will reject hosts with ambiguous names such as ``localhost`` or + ``127.0.0.1`` for this configuration as ``condemned hosts`` are evaluated + on the suite host server. + To prevent large numbers of suites attempting to restart simultaneously the ``auto restart delay`` setting defines a period of time in seconds. Suites will wait for a random period of time between zero and diff --git a/doc/src/screenshots.rst b/doc/src/screenshots.rst deleted file mode 100644 index 7598a694cf7..00000000000 --- a/doc/src/screenshots.rst +++ /dev/null @@ -1,30 +0,0 @@ -Cylc Screenshots -================ - -.. _fig-gcylc-1: - -.. figure:: graphics/png/orig/gcylc-graph-and-dot-views.png - :align: center - - gcylc graph and dot views. - -.. _fig-gcylc-2: - -.. figure:: graphics/png/orig/gcylc-text-view.png - :align: center - - gcylc text view. - -.. _fig-gscan: - -.. figure:: graphics/png/orig/gscan.png - :align: center - - gscan multi-suite state summary GUI. - -.. _fig-ecox-1: - -.. figure:: graphics/png/orig/ecox-1.png - :align: center - - A large-ish suite graphed by cylc. diff --git a/doc/src/suite-config.rst b/doc/src/suite-config.rst index b670dccb695..1e05f2e77cb 100644 --- a/doc/src/suite-config.rst +++ b/doc/src/suite-config.rst @@ -2044,11 +2044,7 @@ inherit from the *root* namespace (below). Nested namespaces define *task families* that can be used in the graph as convenient shorthand for triggering all member tasks at once, or for triggering other tasks off all members at once - -see :ref:`FamilyTriggers`. Nested namespaces can be -progressively expanded and collapsed in the dependency graph viewer, and -in the gcylc graph and text views. Only the first parent of each -namespace (as for single-inheritance) is used for suite visualization -purposes. +see :ref:`FamilyTriggers`. Namespace Names @@ -2482,62 +2478,9 @@ Visualization ------------- The visualization section of a suite configuration is used to configure -suite graphing, principally graph node (task) and edge (dependency -arrow) style attributes. Tasks can be grouped for the purpose of -applying common style attributes. See :ref:`SuiteRCReference` for details. +the representation of a suite in Cylc visualisation tools. - -Collapsible Families In Suite Graphs -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. code-block:: cylc - - [visualization] - collapsed families = family1, family2 - -Nested families from the runtime inheritance hierarchy can be expanded -and collapsed in suite graphs and the gcylc graph view. All families -are displayed in the collapsed state at first, unless -``[visualization]collapsed families`` is used to single out -specific families for initial collapsing. - -In the gcylc graph view, nodes outside of the main graph (such as the -members of collapsed families) are plotted as rectangular nodes to -the right if they are doing anything interesting (submitted, running, -failed). - -:numref:`fig-namespaces` illustrates successive expansion of nested -task families in the *namespaces* example suite. - -.. todo:: - - Create sub-figures if possible: for now hacked as separate figures with - link and caption on final displayed figure. - -.. figure:: graphics/png/orig/inherit-2.png - :align: center - -.. figure:: graphics/png/orig/inherit-3.png - :align: center - -.. figure:: graphics/png/orig/inherit-4.png - :align: center - -.. figure:: graphics/png/orig/inherit-5.png - :align: center - -.. figure:: graphics/png/orig/inherit-6.png - :align: center - -.. _fig-namespaces: - -.. figure:: graphics/png/orig/inherit-7.png - :align: center - - Graphs of the *namespaces* example suite showing various states of - expansion of the nested namespace family hierarchy, from all families - collapsed (top left) through to all expanded (bottom right). This - can also be done by right-clicking on tasks in the gcylc graph view. +See :ref:`SuiteRCReference` for details. .. _Parameterized Tasks Label: diff --git a/doc/src/suite-design-guide/efficiency.rst b/doc/src/suite-design-guide/efficiency.rst index a8c35680841..c725d42f2fb 100644 --- a/doc/src/suite-design-guide/efficiency.rst +++ b/doc/src/suite-design-guide/efficiency.rst @@ -20,7 +20,7 @@ A properly designed family hierarchy fulfils three purposes in Cylc: tasks - efficient bulk triggering, for clear scheduling graphs - clean suite visualization and monitoring, because families are - collapsible in the GUIs + collapsible .. _Sharing By Inheritance: diff --git a/doc/src/suite-design-guide/general-principles.rst b/doc/src/suite-design-guide/general-principles.rst index 0cc1af198c7..5d78dcfe081 100644 --- a/doc/src/suite-design-guide/general-principles.rst +++ b/doc/src/suite-design-guide/general-principles.rst @@ -54,11 +54,6 @@ through inter-suite triggering. Each approach has its pros and cons, depending on your requirements and preferences with respect to the complexity and manageability of the resulting system. -The ``cylc gscan`` GUI lets you monitor multiple suites at a time, and -you can define virtual groups of suites that collapse into a single state -summary. - - Inter-Suite Triggering ^^^^^^^^^^^^^^^^^^^^^^ @@ -112,7 +107,7 @@ Installing Files At Start-up Use ``rose suite-run`` *file creation mode* or ``R1`` install tasks to copy files to the self-contained suite run directory at start-up. Install tasks are preferred for time-consuming installations because -they don't slow the suite start-up process, they can be monitored in the GUI, +they don't slow the suite start-up process, they can be monitored, they can run directly on target platforms, and you can rerun them later without restarting the suite. If you are using symbolic links to install files under your suite directory it is recommended that the linking should be set up to diff --git a/doc/src/suite-design-guide/style-guide.rst b/doc/src/suite-design-guide/style-guide.rst index 549abdadb29..923728ec8c5 100644 --- a/doc/src/suite-design-guide/style-guide.rst +++ b/doc/src/suite-design-guide/style-guide.rst @@ -235,8 +235,7 @@ Titles, Descriptions, And URLs ------------------------------ Document the suite and its tasks with ``title``, -``description``, and ``url`` items instead of comments. These -can be displayed, or linked to, by the GUI at runtime. +``description``, and ``url`` items instead of comments. .. _Line Length: @@ -276,7 +275,7 @@ Use ``UPPERCASE`` for family names and ``lowercase`` for tasks, so you can distinguish them at a glance. Choose a convention for multi-component names and use it consistently. Put the -most general name components first for natural grouping in the GUI, e.g. +most general name components first for natural grouping, e.g. ``obs_sonde``, ``obs_radar`` (not ``sonde_obs`` etc.) Within your convention keep names as short as possible. diff --git a/doc/src/task-job-submission.rst b/doc/src/task-job-submission.rst index 00e1a735b70..e7e1404a877 100644 --- a/doc/src/task-job-submission.rst +++ b/doc/src/task-job-submission.rst @@ -374,7 +374,7 @@ to here will be complete task job logs. Some batch systems, such as ``pbs``, redirect a job's stdout and stderr streams to a separate cache area while the job is running. The contents are only copied to the normal locations when the job completes. This -means that ``cylc cat-log`` or the gcylc GUI will be unable to find the +means that ``cylc cat-log`` will be unable to find the job's stdout and stderr streams while the job is running. Some sites with these batch systems are known to provide commands for viewing and/or tail-follow a job's stdout and stderr streams that are redirected to these @@ -431,7 +431,7 @@ Polling may be required to update the suite state correctly after unusual events such as a machine being rebooted with tasks running on it, or network problems that prevent task messages from getting back to the suite host. -Tasks can be polled on demand by right-clicking on them in gcylc or using the +Tasks can be polled on demand by using the ``cylc poll`` command. Tasks are polled automatically, once, if they timeout while queueing in a @@ -482,8 +482,7 @@ For supported batch systems, the suite server program can execute a process on the task host, by non-interactive ssh, to kill a submitted or running job according to its batch system. -Tasks can be killed on demand by right-clicking on them in gcylc or using the -``cylc kill`` command. +Tasks can be killed on demand by using the ``cylc kill`` command. Execution Time Limit @@ -553,7 +552,7 @@ batch system handler to change the directive prefix from ``#PBS`` to .. code-block:: python - #!/usr/bin/env python2 + #!/usr/bin/env python3 from cylc.batch_sys_handlers.pbs import PBSHandler diff --git a/doc/src/tutorial.rst b/doc/src/tutorial.rst index 456a2323e76..ca4c2fce7ca 100644 --- a/doc/src/tutorial.rst +++ b/doc/src/tutorial.rst @@ -37,8 +37,8 @@ See :ref:`Configure Site Environment on Job Hosts` for information. User Interfaces --------------- -You should have access to the cylc command line (CLI) and graphical (GUI) user -interfaces once cylc has been installed as described in +You should have access to the cylc command line (CLI) user +interface once cylc has been installed as described in Section :ref:`InstallCylc`. Command Line Interface (CLI) @@ -53,28 +53,11 @@ and their help documentation. $ cylc help # Top level command help. $ cylc run --help # Example command-specific help. -Command help transcripts are printed in :ref:`CommandReference` and are -available from the GUI Help menu. +Command help transcripts are printed in :ref:`CommandReference`. Cylc is *scriptable* - the error status returned by commands can be relied on. -Graphical User Interface (GUI) -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -The cylc GUI covers the same functionality as the CLI, but it has more -sophisticated suite monitoring capability. It can start and stop suites, or -connect to suites that are already running; in either case, shutting down the -GUI does not affect the suite itself. - -.. code-block:: bash - - $ gcylc & # or: - $ cylc gui & # Single suite control GUI. - $ cylc gscan & # Multi-suite monitor GUI. - -Clicking on a suite in gscan, shown in :numref:`fig-gscan`, opens a -gcylc instance for it. Suite Configuration ------------------- @@ -229,14 +212,6 @@ just above and open the *Hello World* suite in your text editor: $ cylc edit tut/oneoff/basic # in-terminal $ cylc edit -g tut/oneoff/basic & # or GUI -Alternatively, start gcylc on the suite: - -.. code-block:: bash - - $ gcylc tut/oneoff/basic & - -and choose *Suite* ``->`` *Edit* from the menu. - The editor will be invoked from within the suite configuration directory for easy access to other suite files (in this case there are none). There are syntax highlighting control files for several text editors under @@ -270,7 +245,7 @@ that you can exit the terminal or even log out without killing the suite: .___! | are welcome to redistribute it under certain !_____! conditions; see `cylc conditions`. - *** listening on https://nwp-1:43027/ *** + *** listening on tcp://nwp-1:43027/ *** To view suite server program contact information: $ cylc get-suite-contact tut/oneoff/basic @@ -308,23 +283,6 @@ overwritten if a job is resubmitted for any reason). The suite shuts down automatically once all tasks have succeeded. -GUI -^^^ - -The cylc GUI can start and stop suites, or (re)connect to suites that -are already running: - -.. code-block:: bash - - $ cylc gui tut/oneoff/basic & - -Use the tool bar *Play* button, or the *Control* ``->`` *Run* menu item, to -run the suite again. You may want to alter the suite configuration slightly -to make the task take longer to run. Try right-clicking on the -``hello`` task to view its output logs. The relative merits of the three -*suite views* - dot, text, and graph - will be more apparent later when we -have more tasks. Closing the GUI does not affect the suite itself. - .. _RemoteSuites: @@ -347,8 +305,7 @@ provided, then ranking according to the given *rank* method. Discovering Running Suites -------------------------- -Suites that are currently running can be detected with command line or -GUI tools: +Suites that are currently running can be detected using ``cylc scan``: .. code-block:: bash @@ -356,12 +313,6 @@ GUI tools: $ cylc scan tut/oneoff/basic oliverh@nwp-1:43001 - # GUI summary view of running suites: - $ cylc gscan & - -The scan GUI is shown in :numref:`fig-gscan`; clicking on a suite in -it opens gcylc. - Task Identifiers ---------------- @@ -388,8 +339,7 @@ specified in the suite configuration (environment, script, etc.) in error trapping code, messaging calls to report task progress back to the suite server program, and so forth. Job scripts are written to the *suite job log directory* where they can be viewed alongside the job output logs. They -can be accessed at run time by right-clicking on the task in the cylc GUI, or -printed to the terminal: +can printed to the terminal: .. code-block:: bash @@ -429,7 +379,7 @@ system. Cylc supports a number of different batch systems. Tasks submitted to external batch queuing systems like ``at``, ``PBS``, ``SLURM``, ``Moab``, or ``LoadLeveler``, are displayed as -*submitted* in the cylc GUI until they start executing. +*submitted* until they start executing. - For more on task job scripts, see :ref:`JobScripts`. - For more on batch systems, see :ref:`AvailableMethods`. @@ -451,10 +401,7 @@ time-stamped suite log file, and task job scripts and job logs | |-- contact # detail on how to contact the running suite | |-- db # private suite run database | |-- passphrase # passphrase for client authentication - | |-- source # symbolic link to source directory - | |-- ssl.cert # SSL certificate for the suite server - | `-- ssl.pem # SSL private key - |-- cylc-suite.db # back compat symlink to public suite run database + | `-- source # symbolic link to source directory |-- share # suite share directory (not used in this example) |-- work # task work space (sub-dirs are deleted if not used) | `-- 1 # task cycle point directory (or 1) @@ -491,10 +438,7 @@ config files if necessary, and the suite share and work locations can be configured separately because of the potentially larger disk space requirement. -Task job logs can be viewed by right-clicking on tasks in the gcylc -GUI (so long as the task proxy is live in the suite), manually -accessed from the log directory (of course), or printed to the terminal -with the ``cylc cat-log`` command: +Task job logs can be viewed with the ``cylc cat-log`` command: .. code-block:: bash @@ -596,8 +540,8 @@ directories, transfer any essential service files, send the task job script over, and submit it to run there by the configured batch system. Remote task job logs are saved to the suite run directory on the task remote, -not on the account running the suite. They can be retrieved by right-clicking -on the task in the GUI, or to have cylc pull them back to the suite account +not on the account running the suite. They can be retrieved by +``cylc cat-log``, or to have cylc pull them back to the suite account automatically do this: .. code-block:: cylc @@ -1331,8 +1275,8 @@ If a task with configured retries fails, it goes into the *retrying* state until the next retry delay is up, then it resubmits. It only enters the *failed* state on a final definitive failure. -If a task with configured retries is *killed* (by ``cylc kill`` or -via the GUI) it goes to the *held* state so that the operator can decide +If a task with configured retries is *killed* +it goes to the *held* state so that the operator can decide whether to release it and continue the retry sequence or to abort the retry sequence by manually resetting it to the *failed* state. @@ -1384,9 +1328,6 @@ and can be copied and modified at will. then modify the suite configuration and ``cylc reload`` it before using ``cylc release`` to continue (you can also reload without holding). -- Use the gcylc View menu to show the task state color key and - watch tasks in the ``task-states`` example evolve - as the suite runs. - Manually re-run a task that has already completed or failed, with ``cylc trigger``. - Use an *internal queue* to prevent more than an alotted number diff --git a/etc/cylc-bash-completion b/etc/cylc-bash-completion index 54753a9c90f..dca886f5072 100644 --- a/etc/cylc-bash-completion +++ b/etc/cylc-bash-completion @@ -38,7 +38,7 @@ _cylc() { cur="${COMP_WORDS[COMP_CWORD]}" sec="${COMP_WORDS[1]}" opts="$(cylc print -x -y 2>/dev/null)" - suite_cmds="broadcast|bcast|cat-log|log|cat-state|check-versions|checkpoint|diff|compare|documentation|browse|dump|edit|ext-trigger|external-trigger|get-directory|get-suite-config|get-config|get-suite-version|get-cylc-version|graph|graph-diff|gui|hold|import-examples|insert|jobscript|kill|list|ls|ls-checkpoints|monitor|nudge|ping|poll|print|profile-battery|register|release|unhold|reload|remove|report-timings|reset|restart|run|start|scan|scp-transfer|search|grep|set-verbosity|show|spawn|stop|shutdown|submit|single|suite-state|test-battery|trigger|validate|view|warrenty" + suite_cmds="broadcast|bcast|cat-log|log|cat-state|check-versions|checkpoint|diff|compare|documentation|browse|dump|edit|ext-trigger|external-trigger|get-directory|get-suite-config|get-config|get-suite-version|get-cylc-version|graph|graph-diff|hold|import-examples|insert|jobscript|kill|list|ls|ls-checkpoints|monitor|nudge|ping|poll|print|profile-battery|register|release|unhold|reload|remove|report-timings|reset|restart|run|start|scan|scp-transfer|search|grep|set-verbosity|show|spawn|stop|shutdown|submit|single|suite-state|test-battery|trigger|validate|view|warrenty" if [[ ${COMP_CWORD} -eq 1 ]]; then diff --git a/etc/cylc-compact.server b/etc/cylc-compact.server deleted file mode 100644 index 0b5b6a330a5..00000000000 --- a/etc/cylc-compact.server +++ /dev/null @@ -1,26 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/etc/cylc.server b/etc/cylc.server deleted file mode 100644 index 2d943413299..00000000000 --- a/etc/cylc.server +++ /dev/null @@ -1,26 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/etc/dev-bin/README.group-demo b/etc/dev-bin/README.group-demo index e663f3741a7..59b971776cc 100644 --- a/etc/dev-bin/README.group-demo +++ b/etc/dev-bin/README.group-demo @@ -1,2 +1 @@ Start and stop a bunch of grouped suites, created on the fly. Good for testing -gscan... diff --git a/etc/dev-bin/defn-order-test.py b/etc/dev-bin/defn-order-test.py index 5556c00325e..e7e67abb293 100755 --- a/etc/dev-bin/defn-order-test.py +++ b/etc/dev-bin/defn-order-test.py @@ -1,6 +1,8 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 -import time, string, random +import time +from string import ascii_letters +import random from copy import deepcopy # This is a standalone performance test of the algorithm used in gcylc to @@ -8,39 +10,40 @@ # defined in the suite.rc file. # Number of namespaces. -N=10000 +N = 10000 # N names of length 5-15 characters each (c.f. namespaces in "definition # order"). names = [] -for i in range(0,N): - names.append( ''.join( random.choice(string.ascii_letters) for n in xrange( 5+random.randrange(10) ))) +for i in range(0, N): + names.append(''.join( + random.choice(ascii_letters) + for n in range(5 + random.randrange(10)))) # N lists with 2-7 names each (c.f. tree view paths of the inheritance # hierarchy). paths1 = [] -for i in range(0,N): +for i in range(0, N): p = [] - for j in range(0, 2+random.randrange(6)): - z = random.randrange(0,N) - p.append( names[z] ) + for j in range(0, 2 + random.randrange(6)): + z = random.randrange(0, N) + p.append(names[z]) paths1.append(p) -paths2 = deepcopy( paths1 ) +paths2 = deepcopy(paths1) # Alphanumeric sort. s = time.time() paths1.sort() t1 = time.time() - s -dict_names = dict( zip( names, range(0,len(names)))) +dict_names = dict(zip(names, range(0, len(names)))) # Definition order sort. s = time.time() -paths2.sort( key=lambda x: map( dict_names.get, x ) ) +paths2.sort(key=lambda x: map(dict_names.get, x)) t2 = time.time() - s -print "Alphanumeric sort:", t1, "sec" -print "Definition sort:", t2, "sec" -print " => factor of", t2/t1 - +print("Alphanumeric sort:", t1, "sec") +print("Definition sort:", t2, "sec") +print(" => factor of", t2 / t1) diff --git a/etc/dev-bin/live-graph-movie.sh b/etc/dev-bin/live-graph-movie.sh index 0f32102e052..ee655124b74 100755 --- a/etc/dev-bin/live-graph-movie.sh +++ b/etc/dev-bin/live-graph-movie.sh @@ -8,8 +8,8 @@ usage() { USAGE: $0 Convert frame-*.dot files in the suite share directory (as generated by the -gcylc graph view if the corresponding menu option is selecte) to a frames.webm -movie, which can be played natively in the Firefox browser. +old gcylc graph view if the corresponding menu option is selecte) to a +frames.webm movie, which can be played natively in the Firefox browser. Safe to re-run with different frame size and rate. EOF diff --git a/etc/dev-suites/xtrigger/kafka/consumer/lib/python/cylc_kafka_consumer.py b/etc/dev-suites/xtrigger/kafka/consumer/lib/python/cylc_kafka_consumer.py index d8a3a3df1a8..93520c5910f 100755 --- a/etc/dev-suites/xtrigger/kafka/consumer/lib/python/cylc_kafka_consumer.py +++ b/etc/dev-suites/xtrigger/kafka/consumer/lib/python/cylc_kafka_consumer.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 """ A generic Kakfa consumer for use as a Cylc external trigger function. @@ -55,7 +55,7 @@ def _match_msg(cylc_msg, kafka_msg): def cylc_kafka_consumer(kafka_server, kafka_topic, group_id, message, debug): - """Look for a matching message in a Kafka topic. + r"""Look for a matching message in a Kafka topic. ARGUMENTS: * kafka_server - Kafka server URL, e.g. "localhost:9092". @@ -89,9 +89,10 @@ def cylc_kafka_consumer(kafka_server, kafka_topic, group_id, message, debug): should be represented as: "system:prod point:2025 data:" - A match occurs Kafka if all message dict items match, and the result returned - is the sub-dict of the actual values of items containing - angle-bracket-delineated regex patterns. E.g. above {'data': 'nwp-2025.nc'}. + A match occurs Kafka if all message dict items match, and the result + returned is the sub-dict of the actual values of items containing + angle-bracket-delineated regex patterns. + E.g. above {'data': 'nwp-2025.nc'}. """ diff --git a/etc/dev-suites/xtrigger/kafka/producer/bin/cylc_kafka_producer.py b/etc/dev-suites/xtrigger/kafka/producer/bin/cylc_kafka_producer.py index 6d2f7168204..a11f851bf0f 100755 --- a/etc/dev-suites/xtrigger/kafka/producer/bin/cylc_kafka_producer.py +++ b/etc/dev-suites/xtrigger/kafka/producer/bin/cylc_kafka_producer.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 """ A generic Kafka producer for use as a Cylc event handler. @@ -30,14 +30,14 @@ def main(): """ if 'help' in sys.argv[1]: - print cleandoc(main.__doc__) + print(cleandoc(main.__doc__)) sys.exit(0) # TODO exception handling for bad inputs etc. kafka_server = sys.argv[1] kafka_topic = sys.argv[2] # Construct a message dict from kwargs. - dmsg = dict([k.split('=') for k in sys.argv[3:]]) + dmsg = {k.split('=') for k in sys.argv[3:]} producer = KafkaProducer( bootstrap_servers=kafka_server, diff --git a/etc/examples/jinja2/filter/Jinja2Filters/pad.py b/etc/examples/jinja2/filter/Jinja2Filters/pad.py index bd689d87282..d89dfc01db5 100644 --- a/etc/examples/jinja2/filter/Jinja2Filters/pad.py +++ b/etc/examples/jinja2/filter/Jinja2Filters/pad.py @@ -1,5 +1,5 @@ -#!/usr/bin/env python2 -def pad( value, length, fillchar ): +#!/usr/bin/env python3 +def pad(value, length, fillchar): """A Jinja2 custom filter. Pad a string to some length with a fill character""" - return str(value).rjust(int(length),str(fillchar)) + return str(value).rjust(int(length), str(fillchar)) diff --git a/etc/examples/queues/suite.rc b/etc/examples/queues/suite.rc index c93aa62a2a5..d3341ebee55 100644 --- a/etc/examples/queues/suite.rc +++ b/etc/examples/queues/suite.rc @@ -1,10 +1,10 @@ [meta] title = demonstrates internal queueing description = """ -Two trees of tasks: the first uses the default queue set to a limit of -two active tasks at once; the second uses another queue limited to three -active tasks at once. Run via the graph control GUI for a clear view. - """ + Two trees of tasks: the first uses the default queue set to a limit of + two active tasks at once; the second uses another queue limited to three + active tasks at once. + """ [scheduling] [[queues]] [[[default]]] diff --git a/etc/gcylc-themes.rc b/etc/gcylc-themes.rc deleted file mode 100644 index 575968b21ae..00000000000 --- a/etc/gcylc-themes.rc +++ /dev/null @@ -1,79 +0,0 @@ -# The following themes come with cylc. -# Define custom themes in $HOME/.cylc/gcylc.rc. - -[themes] - [[default]] - # Default theme rationale: - # * States representing something that happened external to cylc - # are solid; those representing a task proxy states purely - # internal to cylc are hollow. Thus pre-submitted states, and - # the retry state, are hollow. - # * Normal states are pleasant shades of grey, blue, or green. - # * Failure states are bright red ('submit-failed' is hollow). - # * The 'running' state is solid green. The 'queued', - # 'ready', and 'submitted' states are different - # shades of yellow-to-green to indicate "almost running". - # * The operator-held state is pink, to indicate an abnormal but - # non-threatening state. - # * The 'succeeded' state is uninteresting, hence grey. - # * The 'retry' states are hollow (pre-running) orange (almost red) - # as a failure has occurred but we're doing something about it. - # * The 'waiting' state is light blue. - defaults = "color=#bbbbbb", "style=unfilled" - waiting = "color=#88c6ff" - held = "color=#fe83ff" - queued = "color=#dcd901" - ready = "color=#a08f49" - expired = "color=#000000" - submitted = "color=#a1cf25", "style=filled", "fontcolor=black" - submit-failed = "color=#ff007e", "style=filled", "fontcolor=white" - running = "color=#00c140", "style=filled", "fontcolor=black" - succeeded = "color=#ada5a5", "style=filled", "fontcolor=black" - failed = "color=#ff0000", "style=filled", "fontcolor=white" - retrying = "color=#ff7e00" - submit-retrying = "color=#fa7e00" - runahead = "color=#b35bfd" - - [[solid]] - # a solid version of the default theme - inherit = "default" - defaults = "color=#bbbbbb", "style=filled", "fontcolor=black" - expired = "color=#000000", "fontcolor=white" - - [[high-contrast]] - inherit = "default" - defaults = "color=#bbbbbb", "style=filled", "fontcolor=white" - waiting = "color=#008ed3" - held = "color=#fc00ff" - queued = "color=#e6c522", "fontcolor=black" - ready = "color=#9c8320" - expired = "color=#000000", "fontcolor=white" - submitted = "color=#76a200", "fontcolor=black" - submit-failed = "color=#ff6f00" - running = "color=#008e27" - succeeded = "color=#555454" - failed = "color=#ff0035" - retrying = "color=#ff6282" - submit-retrying = "color=#ffaf6f" - runahead = "color=#8000c0" - - [[color-blind]] - # Ten colors that are apparently distinguishable by people with - # most forms of color-blindness: - # http://safecolors.rigdenage.com/colourchoice.html - # Can also use style=filled/unfilled to help discrimate states. - inherit = "default" - defaults = "color=#bbbbbb", "style=filled", "fontcolor=black" - waiting = "color=#00ccff" # blue - queued = "color=#336600" # moss green - ready = "color=#d2800a", "style=unfilled" - expired = "color=#000000", "fontcolor=white" - submitted = "color=#ff9933" # orange - submit-failed = "color=#ff3333", "style=unfilled" - running = "color=#66ff33" # lime green - succeeded = "color=#cccccc" # grey - failed = "color=#ff3333" # red - held = "color=#9900cc" # purple - retrying = "color=#009999" # turquoise - submit-retrying = "color=#007777" # turquoise2 - runahead = "color=#000000" diff --git a/etc/gcylc.rc.eg b/etc/gcylc.rc.eg deleted file mode 100644 index a9fda76db47..00000000000 --- a/etc/gcylc.rc.eg +++ /dev/null @@ -1,21 +0,0 @@ -#------------------------------------------------------------------------------ -# An example gcylc.rc config file for "cylc gui" aka "gcylc". -#------------------------------------------------------------------------------ -# To customize your gcylc task state color themes: -# 1) copy this file to ~/.cylc/gcylc.rc -# 2) add new named themes to it by -# (a) defining all colors yourself (see /etc/gcylc-themes.rc) -# or -# (b) inheriting from an existing theme and overriding specific -# colors (see the "PinkRun" example below). -#------------------------------------------------------------------------------ - -initial views = text, dot # set your default views -ungrouped views = graph # set your initially ungrouped views -use theme = PinkRun # set your default theme -initial side-by-side views = False # set your default orientation - -[themes] - [[PinkRun]] # override the 'running' color in the 'default' theme - inherit = default - running = "color=#ff00ff" # magenta diff --git a/lib/Jinja2Filters/duration_as.py b/lib/Jinja2Filters/duration_as.py index dd9228e73c6..2b71dec7194 100644 --- a/lib/Jinja2Filters/duration_as.py +++ b/lib/Jinja2Filters/duration_as.py @@ -57,11 +57,10 @@ def duration_as(iso8601_duration, units): 3600.0 >>> # Exceptions. - >>> try: - ... duration_as('invalid', 's') # Invalid duration - ... except Exception as exc: - ... print type(exc) - + >>> duration_as('invalid', 's') # doctest: +NORMALIZE_WHITESPACE + Traceback (most recent call last): + isodatetime.parsers.ISO8601SyntaxError: Invalid ISO 8601 duration \ + representation: invalid """ for converter_names in CONVERSIONS: if units.lower() in converter_names: @@ -75,5 +74,5 @@ def duration_as(iso8601_duration, units): if __name__ == "__main__": for duration in ['PT1H', 'P1D', 'P7D']: for short_name, _ in CONVERSIONS: - print short_name, duration_as(duration, short_name) - print '\n' + print(short_name, duration_as(duration, short_name)) + print('\n') diff --git a/lib/Jinja2Filters/strftime.py b/lib/Jinja2Filters/strftime.py index c105aa8f7e3..029b517f8b7 100644 --- a/lib/Jinja2Filters/strftime.py +++ b/lib/Jinja2Filters/strftime.py @@ -56,21 +56,20 @@ def strftime(iso8601_datetime, strftime_str, strptime_str=None): '10661014T08' >>> # Exceptions. - >>> try: - ... strftime('invalid', '%H') # Invalid datetime. - ... except Exception as exc: - ... print type(exc) + >>> strftime('invalid', '%H') # doctest: +NORMALIZE_WHITESPACE + Traceback (most recent call last): - >>> try: - ... strftime('2000', '%invalid') # Invalid strftime. - ... except Exception as exc: - ... print type(exc) - - >>> try: - ... strftime('2000', '%Y', '%invalid') # Invalid strptime. - ... except Exception as exc: - ... print type(exc) - + isodatetime.parsers.ISO8601SyntaxError: Invalid ISO 8601 date \ + representation: invalid + >>> strftime('2000', '%invalid') # doctest: +NORMALIZE_WHITESPACE + Traceback (most recent call last): + isodatetime.parser_spec.StrftimeSyntaxError: Invalid \ + strftime/strptime representation: %i + >>> strftime('2000', '%Y', '%invalid') + ... # doctest: +NORMALIZE_WHITESPACE + Traceback (most recent call last): + isodatetime.parser_spec.StrftimeSyntaxError: Invalid \ + strftime/strptime representation: %i """ if not strptime_str: return TimePointParser().parse(iso8601_datetime).strftime(strftime_str) diff --git a/lib/cherrypy/LICENSE.txt b/lib/cherrypy/LICENSE.txt deleted file mode 100644 index 34d1505f466..00000000000 --- a/lib/cherrypy/LICENSE.txt +++ /dev/null @@ -1,25 +0,0 @@ -Copyright (c) 2004-2016, CherryPy Team (team@cherrypy.org) -All rights reserved. - -Redistribution and use in source and binary forms, with or without modification, -are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - * Neither the name of the CherryPy Team nor the names of its contributors - may be used to endorse or promote products derived from this software - without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE -FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/lib/cherrypy/__init__.py b/lib/cherrypy/__init__.py deleted file mode 100644 index 3b1a617e7bf..00000000000 --- a/lib/cherrypy/__init__.py +++ /dev/null @@ -1,380 +0,0 @@ -"""CherryPy is a pythonic, object-oriented HTTP framework. - - -CherryPy consists of not one, but four separate API layers. - -The APPLICATION LAYER is the simplest. CherryPy applications are written as -a tree of classes and methods, where each branch in the tree corresponds to -a branch in the URL path. Each method is a 'page handler', which receives -GET and POST params as keyword arguments, and returns or yields the (HTML) -body of the response. The special method name 'index' is used for paths -that end in a slash, and the special method name 'default' is used to -handle multiple paths via a single handler. This layer also includes: - - * the 'exposed' attribute (and cherrypy.expose) - * cherrypy.quickstart() - * _cp_config attributes - * cherrypy.tools (including cherrypy.session) - * cherrypy.url() - -The ENVIRONMENT LAYER is used by developers at all levels. It provides -information about the current request and response, plus the application -and server environment, via a (default) set of top-level objects: - - * cherrypy.request - * cherrypy.response - * cherrypy.engine - * cherrypy.server - * cherrypy.tree - * cherrypy.config - * cherrypy.thread_data - * cherrypy.log - * cherrypy.HTTPError, NotFound, and HTTPRedirect - * cherrypy.lib - -The EXTENSION LAYER allows advanced users to construct and share their own -plugins. It consists of: - - * Hook API - * Tool API - * Toolbox API - * Dispatch API - * Config Namespace API - -Finally, there is the CORE LAYER, which uses the core API's to construct -the default components which are available at higher layers. You can think -of the default components as the 'reference implementation' for CherryPy. -Megaframeworks (and advanced users) may replace the default components -with customized or extended components. The core API's are: - - * Application API - * Engine API - * Request API - * Server API - * WSGI API - -These API's are described in the `CherryPy specification `_. -""" - -import warnings - -warnings.filterwarnings('ignore', 'Module cherrypy was already imported') -warnings.filterwarnings('ignore', 'Module jinja2 was already imported') -warnings.filterwarnings('ignore', 'Module markupsafe was already imported') -try: - import pkg_resources -except ImportError: - pass - -from cherrypy._cperror import HTTPError, HTTPRedirect, InternalRedirect -from cherrypy._cperror import NotFound, CherryPyException, TimeoutError - -from cherrypy import _cpdispatch as dispatch - -from cherrypy import _cptools -tools = _cptools.default_toolbox -Tool = _cptools.Tool - -from cherrypy import _cprequest -from cherrypy.lib import httputil as _httputil - -from cherrypy import _cptree -tree = _cptree.Tree() -from cherrypy._cptree import Application -from cherrypy import _cpwsgi as wsgi - -from cherrypy import process -try: - from cherrypy.process import win32 - engine = win32.Win32Bus() - engine.console_control_handler = win32.ConsoleCtrlHandler(engine) - del win32 -except ImportError: - engine = process.bus - - -try: - __version__ = pkg_resources.require('cherrypy')[0].version -except Exception: - __version__ = 'unknown' - - -# Timeout monitor. We add two channels to the engine -# to which cherrypy.Application will publish. -engine.listeners['before_request'] = set() -engine.listeners['after_request'] = set() - - -class _TimeoutMonitor(process.plugins.Monitor): - - def __init__(self, bus): - self.servings = [] - process.plugins.Monitor.__init__(self, bus, self.run) - - def before_request(self): - self.servings.append((serving.request, serving.response)) - - def after_request(self): - try: - self.servings.remove((serving.request, serving.response)) - except ValueError: - pass - - def run(self): - """Check timeout on all responses. (Internal)""" - for req, resp in self.servings: - resp.check_timeout() -engine.timeout_monitor = _TimeoutMonitor(engine) -engine.timeout_monitor.subscribe() - -engine.autoreload = process.plugins.Autoreloader(engine) -engine.autoreload.subscribe() - -engine.thread_manager = process.plugins.ThreadManager(engine) -engine.thread_manager.subscribe() - -engine.signal_handler = process.plugins.SignalHandler(engine) - - -class _HandleSignalsPlugin(object): - - """Handle signals from other processes based on the configured - platform handlers above.""" - - def __init__(self, bus): - self.bus = bus - - def subscribe(self): - """Add the handlers based on the platform""" - if hasattr(self.bus, "signal_handler"): - self.bus.signal_handler.subscribe() - if hasattr(self.bus, "console_control_handler"): - self.bus.console_control_handler.subscribe() - -engine.signals = _HandleSignalsPlugin(engine) - - -from cherrypy import _cpserver -server = _cpserver.Server() -server.subscribe() - - -def quickstart(root=None, script_name="", config=None): - """Mount the given root, start the builtin server (and engine), then block. - - root: an instance of a "controller class" (a collection of page handler - methods) which represents the root of the application. - script_name: a string containing the "mount point" of the application. - This should start with a slash, and be the path portion of the URL - at which to mount the given root. For example, if root.index() will - handle requests to "http://www.example.com:8080/dept/app1/", then - the script_name argument would be "/dept/app1". - - It MUST NOT end in a slash. If the script_name refers to the root - of the URI, it MUST be an empty string (not "/"). - config: a file or dict containing application config. If this contains - a [global] section, those entries will be used in the global - (site-wide) config. - """ - if config: - _global_conf_alias.update(config) - - tree.mount(root, script_name, config) - - engine.signals.subscribe() - engine.start() - engine.block() - - -from cherrypy._cpcompat import threadlocal as _local - - -class _Serving(_local): - - """An interface for registering request and response objects. - - Rather than have a separate "thread local" object for the request and - the response, this class works as a single threadlocal container for - both objects (and any others which developers wish to define). In this - way, we can easily dump those objects when we stop/start a new HTTP - conversation, yet still refer to them as module-level globals in a - thread-safe way. - """ - - request = _cprequest.Request(_httputil.Host("127.0.0.1", 80), - _httputil.Host("127.0.0.1", 1111)) - """ - The request object for the current thread. In the main thread, - and any threads which are not receiving HTTP requests, this is None.""" - - response = _cprequest.Response() - """ - The response object for the current thread. In the main thread, - and any threads which are not receiving HTTP requests, this is None.""" - - def load(self, request, response): - self.request = request - self.response = response - - def clear(self): - """Remove all attributes of self.""" - self.__dict__.clear() - -serving = _Serving() - - -class _ThreadLocalProxy(object): - - __slots__ = ['__attrname__', '__dict__'] - - def __init__(self, attrname): - self.__attrname__ = attrname - - def __getattr__(self, name): - child = getattr(serving, self.__attrname__) - return getattr(child, name) - - def __setattr__(self, name, value): - if name in ("__attrname__", ): - object.__setattr__(self, name, value) - else: - child = getattr(serving, self.__attrname__) - setattr(child, name, value) - - def __delattr__(self, name): - child = getattr(serving, self.__attrname__) - delattr(child, name) - - def _get_dict(self): - child = getattr(serving, self.__attrname__) - d = child.__class__.__dict__.copy() - d.update(child.__dict__) - return d - __dict__ = property(_get_dict) - - def __getitem__(self, key): - child = getattr(serving, self.__attrname__) - return child[key] - - def __setitem__(self, key, value): - child = getattr(serving, self.__attrname__) - child[key] = value - - def __delitem__(self, key): - child = getattr(serving, self.__attrname__) - del child[key] - - def __contains__(self, key): - child = getattr(serving, self.__attrname__) - return key in child - - def __len__(self): - child = getattr(serving, self.__attrname__) - return len(child) - - def __nonzero__(self): - child = getattr(serving, self.__attrname__) - return bool(child) - # Python 3 - __bool__ = __nonzero__ - -# Create request and response object (the same objects will be used -# throughout the entire life of the webserver, but will redirect -# to the "serving" object) -request = _ThreadLocalProxy('request') -response = _ThreadLocalProxy('response') - -# Create thread_data object as a thread-specific all-purpose storage - - -class _ThreadData(_local): - - """A container for thread-specific data.""" -thread_data = _ThreadData() - - -# Monkeypatch pydoc to allow help() to go through the threadlocal proxy. -# Jan 2007: no Googleable examples of anyone else replacing pydoc.resolve. -# The only other way would be to change what is returned from type(request) -# and that's not possible in pure Python (you'd have to fake ob_type). -def _cherrypy_pydoc_resolve(thing, forceload=0): - """Given an object or a path to an object, get the object and its name.""" - if isinstance(thing, _ThreadLocalProxy): - thing = getattr(serving, thing.__attrname__) - return _pydoc._builtin_resolve(thing, forceload) - -try: - import pydoc as _pydoc - _pydoc._builtin_resolve = _pydoc.resolve - _pydoc.resolve = _cherrypy_pydoc_resolve -except ImportError: - pass - - -from cherrypy import _cplogging - - -class _GlobalLogManager(_cplogging.LogManager): - - """A site-wide LogManager; routes to app.log or global log as appropriate. - - This :class:`LogManager` implements - cherrypy.log() and cherrypy.log.access(). If either - function is called during a request, the message will be sent to the - logger for the current Application. If they are called outside of a - request, the message will be sent to the site-wide logger. - """ - - def __call__(self, *args, **kwargs): - """Log the given message to the app.log or global log as appropriate. - """ - # Do NOT use try/except here. See - # https://github.com/cherrypy/cherrypy/issues/945 - if hasattr(request, 'app') and hasattr(request.app, 'log'): - log = request.app.log - else: - log = self - return log.error(*args, **kwargs) - - def access(self): - """Log an access message to the app.log or global log as appropriate. - """ - try: - return request.app.log.access() - except AttributeError: - return _cplogging.LogManager.access(self) - - -log = _GlobalLogManager() -# Set a default screen handler on the global log. -log.screen = True -log.error_file = '' -# Using an access file makes CP about 10% slower. Leave off by default. -log.access_file = '' - - -def _buslog(msg, level): - log.error(msg, 'ENGINE', severity=level) -engine.subscribe('log', _buslog) - -from cherrypy._helper import expose, popargs, url - -# import _cpconfig last so it can reference other top-level objects -from cherrypy import _cpconfig -# Use _global_conf_alias so quickstart can use 'config' as an arg -# without shadowing cherrypy.config. -config = _global_conf_alias = _cpconfig.Config() -config.defaults = { - 'tools.log_tracebacks.on': True, - 'tools.log_headers.on': True, - 'tools.trailing_slash.on': True, - 'tools.encode.on': True -} -config.namespaces["log"] = lambda k, v: setattr(log, k, v) -config.namespaces["checker"] = lambda k, v: setattr(checker, k, v) -# Must reset to get our defaults applied. -config.reset() - -from cherrypy import _cpchecker -checker = _cpchecker.Checker() -engine.subscribe('start', checker) diff --git a/lib/cherrypy/__main__.py b/lib/cherrypy/__main__.py deleted file mode 100755 index b1c9c012f6a..00000000000 --- a/lib/cherrypy/__main__.py +++ /dev/null @@ -1,4 +0,0 @@ -import cherrypy.daemon - -if __name__ == '__main__': - cherrypy.daemon.run() diff --git a/lib/cherrypy/_cpchecker.py b/lib/cherrypy/_cpchecker.py deleted file mode 100644 index 4ef82597141..00000000000 --- a/lib/cherrypy/_cpchecker.py +++ /dev/null @@ -1,332 +0,0 @@ -import os -import warnings - -import cherrypy -from cherrypy._cpcompat import iteritems, copykeys, builtins - - -class Checker(object): - - """A checker for CherryPy sites and their mounted applications. - - When this object is called at engine startup, it executes each - of its own methods whose names start with ``check_``. If you wish - to disable selected checks, simply add a line in your global - config which sets the appropriate method to False:: - - [global] - checker.check_skipped_app_config = False - - You may also dynamically add or replace ``check_*`` methods in this way. - """ - - on = True - """If True (the default), run all checks; if False, turn off all checks.""" - - def __init__(self): - self._populate_known_types() - - def __call__(self): - """Run all check_* methods.""" - if self.on: - oldformatwarning = warnings.formatwarning - warnings.formatwarning = self.formatwarning - try: - for name in dir(self): - if name.startswith("check_"): - method = getattr(self, name) - if method and hasattr(method, '__call__'): - method() - finally: - warnings.formatwarning = oldformatwarning - - def formatwarning(self, message, category, filename, lineno, line=None): - """Function to format a warning.""" - return "CherryPy Checker:\n%s\n\n" % message - - # This value should be set inside _cpconfig. - global_config_contained_paths = False - - def check_app_config_entries_dont_start_with_script_name(self): - """Check for Application config with sections that repeat script_name. - """ - for sn, app in cherrypy.tree.apps.items(): - if not isinstance(app, cherrypy.Application): - continue - if not app.config: - continue - if sn == '': - continue - sn_atoms = sn.strip("/").split("/") - for key in app.config.keys(): - key_atoms = key.strip("/").split("/") - if key_atoms[:len(sn_atoms)] == sn_atoms: - warnings.warn( - "The application mounted at %r has config " - "entries that start with its script name: %r" % (sn, - key)) - - def check_site_config_entries_in_app_config(self): - """Check for mounted Applications that have site-scoped config.""" - for sn, app in iteritems(cherrypy.tree.apps): - if not isinstance(app, cherrypy.Application): - continue - - msg = [] - for section, entries in iteritems(app.config): - if section.startswith('/'): - for key, value in iteritems(entries): - for n in ("engine.", "server.", "tree.", "checker."): - if key.startswith(n): - msg.append("[%s] %s = %s" % - (section, key, value)) - if msg: - msg.insert(0, - "The application mounted at %r contains the " - "following config entries, which are only allowed " - "in site-wide config. Move them to a [global] " - "section and pass them to cherrypy.config.update() " - "instead of tree.mount()." % sn) - warnings.warn(os.linesep.join(msg)) - - def check_skipped_app_config(self): - """Check for mounted Applications that have no config.""" - for sn, app in cherrypy.tree.apps.items(): - if not isinstance(app, cherrypy.Application): - continue - if not app.config: - msg = "The Application mounted at %r has an empty config." % sn - if self.global_config_contained_paths: - msg += (" It looks like the config you passed to " - "cherrypy.config.update() contains application-" - "specific sections. You must explicitly pass " - "application config via " - "cherrypy.tree.mount(..., config=app_config)") - warnings.warn(msg) - return - - def check_app_config_brackets(self): - """Check for Application config with extraneous brackets in section - names. - """ - for sn, app in cherrypy.tree.apps.items(): - if not isinstance(app, cherrypy.Application): - continue - if not app.config: - continue - for key in app.config.keys(): - if key.startswith("[") or key.endswith("]"): - warnings.warn( - "The application mounted at %r has config " - "section names with extraneous brackets: %r. " - "Config *files* need brackets; config *dicts* " - "(e.g. passed to tree.mount) do not." % (sn, key)) - - def check_static_paths(self): - """Check Application config for incorrect static paths.""" - # Use the dummy Request object in the main thread. - request = cherrypy.request - for sn, app in cherrypy.tree.apps.items(): - if not isinstance(app, cherrypy.Application): - continue - request.app = app - for section in app.config: - # get_resource will populate request.config - request.get_resource(section + "/dummy.html") - conf = request.config.get - - if conf("tools.staticdir.on", False): - msg = "" - root = conf("tools.staticdir.root") - dir = conf("tools.staticdir.dir") - if dir is None: - msg = "tools.staticdir.dir is not set." - else: - fulldir = "" - if os.path.isabs(dir): - fulldir = dir - if root: - msg = ("dir is an absolute path, even " - "though a root is provided.") - testdir = os.path.join(root, dir[1:]) - if os.path.exists(testdir): - msg += ( - "\nIf you meant to serve the " - "filesystem folder at %r, remove the " - "leading slash from dir." % (testdir,)) - else: - if not root: - msg = ( - "dir is a relative path and " - "no root provided.") - else: - fulldir = os.path.join(root, dir) - if not os.path.isabs(fulldir): - msg = ("%r is not an absolute path." % ( - fulldir,)) - - if fulldir and not os.path.exists(fulldir): - if msg: - msg += "\n" - msg += ("%r (root + dir) is not an existing " - "filesystem path." % fulldir) - - if msg: - warnings.warn("%s\nsection: [%s]\nroot: %r\ndir: %r" - % (msg, section, root, dir)) - - # -------------------------- Compatibility -------------------------- # - obsolete = { - 'server.default_content_type': 'tools.response_headers.headers', - 'log_access_file': 'log.access_file', - 'log_config_options': None, - 'log_file': 'log.error_file', - 'log_file_not_found': None, - 'log_request_headers': 'tools.log_headers.on', - 'log_to_screen': 'log.screen', - 'show_tracebacks': 'request.show_tracebacks', - 'throw_errors': 'request.throw_errors', - 'profiler.on': ('cherrypy.tree.mount(profiler.make_app(' - 'cherrypy.Application(Root())))'), - } - - deprecated = {} - - def _compat(self, config): - """Process config and warn on each obsolete or deprecated entry.""" - for section, conf in config.items(): - if isinstance(conf, dict): - for k, v in conf.items(): - if k in self.obsolete: - warnings.warn("%r is obsolete. Use %r instead.\n" - "section: [%s]" % - (k, self.obsolete[k], section)) - elif k in self.deprecated: - warnings.warn("%r is deprecated. Use %r instead.\n" - "section: [%s]" % - (k, self.deprecated[k], section)) - else: - if section in self.obsolete: - warnings.warn("%r is obsolete. Use %r instead." - % (section, self.obsolete[section])) - elif section in self.deprecated: - warnings.warn("%r is deprecated. Use %r instead." - % (section, self.deprecated[section])) - - def check_compatibility(self): - """Process config and warn on each obsolete or deprecated entry.""" - self._compat(cherrypy.config) - for sn, app in cherrypy.tree.apps.items(): - if not isinstance(app, cherrypy.Application): - continue - self._compat(app.config) - - # ------------------------ Known Namespaces ------------------------ # - extra_config_namespaces = [] - - def _known_ns(self, app): - ns = ["wsgi"] - ns.extend(copykeys(app.toolboxes)) - ns.extend(copykeys(app.namespaces)) - ns.extend(copykeys(app.request_class.namespaces)) - ns.extend(copykeys(cherrypy.config.namespaces)) - ns += self.extra_config_namespaces - - for section, conf in app.config.items(): - is_path_section = section.startswith("/") - if is_path_section and isinstance(conf, dict): - for k, v in conf.items(): - atoms = k.split(".") - if len(atoms) > 1: - if atoms[0] not in ns: - # Spit out a special warning if a known - # namespace is preceded by "cherrypy." - if atoms[0] == "cherrypy" and atoms[1] in ns: - msg = ( - "The config entry %r is invalid; " - "try %r instead.\nsection: [%s]" - % (k, ".".join(atoms[1:]), section)) - else: - msg = ( - "The config entry %r is invalid, " - "because the %r config namespace " - "is unknown.\n" - "section: [%s]" % (k, atoms[0], section)) - warnings.warn(msg) - elif atoms[0] == "tools": - if atoms[1] not in dir(cherrypy.tools): - msg = ( - "The config entry %r may be invalid, " - "because the %r tool was not found.\n" - "section: [%s]" % (k, atoms[1], section)) - warnings.warn(msg) - - def check_config_namespaces(self): - """Process config and warn on each unknown config namespace.""" - for sn, app in cherrypy.tree.apps.items(): - if not isinstance(app, cherrypy.Application): - continue - self._known_ns(app) - - # -------------------------- Config Types -------------------------- # - known_config_types = {} - - def _populate_known_types(self): - b = [x for x in vars(builtins).values() - if type(x) is type(str)] - - def traverse(obj, namespace): - for name in dir(obj): - # Hack for 3.2's warning about body_params - if name == 'body_params': - continue - vtype = type(getattr(obj, name, None)) - if vtype in b: - self.known_config_types[namespace + "." + name] = vtype - - traverse(cherrypy.request, "request") - traverse(cherrypy.response, "response") - traverse(cherrypy.server, "server") - traverse(cherrypy.engine, "engine") - traverse(cherrypy.log, "log") - - def _known_types(self, config): - msg = ("The config entry %r in section %r is of type %r, " - "which does not match the expected type %r.") - - for section, conf in config.items(): - if isinstance(conf, dict): - for k, v in conf.items(): - if v is not None: - expected_type = self.known_config_types.get(k, None) - vtype = type(v) - if expected_type and vtype != expected_type: - warnings.warn(msg % (k, section, vtype.__name__, - expected_type.__name__)) - else: - k, v = section, conf - if v is not None: - expected_type = self.known_config_types.get(k, None) - vtype = type(v) - if expected_type and vtype != expected_type: - warnings.warn(msg % (k, section, vtype.__name__, - expected_type.__name__)) - - def check_config_types(self): - """Assert that config values are of the same type as default values.""" - self._known_types(cherrypy.config) - for sn, app in cherrypy.tree.apps.items(): - if not isinstance(app, cherrypy.Application): - continue - self._known_types(app.config) - - # -------------------- Specific config warnings -------------------- # - def check_localhost(self): - """Warn if any socket_host is 'localhost'. See #711.""" - for k, v in cherrypy.config.items(): - if k == 'server.socket_host' and v == 'localhost': - warnings.warn("The use of 'localhost' as a socket host can " - "cause problems on newer systems, since " - "'localhost' can map to either an IPv4 or an " - "IPv6 address. You should use '127.0.0.1' " - "or '[::1]' instead.") diff --git a/lib/cherrypy/_cpcompat.py b/lib/cherrypy/_cpcompat.py deleted file mode 100644 index 35d134ec523..00000000000 --- a/lib/cherrypy/_cpcompat.py +++ /dev/null @@ -1,353 +0,0 @@ -"""Compatibility code for using CherryPy with various versions of Python. - -CherryPy 3.2 is compatible with Python versions 2.3+. This module provides a -useful abstraction over the differences between Python versions, sometimes by -preferring a newer idiom, sometimes an older one, and sometimes a custom one. - -In particular, Python 2 uses str and '' for byte strings, while Python 3 -uses str and '' for unicode strings. We will call each of these the 'native -string' type for each version. Because of this major difference, this module -provides -two functions: 'ntob', which translates native strings (of type 'str') into -byte strings regardless of Python version, and 'ntou', which translates native -strings to unicode strings. This also provides a 'BytesIO' name for dealing -specifically with bytes, and a 'StringIO' name for dealing with native strings. -It also provides a 'base64_decode' function with native strings as input and -output. -""" -import os -import re -import sys -import threading - -IS_PY3 = sys.version_info[0] == 3 - -try: - import six -except ImportError: - if IS_PY3: - _unicode_type = str - else: - _unicode_type = unicode -else: - _unicode_type = six.text_type - - - -if IS_PY3: - basestring = (bytes, str) - - def ntob(n, encoding='ISO-8859-1'): - """Return the given native string as a byte string in the given - encoding. - """ - assert_native(n) - # In Python 3, the native string type is unicode - return n.encode(encoding) - - def ntou(n, encoding='ISO-8859-1'): - """Return the given native string as a unicode string with the given - encoding. - """ - assert_native(n) - # In Python 3, the native string type is unicode - return n - - def tonative(n, encoding='ISO-8859-1'): - """Return the given string as a native string in the given encoding.""" - # In Python 3, the native string type is unicode - if isinstance(n, bytes): - return n.decode(encoding) - return n -else: - # Python 2 - basestring = basestring - - def ntob(n, encoding='ISO-8859-1'): - """Return the given native string as a byte string in the given - encoding. - """ - assert_native(n) - # In Python 2, the native string type is bytes. Assume it's already - # in the given encoding, which for ISO-8859-1 is almost always what - # was intended. - return n - - def ntou(n, encoding='ISO-8859-1'): - """Return the given native string as a unicode string with the given - encoding. - """ - assert_native(n) - # In Python 2, the native string type is bytes. - # First, check for the special encoding 'escape'. The test suite uses - # this to signal that it wants to pass a string with embedded \uXXXX - # escapes, but without having to prefix it with u'' for Python 2, - # but no prefix for Python 3. - if encoding == 'escape': - return unicode( - re.sub(r'\\u([0-9a-zA-Z]{4})', - lambda m: unichr(int(m.group(1), 16)), - n.decode('ISO-8859-1'))) - # Assume it's already in the given encoding, which for ISO-8859-1 - # is almost always what was intended. - return n.decode(encoding) - - def tonative(n, encoding='ISO-8859-1'): - """Return the given string as a native string in the given encoding.""" - # In Python 2, the native string type is bytes. - if isinstance(n, unicode): - return n.encode(encoding) - return n - - -def assert_native(n): - if not isinstance(n, str): - raise TypeError("n must be a native str (got %s)" % type(n).__name__) - -try: - # Python 3.1+ - from base64 import decodebytes as _base64_decodebytes -except ImportError: - # Python 3.0- - # since CherryPy claims compability with Python 2.3, we must use - # the legacy API of base64 - from base64 import decodestring as _base64_decodebytes - - -def base64_decode(n, encoding='ISO-8859-1'): - """Return the native string base64-decoded (as a native string).""" - if isinstance(n, _unicode_type): - b = n.encode(encoding) - else: - b = n - b = _base64_decodebytes(b) - if str is _unicode_type: - return b.decode(encoding) - else: - return b - - -try: - sorted = sorted -except NameError: - def sorted(i): - i = i[:] - i.sort() - return i - -try: - reversed = reversed -except NameError: - def reversed(x): - i = len(x) - while i > 0: - i -= 1 - yield x[i] - -try: - # Python 3 - from urllib.parse import urljoin, urlencode - from urllib.parse import quote, quote_plus - from urllib.request import unquote, urlopen - from urllib.request import parse_http_list, parse_keqv_list -except ImportError: - # Python 2 - from urlparse import urljoin - from urllib import urlencode, urlopen - from urllib import quote, quote_plus - from urllib import unquote - from urllib2 import parse_http_list, parse_keqv_list - -try: - from threading import local as threadlocal -except ImportError: - from cherrypy._cpthreadinglocal import local as threadlocal - -try: - dict.iteritems - # Python 2 - iteritems = lambda d: d.iteritems() - copyitems = lambda d: d.items() -except AttributeError: - # Python 3 - iteritems = lambda d: d.items() - copyitems = lambda d: list(d.items()) - -try: - dict.iterkeys - # Python 2 - iterkeys = lambda d: d.iterkeys() - copykeys = lambda d: d.keys() -except AttributeError: - # Python 3 - iterkeys = lambda d: d.keys() - copykeys = lambda d: list(d.keys()) - -try: - dict.itervalues - # Python 2 - itervalues = lambda d: d.itervalues() - copyvalues = lambda d: d.values() -except AttributeError: - # Python 3 - itervalues = lambda d: d.values() - copyvalues = lambda d: list(d.values()) - -try: - # Python 3 - import builtins -except ImportError: - # Python 2 - import __builtin__ as builtins - -try: - # Python 2. We try Python 2 first clients on Python 2 - # don't try to import the 'http' module from cherrypy.lib - from Cookie import SimpleCookie, CookieError - from httplib import BadStatusLine, HTTPConnection, IncompleteRead - from httplib import NotConnected - from BaseHTTPServer import BaseHTTPRequestHandler -except ImportError: - # Python 3 - from http.cookies import SimpleCookie, CookieError - from http.client import BadStatusLine, HTTPConnection, IncompleteRead - from http.client import NotConnected - from http.server import BaseHTTPRequestHandler - -# Some platforms don't expose HTTPSConnection, so handle it separately -if IS_PY3: - try: - from http.client import HTTPSConnection - except ImportError: - # Some platforms which don't have SSL don't expose HTTPSConnection - HTTPSConnection = None -else: - try: - from httplib import HTTPSConnection - except ImportError: - HTTPSConnection = None - -try: - # Python 2 - xrange = xrange -except NameError: - # Python 3 - xrange = range - -import threading -if hasattr(threading.Thread, "daemon"): - # Python 2.6+ - def get_daemon(t): - return t.daemon - - def set_daemon(t, val): - t.daemon = val -else: - def get_daemon(t): - return t.isDaemon() - - def set_daemon(t, val): - t.setDaemon(val) - -try: - from email.utils import formatdate - - def HTTPDate(timeval=None): - return formatdate(timeval, usegmt=True) -except ImportError: - from rfc822 import formatdate as HTTPDate - -try: - # Python 3 - from urllib.parse import unquote as parse_unquote - - def unquote_qs(atom, encoding, errors='strict'): - return parse_unquote( - atom.replace('+', ' '), - encoding=encoding, - errors=errors) -except ImportError: - # Python 2 - from urllib import unquote as parse_unquote - - def unquote_qs(atom, encoding, errors='strict'): - return parse_unquote(atom.replace('+', ' ')).decode(encoding, errors) - -try: - # Prefer simplejson, which is usually more advanced than the builtin - # module. - import simplejson as json - json_decode = json.JSONDecoder().decode - _json_encode = json.JSONEncoder().iterencode -except ImportError: - if sys.version_info >= (2, 6): - # Python >=2.6 : json is part of the standard library - import json - json_decode = json.JSONDecoder().decode - _json_encode = json.JSONEncoder().iterencode - else: - json = None - - def json_decode(s): - raise ValueError('No JSON library is available') - - def _json_encode(s): - raise ValueError('No JSON library is available') -finally: - if json and IS_PY3: - # The two Python 3 implementations (simplejson/json) - # outputs str. We need bytes. - def json_encode(value): - for chunk in _json_encode(value): - yield chunk.encode('utf8') - else: - json_encode = _json_encode - - -try: - import cPickle as pickle -except ImportError: - # In Python 2, pickle is a Python version. - # In Python 3, pickle is the sped-up C version. - import pickle - -import binascii - -def random20(): - return binascii.hexlify(os.urandom(20)).decode('ascii') - -try: - from _thread import get_ident as get_thread_ident -except ImportError: - from thread import get_ident as get_thread_ident - -try: - # Python 3 - next = next -except NameError: - # Python 2 - def next(i): - return i.next() - -if sys.version_info >= (3, 3): - Timer = threading.Timer - Event = threading.Event -else: - # Python 3.2 and earlier - Timer = threading._Timer - Event = threading._Event - -# Prior to Python 2.6, the Thread class did not have a .daemon property. -# This mix-in adds that property. - - -class SetDaemonProperty: - - def __get_daemon(self): - return self.isDaemon() - - def __set_daemon(self, daemon): - self.setDaemon(daemon) - - if sys.version_info < (2, 6): - daemon = property(__get_daemon, __set_daemon) diff --git a/lib/cherrypy/_cpcompat_subprocess.py b/lib/cherrypy/_cpcompat_subprocess.py deleted file mode 100644 index 517b8d44094..00000000000 --- a/lib/cherrypy/_cpcompat_subprocess.py +++ /dev/null @@ -1,1544 +0,0 @@ -# subprocess - Subprocesses with accessible I/O streams -# -# For more information about this module, see PEP 324. -# -# This module should remain compatible with Python 2.2, see PEP 291. -# -# Copyright (c) 2003-2005 by Peter Astrand -# -# Licensed to PSF under a Contributor Agreement. -# See http://www.python.org/2.4/license for licensing details. - -r"""subprocess - Subprocesses with accessible I/O streams - -This module allows you to spawn processes, connect to their -input/output/error pipes, and obtain their return codes. This module -intends to replace several other, older modules and functions, like: - -os.system -os.spawn* -os.popen* -popen2.* -commands.* - -Information about how the subprocess module can be used to replace these -modules and functions can be found below. - - - -Using the subprocess module -=========================== -This module defines one class called Popen: - -class Popen(args, bufsize=0, executable=None, - stdin=None, stdout=None, stderr=None, - preexec_fn=None, close_fds=False, shell=False, - cwd=None, env=None, universal_newlines=False, - startupinfo=None, creationflags=0): - - -Arguments are: - -args should be a string, or a sequence of program arguments. The -program to execute is normally the first item in the args sequence or -string, but can be explicitly set by using the executable argument. - -On UNIX, with shell=False (default): In this case, the Popen class -uses os.execvp() to execute the child program. args should normally -be a sequence. A string will be treated as a sequence with the string -as the only item (the program to execute). - -On UNIX, with shell=True: If args is a string, it specifies the -command string to execute through the shell. If args is a sequence, -the first item specifies the command string, and any additional items -will be treated as additional shell arguments. - -On Windows: the Popen class uses CreateProcess() to execute the child -program, which operates on strings. If args is a sequence, it will be -converted to a string using the list2cmdline method. Please note that -not all MS Windows applications interpret the command line the same -way: The list2cmdline is designed for applications using the same -rules as the MS C runtime. - -bufsize, if given, has the same meaning as the corresponding argument -to the built-in open() function: 0 means unbuffered, 1 means line -buffered, any other positive value means use a buffer of -(approximately) that size. A negative bufsize means to use the system -default, which usually means fully buffered. The default value for -bufsize is 0 (unbuffered). - -stdin, stdout and stderr specify the executed programs' standard -input, standard output and standard error file handles, respectively. -Valid values are PIPE, an existing file descriptor (a positive -integer), an existing file object, and None. PIPE indicates that a -new pipe to the child should be created. With None, no redirection -will occur; the child's file handles will be inherited from the -parent. Additionally, stderr can be STDOUT, which indicates that the -stderr data from the applications should be captured into the same -file handle as for stdout. - -If preexec_fn is set to a callable object, this object will be called -in the child process just before the child is executed. - -If close_fds is true, all file descriptors except 0, 1 and 2 will be -closed before the child process is executed. - -if shell is true, the specified command will be executed through the -shell. - -If cwd is not None, the current directory will be changed to cwd -before the child is executed. - -If env is not None, it defines the environment variables for the new -process. - -If universal_newlines is true, the file objects stdout and stderr are -opened as a text files, but lines may be terminated by any of '\n', -the Unix end-of-line convention, '\r', the Macintosh convention or -'\r\n', the Windows convention. All of these external representations -are seen as '\n' by the Python program. Note: This feature is only -available if Python is built with universal newline support (the -default). Also, the newlines attribute of the file objects stdout, -stdin and stderr are not updated by the communicate() method. - -The startupinfo and creationflags, if given, will be passed to the -underlying CreateProcess() function. They can specify things such as -appearance of the main window and priority for the new process. -(Windows only) - - -This module also defines some shortcut functions: - -call(*popenargs, **kwargs): - Run command with arguments. Wait for command to complete, then - return the returncode attribute. - - The arguments are the same as for the Popen constructor. Example: - - retcode = call(["ls", "-l"]) - -check_call(*popenargs, **kwargs): - Run command with arguments. Wait for command to complete. If the - exit code was zero then return, otherwise raise - CalledProcessError. The CalledProcessError object will have the - return code in the returncode attribute. - - The arguments are the same as for the Popen constructor. Example: - - check_call(["ls", "-l"]) - -check_output(*popenargs, **kwargs): - Run command with arguments and return its output as a byte string. - - If the exit code was non-zero it raises a CalledProcessError. The - CalledProcessError object will have the return code in the returncode - attribute and output in the output attribute. - - The arguments are the same as for the Popen constructor. Example: - - output = check_output(["ls", "-l", "/dev/null"]) - - -Exceptions ----------- -Exceptions raised in the child process, before the new program has -started to execute, will be re-raised in the parent. Additionally, -the exception object will have one extra attribute called -'child_traceback', which is a string containing traceback information -from the childs point of view. - -The most common exception raised is OSError. This occurs, for -example, when trying to execute a non-existent file. Applications -should prepare for OSErrors. - -A ValueError will be raised if Popen is called with invalid arguments. - -check_call() and check_output() will raise CalledProcessError, if the -called process returns a non-zero return code. - - -Security --------- -Unlike some other popen functions, this implementation will never call -/bin/sh implicitly. This means that all characters, including shell -metacharacters, can safely be passed to child processes. - - -Popen objects -============= -Instances of the Popen class have the following methods: - -poll() - Check if child process has terminated. Returns returncode - attribute. - -wait() - Wait for child process to terminate. Returns returncode attribute. - -communicate(input=None) - Interact with process: Send data to stdin. Read data from stdout - and stderr, until end-of-file is reached. Wait for process to - terminate. The optional input argument should be a string to be - sent to the child process, or None, if no data should be sent to - the child. - - communicate() returns a tuple (stdout, stderr). - - Note: The data read is buffered in memory, so do not use this - method if the data size is large or unlimited. - -The following attributes are also available: - -stdin - If the stdin argument is PIPE, this attribute is a file object - that provides input to the child process. Otherwise, it is None. - -stdout - If the stdout argument is PIPE, this attribute is a file object - that provides output from the child process. Otherwise, it is - None. - -stderr - If the stderr argument is PIPE, this attribute is file object that - provides error output from the child process. Otherwise, it is - None. - -pid - The process ID of the child process. - -returncode - The child return code. A None value indicates that the process - hasn't terminated yet. A negative value -N indicates that the - child was terminated by signal N (UNIX only). - - -Replacing older functions with the subprocess module -==================================================== -In this section, "a ==> b" means that b can be used as a replacement -for a. - -Note: All functions in this section fail (more or less) silently if -the executed program cannot be found; this module raises an OSError -exception. - -In the following examples, we assume that the subprocess module is -imported with "from subprocess import *". - - -Replacing /bin/sh shell backquote ---------------------------------- -output=`mycmd myarg` -==> -output = Popen(["mycmd", "myarg"], stdout=PIPE).communicate()[0] - - -Replacing shell pipe line -------------------------- -output=`dmesg | grep hda` -==> -p1 = Popen(["dmesg"], stdout=PIPE) -p2 = Popen(["grep", "hda"], stdin=p1.stdout, stdout=PIPE) -output = p2.communicate()[0] - - -Replacing os.system() ---------------------- -sts = os.system("mycmd" + " myarg") -==> -p = Popen("mycmd" + " myarg", shell=True) -pid, sts = os.waitpid(p.pid, 0) - -Note: - -* Calling the program through the shell is usually not required. - -* It's easier to look at the returncode attribute than the - exitstatus. - -A more real-world example would look like this: - -try: - retcode = call("mycmd" + " myarg", shell=True) - if retcode < 0: - print >>sys.stderr, "Child was terminated by signal", -retcode - else: - print >>sys.stderr, "Child returned", retcode -except OSError, e: - print >>sys.stderr, "Execution failed:", e - - -Replacing os.spawn* -------------------- -P_NOWAIT example: - -pid = os.spawnlp(os.P_NOWAIT, "/bin/mycmd", "mycmd", "myarg") -==> -pid = Popen(["/bin/mycmd", "myarg"]).pid - - -P_WAIT example: - -retcode = os.spawnlp(os.P_WAIT, "/bin/mycmd", "mycmd", "myarg") -==> -retcode = call(["/bin/mycmd", "myarg"]) - - -Vector example: - -os.spawnvp(os.P_NOWAIT, path, args) -==> -Popen([path] + args[1:]) - - -Environment example: - -os.spawnlpe(os.P_NOWAIT, "/bin/mycmd", "mycmd", "myarg", env) -==> -Popen(["/bin/mycmd", "myarg"], env={"PATH": "/usr/bin"}) - - -Replacing os.popen* -------------------- -pipe = os.popen("cmd", mode='r', bufsize) -==> -pipe = Popen("cmd", shell=True, bufsize=bufsize, stdout=PIPE).stdout - -pipe = os.popen("cmd", mode='w', bufsize) -==> -pipe = Popen("cmd", shell=True, bufsize=bufsize, stdin=PIPE).stdin - - -(child_stdin, child_stdout) = os.popen2("cmd", mode, bufsize) -==> -p = Popen("cmd", shell=True, bufsize=bufsize, - stdin=PIPE, stdout=PIPE, close_fds=True) -(child_stdin, child_stdout) = (p.stdin, p.stdout) - - -(child_stdin, - child_stdout, - child_stderr) = os.popen3("cmd", mode, bufsize) -==> -p = Popen("cmd", shell=True, bufsize=bufsize, - stdin=PIPE, stdout=PIPE, stderr=PIPE, close_fds=True) -(child_stdin, - child_stdout, - child_stderr) = (p.stdin, p.stdout, p.stderr) - - -(child_stdin, child_stdout_and_stderr) = os.popen4("cmd", mode, - bufsize) -==> -p = Popen("cmd", shell=True, bufsize=bufsize, - stdin=PIPE, stdout=PIPE, stderr=STDOUT, close_fds=True) -(child_stdin, child_stdout_and_stderr) = (p.stdin, p.stdout) - -On Unix, os.popen2, os.popen3 and os.popen4 also accept a sequence as -the command to execute, in which case arguments will be passed -directly to the program without shell intervention. This usage can be -replaced as follows: - -(child_stdin, child_stdout) = os.popen2(["/bin/ls", "-l"], mode, - bufsize) -==> -p = Popen(["/bin/ls", "-l"], bufsize=bufsize, stdin=PIPE, stdout=PIPE) -(child_stdin, child_stdout) = (p.stdin, p.stdout) - -Return code handling translates as follows: - -pipe = os.popen("cmd", 'w') -... -rc = pipe.close() -if rc is not None and rc % 256: - print "There were some errors" -==> -process = Popen("cmd", 'w', shell=True, stdin=PIPE) -... -process.stdin.close() -if process.wait() != 0: - print "There were some errors" - - -Replacing popen2.* ------------------- -(child_stdout, child_stdin) = popen2.popen2("somestring", bufsize, mode) -==> -p = Popen(["somestring"], shell=True, bufsize=bufsize - stdin=PIPE, stdout=PIPE, close_fds=True) -(child_stdout, child_stdin) = (p.stdout, p.stdin) - -On Unix, popen2 also accepts a sequence as the command to execute, in -which case arguments will be passed directly to the program without -shell intervention. This usage can be replaced as follows: - -(child_stdout, child_stdin) = popen2.popen2(["mycmd", "myarg"], bufsize, - mode) -==> -p = Popen(["mycmd", "myarg"], bufsize=bufsize, - stdin=PIPE, stdout=PIPE, close_fds=True) -(child_stdout, child_stdin) = (p.stdout, p.stdin) - -The popen2.Popen3 and popen2.Popen4 basically works as subprocess.Popen, -except that: - -* subprocess.Popen raises an exception if the execution fails -* the capturestderr argument is replaced with the stderr argument. -* stdin=PIPE and stdout=PIPE must be specified. -* popen2 closes all filedescriptors by default, but you have to specify - close_fds=True with subprocess.Popen. -""" - -import sys -mswindows = (sys.platform == "win32") - -import os -import types -import traceback -import gc -import signal -import errno - -try: - set -except NameError: - from sets import Set as set - -# Exception classes used by this module. - - -class CalledProcessError(Exception): - - """This exception is raised when a process run by check_call() or - check_output() returns a non-zero exit status. - The exit status will be stored in the returncode attribute; - check_output() will also store the output in the output attribute. - """ - - def __init__(self, returncode, cmd, output=None): - self.returncode = returncode - self.cmd = cmd - self.output = output - - def __str__(self): - return "Command '%s' returned non-zero exit status %d" % ( - self.cmd, self.returncode) - - -if mswindows: - import threading - import msvcrt - import _subprocess - - class STARTUPINFO: - dwFlags = 0 - hStdInput = None - hStdOutput = None - hStdError = None - wShowWindow = 0 - - class pywintypes: - error = IOError -else: - import select - _has_poll = hasattr(select, 'poll') - import fcntl - import pickle - - # When select or poll has indicated that the file is writable, - # we can write up to _PIPE_BUF bytes without risk of blocking. - # POSIX defines PIPE_BUF as >= 512. - _PIPE_BUF = getattr(select, 'PIPE_BUF', 512) - - -__all__ = ["Popen", "PIPE", "STDOUT", "call", "check_call", - "check_output", "CalledProcessError"] - -if mswindows: - from _subprocess import CREATE_NEW_CONSOLE, CREATE_NEW_PROCESS_GROUP, \ - STD_INPUT_HANDLE, STD_OUTPUT_HANDLE, \ - STD_ERROR_HANDLE, SW_HIDE, \ - STARTF_USESTDHANDLES, STARTF_USESHOWWINDOW - - __all__.extend(["CREATE_NEW_CONSOLE", "CREATE_NEW_PROCESS_GROUP", - "STD_INPUT_HANDLE", "STD_OUTPUT_HANDLE", - "STD_ERROR_HANDLE", "SW_HIDE", - "STARTF_USESTDHANDLES", "STARTF_USESHOWWINDOW"]) -try: - MAXFD = os.sysconf("SC_OPEN_MAX") -except: - MAXFD = 256 - -_active = [] - - -def _cleanup(): - for inst in _active[:]: - res = inst._internal_poll(_deadstate=sys.maxint) - if res is not None: - try: - _active.remove(inst) - except ValueError: - # This can happen if two threads create a new Popen instance. - # It's harmless that it was already removed, so ignore. - pass - -PIPE = -1 -STDOUT = -2 - - -def _eintr_retry_call(func, *args): - while True: - try: - return func(*args) - except (OSError, IOError), e: - if e.errno == errno.EINTR: - continue - raise - - -def call(*popenargs, **kwargs): - """Run command with arguments. Wait for command to complete, then - return the returncode attribute. - - The arguments are the same as for the Popen constructor. Example: - - retcode = call(["ls", "-l"]) - """ - return Popen(*popenargs, **kwargs).wait() - - -def check_call(*popenargs, **kwargs): - """Run command with arguments. Wait for command to complete. If - the exit code was zero then return, otherwise raise - CalledProcessError. The CalledProcessError object will have the - return code in the returncode attribute. - - The arguments are the same as for the Popen constructor. Example: - - check_call(["ls", "-l"]) - """ - retcode = call(*popenargs, **kwargs) - if retcode: - cmd = kwargs.get("args") - if cmd is None: - cmd = popenargs[0] - raise CalledProcessError(retcode, cmd) - return 0 - - -def check_output(*popenargs, **kwargs): - r"""Run command with arguments and return its output as a byte string. - - If the exit code was non-zero it raises a CalledProcessError. The - CalledProcessError object will have the return code in the returncode - attribute and output in the output attribute. - - The arguments are the same as for the Popen constructor. Example: - - >>> check_output(["ls", "-l", "/dev/null"]) - 'crw-rw-rw- 1 root root 1, 3 Oct 18 2007 /dev/null\n' - - The stdout argument is not allowed as it is used internally. - To capture standard error in the result, use stderr=STDOUT. - - >>> check_output(["/bin/sh", "-c", - ... "ls -l non_existent_file ; exit 0"], - ... stderr=STDOUT) - 'ls: non_existent_file: No such file or directory\n' - """ - if 'stdout' in kwargs: - raise ValueError('stdout argument not allowed, it will be overridden.') - process = Popen(stdout=PIPE, *popenargs, **kwargs) - output, unused_err = process.communicate() - retcode = process.poll() - if retcode: - cmd = kwargs.get("args") - if cmd is None: - cmd = popenargs[0] - raise CalledProcessError(retcode, cmd, output=output) - return output - - -def list2cmdline(seq): - """ - Translate a sequence of arguments into a command line - string, using the same rules as the MS C runtime: - - 1) Arguments are delimited by white space, which is either a - space or a tab. - - 2) A string surrounded by double quotation marks is - interpreted as a single argument, regardless of white space - contained within. A quoted string can be embedded in an - argument. - - 3) A double quotation mark preceded by a backslash is - interpreted as a literal double quotation mark. - - 4) Backslashes are interpreted literally, unless they - immediately precede a double quotation mark. - - 5) If backslashes immediately precede a double quotation mark, - every pair of backslashes is interpreted as a literal - backslash. If the number of backslashes is odd, the last - backslash escapes the next double quotation mark as - described in rule 3. - """ - - # See - # http://msdn.microsoft.com/en-us/library/17w5ykft.aspx - # or search http://msdn.microsoft.com for - # "Parsing C++ Command-Line Arguments" - result = [] - needquote = False - for arg in seq: - bs_buf = [] - - # Add a space to separate this argument from the others - if result: - result.append(' ') - - needquote = (" " in arg) or ("\t" in arg) or not arg - if needquote: - result.append('"') - - for c in arg: - if c == '\\': - # Don't know if we need to double yet. - bs_buf.append(c) - elif c == '"': - # Double backslashes. - result.append('\\' * len(bs_buf) * 2) - bs_buf = [] - result.append('\\"') - else: - # Normal char - if bs_buf: - result.extend(bs_buf) - bs_buf = [] - result.append(c) - - # Add remaining backslashes, if any. - if bs_buf: - result.extend(bs_buf) - - if needquote: - result.extend(bs_buf) - result.append('"') - - return ''.join(result) - - -class Popen(object): - - def __init__(self, args, bufsize=0, executable=None, - stdin=None, stdout=None, stderr=None, - preexec_fn=None, close_fds=False, shell=False, - cwd=None, env=None, universal_newlines=False, - startupinfo=None, creationflags=0): - """Create new Popen instance.""" - _cleanup() - - self._child_created = False - if not isinstance(bufsize, (int, long)): - raise TypeError("bufsize must be an integer") - - if mswindows: - if preexec_fn is not None: - raise ValueError("preexec_fn is not supported on Windows " - "platforms") - if close_fds and (stdin is not None or stdout is not None or - stderr is not None): - raise ValueError("close_fds is not supported on Windows " - "platforms if you redirect " - "stdin/stdout/stderr") - else: - # POSIX - if startupinfo is not None: - raise ValueError("startupinfo is only supported on Windows " - "platforms") - if creationflags != 0: - raise ValueError("creationflags is only supported on Windows " - "platforms") - - self.stdin = None - self.stdout = None - self.stderr = None - self.pid = None - self.returncode = None - self.universal_newlines = universal_newlines - - # Input and output objects. The general principle is like - # this: - # - # Parent Child - # ------ ----- - # p2cwrite ---stdin---> p2cread - # c2pread <--stdout--- c2pwrite - # errread <--stderr--- errwrite - # - # On POSIX, the child objects are file descriptors. On - # Windows, these are Windows file handles. The parent objects - # are file descriptors on both platforms. The parent objects - # are None when not using PIPEs. The child objects are None - # when not redirecting. - - (p2cread, p2cwrite, - c2pread, c2pwrite, - errread, errwrite) = self._get_handles(stdin, stdout, stderr) - - self._execute_child(args, executable, preexec_fn, close_fds, - cwd, env, universal_newlines, - startupinfo, creationflags, shell, - p2cread, p2cwrite, - c2pread, c2pwrite, - errread, errwrite) - - if mswindows: - if p2cwrite is not None: - p2cwrite = msvcrt.open_osfhandle(p2cwrite.Detach(), 0) - if c2pread is not None: - c2pread = msvcrt.open_osfhandle(c2pread.Detach(), 0) - if errread is not None: - errread = msvcrt.open_osfhandle(errread.Detach(), 0) - - if p2cwrite is not None: - self.stdin = os.fdopen(p2cwrite, 'wb', bufsize) - if c2pread is not None: - if universal_newlines: - self.stdout = os.fdopen(c2pread, 'rU', bufsize) - else: - self.stdout = os.fdopen(c2pread, 'rb', bufsize) - if errread is not None: - if universal_newlines: - self.stderr = os.fdopen(errread, 'rU', bufsize) - else: - self.stderr = os.fdopen(errread, 'rb', bufsize) - - def _translate_newlines(self, data): - data = data.replace("\r\n", "\n") - data = data.replace("\r", "\n") - return data - - def __del__(self, _maxint=sys.maxint, _active=_active): - # If __init__ hasn't had a chance to execute (e.g. if it - # was passed an undeclared keyword argument), we don't - # have a _child_created attribute at all. - if not getattr(self, '_child_created', False): - # We didn't get to successfully create a child process. - return - # In case the child hasn't been waited on, check if it's done. - self._internal_poll(_deadstate=_maxint) - if self.returncode is None and _active is not None: - # Child is still running, keep us alive until we can wait on it. - _active.append(self) - - def communicate(self, input=None): - """Interact with process: Send data to stdin. Read data from - stdout and stderr, until end-of-file is reached. Wait for - process to terminate. The optional input argument should be a - string to be sent to the child process, or None, if no data - should be sent to the child. - - communicate() returns a tuple (stdout, stderr).""" - - # Optimization: If we are only using one pipe, or no pipe at - # all, using select() or threads is unnecessary. - if [self.stdin, self.stdout, self.stderr].count(None) >= 2: - stdout = None - stderr = None - if self.stdin: - if input: - try: - self.stdin.write(input) - except IOError, e: - if e.errno != errno.EPIPE and e.errno != errno.EINVAL: - raise - self.stdin.close() - elif self.stdout: - stdout = _eintr_retry_call(self.stdout.read) - self.stdout.close() - elif self.stderr: - stderr = _eintr_retry_call(self.stderr.read) - self.stderr.close() - self.wait() - return (stdout, stderr) - - return self._communicate(input) - - def poll(self): - return self._internal_poll() - - if mswindows: - # - # Windows methods - # - def _get_handles(self, stdin, stdout, stderr): - """Construct and return tuple with IO objects: - p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite - """ - if stdin is None and stdout is None and stderr is None: - return (None, None, None, None, None, None) - - p2cread, p2cwrite = None, None - c2pread, c2pwrite = None, None - errread, errwrite = None, None - - if stdin is None: - p2cread = _subprocess.GetStdHandle( - _subprocess.STD_INPUT_HANDLE) - if p2cread is None: - p2cread, _ = _subprocess.CreatePipe(None, 0) - elif stdin == PIPE: - p2cread, p2cwrite = _subprocess.CreatePipe(None, 0) - elif isinstance(stdin, int): - p2cread = msvcrt.get_osfhandle(stdin) - else: - # Assuming file-like object - p2cread = msvcrt.get_osfhandle(stdin.fileno()) - p2cread = self._make_inheritable(p2cread) - - if stdout is None: - c2pwrite = _subprocess.GetStdHandle( - _subprocess.STD_OUTPUT_HANDLE) - if c2pwrite is None: - _, c2pwrite = _subprocess.CreatePipe(None, 0) - elif stdout == PIPE: - c2pread, c2pwrite = _subprocess.CreatePipe(None, 0) - elif isinstance(stdout, int): - c2pwrite = msvcrt.get_osfhandle(stdout) - else: - # Assuming file-like object - c2pwrite = msvcrt.get_osfhandle(stdout.fileno()) - c2pwrite = self._make_inheritable(c2pwrite) - - if stderr is None: - errwrite = _subprocess.GetStdHandle( - _subprocess.STD_ERROR_HANDLE) - if errwrite is None: - _, errwrite = _subprocess.CreatePipe(None, 0) - elif stderr == PIPE: - errread, errwrite = _subprocess.CreatePipe(None, 0) - elif stderr == STDOUT: - errwrite = c2pwrite - elif isinstance(stderr, int): - errwrite = msvcrt.get_osfhandle(stderr) - else: - # Assuming file-like object - errwrite = msvcrt.get_osfhandle(stderr.fileno()) - errwrite = self._make_inheritable(errwrite) - - return (p2cread, p2cwrite, - c2pread, c2pwrite, - errread, errwrite) - - def _make_inheritable(self, handle): - """Return a duplicate of handle, which is inheritable""" - return _subprocess.DuplicateHandle( - _subprocess.GetCurrentProcess(), - handle, - _subprocess.GetCurrentProcess(), - 0, - 1, - _subprocess.DUPLICATE_SAME_ACCESS - ) - - def _find_w9xpopen(self): - """Find and return absolut path to w9xpopen.exe""" - w9xpopen = os.path.join( - os.path.dirname(_subprocess.GetModuleFileName(0)), - "w9xpopen.exe") - if not os.path.exists(w9xpopen): - # Eeek - file-not-found - possibly an embedding - # situation - see if we can locate it in sys.exec_prefix - w9xpopen = os.path.join(os.path.dirname(sys.exec_prefix), - "w9xpopen.exe") - if not os.path.exists(w9xpopen): - raise RuntimeError("Cannot locate w9xpopen.exe, which is " - "needed for Popen to work with your " - "shell or platform.") - return w9xpopen - - def _execute_child(self, args, executable, preexec_fn, close_fds, - cwd, env, universal_newlines, - startupinfo, creationflags, shell, - p2cread, p2cwrite, - c2pread, c2pwrite, - errread, errwrite): - """Execute program (MS Windows version)""" - - if not isinstance(args, types.StringTypes): - args = list2cmdline(args) - - # Process startup details - if startupinfo is None: - startupinfo = STARTUPINFO() - if None not in (p2cread, c2pwrite, errwrite): - startupinfo.dwFlags |= _subprocess.STARTF_USESTDHANDLES - startupinfo.hStdInput = p2cread - startupinfo.hStdOutput = c2pwrite - startupinfo.hStdError = errwrite - - if shell: - startupinfo.dwFlags |= _subprocess.STARTF_USESHOWWINDOW - startupinfo.wShowWindow = _subprocess.SW_HIDE - comspec = os.environ.get("COMSPEC", "cmd.exe") - args = '{0} /c "{1}"'.format(comspec, args) - if (_subprocess.GetVersion() >= 0x80000000 or - os.path.basename(comspec).lower() == "command.com"): - # Win9x, or using command.com on NT. We need to - # use the w9xpopen intermediate program. For more - # information, see KB Q150956 - # (http://web.archive.org/web/20011105084002/http://support.microsoft.com/support/kb/articles/Q150/9/56.asp) - w9xpopen = self._find_w9xpopen() - args = '"%s" %s' % (w9xpopen, args) - # Not passing CREATE_NEW_CONSOLE has been known to - # cause random failures on win9x. Specifically a - # dialog: "Your program accessed mem currently in - # use at xxx" and a hopeful warning about the - # stability of your system. Cost is Ctrl+C wont - # kill children. - creationflags |= _subprocess.CREATE_NEW_CONSOLE - - # Start the process - try: - try: - hp, ht, pid, tid = _subprocess.CreateProcess( - executable, args, - # no special - # security - None, None, - int(not close_fds), - creationflags, - env, - cwd, - startupinfo) - except pywintypes.error, e: - # Translate pywintypes.error to WindowsError, which is - # a subclass of OSError. FIXME: We should really - # translate errno using _sys_errlist (or similar), but - # how can this be done from Python? - raise WindowsError(*e.args) - finally: - # Child is launched. Close the parent's copy of those pipe - # handles that only the child should have open. You need - # to make sure that no handles to the write end of the - # output pipe are maintained in this process or else the - # pipe will not close when the child process exits and the - # ReadFile will hang. - if p2cread is not None: - p2cread.Close() - if c2pwrite is not None: - c2pwrite.Close() - if errwrite is not None: - errwrite.Close() - - # Retain the process handle, but close the thread handle - self._child_created = True - self._handle = hp - self.pid = pid - ht.Close() - - def _internal_poll( - self, _deadstate=None, - _WaitForSingleObject=_subprocess.WaitForSingleObject, - _WAIT_OBJECT_0=_subprocess.WAIT_OBJECT_0, - _GetExitCodeProcess=_subprocess.GetExitCodeProcess - ): - """Check if child process has terminated. Returns returncode - attribute. - - This method is called by __del__, so it can only refer to objects - in its local scope. - - """ - if self.returncode is None: - if _WaitForSingleObject(self._handle, 0) == _WAIT_OBJECT_0: - self.returncode = _GetExitCodeProcess(self._handle) - return self.returncode - - def wait(self): - """Wait for child process to terminate. Returns returncode - attribute.""" - if self.returncode is None: - _subprocess.WaitForSingleObject(self._handle, - _subprocess.INFINITE) - self.returncode = _subprocess.GetExitCodeProcess(self._handle) - return self.returncode - - def _readerthread(self, fh, buffer): - buffer.append(fh.read()) - - def _communicate(self, input): - stdout = None # Return - stderr = None # Return - - if self.stdout: - stdout = [] - stdout_thread = threading.Thread(target=self._readerthread, - args=(self.stdout, stdout)) - stdout_thread.setDaemon(True) - stdout_thread.start() - if self.stderr: - stderr = [] - stderr_thread = threading.Thread(target=self._readerthread, - args=(self.stderr, stderr)) - stderr_thread.setDaemon(True) - stderr_thread.start() - - if self.stdin: - if input is not None: - try: - self.stdin.write(input) - except IOError, e: - if e.errno != errno.EPIPE: - raise - self.stdin.close() - - if self.stdout: - stdout_thread.join() - if self.stderr: - stderr_thread.join() - - # All data exchanged. Translate lists into strings. - if stdout is not None: - stdout = stdout[0] - if stderr is not None: - stderr = stderr[0] - - # Translate newlines, if requested. We cannot let the file - # object do the translation: It is based on stdio, which is - # impossible to combine with select (unless forcing no - # buffering). - if self.universal_newlines and hasattr(file, 'newlines'): - if stdout: - stdout = self._translate_newlines(stdout) - if stderr: - stderr = self._translate_newlines(stderr) - - self.wait() - return (stdout, stderr) - - def send_signal(self, sig): - """Send a signal to the process - """ - if sig == signal.SIGTERM: - self.terminate() - elif sig == signal.CTRL_C_EVENT: - os.kill(self.pid, signal.CTRL_C_EVENT) - elif sig == signal.CTRL_BREAK_EVENT: - os.kill(self.pid, signal.CTRL_BREAK_EVENT) - else: - raise ValueError("Unsupported signal: {0}".format(sig)) - - def terminate(self): - """Terminates the process - """ - _subprocess.TerminateProcess(self._handle, 1) - - kill = terminate - - else: - # - # POSIX methods - # - def _get_handles(self, stdin, stdout, stderr): - """Construct and return tuple with IO objects: - p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite - """ - p2cread, p2cwrite = None, None - c2pread, c2pwrite = None, None - errread, errwrite = None, None - - if stdin is None: - pass - elif stdin == PIPE: - p2cread, p2cwrite = self.pipe_cloexec() - elif isinstance(stdin, int): - p2cread = stdin - else: - # Assuming file-like object - p2cread = stdin.fileno() - - if stdout is None: - pass - elif stdout == PIPE: - c2pread, c2pwrite = self.pipe_cloexec() - elif isinstance(stdout, int): - c2pwrite = stdout - else: - # Assuming file-like object - c2pwrite = stdout.fileno() - - if stderr is None: - pass - elif stderr == PIPE: - errread, errwrite = self.pipe_cloexec() - elif stderr == STDOUT: - errwrite = c2pwrite - elif isinstance(stderr, int): - errwrite = stderr - else: - # Assuming file-like object - errwrite = stderr.fileno() - - return (p2cread, p2cwrite, - c2pread, c2pwrite, - errread, errwrite) - - def _set_cloexec_flag(self, fd, cloexec=True): - try: - cloexec_flag = fcntl.FD_CLOEXEC - except AttributeError: - cloexec_flag = 1 - - old = fcntl.fcntl(fd, fcntl.F_GETFD) - if cloexec: - fcntl.fcntl(fd, fcntl.F_SETFD, old | cloexec_flag) - else: - fcntl.fcntl(fd, fcntl.F_SETFD, old & ~cloexec_flag) - - def pipe_cloexec(self): - """Create a pipe with FDs set CLOEXEC.""" - # Pipes' FDs are set CLOEXEC by default because we don't want them - # to be inherited by other subprocesses: the CLOEXEC flag is - # removed from the child's FDs by _dup2(), between fork() and - # exec(). - # This is not atomic: we would need the pipe2() syscall for that. - r, w = os.pipe() - self._set_cloexec_flag(r) - self._set_cloexec_flag(w) - return r, w - - def _close_fds(self, but): - if hasattr(os, 'closerange'): - os.closerange(3, but) - os.closerange(but + 1, MAXFD) - else: - for i in xrange(3, MAXFD): - if i == but: - continue - try: - os.close(i) - except: - pass - - def _execute_child(self, args, executable, preexec_fn, close_fds, - cwd, env, universal_newlines, - startupinfo, creationflags, shell, - p2cread, p2cwrite, - c2pread, c2pwrite, - errread, errwrite): - """Execute program (POSIX version)""" - - if isinstance(args, types.StringTypes): - args = [args] - else: - args = list(args) - - if shell: - args = ["/bin/sh", "-c"] + args - if executable: - args[0] = executable - - if executable is None: - executable = args[0] - - # For transferring possible exec failure from child to parent - # The first char specifies the exception type: 0 means - # OSError, 1 means some other error. - errpipe_read, errpipe_write = self.pipe_cloexec() - try: - try: - gc_was_enabled = gc.isenabled() - # Disable gc to avoid bug where gc -> file_dealloc -> - # write to stderr -> hang. - # http://bugs.python.org/issue1336 - gc.disable() - try: - self.pid = os.fork() - except: - if gc_was_enabled: - gc.enable() - raise - self._child_created = True - if self.pid == 0: - # Child - try: - # Close parent's pipe ends - if p2cwrite is not None: - os.close(p2cwrite) - if c2pread is not None: - os.close(c2pread) - if errread is not None: - os.close(errread) - os.close(errpipe_read) - - # When duping fds, if there arises a situation - # where one of the fds is either 0, 1 or 2, it - # is possible that it is overwritten (#12607). - if c2pwrite == 0: - c2pwrite = os.dup(c2pwrite) - if errwrite == 0 or errwrite == 1: - errwrite = os.dup(errwrite) - - # Dup fds for child - def _dup2(a, b): - # dup2() removes the CLOEXEC flag but - # we must do it ourselves if dup2() - # would be a no-op (issue #10806). - if a == b: - self._set_cloexec_flag(a, False) - elif a is not None: - os.dup2(a, b) - _dup2(p2cread, 0) - _dup2(c2pwrite, 1) - _dup2(errwrite, 2) - - # Close pipe fds. Make sure we don't close the - # same fd more than once, or standard fds. - closed = set([None]) - for fd in [p2cread, c2pwrite, errwrite]: - if fd not in closed and fd > 2: - os.close(fd) - closed.add(fd) - - # Close all other fds, if asked for - if close_fds: - self._close_fds(but=errpipe_write) - - if cwd is not None: - os.chdir(cwd) - - if preexec_fn: - preexec_fn() - - if env is None: - os.execvp(executable, args) - else: - os.execvpe(executable, args, env) - - except: - exc_type, exc_value, tb = sys.exc_info() - # Save the traceback and attach it to the exception - # object - exc_lines = traceback.format_exception(exc_type, - exc_value, - tb) - exc_value.child_traceback = ''.join(exc_lines) - os.write(errpipe_write, pickle.dumps(exc_value)) - - # This exitcode won't be reported to applications, - # so it really doesn't matter what we return. - os._exit(255) - - # Parent - if gc_was_enabled: - gc.enable() - finally: - # be sure the FD is closed no matter what - os.close(errpipe_write) - - if p2cread is not None and p2cwrite is not None: - os.close(p2cread) - if c2pwrite is not None and c2pread is not None: - os.close(c2pwrite) - if errwrite is not None and errread is not None: - os.close(errwrite) - - # Wait for exec to fail or succeed; possibly raising exception - # Exception limited to 1M - data = _eintr_retry_call(os.read, errpipe_read, 1048576) - finally: - # be sure the FD is closed no matter what - os.close(errpipe_read) - - if data != "": - try: - _eintr_retry_call(os.waitpid, self.pid, 0) - except OSError, e: - if e.errno != errno.ECHILD: - raise - child_exception = pickle.loads(data) - for fd in (p2cwrite, c2pread, errread): - if fd is not None: - os.close(fd) - raise child_exception - - def _handle_exitstatus(self, sts, _WIFSIGNALED=os.WIFSIGNALED, - _WTERMSIG=os.WTERMSIG, _WIFEXITED=os.WIFEXITED, - _WEXITSTATUS=os.WEXITSTATUS): - # This method is called (indirectly) by __del__, so it cannot - # refer to anything outside of its local scope.""" - if _WIFSIGNALED(sts): - self.returncode = -_WTERMSIG(sts) - elif _WIFEXITED(sts): - self.returncode = _WEXITSTATUS(sts) - else: - # Should never happen - raise RuntimeError("Unknown child exit status!") - - def _internal_poll(self, _deadstate=None, _waitpid=os.waitpid, - _WNOHANG=os.WNOHANG, _os_error=os.error): - """Check if child process has terminated. Returns returncode - attribute. - - This method is called by __del__, so it cannot reference anything - outside of the local scope (nor can any methods it calls). - - """ - if self.returncode is None: - try: - pid, sts = _waitpid(self.pid, _WNOHANG) - if pid == self.pid: - self._handle_exitstatus(sts) - except _os_error: - if _deadstate is not None: - self.returncode = _deadstate - return self.returncode - - def wait(self): - """Wait for child process to terminate. Returns returncode - attribute.""" - if self.returncode is None: - try: - pid, sts = _eintr_retry_call(os.waitpid, self.pid, 0) - except OSError, e: - if e.errno != errno.ECHILD: - raise - # This happens if SIGCLD is set to be ignored or waiting - # for child processes has otherwise been disabled for our - # process. This child is dead, we can't get the status. - sts = 0 - self._handle_exitstatus(sts) - return self.returncode - - def _communicate(self, input): - if self.stdin: - # Flush stdio buffer. This might block, if the user has - # been writing to .stdin in an uncontrolled fashion. - self.stdin.flush() - if not input: - self.stdin.close() - - if _has_poll: - stdout, stderr = self._communicate_with_poll(input) - else: - stdout, stderr = self._communicate_with_select(input) - - # All data exchanged. Translate lists into strings. - if stdout is not None: - stdout = ''.join(stdout) - if stderr is not None: - stderr = ''.join(stderr) - - # Translate newlines, if requested. We cannot let the file - # object do the translation: It is based on stdio, which is - # impossible to combine with select (unless forcing no - # buffering). - if self.universal_newlines and hasattr(file, 'newlines'): - if stdout: - stdout = self._translate_newlines(stdout) - if stderr: - stderr = self._translate_newlines(stderr) - - self.wait() - return (stdout, stderr) - - def _communicate_with_poll(self, input): - stdout = None # Return - stderr = None # Return - fd2file = {} - fd2output = {} - - poller = select.poll() - - def register_and_append(file_obj, eventmask): - poller.register(file_obj.fileno(), eventmask) - fd2file[file_obj.fileno()] = file_obj - - def close_unregister_and_remove(fd): - poller.unregister(fd) - fd2file[fd].close() - fd2file.pop(fd) - - if self.stdin and input: - register_and_append(self.stdin, select.POLLOUT) - - select_POLLIN_POLLPRI = select.POLLIN | select.POLLPRI - if self.stdout: - register_and_append(self.stdout, select_POLLIN_POLLPRI) - fd2output[self.stdout.fileno()] = stdout = [] - if self.stderr: - register_and_append(self.stderr, select_POLLIN_POLLPRI) - fd2output[self.stderr.fileno()] = stderr = [] - - input_offset = 0 - while fd2file: - try: - ready = poller.poll() - except select.error, e: - if e.args[0] == errno.EINTR: - continue - raise - - for fd, mode in ready: - if mode & select.POLLOUT: - chunk = input[input_offset: input_offset + _PIPE_BUF] - try: - input_offset += os.write(fd, chunk) - except OSError, e: - if e.errno == errno.EPIPE: - close_unregister_and_remove(fd) - else: - raise - else: - if input_offset >= len(input): - close_unregister_and_remove(fd) - elif mode & select_POLLIN_POLLPRI: - data = os.read(fd, 4096) - if not data: - close_unregister_and_remove(fd) - fd2output[fd].append(data) - else: - # Ignore hang up or errors. - close_unregister_and_remove(fd) - - return (stdout, stderr) - - def _communicate_with_select(self, input): - read_set = [] - write_set = [] - stdout = None # Return - stderr = None # Return - - if self.stdin and input: - write_set.append(self.stdin) - if self.stdout: - read_set.append(self.stdout) - stdout = [] - if self.stderr: - read_set.append(self.stderr) - stderr = [] - - input_offset = 0 - while read_set or write_set: - try: - rlist, wlist, xlist = select.select( - read_set, write_set, []) - except select.error, e: - if e.args[0] == errno.EINTR: - continue - raise - - if self.stdin in wlist: - chunk = input[input_offset: input_offset + _PIPE_BUF] - try: - bytes_written = os.write(self.stdin.fileno(), chunk) - except OSError, e: - if e.errno == errno.EPIPE: - self.stdin.close() - write_set.remove(self.stdin) - else: - raise - else: - input_offset += bytes_written - if input_offset >= len(input): - self.stdin.close() - write_set.remove(self.stdin) - - if self.stdout in rlist: - data = os.read(self.stdout.fileno(), 1024) - if data == "": - self.stdout.close() - read_set.remove(self.stdout) - stdout.append(data) - - if self.stderr in rlist: - data = os.read(self.stderr.fileno(), 1024) - if data == "": - self.stderr.close() - read_set.remove(self.stderr) - stderr.append(data) - - return (stdout, stderr) - - def send_signal(self, sig): - """Send a signal to the process - """ - os.kill(self.pid, sig) - - def terminate(self): - """Terminate the process with SIGTERM - """ - self.send_signal(signal.SIGTERM) - - def kill(self): - """Kill the process with SIGKILL - """ - self.send_signal(signal.SIGKILL) - - -def _demo_posix(): - # - # Example 1: Simple redirection: Get process list - # - plist = Popen(["ps"], stdout=PIPE).communicate()[0] - print "Process list:" - print plist - - # - # Example 2: Change uid before executing child - # - if os.getuid() == 0: - p = Popen(["id"], preexec_fn=lambda: os.setuid(100)) - p.wait() - - # - # Example 3: Connecting several subprocesses - # - print "Looking for 'hda'..." - p1 = Popen(["dmesg"], stdout=PIPE) - p2 = Popen(["grep", "hda"], stdin=p1.stdout, stdout=PIPE) - print repr(p2.communicate()[0]) - - # - # Example 4: Catch execution error - # - print - print "Trying a weird file..." - try: - print Popen(["/this/path/does/not/exist"]).communicate() - except OSError, e: - if e.errno == errno.ENOENT: - print "The file didn't exist. I thought so..." - print "Child traceback:" - print e.child_traceback - else: - print "Error", e.errno - else: - print >>sys.stderr, "Gosh. No error." - - -def _demo_windows(): - # - # Example 1: Connecting several subprocesses - # - print "Looking for 'PROMPT' in set output..." - p1 = Popen("set", stdout=PIPE, shell=True) - p2 = Popen('find "PROMPT"', stdin=p1.stdout, stdout=PIPE) - print repr(p2.communicate()[0]) - - # - # Example 2: Simple execution of program - # - print "Executing calc..." - p = Popen("calc") - p.wait() - - -if __name__ == "__main__": - if mswindows: - _demo_windows() - else: - _demo_posix() diff --git a/lib/cherrypy/_cpconfig.py b/lib/cherrypy/_cpconfig.py deleted file mode 100644 index 129ddcb621b..00000000000 --- a/lib/cherrypy/_cpconfig.py +++ /dev/null @@ -1,329 +0,0 @@ -""" -Configuration system for CherryPy. - -Configuration in CherryPy is implemented via dictionaries. Keys are strings -which name the mapped value, which may be of any type. - - -Architecture ------------- - -CherryPy Requests are part of an Application, which runs in a global context, -and configuration data may apply to any of those three scopes: - -Global - Configuration entries which apply everywhere are stored in - cherrypy.config. - -Application - Entries which apply to each mounted application are stored - on the Application object itself, as 'app.config'. This is a two-level - dict where each key is a path, or "relative URL" (for example, "/" or - "/path/to/my/page"), and each value is a config dict. Usually, this - data is provided in the call to tree.mount(root(), config=conf), - although you may also use app.merge(conf). - -Request - Each Request object possesses a single 'Request.config' dict. - Early in the request process, this dict is populated by merging global - config entries, Application entries (whose path equals or is a parent - of Request.path_info), and any config acquired while looking up the - page handler (see next). - - -Declaration ------------ - -Configuration data may be supplied as a Python dictionary, as a filename, -or as an open file object. When you supply a filename or file, CherryPy -uses Python's builtin ConfigParser; you declare Application config by -writing each path as a section header:: - - [/path/to/my/page] - request.stream = True - -To declare global configuration entries, place them in a [global] section. - -You may also declare config entries directly on the classes and methods -(page handlers) that make up your CherryPy application via the ``_cp_config`` -attribute, set with the ``cherrypy.config`` decorator. For example:: - - @cherrypy.config(**{'tools.gzip.on': True}) - class Demo: - - @cherrypy.expose - @cherrypy.config(**{'request.show_tracebacks': False}) - def index(self): - return "Hello world" - -.. note:: - - This behavior is only guaranteed for the default dispatcher. - Other dispatchers may have different restrictions on where - you can attach config attributes. - - -Namespaces ----------- - -Configuration keys are separated into namespaces by the first "." in the key. -Current namespaces: - -engine - Controls the 'application engine', including autoreload. - These can only be declared in the global config. - -tree - Grafts cherrypy.Application objects onto cherrypy.tree. - These can only be declared in the global config. - -hooks - Declares additional request-processing functions. - -log - Configures the logging for each application. - These can only be declared in the global or / config. - -request - Adds attributes to each Request. - -response - Adds attributes to each Response. - -server - Controls the default HTTP server via cherrypy.server. - These can only be declared in the global config. - -tools - Runs and configures additional request-processing packages. - -wsgi - Adds WSGI middleware to an Application's "pipeline". - These can only be declared in the app's root config ("/"). - -checker - Controls the 'checker', which looks for common errors in - app state (including config) when the engine starts. - Global config only. - -The only key that does not exist in a namespace is the "environment" entry. -This special entry 'imports' other config entries from a template stored in -cherrypy._cpconfig.environments[environment]. It only applies to the global -config, and only when you use cherrypy.config.update. - -You can define your own namespaces to be called at the Global, Application, -or Request level, by adding a named handler to cherrypy.config.namespaces, -app.namespaces, or app.request_class.namespaces. The name can -be any string, and the handler must be either a callable or a (Python 2.5 -style) context manager. -""" - -import cherrypy -from cherrypy._cpcompat import basestring -from cherrypy.lib import reprconf - -# Deprecated in CherryPy 3.2--remove in 3.3 -NamespaceSet = reprconf.NamespaceSet - - -def merge(base, other): - """Merge one app config (from a dict, file, or filename) into another. - - If the given config is a filename, it will be appended to - the list of files to monitor for "autoreload" changes. - """ - if isinstance(other, basestring): - cherrypy.engine.autoreload.files.add(other) - - # Load other into base - for section, value_map in reprconf.as_dict(other).items(): - if not isinstance(value_map, dict): - raise ValueError( - "Application config must include section headers, but the " - "config you tried to merge doesn't have any sections. " - "Wrap your config in another dict with paths as section " - "headers, for example: {'/': config}.") - base.setdefault(section, {}).update(value_map) - - -class Config(reprconf.Config): - - """The 'global' configuration data for the entire CherryPy process.""" - - def update(self, config): - """Update self from a dict, file or filename.""" - if isinstance(config, basestring): - # Filename - cherrypy.engine.autoreload.files.add(config) - reprconf.Config.update(self, config) - - def _apply(self, config): - """Update self from a dict.""" - if isinstance(config.get("global"), dict): - if len(config) > 1: - cherrypy.checker.global_config_contained_paths = True - config = config["global"] - if 'tools.staticdir.dir' in config: - config['tools.staticdir.section'] = "global" - reprconf.Config._apply(self, config) - - @staticmethod - def __call__(*args, **kwargs): - """Decorator for page handlers to set _cp_config.""" - if args: - raise TypeError( - "The cherrypy.config decorator does not accept positional " - "arguments; you must use keyword arguments.") - - def tool_decorator(f): - _Vars(f).setdefault('_cp_config', {}).update(kwargs) - return f - return tool_decorator - - -class _Vars(object): - """ - Adapter that allows setting a default attribute on a function - or class. - """ - def __init__(self, target): - self.target = target - - def setdefault(self, key, default): - if not hasattr(self.target, key): - setattr(self.target, key, default) - return getattr(self.target, key) - - -# Sphinx begin config.environments -Config.environments = environments = { - "staging": { - 'engine.autoreload.on': False, - 'checker.on': False, - 'tools.log_headers.on': False, - 'request.show_tracebacks': False, - 'request.show_mismatched_params': False, - }, - "production": { - 'engine.autoreload.on': False, - 'checker.on': False, - 'tools.log_headers.on': False, - 'request.show_tracebacks': False, - 'request.show_mismatched_params': False, - 'log.screen': False, - }, - "embedded": { - # For use with CherryPy embedded in another deployment stack. - 'engine.autoreload.on': False, - 'checker.on': False, - 'tools.log_headers.on': False, - 'request.show_tracebacks': False, - 'request.show_mismatched_params': False, - 'log.screen': False, - 'engine.SIGHUP': None, - 'engine.SIGTERM': None, - }, - "test_suite": { - 'engine.autoreload.on': False, - 'checker.on': False, - 'tools.log_headers.on': False, - 'request.show_tracebacks': True, - 'request.show_mismatched_params': True, - 'log.screen': False, - }, -} -# Sphinx end config.environments - - -def _server_namespace_handler(k, v): - """Config handler for the "server" namespace.""" - atoms = k.split(".", 1) - if len(atoms) > 1: - # Special-case config keys of the form 'server.servername.socket_port' - # to configure additional HTTP servers. - if not hasattr(cherrypy, "servers"): - cherrypy.servers = {} - - servername, k = atoms - if servername not in cherrypy.servers: - from cherrypy import _cpserver - cherrypy.servers[servername] = _cpserver.Server() - # On by default, but 'on = False' can unsubscribe it (see below). - cherrypy.servers[servername].subscribe() - - if k == 'on': - if v: - cherrypy.servers[servername].subscribe() - else: - cherrypy.servers[servername].unsubscribe() - else: - setattr(cherrypy.servers[servername], k, v) - else: - setattr(cherrypy.server, k, v) -Config.namespaces["server"] = _server_namespace_handler - - -def _engine_namespace_handler(k, v): - """Backward compatibility handler for the "engine" namespace.""" - engine = cherrypy.engine - - deprecated = { - 'autoreload_on': 'autoreload.on', - 'autoreload_frequency': 'autoreload.frequency', - 'autoreload_match': 'autoreload.match', - 'reload_files': 'autoreload.files', - 'deadlock_poll_freq': 'timeout_monitor.frequency' - } - - if k in deprecated: - engine.log( - 'WARNING: Use of engine.%s is deprecated and will be removed in a ' - 'future version. Use engine.%s instead.' % (k, deprecated[k])) - - if k == 'autoreload_on': - if v: - engine.autoreload.subscribe() - else: - engine.autoreload.unsubscribe() - elif k == 'autoreload_frequency': - engine.autoreload.frequency = v - elif k == 'autoreload_match': - engine.autoreload.match = v - elif k == 'reload_files': - engine.autoreload.files = set(v) - elif k == 'deadlock_poll_freq': - engine.timeout_monitor.frequency = v - elif k == 'SIGHUP': - engine.listeners['SIGHUP'] = set([v]) - elif k == 'SIGTERM': - engine.listeners['SIGTERM'] = set([v]) - elif "." in k: - plugin, attrname = k.split(".", 1) - plugin = getattr(engine, plugin) - if attrname == 'on': - if v and hasattr(getattr(plugin, 'subscribe', None), '__call__'): - plugin.subscribe() - return - elif ( - (not v) and - hasattr(getattr(plugin, 'unsubscribe', None), '__call__') - ): - plugin.unsubscribe() - return - setattr(plugin, attrname, v) - else: - setattr(engine, k, v) -Config.namespaces["engine"] = _engine_namespace_handler - - -def _tree_namespace_handler(k, v): - """Namespace handler for the 'tree' config namespace.""" - if isinstance(v, dict): - for script_name, app in v.items(): - cherrypy.tree.graft(app, script_name) - msg = "Mounted: %s on %s" % (app, script_name or "/") - cherrypy.engine.log(msg) - else: - cherrypy.tree.graft(v, v.script_name) - cherrypy.engine.log("Mounted: %s on %s" % (v, v.script_name or "/")) -Config.namespaces["tree"] = _tree_namespace_handler diff --git a/lib/cherrypy/_cpdispatch.py b/lib/cherrypy/_cpdispatch.py deleted file mode 100644 index 2cb03c7e00e..00000000000 --- a/lib/cherrypy/_cpdispatch.py +++ /dev/null @@ -1,685 +0,0 @@ -"""CherryPy dispatchers. - -A 'dispatcher' is the object which looks up the 'page handler' callable -and collects config for the current request based on the path_info, other -request attributes, and the application architecture. The core calls the -dispatcher as early as possible, passing it a 'path_info' argument. - -The default dispatcher discovers the page handler by matching path_info -to a hierarchical arrangement of objects, starting at request.app.root. -""" - -import string -import sys -import types -try: - classtype = (type, types.ClassType) -except AttributeError: - classtype = type - -import cherrypy - - -class PageHandler(object): - - """Callable which sets response.body.""" - - def __init__(self, callable, *args, **kwargs): - self.callable = callable - self.args = args - self.kwargs = kwargs - - def get_args(self): - return cherrypy.serving.request.args - - def set_args(self, args): - cherrypy.serving.request.args = args - return cherrypy.serving.request.args - - args = property( - get_args, - set_args, - doc="The ordered args should be accessible from post dispatch hooks" - ) - - def get_kwargs(self): - return cherrypy.serving.request.kwargs - - def set_kwargs(self, kwargs): - cherrypy.serving.request.kwargs = kwargs - return cherrypy.serving.request.kwargs - - kwargs = property( - get_kwargs, - set_kwargs, - doc="The named kwargs should be accessible from post dispatch hooks" - ) - - def __call__(self): - try: - return self.callable(*self.args, **self.kwargs) - except TypeError: - x = sys.exc_info()[1] - try: - test_callable_spec(self.callable, self.args, self.kwargs) - except cherrypy.HTTPError: - raise sys.exc_info()[1] - except: - raise x - raise - - -def test_callable_spec(callable, callable_args, callable_kwargs): - """ - Inspect callable and test to see if the given args are suitable for it. - - When an error occurs during the handler's invoking stage there are 2 - erroneous cases: - 1. Too many parameters passed to a function which doesn't define - one of *args or **kwargs. - 2. Too little parameters are passed to the function. - - There are 3 sources of parameters to a cherrypy handler. - 1. query string parameters are passed as keyword parameters to the - handler. - 2. body parameters are also passed as keyword parameters. - 3. when partial matching occurs, the final path atoms are passed as - positional args. - Both the query string and path atoms are part of the URI. If they are - incorrect, then a 404 Not Found should be raised. Conversely the body - parameters are part of the request; if they are invalid a 400 Bad Request. - """ - show_mismatched_params = getattr( - cherrypy.serving.request, 'show_mismatched_params', False) - try: - (args, varargs, varkw, defaults) = getargspec(callable) - except TypeError: - if isinstance(callable, object) and hasattr(callable, '__call__'): - (args, varargs, varkw, - defaults) = getargspec(callable.__call__) - else: - # If it wasn't one of our own types, re-raise - # the original error - raise - - if args and args[0] == 'self': - args = args[1:] - - arg_usage = dict([(arg, 0,) for arg in args]) - vararg_usage = 0 - varkw_usage = 0 - extra_kwargs = set() - - for i, value in enumerate(callable_args): - try: - arg_usage[args[i]] += 1 - except IndexError: - vararg_usage += 1 - - for key in callable_kwargs.keys(): - try: - arg_usage[key] += 1 - except KeyError: - varkw_usage += 1 - extra_kwargs.add(key) - - # figure out which args have defaults. - args_with_defaults = args[-len(defaults or []):] - for i, val in enumerate(defaults or []): - # Defaults take effect only when the arg hasn't been used yet. - if arg_usage[args_with_defaults[i]] == 0: - arg_usage[args_with_defaults[i]] += 1 - - missing_args = [] - multiple_args = [] - for key, usage in arg_usage.items(): - if usage == 0: - missing_args.append(key) - elif usage > 1: - multiple_args.append(key) - - if missing_args: - # In the case where the method allows body arguments - # there are 3 potential errors: - # 1. not enough query string parameters -> 404 - # 2. not enough body parameters -> 400 - # 3. not enough path parts (partial matches) -> 404 - # - # We can't actually tell which case it is, - # so I'm raising a 404 because that covers 2/3 of the - # possibilities - # - # In the case where the method does not allow body - # arguments it's definitely a 404. - message = None - if show_mismatched_params: - message = "Missing parameters: %s" % ",".join(missing_args) - raise cherrypy.HTTPError(404, message=message) - - # the extra positional arguments come from the path - 404 Not Found - if not varargs and vararg_usage > 0: - raise cherrypy.HTTPError(404) - - body_params = cherrypy.serving.request.body.params or {} - body_params = set(body_params.keys()) - qs_params = set(callable_kwargs.keys()) - body_params - - if multiple_args: - if qs_params.intersection(set(multiple_args)): - # If any of the multiple parameters came from the query string then - # it's a 404 Not Found - error = 404 - else: - # Otherwise it's a 400 Bad Request - error = 400 - - message = None - if show_mismatched_params: - message = "Multiple values for parameters: "\ - "%s" % ",".join(multiple_args) - raise cherrypy.HTTPError(error, message=message) - - if not varkw and varkw_usage > 0: - - # If there were extra query string parameters, it's a 404 Not Found - extra_qs_params = set(qs_params).intersection(extra_kwargs) - if extra_qs_params: - message = None - if show_mismatched_params: - message = "Unexpected query string "\ - "parameters: %s" % ", ".join(extra_qs_params) - raise cherrypy.HTTPError(404, message=message) - - # If there were any extra body parameters, it's a 400 Not Found - extra_body_params = set(body_params).intersection(extra_kwargs) - if extra_body_params: - message = None - if show_mismatched_params: - message = "Unexpected body parameters: "\ - "%s" % ", ".join(extra_body_params) - raise cherrypy.HTTPError(400, message=message) - - -try: - import inspect -except ImportError: - test_callable_spec = lambda callable, args, kwargs: None -else: - getargspec = inspect.getargspec - # Python 3 requires using getfullargspec if keyword-only arguments are present - if hasattr(inspect, 'getfullargspec'): - def getargspec(callable): - return inspect.getfullargspec(callable)[:4] - - -class LateParamPageHandler(PageHandler): - - """When passing cherrypy.request.params to the page handler, we do not - want to capture that dict too early; we want to give tools like the - decoding tool a chance to modify the params dict in-between the lookup - of the handler and the actual calling of the handler. This subclass - takes that into account, and allows request.params to be 'bound late' - (it's more complicated than that, but that's the effect). - """ - - def _get_kwargs(self): - kwargs = cherrypy.serving.request.params.copy() - if self._kwargs: - kwargs.update(self._kwargs) - return kwargs - - def _set_kwargs(self, kwargs): - cherrypy.serving.request.kwargs = kwargs - self._kwargs = kwargs - - kwargs = property(_get_kwargs, _set_kwargs, - doc='page handler kwargs (with ' - 'cherrypy.request.params copied in)') - - -if sys.version_info < (3, 0): - punctuation_to_underscores = string.maketrans( - string.punctuation, '_' * len(string.punctuation)) - - def validate_translator(t): - if not isinstance(t, str) or len(t) != 256: - raise ValueError( - "The translate argument must be a str of len 256.") -else: - punctuation_to_underscores = str.maketrans( - string.punctuation, '_' * len(string.punctuation)) - - def validate_translator(t): - if not isinstance(t, dict): - raise ValueError("The translate argument must be a dict.") - - -class Dispatcher(object): - - """CherryPy Dispatcher which walks a tree of objects to find a handler. - - The tree is rooted at cherrypy.request.app.root, and each hierarchical - component in the path_info argument is matched to a corresponding nested - attribute of the root object. Matching handlers must have an 'exposed' - attribute which evaluates to True. The special method name "index" - matches a URI which ends in a slash ("/"). The special method name - "default" may match a portion of the path_info (but only when no longer - substring of the path_info matches some other object). - - This is the default, built-in dispatcher for CherryPy. - """ - - dispatch_method_name = '_cp_dispatch' - """ - The name of the dispatch method that nodes may optionally implement - to provide their own dynamic dispatch algorithm. - """ - - def __init__(self, dispatch_method_name=None, - translate=punctuation_to_underscores): - validate_translator(translate) - self.translate = translate - if dispatch_method_name: - self.dispatch_method_name = dispatch_method_name - - def __call__(self, path_info): - """Set handler and config for the current request.""" - request = cherrypy.serving.request - func, vpath = self.find_handler(path_info) - - if func: - # Decode any leftover %2F in the virtual_path atoms. - vpath = [x.replace("%2F", "/") for x in vpath] - request.handler = LateParamPageHandler(func, *vpath) - else: - request.handler = cherrypy.NotFound() - - def find_handler(self, path): - """Return the appropriate page handler, plus any virtual path. - - This will return two objects. The first will be a callable, - which can be used to generate page output. Any parameters from - the query string or request body will be sent to that callable - as keyword arguments. - - The callable is found by traversing the application's tree, - starting from cherrypy.request.app.root, and matching path - components to successive objects in the tree. For example, the - URL "/path/to/handler" might return root.path.to.handler. - - The second object returned will be a list of names which are - 'virtual path' components: parts of the URL which are dynamic, - and were not used when looking up the handler. - These virtual path components are passed to the handler as - positional arguments. - """ - request = cherrypy.serving.request - app = request.app - root = app.root - dispatch_name = self.dispatch_method_name - - # Get config for the root object/path. - fullpath = [x for x in path.strip('/').split('/') if x] + ['index'] - fullpath_len = len(fullpath) - segleft = fullpath_len - nodeconf = {} - if hasattr(root, "_cp_config"): - nodeconf.update(root._cp_config) - if "/" in app.config: - nodeconf.update(app.config["/"]) - object_trail = [['root', root, nodeconf, segleft]] - - node = root - iternames = fullpath[:] - while iternames: - name = iternames[0] - # map to legal Python identifiers (e.g. replace '.' with '_') - objname = name.translate(self.translate) - - nodeconf = {} - subnode = getattr(node, objname, None) - pre_len = len(iternames) - if subnode is None: - dispatch = getattr(node, dispatch_name, None) - if dispatch and hasattr(dispatch, '__call__') and not \ - getattr(dispatch, 'exposed', False) and \ - pre_len > 1: - # Don't expose the hidden 'index' token to _cp_dispatch - # We skip this if pre_len == 1 since it makes no sense - # to call a dispatcher when we have no tokens left. - index_name = iternames.pop() - subnode = dispatch(vpath=iternames) - iternames.append(index_name) - else: - # We didn't find a path, but keep processing in case there - # is a default() handler. - iternames.pop(0) - else: - # We found the path, remove the vpath entry - iternames.pop(0) - segleft = len(iternames) - if segleft > pre_len: - # No path segment was removed. Raise an error. - raise cherrypy.CherryPyException( - "A vpath segment was added. Custom dispatchers may only " - + "remove elements. While trying to process " - + "{0} in {1}".format(name, fullpath) - ) - elif segleft == pre_len: - # Assume that the handler used the current path segment, but - # did not pop it. This allows things like - # return getattr(self, vpath[0], None) - iternames.pop(0) - segleft -= 1 - node = subnode - - if node is not None: - # Get _cp_config attached to this node. - if hasattr(node, "_cp_config"): - nodeconf.update(node._cp_config) - - # Mix in values from app.config for this path. - existing_len = fullpath_len - pre_len - if existing_len != 0: - curpath = '/' + '/'.join(fullpath[0:existing_len]) - else: - curpath = '' - new_segs = fullpath[fullpath_len - pre_len:fullpath_len - segleft] - for seg in new_segs: - curpath += '/' + seg - if curpath in app.config: - nodeconf.update(app.config[curpath]) - - object_trail.append([name, node, nodeconf, segleft]) - - def set_conf(): - """Collapse all object_trail config into cherrypy.request.config. - """ - base = cherrypy.config.copy() - # Note that we merge the config from each node - # even if that node was None. - for name, obj, conf, segleft in object_trail: - base.update(conf) - if 'tools.staticdir.dir' in conf: - base['tools.staticdir.section'] = '/' + \ - '/'.join(fullpath[0:fullpath_len - segleft]) - return base - - # Try successive objects (reverse order) - num_candidates = len(object_trail) - 1 - for i in range(num_candidates, -1, -1): - - name, candidate, nodeconf, segleft = object_trail[i] - if candidate is None: - continue - - # Try a "default" method on the current leaf. - if hasattr(candidate, "default"): - defhandler = candidate.default - if getattr(defhandler, 'exposed', False): - # Insert any extra _cp_config from the default handler. - conf = getattr(defhandler, "_cp_config", {}) - object_trail.insert( - i + 1, ["default", defhandler, conf, segleft]) - request.config = set_conf() - # See https://github.com/cherrypy/cherrypy/issues/613 - request.is_index = path.endswith("/") - return defhandler, fullpath[fullpath_len - segleft:-1] - - # Uncomment the next line to restrict positional params to - # "default". - # if i < num_candidates - 2: continue - - # Try the current leaf. - if getattr(candidate, 'exposed', False): - request.config = set_conf() - if i == num_candidates: - # We found the extra ".index". Mark request so tools - # can redirect if path_info has no trailing slash. - request.is_index = True - else: - # We're not at an 'index' handler. Mark request so tools - # can redirect if path_info has NO trailing slash. - # Note that this also includes handlers which take - # positional parameters (virtual paths). - request.is_index = False - return candidate, fullpath[fullpath_len - segleft:-1] - - # We didn't find anything - request.config = set_conf() - return None, [] - - -class MethodDispatcher(Dispatcher): - - """Additional dispatch based on cherrypy.request.method.upper(). - - Methods named GET, POST, etc will be called on an exposed class. - The method names must be all caps; the appropriate Allow header - will be output showing all capitalized method names as allowable - HTTP verbs. - - Note that the containing class must be exposed, not the methods. - """ - - def __call__(self, path_info): - """Set handler and config for the current request.""" - request = cherrypy.serving.request - resource, vpath = self.find_handler(path_info) - - if resource: - # Set Allow header - avail = [m for m in dir(resource) if m.isupper()] - if "GET" in avail and "HEAD" not in avail: - avail.append("HEAD") - avail.sort() - cherrypy.serving.response.headers['Allow'] = ", ".join(avail) - - # Find the subhandler - meth = request.method.upper() - func = getattr(resource, meth, None) - if func is None and meth == "HEAD": - func = getattr(resource, "GET", None) - if func: - # Grab any _cp_config on the subhandler. - if hasattr(func, "_cp_config"): - request.config.update(func._cp_config) - - # Decode any leftover %2F in the virtual_path atoms. - vpath = [x.replace("%2F", "/") for x in vpath] - request.handler = LateParamPageHandler(func, *vpath) - else: - request.handler = cherrypy.HTTPError(405) - else: - request.handler = cherrypy.NotFound() - - -class RoutesDispatcher(object): - - """A Routes based dispatcher for CherryPy.""" - - def __init__(self, full_result=False, **mapper_options): - """ - Routes dispatcher - - Set full_result to True if you wish the controller - and the action to be passed on to the page handler - parameters. By default they won't be. - """ - import routes - self.full_result = full_result - self.controllers = {} - self.mapper = routes.Mapper(**mapper_options) - self.mapper.controller_scan = self.controllers.keys - - def connect(self, name, route, controller, **kwargs): - self.controllers[name] = controller - self.mapper.connect(name, route, controller=name, **kwargs) - - def redirect(self, url): - raise cherrypy.HTTPRedirect(url) - - def __call__(self, path_info): - """Set handler and config for the current request.""" - func = self.find_handler(path_info) - if func: - cherrypy.serving.request.handler = LateParamPageHandler(func) - else: - cherrypy.serving.request.handler = cherrypy.NotFound() - - def find_handler(self, path_info): - """Find the right page handler, and set request.config.""" - import routes - - request = cherrypy.serving.request - - config = routes.request_config() - config.mapper = self.mapper - if hasattr(request, 'wsgi_environ'): - config.environ = request.wsgi_environ - config.host = request.headers.get('Host', None) - config.protocol = request.scheme - config.redirect = self.redirect - - result = self.mapper.match(path_info) - - config.mapper_dict = result - params = {} - if result: - params = result.copy() - if not self.full_result: - params.pop('controller', None) - params.pop('action', None) - request.params.update(params) - - # Get config for the root object/path. - request.config = base = cherrypy.config.copy() - curpath = "" - - def merge(nodeconf): - if 'tools.staticdir.dir' in nodeconf: - nodeconf['tools.staticdir.section'] = curpath or "/" - base.update(nodeconf) - - app = request.app - root = app.root - if hasattr(root, "_cp_config"): - merge(root._cp_config) - if "/" in app.config: - merge(app.config["/"]) - - # Mix in values from app.config. - atoms = [x for x in path_info.split("/") if x] - if atoms: - last = atoms.pop() - else: - last = None - for atom in atoms: - curpath = "/".join((curpath, atom)) - if curpath in app.config: - merge(app.config[curpath]) - - handler = None - if result: - controller = result.get('controller') - controller = self.controllers.get(controller, controller) - if controller: - if isinstance(controller, classtype): - controller = controller() - # Get config from the controller. - if hasattr(controller, "_cp_config"): - merge(controller._cp_config) - - action = result.get('action') - if action is not None: - handler = getattr(controller, action, None) - # Get config from the handler - if hasattr(handler, "_cp_config"): - merge(handler._cp_config) - else: - handler = controller - - # Do the last path atom here so it can - # override the controller's _cp_config. - if last: - curpath = "/".join((curpath, last)) - if curpath in app.config: - merge(app.config[curpath]) - - return handler - - -def XMLRPCDispatcher(next_dispatcher=Dispatcher()): - from cherrypy.lib import xmlrpcutil - - def xmlrpc_dispatch(path_info): - path_info = xmlrpcutil.patched_path(path_info) - return next_dispatcher(path_info) - return xmlrpc_dispatch - - -def VirtualHost(next_dispatcher=Dispatcher(), use_x_forwarded_host=True, - **domains): - """ - Select a different handler based on the Host header. - - This can be useful when running multiple sites within one CP server. - It allows several domains to point to different parts of a single - website structure. For example:: - - http://www.domain.example -> root - http://www.domain2.example -> root/domain2/ - http://www.domain2.example:443 -> root/secure - - can be accomplished via the following config:: - - [/] - request.dispatch = cherrypy.dispatch.VirtualHost( - **{'www.domain2.example': '/domain2', - 'www.domain2.example:443': '/secure', - }) - - next_dispatcher - The next dispatcher object in the dispatch chain. - The VirtualHost dispatcher adds a prefix to the URL and calls - another dispatcher. Defaults to cherrypy.dispatch.Dispatcher(). - - use_x_forwarded_host - If True (the default), any "X-Forwarded-Host" - request header will be used instead of the "Host" header. This - is commonly added by HTTP servers (such as Apache) when proxying. - - ``**domains`` - A dict of {host header value: virtual prefix} pairs. - The incoming "Host" request header is looked up in this dict, - and, if a match is found, the corresponding "virtual prefix" - value will be prepended to the URL path before calling the - next dispatcher. Note that you often need separate entries - for "example.com" and "www.example.com". In addition, "Host" - headers may contain the port number. - """ - from cherrypy.lib import httputil - - def vhost_dispatch(path_info): - request = cherrypy.serving.request - header = request.headers.get - - domain = header('Host', '') - if use_x_forwarded_host: - domain = header("X-Forwarded-Host", domain) - - prefix = domains.get(domain, "") - if prefix: - path_info = httputil.urljoin(prefix, path_info) - - result = next_dispatcher(path_info) - - # Touch up staticdir config. See - # https://github.com/cherrypy/cherrypy/issues/614. - section = request.config.get('tools.staticdir.section') - if section: - section = section[len(prefix):] - request.config['tools.staticdir.section'] = section - - return result - return vhost_dispatch diff --git a/lib/cherrypy/_cperror.py b/lib/cherrypy/_cperror.py deleted file mode 100644 index 8266230544b..00000000000 --- a/lib/cherrypy/_cperror.py +++ /dev/null @@ -1,620 +0,0 @@ -"""Exception classes for CherryPy. - -CherryPy provides (and uses) exceptions for declaring that the HTTP response -should be a status other than the default "200 OK". You can ``raise`` them like -normal Python exceptions. You can also call them and they will raise -themselves; this means you can set an -:class:`HTTPError` -or :class:`HTTPRedirect` as the -:attr:`request.handler`. - -.. _redirectingpost: - -Redirecting POST -================ - -When you GET a resource and are redirected by the server to another Location, -there's generally no problem since GET is both a "safe method" (there should -be no side-effects) and an "idempotent method" (multiple calls are no different -than a single call). - -POST, however, is neither safe nor idempotent--if you -charge a credit card, you don't want to be charged twice by a redirect! - -For this reason, *none* of the 3xx responses permit a user-agent (browser) to -resubmit a POST on redirection without first confirming the action with the -user: - -===== ================================= =========== -300 Multiple Choices Confirm with the user -301 Moved Permanently Confirm with the user -302 Found (Object moved temporarily) Confirm with the user -303 See Other GET the new URI--no confirmation -304 Not modified (for conditional GET only--POST should not raise this error) -305 Use Proxy Confirm with the user -307 Temporary Redirect Confirm with the user -===== ================================= =========== - -However, browsers have historically implemented these restrictions poorly; -in particular, many browsers do not force the user to confirm 301, 302 -or 307 when redirecting POST. For this reason, CherryPy defaults to 303, -which most user-agents appear to have implemented correctly. Therefore, if -you raise HTTPRedirect for a POST request, the user-agent will most likely -attempt to GET the new URI (without asking for confirmation from the user). -We realize this is confusing for developers, but it's the safest thing we -could do. You are of course free to raise ``HTTPRedirect(uri, status=302)`` -or any other 3xx status if you know what you're doing, but given the -environment, we couldn't let any of those be the default. - -Custom Error Handling -===================== - -.. image:: /refman/cperrors.gif - -Anticipated HTTP responses --------------------------- - -The 'error_page' config namespace can be used to provide custom HTML output for -expected responses (like 404 Not Found). Supply a filename from which the -output will be read. The contents will be interpolated with the values -%(status)s, %(message)s, %(traceback)s, and %(version)s using plain old Python -`string formatting `_. - -:: - - _cp_config = { - 'error_page.404': os.path.join(localDir, "static/index.html") - } - - -Beginning in version 3.1, you may also provide a function or other callable as -an error_page entry. It will be passed the same status, message, traceback and -version arguments that are interpolated into templates:: - - def error_page_402(status, message, traceback, version): - return "Error %s - Well, I'm very sorry but you haven't paid!" % status - cherrypy.config.update({'error_page.402': error_page_402}) - -Also in 3.1, in addition to the numbered error codes, you may also supply -"error_page.default" to handle all codes which do not have their own error_page -entry. - - - -Unanticipated errors --------------------- - -CherryPy also has a generic error handling mechanism: whenever an unanticipated -error occurs in your code, it will call -:func:`Request.error_response` to -set the response status, headers, and body. By default, this is the same -output as -:class:`HTTPError(500) `. If you want to provide -some other behavior, you generally replace "request.error_response". - -Here is some sample code that shows how to display a custom error message and -send an e-mail containing the error:: - - from cherrypy import _cperror - - def handle_error(): - cherrypy.response.status = 500 - cherrypy.response.body = [ - "Sorry, an error occured" - ] - sendMail('error@domain.com', - 'Error in your web app', - _cperror.format_exc()) - - @cherrypy.config(**{'request.error_response': handle_error}) - class Root: - pass - -Note that you have to explicitly set -:attr:`response.body ` -and not simply return an error message as a result. -""" - -from cgi import escape as _escape -from sys import exc_info as _exc_info, version_info as _version_info -from traceback import format_exception as _format_exception - -try: - import six -except ImportError: - if _version_info[0] == 2: - _unicode_type = unicode - else: - _unicode_type = str -else: - _unicode_type = six.text_type - -from cherrypy._cpcompat import basestring, iteritems, ntob -from cherrypy._cpcompat import tonative, urljoin as _urljoin -from cherrypy.lib import httputil as _httputil - - -class CherryPyException(Exception): - - """A base class for CherryPy exceptions.""" - pass - - -class TimeoutError(CherryPyException): - - """Exception raised when Response.timed_out is detected.""" - pass - - -class InternalRedirect(CherryPyException): - - """Exception raised to switch to the handler for a different URL. - - This exception will redirect processing to another path within the site - (without informing the client). Provide the new path as an argument when - raising the exception. Provide any params in the querystring for the new - URL. - """ - - def __init__(self, path, query_string=""): - import cherrypy - self.request = cherrypy.serving.request - - self.query_string = query_string - if "?" in path: - # Separate any params included in the path - path, self.query_string = path.split("?", 1) - - # Note that urljoin will "do the right thing" whether url is: - # 1. a URL relative to root (e.g. "/dummy") - # 2. a URL relative to the current path - # Note that any query string will be discarded. - path = _urljoin(self.request.path_info, path) - - # Set a 'path' member attribute so that code which traps this - # error can have access to it. - self.path = path - - CherryPyException.__init__(self, path, self.query_string) - - -class HTTPRedirect(CherryPyException): - - """Exception raised when the request should be redirected. - - This exception will force a HTTP redirect to the URL or URL's you give it. - The new URL must be passed as the first argument to the Exception, - e.g., HTTPRedirect(newUrl). Multiple URLs are allowed in a list. - If a URL is absolute, it will be used as-is. If it is relative, it is - assumed to be relative to the current cherrypy.request.path_info. - - If one of the provided URL is a unicode object, it will be encoded - using the default encoding or the one passed in parameter. - - There are multiple types of redirect, from which you can select via the - ``status`` argument. If you do not provide a ``status`` arg, it defaults to - 303 (or 302 if responding with HTTP/1.0). - - Examples:: - - raise cherrypy.HTTPRedirect("") - raise cherrypy.HTTPRedirect("/abs/path", 307) - raise cherrypy.HTTPRedirect(["path1", "path2?a=1&b=2"], 301) - - See :ref:`redirectingpost` for additional caveats. - """ - - status = None - """The integer HTTP status code to emit.""" - - urls = None - """The list of URL's to emit.""" - - encoding = 'utf-8' - """The encoding when passed urls are not native strings""" - - def __init__(self, urls, status=None, encoding=None): - import cherrypy - request = cherrypy.serving.request - - if isinstance(urls, basestring): - urls = [urls] - - abs_urls = [] - for url in urls: - url = tonative(url, encoding or self.encoding) - - # Note that urljoin will "do the right thing" whether url is: - # 1. a complete URL with host (e.g. "http://www.example.com/test") - # 2. a URL relative to root (e.g. "/dummy") - # 3. a URL relative to the current path - # Note that any query string in cherrypy.request is discarded. - url = _urljoin(cherrypy.url(), url) - abs_urls.append(url) - self.urls = abs_urls - - # RFC 2616 indicates a 301 response code fits our goal; however, - # browser support for 301 is quite messy. Do 302/303 instead. See - # http://www.alanflavell.org.uk/www/post-redirect.html - if status is None: - if request.protocol >= (1, 1): - status = 303 - else: - status = 302 - else: - status = int(status) - if status < 300 or status > 399: - raise ValueError("status must be between 300 and 399.") - - self.status = status - CherryPyException.__init__(self, abs_urls, status) - - def set_response(self): - """Modify cherrypy.response status, headers, and body to represent - self. - - CherryPy uses this internally, but you can also use it to create an - HTTPRedirect object and set its output without *raising* the exception. - """ - import cherrypy - response = cherrypy.serving.response - response.status = status = self.status - - if status in (300, 301, 302, 303, 307): - response.headers['Content-Type'] = "text/html;charset=utf-8" - # "The ... URI SHOULD be given by the Location field - # in the response." - response.headers['Location'] = self.urls[0] - - # "Unless the request method was HEAD, the entity of the response - # SHOULD contain a short hypertext note with a hyperlink to the - # new URI(s)." - msg = { - 300: "This resource can be found at ", - 301: "This resource has permanently moved to ", - 302: "This resource resides temporarily at ", - 303: "This resource can be found at ", - 307: "This resource has moved temporarily to ", - }[status] - msg += '%s.' - from xml.sax import saxutils - msgs = [msg % (saxutils.quoteattr(u), u) for u in self.urls] - response.body = ntob("
\n".join(msgs), 'utf-8') - # Previous code may have set C-L, so we have to reset it - # (allow finalize to set it). - response.headers.pop('Content-Length', None) - elif status == 304: - # Not Modified. - # "The response MUST include the following header fields: - # Date, unless its omission is required by section 14.18.1" - # The "Date" header should have been set in Response.__init__ - - # "...the response SHOULD NOT include other entity-headers." - for key in ('Allow', 'Content-Encoding', 'Content-Language', - 'Content-Length', 'Content-Location', 'Content-MD5', - 'Content-Range', 'Content-Type', 'Expires', - 'Last-Modified'): - if key in response.headers: - del response.headers[key] - - # "The 304 response MUST NOT contain a message-body." - response.body = None - # Previous code may have set C-L, so we have to reset it. - response.headers.pop('Content-Length', None) - elif status == 305: - # Use Proxy. - # self.urls[0] should be the URI of the proxy. - response.headers['Location'] = ntob(self.urls[0], 'utf-8') - response.body = None - # Previous code may have set C-L, so we have to reset it. - response.headers.pop('Content-Length', None) - else: - raise ValueError("The %s status code is unknown." % status) - - def __call__(self): - """Use this exception as a request.handler (raise self).""" - raise self - - -def clean_headers(status): - """Remove any headers which should not apply to an error response.""" - import cherrypy - - response = cherrypy.serving.response - - # Remove headers which applied to the original content, - # but do not apply to the error page. - respheaders = response.headers - for key in ["Accept-Ranges", "Age", "ETag", "Location", "Retry-After", - "Vary", "Content-Encoding", "Content-Length", "Expires", - "Content-Location", "Content-MD5", "Last-Modified"]: - if key in respheaders: - del respheaders[key] - - if status != 416: - # A server sending a response with status code 416 (Requested - # range not satisfiable) SHOULD include a Content-Range field - # with a byte-range-resp-spec of "*". The instance-length - # specifies the current length of the selected resource. - # A response with status code 206 (Partial Content) MUST NOT - # include a Content-Range field with a byte-range- resp-spec of "*". - if "Content-Range" in respheaders: - del respheaders["Content-Range"] - - -class HTTPError(CherryPyException): - - """Exception used to return an HTTP error code (4xx-5xx) to the client. - - This exception can be used to automatically send a response using a - http status code, with an appropriate error page. It takes an optional - ``status`` argument (which must be between 400 and 599); it defaults to 500 - ("Internal Server Error"). It also takes an optional ``message`` argument, - which will be returned in the response body. See - `RFC2616 `_ - for a complete list of available error codes and when to use them. - - Examples:: - - raise cherrypy.HTTPError(403) - raise cherrypy.HTTPError( - "403 Forbidden", "You are not allowed to access this resource.") - """ - - status = None - """The HTTP status code. May be of type int or str (with a Reason-Phrase). - """ - - code = None - """The integer HTTP status code.""" - - reason = None - """The HTTP Reason-Phrase string.""" - - def __init__(self, status=500, message=None): - self.status = status - try: - self.code, self.reason, defaultmsg = _httputil.valid_status(status) - except ValueError: - raise self.__class__(500, _exc_info()[1].args[0]) - - if self.code < 400 or self.code > 599: - raise ValueError("status must be between 400 and 599.") - - # See http://www.python.org/dev/peps/pep-0352/ - # self.message = message - self._message = message or defaultmsg - CherryPyException.__init__(self, status, message) - - def set_response(self): - """Modify cherrypy.response status, headers, and body to represent - self. - - CherryPy uses this internally, but you can also use it to create an - HTTPError object and set its output without *raising* the exception. - """ - import cherrypy - - response = cherrypy.serving.response - - clean_headers(self.code) - - # In all cases, finalize will be called after this method, - # so don't bother cleaning up response values here. - response.status = self.status - tb = None - if cherrypy.serving.request.show_tracebacks: - tb = format_exc() - - response.headers.pop('Content-Length', None) - - content = self.get_error_page(self.status, traceback=tb, - message=self._message) - response.body = content - - _be_ie_unfriendly(self.code) - - def get_error_page(self, *args, **kwargs): - return get_error_page(*args, **kwargs) - - def __call__(self): - """Use this exception as a request.handler (raise self).""" - raise self - - -class NotFound(HTTPError): - - """Exception raised when a URL could not be mapped to any handler (404). - - This is equivalent to raising - :class:`HTTPError("404 Not Found") `. - """ - - def __init__(self, path=None): - if path is None: - import cherrypy - request = cherrypy.serving.request - path = request.script_name + request.path_info - self.args = (path,) - HTTPError.__init__(self, 404, "The path '%s' was not found." % path) - - -_HTTPErrorTemplate = ''' - - - - %(status)s - - - -

%(status)s

-

%(message)s

- 
%(traceback)s
-
- - Powered by CherryPy %(version)s - -
- - -''' - - -def get_error_page(status, **kwargs): - """Return an HTML page, containing a pretty error response. - - status should be an int or a str. - kwargs will be interpolated into the page template. - """ - import cherrypy - - try: - code, reason, message = _httputil.valid_status(status) - except ValueError: - raise cherrypy.HTTPError(500, _exc_info()[1].args[0]) - - # We can't use setdefault here, because some - # callers send None for kwarg values. - if kwargs.get('status') is None: - kwargs['status'] = "%s %s" % (code, reason) - if kwargs.get('message') is None: - kwargs['message'] = message - if kwargs.get('traceback') is None: - kwargs['traceback'] = '' - if kwargs.get('version') is None: - kwargs['version'] = cherrypy.__version__ - - for k, v in iteritems(kwargs): - if v is None: - kwargs[k] = "" - else: - kwargs[k] = _escape(kwargs[k]) - - # Use a custom template or callable for the error page? - pages = cherrypy.serving.request.error_page - error_page = pages.get(code) or pages.get('default') - - # Default template, can be overridden below. - template = _HTTPErrorTemplate - if error_page: - try: - if hasattr(error_page, '__call__'): - # The caller function may be setting headers manually, - # so we delegate to it completely. We may be returning - # an iterator as well as a string here. - # - # We *must* make sure any content is not unicode. - result = error_page(**kwargs) - if cherrypy.lib.is_iterator(result): - from cherrypy.lib.encoding import UTF8StreamEncoder - return UTF8StreamEncoder(result) - elif isinstance(result, _unicode_type): - return result.encode('utf-8') - else: - if not isinstance(result, bytes): - raise ValueError('error page function did not ' - 'return a bytestring, unicode type or an ' - 'iterator - returned object of type %s.' - % (type(result).__name__)) - return result - else: - # Load the template from this path. - template = tonative(open(error_page, 'rb').read()) - except: - e = _format_exception(*_exc_info())[-1] - m = kwargs['message'] - if m: - m += "
" - m += "In addition, the custom error page failed:\n
%s" % e - kwargs['message'] = m - - response = cherrypy.serving.response - response.headers['Content-Type'] = "text/html;charset=utf-8" - result = template % kwargs - return result.encode('utf-8') - - - -_ie_friendly_error_sizes = { - 400: 512, 403: 256, 404: 512, 405: 256, - 406: 512, 408: 512, 409: 512, 410: 256, - 500: 512, 501: 512, 505: 512, -} - - -def _be_ie_unfriendly(status): - import cherrypy - response = cherrypy.serving.response - - # For some statuses, Internet Explorer 5+ shows "friendly error - # messages" instead of our response.body if the body is smaller - # than a given size. Fix this by returning a body over that size - # (by adding whitespace). - # See http://support.microsoft.com/kb/q218155/ - s = _ie_friendly_error_sizes.get(status, 0) - if s: - s += 1 - # Since we are issuing an HTTP error status, we assume that - # the entity is short, and we should just collapse it. - content = response.collapse_body() - l = len(content) - if l and l < s: - # IN ADDITION: the response must be written to IE - # in one chunk or it will still get replaced! Bah. - content = content + (ntob(" ") * (s - l)) - response.body = content - response.headers['Content-Length'] = str(len(content)) - - -def format_exc(exc=None): - """Return exc (or sys.exc_info if None), formatted.""" - try: - if exc is None: - exc = _exc_info() - if exc == (None, None, None): - return "" - import traceback - return "".join(traceback.format_exception(*exc)) - finally: - del exc - - -def bare_error(extrabody=None): - """Produce status, headers, body for a critical error. - - Returns a triple without calling any other questionable functions, - so it should be as error-free as possible. Call it from an HTTP server - if you get errors outside of the request. - - If extrabody is None, a friendly but rather unhelpful error message - is set in the body. If extrabody is a string, it will be appended - as-is to the body. - """ - - # The whole point of this function is to be a last line-of-defense - # in handling errors. That is, it must not raise any errors itself; - # it cannot be allowed to fail. Therefore, don't add to it! - # In particular, don't call any other CP functions. - - body = ntob("Unrecoverable error in the server.") - if extrabody is not None: - if not isinstance(extrabody, bytes): - extrabody = extrabody.encode('utf-8') - body += ntob("\n") + extrabody - - return (ntob("500 Internal Server Error"), - [(ntob('Content-Type'), ntob('text/plain')), - (ntob('Content-Length'), ntob(str(len(body)), 'ISO-8859-1'))], - [body]) diff --git a/lib/cherrypy/_cplogging.py b/lib/cherrypy/_cplogging.py deleted file mode 100644 index fc8c16b8433..00000000000 --- a/lib/cherrypy/_cplogging.py +++ /dev/null @@ -1,472 +0,0 @@ -""" -Simple config -============= - -Although CherryPy uses the :mod:`Python logging module `, it does so -behind the scenes so that simple logging is simple, but complicated logging -is still possible. "Simple" logging means that you can log to the screen -(i.e. console/stdout) or to a file, and that you can easily have separate -error and access log files. - -Here are the simplified logging settings. You use these by adding lines to -your config file or dict. You should set these at either the global level or -per application (see next), but generally not both. - - * ``log.screen``: Set this to True to have both "error" and "access" messages - printed to stdout. - * ``log.access_file``: Set this to an absolute filename where you want - "access" messages written. - * ``log.error_file``: Set this to an absolute filename where you want "error" - messages written. - -Many events are automatically logged; to log your own application events, call -:func:`cherrypy.log`. - -Architecture -============ - -Separate scopes ---------------- - -CherryPy provides log managers at both the global and application layers. -This means you can have one set of logging rules for your entire site, -and another set of rules specific to each application. The global log -manager is found at :func:`cherrypy.log`, and the log manager for each -application is found at :attr:`app.log`. -If you're inside a request, the latter is reachable from -``cherrypy.request.app.log``; if you're outside a request, you'll have to -obtain a reference to the ``app``: either the return value of -:func:`tree.mount()` or, if you used -:func:`quickstart()` instead, via -``cherrypy.tree.apps['/']``. - -By default, the global logs are named "cherrypy.error" and "cherrypy.access", -and the application logs are named "cherrypy.error.2378745" and -"cherrypy.access.2378745" (the number is the id of the Application object). -This means that the application logs "bubble up" to the site logs, so if your -application has no log handlers, the site-level handlers will still log the -messages. - -Errors vs. Access ------------------ - -Each log manager handles both "access" messages (one per HTTP request) and -"error" messages (everything else). Note that the "error" log is not just for -errors! The format of access messages is highly formalized, but the error log -isn't--it receives messages from a variety of sources (including full error -tracebacks, if enabled). - -If you are logging the access log and error log to the same source, then there -is a possibility that a specially crafted error message may replicate an access -log message as described in CWE-117. In this case it is the application -developer's responsibility to manually escape data before using CherryPy's log() -functionality, or they may create an application that is vulnerable to CWE-117. -This would be achieved by using a custom handler escape any special characters, -and attached as described below. - -Custom Handlers -=============== - -The simple settings above work by manipulating Python's standard :mod:`logging` -module. So when you need something more complex, the full power of the standard -module is yours to exploit. You can borrow or create custom handlers, formats, -filters, and much more. Here's an example that skips the standard FileHandler -and uses a RotatingFileHandler instead: - -:: - - #python - log = app.log - - # Remove the default FileHandlers if present. - log.error_file = "" - log.access_file = "" - - maxBytes = getattr(log, "rot_maxBytes", 10000000) - backupCount = getattr(log, "rot_backupCount", 1000) - - # Make a new RotatingFileHandler for the error log. - fname = getattr(log, "rot_error_file", "error.log") - h = handlers.RotatingFileHandler(fname, 'a', maxBytes, backupCount) - h.setLevel(DEBUG) - h.setFormatter(_cplogging.logfmt) - log.error_log.addHandler(h) - - # Make a new RotatingFileHandler for the access log. - fname = getattr(log, "rot_access_file", "access.log") - h = handlers.RotatingFileHandler(fname, 'a', maxBytes, backupCount) - h.setLevel(DEBUG) - h.setFormatter(_cplogging.logfmt) - log.access_log.addHandler(h) - - -The ``rot_*`` attributes are pulled straight from the application log object. -Since "log.*" config entries simply set attributes on the log object, you can -add custom attributes to your heart's content. Note that these handlers are -used ''instead'' of the default, simple handlers outlined above (so don't set -the "log.error_file" config entry, for example). -""" - -import datetime -import logging -# Silence the no-handlers "warning" (stderr write!) in stdlib logging -logging.Logger.manager.emittedNoHandlerWarning = 1 -logfmt = logging.Formatter("%(message)s") -import os -import sys - -IS_PY3 = sys.version_info[0] == 3 -try: - import six -except ImportError: - if IS_PY3: - _unicode_type = str - else: - _unicode_type = unicode -else: - _unicode_type = six.text_type - - -import cherrypy -from cherrypy import _cperror -from cherrypy._cpcompat import ntob - - -class NullHandler(logging.Handler): - - """A no-op logging handler to silence the logging.lastResort handler.""" - - def handle(self, record): - pass - - def emit(self, record): - pass - - def createLock(self): - self.lock = None - - -class LogManager(object): - - """An object to assist both simple and advanced logging. - - ``cherrypy.log`` is an instance of this class. - """ - - appid = None - """The id() of the Application object which owns this log manager. If this - is a global log manager, appid is None.""" - - error_log = None - """The actual :class:`logging.Logger` instance for error messages.""" - - access_log = None - """The actual :class:`logging.Logger` instance for access messages.""" - - access_log_format = ( - '{h} {l} {u} {t} "{r}" {s} {b} "{f}" "{a}"' - if IS_PY3 else - '%(h)s %(l)s %(u)s %(t)s "%(r)s" %(s)s %(b)s "%(f)s" "%(a)s"' - ) - - logger_root = None - """The "top-level" logger name. - - This string will be used as the first segment in the Logger names. - The default is "cherrypy", for example, in which case the Logger names - will be of the form:: - - cherrypy.error. - cherrypy.access. - """ - - def __init__(self, appid=None, logger_root="cherrypy"): - self.logger_root = logger_root - self.appid = appid - if appid is None: - self.error_log = logging.getLogger("%s.error" % logger_root) - self.access_log = logging.getLogger("%s.access" % logger_root) - else: - self.error_log = logging.getLogger( - "%s.error.%s" % (logger_root, appid)) - self.access_log = logging.getLogger( - "%s.access.%s" % (logger_root, appid)) - self.error_log.setLevel(logging.INFO) - self.access_log.setLevel(logging.INFO) - - # Silence the no-handlers "warning" (stderr write!) in stdlib logging - self.error_log.addHandler(NullHandler()) - self.access_log.addHandler(NullHandler()) - - cherrypy.engine.subscribe('graceful', self.reopen_files) - - def reopen_files(self): - """Close and reopen all file handlers.""" - for log in (self.error_log, self.access_log): - for h in log.handlers: - if isinstance(h, logging.FileHandler): - h.acquire() - h.stream.close() - h.stream = open(h.baseFilename, h.mode) - h.release() - - def error(self, msg='', context='', severity=logging.INFO, - traceback=False): - """Write the given ``msg`` to the error log. - - This is not just for errors! Applications may call this at any time - to log application-specific information. - - If ``traceback`` is True, the traceback of the current exception - (if any) will be appended to ``msg``. - """ - exc_info = None - if traceback: - exc_info = _cperror._exc_info() - - self.error_log.log(severity, ' '.join((self.time(), context, msg)), exc_info=exc_info) - - def __call__(self, *args, **kwargs): - """An alias for ``error``.""" - return self.error(*args, **kwargs) - - def access(self): - """Write to the access log (in Apache/NCSA Combined Log format). - - See the - `apache documentation `_ - for format details. - - CherryPy calls this automatically for you. Note there are no arguments; - it collects the data itself from - :class:`cherrypy.request`. - - Like Apache started doing in 2.0.46, non-printable and other special - characters in %r (and we expand that to all parts) are escaped using - \\xhh sequences, where hh stands for the hexadecimal representation - of the raw byte. Exceptions from this rule are " and \\, which are - escaped by prepending a backslash, and all whitespace characters, - which are written in their C-style notation (\\n, \\t, etc). - """ - request = cherrypy.serving.request - remote = request.remote - response = cherrypy.serving.response - outheaders = response.headers - inheaders = request.headers - if response.output_status is None: - status = "-" - else: - status = response.output_status.split(ntob(" "), 1)[0] - if IS_PY3: - status = status.decode('ISO-8859-1') - - atoms = {'h': remote.name or remote.ip, - 'l': '-', - 'u': getattr(request, "login", None) or "-", - 't': self.time(), - 'r': request.request_line, - 's': status, - 'b': dict.get(outheaders, 'Content-Length', '') or "-", - 'f': dict.get(inheaders, 'Referer', ''), - 'a': dict.get(inheaders, 'User-Agent', ''), - 'o': dict.get(inheaders, 'Host', '-'), - } - if IS_PY3: - for k, v in atoms.items(): - if not isinstance(v, str): - v = str(v) - v = v.replace('"', '\\"').encode('utf8') - # Fortunately, repr(str) escapes unprintable chars, \n, \t, etc - # and backslash for us. All we have to do is strip the quotes. - v = repr(v)[2:-1] - - # in python 3.0 the repr of bytes (as returned by encode) - # uses double \'s. But then the logger escapes them yet, again - # resulting in quadruple slashes. Remove the extra one here. - v = v.replace('\\\\', '\\') - - # Escape double-quote. - atoms[k] = v - - try: - self.access_log.log( - logging.INFO, self.access_log_format.format(**atoms)) - except: - self(traceback=True) - else: - for k, v in atoms.items(): - if isinstance(v, _unicode_type): - v = v.encode('utf8') - elif not isinstance(v, str): - v = str(v) - # Fortunately, repr(str) escapes unprintable chars, \n, \t, etc - # and backslash for us. All we have to do is strip the quotes. - v = repr(v)[1:-1] - # Escape double-quote. - atoms[k] = v.replace('"', '\\"') - - try: - self.access_log.log( - logging.INFO, self.access_log_format % atoms) - except: - self(traceback=True) - - def time(self): - """Return now() in Apache Common Log Format (no timezone).""" - now = datetime.datetime.now() - monthnames = ['jan', 'feb', 'mar', 'apr', 'may', 'jun', - 'jul', 'aug', 'sep', 'oct', 'nov', 'dec'] - month = monthnames[now.month - 1].capitalize() - return ('[%02d/%s/%04d:%02d:%02d:%02d]' % - (now.day, month, now.year, now.hour, now.minute, now.second)) - - def _get_builtin_handler(self, log, key): - for h in log.handlers: - if getattr(h, "_cpbuiltin", None) == key: - return h - - # ------------------------- Screen handlers ------------------------- # - def _set_screen_handler(self, log, enable, stream=None): - h = self._get_builtin_handler(log, "screen") - if enable: - if not h: - if stream is None: - stream = sys.stderr - h = logging.StreamHandler(stream) - h.setFormatter(logfmt) - h._cpbuiltin = "screen" - log.addHandler(h) - elif h: - log.handlers.remove(h) - - def _get_screen(self): - h = self._get_builtin_handler - has_h = h(self.error_log, "screen") or h(self.access_log, "screen") - return bool(has_h) - - def _set_screen(self, newvalue): - self._set_screen_handler(self.error_log, newvalue, stream=sys.stderr) - self._set_screen_handler(self.access_log, newvalue, stream=sys.stdout) - screen = property(_get_screen, _set_screen, - doc="""Turn stderr/stdout logging on or off. - - If you set this to True, it'll add the appropriate StreamHandler for - you. If you set it to False, it will remove the handler. - """) - - # -------------------------- File handlers -------------------------- # - - def _add_builtin_file_handler(self, log, fname): - h = logging.FileHandler(fname) - h.setFormatter(logfmt) - h._cpbuiltin = "file" - log.addHandler(h) - - def _set_file_handler(self, log, filename): - h = self._get_builtin_handler(log, "file") - if filename: - if h: - if h.baseFilename != os.path.abspath(filename): - h.close() - log.handlers.remove(h) - self._add_builtin_file_handler(log, filename) - else: - self._add_builtin_file_handler(log, filename) - else: - if h: - h.close() - log.handlers.remove(h) - - def _get_error_file(self): - h = self._get_builtin_handler(self.error_log, "file") - if h: - return h.baseFilename - return '' - - def _set_error_file(self, newvalue): - self._set_file_handler(self.error_log, newvalue) - error_file = property(_get_error_file, _set_error_file, - doc="""The filename for self.error_log. - - If you set this to a string, it'll add the appropriate FileHandler for - you. If you set it to ``None`` or ``''``, it will remove the handler. - """) - - def _get_access_file(self): - h = self._get_builtin_handler(self.access_log, "file") - if h: - return h.baseFilename - return '' - - def _set_access_file(self, newvalue): - self._set_file_handler(self.access_log, newvalue) - access_file = property(_get_access_file, _set_access_file, - doc="""The filename for self.access_log. - - If you set this to a string, it'll add the appropriate FileHandler for - you. If you set it to ``None`` or ``''``, it will remove the handler. - """) - - # ------------------------- WSGI handlers ------------------------- # - - def _set_wsgi_handler(self, log, enable): - h = self._get_builtin_handler(log, "wsgi") - if enable: - if not h: - h = WSGIErrorHandler() - h.setFormatter(logfmt) - h._cpbuiltin = "wsgi" - log.addHandler(h) - elif h: - log.handlers.remove(h) - - def _get_wsgi(self): - return bool(self._get_builtin_handler(self.error_log, "wsgi")) - - def _set_wsgi(self, newvalue): - self._set_wsgi_handler(self.error_log, newvalue) - wsgi = property(_get_wsgi, _set_wsgi, - doc="""Write errors to wsgi.errors. - - If you set this to True, it'll add the appropriate - :class:`WSGIErrorHandler` for you - (which writes errors to ``wsgi.errors``). - If you set it to False, it will remove the handler. - """) - - -class WSGIErrorHandler(logging.Handler): - - "A handler class which writes logging records to environ['wsgi.errors']." - - def flush(self): - """Flushes the stream.""" - try: - stream = cherrypy.serving.request.wsgi_environ.get('wsgi.errors') - except (AttributeError, KeyError): - pass - else: - stream.flush() - - def emit(self, record): - """Emit a record.""" - try: - stream = cherrypy.serving.request.wsgi_environ.get('wsgi.errors') - except (AttributeError, KeyError): - pass - else: - try: - msg = self.format(record) - fs = "%s\n" - import types - # if no unicode support... - if not hasattr(types, "UnicodeType"): - stream.write(fs % msg) - else: - try: - stream.write(fs % msg) - except UnicodeError: - stream.write(fs % msg.encode("UTF-8")) - self.flush() - except: - self.handleError(record) diff --git a/lib/cherrypy/_cpmodpy.py b/lib/cherrypy/_cpmodpy.py deleted file mode 100644 index 82f6ab3ba17..00000000000 --- a/lib/cherrypy/_cpmodpy.py +++ /dev/null @@ -1,354 +0,0 @@ -"""Native adapter for serving CherryPy via mod_python - -Basic usage: - -########################################## -# Application in a module called myapp.py -########################################## - -import cherrypy - -class Root: - @cherrypy.expose - def index(self): - return 'Hi there, Ho there, Hey there' - - -# We will use this method from the mod_python configuration -# as the entry point to our application -def setup_server(): - cherrypy.tree.mount(Root()) - cherrypy.config.update({'environment': 'production', - 'log.screen': False, - 'show_tracebacks': False}) - -########################################## -# mod_python settings for apache2 -# This should reside in your httpd.conf -# or a file that will be loaded at -# apache startup -########################################## - -# Start -DocumentRoot "/" -Listen 8080 -LoadModule python_module /usr/lib/apache2/modules/mod_python.so - - - PythonPath "sys.path+['/path/to/my/application']" - SetHandler python-program - PythonHandler cherrypy._cpmodpy::handler - PythonOption cherrypy.setup myapp::setup_server - PythonDebug On - -# End - -The actual path to your mod_python.so is dependent on your -environment. In this case we suppose a global mod_python -installation on a Linux distribution such as Ubuntu. - -We do set the PythonPath configuration setting so that -your application can be found by from the user running -the apache2 instance. Of course if your application -resides in the global site-package this won't be needed. - -Then restart apache2 and access http://127.0.0.1:8080 -""" - -import logging -import sys -import io - -import cherrypy -from cherrypy._cpcompat import copyitems, ntob -from cherrypy._cperror import format_exc, bare_error -from cherrypy.lib import httputil - - -# ------------------------------ Request-handling - - -def setup(req): - from mod_python import apache - - # Run any setup functions defined by a "PythonOption cherrypy.setup" - # directive. - options = req.get_options() - if 'cherrypy.setup' in options: - for function in options['cherrypy.setup'].split(): - atoms = function.split('::', 1) - if len(atoms) == 1: - mod = __import__(atoms[0], globals(), locals()) - else: - modname, fname = atoms - mod = __import__(modname, globals(), locals(), [fname]) - func = getattr(mod, fname) - func() - - cherrypy.config.update({'log.screen': False, - "tools.ignore_headers.on": True, - "tools.ignore_headers.headers": ['Range'], - }) - - engine = cherrypy.engine - if hasattr(engine, "signal_handler"): - engine.signal_handler.unsubscribe() - if hasattr(engine, "console_control_handler"): - engine.console_control_handler.unsubscribe() - engine.autoreload.unsubscribe() - cherrypy.server.unsubscribe() - - def _log(msg, level): - newlevel = apache.APLOG_ERR - if logging.DEBUG >= level: - newlevel = apache.APLOG_DEBUG - elif logging.INFO >= level: - newlevel = apache.APLOG_INFO - elif logging.WARNING >= level: - newlevel = apache.APLOG_WARNING - # On Windows, req.server is required or the msg will vanish. See - # http://www.modpython.org/pipermail/mod_python/2003-October/014291.html - # Also, "When server is not specified...LogLevel does not apply..." - apache.log_error(msg, newlevel, req.server) - engine.subscribe('log', _log) - - engine.start() - - def cherrypy_cleanup(data): - engine.exit() - try: - # apache.register_cleanup wasn't available until 3.1.4. - apache.register_cleanup(cherrypy_cleanup) - except AttributeError: - req.server.register_cleanup(req, cherrypy_cleanup) - - -class _ReadOnlyRequest: - expose = ('read', 'readline', 'readlines') - - def __init__(self, req): - for method in self.expose: - self.__dict__[method] = getattr(req, method) - - -recursive = False - -_isSetUp = False - - -def handler(req): - from mod_python import apache - try: - global _isSetUp - if not _isSetUp: - setup(req) - _isSetUp = True - - # Obtain a Request object from CherryPy - local = req.connection.local_addr - local = httputil.Host( - local[0], local[1], req.connection.local_host or "") - remote = req.connection.remote_addr - remote = httputil.Host( - remote[0], remote[1], req.connection.remote_host or "") - - scheme = req.parsed_uri[0] or 'http' - req.get_basic_auth_pw() - - try: - # apache.mpm_query only became available in mod_python 3.1 - q = apache.mpm_query - threaded = q(apache.AP_MPMQ_IS_THREADED) - forked = q(apache.AP_MPMQ_IS_FORKED) - except AttributeError: - bad_value = ("You must provide a PythonOption '%s', " - "either 'on' or 'off', when running a version " - "of mod_python < 3.1") - - threaded = options.get('multithread', '').lower() - if threaded == 'on': - threaded = True - elif threaded == 'off': - threaded = False - else: - raise ValueError(bad_value % "multithread") - - forked = options.get('multiprocess', '').lower() - if forked == 'on': - forked = True - elif forked == 'off': - forked = False - else: - raise ValueError(bad_value % "multiprocess") - - sn = cherrypy.tree.script_name(req.uri or "/") - if sn is None: - send_response(req, '404 Not Found', [], '') - else: - app = cherrypy.tree.apps[sn] - method = req.method - path = req.uri - qs = req.args or "" - reqproto = req.protocol - headers = copyitems(req.headers_in) - rfile = _ReadOnlyRequest(req) - prev = None - - try: - redirections = [] - while True: - request, response = app.get_serving(local, remote, scheme, - "HTTP/1.1") - request.login = req.user - request.multithread = bool(threaded) - request.multiprocess = bool(forked) - request.app = app - request.prev = prev - - # Run the CherryPy Request object and obtain the response - try: - request.run(method, path, qs, reqproto, headers, rfile) - break - except cherrypy.InternalRedirect: - ir = sys.exc_info()[1] - app.release_serving() - prev = request - - if not recursive: - if ir.path in redirections: - raise RuntimeError( - "InternalRedirector visited the same URL " - "twice: %r" % ir.path) - else: - # Add the *previous* path_info + qs to - # redirections. - if qs: - qs = "?" + qs - redirections.append(sn + path + qs) - - # Munge environment and try again. - method = "GET" - path = ir.path - qs = ir.query_string - rfile = io.BytesIO() - - send_response( - req, response.output_status, response.header_list, - response.body, response.stream) - finally: - app.release_serving() - except: - tb = format_exc() - cherrypy.log(tb, 'MOD_PYTHON', severity=logging.ERROR) - s, h, b = bare_error() - send_response(req, s, h, b) - return apache.OK - - -def send_response(req, status, headers, body, stream=False): - # Set response status - req.status = int(status[:3]) - - # Set response headers - req.content_type = "text/plain" - for header, value in headers: - if header.lower() == 'content-type': - req.content_type = value - continue - req.headers_out.add(header, value) - - if stream: - # Flush now so the status and headers are sent immediately. - req.flush() - - # Set response body - if isinstance(body, basestring): - req.write(body) - else: - for seg in body: - req.write(seg) - - -# --------------- Startup tools for CherryPy + mod_python --------------- # -import os -import re -try: - import subprocess - - def popen(fullcmd): - p = subprocess.Popen(fullcmd, shell=True, - stdout=subprocess.PIPE, stderr=subprocess.STDOUT, - close_fds=True) - return p.stdout -except ImportError: - def popen(fullcmd): - pipein, pipeout = os.popen4(fullcmd) - return pipeout - - -def read_process(cmd, args=""): - fullcmd = "%s %s" % (cmd, args) - pipeout = popen(fullcmd) - try: - firstline = pipeout.readline() - cmd_not_found = re.search( - ntob("(not recognized|No such file|not found)"), - firstline, - re.IGNORECASE - ) - if cmd_not_found: - raise IOError('%s must be on your system path.' % cmd) - output = firstline + pipeout.read() - finally: - pipeout.close() - return output - - -class ModPythonServer(object): - - template = """ -# Apache2 server configuration file for running CherryPy with mod_python. - -DocumentRoot "/" -Listen %(port)s -LoadModule python_module modules/mod_python.so - - - SetHandler python-program - PythonHandler %(handler)s - PythonDebug On -%(opts)s - -""" - - def __init__(self, loc="/", port=80, opts=None, apache_path="apache", - handler="cherrypy._cpmodpy::handler"): - self.loc = loc - self.port = port - self.opts = opts - self.apache_path = apache_path - self.handler = handler - - def start(self): - opts = "".join([" PythonOption %s %s\n" % (k, v) - for k, v in self.opts]) - conf_data = self.template % {"port": self.port, - "loc": self.loc, - "opts": opts, - "handler": self.handler, - } - - mpconf = os.path.join(os.path.dirname(__file__), "cpmodpy.conf") - f = open(mpconf, 'wb') - try: - f.write(conf_data) - finally: - f.close() - - response = read_process(self.apache_path, "-k start -f %s" % mpconf) - self.ready = True - return response - - def stop(self): - os.popen("apache -k stop") - self.ready = False diff --git a/lib/cherrypy/_cpnative_server.py b/lib/cherrypy/_cpnative_server.py deleted file mode 100644 index 8edb89c1420..00000000000 --- a/lib/cherrypy/_cpnative_server.py +++ /dev/null @@ -1,154 +0,0 @@ -"""Native adapter for serving CherryPy via its builtin server.""" - -import logging -import sys -import io - -import cherrypy -from cherrypy._cperror import format_exc, bare_error -from cherrypy.lib import httputil -from cherrypy import wsgiserver - - -class NativeGateway(wsgiserver.Gateway): - - recursive = False - - def respond(self): - req = self.req - try: - # Obtain a Request object from CherryPy - local = req.server.bind_addr - local = httputil.Host(local[0], local[1], "") - remote = req.conn.remote_addr, req.conn.remote_port - remote = httputil.Host(remote[0], remote[1], "") - - scheme = req.scheme - sn = cherrypy.tree.script_name(req.uri or "/") - if sn is None: - self.send_response('404 Not Found', [], ['']) - else: - app = cherrypy.tree.apps[sn] - method = req.method - path = req.path - qs = req.qs or "" - headers = req.inheaders.items() - rfile = req.rfile - prev = None - - try: - redirections = [] - while True: - request, response = app.get_serving( - local, remote, scheme, "HTTP/1.1") - request.multithread = True - request.multiprocess = False - request.app = app - request.prev = prev - - # Run the CherryPy Request object and obtain the - # response - try: - request.run(method, path, qs, - req.request_protocol, headers, rfile) - break - except cherrypy.InternalRedirect: - ir = sys.exc_info()[1] - app.release_serving() - prev = request - - if not self.recursive: - if ir.path in redirections: - raise RuntimeError( - "InternalRedirector visited the same " - "URL twice: %r" % ir.path) - else: - # Add the *previous* path_info + qs to - # redirections. - if qs: - qs = "?" + qs - redirections.append(sn + path + qs) - - # Munge environment and try again. - method = "GET" - path = ir.path - qs = ir.query_string - rfile = io.BytesIO() - - self.send_response( - response.output_status, response.header_list, - response.body) - finally: - app.release_serving() - except: - tb = format_exc() - # print tb - cherrypy.log(tb, 'NATIVE_ADAPTER', severity=logging.ERROR) - s, h, b = bare_error() - self.send_response(s, h, b) - - def send_response(self, status, headers, body): - req = self.req - - # Set response status - req.status = str(status or "500 Server Error") - - # Set response headers - for header, value in headers: - req.outheaders.append((header, value)) - if (req.ready and not req.sent_headers): - req.sent_headers = True - req.send_headers() - - # Set response body - for seg in body: - req.write(seg) - - -class CPHTTPServer(wsgiserver.HTTPServer): - - """Wrapper for wsgiserver.HTTPServer. - - wsgiserver has been designed to not reference CherryPy in any way, - so that it can be used in other frameworks and applications. - Therefore, we wrap it here, so we can apply some attributes - from config -> cherrypy.server -> HTTPServer. - """ - - def __init__(self, server_adapter=cherrypy.server): - self.server_adapter = server_adapter - - server_name = (self.server_adapter.socket_host or - self.server_adapter.socket_file or - None) - - wsgiserver.HTTPServer.__init__( - self, server_adapter.bind_addr, NativeGateway, - minthreads=server_adapter.thread_pool, - maxthreads=server_adapter.thread_pool_max, - server_name=server_name) - - self.max_request_header_size = ( - self.server_adapter.max_request_header_size or 0) - self.max_request_body_size = ( - self.server_adapter.max_request_body_size or 0) - self.request_queue_size = self.server_adapter.socket_queue_size - self.timeout = self.server_adapter.socket_timeout - self.shutdown_timeout = self.server_adapter.shutdown_timeout - self.protocol = self.server_adapter.protocol_version - self.nodelay = self.server_adapter.nodelay - - ssl_module = self.server_adapter.ssl_module or 'pyopenssl' - if self.server_adapter.ssl_context: - adapter_class = wsgiserver.get_ssl_adapter_class(ssl_module) - self.ssl_adapter = adapter_class( - self.server_adapter.ssl_certificate, - self.server_adapter.ssl_private_key, - self.server_adapter.ssl_certificate_chain) - self.ssl_adapter.context = self.server_adapter.ssl_context - elif self.server_adapter.ssl_certificate: - adapter_class = wsgiserver.get_ssl_adapter_class(ssl_module) - self.ssl_adapter = adapter_class( - self.server_adapter.ssl_certificate, - self.server_adapter.ssl_private_key, - self.server_adapter.ssl_certificate_chain) diff --git a/lib/cherrypy/_cpreqbody.py b/lib/cherrypy/_cpreqbody.py deleted file mode 100644 index e2d342b2e5f..00000000000 --- a/lib/cherrypy/_cpreqbody.py +++ /dev/null @@ -1,1018 +0,0 @@ -"""Request body processing for CherryPy. - -.. versionadded:: 3.2 - -Application authors have complete control over the parsing of HTTP request -entities. In short, -:attr:`cherrypy.request.body` -is now always set to an instance of -:class:`RequestBody`, -and *that* class is a subclass of :class:`Entity`. - -When an HTTP request includes an entity body, it is often desirable to -provide that information to applications in a form other than the raw bytes. -Different content types demand different approaches. Examples: - - * For a GIF file, we want the raw bytes in a stream. - * An HTML form is better parsed into its component fields, and each text field - decoded from bytes to unicode. - * A JSON body should be deserialized into a Python dict or list. - -When the request contains a Content-Type header, the media type is used as a -key to look up a value in the -:attr:`request.body.processors` dict. -If the full media -type is not found, then the major type is tried; for example, if no processor -is found for the 'image/jpeg' type, then we look for a processor for the -'image' types altogether. If neither the full type nor the major type has a -matching processor, then a default processor is used -(:func:`default_proc`). For most -types, this means no processing is done, and the body is left unread as a -raw byte stream. Processors are configurable in an 'on_start_resource' hook. - -Some processors, especially those for the 'text' types, attempt to decode bytes -to unicode. If the Content-Type request header includes a 'charset' parameter, -this is used to decode the entity. Otherwise, one or more default charsets may -be attempted, although this decision is up to each processor. If a processor -successfully decodes an Entity or Part, it should set the -:attr:`charset` attribute -on the Entity or Part to the name of the successful charset, so that -applications can easily re-encode or transcode the value if they wish. - -If the Content-Type of the request entity is of major type 'multipart', then -the above parsing process, and possibly a decoding process, is performed for -each part. - -For both the full entity and multipart parts, a Content-Disposition header may -be used to fill :attr:`name` and -:attr:`filename` attributes on the -request.body or the Part. - -.. _custombodyprocessors: - -Custom Processors -================= - -You can add your own processors for any specific or major MIME type. Simply add -it to the :attr:`processors` dict in a -hook/tool that runs at ``on_start_resource`` or ``before_request_body``. -Here's the built-in JSON tool for an example:: - - def json_in(force=True, debug=False): - request = cherrypy.serving.request - def json_processor(entity): - \"""Read application/json data into request.json.\""" - if not entity.headers.get("Content-Length", ""): - raise cherrypy.HTTPError(411) - - body = entity.fp.read() - try: - request.json = json_decode(body) - except ValueError: - raise cherrypy.HTTPError(400, 'Invalid JSON document') - if force: - request.body.processors.clear() - request.body.default_proc = cherrypy.HTTPError( - 415, 'Expected an application/json content type') - request.body.processors['application/json'] = json_processor - -We begin by defining a new ``json_processor`` function to stick in the -``processors`` dictionary. All processor functions take a single argument, -the ``Entity`` instance they are to process. It will be called whenever a -request is received (for those URI's where the tool is turned on) which -has a ``Content-Type`` of "application/json". - -First, it checks for a valid ``Content-Length`` (raising 411 if not valid), -then reads the remaining bytes on the socket. The ``fp`` object knows its -own length, so it won't hang waiting for data that never arrives. It will -return when all data has been read. Then, we decode those bytes using -Python's built-in ``json`` module, and stick the decoded result onto -``request.json`` . If it cannot be decoded, we raise 400. - -If the "force" argument is True (the default), the ``Tool`` clears the -``processors`` dict so that request entities of other ``Content-Types`` -aren't parsed at all. Since there's no entry for those invalid MIME -types, the ``default_proc`` method of ``cherrypy.request.body`` is -called. But this does nothing by default (usually to provide the page -handler an opportunity to handle it.) -But in our case, we want to raise 415, so we replace -``request.body.default_proc`` -with the error (``HTTPError`` instances, when called, raise themselves). - -If we were defining a custom processor, we can do so without making a ``Tool``. -Just add the config entry:: - - request.body.processors = {'application/json': json_processor} - -Note that you can only replace the ``processors`` dict wholesale this way, -not update the existing one. -""" - -try: - from io import DEFAULT_BUFFER_SIZE -except ImportError: - DEFAULT_BUFFER_SIZE = 8192 -import re -import sys -import tempfile -try: - from urllib import unquote_plus -except ImportError: - def unquote_plus(bs): - """Bytes version of urllib.parse.unquote_plus.""" - bs = bs.replace(ntob('+'), ntob(' ')) - atoms = bs.split(ntob('%')) - for i in range(1, len(atoms)): - item = atoms[i] - try: - pct = int(item[:2], 16) - atoms[i] = bytes([pct]) + item[2:] - except ValueError: - pass - return ntob('').join(atoms) - -import cherrypy -from cherrypy._cpcompat import basestring, ntob, ntou -from cherrypy.lib import httputil - - -# ------------------------------- Processors -------------------------------- # - -def process_urlencoded(entity): - """Read application/x-www-form-urlencoded data into entity.params.""" - qs = entity.fp.read() - for charset in entity.attempt_charsets: - try: - params = {} - for aparam in qs.split(ntob('&')): - for pair in aparam.split(ntob(';')): - if not pair: - continue - - atoms = pair.split(ntob('='), 1) - if len(atoms) == 1: - atoms.append(ntob('')) - - key = unquote_plus(atoms[0]).decode(charset) - value = unquote_plus(atoms[1]).decode(charset) - - if key in params: - if not isinstance(params[key], list): - params[key] = [params[key]] - params[key].append(value) - else: - params[key] = value - except UnicodeDecodeError: - pass - else: - entity.charset = charset - break - else: - raise cherrypy.HTTPError( - 400, "The request entity could not be decoded. The following " - "charsets were attempted: %s" % repr(entity.attempt_charsets)) - - # Now that all values have been successfully parsed and decoded, - # apply them to the entity.params dict. - for key, value in params.items(): - if key in entity.params: - if not isinstance(entity.params[key], list): - entity.params[key] = [entity.params[key]] - entity.params[key].append(value) - else: - entity.params[key] = value - - -def process_multipart(entity): - """Read all multipart parts into entity.parts.""" - ib = "" - if 'boundary' in entity.content_type.params: - # http://tools.ietf.org/html/rfc2046#section-5.1.1 - # "The grammar for parameters on the Content-type field is such that it - # is often necessary to enclose the boundary parameter values in quotes - # on the Content-type line" - ib = entity.content_type.params['boundary'].strip('"') - - if not re.match("^[ -~]{0,200}[!-~]$", ib): - raise ValueError('Invalid boundary in multipart form: %r' % (ib,)) - - ib = ('--' + ib).encode('ascii') - - # Find the first marker - while True: - b = entity.readline() - if not b: - return - - b = b.strip() - if b == ib: - break - - # Read all parts - while True: - part = entity.part_class.from_fp(entity.fp, ib) - entity.parts.append(part) - part.process() - if part.fp.done: - break - - -def process_multipart_form_data(entity): - """Read all multipart/form-data parts into entity.parts or entity.params. - """ - process_multipart(entity) - - kept_parts = [] - for part in entity.parts: - if part.name is None: - kept_parts.append(part) - else: - if part.filename is None: - # It's a regular field - value = part.fullvalue() - else: - # It's a file upload. Retain the whole part so consumer code - # has access to its .file and .filename attributes. - value = part - - if part.name in entity.params: - if not isinstance(entity.params[part.name], list): - entity.params[part.name] = [entity.params[part.name]] - entity.params[part.name].append(value) - else: - entity.params[part.name] = value - - entity.parts = kept_parts - - -def _old_process_multipart(entity): - """The behavior of 3.2 and lower. Deprecated and will be changed in 3.3.""" - process_multipart(entity) - - params = entity.params - - for part in entity.parts: - if part.name is None: - key = ntou('parts') - else: - key = part.name - - if part.filename is None: - # It's a regular field - value = part.fullvalue() - else: - # It's a file upload. Retain the whole part so consumer code - # has access to its .file and .filename attributes. - value = part - - if key in params: - if not isinstance(params[key], list): - params[key] = [params[key]] - params[key].append(value) - else: - params[key] = value - - -# -------------------------------- Entities --------------------------------- # -class Entity(object): - - """An HTTP request body, or MIME multipart body. - - This class collects information about the HTTP request entity. When a - given entity is of MIME type "multipart", each part is parsed into its own - Entity instance, and the set of parts stored in - :attr:`entity.parts`. - - Between the ``before_request_body`` and ``before_handler`` tools, CherryPy - tries to process the request body (if any) by calling - :func:`request.body.process`. - This uses the ``content_type`` of the Entity to look up a suitable - processor in - :attr:`Entity.processors`, - a dict. - If a matching processor cannot be found for the complete Content-Type, - it tries again using the major type. For example, if a request with an - entity of type "image/jpeg" arrives, but no processor can be found for - that complete type, then one is sought for the major type "image". If a - processor is still not found, then the - :func:`default_proc` method - of the Entity is called (which does nothing by default; you can - override this too). - - CherryPy includes processors for the "application/x-www-form-urlencoded" - type, the "multipart/form-data" type, and the "multipart" major type. - CherryPy 3.2 processes these types almost exactly as older versions. - Parts are passed as arguments to the page handler using their - ``Content-Disposition.name`` if given, otherwise in a generic "parts" - argument. Each such part is either a string, or the - :class:`Part` itself if it's a file. (In this - case it will have ``file`` and ``filename`` attributes, or possibly a - ``value`` attribute). Each Part is itself a subclass of - Entity, and has its own ``process`` method and ``processors`` dict. - - There is a separate processor for the "multipart" major type which is more - flexible, and simply stores all multipart parts in - :attr:`request.body.parts`. You can - enable it with:: - - cherrypy.request.body.processors['multipart'] = _cpreqbody.process_multipart - - in an ``on_start_resource`` tool. - """ - - # http://tools.ietf.org/html/rfc2046#section-4.1.2: - # "The default character set, which must be assumed in the - # absence of a charset parameter, is US-ASCII." - # However, many browsers send data in utf-8 with no charset. - attempt_charsets = ['utf-8'] - """A list of strings, each of which should be a known encoding. - - When the Content-Type of the request body warrants it, each of the given - encodings will be tried in order. The first one to successfully decode the - entity without raising an error is stored as - :attr:`entity.charset`. This defaults - to ``['utf-8']`` (plus 'ISO-8859-1' for "text/\*" types, as required by - `HTTP/1.1 `_), - but ``['us-ascii', 'utf-8']`` for multipart parts. - """ - - charset = None - """The successful decoding; see "attempt_charsets" above.""" - - content_type = None - """The value of the Content-Type request header. - - If the Entity is part of a multipart payload, this will be the Content-Type - given in the MIME headers for this part. - """ - - default_content_type = 'application/x-www-form-urlencoded' - """This defines a default ``Content-Type`` to use if no Content-Type header - is given. The empty string is used for RequestBody, which results in the - request body not being read or parsed at all. This is by design; a missing - ``Content-Type`` header in the HTTP request entity is an error at best, - and a security hole at worst. For multipart parts, however, the MIME spec - declares that a part with no Content-Type defaults to "text/plain" - (see :class:`Part`). - """ - - filename = None - """The ``Content-Disposition.filename`` header, if available.""" - - fp = None - """The readable socket file object.""" - - headers = None - """A dict of request/multipart header names and values. - - This is a copy of the ``request.headers`` for the ``request.body``; - for multipart parts, it is the set of headers for that part. - """ - - length = None - """The value of the ``Content-Length`` header, if provided.""" - - name = None - """The "name" parameter of the ``Content-Disposition`` header, if any.""" - - params = None - """ - If the request Content-Type is 'application/x-www-form-urlencoded' or - multipart, this will be a dict of the params pulled from the entity - body; that is, it will be the portion of request.params that come - from the message body (sometimes called "POST params", although they - can be sent with various HTTP method verbs). This value is set between - the 'before_request_body' and 'before_handler' hooks (assuming that - process_request_body is True).""" - - processors = {'application/x-www-form-urlencoded': process_urlencoded, - 'multipart/form-data': process_multipart_form_data, - 'multipart': process_multipart, - } - """A dict of Content-Type names to processor methods.""" - - parts = None - """A list of Part instances if ``Content-Type`` is of major type - "multipart".""" - - part_class = None - """The class used for multipart parts. - - You can replace this with custom subclasses to alter the processing of - multipart parts. - """ - - def __init__(self, fp, headers, params=None, parts=None): - # Make an instance-specific copy of the class processors - # so Tools, etc. can replace them per-request. - self.processors = self.processors.copy() - - self.fp = fp - self.headers = headers - - if params is None: - params = {} - self.params = params - - if parts is None: - parts = [] - self.parts = parts - - # Content-Type - self.content_type = headers.elements('Content-Type') - if self.content_type: - self.content_type = self.content_type[0] - else: - self.content_type = httputil.HeaderElement.from_str( - self.default_content_type) - - # Copy the class 'attempt_charsets', prepending any Content-Type - # charset - dec = self.content_type.params.get("charset", None) - if dec: - self.attempt_charsets = [dec] + [c for c in self.attempt_charsets - if c != dec] - else: - self.attempt_charsets = self.attempt_charsets[:] - - # Length - self.length = None - clen = headers.get('Content-Length', None) - # If Transfer-Encoding is 'chunked', ignore any Content-Length. - if ( - clen is not None and - 'chunked' not in headers.get('Transfer-Encoding', '') - ): - try: - self.length = int(clen) - except ValueError: - pass - - # Content-Disposition - self.name = None - self.filename = None - disp = headers.elements('Content-Disposition') - if disp: - disp = disp[0] - if 'name' in disp.params: - self.name = disp.params['name'] - if self.name.startswith('"') and self.name.endswith('"'): - self.name = self.name[1:-1] - if 'filename' in disp.params: - self.filename = disp.params['filename'] - if ( - self.filename.startswith('"') and - self.filename.endswith('"') - ): - self.filename = self.filename[1:-1] - - # The 'type' attribute is deprecated in 3.2; remove it in 3.3. - type = property( - lambda self: self.content_type, - doc="A deprecated alias for " - ":attr:`content_type`." - ) - - def read(self, size=None, fp_out=None): - return self.fp.read(size, fp_out) - - def readline(self, size=None): - return self.fp.readline(size) - - def readlines(self, sizehint=None): - return self.fp.readlines(sizehint) - - def __iter__(self): - return self - - def __next__(self): - line = self.readline() - if not line: - raise StopIteration - return line - - def next(self): - return self.__next__() - - def read_into_file(self, fp_out=None): - """Read the request body into fp_out (or make_file() if None). - - Return fp_out. - """ - if fp_out is None: - fp_out = self.make_file() - self.read(fp_out=fp_out) - return fp_out - - def make_file(self): - """Return a file-like object into which the request body will be read. - - By default, this will return a TemporaryFile. Override as needed. - See also :attr:`cherrypy._cpreqbody.Part.maxrambytes`.""" - return tempfile.TemporaryFile() - - def fullvalue(self): - """Return this entity as a string, whether stored in a file or not.""" - if self.file: - # It was stored in a tempfile. Read it. - self.file.seek(0) - value = self.file.read() - self.file.seek(0) - else: - value = self.value - value = self.decode_entity(value) - return value - - def decode_entity(self , value): - """Return a given byte encoded value as a string""" - for charset in self.attempt_charsets: - try: - value = value.decode(charset) - except UnicodeDecodeError: - pass - else: - self.charset = charset - return value - else: - raise cherrypy.HTTPError( - 400, - "The request entity could not be decoded. The following " - "charsets were attempted: %s" % repr(self.attempt_charsets) - ) - - def process(self): - """Execute the best-match processor for the given media type.""" - proc = None - ct = self.content_type.value - try: - proc = self.processors[ct] - except KeyError: - toptype = ct.split('/', 1)[0] - try: - proc = self.processors[toptype] - except KeyError: - pass - if proc is None: - self.default_proc() - else: - proc(self) - - def default_proc(self): - """Called if a more-specific processor is not found for the - ``Content-Type``. - """ - # Leave the fp alone for someone else to read. This works fine - # for request.body, but the Part subclasses need to override this - # so they can move on to the next part. - pass - - -class Part(Entity): - - """A MIME part entity, part of a multipart entity.""" - - # "The default character set, which must be assumed in the absence of a - # charset parameter, is US-ASCII." - attempt_charsets = ['us-ascii', 'utf-8'] - """A list of strings, each of which should be a known encoding. - - When the Content-Type of the request body warrants it, each of the given - encodings will be tried in order. The first one to successfully decode the - entity without raising an error is stored as - :attr:`entity.charset`. This defaults - to ``['utf-8']`` (plus 'ISO-8859-1' for "text/\*" types, as required by - `HTTP/1.1 `_), - but ``['us-ascii', 'utf-8']`` for multipart parts. - """ - - boundary = None - """The MIME multipart boundary.""" - - default_content_type = 'text/plain' - """This defines a default ``Content-Type`` to use if no Content-Type header - is given. The empty string is used for RequestBody, which results in the - request body not being read or parsed at all. This is by design; a missing - ``Content-Type`` header in the HTTP request entity is an error at best, - and a security hole at worst. For multipart parts, however (this class), - the MIME spec declares that a part with no Content-Type defaults to - "text/plain". - """ - - # This is the default in stdlib cgi. We may want to increase it. - maxrambytes = 1000 - """The threshold of bytes after which point the ``Part`` will store - its data in a file (generated by - :func:`make_file`) - instead of a string. Defaults to 1000, just like the :mod:`cgi` - module in Python's standard library. - """ - - def __init__(self, fp, headers, boundary): - Entity.__init__(self, fp, headers) - self.boundary = boundary - self.file = None - self.value = None - - def from_fp(cls, fp, boundary): - headers = cls.read_headers(fp) - return cls(fp, headers, boundary) - from_fp = classmethod(from_fp) - - def read_headers(cls, fp): - headers = httputil.HeaderMap() - while True: - line = fp.readline() - if not line: - # No more data--illegal end of headers - raise EOFError("Illegal end of headers.") - - if line == ntob('\r\n'): - # Normal end of headers - break - if not line.endswith(ntob('\r\n')): - raise ValueError("MIME requires CRLF terminators: %r" % line) - - if line[0] in ntob(' \t'): - # It's a continuation line. - v = line.strip().decode('ISO-8859-1') - else: - k, v = line.split(ntob(":"), 1) - k = k.strip().decode('ISO-8859-1') - v = v.strip().decode('ISO-8859-1') - - existing = headers.get(k) - if existing: - v = ", ".join((existing, v)) - headers[k] = v - - return headers - read_headers = classmethod(read_headers) - - def read_lines_to_boundary(self, fp_out=None): - """Read bytes from self.fp and return or write them to a file. - - If the 'fp_out' argument is None (the default), all bytes read are - returned in a single byte string. - - If the 'fp_out' argument is not None, it must be a file-like - object that supports the 'write' method; all bytes read will be - written to the fp, and that fp is returned. - """ - endmarker = self.boundary + ntob("--") - delim = ntob("") - prev_lf = True - lines = [] - seen = 0 - while True: - line = self.fp.readline(1 << 16) - if not line: - raise EOFError("Illegal end of multipart body.") - if line.startswith(ntob("--")) and prev_lf: - strippedline = line.strip() - if strippedline == self.boundary: - break - if strippedline == endmarker: - self.fp.finish() - break - - line = delim + line - - if line.endswith(ntob("\r\n")): - delim = ntob("\r\n") - line = line[:-2] - prev_lf = True - elif line.endswith(ntob("\n")): - delim = ntob("\n") - line = line[:-1] - prev_lf = True - else: - delim = ntob("") - prev_lf = False - - if fp_out is None: - lines.append(line) - seen += len(line) - if seen > self.maxrambytes: - fp_out = self.make_file() - for line in lines: - fp_out.write(line) - else: - fp_out.write(line) - - if fp_out is None: - result = ntob('').join(lines) - return result - else: - fp_out.seek(0) - return fp_out - - def default_proc(self): - """Called if a more-specific processor is not found for the - ``Content-Type``. - """ - if self.filename: - # Always read into a file if a .filename was given. - self.file = self.read_into_file() - else: - result = self.read_lines_to_boundary() - if isinstance(result, basestring): - self.value = result - else: - self.file = result - - def read_into_file(self, fp_out=None): - """Read the request body into fp_out (or make_file() if None). - - Return fp_out. - """ - if fp_out is None: - fp_out = self.make_file() - self.read_lines_to_boundary(fp_out=fp_out) - return fp_out - -Entity.part_class = Part - -try: - inf = float('inf') -except ValueError: - # Python 2.4 and lower - class Infinity(object): - - def __cmp__(self, other): - return 1 - - def __sub__(self, other): - return self - inf = Infinity() - - -comma_separated_headers = [ - 'Accept', 'Accept-Charset', 'Accept-Encoding', - 'Accept-Language', 'Accept-Ranges', 'Allow', - 'Cache-Control', 'Connection', 'Content-Encoding', - 'Content-Language', 'Expect', 'If-Match', - 'If-None-Match', 'Pragma', 'Proxy-Authenticate', - 'Te', 'Trailer', 'Transfer-Encoding', 'Upgrade', - 'Vary', 'Via', 'Warning', 'Www-Authenticate' -] - - -class SizedReader: - - def __init__(self, fp, length, maxbytes, bufsize=DEFAULT_BUFFER_SIZE, - has_trailers=False): - # Wrap our fp in a buffer so peek() works - self.fp = fp - self.length = length - self.maxbytes = maxbytes - self.buffer = ntob('') - self.bufsize = bufsize - self.bytes_read = 0 - self.done = False - self.has_trailers = has_trailers - - def read(self, size=None, fp_out=None): - """Read bytes from the request body and return or write them to a file. - - A number of bytes less than or equal to the 'size' argument are read - off the socket. The actual number of bytes read are tracked in - self.bytes_read. The number may be smaller than 'size' when 1) the - client sends fewer bytes, 2) the 'Content-Length' request header - specifies fewer bytes than requested, or 3) the number of bytes read - exceeds self.maxbytes (in which case, 413 is raised). - - If the 'fp_out' argument is None (the default), all bytes read are - returned in a single byte string. - - If the 'fp_out' argument is not None, it must be a file-like - object that supports the 'write' method; all bytes read will be - written to the fp, and None is returned. - """ - - if self.length is None: - if size is None: - remaining = inf - else: - remaining = size - else: - remaining = self.length - self.bytes_read - if size and size < remaining: - remaining = size - if remaining == 0: - self.finish() - if fp_out is None: - return ntob('') - else: - return None - - chunks = [] - - # Read bytes from the buffer. - if self.buffer: - if remaining is inf: - data = self.buffer - self.buffer = ntob('') - else: - data = self.buffer[:remaining] - self.buffer = self.buffer[remaining:] - datalen = len(data) - remaining -= datalen - - # Check lengths. - self.bytes_read += datalen - if self.maxbytes and self.bytes_read > self.maxbytes: - raise cherrypy.HTTPError(413) - - # Store the data. - if fp_out is None: - chunks.append(data) - else: - fp_out.write(data) - - # Read bytes from the socket. - while remaining > 0: - chunksize = min(remaining, self.bufsize) - try: - data = self.fp.read(chunksize) - except Exception: - e = sys.exc_info()[1] - if e.__class__.__name__ == 'MaxSizeExceeded': - # Post data is too big - raise cherrypy.HTTPError( - 413, "Maximum request length: %r" % e.args[1]) - else: - raise - if not data: - self.finish() - break - datalen = len(data) - remaining -= datalen - - # Check lengths. - self.bytes_read += datalen - if self.maxbytes and self.bytes_read > self.maxbytes: - raise cherrypy.HTTPError(413) - - # Store the data. - if fp_out is None: - chunks.append(data) - else: - fp_out.write(data) - - if fp_out is None: - return ntob('').join(chunks) - - def readline(self, size=None): - """Read a line from the request body and return it.""" - chunks = [] - while size is None or size > 0: - chunksize = self.bufsize - if size is not None and size < self.bufsize: - chunksize = size - data = self.read(chunksize) - if not data: - break - pos = data.find(ntob('\n')) + 1 - if pos: - chunks.append(data[:pos]) - remainder = data[pos:] - self.buffer += remainder - self.bytes_read -= len(remainder) - break - else: - chunks.append(data) - return ntob('').join(chunks) - - def readlines(self, sizehint=None): - """Read lines from the request body and return them.""" - if self.length is not None: - if sizehint is None: - sizehint = self.length - self.bytes_read - else: - sizehint = min(sizehint, self.length - self.bytes_read) - - lines = [] - seen = 0 - while True: - line = self.readline() - if not line: - break - lines.append(line) - seen += len(line) - if seen >= sizehint: - break - return lines - - def finish(self): - self.done = True - if self.has_trailers and hasattr(self.fp, 'read_trailer_lines'): - self.trailers = {} - - try: - for line in self.fp.read_trailer_lines(): - if line[0] in ntob(' \t'): - # It's a continuation line. - v = line.strip() - else: - try: - k, v = line.split(ntob(":"), 1) - except ValueError: - raise ValueError("Illegal header line.") - k = k.strip().title() - v = v.strip() - - if k in comma_separated_headers: - existing = self.trailers.get(envname) - if existing: - v = ntob(", ").join((existing, v)) - self.trailers[k] = v - except Exception: - e = sys.exc_info()[1] - if e.__class__.__name__ == 'MaxSizeExceeded': - # Post data is too big - raise cherrypy.HTTPError( - 413, "Maximum request length: %r" % e.args[1]) - else: - raise - - -class RequestBody(Entity): - - """The entity of the HTTP request.""" - - bufsize = 8 * 1024 - """The buffer size used when reading the socket.""" - - # Don't parse the request body at all if the client didn't provide - # a Content-Type header. See - # https://github.com/cherrypy/cherrypy/issues/790 - default_content_type = '' - """This defines a default ``Content-Type`` to use if no Content-Type header - is given. The empty string is used for RequestBody, which results in the - request body not being read or parsed at all. This is by design; a missing - ``Content-Type`` header in the HTTP request entity is an error at best, - and a security hole at worst. For multipart parts, however, the MIME spec - declares that a part with no Content-Type defaults to "text/plain" - (see :class:`Part`). - """ - - maxbytes = None - """Raise ``MaxSizeExceeded`` if more bytes than this are read from - the socket. - """ - - def __init__(self, fp, headers, params=None, request_params=None): - Entity.__init__(self, fp, headers, params) - - # http://www.w3.org/Protocols/rfc2616/rfc2616-sec3.html#sec3.7.1 - # When no explicit charset parameter is provided by the - # sender, media subtypes of the "text" type are defined - # to have a default charset value of "ISO-8859-1" when - # received via HTTP. - if self.content_type.value.startswith('text/'): - for c in ('ISO-8859-1', 'iso-8859-1', 'Latin-1', 'latin-1'): - if c in self.attempt_charsets: - break - else: - self.attempt_charsets.append('ISO-8859-1') - - # Temporary fix while deprecating passing .parts as .params. - self.processors['multipart'] = _old_process_multipart - - if request_params is None: - request_params = {} - self.request_params = request_params - - def process(self): - """Process the request entity based on its Content-Type.""" - # "The presence of a message-body in a request is signaled by the - # inclusion of a Content-Length or Transfer-Encoding header field in - # the request's message-headers." - # It is possible to send a POST request with no body, for example; - # however, app developers are responsible in that case to set - # cherrypy.request.process_body to False so this method isn't called. - h = cherrypy.serving.request.headers - if 'Content-Length' not in h and 'Transfer-Encoding' not in h: - raise cherrypy.HTTPError(411) - - self.fp = SizedReader(self.fp, self.length, - self.maxbytes, bufsize=self.bufsize, - has_trailers='Trailer' in h) - super(RequestBody, self).process() - - # Body params should also be a part of the request_params - # add them in here. - request_params = self.request_params - for key, value in self.params.items(): - # Python 2 only: keyword arguments must be byte strings (type - # 'str'). - if sys.version_info < (3, 0): - if isinstance(key, unicode): - key = key.encode('ISO-8859-1') - - if key in request_params: - if not isinstance(request_params[key], list): - request_params[key] = [request_params[key]] - request_params[key].append(value) - else: - request_params[key] = value diff --git a/lib/cherrypy/_cprequest.py b/lib/cherrypy/_cprequest.py deleted file mode 100644 index fbcaad0f446..00000000000 --- a/lib/cherrypy/_cprequest.py +++ /dev/null @@ -1,982 +0,0 @@ -import sys -import time -import warnings - -IS_PY3 = sys.version_info[0] == 3 -try: - import six -except ImportError: - if IS_PY3: - _unicode_type = str - else: - _unicode_type = unicode -else: - _unicode_type = six.text_type - -import cherrypy -from cherrypy._cpcompat import basestring, copykeys, ntob -from cherrypy._cpcompat import SimpleCookie, CookieError -from cherrypy import _cpreqbody, _cpconfig -from cherrypy._cperror import format_exc, bare_error -from cherrypy.lib import httputil, file_generator - - -class Hook(object): - - """A callback and its metadata: failsafe, priority, and kwargs.""" - - callback = None - """ - The bare callable that this Hook object is wrapping, which will - be called when the Hook is called.""" - - failsafe = False - """ - If True, the callback is guaranteed to run even if other callbacks - from the same call point raise exceptions.""" - - priority = 50 - """ - Defines the order of execution for a list of Hooks. Priority numbers - should be limited to the closed interval [0, 100], but values outside - this range are acceptable, as are fractional values.""" - - kwargs = {} - """ - A set of keyword arguments that will be passed to the - callable on each call.""" - - def __init__(self, callback, failsafe=None, priority=None, **kwargs): - self.callback = callback - - if failsafe is None: - failsafe = getattr(callback, "failsafe", False) - self.failsafe = failsafe - - if priority is None: - priority = getattr(callback, "priority", 50) - self.priority = priority - - self.kwargs = kwargs - - def __lt__(self, other): - # Python 3 - return self.priority < other.priority - - def __cmp__(self, other): - # Python 2 - return cmp(self.priority, other.priority) - - def __call__(self): - """Run self.callback(**self.kwargs).""" - return self.callback(**self.kwargs) - - def __repr__(self): - cls = self.__class__ - return ("%s.%s(callback=%r, failsafe=%r, priority=%r, %s)" - % (cls.__module__, cls.__name__, self.callback, - self.failsafe, self.priority, - ", ".join(['%s=%r' % (k, v) - for k, v in self.kwargs.items()]))) - - -class HookMap(dict): - - """A map of call points to lists of callbacks (Hook objects).""" - - def __new__(cls, points=None): - d = dict.__new__(cls) - for p in points or []: - d[p] = [] - return d - - def __init__(self, *a, **kw): - pass - - def attach(self, point, callback, failsafe=None, priority=None, **kwargs): - """Append a new Hook made from the supplied arguments.""" - self[point].append(Hook(callback, failsafe, priority, **kwargs)) - - def run(self, point): - """Execute all registered Hooks (callbacks) for the given point.""" - exc = None - hooks = self[point] - hooks.sort() - for hook in hooks: - # Some hooks are guaranteed to run even if others at - # the same hookpoint fail. We will still log the failure, - # but proceed on to the next hook. The only way - # to stop all processing from one of these hooks is - # to raise SystemExit and stop the whole server. - if exc is None or hook.failsafe: - try: - hook() - except (KeyboardInterrupt, SystemExit): - raise - except (cherrypy.HTTPError, cherrypy.HTTPRedirect, - cherrypy.InternalRedirect): - exc = sys.exc_info()[1] - except: - exc = sys.exc_info()[1] - cherrypy.log(traceback=True, severity=40) - if exc: - raise exc - - def __copy__(self): - newmap = self.__class__() - # We can't just use 'update' because we want copies of the - # mutable values (each is a list) as well. - for k, v in self.items(): - newmap[k] = v[:] - return newmap - copy = __copy__ - - def __repr__(self): - cls = self.__class__ - return "%s.%s(points=%r)" % ( - cls.__module__, - cls.__name__, - copykeys(self) - ) - - -# Config namespace handlers - -def hooks_namespace(k, v): - """Attach bare hooks declared in config.""" - # Use split again to allow multiple hooks for a single - # hookpoint per path (e.g. "hooks.before_handler.1"). - # Little-known fact you only get from reading source ;) - hookpoint = k.split(".", 1)[0] - if isinstance(v, basestring): - v = cherrypy.lib.attributes(v) - if not isinstance(v, Hook): - v = Hook(v) - cherrypy.serving.request.hooks[hookpoint].append(v) - - -def request_namespace(k, v): - """Attach request attributes declared in config.""" - # Provides config entries to set request.body attrs (like - # attempt_charsets). - if k[:5] == 'body.': - setattr(cherrypy.serving.request.body, k[5:], v) - else: - setattr(cherrypy.serving.request, k, v) - - -def response_namespace(k, v): - """Attach response attributes declared in config.""" - # Provides config entries to set default response headers - # http://cherrypy.org/ticket/889 - if k[:8] == 'headers.': - cherrypy.serving.response.headers[k.split('.', 1)[1]] = v - else: - setattr(cherrypy.serving.response, k, v) - - -def error_page_namespace(k, v): - """Attach error pages declared in config.""" - if k != 'default': - k = int(k) - cherrypy.serving.request.error_page[k] = v - - -hookpoints = ['on_start_resource', 'before_request_body', - 'before_handler', 'before_finalize', - 'on_end_resource', 'on_end_request', - 'before_error_response', 'after_error_response'] - - -class Request(object): - - """An HTTP request. - - This object represents the metadata of an HTTP request message; - that is, it contains attributes which describe the environment - in which the request URL, headers, and body were sent (if you - want tools to interpret the headers and body, those are elsewhere, - mostly in Tools). This 'metadata' consists of socket data, - transport characteristics, and the Request-Line. This object - also contains data regarding the configuration in effect for - the given URL, and the execution plan for generating a response. - """ - - prev = None - """ - The previous Request object (if any). This should be None - unless we are processing an InternalRedirect.""" - - # Conversation/connection attributes - local = httputil.Host("127.0.0.1", 80) - "An httputil.Host(ip, port, hostname) object for the server socket." - - remote = httputil.Host("127.0.0.1", 1111) - "An httputil.Host(ip, port, hostname) object for the client socket." - - scheme = "http" - """ - The protocol used between client and server. In most cases, - this will be either 'http' or 'https'.""" - - server_protocol = "HTTP/1.1" - """ - The HTTP version for which the HTTP server is at least - conditionally compliant.""" - - base = "" - """The (scheme://host) portion of the requested URL. - In some cases (e.g. when proxying via mod_rewrite), this may contain - path segments which cherrypy.url uses when constructing url's, but - which otherwise are ignored by CherryPy. Regardless, this value - MUST NOT end in a slash.""" - - # Request-Line attributes - request_line = "" - """ - The complete Request-Line received from the client. This is a - single string consisting of the request method, URI, and protocol - version (joined by spaces). Any final CRLF is removed.""" - - method = "GET" - """ - Indicates the HTTP method to be performed on the resource identified - by the Request-URI. Common methods include GET, HEAD, POST, PUT, and - DELETE. CherryPy allows any extension method; however, various HTTP - servers and gateways may restrict the set of allowable methods. - CherryPy applications SHOULD restrict the set (on a per-URI basis).""" - - query_string = "" - """ - The query component of the Request-URI, a string of information to be - interpreted by the resource. The query portion of a URI follows the - path component, and is separated by a '?'. For example, the URI - 'http://www.cherrypy.org/wiki?a=3&b=4' has the query component, - 'a=3&b=4'.""" - - query_string_encoding = 'utf8' - """ - The encoding expected for query string arguments after % HEX HEX decoding). - If a query string is provided that cannot be decoded with this encoding, - 404 is raised (since technically it's a different URI). If you want - arbitrary encodings to not error, set this to 'Latin-1'; you can then - encode back to bytes and re-decode to whatever encoding you like later. - """ - - protocol = (1, 1) - """The HTTP protocol version corresponding to the set - of features which should be allowed in the response. If BOTH - the client's request message AND the server's level of HTTP - compliance is HTTP/1.1, this attribute will be the tuple (1, 1). - If either is 1.0, this attribute will be the tuple (1, 0). - Lower HTTP protocol versions are not explicitly supported.""" - - params = {} - """ - A dict which combines query string (GET) and request entity (POST) - variables. This is populated in two stages: GET params are added - before the 'on_start_resource' hook, and POST params are added - between the 'before_request_body' and 'before_handler' hooks.""" - - # Message attributes - header_list = [] - """ - A list of the HTTP request headers as (name, value) tuples. - In general, you should use request.headers (a dict) instead.""" - - headers = httputil.HeaderMap() - """ - A dict-like object containing the request headers. Keys are header - names (in Title-Case format); however, you may get and set them in - a case-insensitive manner. That is, headers['Content-Type'] and - headers['content-type'] refer to the same value. Values are header - values (decoded according to :rfc:`2047` if necessary). See also: - httputil.HeaderMap, httputil.HeaderElement.""" - - cookie = SimpleCookie() - """See help(Cookie).""" - - rfile = None - """ - If the request included an entity (body), it will be available - as a stream in this attribute. However, the rfile will normally - be read for you between the 'before_request_body' hook and the - 'before_handler' hook, and the resulting string is placed into - either request.params or the request.body attribute. - - You may disable the automatic consumption of the rfile by setting - request.process_request_body to False, either in config for the desired - path, or in an 'on_start_resource' or 'before_request_body' hook. - - WARNING: In almost every case, you should not attempt to read from the - rfile stream after CherryPy's automatic mechanism has read it. If you - turn off the automatic parsing of rfile, you should read exactly the - number of bytes specified in request.headers['Content-Length']. - Ignoring either of these warnings may result in a hung request thread - or in corruption of the next (pipelined) request. - """ - - process_request_body = True - """ - If True, the rfile (if any) is automatically read and parsed, - and the result placed into request.params or request.body.""" - - methods_with_bodies = ("POST", "PUT") - """ - A sequence of HTTP methods for which CherryPy will automatically - attempt to read a body from the rfile. If you are going to change - this property, modify it on the configuration (recommended) - or on the "hook point" `on_start_resource`. - """ - - body = None - """ - If the request Content-Type is 'application/x-www-form-urlencoded' - or multipart, this will be None. Otherwise, this will be an instance - of :class:`RequestBody` (which you - can .read()); this value is set between the 'before_request_body' and - 'before_handler' hooks (assuming that process_request_body is True).""" - - # Dispatch attributes - dispatch = cherrypy.dispatch.Dispatcher() - """ - The object which looks up the 'page handler' callable and collects - config for the current request based on the path_info, other - request attributes, and the application architecture. The core - calls the dispatcher as early as possible, passing it a 'path_info' - argument. - - The default dispatcher discovers the page handler by matching path_info - to a hierarchical arrangement of objects, starting at request.app.root. - See help(cherrypy.dispatch) for more information.""" - - script_name = "" - """ - The 'mount point' of the application which is handling this request. - - This attribute MUST NOT end in a slash. If the script_name refers to - the root of the URI, it MUST be an empty string (not "/"). - """ - - path_info = "/" - """ - The 'relative path' portion of the Request-URI. This is relative - to the script_name ('mount point') of the application which is - handling this request.""" - - login = None - """ - When authentication is used during the request processing this is - set to 'False' if it failed and to the 'username' value if it succeeded. - The default 'None' implies that no authentication happened.""" - - # Note that cherrypy.url uses "if request.app:" to determine whether - # the call is during a real HTTP request or not. So leave this None. - app = None - """The cherrypy.Application object which is handling this request.""" - - handler = None - """ - The function, method, or other callable which CherryPy will call to - produce the response. The discovery of the handler and the arguments - it will receive are determined by the request.dispatch object. - By default, the handler is discovered by walking a tree of objects - starting at request.app.root, and is then passed all HTTP params - (from the query string and POST body) as keyword arguments.""" - - toolmaps = {} - """ - A nested dict of all Toolboxes and Tools in effect for this request, - of the form: {Toolbox.namespace: {Tool.name: config dict}}.""" - - config = None - """ - A flat dict of all configuration entries which apply to the - current request. These entries are collected from global config, - application config (based on request.path_info), and from handler - config (exactly how is governed by the request.dispatch object in - effect for this request; by default, handler config can be attached - anywhere in the tree between request.app.root and the final handler, - and inherits downward).""" - - is_index = None - """ - This will be True if the current request is mapped to an 'index' - resource handler (also, a 'default' handler if path_info ends with - a slash). The value may be used to automatically redirect the - user-agent to a 'more canonical' URL which either adds or removes - the trailing slash. See cherrypy.tools.trailing_slash.""" - - hooks = HookMap(hookpoints) - """ - A HookMap (dict-like object) of the form: {hookpoint: [hook, ...]}. - Each key is a str naming the hook point, and each value is a list - of hooks which will be called at that hook point during this request. - The list of hooks is generally populated as early as possible (mostly - from Tools specified in config), but may be extended at any time. - See also: _cprequest.Hook, _cprequest.HookMap, and cherrypy.tools.""" - - error_response = cherrypy.HTTPError(500).set_response - """ - The no-arg callable which will handle unexpected, untrapped errors - during request processing. This is not used for expected exceptions - (like NotFound, HTTPError, or HTTPRedirect) which are raised in - response to expected conditions (those should be customized either - via request.error_page or by overriding HTTPError.set_response). - By default, error_response uses HTTPError(500) to return a generic - error response to the user-agent.""" - - error_page = {} - """ - A dict of {error code: response filename or callable} pairs. - - The error code must be an int representing a given HTTP error code, - or the string 'default', which will be used if no matching entry - is found for a given numeric code. - - If a filename is provided, the file should contain a Python string- - formatting template, and can expect by default to receive format - values with the mapping keys %(status)s, %(message)s, %(traceback)s, - and %(version)s. The set of format mappings can be extended by - overriding HTTPError.set_response. - - If a callable is provided, it will be called by default with keyword - arguments 'status', 'message', 'traceback', and 'version', as for a - string-formatting template. The callable must return a string or - iterable of strings which will be set to response.body. It may also - override headers or perform any other processing. - - If no entry is given for an error code, and no 'default' entry exists, - a default template will be used. - """ - - show_tracebacks = True - """ - If True, unexpected errors encountered during request processing will - include a traceback in the response body.""" - - show_mismatched_params = True - """ - If True, mismatched parameters encountered during PageHandler invocation - processing will be included in the response body.""" - - throws = (KeyboardInterrupt, SystemExit, cherrypy.InternalRedirect) - """The sequence of exceptions which Request.run does not trap.""" - - throw_errors = False - """ - If True, Request.run will not trap any errors (except HTTPRedirect and - HTTPError, which are more properly called 'exceptions', not errors).""" - - closed = False - """True once the close method has been called, False otherwise.""" - - stage = None - """ - A string containing the stage reached in the request-handling process. - This is useful when debugging a live server with hung requests.""" - - namespaces = _cpconfig.NamespaceSet( - **{"hooks": hooks_namespace, - "request": request_namespace, - "response": response_namespace, - "error_page": error_page_namespace, - "tools": cherrypy.tools, - }) - - def __init__(self, local_host, remote_host, scheme="http", - server_protocol="HTTP/1.1"): - """Populate a new Request object. - - local_host should be an httputil.Host object with the server info. - remote_host should be an httputil.Host object with the client info. - scheme should be a string, either "http" or "https". - """ - self.local = local_host - self.remote = remote_host - self.scheme = scheme - self.server_protocol = server_protocol - - self.closed = False - - # Put a *copy* of the class error_page into self. - self.error_page = self.error_page.copy() - - # Put a *copy* of the class namespaces into self. - self.namespaces = self.namespaces.copy() - - self.stage = None - - def close(self): - """Run cleanup code. (Core)""" - if not self.closed: - self.closed = True - self.stage = 'on_end_request' - self.hooks.run('on_end_request') - self.stage = 'close' - - def run(self, method, path, query_string, req_protocol, headers, rfile): - r"""Process the Request. (Core) - - method, path, query_string, and req_protocol should be pulled directly - from the Request-Line (e.g. "GET /path?key=val HTTP/1.0"). - - path - This should be %XX-unquoted, but query_string should not be. - - When using Python 2, they both MUST be byte strings, - not unicode strings. - - When using Python 3, they both MUST be unicode strings, - not byte strings, and preferably not bytes \x00-\xFF - disguised as unicode. - - headers - A list of (name, value) tuples. - - rfile - A file-like object containing the HTTP request entity. - - When run() is done, the returned object should have 3 attributes: - - * status, e.g. "200 OK" - * header_list, a list of (name, value) tuples - * body, an iterable yielding strings - - Consumer code (HTTP servers) should then access these response - attributes to build the outbound stream. - - """ - response = cherrypy.serving.response - self.stage = 'run' - try: - self.error_response = cherrypy.HTTPError(500).set_response - - self.method = method - path = path or "/" - self.query_string = query_string or '' - self.params = {} - - # Compare request and server HTTP protocol versions, in case our - # server does not support the requested protocol. Limit our output - # to min(req, server). We want the following output: - # request server actual written supported response - # protocol protocol response protocol feature set - # a 1.0 1.0 1.0 1.0 - # b 1.0 1.1 1.1 1.0 - # c 1.1 1.0 1.0 1.0 - # d 1.1 1.1 1.1 1.1 - # Notice that, in (b), the response will be "HTTP/1.1" even though - # the client only understands 1.0. RFC 2616 10.5.6 says we should - # only return 505 if the _major_ version is different. - rp = int(req_protocol[5]), int(req_protocol[7]) - sp = int(self.server_protocol[5]), int(self.server_protocol[7]) - self.protocol = min(rp, sp) - response.headers.protocol = self.protocol - - # Rebuild first line of the request (e.g. "GET /path HTTP/1.0"). - url = path - if query_string: - url += '?' + query_string - self.request_line = '%s %s %s' % (method, url, req_protocol) - - self.header_list = list(headers) - self.headers = httputil.HeaderMap() - - self.rfile = rfile - self.body = None - - self.cookie = SimpleCookie() - self.handler = None - - # path_info should be the path from the - # app root (script_name) to the handler. - self.script_name = self.app.script_name - self.path_info = pi = path[len(self.script_name):] - - self.stage = 'respond' - self.respond(pi) - - except self.throws: - raise - except: - if self.throw_errors: - raise - else: - # Failure in setup, error handler or finalize. Bypass them. - # Can't use handle_error because we may not have hooks yet. - cherrypy.log(traceback=True, severity=40) - if self.show_tracebacks: - body = format_exc() - else: - body = "" - r = bare_error(body) - response.output_status, response.header_list, response.body = r - - if self.method == "HEAD": - # HEAD requests MUST NOT return a message-body in the response. - response.body = [] - - try: - cherrypy.log.access() - except: - cherrypy.log.error(traceback=True) - - if response.timed_out: - raise cherrypy.TimeoutError() - - return response - - # Uncomment for stage debugging - # stage = property(lambda self: self._stage, lambda self, v: print(v)) - - def respond(self, path_info): - """Generate a response for the resource at self.path_info. (Core)""" - response = cherrypy.serving.response - try: - try: - try: - if self.app is None: - raise cherrypy.NotFound() - - # Get the 'Host' header, so we can HTTPRedirect properly. - self.stage = 'process_headers' - self.process_headers() - - # Make a copy of the class hooks - self.hooks = self.__class__.hooks.copy() - self.toolmaps = {} - - self.stage = 'get_resource' - self.get_resource(path_info) - - self.body = _cpreqbody.RequestBody( - self.rfile, self.headers, request_params=self.params) - - self.namespaces(self.config) - - self.stage = 'on_start_resource' - self.hooks.run('on_start_resource') - - # Parse the querystring - self.stage = 'process_query_string' - self.process_query_string() - - # Process the body - if self.process_request_body: - if self.method not in self.methods_with_bodies: - self.process_request_body = False - self.stage = 'before_request_body' - self.hooks.run('before_request_body') - if self.process_request_body: - self.body.process() - - # Run the handler - self.stage = 'before_handler' - self.hooks.run('before_handler') - if self.handler: - self.stage = 'handler' - response.body = self.handler() - - # Finalize - self.stage = 'before_finalize' - self.hooks.run('before_finalize') - response.finalize() - except (cherrypy.HTTPRedirect, cherrypy.HTTPError): - inst = sys.exc_info()[1] - inst.set_response() - self.stage = 'before_finalize (HTTPError)' - self.hooks.run('before_finalize') - response.finalize() - finally: - self.stage = 'on_end_resource' - self.hooks.run('on_end_resource') - except self.throws: - raise - except: - if self.throw_errors: - raise - self.handle_error() - - def process_query_string(self): - """Parse the query string into Python structures. (Core)""" - try: - p = httputil.parse_query_string( - self.query_string, encoding=self.query_string_encoding) - except UnicodeDecodeError: - raise cherrypy.HTTPError( - 404, "The given query string could not be processed. Query " - "strings for this resource must be encoded with %r." % - self.query_string_encoding) - - # Python 2 only: keyword arguments must be byte strings (type 'str'). - if not IS_PY3: - for key, value in p.items(): - if isinstance(key, _unicode_type): - del p[key] - p[key.encode(self.query_string_encoding)] = value - self.params.update(p) - - def process_headers(self): - """Parse HTTP header data into Python structures. (Core)""" - # Process the headers into self.headers - headers = self.headers - for name, value in self.header_list: - # Call title() now (and use dict.__method__(headers)) - # so title doesn't have to be called twice. - name = name.title() - value = value.strip() - - # Warning: if there is more than one header entry for cookies - # (AFAIK, only Konqueror does that), only the last one will - # remain in headers (but they will be correctly stored in - # request.cookie). - if "=?" in value: - dict.__setitem__(headers, name, httputil.decode_TEXT(value)) - else: - dict.__setitem__(headers, name, value) - - # Handle cookies differently because on Konqueror, multiple - # cookies come on different lines with the same key - if name == 'Cookie': - try: - self.cookie.load(value) - except CookieError: - msg = "Illegal cookie name %s" % value.split('=')[0] - raise cherrypy.HTTPError(400, msg) - - if not dict.__contains__(headers, 'Host'): - # All Internet-based HTTP/1.1 servers MUST respond with a 400 - # (Bad Request) status code to any HTTP/1.1 request message - # which lacks a Host header field. - if self.protocol >= (1, 1): - msg = "HTTP/1.1 requires a 'Host' request header." - raise cherrypy.HTTPError(400, msg) - host = dict.get(headers, 'Host') - if not host: - host = self.local.name or self.local.ip - self.base = "%s://%s" % (self.scheme, host) - - def get_resource(self, path): - """Call a dispatcher (which sets self.handler and .config). (Core)""" - # First, see if there is a custom dispatch at this URI. Custom - # dispatchers can only be specified in app.config, not in _cp_config - # (since custom dispatchers may not even have an app.root). - dispatch = self.app.find_config( - path, "request.dispatch", self.dispatch) - - # dispatch() should set self.handler and self.config - dispatch(path) - - def handle_error(self): - """Handle the last unanticipated exception. (Core)""" - try: - self.hooks.run("before_error_response") - if self.error_response: - self.error_response() - self.hooks.run("after_error_response") - cherrypy.serving.response.finalize() - except cherrypy.HTTPRedirect: - inst = sys.exc_info()[1] - inst.set_response() - cherrypy.serving.response.finalize() - - # ------------------------- Properties ------------------------- # - - def _get_body_params(self): - warnings.warn( - "body_params is deprecated in CherryPy 3.2, will be removed in " - "CherryPy 3.3.", - DeprecationWarning - ) - return self.body.params - body_params = property(_get_body_params, - doc=""" - If the request Content-Type is 'application/x-www-form-urlencoded' or - multipart, this will be a dict of the params pulled from the entity - body; that is, it will be the portion of request.params that come - from the message body (sometimes called "POST params", although they - can be sent with various HTTP method verbs). This value is set between - the 'before_request_body' and 'before_handler' hooks (assuming that - process_request_body is True). - - Deprecated in 3.2, will be removed for 3.3 in favor of - :attr:`request.body.params`.""") - - -class ResponseBody(object): - - """The body of the HTTP response (the response entity).""" - - if IS_PY3: - unicode_err = ("Page handlers MUST return bytes. Use tools.encode " - "if you wish to return unicode.") - - def __get__(self, obj, objclass=None): - if obj is None: - # When calling on the class instead of an instance... - return self - else: - return obj._body - - def __set__(self, obj, value): - # Convert the given value to an iterable object. - if IS_PY3 and isinstance(value, str): - raise ValueError(self.unicode_err) - - if isinstance(value, basestring): - # strings get wrapped in a list because iterating over a single - # item list is much faster than iterating over every character - # in a long string. - if value: - value = [value] - else: - # [''] doesn't evaluate to False, so replace it with []. - value = [] - elif IS_PY3 and isinstance(value, list): - # every item in a list must be bytes... - for i, item in enumerate(value): - if isinstance(item, str): - raise ValueError(self.unicode_err) - # Don't use isinstance here; io.IOBase which has an ABC takes - # 1000 times as long as, say, isinstance(value, str) - elif hasattr(value, 'read'): - value = file_generator(value) - elif value is None: - value = [] - obj._body = value - - -class Response(object): - - """An HTTP Response, including status, headers, and body.""" - - status = "" - """The HTTP Status-Code and Reason-Phrase.""" - - header_list = [] - """ - A list of the HTTP response headers as (name, value) tuples. - In general, you should use response.headers (a dict) instead. This - attribute is generated from response.headers and is not valid until - after the finalize phase.""" - - headers = httputil.HeaderMap() - """ - A dict-like object containing the response headers. Keys are header - names (in Title-Case format); however, you may get and set them in - a case-insensitive manner. That is, headers['Content-Type'] and - headers['content-type'] refer to the same value. Values are header - values (decoded according to :rfc:`2047` if necessary). - - .. seealso:: classes :class:`HeaderMap`, :class:`HeaderElement` - """ - - cookie = SimpleCookie() - """See help(Cookie).""" - - body = ResponseBody() - """The body (entity) of the HTTP response.""" - - time = None - """The value of time.time() when created. Use in HTTP dates.""" - - timeout = 300 - """Seconds after which the response will be aborted.""" - - timed_out = False - """ - Flag to indicate the response should be aborted, because it has - exceeded its timeout.""" - - stream = False - """If False, buffer the response body.""" - - def __init__(self): - self.status = None - self.header_list = None - self._body = [] - self.time = time.time() - - self.headers = httputil.HeaderMap() - # Since we know all our keys are titled strings, we can - # bypass HeaderMap.update and get a big speed boost. - dict.update(self.headers, { - "Content-Type": 'text/html', - "Server": "CherryPy/" + cherrypy.__version__, - "Date": httputil.HTTPDate(self.time), - }) - self.cookie = SimpleCookie() - - def collapse_body(self): - """Collapse self.body to a single string; replace it and return it.""" - if isinstance(self.body, basestring): - return self.body - - newbody = [] - for chunk in self.body: - if IS_PY3 and not isinstance(chunk, bytes): - raise TypeError("Chunk %s is not of type 'bytes'." % - repr(chunk)) - newbody.append(chunk) - newbody = ntob('').join(newbody) - - self.body = newbody - return newbody - - def finalize(self): - """Transform headers (and cookies) into self.header_list. (Core)""" - try: - code, reason, _ = httputil.valid_status(self.status) - except ValueError: - raise cherrypy.HTTPError(500, sys.exc_info()[1].args[0]) - - headers = self.headers - - self.status = "%s %s" % (code, reason) - self.output_status = ntob(str(code), 'ascii') + \ - ntob(" ") + headers.encode(reason) - - if self.stream: - # The upshot: wsgiserver will chunk the response if - # you pop Content-Length (or set it explicitly to None). - # Note that lib.static sets C-L to the file's st_size. - if dict.get(headers, 'Content-Length') is None: - dict.pop(headers, 'Content-Length', None) - elif code < 200 or code in (204, 205, 304): - # "All 1xx (informational), 204 (no content), - # and 304 (not modified) responses MUST NOT - # include a message-body." - dict.pop(headers, 'Content-Length', None) - self.body = ntob("") - else: - # Responses which are not streamed should have a Content-Length, - # but allow user code to set Content-Length if desired. - if dict.get(headers, 'Content-Length') is None: - content = self.collapse_body() - dict.__setitem__(headers, 'Content-Length', len(content)) - - # Transform our header dict into a list of tuples. - self.header_list = h = headers.output() - - cookie = self.cookie.output() - if cookie: - for line in cookie.split("\n"): - if line.endswith("\r"): - # Python 2.4 emits cookies joined by LF but 2.5+ by CRLF. - line = line[:-1] - name, value = line.split(": ", 1) - if isinstance(name, _unicode_type): - name = name.encode("ISO-8859-1") - if isinstance(value, _unicode_type): - value = headers.encode(value) - h.append((name, value)) - - def check_timeout(self): - """If now > self.time + self.timeout, set self.timed_out. - - This purposefully sets a flag, rather than raising an error, - so that a monitor thread can interrupt the Response thread. - """ - if time.time() > self.time + self.timeout: - self.timed_out = True diff --git a/lib/cherrypy/_cpserver.py b/lib/cherrypy/_cpserver.py deleted file mode 100644 index 99ce04ec5d9..00000000000 --- a/lib/cherrypy/_cpserver.py +++ /dev/null @@ -1,228 +0,0 @@ -"""Manage HTTP servers with CherryPy.""" - -import sys - -IS_PY3 = sys.version_info[0] == 3 - -import cherrypy -from cherrypy.lib import attributes -from cherrypy._cpcompat import basestring - -# We import * because we want to export check_port -# et al as attributes of this module. -from cherrypy.process.servers import * - - -class Server(ServerAdapter): - - """An adapter for an HTTP server. - - You can set attributes (like socket_host and socket_port) - on *this* object (which is probably cherrypy.server), and call - quickstart. For example:: - - cherrypy.server.socket_port = 80 - cherrypy.quickstart() - """ - - socket_port = 8080 - """The TCP port on which to listen for connections.""" - - _socket_host = '127.0.0.1' - - def _get_socket_host(self): - return self._socket_host - - def _set_socket_host(self, value): - if value == '': - raise ValueError("The empty string ('') is not an allowed value. " - "Use '0.0.0.0' instead to listen on all active " - "interfaces (INADDR_ANY).") - self._socket_host = value - socket_host = property( - _get_socket_host, - _set_socket_host, - doc="""The hostname or IP address on which to listen for connections. - - Host values may be any IPv4 or IPv6 address, or any valid hostname. - The string 'localhost' is a synonym for '127.0.0.1' (or '::1', if - your hosts file prefers IPv6). The string '0.0.0.0' is a special - IPv4 entry meaning "any active interface" (INADDR_ANY), and '::' - is the similar IN6ADDR_ANY for IPv6. The empty string or None are - not allowed.""") - - socket_file = None - """If given, the name of the UNIX socket to use instead of TCP/IP. - - When this option is not None, the `socket_host` and `socket_port` options - are ignored.""" - - socket_queue_size = 5 - """The 'backlog' argument to socket.listen(); specifies the maximum number - of queued connections (default 5).""" - - socket_timeout = 10 - """The timeout in seconds for accepted connections (default 10).""" - - accepted_queue_size = -1 - """The maximum number of requests which will be queued up before - the server refuses to accept it (default -1, meaning no limit).""" - - accepted_queue_timeout = 10 - """The timeout in seconds for attempting to add a request to the - queue when the queue is full (default 10).""" - - shutdown_timeout = 5 - """The time to wait for HTTP worker threads to clean up.""" - - protocol_version = 'HTTP/1.1' - """The version string to write in the Status-Line of all HTTP responses, - for example, "HTTP/1.1" (the default). Depending on the HTTP server used, - this should also limit the supported features used in the response.""" - - thread_pool = 10 - """The number of worker threads to start up in the pool.""" - - thread_pool_max = -1 - """The maximum size of the worker-thread pool. Use -1 to indicate no limit. - """ - - max_request_header_size = 500 * 1024 - """The maximum number of bytes allowable in the request headers. - If exceeded, the HTTP server should return "413 Request Entity Too Large". - """ - - max_request_body_size = 100 * 1024 * 1024 - """The maximum number of bytes allowable in the request body. If exceeded, - the HTTP server should return "413 Request Entity Too Large".""" - - instance = None - """If not None, this should be an HTTP server instance (such as - CPWSGIServer) which cherrypy.server will control. Use this when you need - more control over object instantiation than is available in the various - configuration options.""" - - ssl_context = None - """When using PyOpenSSL, an instance of SSL.Context.""" - - ssl_certificate = None - """The filename of the SSL certificate to use.""" - - ssl_certificate_chain = None - """When using PyOpenSSL, the certificate chain to pass to - Context.load_verify_locations.""" - - ssl_private_key = None - """The filename of the private key to use with SSL.""" - - if IS_PY3: - ssl_module = 'builtin' - """The name of a registered SSL adaptation module to use with - the builtin WSGI server. Builtin options are: 'builtin' (to - use the SSL library built into recent versions of Python). - You may also register your own classes in the - wsgiserver.ssl_adapters dict.""" - else: - ssl_module = 'pyopenssl' - """The name of a registered SSL adaptation module to use with the - builtin WSGI server. Builtin options are 'builtin' (to use the SSL - library built into recent versions of Python) and 'pyopenssl' (to - use the PyOpenSSL project, which you must install separately). You - may also register your own classes in the wsgiserver.ssl_adapters - dict.""" - - statistics = False - """Turns statistics-gathering on or off for aware HTTP servers.""" - - nodelay = True - """If True (the default since 3.1), sets the TCP_NODELAY socket option.""" - - wsgi_version = (1, 0) - """The WSGI version tuple to use with the builtin WSGI server. - The provided options are (1, 0) [which includes support for PEP 3333, - which declares it covers WSGI version 1.0.1 but still mandates the - wsgi.version (1, 0)] and ('u', 0), an experimental unicode version. - You may create and register your own experimental versions of the WSGI - protocol by adding custom classes to the wsgiserver.wsgi_gateways dict.""" - - def __init__(self): - self.bus = cherrypy.engine - self.httpserver = None - self.interrupt = None - self.running = False - - def httpserver_from_self(self, httpserver=None): - """Return a (httpserver, bind_addr) pair based on self attributes.""" - if httpserver is None: - httpserver = self.instance - if httpserver is None: - from cherrypy import _cpwsgi_server - httpserver = _cpwsgi_server.CPWSGIServer(self) - if isinstance(httpserver, basestring): - # Is anyone using this? Can I add an arg? - httpserver = attributes(httpserver)(self) - return httpserver, self.bind_addr - - def start(self): - """Start the HTTP server.""" - if not self.httpserver: - self.httpserver, self.bind_addr = self.httpserver_from_self() - ServerAdapter.start(self) - start.priority = 75 - - def _get_bind_addr(self): - if self.socket_file: - return self.socket_file - if self.socket_host is None and self.socket_port is None: - return None - return (self.socket_host, self.socket_port) - - def _set_bind_addr(self, value): - if value is None: - self.socket_file = None - self.socket_host = None - self.socket_port = None - elif isinstance(value, basestring): - self.socket_file = value - self.socket_host = None - self.socket_port = None - else: - try: - self.socket_host, self.socket_port = value - self.socket_file = None - except ValueError: - raise ValueError("bind_addr must be a (host, port) tuple " - "(for TCP sockets) or a string (for Unix " - "domain sockets), not %r" % value) - bind_addr = property( - _get_bind_addr, - _set_bind_addr, - doc='A (host, port) tuple for TCP sockets or ' - 'a str for Unix domain sockets.') - - def base(self): - """Return the base (scheme://host[:port] or sock file) for this server. - """ - if self.socket_file: - return self.socket_file - - host = self.socket_host - if host in ('0.0.0.0', '::'): - # 0.0.0.0 is INADDR_ANY and :: is IN6ADDR_ANY. - # Look up the host name, which should be the - # safest thing to spit out in a URL. - import socket - host = socket.gethostname() - - port = self.socket_port - - if self.ssl_certificate: - scheme = "https" - if port != 443: - host += ":%s" % port - else: - scheme = "http" - if port != 80: - host += ":%s" % port - - return "%s://%s" % (scheme, host) diff --git a/lib/cherrypy/_cpthreadinglocal.py b/lib/cherrypy/_cpthreadinglocal.py deleted file mode 100644 index 238c322496b..00000000000 --- a/lib/cherrypy/_cpthreadinglocal.py +++ /dev/null @@ -1,241 +0,0 @@ -# This is a backport of Python-2.4's threading.local() implementation - -"""Thread-local objects - -(Note that this module provides a Python version of thread - threading.local class. Depending on the version of Python you're - using, there may be a faster one available. You should always import - the local class from threading.) - -Thread-local objects support the management of thread-local data. -If you have data that you want to be local to a thread, simply create -a thread-local object and use its attributes: - - >>> mydata = local() - >>> mydata.number = 42 - >>> mydata.number - 42 - -You can also access the local-object's dictionary: - - >>> mydata.__dict__ - {'number': 42} - >>> mydata.__dict__.setdefault('widgets', []) - [] - >>> mydata.widgets - [] - -What's important about thread-local objects is that their data are -local to a thread. If we access the data in a different thread: - - >>> log = [] - >>> def f(): - ... items = mydata.__dict__.items() - ... items.sort() - ... log.append(items) - ... mydata.number = 11 - ... log.append(mydata.number) - - >>> import threading - >>> thread = threading.Thread(target=f) - >>> thread.start() - >>> thread.join() - >>> log - [[], 11] - -we get different data. Furthermore, changes made in the other thread -don't affect data seen in this thread: - - >>> mydata.number - 42 - -Of course, values you get from a local object, including a __dict__ -attribute, are for whatever thread was current at the time the -attribute was read. For that reason, you generally don't want to save -these values across threads, as they apply only to the thread they -came from. - -You can create custom local objects by subclassing the local class: - - >>> class MyLocal(local): - ... number = 2 - ... initialized = False - ... def __init__(self, **kw): - ... if self.initialized: - ... raise SystemError('__init__ called too many times') - ... self.initialized = True - ... self.__dict__.update(kw) - ... def squared(self): - ... return self.number ** 2 - -This can be useful to support default values, methods and -initialization. Note that if you define an __init__ method, it will be -called each time the local object is used in a separate thread. This -is necessary to initialize each thread's dictionary. - -Now if we create a local object: - - >>> mydata = MyLocal(color='red') - -Now we have a default number: - - >>> mydata.number - 2 - -an initial color: - - >>> mydata.color - 'red' - >>> del mydata.color - -And a method that operates on the data: - - >>> mydata.squared() - 4 - -As before, we can access the data in a separate thread: - - >>> log = [] - >>> thread = threading.Thread(target=f) - >>> thread.start() - >>> thread.join() - >>> log - [[('color', 'red'), ('initialized', True)], 11] - -without affecting this thread's data: - - >>> mydata.number - 2 - >>> mydata.color - Traceback (most recent call last): - ... - AttributeError: 'MyLocal' object has no attribute 'color' - -Note that subclasses can define slots, but they are not thread -local. They are shared across threads: - - >>> class MyLocal(local): - ... __slots__ = 'number' - - >>> mydata = MyLocal() - >>> mydata.number = 42 - >>> mydata.color = 'red' - -So, the separate thread: - - >>> thread = threading.Thread(target=f) - >>> thread.start() - >>> thread.join() - -affects what we see: - - >>> mydata.number - 11 - ->>> del mydata -""" - -# Threading import is at end - - -class _localbase(object): - __slots__ = '_local__key', '_local__args', '_local__lock' - - def __new__(cls, *args, **kw): - self = object.__new__(cls) - key = 'thread.local.' + str(id(self)) - object.__setattr__(self, '_local__key', key) - object.__setattr__(self, '_local__args', (args, kw)) - object.__setattr__(self, '_local__lock', RLock()) - - if args or kw and (cls.__init__ is object.__init__): - raise TypeError("Initialization arguments are not supported") - - # We need to create the thread dict in anticipation of - # __init__ being called, to make sure we don't call it - # again ourselves. - dict = object.__getattribute__(self, '__dict__') - currentThread().__dict__[key] = dict - - return self - - -def _patch(self): - key = object.__getattribute__(self, '_local__key') - d = currentThread().__dict__.get(key) - if d is None: - d = {} - currentThread().__dict__[key] = d - object.__setattr__(self, '__dict__', d) - - # we have a new instance dict, so call out __init__ if we have - # one - cls = type(self) - if cls.__init__ is not object.__init__: - args, kw = object.__getattribute__(self, '_local__args') - cls.__init__(self, *args, **kw) - else: - object.__setattr__(self, '__dict__', d) - - -class local(_localbase): - - def __getattribute__(self, name): - lock = object.__getattribute__(self, '_local__lock') - lock.acquire() - try: - _patch(self) - return object.__getattribute__(self, name) - finally: - lock.release() - - def __setattr__(self, name, value): - lock = object.__getattribute__(self, '_local__lock') - lock.acquire() - try: - _patch(self) - return object.__setattr__(self, name, value) - finally: - lock.release() - - def __delattr__(self, name): - lock = object.__getattribute__(self, '_local__lock') - lock.acquire() - try: - _patch(self) - return object.__delattr__(self, name) - finally: - lock.release() - - def __del__(): - threading_enumerate = enumerate - __getattribute__ = object.__getattribute__ - - def __del__(self): - key = __getattribute__(self, '_local__key') - - try: - threads = list(threading_enumerate()) - except: - # if enumerate fails, as it seems to do during - # shutdown, we'll skip cleanup under the assumption - # that there is nothing to clean up - return - - for thread in threads: - try: - __dict__ = thread.__dict__ - except AttributeError: - # Thread is dying, rest in peace - continue - - if key in __dict__: - try: - del __dict__[key] - except KeyError: - pass # didn't have anything in this thread - - return __del__ - __del__ = __del__() - -from threading import currentThread, enumerate, RLock diff --git a/lib/cherrypy/_cptools.py b/lib/cherrypy/_cptools.py deleted file mode 100644 index 54c63734572..00000000000 --- a/lib/cherrypy/_cptools.py +++ /dev/null @@ -1,537 +0,0 @@ -"""CherryPy tools. A "tool" is any helper, adapted to CP. - -Tools are usually designed to be used in a variety of ways (although some -may only offer one if they choose): - - Library calls - All tools are callables that can be used wherever needed. - The arguments are straightforward and should be detailed within the - docstring. - - Function decorators - All tools, when called, may be used as decorators which configure - individual CherryPy page handlers (methods on the CherryPy tree). - That is, "@tools.anytool()" should "turn on" the tool via the - decorated function's _cp_config attribute. - - CherryPy config - If a tool exposes a "_setup" callable, it will be called - once per Request (if the feature is "turned on" via config). - -Tools may be implemented as any object with a namespace. The builtins -are generally either modules or instances of the tools.Tool class. -""" - -import sys -import warnings - -import cherrypy -from cherrypy._helper import expose - - -def _getargs(func): - """Return the names of all static arguments to the given function.""" - # Use this instead of importing inspect for less mem overhead. - import types - if sys.version_info >= (3, 0): - if isinstance(func, types.MethodType): - func = func.__func__ - co = func.__code__ - else: - if isinstance(func, types.MethodType): - func = func.im_func - co = func.func_code - return co.co_varnames[:co.co_argcount] - - -_attr_error = ( - "CherryPy Tools cannot be turned on directly. Instead, turn them " - "on via config, or use them as decorators on your page handlers." -) - - -class Tool(object): - - """A registered function for use with CherryPy request-processing hooks. - - help(tool.callable) should give you more information about this Tool. - """ - - namespace = "tools" - - def __init__(self, point, callable, name=None, priority=50): - self._point = point - self.callable = callable - self._name = name - self._priority = priority - self.__doc__ = self.callable.__doc__ - self._setargs() - - def _get_on(self): - raise AttributeError(_attr_error) - - def _set_on(self, value): - raise AttributeError(_attr_error) - on = property(_get_on, _set_on) - - def _setargs(self): - """Copy func parameter names to obj attributes.""" - try: - for arg in _getargs(self.callable): - setattr(self, arg, None) - except (TypeError, AttributeError): - if hasattr(self.callable, "__call__"): - for arg in _getargs(self.callable.__call__): - setattr(self, arg, None) - # IronPython 1.0 raises NotImplementedError because - # inspect.getargspec tries to access Python bytecode - # in co_code attribute. - except NotImplementedError: - pass - # IronPython 1B1 may raise IndexError in some cases, - # but if we trap it here it doesn't prevent CP from working. - except IndexError: - pass - - def _merged_args(self, d=None): - """Return a dict of configuration entries for this Tool.""" - if d: - conf = d.copy() - else: - conf = {} - - tm = cherrypy.serving.request.toolmaps[self.namespace] - if self._name in tm: - conf.update(tm[self._name]) - - if "on" in conf: - del conf["on"] - - return conf - - def __call__(self, *args, **kwargs): - """Compile-time decorator (turn on the tool in config). - - For example:: - - @expose - @tools.proxy() - def whats_my_base(self): - return cherrypy.request.base - """ - if args: - raise TypeError("The %r Tool does not accept positional " - "arguments; you must use keyword arguments." - % self._name) - - def tool_decorator(f): - if not hasattr(f, "_cp_config"): - f._cp_config = {} - subspace = self.namespace + "." + self._name + "." - f._cp_config[subspace + "on"] = True - for k, v in kwargs.items(): - f._cp_config[subspace + k] = v - return f - return tool_decorator - - def _setup(self): - """Hook this tool into cherrypy.request. - - The standard CherryPy request object will automatically call this - method when the tool is "turned on" in config. - """ - conf = self._merged_args() - p = conf.pop("priority", None) - if p is None: - p = getattr(self.callable, "priority", self._priority) - cherrypy.serving.request.hooks.attach(self._point, self.callable, - priority=p, **conf) - - -class HandlerTool(Tool): - - """Tool which is called 'before main', that may skip normal handlers. - - If the tool successfully handles the request (by setting response.body), - if should return True. This will cause CherryPy to skip any 'normal' page - handler. If the tool did not handle the request, it should return False - to tell CherryPy to continue on and call the normal page handler. If the - tool is declared AS a page handler (see the 'handler' method), returning - False will raise NotFound. - """ - - def __init__(self, callable, name=None): - Tool.__init__(self, 'before_handler', callable, name) - - def handler(self, *args, **kwargs): - """Use this tool as a CherryPy page handler. - - For example:: - - class Root: - nav = tools.staticdir.handler(section="/nav", dir="nav", - root=absDir) - """ - @expose - def handle_func(*a, **kw): - handled = self.callable(*args, **self._merged_args(kwargs)) - if not handled: - raise cherrypy.NotFound() - return cherrypy.serving.response.body - return handle_func - - def _wrapper(self, **kwargs): - if self.callable(**kwargs): - cherrypy.serving.request.handler = None - - def _setup(self): - """Hook this tool into cherrypy.request. - - The standard CherryPy request object will automatically call this - method when the tool is "turned on" in config. - """ - conf = self._merged_args() - p = conf.pop("priority", None) - if p is None: - p = getattr(self.callable, "priority", self._priority) - cherrypy.serving.request.hooks.attach(self._point, self._wrapper, - priority=p, **conf) - - -class HandlerWrapperTool(Tool): - - """Tool which wraps request.handler in a provided wrapper function. - - The 'newhandler' arg must be a handler wrapper function that takes a - 'next_handler' argument, plus ``*args`` and ``**kwargs``. Like all - page handler - functions, it must return an iterable for use as cherrypy.response.body. - - For example, to allow your 'inner' page handlers to return dicts - which then get interpolated into a template:: - - def interpolator(next_handler, *args, **kwargs): - filename = cherrypy.request.config.get('template') - cherrypy.response.template = env.get_template(filename) - response_dict = next_handler(*args, **kwargs) - return cherrypy.response.template.render(**response_dict) - cherrypy.tools.jinja = HandlerWrapperTool(interpolator) - """ - - def __init__(self, newhandler, point='before_handler', name=None, - priority=50): - self.newhandler = newhandler - self._point = point - self._name = name - self._priority = priority - - def callable(self, *args, **kwargs): - innerfunc = cherrypy.serving.request.handler - - def wrap(*args, **kwargs): - return self.newhandler(innerfunc, *args, **kwargs) - cherrypy.serving.request.handler = wrap - - -class ErrorTool(Tool): - - """Tool which is used to replace the default request.error_response.""" - - def __init__(self, callable, name=None): - Tool.__init__(self, None, callable, name) - - def _wrapper(self): - self.callable(**self._merged_args()) - - def _setup(self): - """Hook this tool into cherrypy.request. - - The standard CherryPy request object will automatically call this - method when the tool is "turned on" in config. - """ - cherrypy.serving.request.error_response = self._wrapper - - -# Builtin tools # - -from cherrypy.lib import cptools, encoding, auth, static, jsontools -from cherrypy.lib import sessions as _sessions, xmlrpcutil as _xmlrpc -from cherrypy.lib import caching as _caching -from cherrypy.lib import auth_basic, auth_digest - - -class SessionTool(Tool): - - """Session Tool for CherryPy. - - sessions.locking - When 'implicit' (the default), the session will be locked for you, - just before running the page handler. - - When 'early', the session will be locked before reading the request - body. This is off by default for safety reasons; for example, - a large upload would block the session, denying an AJAX - progress meter - (`issue `_). - - When 'explicit' (or any other value), you need to call - cherrypy.session.acquire_lock() yourself before using - session data. - """ - - def __init__(self): - # _sessions.init must be bound after headers are read - Tool.__init__(self, 'before_request_body', _sessions.init) - - def _lock_session(self): - cherrypy.serving.session.acquire_lock() - - def _setup(self): - """Hook this tool into cherrypy.request. - - The standard CherryPy request object will automatically call this - method when the tool is "turned on" in config. - """ - hooks = cherrypy.serving.request.hooks - - conf = self._merged_args() - - p = conf.pop("priority", None) - if p is None: - p = getattr(self.callable, "priority", self._priority) - - hooks.attach(self._point, self.callable, priority=p, **conf) - - locking = conf.pop('locking', 'implicit') - if locking == 'implicit': - hooks.attach('before_handler', self._lock_session) - elif locking == 'early': - # Lock before the request body (but after _sessions.init runs!) - hooks.attach('before_request_body', self._lock_session, - priority=60) - else: - # Don't lock - pass - - hooks.attach('before_finalize', _sessions.save) - hooks.attach('on_end_request', _sessions.close) - - def regenerate(self): - """Drop the current session and make a new one (with a new id).""" - sess = cherrypy.serving.session - sess.regenerate() - - # Grab cookie-relevant tool args - conf = dict([(k, v) for k, v in self._merged_args().items() - if k in ('path', 'path_header', 'name', 'timeout', - 'domain', 'secure')]) - _sessions.set_response_cookie(**conf) - - -class XMLRPCController(object): - - """A Controller (page handler collection) for XML-RPC. - - To use it, have your controllers subclass this base class (it will - turn on the tool for you). - - You can also supply the following optional config entries:: - - tools.xmlrpc.encoding: 'utf-8' - tools.xmlrpc.allow_none: 0 - - XML-RPC is a rather discontinuous layer over HTTP; dispatching to the - appropriate handler must first be performed according to the URL, and - then a second dispatch step must take place according to the RPC method - specified in the request body. It also allows a superfluous "/RPC2" - prefix in the URL, supplies its own handler args in the body, and - requires a 200 OK "Fault" response instead of 404 when the desired - method is not found. - - Therefore, XML-RPC cannot be implemented for CherryPy via a Tool alone. - This Controller acts as the dispatch target for the first half (based - on the URL); it then reads the RPC method from the request body and - does its own second dispatch step based on that method. It also reads - body params, and returns a Fault on error. - - The XMLRPCDispatcher strips any /RPC2 prefix; if you aren't using /RPC2 - in your URL's, you can safely skip turning on the XMLRPCDispatcher. - Otherwise, you need to use declare it in config:: - - request.dispatch: cherrypy.dispatch.XMLRPCDispatcher() - """ - - # Note we're hard-coding this into the 'tools' namespace. We could do - # a huge amount of work to make it relocatable, but the only reason why - # would be if someone actually disabled the default_toolbox. Meh. - _cp_config = {'tools.xmlrpc.on': True} - - @expose - def default(self, *vpath, **params): - rpcparams, rpcmethod = _xmlrpc.process_body() - - subhandler = self - for attr in str(rpcmethod).split('.'): - subhandler = getattr(subhandler, attr, None) - - if subhandler and getattr(subhandler, "exposed", False): - body = subhandler(*(vpath + rpcparams), **params) - - else: - # https://github.com/cherrypy/cherrypy/issues/533 - # if a method is not found, an xmlrpclib.Fault should be returned - # raising an exception here will do that; see - # cherrypy.lib.xmlrpcutil.on_error - raise Exception('method "%s" is not supported' % attr) - - conf = cherrypy.serving.request.toolmaps['tools'].get("xmlrpc", {}) - _xmlrpc.respond(body, - conf.get('encoding', 'utf-8'), - conf.get('allow_none', 0)) - return cherrypy.serving.response.body - - -class SessionAuthTool(HandlerTool): - - def _setargs(self): - for name in dir(cptools.SessionAuth): - if not name.startswith("__"): - setattr(self, name, None) - - -class CachingTool(Tool): - - """Caching Tool for CherryPy.""" - - def _wrapper(self, **kwargs): - request = cherrypy.serving.request - if _caching.get(**kwargs): - request.handler = None - else: - if request.cacheable: - # Note the devious technique here of adding hooks on the fly - request.hooks.attach('before_finalize', _caching.tee_output, - priority=90) - _wrapper.priority = 20 - - def _setup(self): - """Hook caching into cherrypy.request.""" - conf = self._merged_args() - - p = conf.pop("priority", None) - cherrypy.serving.request.hooks.attach('before_handler', self._wrapper, - priority=p, **conf) - - -class Toolbox(object): - - """A collection of Tools. - - This object also functions as a config namespace handler for itself. - Custom toolboxes should be added to each Application's toolboxes dict. - """ - - def __init__(self, namespace): - self.namespace = namespace - - def __setattr__(self, name, value): - # If the Tool._name is None, supply it from the attribute name. - if isinstance(value, Tool): - if value._name is None: - value._name = name - value.namespace = self.namespace - object.__setattr__(self, name, value) - - def __enter__(self): - """Populate request.toolmaps from tools specified in config.""" - cherrypy.serving.request.toolmaps[self.namespace] = map = {} - - def populate(k, v): - toolname, arg = k.split(".", 1) - bucket = map.setdefault(toolname, {}) - bucket[arg] = v - return populate - - def __exit__(self, exc_type, exc_val, exc_tb): - """Run tool._setup() for each tool in our toolmap.""" - map = cherrypy.serving.request.toolmaps.get(self.namespace) - if map: - for name, settings in map.items(): - if settings.get("on", False): - tool = getattr(self, name) - tool._setup() - - def register(self, point, **kwargs): - """Return a decorator which registers the function at the given hook point.""" - def decorator(func): - setattr(self, kwargs.get('name', func.__name__), Tool(point, func, **kwargs)) - return func - return decorator - - -class DeprecatedTool(Tool): - - _name = None - warnmsg = "This Tool is deprecated." - - def __init__(self, point, warnmsg=None): - self.point = point - if warnmsg is not None: - self.warnmsg = warnmsg - - def __call__(self, *args, **kwargs): - warnings.warn(self.warnmsg) - - def tool_decorator(f): - return f - return tool_decorator - - def _setup(self): - warnings.warn(self.warnmsg) - - -default_toolbox = _d = Toolbox("tools") -_d.session_auth = SessionAuthTool(cptools.session_auth) -_d.allow = Tool('on_start_resource', cptools.allow) -_d.proxy = Tool('before_request_body', cptools.proxy, priority=30) -_d.response_headers = Tool('on_start_resource', cptools.response_headers) -_d.log_tracebacks = Tool('before_error_response', cptools.log_traceback) -_d.log_headers = Tool('before_error_response', cptools.log_request_headers) -_d.log_hooks = Tool('on_end_request', cptools.log_hooks, priority=100) -_d.err_redirect = ErrorTool(cptools.redirect) -_d.etags = Tool('before_finalize', cptools.validate_etags, priority=75) -_d.decode = Tool('before_request_body', encoding.decode) -# the order of encoding, gzip, caching is important -_d.encode = Tool('before_handler', encoding.ResponseEncoder, priority=70) -_d.gzip = Tool('before_finalize', encoding.gzip, priority=80) -_d.staticdir = HandlerTool(static.staticdir) -_d.staticfile = HandlerTool(static.staticfile) -_d.sessions = SessionTool() -_d.xmlrpc = ErrorTool(_xmlrpc.on_error) -_d.caching = CachingTool('before_handler', _caching.get, 'caching') -_d.expires = Tool('before_finalize', _caching.expires) -_d.tidy = DeprecatedTool( - 'before_finalize', - "The tidy tool has been removed from the standard distribution of " - "CherryPy. The most recent version can be found at " - "http://tools.cherrypy.org/browser.") -_d.nsgmls = DeprecatedTool( - 'before_finalize', - "The nsgmls tool has been removed from the standard distribution of " - "CherryPy. The most recent version can be found at " - "http://tools.cherrypy.org/browser.") -_d.ignore_headers = Tool('before_request_body', cptools.ignore_headers) -_d.referer = Tool('before_request_body', cptools.referer) -_d.basic_auth = Tool('on_start_resource', auth.basic_auth) -_d.digest_auth = Tool('on_start_resource', auth.digest_auth) -_d.trailing_slash = Tool('before_handler', cptools.trailing_slash, priority=60) -_d.flatten = Tool('before_finalize', cptools.flatten) -_d.accept = Tool('on_start_resource', cptools.accept) -_d.redirect = Tool('on_start_resource', cptools.redirect) -_d.autovary = Tool('on_start_resource', cptools.autovary, priority=0) -_d.json_in = Tool('before_request_body', jsontools.json_in, priority=30) -_d.json_out = Tool('before_handler', jsontools.json_out, priority=30) -_d.auth_basic = Tool('before_handler', auth_basic.basic_auth, priority=1) -_d.auth_digest = Tool('before_handler', auth_digest.digest_auth, priority=1) - -del _d, cptools, encoding, auth, static diff --git a/lib/cherrypy/_cptree.py b/lib/cherrypy/_cptree.py deleted file mode 100644 index 1e77c435444..00000000000 --- a/lib/cherrypy/_cptree.py +++ /dev/null @@ -1,288 +0,0 @@ -"""CherryPy Application and Tree objects.""" - -import os -import sys - -IS_PY3 = sys.version_info[0] == 3 - -import cherrypy -from cherrypy._cpcompat import ntou -from cherrypy import _cpconfig, _cplogging, _cprequest, _cpwsgi, tools -from cherrypy.lib import httputil - - -class Application(object): - - """A CherryPy Application. - - Servers and gateways should not instantiate Request objects directly. - Instead, they should ask an Application object for a request object. - - An instance of this class may also be used as a WSGI callable - (WSGI application object) for itself. - """ - - root = None - """The top-most container of page handlers for this app. Handlers should - be arranged in a hierarchy of attributes, matching the expected URI - hierarchy; the default dispatcher then searches this hierarchy for a - matching handler. When using a dispatcher other than the default, - this value may be None.""" - - config = {} - """A dict of {path: pathconf} pairs, where 'pathconf' is itself a dict - of {key: value} pairs.""" - - namespaces = _cpconfig.NamespaceSet() - toolboxes = {'tools': cherrypy.tools} - - log = None - """A LogManager instance. See _cplogging.""" - - wsgiapp = None - """A CPWSGIApp instance. See _cpwsgi.""" - - request_class = _cprequest.Request - response_class = _cprequest.Response - - relative_urls = False - - def __init__(self, root, script_name="", config=None): - self.log = _cplogging.LogManager(id(self), cherrypy.log.logger_root) - self.root = root - self.script_name = script_name - self.wsgiapp = _cpwsgi.CPWSGIApp(self) - - self.namespaces = self.namespaces.copy() - self.namespaces["log"] = lambda k, v: setattr(self.log, k, v) - self.namespaces["wsgi"] = self.wsgiapp.namespace_handler - - self.config = self.__class__.config.copy() - if config: - self.merge(config) - - def __repr__(self): - return "%s.%s(%r, %r)" % (self.__module__, self.__class__.__name__, - self.root, self.script_name) - - script_name_doc = """The URI "mount point" for this app. A mount point - is that portion of the URI which is constant for all URIs that are - serviced by this application; it does not include scheme, host, or proxy - ("virtual host") portions of the URI. - - For example, if script_name is "/my/cool/app", then the URL - "http://www.example.com/my/cool/app/page1" might be handled by a - "page1" method on the root object. - - The value of script_name MUST NOT end in a slash. If the script_name - refers to the root of the URI, it MUST be an empty string (not "/"). - - If script_name is explicitly set to None, then the script_name will be - provided for each call from request.wsgi_environ['SCRIPT_NAME']. - """ - - def _get_script_name(self): - if self._script_name is not None: - return self._script_name - - # A `_script_name` with a value of None signals that the script name - # should be pulled from WSGI environ. - return cherrypy.serving.request.wsgi_environ['SCRIPT_NAME'].rstrip("/") - - def _set_script_name(self, value): - if value: - value = value.rstrip("/") - self._script_name = value - script_name = property(fget=_get_script_name, fset=_set_script_name, - doc=script_name_doc) - - def merge(self, config): - """Merge the given config into self.config.""" - _cpconfig.merge(self.config, config) - - # Handle namespaces specified in config. - self.namespaces(self.config.get("/", {})) - - def find_config(self, path, key, default=None): - """Return the most-specific value for key along path, or default.""" - trail = path or "/" - while trail: - nodeconf = self.config.get(trail, {}) - - if key in nodeconf: - return nodeconf[key] - - lastslash = trail.rfind("/") - if lastslash == -1: - break - elif lastslash == 0 and trail != "/": - trail = "/" - else: - trail = trail[:lastslash] - - return default - - def get_serving(self, local, remote, scheme, sproto): - """Create and return a Request and Response object.""" - req = self.request_class(local, remote, scheme, sproto) - req.app = self - - for name, toolbox in self.toolboxes.items(): - req.namespaces[name] = toolbox - - resp = self.response_class() - cherrypy.serving.load(req, resp) - cherrypy.engine.publish('acquire_thread') - cherrypy.engine.publish('before_request') - - return req, resp - - def release_serving(self): - """Release the current serving (request and response).""" - req = cherrypy.serving.request - - cherrypy.engine.publish('after_request') - - try: - req.close() - except: - cherrypy.log(traceback=True, severity=40) - - cherrypy.serving.clear() - - def __call__(self, environ, start_response): - return self.wsgiapp(environ, start_response) - - -class Tree(object): - - """A registry of CherryPy applications, mounted at diverse points. - - An instance of this class may also be used as a WSGI callable - (WSGI application object), in which case it dispatches to all - mounted apps. - """ - - apps = {} - """ - A dict of the form {script name: application}, where "script name" - is a string declaring the URI mount point (no trailing slash), and - "application" is an instance of cherrypy.Application (or an arbitrary - WSGI callable if you happen to be using a WSGI server).""" - - def __init__(self): - self.apps = {} - - def mount(self, root, script_name="", config=None): - """Mount a new app from a root object, script_name, and config. - - root - An instance of a "controller class" (a collection of page - handler methods) which represents the root of the application. - This may also be an Application instance, or None if using - a dispatcher other than the default. - - script_name - A string containing the "mount point" of the application. - This should start with a slash, and be the path portion of the - URL at which to mount the given root. For example, if root.index() - will handle requests to "http://www.example.com:8080/dept/app1/", - then the script_name argument would be "/dept/app1". - - It MUST NOT end in a slash. If the script_name refers to the - root of the URI, it MUST be an empty string (not "/"). - - config - A file or dict containing application config. - """ - if script_name is None: - raise TypeError( - "The 'script_name' argument may not be None. Application " - "objects may, however, possess a script_name of None (in " - "order to inpect the WSGI environ for SCRIPT_NAME upon each " - "request). You cannot mount such Applications on this Tree; " - "you must pass them to a WSGI server interface directly.") - - # Next line both 1) strips trailing slash and 2) maps "/" -> "". - script_name = script_name.rstrip("/") - - if isinstance(root, Application): - app = root - if script_name != "" and script_name != app.script_name: - raise ValueError( - "Cannot specify a different script name and pass an " - "Application instance to cherrypy.mount") - script_name = app.script_name - else: - app = Application(root, script_name) - - # If mounted at "", add favicon.ico - if (script_name == "" and root is not None - and not hasattr(root, "favicon_ico")): - favicon = os.path.join(os.getcwd(), os.path.dirname(__file__), - "favicon.ico") - root.favicon_ico = tools.staticfile.handler(favicon) - - if config: - app.merge(config) - - self.apps[script_name] = app - - return app - - def graft(self, wsgi_callable, script_name=""): - """Mount a wsgi callable at the given script_name.""" - # Next line both 1) strips trailing slash and 2) maps "/" -> "". - script_name = script_name.rstrip("/") - self.apps[script_name] = wsgi_callable - - def script_name(self, path=None): - """The script_name of the app at the given path, or None. - - If path is None, cherrypy.request is used. - """ - if path is None: - try: - request = cherrypy.serving.request - path = httputil.urljoin(request.script_name, - request.path_info) - except AttributeError: - return None - - while True: - if path in self.apps: - return path - - if path == "": - return None - - # Move one node up the tree and try again. - path = path[:path.rfind("/")] - - def __call__(self, environ, start_response): - # If you're calling this, then you're probably setting SCRIPT_NAME - # to '' (some WSGI servers always set SCRIPT_NAME to ''). - # Try to look up the app using the full path. - env1x = environ - if not IS_PY3 and environ.get(ntou('wsgi.version')) == (ntou('u'), 0): - env1x = _cpwsgi.downgrade_wsgi_ux_to_1x(environ) - path = httputil.urljoin(env1x.get('SCRIPT_NAME', ''), - env1x.get('PATH_INFO', '')) - sn = self.script_name(path or "/") - if sn is None: - start_response('404 Not Found', []) - return [] - - app = self.apps[sn] - - # Correct the SCRIPT_NAME and PATH_INFO environ entries. - environ = environ.copy() - if not IS_PY3 and environ.get(ntou('wsgi.version')) == (ntou('u'), 0): - # Python 2/WSGI u.0: all strings MUST be of type unicode - enc = environ[ntou('wsgi.url_encoding')] - environ[ntou('SCRIPT_NAME')] = sn.decode(enc) - environ[ntou('PATH_INFO')] = path[len(sn.rstrip("/")):].decode(enc) - else: - environ['SCRIPT_NAME'] = sn - environ['PATH_INFO'] = path[len(sn.rstrip("/")):] - return app(environ, start_response) diff --git a/lib/cherrypy/_cpwsgi.py b/lib/cherrypy/_cpwsgi.py deleted file mode 100644 index ba92cd7f5c1..00000000000 --- a/lib/cherrypy/_cpwsgi.py +++ /dev/null @@ -1,451 +0,0 @@ -"""WSGI interface (see PEP 333 and 3333). - -Note that WSGI environ keys and values are 'native strings'; that is, -whatever the type of "" is. For Python 2, that's a byte string; for Python 3, -it's a unicode string. But PEP 3333 says: "even if Python's str type is -actually Unicode "under the hood", the content of native strings must -still be translatable to bytes via the Latin-1 encoding!" -""" - -import sys as _sys -import io - -IS_PY3 = _sys.version_info[0] == 3 -try: - import six -except ImportError: - if IS_PY3: - _unicode_type = str - else: - _unicode_type = unicode -else: - _unicode_type = six.text_type - -import cherrypy as _cherrypy -from cherrypy._cpcompat import ntob, ntou -from cherrypy import _cperror -from cherrypy.lib import httputil -from cherrypy.lib import is_closable_iterator - -def downgrade_wsgi_ux_to_1x(environ): - """Return a new environ dict for WSGI 1.x from the given WSGI u.x environ. - """ - env1x = {} - - url_encoding = environ[ntou('wsgi.url_encoding')] - for k, v in list(environ.items()): - if k in [ntou('PATH_INFO'), ntou('SCRIPT_NAME'), ntou('QUERY_STRING')]: - v = v.encode(url_encoding) - elif isinstance(v, _unicode_type): - v = v.encode('ISO-8859-1') - env1x[k.encode('ISO-8859-1')] = v - - return env1x - - -class VirtualHost(object): - - """Select a different WSGI application based on the Host header. - - This can be useful when running multiple sites within one CP server. - It allows several domains to point to different applications. For example:: - - root = Root() - RootApp = cherrypy.Application(root) - Domain2App = cherrypy.Application(root) - SecureApp = cherrypy.Application(Secure()) - - vhost = cherrypy._cpwsgi.VirtualHost(RootApp, - domains={'www.domain2.example': Domain2App, - 'www.domain2.example:443': SecureApp, - }) - - cherrypy.tree.graft(vhost) - """ - default = None - """Required. The default WSGI application.""" - - use_x_forwarded_host = True - """If True (the default), any "X-Forwarded-Host" - request header will be used instead of the "Host" header. This - is commonly added by HTTP servers (such as Apache) when proxying.""" - - domains = {} - """A dict of {host header value: application} pairs. - The incoming "Host" request header is looked up in this dict, - and, if a match is found, the corresponding WSGI application - will be called instead of the default. Note that you often need - separate entries for "example.com" and "www.example.com". - In addition, "Host" headers may contain the port number. - """ - - def __init__(self, default, domains=None, use_x_forwarded_host=True): - self.default = default - self.domains = domains or {} - self.use_x_forwarded_host = use_x_forwarded_host - - def __call__(self, environ, start_response): - domain = environ.get('HTTP_HOST', '') - if self.use_x_forwarded_host: - domain = environ.get("HTTP_X_FORWARDED_HOST", domain) - - nextapp = self.domains.get(domain) - if nextapp is None: - nextapp = self.default - return nextapp(environ, start_response) - - -class InternalRedirector(object): - - """WSGI middleware that handles raised cherrypy.InternalRedirect.""" - - def __init__(self, nextapp, recursive=False): - self.nextapp = nextapp - self.recursive = recursive - - def __call__(self, environ, start_response): - redirections = [] - while True: - environ = environ.copy() - try: - return self.nextapp(environ, start_response) - except _cherrypy.InternalRedirect: - ir = _sys.exc_info()[1] - sn = environ.get('SCRIPT_NAME', '') - path = environ.get('PATH_INFO', '') - qs = environ.get('QUERY_STRING', '') - - # Add the *previous* path_info + qs to redirections. - old_uri = sn + path - if qs: - old_uri += "?" + qs - redirections.append(old_uri) - - if not self.recursive: - # Check to see if the new URI has been redirected to - # already - new_uri = sn + ir.path - if ir.query_string: - new_uri += "?" + ir.query_string - if new_uri in redirections: - ir.request.close() - raise RuntimeError("InternalRedirector visited the " - "same URL twice: %r" % new_uri) - - # Munge the environment and try again. - environ['REQUEST_METHOD'] = "GET" - environ['PATH_INFO'] = ir.path - environ['QUERY_STRING'] = ir.query_string - environ['wsgi.input'] = io.BytesIO() - environ['CONTENT_LENGTH'] = "0" - environ['cherrypy.previous_request'] = ir.request - - -class ExceptionTrapper(object): - - """WSGI middleware that traps exceptions.""" - - def __init__(self, nextapp, throws=(KeyboardInterrupt, SystemExit)): - self.nextapp = nextapp - self.throws = throws - - def __call__(self, environ, start_response): - return _TrappedResponse( - self.nextapp, - environ, - start_response, - self.throws - ) - - -class _TrappedResponse(object): - - response = iter([]) - - def __init__(self, nextapp, environ, start_response, throws): - self.nextapp = nextapp - self.environ = environ - self.start_response = start_response - self.throws = throws - self.started_response = False - self.response = self.trap( - self.nextapp, self.environ, self.start_response) - self.iter_response = iter(self.response) - - def __iter__(self): - self.started_response = True - return self - - def __next__(self): - return self.trap(next, self.iter_response) - - # todo: https://pythonhosted.org/six/#six.Iterator - if not IS_PY3: - next = __next__ - - def close(self): - if hasattr(self.response, 'close'): - self.response.close() - - def trap(self, func, *args, **kwargs): - try: - return func(*args, **kwargs) - except self.throws: - raise - except StopIteration: - raise - except: - tb = _cperror.format_exc() - #print('trapped (started %s):' % self.started_response, tb) - _cherrypy.log(tb, severity=40) - if not _cherrypy.request.show_tracebacks: - tb = "" - s, h, b = _cperror.bare_error(tb) - if IS_PY3: - # What fun. - s = s.decode('ISO-8859-1') - h = [(k.decode('ISO-8859-1'), v.decode('ISO-8859-1')) - for k, v in h] - if self.started_response: - # Empty our iterable (so future calls raise StopIteration) - self.iter_response = iter([]) - else: - self.iter_response = iter(b) - - try: - self.start_response(s, h, _sys.exc_info()) - except: - # "The application must not trap any exceptions raised by - # start_response, if it called start_response with exc_info. - # Instead, it should allow such exceptions to propagate - # back to the server or gateway." - # But we still log and call close() to clean up ourselves. - _cherrypy.log(traceback=True, severity=40) - raise - - if self.started_response: - return ntob("").join(b) - else: - return b - - -# WSGI-to-CP Adapter # - - -class AppResponse(object): - - """WSGI response iterable for CherryPy applications.""" - - def __init__(self, environ, start_response, cpapp): - self.cpapp = cpapp - try: - if not IS_PY3: - if environ.get(ntou('wsgi.version')) == (ntou('u'), 0): - environ = downgrade_wsgi_ux_to_1x(environ) - self.environ = environ - self.run() - - r = _cherrypy.serving.response - - outstatus = r.output_status - if not isinstance(outstatus, bytes): - raise TypeError("response.output_status is not a byte string.") - - outheaders = [] - for k, v in r.header_list: - if not isinstance(k, bytes): - raise TypeError( - "response.header_list key %r is not a byte string." % - k) - if not isinstance(v, bytes): - raise TypeError( - "response.header_list value %r is not a byte string." % - v) - outheaders.append((k, v)) - - if IS_PY3: - # According to PEP 3333, when using Python 3, the response - # status and headers must be bytes masquerading as unicode; - # that is, they must be of type "str" but are restricted to - # code points in the "latin-1" set. - outstatus = outstatus.decode('ISO-8859-1') - outheaders = [(k.decode('ISO-8859-1'), v.decode('ISO-8859-1')) - for k, v in outheaders] - - self.iter_response = iter(r.body) - self.write = start_response(outstatus, outheaders) - except: - self.close() - raise - - def __iter__(self): - return self - - def __next__(self): - return next(self.iter_response) - - # todo: https://pythonhosted.org/six/#six.Iterator - if not IS_PY3: - next = __next__ - - def close(self): - """Close and de-reference the current request and response. (Core)""" - streaming = _cherrypy.serving.response.stream - self.cpapp.release_serving() - - # We avoid the expense of examining the iterator to see if it's - # closable unless we are streaming the response, as that's the - # only situation where we are going to have an iterator which - # may not have been exhausted yet. - if streaming and is_closable_iterator(self.iter_response): - iter_close = self.iter_response.close - try: - iter_close() - except Exception: - _cherrypy.log(traceback=True, severity=40) - - def run(self): - """Create a Request object using environ.""" - env = self.environ.get - - local = httputil.Host('', - int(env('SERVER_PORT', 80) or -1), - env('SERVER_NAME', '')) - remote = httputil.Host(env('REMOTE_ADDR', ''), - int(env('REMOTE_PORT', -1) or -1), - env('REMOTE_HOST', '')) - scheme = env('wsgi.url_scheme') - sproto = env('ACTUAL_SERVER_PROTOCOL', "HTTP/1.1") - request, resp = self.cpapp.get_serving(local, remote, scheme, sproto) - - # LOGON_USER is served by IIS, and is the name of the - # user after having been mapped to a local account. - # Both IIS and Apache set REMOTE_USER, when possible. - request.login = env('LOGON_USER') or env('REMOTE_USER') or None - request.multithread = self.environ['wsgi.multithread'] - request.multiprocess = self.environ['wsgi.multiprocess'] - request.wsgi_environ = self.environ - request.prev = env('cherrypy.previous_request', None) - - meth = self.environ['REQUEST_METHOD'] - - path = httputil.urljoin(self.environ.get('SCRIPT_NAME', ''), - self.environ.get('PATH_INFO', '')) - qs = self.environ.get('QUERY_STRING', '') - - if IS_PY3: - # This isn't perfect; if the given PATH_INFO is in the - # wrong encoding, it may fail to match the appropriate config - # section URI. But meh. - old_enc = self.environ.get('wsgi.url_encoding', 'ISO-8859-1') - new_enc = self.cpapp.find_config(self.environ.get('PATH_INFO', ''), - "request.uri_encoding", 'utf-8') - if new_enc.lower() != old_enc.lower(): - # Even though the path and qs are unicode, the WSGI server - # is required by PEP 3333 to coerce them to ISO-8859-1 - # masquerading as unicode. So we have to encode back to - # bytes and then decode again using the "correct" encoding. - try: - u_path = path.encode(old_enc).decode(new_enc) - u_qs = qs.encode(old_enc).decode(new_enc) - except (UnicodeEncodeError, UnicodeDecodeError): - # Just pass them through without transcoding and hope. - pass - else: - # Only set transcoded values if they both succeed. - path = u_path - qs = u_qs - - rproto = self.environ.get('SERVER_PROTOCOL') - headers = self.translate_headers(self.environ) - rfile = self.environ['wsgi.input'] - request.run(meth, path, qs, rproto, headers, rfile) - - headerNames = {'HTTP_CGI_AUTHORIZATION': 'Authorization', - 'CONTENT_LENGTH': 'Content-Length', - 'CONTENT_TYPE': 'Content-Type', - 'REMOTE_HOST': 'Remote-Host', - 'REMOTE_ADDR': 'Remote-Addr', - } - - def translate_headers(self, environ): - """Translate CGI-environ header names to HTTP header names.""" - for cgiName in environ: - # We assume all incoming header keys are uppercase already. - if cgiName in self.headerNames: - yield self.headerNames[cgiName], environ[cgiName] - elif cgiName[:5] == "HTTP_": - # Hackish attempt at recovering original header names. - translatedHeader = cgiName[5:].replace("_", "-") - yield translatedHeader, environ[cgiName] - - -class CPWSGIApp(object): - - """A WSGI application object for a CherryPy Application.""" - - pipeline = [('ExceptionTrapper', ExceptionTrapper), - ('InternalRedirector', InternalRedirector), - ] - """A list of (name, wsgiapp) pairs. Each 'wsgiapp' MUST be a - constructor that takes an initial, positional 'nextapp' argument, - plus optional keyword arguments, and returns a WSGI application - (that takes environ and start_response arguments). The 'name' can - be any you choose, and will correspond to keys in self.config.""" - - head = None - """Rather than nest all apps in the pipeline on each call, it's only - done the first time, and the result is memoized into self.head. Set - this to None again if you change self.pipeline after calling self.""" - - config = {} - """A dict whose keys match names listed in the pipeline. Each - value is a further dict which will be passed to the corresponding - named WSGI callable (from the pipeline) as keyword arguments.""" - - response_class = AppResponse - """The class to instantiate and return as the next app in the WSGI chain. - """ - - def __init__(self, cpapp, pipeline=None): - self.cpapp = cpapp - self.pipeline = self.pipeline[:] - if pipeline: - self.pipeline.extend(pipeline) - self.config = self.config.copy() - - def tail(self, environ, start_response): - """WSGI application callable for the actual CherryPy application. - - You probably shouldn't call this; call self.__call__ instead, - so that any WSGI middleware in self.pipeline can run first. - """ - return self.response_class(environ, start_response, self.cpapp) - - def __call__(self, environ, start_response): - head = self.head - if head is None: - # Create and nest the WSGI apps in our pipeline (in reverse order). - # Then memoize the result in self.head. - head = self.tail - for name, callable in self.pipeline[::-1]: - conf = self.config.get(name, {}) - head = callable(head, **conf) - self.head = head - return head(environ, start_response) - - def namespace_handler(self, k, v): - """Config handler for the 'wsgi' namespace.""" - if k == "pipeline": - # Note this allows multiple 'wsgi.pipeline' config entries - # (but each entry will be processed in a 'random' order). - # It should also allow developers to set default middleware - # in code (passed to self.__init__) that deployers can add to - # (but not remove) via config. - self.pipeline.extend(v) - elif k == "response_class": - self.response_class = v - else: - name, arg = k.split(".", 1) - bucket = self.config.setdefault(name, {}) - bucket[arg] = v diff --git a/lib/cherrypy/_cpwsgi_server.py b/lib/cherrypy/_cpwsgi_server.py deleted file mode 100644 index 874e2e9fe73..00000000000 --- a/lib/cherrypy/_cpwsgi_server.py +++ /dev/null @@ -1,70 +0,0 @@ -"""WSGI server interface (see PEP 333). This adds some CP-specific bits to -the framework-agnostic wsgiserver package. -""" -import sys - -import cherrypy -from cherrypy import wsgiserver - - -class CPWSGIServer(wsgiserver.CherryPyWSGIServer): - - """Wrapper for wsgiserver.CherryPyWSGIServer. - - wsgiserver has been designed to not reference CherryPy in any way, - so that it can be used in other frameworks and applications. Therefore, - we wrap it here, so we can set our own mount points from cherrypy.tree - and apply some attributes from config -> cherrypy.server -> wsgiserver. - """ - - def __init__(self, server_adapter=cherrypy.server): - self.server_adapter = server_adapter - self.max_request_header_size = ( - self.server_adapter.max_request_header_size or 0 - ) - self.max_request_body_size = ( - self.server_adapter.max_request_body_size or 0 - ) - - server_name = (self.server_adapter.socket_host or - self.server_adapter.socket_file or - None) - - self.wsgi_version = self.server_adapter.wsgi_version - s = wsgiserver.CherryPyWSGIServer - s.__init__(self, server_adapter.bind_addr, cherrypy.tree, - self.server_adapter.thread_pool, - server_name, - max=self.server_adapter.thread_pool_max, - request_queue_size=self.server_adapter.socket_queue_size, - timeout=self.server_adapter.socket_timeout, - shutdown_timeout=self.server_adapter.shutdown_timeout, - accepted_queue_size=self.server_adapter.accepted_queue_size, - accepted_queue_timeout=self.server_adapter.accepted_queue_timeout, - ) - self.protocol = self.server_adapter.protocol_version - self.nodelay = self.server_adapter.nodelay - - if sys.version_info >= (3, 0): - ssl_module = self.server_adapter.ssl_module or 'builtin' - else: - ssl_module = self.server_adapter.ssl_module or 'pyopenssl' - if self.server_adapter.ssl_context: - adapter_class = wsgiserver.get_ssl_adapter_class(ssl_module) - self.ssl_adapter = adapter_class( - self.server_adapter.ssl_certificate, - self.server_adapter.ssl_private_key, - self.server_adapter.ssl_certificate_chain) - self.ssl_adapter.context = self.server_adapter.ssl_context - elif self.server_adapter.ssl_certificate: - adapter_class = wsgiserver.get_ssl_adapter_class(ssl_module) - self.ssl_adapter = adapter_class( - self.server_adapter.ssl_certificate, - self.server_adapter.ssl_private_key, - self.server_adapter.ssl_certificate_chain) - - self.stats['Enabled'] = getattr( - self.server_adapter, 'statistics', False) - - def error_log(self, msg="", level=20, traceback=False): - cherrypy.engine.log(msg, level, traceback) diff --git a/lib/cherrypy/_helper.py b/lib/cherrypy/_helper.py deleted file mode 100644 index 13603e95030..00000000000 --- a/lib/cherrypy/_helper.py +++ /dev/null @@ -1,299 +0,0 @@ -""" -Helper functions for CP apps -""" - -import sys -IS_PY3 = sys.version_info[0] == 3 - -from cherrypy._cpcompat import urljoin as _urljoin, urlencode as _urlencode -from cherrypy._cpcompat import basestring - -import cherrypy - - -def expose(func=None, alias=None): - """ - Expose the function or class, optionally providing an alias or set of aliases. - """ - def expose_(func): - func.exposed = True - if alias is not None: - if isinstance(alias, basestring): - parents[alias.replace(".", "_")] = func - else: - for a in alias: - parents[a.replace(".", "_")] = func - return func - - import sys - import types - decoratable_types = types.FunctionType, types.MethodType, type, - if not IS_PY3: - # Old-style classes are type types.ClassType. - decoratable_types += types.ClassType, - if isinstance(func, decoratable_types): - if alias is None: - # @expose - func.exposed = True - return func - else: - # func = expose(func, alias) - parents = sys._getframe(1).f_locals - return expose_(func) - elif func is None: - if alias is None: - # @expose() - parents = sys._getframe(1).f_locals - return expose_ - else: - # @expose(alias="alias") or - # @expose(alias=["alias1", "alias2"]) - parents = sys._getframe(1).f_locals - return expose_ - else: - # @expose("alias") or - # @expose(["alias1", "alias2"]) - parents = sys._getframe(1).f_locals - alias = func - return expose_ - - -def popargs(*args, **kwargs): - """A decorator for _cp_dispatch - (cherrypy.dispatch.Dispatcher.dispatch_method_name). - - Optional keyword argument: handler=(Object or Function) - - Provides a _cp_dispatch function that pops off path segments into - cherrypy.request.params under the names specified. The dispatch - is then forwarded on to the next vpath element. - - Note that any existing (and exposed) member function of the class that - popargs is applied to will override that value of the argument. For - instance, if you have a method named "list" on the class decorated with - popargs, then accessing "/list" will call that function instead of popping - it off as the requested parameter. This restriction applies to all - _cp_dispatch functions. The only way around this restriction is to create - a "blank class" whose only function is to provide _cp_dispatch. - - If there are path elements after the arguments, or more arguments - are requested than are available in the vpath, then the 'handler' - keyword argument specifies the next object to handle the parameterized - request. If handler is not specified or is None, then self is used. - If handler is a function rather than an instance, then that function - will be called with the args specified and the return value from that - function used as the next object INSTEAD of adding the parameters to - cherrypy.request.args. - - This decorator may be used in one of two ways: - - As a class decorator: - @cherrypy.popargs('year', 'month', 'day') - class Blog: - def index(self, year=None, month=None, day=None): - #Process the parameters here; any url like - #/, /2009, /2009/12, or /2009/12/31 - #will fill in the appropriate parameters. - - def create(self): - #This link will still be available at /create. Defined functions - #take precedence over arguments. - - Or as a member of a class: - class Blog: - _cp_dispatch = cherrypy.popargs('year', 'month', 'day') - #... - - The handler argument may be used to mix arguments with built in functions. - For instance, the following setup allows different activities at the - day, month, and year level: - - class DayHandler: - def index(self, year, month, day): - #Do something with this day; probably list entries - - def delete(self, year, month, day): - #Delete all entries for this day - - @cherrypy.popargs('day', handler=DayHandler()) - class MonthHandler: - def index(self, year, month): - #Do something with this month; probably list entries - - def delete(self, year, month): - #Delete all entries for this month - - @cherrypy.popargs('month', handler=MonthHandler()) - class YearHandler: - def index(self, year): - #Do something with this year - - #... - - @cherrypy.popargs('year', handler=YearHandler()) - class Root: - def index(self): - #... - - """ - - # Since keyword arg comes after *args, we have to process it ourselves - # for lower versions of python. - - handler = None - handler_call = False - for k, v in kwargs.items(): - if k == 'handler': - handler = v - else: - raise TypeError( - "cherrypy.popargs() got an unexpected keyword argument '{0}'" - .format(k) - ) - - import inspect - - if handler is not None \ - and (hasattr(handler, '__call__') or inspect.isclass(handler)): - handler_call = True - - def decorated(cls_or_self=None, vpath=None): - if inspect.isclass(cls_or_self): - # cherrypy.popargs is a class decorator - cls = cls_or_self - setattr(cls, cherrypy.dispatch.Dispatcher.dispatch_method_name, decorated) - return cls - - # We're in the actual function - self = cls_or_self - parms = {} - for arg in args: - if not vpath: - break - parms[arg] = vpath.pop(0) - - if handler is not None: - if handler_call: - return handler(**parms) - else: - cherrypy.request.params.update(parms) - return handler - - cherrypy.request.params.update(parms) - - # If we are the ultimate handler, then to prevent our _cp_dispatch - # from being called again, we will resolve remaining elements through - # getattr() directly. - if vpath: - return getattr(self, vpath.pop(0), None) - else: - return self - - return decorated - - -def url(path="", qs="", script_name=None, base=None, relative=None): - """Create an absolute URL for the given path. - - If 'path' starts with a slash ('/'), this will return - (base + script_name + path + qs). - If it does not start with a slash, this returns - (base + script_name [+ request.path_info] + path + qs). - - If script_name is None, cherrypy.request will be used - to find a script_name, if available. - - If base is None, cherrypy.request.base will be used (if available). - Note that you can use cherrypy.tools.proxy to change this. - - Finally, note that this function can be used to obtain an absolute URL - for the current request path (minus the querystring) by passing no args. - If you call url(qs=cherrypy.request.query_string), you should get the - original browser URL (assuming no internal redirections). - - If relative is None or not provided, request.app.relative_urls will - be used (if available, else False). If False, the output will be an - absolute URL (including the scheme, host, vhost, and script_name). - If True, the output will instead be a URL that is relative to the - current request path, perhaps including '..' atoms. If relative is - the string 'server', the output will instead be a URL that is - relative to the server root; i.e., it will start with a slash. - """ - if isinstance(qs, (tuple, list, dict)): - qs = _urlencode(qs) - if qs: - qs = '?' + qs - - if cherrypy.request.app: - if not path.startswith("/"): - # Append/remove trailing slash from path_info as needed - # (this is to support mistyped URL's without redirecting; - # if you want to redirect, use tools.trailing_slash). - pi = cherrypy.request.path_info - if cherrypy.request.is_index is True: - if not pi.endswith('/'): - pi = pi + '/' - elif cherrypy.request.is_index is False: - if pi.endswith('/') and pi != '/': - pi = pi[:-1] - - if path == "": - path = pi - else: - path = _urljoin(pi, path) - - if script_name is None: - script_name = cherrypy.request.script_name - if base is None: - base = cherrypy.request.base - - newurl = base + script_name + path + qs - else: - # No request.app (we're being called outside a request). - # We'll have to guess the base from server.* attributes. - # This will produce very different results from the above - # if you're using vhosts or tools.proxy. - if base is None: - base = cherrypy.server.base() - - path = (script_name or "") + path - newurl = base + path + qs - - if './' in newurl: - # Normalize the URL by removing ./ and ../ - atoms = [] - for atom in newurl.split('/'): - if atom == '.': - pass - elif atom == '..': - atoms.pop() - else: - atoms.append(atom) - newurl = '/'.join(atoms) - - # At this point, we should have a fully-qualified absolute URL. - - if relative is None: - relative = getattr(cherrypy.request.app, "relative_urls", False) - - # See http://www.ietf.org/rfc/rfc2396.txt - if relative == 'server': - # "A relative reference beginning with a single slash character is - # termed an absolute-path reference, as defined by ..." - # This is also sometimes called "server-relative". - newurl = '/' + '/'.join(newurl.split('/', 3)[3:]) - elif relative: - # "A relative reference that does not begin with a scheme name - # or a slash character is termed a relative-path reference." - old = url(relative=False).split('/')[:-1] - new = newurl.split('/') - while old and new: - a, b = old[0], new[0] - if a != b: - break - old.pop(0) - new.pop(0) - new = (['..'] * len(old)) + new - newurl = '/'.join(new) - - return newurl diff --git a/lib/cherrypy/cherryd b/lib/cherrypy/cherryd deleted file mode 100755 index fb47f421900..00000000000 --- a/lib/cherrypy/cherryd +++ /dev/null @@ -1,6 +0,0 @@ -#! /usr/bin/env python2 - -import cherrypy.daemon - -if __name__ == '__main__': - cherrypy.daemon.run() diff --git a/lib/cherrypy/daemon.py b/lib/cherrypy/daemon.py deleted file mode 100755 index 395a2e68374..00000000000 --- a/lib/cherrypy/daemon.py +++ /dev/null @@ -1,106 +0,0 @@ -"""The CherryPy daemon.""" - -import sys - -import cherrypy -from cherrypy.process import plugins, servers -from cherrypy import Application - - -def start(configfiles=None, daemonize=False, environment=None, - fastcgi=False, scgi=False, pidfile=None, imports=None, - cgi=False): - """Subscribe all engine plugins and start the engine.""" - sys.path = [''] + sys.path - for i in imports or []: - exec("import %s" % i) - - for c in configfiles or []: - cherrypy.config.update(c) - # If there's only one app mounted, merge config into it. - if len(cherrypy.tree.apps) == 1: - for app in cherrypy.tree.apps.values(): - if isinstance(app, Application): - app.merge(c) - - engine = cherrypy.engine - - if environment is not None: - cherrypy.config.update({'environment': environment}) - - # Only daemonize if asked to. - if daemonize: - # Don't print anything to stdout/sterr. - cherrypy.config.update({'log.screen': False}) - plugins.Daemonizer(engine).subscribe() - - if pidfile: - plugins.PIDFile(engine, pidfile).subscribe() - - if hasattr(engine, "signal_handler"): - engine.signal_handler.subscribe() - if hasattr(engine, "console_control_handler"): - engine.console_control_handler.subscribe() - - if (fastcgi and (scgi or cgi)) or (scgi and cgi): - cherrypy.log.error("You may only specify one of the cgi, fastcgi, and " - "scgi options.", 'ENGINE') - sys.exit(1) - elif fastcgi or scgi or cgi: - # Turn off autoreload when using *cgi. - cherrypy.config.update({'engine.autoreload_on': False}) - # Turn off the default HTTP server (which is subscribed by default). - cherrypy.server.unsubscribe() - - addr = cherrypy.server.bind_addr - cls = ( - servers.FlupFCGIServer if fastcgi else - servers.FlupSCGIServer if scgi else - servers.FlupCGIServer - ) - f = cls(application=cherrypy.tree, bindAddress=addr) - s = servers.ServerAdapter(engine, httpserver=f, bind_addr=addr) - s.subscribe() - - # Always start the engine; this will start all other services - try: - engine.start() - except: - # Assume the error has been logged already via bus.log. - sys.exit(1) - else: - engine.block() - - -def run(): - from optparse import OptionParser - - p = OptionParser() - p.add_option('-c', '--config', action="append", dest='config', - help="specify config file(s)") - p.add_option('-d', action="store_true", dest='daemonize', - help="run the server as a daemon") - p.add_option('-e', '--environment', dest='environment', default=None, - help="apply the given config environment") - p.add_option('-f', action="store_true", dest='fastcgi', - help="start a fastcgi server instead of the default HTTP " - "server") - p.add_option('-s', action="store_true", dest='scgi', - help="start a scgi server instead of the default HTTP server") - p.add_option('-x', action="store_true", dest='cgi', - help="start a cgi server instead of the default HTTP server") - p.add_option('-i', '--import', action="append", dest='imports', - help="specify modules to import") - p.add_option('-p', '--pidfile', dest='pidfile', default=None, - help="store the process id in the given file") - p.add_option('-P', '--Path', action="append", dest='Path', - help="add the given paths to sys.path") - options, args = p.parse_args() - - if options.Path: - for p in options.Path: - sys.path.insert(0, p) - - start(options.config, options.daemonize, - options.environment, options.fastcgi, options.scgi, - options.pidfile, options.imports, options.cgi) diff --git a/lib/cherrypy/favicon.ico b/lib/cherrypy/favicon.ico deleted file mode 100644 index f0d7e61bada..00000000000 Binary files a/lib/cherrypy/favicon.ico and /dev/null differ diff --git a/lib/cherrypy/lib/__init__.py b/lib/cherrypy/lib/__init__.py deleted file mode 100644 index a75a53da47c..00000000000 --- a/lib/cherrypy/lib/__init__.py +++ /dev/null @@ -1,85 +0,0 @@ -"""CherryPy Library""" - -# Deprecated in CherryPy 3.2 -- remove in CherryPy 3.3 -from cherrypy.lib.reprconf import unrepr, modules, attributes - -def is_iterator(obj): - '''Returns a boolean indicating if the object provided implements - the iterator protocol (i.e. like a generator). This will return - false for objects which iterable, but not iterators themselves.''' - from types import GeneratorType - if isinstance(obj, GeneratorType): - return True - elif not hasattr(obj, '__iter__'): - return False - else: - # Types which implement the protocol must return themselves when - # invoking 'iter' upon them. - return iter(obj) is obj - -def is_closable_iterator(obj): - - # Not an iterator. - if not is_iterator(obj): - return False - - # A generator - the easiest thing to deal with. - import inspect - if inspect.isgenerator(obj): - return True - - # A custom iterator. Look for a close method... - if not (hasattr(obj, 'close') and callable(obj.close)): - return False - - # ... which doesn't require any arguments. - try: - inspect.getcallargs(obj.close) - except TypeError: - return False - else: - return True - -class file_generator(object): - - """Yield the given input (a file object) in chunks (default 64k). (Core)""" - - def __init__(self, input, chunkSize=65536): - self.input = input - self.chunkSize = chunkSize - - def __iter__(self): - return self - - def __next__(self): - chunk = self.input.read(self.chunkSize) - if chunk: - return chunk - else: - if hasattr(self.input, 'close'): - self.input.close() - raise StopIteration() - next = __next__ - - -def file_generator_limited(fileobj, count, chunk_size=65536): - """Yield the given file object in chunks, stopping after `count` - bytes has been emitted. Default chunk size is 64kB. (Core) - """ - remaining = count - while remaining > 0: - chunk = fileobj.read(min(chunk_size, remaining)) - chunklen = len(chunk) - if chunklen == 0: - return - remaining -= chunklen - yield chunk - - -def set_vary_header(response, header_name): - "Add a Vary header to a response" - varies = response.headers.get("Vary", "") - varies = [x.strip() for x in varies.split(",") if x.strip()] - if header_name not in varies: - varies.append(header_name) - response.headers['Vary'] = ", ".join(varies) diff --git a/lib/cherrypy/lib/auth.py b/lib/cherrypy/lib/auth.py deleted file mode 100644 index 71591aaa8f6..00000000000 --- a/lib/cherrypy/lib/auth.py +++ /dev/null @@ -1,97 +0,0 @@ -import cherrypy -from cherrypy.lib import httpauth - - -def check_auth(users, encrypt=None, realm=None): - """If an authorization header contains credentials, return True or False. - """ - request = cherrypy.serving.request - if 'authorization' in request.headers: - # make sure the provided credentials are correctly set - ah = httpauth.parseAuthorization(request.headers['authorization']) - if ah is None: - raise cherrypy.HTTPError(400, 'Bad Request') - - if not encrypt: - encrypt = httpauth.DIGEST_AUTH_ENCODERS[httpauth.MD5] - - if hasattr(users, '__call__'): - try: - # backward compatibility - users = users() # expect it to return a dictionary - - if not isinstance(users, dict): - raise ValueError( - "Authentication users must be a dictionary") - - # fetch the user password - password = users.get(ah["username"], None) - except TypeError: - # returns a password (encrypted or clear text) - password = users(ah["username"]) - else: - if not isinstance(users, dict): - raise ValueError("Authentication users must be a dictionary") - - # fetch the user password - password = users.get(ah["username"], None) - - # validate the authorization by re-computing it here - # and compare it with what the user-agent provided - if httpauth.checkResponse(ah, password, method=request.method, - encrypt=encrypt, realm=realm): - request.login = ah["username"] - return True - - request.login = False - return False - - -def basic_auth(realm, users, encrypt=None, debug=False): - """If auth fails, raise 401 with a basic authentication header. - - realm - A string containing the authentication realm. - - users - A dict of the form: {username: password} or a callable returning - a dict. - - encrypt - callable used to encrypt the password returned from the user-agent. - if None it defaults to a md5 encryption. - - """ - if check_auth(users, encrypt): - if debug: - cherrypy.log('Auth successful', 'TOOLS.BASIC_AUTH') - return - - # inform the user-agent this path is protected - cherrypy.serving.response.headers[ - 'www-authenticate'] = httpauth.basicAuth(realm) - - raise cherrypy.HTTPError( - 401, "You are not authorized to access that resource") - - -def digest_auth(realm, users, debug=False): - """If auth fails, raise 401 with a digest authentication header. - - realm - A string containing the authentication realm. - users - A dict of the form: {username: password} or a callable returning - a dict. - """ - if check_auth(users, realm=realm): - if debug: - cherrypy.log('Auth successful', 'TOOLS.DIGEST_AUTH') - return - - # inform the user-agent this path is protected - cherrypy.serving.response.headers[ - 'www-authenticate'] = httpauth.digestAuth(realm) - - raise cherrypy.HTTPError( - 401, "You are not authorized to access that resource") diff --git a/lib/cherrypy/lib/auth_basic.py b/lib/cherrypy/lib/auth_basic.py deleted file mode 100644 index 5ba16f7fb31..00000000000 --- a/lib/cherrypy/lib/auth_basic.py +++ /dev/null @@ -1,90 +0,0 @@ -# This file is part of CherryPy -# -*- coding: utf-8 -*- -# vim:ts=4:sw=4:expandtab:fileencoding=utf-8 - -__doc__ = """This module provides a CherryPy 3.x tool which implements -the server-side of HTTP Basic Access Authentication, as described in -:rfc:`2617`. - -Example usage, using the built-in checkpassword_dict function which uses a dict -as the credentials store:: - - userpassdict = {'bird' : 'bebop', 'ornette' : 'wayout'} - checkpassword = cherrypy.lib.auth_basic.checkpassword_dict(userpassdict) - basic_auth = {'tools.auth_basic.on': True, - 'tools.auth_basic.realm': 'earth', - 'tools.auth_basic.checkpassword': checkpassword, - } - app_config = { '/' : basic_auth } - -""" - -__author__ = 'visteya' -__date__ = 'April 2009' - -import binascii -from cherrypy._cpcompat import base64_decode -import cherrypy - - -def checkpassword_dict(user_password_dict): - """Returns a checkpassword function which checks credentials - against a dictionary of the form: {username : password}. - - If you want a simple dictionary-based authentication scheme, use - checkpassword_dict(my_credentials_dict) as the value for the - checkpassword argument to basic_auth(). - """ - def checkpassword(realm, user, password): - p = user_password_dict.get(user) - return p and p == password or False - - return checkpassword - - -def basic_auth(realm, checkpassword, debug=False): - """A CherryPy tool which hooks at before_handler to perform - HTTP Basic Access Authentication, as specified in :rfc:`2617`. - - If the request has an 'authorization' header with a 'Basic' scheme, this - tool attempts to authenticate the credentials supplied in that header. If - the request has no 'authorization' header, or if it does but the scheme is - not 'Basic', or if authentication fails, the tool sends a 401 response with - a 'WWW-Authenticate' Basic header. - - realm - A string containing the authentication realm. - - checkpassword - A callable which checks the authentication credentials. - Its signature is checkpassword(realm, username, password). where - username and password are the values obtained from the request's - 'authorization' header. If authentication succeeds, checkpassword - returns True, else it returns False. - - """ - - if '"' in realm: - raise ValueError('Realm cannot contain the " (quote) character.') - request = cherrypy.serving.request - - auth_header = request.headers.get('authorization') - if auth_header is not None: - try: - scheme, params = auth_header.split(' ', 1) - if scheme.lower() == 'basic': - username, password = base64_decode(params).split(':', 1) - if checkpassword(realm, username, password): - if debug: - cherrypy.log('Auth succeeded', 'TOOLS.AUTH_BASIC') - request.login = username - return # successful authentication - # split() error, base64.decodestring() error - except (ValueError, binascii.Error): - raise cherrypy.HTTPError(400, 'Bad Request') - - # Respond with 401 status and a WWW-Authenticate header - cherrypy.serving.response.headers[ - 'www-authenticate'] = 'Basic realm="%s"' % realm - raise cherrypy.HTTPError( - 401, "You are not authorized to access that resource") diff --git a/lib/cherrypy/lib/auth_digest.py b/lib/cherrypy/lib/auth_digest.py deleted file mode 100644 index ae5a1ec0d3c..00000000000 --- a/lib/cherrypy/lib/auth_digest.py +++ /dev/null @@ -1,410 +0,0 @@ -# This file is part of CherryPy -# -*- coding: utf-8 -*- -# vim:ts=4:sw=4:expandtab:fileencoding=utf-8 - -__doc__ = """An implementation of the server-side of HTTP Digest Access -Authentication, which is described in :rfc:`2617`. - -Example usage, using the built-in get_ha1_dict_plain function which uses a dict -of plaintext passwords as the credentials store:: - - userpassdict = {'alice' : '4x5istwelve'} - get_ha1 = cherrypy.lib.auth_digest.get_ha1_dict_plain(userpassdict) - digest_auth = {'tools.auth_digest.on': True, - 'tools.auth_digest.realm': 'wonderland', - 'tools.auth_digest.get_ha1': get_ha1, - 'tools.auth_digest.key': 'a565c27146791cfb', - } - app_config = { '/' : digest_auth } -""" - -__author__ = 'visteya' -__date__ = 'April 2009' - -import hashlib -import time -from cherrypy._cpcompat import parse_http_list, parse_keqv_list - -import cherrypy -from cherrypy._cpcompat import ntob - -qop_auth = 'auth' -qop_auth_int = 'auth-int' -valid_qops = (qop_auth, qop_auth_int) - -valid_algorithms = ('MD5', 'MD5-sess', 'SHA') - - -def hexdigest(value, algorithm='MD5'): - """Provide the checksum hexdigest of value.""" - if algorithm in ['MD5', 'MD5-sess']: - return md5_hex(value) - return sha1_hex(value) - - -def md5_hex(value): - """Provide the MD5 hexdigest of value.""" - return hashlib.md5(ntob(value)).hexdigest() - - -def sha1_hex(value): - """Provide the SHA1 hexdigest of value.""" - return hashlib.sha1(ntob(value)).hexdigest() - - -def TRACE(msg): - cherrypy.log(msg, context='TOOLS.AUTH_DIGEST') - -# Three helper functions for users of the tool, providing three variants -# of get_ha1() functions for three different kinds of credential stores. - - -def get_ha1_dict_plain(user_password_dict, algorithm='MD5'): - """Returns a get_ha1 function which obtains a plaintext password from a - dictionary of the form: {username : password}. - - If you want a simple dictionary-based authentication scheme, with plaintext - passwords, use get_ha1_dict_plain(my_userpass_dict) as the value for the - get_ha1 argument to digest_auth(). - """ - def get_ha1(realm, username): - password = user_password_dict.get(username) - if password: - return hexdigest('%s:%s:%s' % (username, realm, password), - algorithm=algorithm) - return None - - return get_ha1 - - -def get_ha1_dict(user_ha1_dict): - """Returns a get_ha1 function which obtains a HA1 password hash from a - dictionary of the form: {username : HA1}. - - If you want a dictionary-based authentication scheme, but with - pre-computed HA1 hashes instead of plain-text passwords, use - get_ha1_dict(my_userha1_dict) as the value for the get_ha1 - argument to digest_auth(). - """ - def get_ha1(realm, username): - return user_ha1_dict.get(username) - - return get_ha1 - - -def get_ha1_file_htdigest(filename): - """Returns a get_ha1 function which obtains a HA1 password hash from a - flat file with lines of the same format as that produced by the Apache - htdigest utility. For example, for realm 'wonderland', username 'alice', - and password '4x5istwelve', the htdigest line would be:: - - alice:wonderland:3238cdfe91a8b2ed8e39646921a02d4c - - If you want to use an Apache htdigest file as the credentials store, - then use get_ha1_file_htdigest(my_htdigest_file) as the value for the - get_ha1 argument to digest_auth(). It is recommended that the filename - argument be an absolute path, to avoid problems. - """ - def get_ha1(realm, username): - result = None - f = open(filename, 'r') - for line in f: - u, r, ha1 = line.rstrip().split(':') - if u == username and r == realm: - result = ha1 - break - f.close() - return result - - return get_ha1 - - -def synthesize_nonce(s, key, timestamp=None, algorithm='MD5'): - """Synthesize a nonce value which resists spoofing and can be checked - for staleness. Returns a string suitable as the value for 'nonce' in - the www-authenticate header. - - s - A string related to the resource, such as the hostname of the server. - - key - A secret string known only to the server. - - timestamp - An integer seconds-since-the-epoch timestamp - - """ - if timestamp is None: - timestamp = int(time.time()) - h = hexdigest('%s:%s:%s' % (timestamp, s, key), algorithm=algorithm) - nonce = '%s:%s' % (timestamp, h) - return nonce - - -def H(s, algorithm='MD5'): - """The hash function H""" - return hexdigest(s, algorithm=algorithm) - - -class HttpDigestAuthorization (object): - - """Class to parse a Digest Authorization header and perform re-calculation - of the digest. - """ - - def errmsg(self, s): - return 'Digest Authorization header: %s' % s - - def __init__(self, auth_header, http_method, debug=False): - self.http_method = http_method - self.debug = debug - scheme, params = auth_header.split(" ", 1) - self.scheme = scheme.lower() - if self.scheme != 'digest': - raise ValueError('Authorization scheme is not "Digest"') - - self.auth_header = auth_header - - # make a dict of the params - items = parse_http_list(params) - paramsd = parse_keqv_list(items) - - self.realm = paramsd.get('realm') - self.username = paramsd.get('username') - self.nonce = paramsd.get('nonce') - self.uri = paramsd.get('uri') - self.method = paramsd.get('method') - self.response = paramsd.get('response') # the response digest - self.algorithm = paramsd.get('algorithm', 'MD5').upper() - self.cnonce = paramsd.get('cnonce') - self.opaque = paramsd.get('opaque') - self.qop = paramsd.get('qop') # qop - self.nc = paramsd.get('nc') # nonce count - - # perform some correctness checks - if self.algorithm not in valid_algorithms: - raise ValueError( - self.errmsg("Unsupported value for algorithm: '%s'" % - self.algorithm)) - - has_reqd = ( - self.username and - self.realm and - self.nonce and - self.uri and - self.response - ) - if not has_reqd: - raise ValueError( - self.errmsg("Not all required parameters are present.")) - - if self.qop: - if self.qop not in valid_qops: - raise ValueError( - self.errmsg("Unsupported value for qop: '%s'" % self.qop)) - if not (self.cnonce and self.nc): - raise ValueError( - self.errmsg("If qop is sent then " - "cnonce and nc MUST be present")) - else: - if self.cnonce or self.nc: - raise ValueError( - self.errmsg("If qop is not sent, " - "neither cnonce nor nc can be present")) - - def __str__(self): - return 'authorization : %s' % self.auth_header - - def validate_nonce(self, s, key): - """Validate the nonce. - Returns True if nonce was generated by synthesize_nonce() and the - timestamp is not spoofed, else returns False. - - s - A string related to the resource, such as the hostname of - the server. - - key - A secret string known only to the server. - - Both s and key must be the same values which were used to synthesize - the nonce we are trying to validate. - """ - try: - timestamp, hashpart = self.nonce.split(':', 1) - s_timestamp, s_hashpart = synthesize_nonce( - s, key, timestamp, algorithm=self.algorithm).split(':', 1) - is_valid = s_hashpart == hashpart - if self.debug: - TRACE('validate_nonce: %s' % is_valid) - return is_valid - except ValueError: # split() error - pass - return False - - def is_nonce_stale(self, max_age_seconds=600): - """Returns True if a validated nonce is stale. The nonce contains a - timestamp in plaintext and also a secure hash of the timestamp. - You should first validate the nonce to ensure the plaintext - timestamp is not spoofed. - """ - try: - timestamp, hashpart = self.nonce.split(':', 1) - if int(timestamp) + max_age_seconds > int(time.time()): - return False - except ValueError: # int() error - pass - if self.debug: - TRACE("nonce is stale") - return True - - def HA2(self, entity_body=''): - """Returns the H(A2) string. See :rfc:`2617` section 3.2.2.3.""" - # RFC 2617 3.2.2.3 - # If the "qop" directive's value is "auth" or is unspecified, - # then A2 is: - # A2 = method ":" digest-uri-value - # - # If the "qop" value is "auth-int", then A2 is: - # A2 = method ":" digest-uri-value ":" H(entity-body) - if self.qop is None or self.qop == "auth": - a2 = '%s:%s' % (self.http_method, self.uri) - elif self.qop == "auth-int": - a2 = "%s:%s:%s" % (self.http_method, self.uri, - H(entity_body, algorithm=self.algorithm)) - else: - # in theory, this should never happen, since I validate qop in - # __init__() - raise ValueError(self.errmsg("Unrecognized value for qop!")) - return H(a2, algorithm=self.algorithm) - - def request_digest(self, ha1, entity_body=''): - """Calculates the Request-Digest. See :rfc:`2617` section 3.2.2.1. - - ha1 - The HA1 string obtained from the credentials store. - - entity_body - If 'qop' is set to 'auth-int', then A2 includes a hash - of the "entity body". The entity body is the part of the - message which follows the HTTP headers. See :rfc:`2617` section - 4.3. This refers to the entity the user agent sent in the - request which has the Authorization header. Typically GET - requests don't have an entity, and POST requests do. - - """ - ha2 = self.HA2(entity_body) - # Request-Digest -- RFC 2617 3.2.2.1 - if self.qop: - req = "%s:%s:%s:%s:%s" % ( - self.nonce, self.nc, self.cnonce, self.qop, ha2) - else: - req = "%s:%s" % (self.nonce, ha2) - - # RFC 2617 3.2.2.2 - # - # If the "algorithm" directive's value is "MD5" or is unspecified, - # then A1 is: - # A1 = unq(username-value) ":" unq(realm-value) ":" passwd - # - # If the "algorithm" directive's value is "MD5-sess", then A1 is - # calculated only once - on the first request by the client following - # receipt of a WWW-Authenticate challenge from the server. - # A1 = H( unq(username-value) ":" unq(realm-value) ":" passwd ) - # ":" unq(nonce-value) ":" unq(cnonce-value) - if self.algorithm == 'MD5-sess': - ha1 = H('%s:%s:%s' % (ha1, self.nonce, self.cnonce), - algorithm=self.algorithm) - - digest = H('%s:%s' % (ha1, req), algorithm=self.algorithm) - return digest - - -def www_authenticate(realm, key, algorithm='MD5', nonce=None, qop=qop_auth, - stale=False): - """Constructs a WWW-Authenticate header for Digest authentication.""" - if qop not in valid_qops: - raise ValueError("Unsupported value for qop: '%s'" % qop) - if algorithm not in valid_algorithms: - raise ValueError("Unsupported value for algorithm: '%s'" % algorithm) - - if nonce is None: - nonce = synthesize_nonce(realm, key, algorithm=algorithm) - s = 'Digest realm="%s", nonce="%s", algorithm="%s", qop="%s"' % ( - realm, nonce, algorithm, qop) - if stale: - s += ', stale="true"' - return s - - -def digest_auth(realm, get_ha1, key, algorithm='MD5', debug=False): - """A CherryPy tool which hooks at before_handler to perform - HTTP Digest Access Authentication, as specified in :rfc:`2617`. - - If the request has an 'authorization' header with a 'Digest' scheme, - this tool authenticates the credentials supplied in that header. - If the request has no 'authorization' header, or if it does but the - scheme is not "Digest", or if authentication fails, the tool sends - a 401 response with a 'WWW-Authenticate' Digest header. - - realm - A string containing the authentication realm. - - get_ha1 - A callable which looks up a username in a credentials store - and returns the HA1 string, which is defined in the RFC to be - MD5(username : realm : password). The function's signature is: - ``get_ha1(realm, username)`` - where username is obtained from the request's 'authorization' header. - If username is not found in the credentials store, get_ha1() returns - None. - - key - A secret string known only to the server, used in the synthesis - of nonces. - - """ - request = cherrypy.serving.request - - auth_header = request.headers.get('authorization') - nonce_is_stale = False - if auth_header is not None: - try: - auth = HttpDigestAuthorization( - auth_header, request.method, debug=debug) - except ValueError: - raise cherrypy.HTTPError( - 400, "The Authorization header could not be parsed.") - - if debug: - TRACE(str(auth)) - - if auth.validate_nonce(realm, key): - ha1 = get_ha1(realm, auth.username) - if ha1 is not None: - # note that for request.body to be available we need to - # hook in at before_handler, not on_start_resource like - # 3.1.x digest_auth does. - digest = auth.request_digest(ha1, entity_body=request.body) - if digest == auth.response: # authenticated - if debug: - TRACE("digest matches auth.response") - # Now check if nonce is stale. - # The choice of ten minutes' lifetime for nonce is somewhat - # arbitrary - nonce_is_stale = auth.is_nonce_stale(max_age_seconds=600) - if not nonce_is_stale: - request.login = auth.username - if debug: - TRACE("authentication of %s successful" % - auth.username) - return - - # Respond with 401 status and a WWW-Authenticate header - header = www_authenticate( - realm, key, algorithm=algorithm, stale=nonce_is_stale) - if debug: - TRACE(header) - cherrypy.serving.response.headers['WWW-Authenticate'] = header - raise cherrypy.HTTPError( - 401, "You are not authorized to access that resource") diff --git a/lib/cherrypy/lib/caching.py b/lib/cherrypy/lib/caching.py deleted file mode 100644 index 375d5f0ed72..00000000000 --- a/lib/cherrypy/lib/caching.py +++ /dev/null @@ -1,470 +0,0 @@ -""" -CherryPy implements a simple caching system as a pluggable Tool. This tool -tries to be an (in-process) HTTP/1.1-compliant cache. It's not quite there -yet, but it's probably good enough for most sites. - -In general, GET responses are cached (along with selecting headers) and, if -another request arrives for the same resource, the caching Tool will return 304 -Not Modified if possible, or serve the cached response otherwise. It also sets -request.cached to True if serving a cached representation, and sets -request.cacheable to False (so it doesn't get cached again). - -If POST, PUT, or DELETE requests are made for a cached resource, they -invalidate (delete) any cached response. - -Usage -===== - -Configuration file example:: - - [/] - tools.caching.on = True - tools.caching.delay = 3600 - -You may use a class other than the default -:class:`MemoryCache` by supplying the config -entry ``cache_class``; supply the full dotted name of the replacement class -as the config value. It must implement the basic methods ``get``, ``put``, -``delete``, and ``clear``. - -You may set any attribute, including overriding methods, on the cache -instance by providing them in config. The above sets the -:attr:`delay` attribute, for example. -""" - -import datetime -import sys -import threading -import time - -import cherrypy -from cherrypy.lib import cptools, httputil -from cherrypy._cpcompat import copyitems, ntob, set_daemon, sorted, Event - - -class Cache(object): - - """Base class for Cache implementations.""" - - def get(self): - """Return the current variant if in the cache, else None.""" - raise NotImplemented - - def put(self, obj, size): - """Store the current variant in the cache.""" - raise NotImplemented - - def delete(self): - """Remove ALL cached variants of the current resource.""" - raise NotImplemented - - def clear(self): - """Reset the cache to its initial, empty state.""" - raise NotImplemented - - -# ------------------------------ Memory Cache ------------------------------- # -class AntiStampedeCache(dict): - - """A storage system for cached items which reduces stampede collisions.""" - - def wait(self, key, timeout=5, debug=False): - """Return the cached value for the given key, or None. - - If timeout is not None, and the value is already - being calculated by another thread, wait until the given timeout has - elapsed. If the value is available before the timeout expires, it is - returned. If not, None is returned, and a sentinel placed in the cache - to signal other threads to wait. - - If timeout is None, no waiting is performed nor sentinels used. - """ - value = self.get(key) - if isinstance(value, Event): - if timeout is None: - # Ignore the other thread and recalc it ourselves. - if debug: - cherrypy.log('No timeout', 'TOOLS.CACHING') - return None - - # Wait until it's done or times out. - if debug: - cherrypy.log('Waiting up to %s seconds' % - timeout, 'TOOLS.CACHING') - value.wait(timeout) - if value.result is not None: - # The other thread finished its calculation. Use it. - if debug: - cherrypy.log('Result!', 'TOOLS.CACHING') - return value.result - # Timed out. Stick an Event in the slot so other threads wait - # on this one to finish calculating the value. - if debug: - cherrypy.log('Timed out', 'TOOLS.CACHING') - e = threading.Event() - e.result = None - dict.__setitem__(self, key, e) - - return None - elif value is None: - # Stick an Event in the slot so other threads wait - # on this one to finish calculating the value. - if debug: - cherrypy.log('Timed out', 'TOOLS.CACHING') - e = threading.Event() - e.result = None - dict.__setitem__(self, key, e) - return value - - def __setitem__(self, key, value): - """Set the cached value for the given key.""" - existing = self.get(key) - dict.__setitem__(self, key, value) - if isinstance(existing, Event): - # Set Event.result so other threads waiting on it have - # immediate access without needing to poll the cache again. - existing.result = value - existing.set() - - -class MemoryCache(Cache): - - """An in-memory cache for varying response content. - - Each key in self.store is a URI, and each value is an AntiStampedeCache. - The response for any given URI may vary based on the values of - "selecting request headers"; that is, those named in the Vary - response header. We assume the list of header names to be constant - for each URI throughout the lifetime of the application, and store - that list in ``self.store[uri].selecting_headers``. - - The items contained in ``self.store[uri]`` have keys which are tuples of - request header values (in the same order as the names in its - selecting_headers), and values which are the actual responses. - """ - - maxobjects = 1000 - """The maximum number of cached objects; defaults to 1000.""" - - maxobj_size = 100000 - """The maximum size of each cached object in bytes; defaults to 100 KB.""" - - maxsize = 10000000 - """The maximum size of the entire cache in bytes; defaults to 10 MB.""" - - delay = 600 - """Seconds until the cached content expires; defaults to 600 (10 minutes). - """ - - antistampede_timeout = 5 - """Seconds to wait for other threads to release a cache lock.""" - - expire_freq = 0.1 - """Seconds to sleep between cache expiration sweeps.""" - - debug = False - - def __init__(self): - self.clear() - - # Run self.expire_cache in a separate daemon thread. - t = threading.Thread(target=self.expire_cache, name='expire_cache') - self.expiration_thread = t - set_daemon(t, True) - t.start() - - def clear(self): - """Reset the cache to its initial, empty state.""" - self.store = {} - self.expirations = {} - self.tot_puts = 0 - self.tot_gets = 0 - self.tot_hist = 0 - self.tot_expires = 0 - self.tot_non_modified = 0 - self.cursize = 0 - - def expire_cache(self): - """Continuously examine cached objects, expiring stale ones. - - This function is designed to be run in its own daemon thread, - referenced at ``self.expiration_thread``. - """ - # It's possible that "time" will be set to None - # arbitrarily, so we check "while time" to avoid exceptions. - # See tickets #99 and #180 for more information. - while time: - now = time.time() - # Must make a copy of expirations so it doesn't change size - # during iteration - for expiration_time, objects in copyitems(self.expirations): - if expiration_time <= now: - for obj_size, uri, sel_header_values in objects: - try: - del self.store[uri][tuple(sel_header_values)] - self.tot_expires += 1 - self.cursize -= obj_size - except KeyError: - # the key may have been deleted elsewhere - pass - del self.expirations[expiration_time] - time.sleep(self.expire_freq) - - def get(self): - """Return the current variant if in the cache, else None.""" - request = cherrypy.serving.request - self.tot_gets += 1 - - uri = cherrypy.url(qs=request.query_string) - uricache = self.store.get(uri) - if uricache is None: - return None - - header_values = [request.headers.get(h, '') - for h in uricache.selecting_headers] - variant = uricache.wait(key=tuple(sorted(header_values)), - timeout=self.antistampede_timeout, - debug=self.debug) - if variant is not None: - self.tot_hist += 1 - return variant - - def put(self, variant, size): - """Store the current variant in the cache.""" - request = cherrypy.serving.request - response = cherrypy.serving.response - - uri = cherrypy.url(qs=request.query_string) - uricache = self.store.get(uri) - if uricache is None: - uricache = AntiStampedeCache() - uricache.selecting_headers = [ - e.value for e in response.headers.elements('Vary')] - self.store[uri] = uricache - - if len(self.store) < self.maxobjects: - total_size = self.cursize + size - - # checks if there's space for the object - if (size < self.maxobj_size and total_size < self.maxsize): - # add to the expirations list - expiration_time = response.time + self.delay - bucket = self.expirations.setdefault(expiration_time, []) - bucket.append((size, uri, uricache.selecting_headers)) - - # add to the cache - header_values = [request.headers.get(h, '') - for h in uricache.selecting_headers] - uricache[tuple(sorted(header_values))] = variant - self.tot_puts += 1 - self.cursize = total_size - - def delete(self): - """Remove ALL cached variants of the current resource.""" - uri = cherrypy.url(qs=cherrypy.serving.request.query_string) - self.store.pop(uri, None) - - -def get(invalid_methods=("POST", "PUT", "DELETE"), debug=False, **kwargs): - """Try to obtain cached output. If fresh enough, raise HTTPError(304). - - If POST, PUT, or DELETE: - * invalidates (deletes) any cached response for this resource - * sets request.cached = False - * sets request.cacheable = False - - else if a cached copy exists: - * sets request.cached = True - * sets request.cacheable = False - * sets response.headers to the cached values - * checks the cached Last-Modified response header against the - current If-(Un)Modified-Since request headers; raises 304 - if necessary. - * sets response.status and response.body to the cached values - * returns True - - otherwise: - * sets request.cached = False - * sets request.cacheable = True - * returns False - """ - request = cherrypy.serving.request - response = cherrypy.serving.response - - if not hasattr(cherrypy, "_cache"): - # Make a process-wide Cache object. - cherrypy._cache = kwargs.pop("cache_class", MemoryCache)() - - # Take all remaining kwargs and set them on the Cache object. - for k, v in kwargs.items(): - setattr(cherrypy._cache, k, v) - cherrypy._cache.debug = debug - - # POST, PUT, DELETE should invalidate (delete) the cached copy. - # See http://www.w3.org/Protocols/rfc2616/rfc2616-sec13.html#sec13.10. - if request.method in invalid_methods: - if debug: - cherrypy.log('request.method %r in invalid_methods %r' % - (request.method, invalid_methods), 'TOOLS.CACHING') - cherrypy._cache.delete() - request.cached = False - request.cacheable = False - return False - - if 'no-cache' in [e.value for e in request.headers.elements('Pragma')]: - request.cached = False - request.cacheable = True - return False - - cache_data = cherrypy._cache.get() - request.cached = bool(cache_data) - request.cacheable = not request.cached - if request.cached: - # Serve the cached copy. - max_age = cherrypy._cache.delay - for v in [e.value for e in request.headers.elements('Cache-Control')]: - atoms = v.split('=', 1) - directive = atoms.pop(0) - if directive == 'max-age': - if len(atoms) != 1 or not atoms[0].isdigit(): - raise cherrypy.HTTPError( - 400, "Invalid Cache-Control header") - max_age = int(atoms[0]) - break - elif directive == 'no-cache': - if debug: - cherrypy.log( - 'Ignoring cache due to Cache-Control: no-cache', - 'TOOLS.CACHING') - request.cached = False - request.cacheable = True - return False - - if debug: - cherrypy.log('Reading response from cache', 'TOOLS.CACHING') - s, h, b, create_time = cache_data - age = int(response.time - create_time) - if (age > max_age): - if debug: - cherrypy.log('Ignoring cache due to age > %d' % max_age, - 'TOOLS.CACHING') - request.cached = False - request.cacheable = True - return False - - # Copy the response headers. See - # https://github.com/cherrypy/cherrypy/issues/721. - response.headers = rh = httputil.HeaderMap() - for k in h: - dict.__setitem__(rh, k, dict.__getitem__(h, k)) - - # Add the required Age header - response.headers["Age"] = str(age) - - try: - # Note that validate_since depends on a Last-Modified header; - # this was put into the cached copy, and should have been - # resurrected just above (response.headers = cache_data[1]). - cptools.validate_since() - except cherrypy.HTTPRedirect: - x = sys.exc_info()[1] - if x.status == 304: - cherrypy._cache.tot_non_modified += 1 - raise - - # serve it & get out from the request - response.status = s - response.body = b - else: - if debug: - cherrypy.log('request is not cached', 'TOOLS.CACHING') - return request.cached - - -def tee_output(): - """Tee response output to cache storage. Internal.""" - # Used by CachingTool by attaching to request.hooks - - request = cherrypy.serving.request - if 'no-store' in request.headers.values('Cache-Control'): - return - - def tee(body): - """Tee response.body into a list.""" - if ('no-cache' in response.headers.values('Pragma') or - 'no-store' in response.headers.values('Cache-Control')): - for chunk in body: - yield chunk - return - - output = [] - for chunk in body: - output.append(chunk) - yield chunk - - # save the cache data - body = ntob('').join(output) - cherrypy._cache.put((response.status, response.headers or {}, - body, response.time), len(body)) - - response = cherrypy.serving.response - response.body = tee(response.body) - - -def expires(secs=0, force=False, debug=False): - """Tool for influencing cache mechanisms using the 'Expires' header. - - secs - Must be either an int or a datetime.timedelta, and indicates the - number of seconds between response.time and when the response should - expire. The 'Expires' header will be set to response.time + secs. - If secs is zero, the 'Expires' header is set one year in the past, and - the following "cache prevention" headers are also set: - - * Pragma: no-cache - * Cache-Control': no-cache, must-revalidate - - force - If False, the following headers are checked: - - * Etag - * Last-Modified - * Age - * Expires - - If any are already present, none of the above response headers are set. - - """ - - response = cherrypy.serving.response - headers = response.headers - - cacheable = False - if not force: - # some header names that indicate that the response can be cached - for indicator in ('Etag', 'Last-Modified', 'Age', 'Expires'): - if indicator in headers: - cacheable = True - break - - if not cacheable and not force: - if debug: - cherrypy.log('request is not cacheable', 'TOOLS.EXPIRES') - else: - if debug: - cherrypy.log('request is cacheable', 'TOOLS.EXPIRES') - if isinstance(secs, datetime.timedelta): - secs = (86400 * secs.days) + secs.seconds - - if secs == 0: - if force or ("Pragma" not in headers): - headers["Pragma"] = "no-cache" - if cherrypy.serving.request.protocol >= (1, 1): - if force or "Cache-Control" not in headers: - headers["Cache-Control"] = "no-cache, must-revalidate" - # Set an explicit Expires date in the past. - expiry = httputil.HTTPDate(1169942400.0) - else: - expiry = httputil.HTTPDate(response.time + secs) - if force or "Expires" not in headers: - headers["Expires"] = expiry diff --git a/lib/cherrypy/lib/covercp.py b/lib/cherrypy/lib/covercp.py deleted file mode 100644 index dd58a0c9b7b..00000000000 --- a/lib/cherrypy/lib/covercp.py +++ /dev/null @@ -1,390 +0,0 @@ -"""Code-coverage tools for CherryPy. - -To use this module, or the coverage tools in the test suite, -you need to download 'coverage.py', either Gareth Rees' `original -implementation `_ -or Ned Batchelder's `enhanced version: -`_ - -To turn on coverage tracing, use the following code:: - - cherrypy.engine.subscribe('start', covercp.start) - -DO NOT subscribe anything on the 'start_thread' channel, as previously -recommended. Calling start once in the main thread should be sufficient -to start coverage on all threads. Calling start again in each thread -effectively clears any coverage data gathered up to that point. - -Run your code, then use the ``covercp.serve()`` function to browse the -results in a web browser. If you run this module from the command line, -it will call ``serve()`` for you. -""" - -import re -import sys -import cgi -import os -import os.path -localFile = os.path.join(os.path.dirname(__file__), "coverage.cache") - -from cherrypy._cpcompat import quote_plus - -import cherrypy - -the_coverage = None -try: - from coverage import coverage - the_coverage = coverage(data_file=localFile) - - def start(): - the_coverage.start() -except ImportError: - # Setting the_coverage to None will raise errors - # that need to be trapped downstream. - the_coverage = None - - import warnings - warnings.warn( - "No code coverage will be performed; " - "coverage.py could not be imported.") - - def start(): - pass -start.priority = 20 - -TEMPLATE_MENU = """ - - CherryPy Coverage Menu - - - -

CherryPy Coverage

""" - -TEMPLATE_FORM = """ -
-
- - Show percentages -
- Hide files over - %%
- Exclude files matching
- -
- - -
-
""" - -TEMPLATE_FRAMESET = """ -CherryPy coverage data - - - - - -""" - -TEMPLATE_COVERAGE = """ - - Coverage for %(name)s - - - -

%(name)s

-

%(fullpath)s

-

Coverage: %(pc)s%%

""" - -TEMPLATE_LOC_COVERED = """ - %s  - %s -\n""" -TEMPLATE_LOC_NOT_COVERED = """ - %s  - %s -\n""" -TEMPLATE_LOC_EXCLUDED = """ - %s  - %s -\n""" - -TEMPLATE_ITEM = ( - "%s%s%s\n" -) - - -def _percent(statements, missing): - s = len(statements) - e = s - len(missing) - if s > 0: - return int(round(100.0 * e / s)) - return 0 - - -def _show_branch(root, base, path, pct=0, showpct=False, exclude="", - coverage=the_coverage): - - # Show the directory name and any of our children - dirs = [k for k, v in root.items() if v] - dirs.sort() - for name in dirs: - newpath = os.path.join(path, name) - - if newpath.lower().startswith(base): - relpath = newpath[len(base):] - yield "| " * relpath.count(os.sep) - yield ( - "%s\n" % - (newpath, quote_plus(exclude), name) - ) - - for chunk in _show_branch( - root[name], base, newpath, pct, showpct, - exclude, coverage=coverage - ): - yield chunk - - # Now list the files - if path.lower().startswith(base): - relpath = path[len(base):] - files = [k for k, v in root.items() if not v] - files.sort() - for name in files: - newpath = os.path.join(path, name) - - pc_str = "" - if showpct: - try: - _, statements, _, missing, _ = coverage.analysis2(newpath) - except: - # Yes, we really want to pass on all errors. - pass - else: - pc = _percent(statements, missing) - pc_str = ("%3d%% " % pc).replace(' ', ' ') - if pc < float(pct) or pc == -1: - pc_str = "%s" % pc_str - else: - pc_str = "%s" % pc_str - - yield TEMPLATE_ITEM % ("| " * (relpath.count(os.sep) + 1), - pc_str, newpath, name) - - -def _skip_file(path, exclude): - if exclude: - return bool(re.search(exclude, path)) - - -def _graft(path, tree): - d = tree - - p = path - atoms = [] - while True: - p, tail = os.path.split(p) - if not tail: - break - atoms.append(tail) - atoms.append(p) - if p != "/": - atoms.append("/") - - atoms.reverse() - for node in atoms: - if node: - d = d.setdefault(node, {}) - - -def get_tree(base, exclude, coverage=the_coverage): - """Return covered module names as a nested dict.""" - tree = {} - runs = coverage.data.executed_files() - for path in runs: - if not _skip_file(path, exclude) and not os.path.isdir(path): - _graft(path, tree) - return tree - - -class CoverStats(object): - - def __init__(self, coverage, root=None): - self.coverage = coverage - if root is None: - # Guess initial depth. Files outside this path will not be - # reachable from the web interface. - import cherrypy - root = os.path.dirname(cherrypy.__file__) - self.root = root - - @cherrypy.expose - def index(self): - return TEMPLATE_FRAMESET % self.root.lower() - - @cherrypy.expose - def menu(self, base="/", pct="50", showpct="", - exclude=r'python\d\.\d|test|tut\d|tutorial'): - - # The coverage module uses all-lower-case names. - base = base.lower().rstrip(os.sep) - - yield TEMPLATE_MENU - yield TEMPLATE_FORM % locals() - - # Start by showing links for parent paths - yield "
" - path = "" - atoms = base.split(os.sep) - atoms.pop() - for atom in atoms: - path += atom + os.sep - yield ("%s %s" - % (path, quote_plus(exclude), atom, os.sep)) - yield "
" - - yield "
" - - # Then display the tree - tree = get_tree(base, exclude, self.coverage) - if not tree: - yield "

No modules covered.

" - else: - for chunk in _show_branch(tree, base, "/", pct, - showpct == 'checked', exclude, - coverage=self.coverage): - yield chunk - - yield "
" - yield "" - - def annotated_file(self, filename, statements, excluded, missing): - source = open(filename, 'r') - buffer = [] - for lineno, line in enumerate(source.readlines()): - lineno += 1 - line = line.strip("\n\r") - empty_the_buffer = True - if lineno in excluded: - template = TEMPLATE_LOC_EXCLUDED - elif lineno in missing: - template = TEMPLATE_LOC_NOT_COVERED - elif lineno in statements: - template = TEMPLATE_LOC_COVERED - else: - empty_the_buffer = False - buffer.append((lineno, line)) - if empty_the_buffer: - for lno, pastline in buffer: - yield template % (lno, cgi.escape(pastline)) - buffer = [] - yield template % (lineno, cgi.escape(line)) - - @cherrypy.expose - def report(self, name): - filename, statements, excluded, missing, _ = self.coverage.analysis2( - name) - pc = _percent(statements, missing) - yield TEMPLATE_COVERAGE % dict(name=os.path.basename(name), - fullpath=name, - pc=pc) - yield '\n' - for line in self.annotated_file(filename, statements, excluded, - missing): - yield line - yield '
' - yield '' - yield '' - - -def serve(path=localFile, port=8080, root=None): - if coverage is None: - raise ImportError("The coverage module could not be imported.") - from coverage import coverage - cov = coverage(data_file=path) - cov.load() - - import cherrypy - cherrypy.config.update({'server.socket_port': int(port), - 'server.thread_pool': 10, - 'environment': "production", - }) - cherrypy.quickstart(CoverStats(cov, root)) - -if __name__ == "__main__": - serve(*tuple(sys.argv[1:])) diff --git a/lib/cherrypy/lib/cpstats.py b/lib/cherrypy/lib/cpstats.py deleted file mode 100644 index 49d1320f479..00000000000 --- a/lib/cherrypy/lib/cpstats.py +++ /dev/null @@ -1,699 +0,0 @@ -"""CPStats, a package for collecting and reporting on program statistics. - -Overview -======== - -Statistics about program operation are an invaluable monitoring and debugging -tool. Unfortunately, the gathering and reporting of these critical values is -usually ad-hoc. This package aims to add a centralized place for gathering -statistical performance data, a structure for recording that data which -provides for extrapolation of that data into more useful information, -and a method of serving that data to both human investigators and -monitoring software. Let's examine each of those in more detail. - -Data Gathering --------------- - -Just as Python's `logging` module provides a common importable for gathering -and sending messages, performance statistics would benefit from a similar -common mechanism, and one that does *not* require each package which wishes -to collect stats to import a third-party module. Therefore, we choose to -re-use the `logging` module by adding a `statistics` object to it. - -That `logging.statistics` object is a nested dict. It is not a custom class, -because that would: - - 1. require libraries and applications to import a third-party module in - order to participate - 2. inhibit innovation in extrapolation approaches and in reporting tools, and - 3. be slow. - -There are, however, some specifications regarding the structure of the dict.:: - - { - +----"SQLAlchemy": { - | "Inserts": 4389745, - | "Inserts per Second": - | lambda s: s["Inserts"] / (time() - s["Start"]), - | C +---"Table Statistics": { - | o | "widgets": {-----------+ - N | l | "Rows": 1.3M, | Record - a | l | "Inserts": 400, | - m | e | },---------------------+ - e | c | "froobles": { - s | t | "Rows": 7845, - p | i | "Inserts": 0, - a | o | }, - c | n +---}, - e | "Slow Queries": - | [{"Query": "SELECT * FROM widgets;", - | "Processing Time": 47.840923343, - | }, - | ], - +----}, - } - -The `logging.statistics` dict has four levels. The topmost level is nothing -more than a set of names to introduce modularity, usually along the lines of -package names. If the SQLAlchemy project wanted to participate, for example, -it might populate the item `logging.statistics['SQLAlchemy']`, whose value -would be a second-layer dict we call a "namespace". Namespaces help multiple -packages to avoid collisions over key names, and make reports easier to read, -to boot. The maintainers of SQLAlchemy should feel free to use more than one -namespace if needed (such as 'SQLAlchemy ORM'). Note that there are no case -or other syntax constraints on the namespace names; they should be chosen -to be maximally readable by humans (neither too short nor too long). - -Each namespace, then, is a dict of named statistical values, such as -'Requests/sec' or 'Uptime'. You should choose names which will look -good on a report: spaces and capitalization are just fine. - -In addition to scalars, values in a namespace MAY be a (third-layer) -dict, or a list, called a "collection". For example, the CherryPy -:class:`StatsTool` keeps track of what each request is doing (or has most -recently done) in a 'Requests' collection, where each key is a thread ID; each -value in the subdict MUST be a fourth dict (whew!) of statistical data about -each thread. We call each subdict in the collection a "record". Similarly, -the :class:`StatsTool` also keeps a list of slow queries, where each record -contains data about each slow query, in order. - -Values in a namespace or record may also be functions, which brings us to: - -Extrapolation -------------- - -The collection of statistical data needs to be fast, as close to unnoticeable -as possible to the host program. That requires us to minimize I/O, for example, -but in Python it also means we need to minimize function calls. So when you -are designing your namespace and record values, try to insert the most basic -scalar values you already have on hand. - -When it comes time to report on the gathered data, however, we usually have -much more freedom in what we can calculate. Therefore, whenever reporting -tools (like the provided :class:`StatsPage` CherryPy class) fetch the contents -of `logging.statistics` for reporting, they first call -`extrapolate_statistics` (passing the whole `statistics` dict as the only -argument). This makes a deep copy of the statistics dict so that the -reporting tool can both iterate over it and even change it without harming -the original. But it also expands any functions in the dict by calling them. -For example, you might have a 'Current Time' entry in the namespace with the -value "lambda scope: time.time()". The "scope" parameter is the current -namespace dict (or record, if we're currently expanding one of those -instead), allowing you access to existing static entries. If you're truly -evil, you can even modify more than one entry at a time. - -However, don't try to calculate an entry and then use its value in further -extrapolations; the order in which the functions are called is not guaranteed. -This can lead to a certain amount of duplicated work (or a redesign of your -schema), but that's better than complicating the spec. - -After the whole thing has been extrapolated, it's time for: - -Reporting ---------- - -The :class:`StatsPage` class grabs the `logging.statistics` dict, extrapolates -it all, and then transforms it to HTML for easy viewing. Each namespace gets -its own header and attribute table, plus an extra table for each collection. -This is NOT part of the statistics specification; other tools can format how -they like. - -You can control which columns are output and how they are formatted by updating -StatsPage.formatting, which is a dict that mirrors the keys and nesting of -`logging.statistics`. The difference is that, instead of data values, it has -formatting values. Use None for a given key to indicate to the StatsPage that a -given column should not be output. Use a string with formatting -(such as '%.3f') to interpolate the value(s), or use a callable (such as -lambda v: v.isoformat()) for more advanced formatting. Any entry which is not -mentioned in the formatting dict is output unchanged. - -Monitoring ----------- - -Although the HTML output takes pains to assign unique id's to each with -statistical data, you're probably better off fetching /cpstats/data, which -outputs the whole (extrapolated) `logging.statistics` dict in JSON format. -That is probably easier to parse, and doesn't have any formatting controls, -so you get the "original" data in a consistently-serialized format. -Note: there's no treatment yet for datetime objects. Try time.time() instead -for now if you can. Nagios will probably thank you. - -Turning Collection Off ----------------------- - -It is recommended each namespace have an "Enabled" item which, if False, -stops collection (but not reporting) of statistical data. Applications -SHOULD provide controls to pause and resume collection by setting these -entries to False or True, if present. - - -Usage -===== - -To collect statistics on CherryPy applications:: - - from cherrypy.lib import cpstats - appconfig['/']['tools.cpstats.on'] = True - -To collect statistics on your own code:: - - import logging - # Initialize the repository - if not hasattr(logging, 'statistics'): logging.statistics = {} - # Initialize my namespace - mystats = logging.statistics.setdefault('My Stuff', {}) - # Initialize my namespace's scalars and collections - mystats.update({ - 'Enabled': True, - 'Start Time': time.time(), - 'Important Events': 0, - 'Events/Second': lambda s: ( - (s['Important Events'] / (time.time() - s['Start Time']))), - }) - ... - for event in events: - ... - # Collect stats - if mystats.get('Enabled', False): - mystats['Important Events'] += 1 - -To report statistics:: - - root.cpstats = cpstats.StatsPage() - -To format statistics reports:: - - See 'Reporting', above. - -""" - -# ------------------------------- Statistics -------------------------------- # - -import logging -if not hasattr(logging, 'statistics'): - logging.statistics = {} - - -def extrapolate_statistics(scope): - """Return an extrapolated copy of the given scope.""" - c = {} - for k, v in list(scope.items()): - if isinstance(v, dict): - v = extrapolate_statistics(v) - elif isinstance(v, (list, tuple)): - v = [extrapolate_statistics(record) for record in v] - elif hasattr(v, '__call__'): - v = v(scope) - c[k] = v - return c - - -# -------------------- CherryPy Applications Statistics --------------------- # - -import sys -import threading -import time - -import cherrypy - -appstats = logging.statistics.setdefault('CherryPy Applications', {}) -appstats.update({ - 'Enabled': True, - 'Bytes Read/Request': lambda s: ( - s['Total Requests'] and - (s['Total Bytes Read'] / float(s['Total Requests'])) or - 0.0 - ), - 'Bytes Read/Second': lambda s: s['Total Bytes Read'] / s['Uptime'](s), - 'Bytes Written/Request': lambda s: ( - s['Total Requests'] and - (s['Total Bytes Written'] / float(s['Total Requests'])) or - 0.0 - ), - 'Bytes Written/Second': lambda s: ( - s['Total Bytes Written'] / s['Uptime'](s) - ), - 'Current Time': lambda s: time.time(), - 'Current Requests': 0, - 'Requests/Second': lambda s: float(s['Total Requests']) / s['Uptime'](s), - 'Server Version': cherrypy.__version__, - 'Start Time': time.time(), - 'Total Bytes Read': 0, - 'Total Bytes Written': 0, - 'Total Requests': 0, - 'Total Time': 0, - 'Uptime': lambda s: time.time() - s['Start Time'], - 'Requests': {}, -}) - -proc_time = lambda s: time.time() - s['Start Time'] - - -class ByteCountWrapper(object): - - """Wraps a file-like object, counting the number of bytes read.""" - - def __init__(self, rfile): - self.rfile = rfile - self.bytes_read = 0 - - def read(self, size=-1): - data = self.rfile.read(size) - self.bytes_read += len(data) - return data - - def readline(self, size=-1): - data = self.rfile.readline(size) - self.bytes_read += len(data) - return data - - def readlines(self, sizehint=0): - # Shamelessly stolen from StringIO - total = 0 - lines = [] - line = self.readline() - while line: - lines.append(line) - total += len(line) - if 0 < sizehint <= total: - break - line = self.readline() - return lines - - def close(self): - self.rfile.close() - - def __iter__(self): - return self - - def next(self): - data = self.rfile.next() - self.bytes_read += len(data) - return data - - -average_uriset_time = lambda s: s['Count'] and (s['Sum'] / s['Count']) or 0 - - -def _get_threading_ident(): - if sys.version_info >= (3, 3): - return threading.get_ident() - return threading._get_ident() - -class StatsTool(cherrypy.Tool): - - """Record various information about the current request.""" - - def __init__(self): - cherrypy.Tool.__init__(self, 'on_end_request', self.record_stop) - - def _setup(self): - """Hook this tool into cherrypy.request. - - The standard CherryPy request object will automatically call this - method when the tool is "turned on" in config. - """ - if appstats.get('Enabled', False): - cherrypy.Tool._setup(self) - self.record_start() - - def record_start(self): - """Record the beginning of a request.""" - request = cherrypy.serving.request - if not hasattr(request.rfile, 'bytes_read'): - request.rfile = ByteCountWrapper(request.rfile) - request.body.fp = request.rfile - - r = request.remote - - appstats['Current Requests'] += 1 - appstats['Total Requests'] += 1 - appstats['Requests'][_get_threading_ident()] = { - 'Bytes Read': None, - 'Bytes Written': None, - # Use a lambda so the ip gets updated by tools.proxy later - 'Client': lambda s: '%s:%s' % (r.ip, r.port), - 'End Time': None, - 'Processing Time': proc_time, - 'Request-Line': request.request_line, - 'Response Status': None, - 'Start Time': time.time(), - } - - def record_stop( - self, uriset=None, slow_queries=1.0, slow_queries_count=100, - debug=False, **kwargs): - """Record the end of a request.""" - resp = cherrypy.serving.response - w = appstats['Requests'][_get_threading_ident()] - - r = cherrypy.request.rfile.bytes_read - w['Bytes Read'] = r - appstats['Total Bytes Read'] += r - - if resp.stream: - w['Bytes Written'] = 'chunked' - else: - cl = int(resp.headers.get('Content-Length', 0)) - w['Bytes Written'] = cl - appstats['Total Bytes Written'] += cl - - w['Response Status'] = getattr( - resp, 'output_status', None) or resp.status - - w['End Time'] = time.time() - p = w['End Time'] - w['Start Time'] - w['Processing Time'] = p - appstats['Total Time'] += p - - appstats['Current Requests'] -= 1 - - if debug: - cherrypy.log('Stats recorded: %s' % repr(w), 'TOOLS.CPSTATS') - - if uriset: - rs = appstats.setdefault('URI Set Tracking', {}) - r = rs.setdefault(uriset, { - 'Min': None, 'Max': None, 'Count': 0, 'Sum': 0, - 'Avg': average_uriset_time}) - if r['Min'] is None or p < r['Min']: - r['Min'] = p - if r['Max'] is None or p > r['Max']: - r['Max'] = p - r['Count'] += 1 - r['Sum'] += p - - if slow_queries and p > slow_queries: - sq = appstats.setdefault('Slow Queries', []) - sq.append(w.copy()) - if len(sq) > slow_queries_count: - sq.pop(0) - - -import cherrypy -cherrypy.tools.cpstats = StatsTool() - - -# ---------------------- CherryPy Statistics Reporting ---------------------- # - -import os -thisdir = os.path.abspath(os.path.dirname(__file__)) - -try: - import json -except ImportError: - try: - import simplejson as json - except ImportError: - json = None - - -missing = object() - -locale_date = lambda v: time.strftime('%c', time.gmtime(v)) -iso_format = lambda v: time.strftime('%Y-%m-%d %H:%M:%S', time.gmtime(v)) - - -def pause_resume(ns): - def _pause_resume(enabled): - pause_disabled = '' - resume_disabled = '' - if enabled: - resume_disabled = 'disabled="disabled" ' - else: - pause_disabled = 'disabled="disabled" ' - return """ -
- - -
-
- - -
- """ % (ns, pause_disabled, ns, resume_disabled) - return _pause_resume - - -class StatsPage(object): - - formatting = { - 'CherryPy Applications': { - 'Enabled': pause_resume('CherryPy Applications'), - 'Bytes Read/Request': '%.3f', - 'Bytes Read/Second': '%.3f', - 'Bytes Written/Request': '%.3f', - 'Bytes Written/Second': '%.3f', - 'Current Time': iso_format, - 'Requests/Second': '%.3f', - 'Start Time': iso_format, - 'Total Time': '%.3f', - 'Uptime': '%.3f', - 'Slow Queries': { - 'End Time': None, - 'Processing Time': '%.3f', - 'Start Time': iso_format, - }, - 'URI Set Tracking': { - 'Avg': '%.3f', - 'Max': '%.3f', - 'Min': '%.3f', - 'Sum': '%.3f', - }, - 'Requests': { - 'Bytes Read': '%s', - 'Bytes Written': '%s', - 'End Time': None, - 'Processing Time': '%.3f', - 'Start Time': None, - }, - }, - 'CherryPy WSGIServer': { - 'Enabled': pause_resume('CherryPy WSGIServer'), - 'Connections/second': '%.3f', - 'Start time': iso_format, - }, - } - - @cherrypy.expose - def index(self): - # Transform the raw data into pretty output for HTML - yield """ - - - Statistics - - - -""" - for title, scalars, collections in self.get_namespaces(): - yield """ -

%s

- - - -""" % title - for i, (key, value) in enumerate(scalars): - colnum = i % 3 - if colnum == 0: - yield """ - """ - yield ( - """ - """ % - vars() - ) - if colnum == 2: - yield """ - """ - - if colnum == 0: - yield """ - - - """ - elif colnum == 1: - yield """ - - """ - yield """ - -
%(key)s%(value)s
""" - - for subtitle, headers, subrows in collections: - yield """ -

%s

- - - """ % subtitle - for key in headers: - yield """ - """ % key - yield """ - - - """ - for subrow in subrows: - yield """ - """ - for value in subrow: - yield """ - """ % value - yield """ - """ - yield """ - -
%s
%s
""" - yield """ - - -""" - - def get_namespaces(self): - """Yield (title, scalars, collections) for each namespace.""" - s = extrapolate_statistics(logging.statistics) - for title, ns in sorted(s.items()): - scalars = [] - collections = [] - ns_fmt = self.formatting.get(title, {}) - for k, v in sorted(ns.items()): - fmt = ns_fmt.get(k, {}) - if isinstance(v, dict): - headers, subrows = self.get_dict_collection(v, fmt) - collections.append((k, ['ID'] + headers, subrows)) - elif isinstance(v, (list, tuple)): - headers, subrows = self.get_list_collection(v, fmt) - collections.append((k, headers, subrows)) - else: - format = ns_fmt.get(k, missing) - if format is None: - # Don't output this column. - continue - if hasattr(format, '__call__'): - v = format(v) - elif format is not missing: - v = format % v - scalars.append((k, v)) - yield title, scalars, collections - - def get_dict_collection(self, v, formatting): - """Return ([headers], [rows]) for the given collection.""" - # E.g., the 'Requests' dict. - headers = [] - try: - # python2 - vals = v.itervalues() - except AttributeError: - # python3 - vals = v.values() - for record in vals: - for k3 in record: - format = formatting.get(k3, missing) - if format is None: - # Don't output this column. - continue - if k3 not in headers: - headers.append(k3) - headers.sort() - - subrows = [] - for k2, record in sorted(v.items()): - subrow = [k2] - for k3 in headers: - v3 = record.get(k3, '') - format = formatting.get(k3, missing) - if format is None: - # Don't output this column. - continue - if hasattr(format, '__call__'): - v3 = format(v3) - elif format is not missing: - v3 = format % v3 - subrow.append(v3) - subrows.append(subrow) - - return headers, subrows - - def get_list_collection(self, v, formatting): - """Return ([headers], [subrows]) for the given collection.""" - # E.g., the 'Slow Queries' list. - headers = [] - for record in v: - for k3 in record: - format = formatting.get(k3, missing) - if format is None: - # Don't output this column. - continue - if k3 not in headers: - headers.append(k3) - headers.sort() - - subrows = [] - for record in v: - subrow = [] - for k3 in headers: - v3 = record.get(k3, '') - format = formatting.get(k3, missing) - if format is None: - # Don't output this column. - continue - if hasattr(format, '__call__'): - v3 = format(v3) - elif format is not missing: - v3 = format % v3 - subrow.append(v3) - subrows.append(subrow) - - return headers, subrows - - if json is not None: - @cherrypy.expose - def data(self): - s = extrapolate_statistics(logging.statistics) - cherrypy.response.headers['Content-Type'] = 'application/json' - return json.dumps(s, sort_keys=True, indent=4) - - @cherrypy.expose - def pause(self, namespace): - logging.statistics.get(namespace, {})['Enabled'] = False - raise cherrypy.HTTPRedirect('./') - pause.cp_config = {'tools.allow.on': True, - 'tools.allow.methods': ['POST']} - - @cherrypy.expose - def resume(self, namespace): - logging.statistics.get(namespace, {})['Enabled'] = True - raise cherrypy.HTTPRedirect('./') - resume.cp_config = {'tools.allow.on': True, - 'tools.allow.methods': ['POST']} diff --git a/lib/cherrypy/lib/cptools.py b/lib/cherrypy/lib/cptools.py deleted file mode 100644 index c4d4ef48eb3..00000000000 --- a/lib/cherrypy/lib/cptools.py +++ /dev/null @@ -1,642 +0,0 @@ -"""Functions for builtin CherryPy tools.""" - -import hashlib -import logging -import re -import sys - -IS_PY3 = sys.version_info[0] == 3 -try: - import six -except ImportError: - if IS_PY3: - _unicode_type = str - else: - _unicode_type = unicode -else: - _unicode_type = six.text_type - -import cherrypy -from cherrypy._cpcompat import basestring -from cherrypy.lib import httputil as _httputil -from cherrypy.lib import is_iterator - - -# Conditional HTTP request support # - -def validate_etags(autotags=False, debug=False): - """Validate the current ETag against If-Match, If-None-Match headers. - - If autotags is True, an ETag response-header value will be provided - from an MD5 hash of the response body (unless some other code has - already provided an ETag header). If False (the default), the ETag - will not be automatic. - - WARNING: the autotags feature is not designed for URL's which allow - methods other than GET. For example, if a POST to the same URL returns - no content, the automatic ETag will be incorrect, breaking a fundamental - use for entity tags in a possibly destructive fashion. Likewise, if you - raise 304 Not Modified, the response body will be empty, the ETag hash - will be incorrect, and your application will break. - See :rfc:`2616` Section 14.24. - """ - response = cherrypy.serving.response - - # Guard against being run twice. - if hasattr(response, "ETag"): - return - - status, reason, msg = _httputil.valid_status(response.status) - - etag = response.headers.get('ETag') - - # Automatic ETag generation. See warning in docstring. - if etag: - if debug: - cherrypy.log('ETag already set: %s' % etag, 'TOOLS.ETAGS') - elif not autotags: - if debug: - cherrypy.log('Autotags off', 'TOOLS.ETAGS') - elif status != 200: - if debug: - cherrypy.log('Status not 200', 'TOOLS.ETAGS') - else: - etag = response.collapse_body() - etag = '"%s"' % hashlib.md5(etag).hexdigest() - if debug: - cherrypy.log('Setting ETag: %s' % etag, 'TOOLS.ETAGS') - response.headers['ETag'] = etag - - response.ETag = etag - - # "If the request would, without the If-Match header field, result in - # anything other than a 2xx or 412 status, then the If-Match header - # MUST be ignored." - if debug: - cherrypy.log('Status: %s' % status, 'TOOLS.ETAGS') - if status >= 200 and status <= 299: - request = cherrypy.serving.request - - conditions = request.headers.elements('If-Match') or [] - conditions = [str(x) for x in conditions] - if debug: - cherrypy.log('If-Match conditions: %s' % repr(conditions), - 'TOOLS.ETAGS') - if conditions and not (conditions == ["*"] or etag in conditions): - raise cherrypy.HTTPError(412, "If-Match failed: ETag %r did " - "not match %r" % (etag, conditions)) - - conditions = request.headers.elements('If-None-Match') or [] - conditions = [str(x) for x in conditions] - if debug: - cherrypy.log('If-None-Match conditions: %s' % repr(conditions), - 'TOOLS.ETAGS') - if conditions == ["*"] or etag in conditions: - if debug: - cherrypy.log('request.method: %s' % - request.method, 'TOOLS.ETAGS') - if request.method in ("GET", "HEAD"): - raise cherrypy.HTTPRedirect([], 304) - else: - raise cherrypy.HTTPError(412, "If-None-Match failed: ETag %r " - "matched %r" % (etag, conditions)) - - -def validate_since(): - """Validate the current Last-Modified against If-Modified-Since headers. - - If no code has set the Last-Modified response header, then no validation - will be performed. - """ - response = cherrypy.serving.response - lastmod = response.headers.get('Last-Modified') - if lastmod: - status, reason, msg = _httputil.valid_status(response.status) - - request = cherrypy.serving.request - - since = request.headers.get('If-Unmodified-Since') - if since and since != lastmod: - if (status >= 200 and status <= 299) or status == 412: - raise cherrypy.HTTPError(412) - - since = request.headers.get('If-Modified-Since') - if since and since == lastmod: - if (status >= 200 and status <= 299) or status == 304: - if request.method in ("GET", "HEAD"): - raise cherrypy.HTTPRedirect([], 304) - else: - raise cherrypy.HTTPError(412) - - -# Tool code # - -def allow(methods=None, debug=False): - """Raise 405 if request.method not in methods (default ['GET', 'HEAD']). - - The given methods are case-insensitive, and may be in any order. - If only one method is allowed, you may supply a single string; - if more than one, supply a list of strings. - - Regardless of whether the current method is allowed or not, this - also emits an 'Allow' response header, containing the given methods. - """ - if not isinstance(methods, (tuple, list)): - methods = [methods] - methods = [m.upper() for m in methods if m] - if not methods: - methods = ['GET', 'HEAD'] - elif 'GET' in methods and 'HEAD' not in methods: - methods.append('HEAD') - - cherrypy.response.headers['Allow'] = ', '.join(methods) - if cherrypy.request.method not in methods: - if debug: - cherrypy.log('request.method %r not in methods %r' % - (cherrypy.request.method, methods), 'TOOLS.ALLOW') - raise cherrypy.HTTPError(405) - else: - if debug: - cherrypy.log('request.method %r in methods %r' % - (cherrypy.request.method, methods), 'TOOLS.ALLOW') - - -def proxy(base=None, local='X-Forwarded-Host', remote='X-Forwarded-For', - scheme='X-Forwarded-Proto', debug=False): - """Change the base URL (scheme://host[:port][/path]). - - For running a CP server behind Apache, lighttpd, or other HTTP server. - - For Apache and lighttpd, you should leave the 'local' argument at the - default value of 'X-Forwarded-Host'. For Squid, you probably want to set - tools.proxy.local = 'Origin'. - - If you want the new request.base to include path info (not just the host), - you must explicitly set base to the full base path, and ALSO set 'local' - to '', so that the X-Forwarded-Host request header (which never includes - path info) does not override it. Regardless, the value for 'base' MUST - NOT end in a slash. - - cherrypy.request.remote.ip (the IP address of the client) will be - rewritten if the header specified by the 'remote' arg is valid. - By default, 'remote' is set to 'X-Forwarded-For'. If you do not - want to rewrite remote.ip, set the 'remote' arg to an empty string. - """ - - request = cherrypy.serving.request - - if scheme: - s = request.headers.get(scheme, None) - if debug: - cherrypy.log('Testing scheme %r:%r' % (scheme, s), 'TOOLS.PROXY') - if s == 'on' and 'ssl' in scheme.lower(): - # This handles e.g. webfaction's 'X-Forwarded-Ssl: on' header - scheme = 'https' - else: - # This is for lighttpd/pound/Mongrel's 'X-Forwarded-Proto: https' - scheme = s - if not scheme: - scheme = request.base[:request.base.find("://")] - - if local: - lbase = request.headers.get(local, None) - if debug: - cherrypy.log('Testing local %r:%r' % (local, lbase), 'TOOLS.PROXY') - if lbase is not None: - base = lbase.split(',')[0] - if not base: - base = request.headers.get('Host', '127.0.0.1') - port = request.local.port - if port != 80: - base += ':%s' % port - - if base.find("://") == -1: - # add http:// or https:// if needed - base = scheme + "://" + base - - request.base = base - - if remote: - xff = request.headers.get(remote) - if debug: - cherrypy.log('Testing remote %r:%r' % (remote, xff), 'TOOLS.PROXY') - if xff: - if remote == 'X-Forwarded-For': - #Bug #1268 - xff = xff.split(',')[0].strip() - request.remote.ip = xff - - -def ignore_headers(headers=('Range',), debug=False): - """Delete request headers whose field names are included in 'headers'. - - This is a useful tool for working behind certain HTTP servers; - for example, Apache duplicates the work that CP does for 'Range' - headers, and will doubly-truncate the response. - """ - request = cherrypy.serving.request - for name in headers: - if name in request.headers: - if debug: - cherrypy.log('Ignoring request header %r' % name, - 'TOOLS.IGNORE_HEADERS') - del request.headers[name] - - -def response_headers(headers=None, debug=False): - """Set headers on the response.""" - if debug: - cherrypy.log('Setting response headers: %s' % repr(headers), - 'TOOLS.RESPONSE_HEADERS') - for name, value in (headers or []): - cherrypy.serving.response.headers[name] = value -response_headers.failsafe = True - - -def referer(pattern, accept=True, accept_missing=False, error=403, - message='Forbidden Referer header.', debug=False): - """Raise HTTPError if Referer header does/does not match the given pattern. - - pattern - A regular expression pattern to test against the Referer. - - accept - If True, the Referer must match the pattern; if False, - the Referer must NOT match the pattern. - - accept_missing - If True, permit requests with no Referer header. - - error - The HTTP error code to return to the client on failure. - - message - A string to include in the response body on failure. - - """ - try: - ref = cherrypy.serving.request.headers['Referer'] - match = bool(re.match(pattern, ref)) - if debug: - cherrypy.log('Referer %r matches %r' % (ref, pattern), - 'TOOLS.REFERER') - if accept == match: - return - except KeyError: - if debug: - cherrypy.log('No Referer header', 'TOOLS.REFERER') - if accept_missing: - return - - raise cherrypy.HTTPError(error, message) - - -class SessionAuth(object): - - """Assert that the user is logged in.""" - - session_key = "username" - debug = False - - def check_username_and_password(self, username, password): - pass - - def anonymous(self): - """Provide a temporary user name for anonymous users.""" - pass - - def on_login(self, username): - pass - - def on_logout(self, username): - pass - - def on_check(self, username): - pass - - def login_screen(self, from_page='..', username='', error_msg='', - **kwargs): - return (_unicode_type(""" -Message: %(error_msg)s -
- Login: -
- Password: -
- -
- -
-""") % vars()).encode("utf-8") - - def do_login(self, username, password, from_page='..', **kwargs): - """Login. May raise redirect, or return True if request handled.""" - response = cherrypy.serving.response - error_msg = self.check_username_and_password(username, password) - if error_msg: - body = self.login_screen(from_page, username, error_msg) - response.body = body - if "Content-Length" in response.headers: - # Delete Content-Length header so finalize() recalcs it. - del response.headers["Content-Length"] - return True - else: - cherrypy.serving.request.login = username - cherrypy.session[self.session_key] = username - self.on_login(username) - raise cherrypy.HTTPRedirect(from_page or "/") - - def do_logout(self, from_page='..', **kwargs): - """Logout. May raise redirect, or return True if request handled.""" - sess = cherrypy.session - username = sess.get(self.session_key) - sess[self.session_key] = None - if username: - cherrypy.serving.request.login = None - self.on_logout(username) - raise cherrypy.HTTPRedirect(from_page) - - def do_check(self): - """Assert username. Raise redirect, or return True if request handled. - """ - sess = cherrypy.session - request = cherrypy.serving.request - response = cherrypy.serving.response - - username = sess.get(self.session_key) - if not username: - sess[self.session_key] = username = self.anonymous() - self._debug_message('No session[username], trying anonymous') - if not username: - url = cherrypy.url(qs=request.query_string) - self._debug_message( - 'No username, routing to login_screen with from_page %(url)r', - locals(), - ) - response.body = self.login_screen(url) - if "Content-Length" in response.headers: - # Delete Content-Length header so finalize() recalcs it. - del response.headers["Content-Length"] - return True - self._debug_message('Setting request.login to %(username)r', locals()) - request.login = username - self.on_check(username) - - def _debug_message(self, template, context={}): - if not self.debug: - return - cherrypy.log(template % context, 'TOOLS.SESSAUTH') - - def run(self): - request = cherrypy.serving.request - response = cherrypy.serving.response - - path = request.path_info - if path.endswith('login_screen'): - self._debug_message('routing %(path)r to login_screen', locals()) - response.body = self.login_screen() - return True - elif path.endswith('do_login'): - if request.method != 'POST': - response.headers['Allow'] = "POST" - self._debug_message('do_login requires POST') - raise cherrypy.HTTPError(405) - self._debug_message('routing %(path)r to do_login', locals()) - return self.do_login(**request.params) - elif path.endswith('do_logout'): - if request.method != 'POST': - response.headers['Allow'] = "POST" - raise cherrypy.HTTPError(405) - self._debug_message('routing %(path)r to do_logout', locals()) - return self.do_logout(**request.params) - else: - self._debug_message('No special path, running do_check') - return self.do_check() - - -def session_auth(**kwargs): - sa = SessionAuth() - for k, v in kwargs.items(): - setattr(sa, k, v) - return sa.run() -session_auth.__doc__ = """Session authentication hook. - -Any attribute of the SessionAuth class may be overridden via a keyword arg -to this function: - -""" + "\n".join(["%s: %s" % (k, type(getattr(SessionAuth, k)).__name__) - for k in dir(SessionAuth) if not k.startswith("__")]) - - -def log_traceback(severity=logging.ERROR, debug=False): - """Write the last error's traceback to the cherrypy error log.""" - cherrypy.log("", "HTTP", severity=severity, traceback=True) - - -def log_request_headers(debug=False): - """Write request headers to the cherrypy error log.""" - h = [" %s: %s" % (k, v) for k, v in cherrypy.serving.request.header_list] - cherrypy.log('\nRequest Headers:\n' + '\n'.join(h), "HTTP") - - -def log_hooks(debug=False): - """Write request.hooks to the cherrypy error log.""" - request = cherrypy.serving.request - - msg = [] - # Sort by the standard points if possible. - from cherrypy import _cprequest - points = _cprequest.hookpoints - for k in request.hooks.keys(): - if k not in points: - points.append(k) - - for k in points: - msg.append(" %s:" % k) - v = request.hooks.get(k, []) - v.sort() - for h in v: - msg.append(" %r" % h) - cherrypy.log('\nRequest Hooks for ' + cherrypy.url() + - ':\n' + '\n'.join(msg), "HTTP") - - -def redirect(url='', internal=True, debug=False): - """Raise InternalRedirect or HTTPRedirect to the given url.""" - if debug: - cherrypy.log('Redirecting %sto: %s' % - ({True: 'internal ', False: ''}[internal], url), - 'TOOLS.REDIRECT') - if internal: - raise cherrypy.InternalRedirect(url) - else: - raise cherrypy.HTTPRedirect(url) - - -def trailing_slash(missing=True, extra=False, status=None, debug=False): - """Redirect if path_info has (missing|extra) trailing slash.""" - request = cherrypy.serving.request - pi = request.path_info - - if debug: - cherrypy.log('is_index: %r, missing: %r, extra: %r, path_info: %r' % - (request.is_index, missing, extra, pi), - 'TOOLS.TRAILING_SLASH') - if request.is_index is True: - if missing: - if not pi.endswith('/'): - new_url = cherrypy.url(pi + '/', request.query_string) - raise cherrypy.HTTPRedirect(new_url, status=status or 301) - elif request.is_index is False: - if extra: - # If pi == '/', don't redirect to ''! - if pi.endswith('/') and pi != '/': - new_url = cherrypy.url(pi[:-1], request.query_string) - raise cherrypy.HTTPRedirect(new_url, status=status or 301) - - -def flatten(debug=False): - """Wrap response.body in a generator that recursively iterates over body. - - This allows cherrypy.response.body to consist of 'nested generators'; - that is, a set of generators that yield generators. - """ - def flattener(input): - numchunks = 0 - for x in input: - if not is_iterator(x): - numchunks += 1 - yield x - else: - for y in flattener(x): - numchunks += 1 - yield y - if debug: - cherrypy.log('Flattened %d chunks' % numchunks, 'TOOLS.FLATTEN') - response = cherrypy.serving.response - response.body = flattener(response.body) - - -def accept(media=None, debug=False): - """Return the client's preferred media-type (from the given Content-Types). - - If 'media' is None (the default), no test will be performed. - - If 'media' is provided, it should be the Content-Type value (as a string) - or values (as a list or tuple of strings) which the current resource - can emit. The client's acceptable media ranges (as declared in the - Accept request header) will be matched in order to these Content-Type - values; the first such string is returned. That is, the return value - will always be one of the strings provided in the 'media' arg (or None - if 'media' is None). - - If no match is found, then HTTPError 406 (Not Acceptable) is raised. - Note that most web browsers send */* as a (low-quality) acceptable - media range, which should match any Content-Type. In addition, "...if - no Accept header field is present, then it is assumed that the client - accepts all media types." - - Matching types are checked in order of client preference first, - and then in the order of the given 'media' values. - - Note that this function does not honor accept-params (other than "q"). - """ - if not media: - return - if isinstance(media, basestring): - media = [media] - request = cherrypy.serving.request - - # Parse the Accept request header, and try to match one - # of the requested media-ranges (in order of preference). - ranges = request.headers.elements('Accept') - if not ranges: - # Any media type is acceptable. - if debug: - cherrypy.log('No Accept header elements', 'TOOLS.ACCEPT') - return media[0] - else: - # Note that 'ranges' is sorted in order of preference - for element in ranges: - if element.qvalue > 0: - if element.value == "*/*": - # Matches any type or subtype - if debug: - cherrypy.log('Match due to */*', 'TOOLS.ACCEPT') - return media[0] - elif element.value.endswith("/*"): - # Matches any subtype - mtype = element.value[:-1] # Keep the slash - for m in media: - if m.startswith(mtype): - if debug: - cherrypy.log('Match due to %s' % element.value, - 'TOOLS.ACCEPT') - return m - else: - # Matches exact value - if element.value in media: - if debug: - cherrypy.log('Match due to %s' % element.value, - 'TOOLS.ACCEPT') - return element.value - - # No suitable media-range found. - ah = request.headers.get('Accept') - if ah is None: - msg = "Your client did not send an Accept header." - else: - msg = "Your client sent this Accept header: %s." % ah - msg += (" But this resource only emits these media types: %s." % - ", ".join(media)) - raise cherrypy.HTTPError(406, msg) - - -class MonitoredHeaderMap(_httputil.HeaderMap): - - def __init__(self): - self.accessed_headers = set() - - def __getitem__(self, key): - self.accessed_headers.add(key) - return _httputil.HeaderMap.__getitem__(self, key) - - def __contains__(self, key): - self.accessed_headers.add(key) - return _httputil.HeaderMap.__contains__(self, key) - - def get(self, key, default=None): - self.accessed_headers.add(key) - return _httputil.HeaderMap.get(self, key, default=default) - - if hasattr({}, 'has_key'): - # Python 2 - def has_key(self, key): - self.accessed_headers.add(key) - return _httputil.HeaderMap.has_key(self, key) - - -def autovary(ignore=None, debug=False): - """Auto-populate the Vary response header based on request.header access. - """ - request = cherrypy.serving.request - - req_h = request.headers - request.headers = MonitoredHeaderMap() - request.headers.update(req_h) - if ignore is None: - ignore = set(['Content-Disposition', 'Content-Length', 'Content-Type']) - - def set_response_header(): - resp_h = cherrypy.serving.response.headers - v = set([e.value for e in resp_h.elements('Vary')]) - if debug: - cherrypy.log( - 'Accessed headers: %s' % request.headers.accessed_headers, - 'TOOLS.AUTOVARY') - v = v.union(request.headers.accessed_headers) - v = v.difference(ignore) - v = list(v) - v.sort() - resp_h['Vary'] = ', '.join(v) - request.hooks.attach('before_finalize', set_response_header, 95) diff --git a/lib/cherrypy/lib/encoding.py b/lib/cherrypy/lib/encoding.py deleted file mode 100644 index b978790e154..00000000000 --- a/lib/cherrypy/lib/encoding.py +++ /dev/null @@ -1,434 +0,0 @@ -import struct -import sys -import time -import io - -IS_PY3 = sys.version_info[0] == 3 -try: - import six -except ImportError: - if IS_PY3: - _unicode_type = str - else: - _unicode_type = unicode -else: - _unicode_type = six.text_type - -import cherrypy -from cherrypy._cpcompat import basestring, ntob -from cherrypy.lib import file_generator -from cherrypy.lib import is_closable_iterator -from cherrypy.lib import set_vary_header - - -def decode(encoding=None, default_encoding='utf-8'): - """Replace or extend the list of charsets used to decode a request entity. - - Either argument may be a single string or a list of strings. - - encoding - If not None, restricts the set of charsets attempted while decoding - a request entity to the given set (even if a different charset is - given in the Content-Type request header). - - default_encoding - Only in effect if the 'encoding' argument is not given. - If given, the set of charsets attempted while decoding a request - entity is *extended* with the given value(s). - - """ - body = cherrypy.request.body - if encoding is not None: - if not isinstance(encoding, list): - encoding = [encoding] - body.attempt_charsets = encoding - elif default_encoding: - if not isinstance(default_encoding, list): - default_encoding = [default_encoding] - body.attempt_charsets = body.attempt_charsets + default_encoding - -class UTF8StreamEncoder: - def __init__(self, iterator): - self._iterator = iterator - - def __iter__(self): - return self - - def next(self): - return self.__next__() - - def __next__(self): - res = next(self._iterator) - if isinstance(res, _unicode_type): - res = res.encode('utf-8') - return res - - def close(self): - if is_closable_iterator(self._iterator): - self._iterator.close() - - def __getattr__(self, attr): - if attr.startswith('__'): - raise AttributeError(self, attr) - return getattr(self._iterator, attr) - - -class ResponseEncoder: - - default_encoding = 'utf-8' - failmsg = "Response body could not be encoded with %r." - encoding = None - errors = 'strict' - text_only = True - add_charset = True - debug = False - - def __init__(self, **kwargs): - for k, v in kwargs.items(): - setattr(self, k, v) - - self.attempted_charsets = set() - request = cherrypy.serving.request - if request.handler is not None: - # Replace request.handler with self - if self.debug: - cherrypy.log('Replacing request.handler', 'TOOLS.ENCODE') - self.oldhandler = request.handler - request.handler = self - - def encode_stream(self, encoding): - """Encode a streaming response body. - - Use a generator wrapper, and just pray it works as the stream is - being written out. - """ - if encoding in self.attempted_charsets: - return False - self.attempted_charsets.add(encoding) - - def encoder(body): - for chunk in body: - if isinstance(chunk, _unicode_type): - chunk = chunk.encode(encoding, self.errors) - yield chunk - self.body = encoder(self.body) - return True - - def encode_string(self, encoding): - """Encode a buffered response body.""" - if encoding in self.attempted_charsets: - return False - self.attempted_charsets.add(encoding) - body = [] - for chunk in self.body: - if isinstance(chunk, _unicode_type): - try: - chunk = chunk.encode(encoding, self.errors) - except (LookupError, UnicodeError): - return False - body.append(chunk) - self.body = body - return True - - def find_acceptable_charset(self): - request = cherrypy.serving.request - response = cherrypy.serving.response - - if self.debug: - cherrypy.log('response.stream %r' % - response.stream, 'TOOLS.ENCODE') - if response.stream: - encoder = self.encode_stream - else: - encoder = self.encode_string - if "Content-Length" in response.headers: - # Delete Content-Length header so finalize() recalcs it. - # Encoded strings may be of different lengths from their - # unicode equivalents, and even from each other. For example: - # >>> t = u"\u7007\u3040" - # >>> len(t) - # 2 - # >>> len(t.encode("UTF-8")) - # 6 - # >>> len(t.encode("utf7")) - # 8 - del response.headers["Content-Length"] - - # Parse the Accept-Charset request header, and try to provide one - # of the requested charsets (in order of user preference). - encs = request.headers.elements('Accept-Charset') - charsets = [enc.value.lower() for enc in encs] - if self.debug: - cherrypy.log('charsets %s' % repr(charsets), 'TOOLS.ENCODE') - - if self.encoding is not None: - # If specified, force this encoding to be used, or fail. - encoding = self.encoding.lower() - if self.debug: - cherrypy.log('Specified encoding %r' % - encoding, 'TOOLS.ENCODE') - if (not charsets) or "*" in charsets or encoding in charsets: - if self.debug: - cherrypy.log('Attempting encoding %r' % - encoding, 'TOOLS.ENCODE') - if encoder(encoding): - return encoding - else: - if not encs: - if self.debug: - cherrypy.log('Attempting default encoding %r' % - self.default_encoding, 'TOOLS.ENCODE') - # Any character-set is acceptable. - if encoder(self.default_encoding): - return self.default_encoding - else: - raise cherrypy.HTTPError(500, self.failmsg % - self.default_encoding) - else: - for element in encs: - if element.qvalue > 0: - if element.value == "*": - # Matches any charset. Try our default. - if self.debug: - cherrypy.log('Attempting default encoding due ' - 'to %r' % element, 'TOOLS.ENCODE') - if encoder(self.default_encoding): - return self.default_encoding - else: - encoding = element.value - if self.debug: - cherrypy.log('Attempting encoding %s (qvalue >' - '0)' % element, 'TOOLS.ENCODE') - if encoder(encoding): - return encoding - - if "*" not in charsets: - # If no "*" is present in an Accept-Charset field, then all - # character sets not explicitly mentioned get a quality - # value of 0, except for ISO-8859-1, which gets a quality - # value of 1 if not explicitly mentioned. - iso = 'iso-8859-1' - if iso not in charsets: - if self.debug: - cherrypy.log('Attempting ISO-8859-1 encoding', - 'TOOLS.ENCODE') - if encoder(iso): - return iso - - # No suitable encoding found. - ac = request.headers.get('Accept-Charset') - if ac is None: - msg = "Your client did not send an Accept-Charset header." - else: - msg = "Your client sent this Accept-Charset header: %s." % ac - _charsets = ", ".join(sorted(self.attempted_charsets)) - msg += " We tried these charsets: %s." % (_charsets,) - raise cherrypy.HTTPError(406, msg) - - def __call__(self, *args, **kwargs): - response = cherrypy.serving.response - self.body = self.oldhandler(*args, **kwargs) - - if isinstance(self.body, basestring): - # strings get wrapped in a list because iterating over a single - # item list is much faster than iterating over every character - # in a long string. - if self.body: - self.body = [self.body] - else: - # [''] doesn't evaluate to False, so replace it with []. - self.body = [] - elif hasattr(self.body, 'read'): - self.body = file_generator(self.body) - elif self.body is None: - self.body = [] - - ct = response.headers.elements("Content-Type") - if self.debug: - cherrypy.log('Content-Type: %r' % [str(h) - for h in ct], 'TOOLS.ENCODE') - if ct and self.add_charset: - ct = ct[0] - if self.text_only: - if ct.value.lower().startswith("text/"): - if self.debug: - cherrypy.log( - 'Content-Type %s starts with "text/"' % ct, - 'TOOLS.ENCODE') - do_find = True - else: - if self.debug: - cherrypy.log('Not finding because Content-Type %s ' - 'does not start with "text/"' % ct, - 'TOOLS.ENCODE') - do_find = False - else: - if self.debug: - cherrypy.log('Finding because not text_only', - 'TOOLS.ENCODE') - do_find = True - - if do_find: - # Set "charset=..." param on response Content-Type header - ct.params['charset'] = self.find_acceptable_charset() - if self.debug: - cherrypy.log('Setting Content-Type %s' % ct, - 'TOOLS.ENCODE') - response.headers["Content-Type"] = str(ct) - - return self.body - -# GZIP - - -def compress(body, compress_level): - """Compress 'body' at the given compress_level.""" - import zlib - - # See http://www.gzip.org/zlib/rfc-gzip.html - yield ntob('\x1f\x8b') # ID1 and ID2: gzip marker - yield ntob('\x08') # CM: compression method - yield ntob('\x00') # FLG: none set - # MTIME: 4 bytes - yield struct.pack(" 0 is present - * The 'identity' value is given with a qvalue > 0. - - """ - request = cherrypy.serving.request - response = cherrypy.serving.response - - set_vary_header(response, "Accept-Encoding") - - if not response.body: - # Response body is empty (might be a 304 for instance) - if debug: - cherrypy.log('No response body', context='TOOLS.GZIP') - return - - # If returning cached content (which should already have been gzipped), - # don't re-zip. - if getattr(request, "cached", False): - if debug: - cherrypy.log('Not gzipping cached response', context='TOOLS.GZIP') - return - - acceptable = request.headers.elements('Accept-Encoding') - if not acceptable: - # If no Accept-Encoding field is present in a request, - # the server MAY assume that the client will accept any - # content coding. In this case, if "identity" is one of - # the available content-codings, then the server SHOULD use - # the "identity" content-coding, unless it has additional - # information that a different content-coding is meaningful - # to the client. - if debug: - cherrypy.log('No Accept-Encoding', context='TOOLS.GZIP') - return - - ct = response.headers.get('Content-Type', '').split(';')[0] - for coding in acceptable: - if coding.value == 'identity' and coding.qvalue != 0: - if debug: - cherrypy.log('Non-zero identity qvalue: %s' % coding, - context='TOOLS.GZIP') - return - if coding.value in ('gzip', 'x-gzip'): - if coding.qvalue == 0: - if debug: - cherrypy.log('Zero gzip qvalue: %s' % coding, - context='TOOLS.GZIP') - return - - if ct not in mime_types: - # If the list of provided mime-types contains tokens - # such as 'text/*' or 'application/*+xml', - # we go through them and find the most appropriate one - # based on the given content-type. - # The pattern matching is only caring about the most - # common cases, as stated above, and doesn't support - # for extra parameters. - found = False - if '/' in ct: - ct_media_type, ct_sub_type = ct.split('/') - for mime_type in mime_types: - if '/' in mime_type: - media_type, sub_type = mime_type.split('/') - if ct_media_type == media_type: - if sub_type == '*': - found = True - break - elif '+' in sub_type and '+' in ct_sub_type: - ct_left, ct_right = ct_sub_type.split('+') - left, right = sub_type.split('+') - if left == '*' and ct_right == right: - found = True - break - - if not found: - if debug: - cherrypy.log('Content-Type %s not in mime_types %r' % - (ct, mime_types), context='TOOLS.GZIP') - return - - if debug: - cherrypy.log('Gzipping', context='TOOLS.GZIP') - # Return a generator that compresses the page - response.headers['Content-Encoding'] = 'gzip' - response.body = compress(response.body, compress_level) - if "Content-Length" in response.headers: - # Delete Content-Length header so finalize() recalcs it. - del response.headers["Content-Length"] - - return - - if debug: - cherrypy.log('No acceptable encoding found.', context='GZIP') - cherrypy.HTTPError(406, "identity, gzip").set_response() diff --git a/lib/cherrypy/lib/gctools.py b/lib/cherrypy/lib/gctools.py deleted file mode 100644 index a5984a63b6e..00000000000 --- a/lib/cherrypy/lib/gctools.py +++ /dev/null @@ -1,217 +0,0 @@ -import gc -import inspect -import os -import sys -import time - -try: - import objgraph -except ImportError: - objgraph = None - -import cherrypy -from cherrypy import _cprequest, _cpwsgi -from cherrypy.process.plugins import SimplePlugin - - -class ReferrerTree(object): - - """An object which gathers all referrers of an object to a given depth.""" - - peek_length = 40 - - def __init__(self, ignore=None, maxdepth=2, maxparents=10): - self.ignore = ignore or [] - self.ignore.append(inspect.currentframe().f_back) - self.maxdepth = maxdepth - self.maxparents = maxparents - - def ascend(self, obj, depth=1): - """Return a nested list containing referrers of the given object.""" - depth += 1 - parents = [] - - # Gather all referrers in one step to minimize - # cascading references due to repr() logic. - refs = gc.get_referrers(obj) - self.ignore.append(refs) - if len(refs) > self.maxparents: - return [("[%s referrers]" % len(refs), [])] - - try: - ascendcode = self.ascend.__code__ - except AttributeError: - ascendcode = self.ascend.im_func.func_code - for parent in refs: - if inspect.isframe(parent) and parent.f_code is ascendcode: - continue - if parent in self.ignore: - continue - if depth <= self.maxdepth: - parents.append((parent, self.ascend(parent, depth))) - else: - parents.append((parent, [])) - - return parents - - def peek(self, s): - """Return s, restricted to a sane length.""" - if len(s) > (self.peek_length + 3): - half = self.peek_length // 2 - return s[:half] + '...' + s[-half:] - else: - return s - - def _format(self, obj, descend=True): - """Return a string representation of a single object.""" - if inspect.isframe(obj): - filename, lineno, func, context, index = inspect.getframeinfo(obj) - return "" % func - - if not descend: - return self.peek(repr(obj)) - - if isinstance(obj, dict): - return "{" + ", ".join(["%s: %s" % (self._format(k, descend=False), - self._format(v, descend=False)) - for k, v in obj.items()]) + "}" - elif isinstance(obj, list): - return "[" + ", ".join([self._format(item, descend=False) - for item in obj]) + "]" - elif isinstance(obj, tuple): - return "(" + ", ".join([self._format(item, descend=False) - for item in obj]) + ")" - - r = self.peek(repr(obj)) - if isinstance(obj, (str, int, float)): - return r - return "%s: %s" % (type(obj), r) - - def format(self, tree): - """Return a list of string reprs from a nested list of referrers.""" - output = [] - - def ascend(branch, depth=1): - for parent, grandparents in branch: - output.append((" " * depth) + self._format(parent)) - if grandparents: - ascend(grandparents, depth + 1) - ascend(tree) - return output - - -def get_instances(cls): - return [x for x in gc.get_objects() if isinstance(x, cls)] - - -class RequestCounter(SimplePlugin): - - def start(self): - self.count = 0 - - def before_request(self): - self.count += 1 - - def after_request(self): - self.count -= 1 -request_counter = RequestCounter(cherrypy.engine) -request_counter.subscribe() - - -def get_context(obj): - if isinstance(obj, _cprequest.Request): - return "path=%s;stage=%s" % (obj.path_info, obj.stage) - elif isinstance(obj, _cprequest.Response): - return "status=%s" % obj.status - elif isinstance(obj, _cpwsgi.AppResponse): - return "PATH_INFO=%s" % obj.environ.get('PATH_INFO', '') - elif hasattr(obj, "tb_lineno"): - return "tb_lineno=%s" % obj.tb_lineno - return "" - - -class GCRoot(object): - - """A CherryPy page handler for testing reference leaks.""" - - classes = [ - (_cprequest.Request, 2, 2, - "Should be 1 in this request thread and 1 in the main thread."), - (_cprequest.Response, 2, 2, - "Should be 1 in this request thread and 1 in the main thread."), - (_cpwsgi.AppResponse, 1, 1, - "Should be 1 in this request thread only."), - ] - - @cherrypy.expose - def index(self): - return "Hello, world!" - - @cherrypy.expose - def stats(self): - output = ["Statistics:"] - - for trial in range(10): - if request_counter.count > 0: - break - time.sleep(0.5) - else: - output.append("\nNot all requests closed properly.") - - # gc_collect isn't perfectly synchronous, because it may - # break reference cycles that then take time to fully - # finalize. Call it thrice and hope for the best. - gc.collect() - gc.collect() - unreachable = gc.collect() - if unreachable: - if objgraph is not None: - final = objgraph.by_type('Nondestructible') - if final: - objgraph.show_backrefs(final, filename='finalizers.png') - - trash = {} - for x in gc.garbage: - trash[type(x)] = trash.get(type(x), 0) + 1 - if trash: - output.insert(0, "\n%s unreachable objects:" % unreachable) - trash = [(v, k) for k, v in trash.items()] - trash.sort() - for pair in trash: - output.append(" " + repr(pair)) - - # Check declared classes to verify uncollected instances. - # These don't have to be part of a cycle; they can be - # any objects that have unanticipated referrers that keep - # them from being collected. - allobjs = {} - for cls, minobj, maxobj, msg in self.classes: - allobjs[cls] = get_instances(cls) - - for cls, minobj, maxobj, msg in self.classes: - objs = allobjs[cls] - lenobj = len(objs) - if lenobj < minobj or lenobj > maxobj: - if minobj == maxobj: - output.append( - "\nExpected %s %r references, got %s." % - (minobj, cls, lenobj)) - else: - output.append( - "\nExpected %s to %s %r references, got %s." % - (minobj, maxobj, cls, lenobj)) - - for obj in objs: - if objgraph is not None: - ig = [id(objs), id(inspect.currentframe())] - fname = "graph_%s_%s.png" % (cls.__name__, id(obj)) - objgraph.show_backrefs( - obj, extra_ignore=ig, max_depth=4, too_many=20, - filename=fname, extra_info=get_context) - output.append("\nReferrers for %s (refcount=%s):" % - (repr(obj), sys.getrefcount(obj))) - t = ReferrerTree(ignore=[objs], maxdepth=3) - tree = t.ascend(obj) - output.extend(t.format(tree)) - - return "\n".join(output) diff --git a/lib/cherrypy/lib/http.py b/lib/cherrypy/lib/http.py deleted file mode 100644 index 12043ad1530..00000000000 --- a/lib/cherrypy/lib/http.py +++ /dev/null @@ -1,6 +0,0 @@ -import warnings -warnings.warn('cherrypy.lib.http has been deprecated and will be removed ' - 'in CherryPy 3.3 use cherrypy.lib.httputil instead.', - DeprecationWarning) - -from cherrypy.lib.httputil import * diff --git a/lib/cherrypy/lib/httpauth.py b/lib/cherrypy/lib/httpauth.py deleted file mode 100644 index 9ecebe32216..00000000000 --- a/lib/cherrypy/lib/httpauth.py +++ /dev/null @@ -1,373 +0,0 @@ -""" -This module defines functions to implement HTTP Digest Authentication -(:rfc:`2617`). -This has full compliance with 'Digest' and 'Basic' authentication methods. In -'Digest' it supports both MD5 and MD5-sess algorithms. - -Usage: - First use 'doAuth' to request the client authentication for a - certain resource. You should send an httplib.UNAUTHORIZED response to the - client so he knows he has to authenticate itself. - - Then use 'parseAuthorization' to retrieve the 'auth_map' used in - 'checkResponse'. - - To use 'checkResponse' you must have already verified the password - associated with the 'username' key in 'auth_map' dict. Then you use the - 'checkResponse' function to verify if the password matches the one sent - by the client. - -SUPPORTED_ALGORITHM - list of supported 'Digest' algorithms -SUPPORTED_QOP - list of supported 'Digest' 'qop'. -""" -__version__ = 1, 0, 1 -__author__ = "Tiago Cogumbreiro " -__credits__ = """ - Peter van Kampen for its recipe which implement most of Digest - authentication: - http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/302378 -""" - -__license__ = """ -Copyright (c) 2005, Tiago Cogumbreiro -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - * Neither the name of Sylvain Hellegouarch nor the names of his - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE -FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -""" - -__all__ = ("digestAuth", "basicAuth", "doAuth", "checkResponse", - "parseAuthorization", "SUPPORTED_ALGORITHM", "md5SessionKey", - "calculateNonce", "SUPPORTED_QOP") - -########################################################################## -import time -import hashlib - -from cherrypy._cpcompat import base64_decode, ntob -from cherrypy._cpcompat import parse_http_list, parse_keqv_list - -MD5 = "MD5" -MD5_SESS = "MD5-sess" -AUTH = "auth" -AUTH_INT = "auth-int" - -SUPPORTED_ALGORITHM = (MD5, MD5_SESS) -SUPPORTED_QOP = (AUTH, AUTH_INT) - -########################################################################## -# doAuth -# -DIGEST_AUTH_ENCODERS = { - MD5: lambda val: hashlib.md5(ntob(val)).hexdigest(), - MD5_SESS: lambda val: hashlib.md5(ntob(val)).hexdigest(), - # SHA: lambda val: sha.new(ntob(val)).hexdigest (), -} - - -def calculateNonce(realm, algorithm=MD5): - """This is an auxaliary function that calculates 'nonce' value. It is used - to handle sessions.""" - - global SUPPORTED_ALGORITHM, DIGEST_AUTH_ENCODERS - assert algorithm in SUPPORTED_ALGORITHM - - try: - encoder = DIGEST_AUTH_ENCODERS[algorithm] - except KeyError: - raise NotImplementedError("The chosen algorithm (%s) does not have " - "an implementation yet" % algorithm) - - return encoder("%d:%s" % (time.time(), realm)) - - -def digestAuth(realm, algorithm=MD5, nonce=None, qop=AUTH): - """Challenges the client for a Digest authentication.""" - global SUPPORTED_ALGORITHM, DIGEST_AUTH_ENCODERS, SUPPORTED_QOP - assert algorithm in SUPPORTED_ALGORITHM - assert qop in SUPPORTED_QOP - - if nonce is None: - nonce = calculateNonce(realm, algorithm) - - return 'Digest realm="%s", nonce="%s", algorithm="%s", qop="%s"' % ( - realm, nonce, algorithm, qop - ) - - -def basicAuth(realm): - """Challengenes the client for a Basic authentication.""" - assert '"' not in realm, "Realms cannot contain the \" (quote) character." - - return 'Basic realm="%s"' % realm - - -def doAuth(realm): - """'doAuth' function returns the challenge string b giving priority over - Digest and fallback to Basic authentication when the browser doesn't - support the first one. - - This should be set in the HTTP header under the key 'WWW-Authenticate'.""" - - return digestAuth(realm) + " " + basicAuth(realm) - - -########################################################################## -# Parse authorization parameters -# -def _parseDigestAuthorization(auth_params): - # Convert the auth params to a dict - items = parse_http_list(auth_params) - params = parse_keqv_list(items) - - # Now validate the params - - # Check for required parameters - required = ["username", "realm", "nonce", "uri", "response"] - for k in required: - if k not in params: - return None - - # If qop is sent then cnonce and nc MUST be present - if "qop" in params and not ("cnonce" in params - and "nc" in params): - return None - - # If qop is not sent, neither cnonce nor nc can be present - if ("cnonce" in params or "nc" in params) and \ - "qop" not in params: - return None - - return params - - -def _parseBasicAuthorization(auth_params): - username, password = base64_decode(auth_params).split(":", 1) - return {"username": username, "password": password} - -AUTH_SCHEMES = { - "basic": _parseBasicAuthorization, - "digest": _parseDigestAuthorization, -} - - -def parseAuthorization(credentials): - """parseAuthorization will convert the value of the 'Authorization' key in - the HTTP header to a map itself. If the parsing fails 'None' is returned. - """ - - global AUTH_SCHEMES - - auth_scheme, auth_params = credentials.split(" ", 1) - auth_scheme = auth_scheme.lower() - - parser = AUTH_SCHEMES[auth_scheme] - params = parser(auth_params) - - if params is None: - return - - assert "auth_scheme" not in params - params["auth_scheme"] = auth_scheme - return params - - -########################################################################## -# Check provided response for a valid password -# -def md5SessionKey(params, password): - """ - If the "algorithm" directive's value is "MD5-sess", then A1 - [the session key] is calculated only once - on the first request by the - client following receipt of a WWW-Authenticate challenge from the server. - - This creates a 'session key' for the authentication of subsequent - requests and responses which is different for each "authentication - session", thus limiting the amount of material hashed with any one - key. - - Because the server need only use the hash of the user - credentials in order to create the A1 value, this construction could - be used in conjunction with a third party authentication service so - that the web server would not need the actual password value. The - specification of such a protocol is beyond the scope of this - specification. -""" - - keys = ("username", "realm", "nonce", "cnonce") - params_copy = {} - for key in keys: - params_copy[key] = params[key] - - params_copy["algorithm"] = MD5_SESS - return _A1(params_copy, password) - - -def _A1(params, password): - algorithm = params.get("algorithm", MD5) - H = DIGEST_AUTH_ENCODERS[algorithm] - - if algorithm == MD5: - # If the "algorithm" directive's value is "MD5" or is - # unspecified, then A1 is: - # A1 = unq(username-value) ":" unq(realm-value) ":" passwd - return "%s:%s:%s" % (params["username"], params["realm"], password) - - elif algorithm == MD5_SESS: - - # This is A1 if qop is set - # A1 = H( unq(username-value) ":" unq(realm-value) ":" passwd ) - # ":" unq(nonce-value) ":" unq(cnonce-value) - h_a1 = H("%s:%s:%s" % (params["username"], params["realm"], password)) - return "%s:%s:%s" % (h_a1, params["nonce"], params["cnonce"]) - - -def _A2(params, method, kwargs): - # If the "qop" directive's value is "auth" or is unspecified, then A2 is: - # A2 = Method ":" digest-uri-value - - qop = params.get("qop", "auth") - if qop == "auth": - return method + ":" + params["uri"] - elif qop == "auth-int": - # If the "qop" value is "auth-int", then A2 is: - # A2 = Method ":" digest-uri-value ":" H(entity-body) - entity_body = kwargs.get("entity_body", "") - H = kwargs["H"] - - return "%s:%s:%s" % ( - method, - params["uri"], - H(entity_body) - ) - - else: - raise NotImplementedError("The 'qop' method is unknown: %s" % qop) - - -def _computeDigestResponse(auth_map, password, method="GET", A1=None, - **kwargs): - """ - Generates a response respecting the algorithm defined in RFC 2617 - """ - params = auth_map - - algorithm = params.get("algorithm", MD5) - - H = DIGEST_AUTH_ENCODERS[algorithm] - KD = lambda secret, data: H(secret + ":" + data) - - qop = params.get("qop", None) - - H_A2 = H(_A2(params, method, kwargs)) - - if algorithm == MD5_SESS and A1 is not None: - H_A1 = H(A1) - else: - H_A1 = H(_A1(params, password)) - - if qop in ("auth", "auth-int"): - # If the "qop" value is "auth" or "auth-int": - # request-digest = <"> < KD ( H(A1), unq(nonce-value) - # ":" nc-value - # ":" unq(cnonce-value) - # ":" unq(qop-value) - # ":" H(A2) - # ) <"> - request = "%s:%s:%s:%s:%s" % ( - params["nonce"], - params["nc"], - params["cnonce"], - params["qop"], - H_A2, - ) - elif qop is None: - # If the "qop" directive is not present (this construction is - # for compatibility with RFC 2069): - # request-digest = - # <"> < KD ( H(A1), unq(nonce-value) ":" H(A2) ) > <"> - request = "%s:%s" % (params["nonce"], H_A2) - - return KD(H_A1, request) - - -def _checkDigestResponse(auth_map, password, method="GET", A1=None, **kwargs): - """This function is used to verify the response given by the client when - he tries to authenticate. - Optional arguments: - entity_body - when 'qop' is set to 'auth-int' you MUST provide the - raw data you are going to send to the client (usually the - HTML page. - request_uri - the uri from the request line compared with the 'uri' - directive of the authorization map. They must represent - the same resource (unused at this time). - """ - - if auth_map['realm'] != kwargs.get('realm', None): - return False - - response = _computeDigestResponse( - auth_map, password, method, A1, **kwargs) - - return response == auth_map["response"] - - -def _checkBasicResponse(auth_map, password, method='GET', encrypt=None, - **kwargs): - # Note that the Basic response doesn't provide the realm value so we cannot - # test it - pass_through = lambda password, username=None: password - encrypt = encrypt or pass_through - try: - candidate = encrypt(auth_map["password"], auth_map["username"]) - except TypeError: - # if encrypt only takes one parameter, it's the password - candidate = encrypt(auth_map["password"]) - return candidate == password - -AUTH_RESPONSES = { - "basic": _checkBasicResponse, - "digest": _checkDigestResponse, -} - - -def checkResponse(auth_map, password, method="GET", encrypt=None, **kwargs): - """'checkResponse' compares the auth_map with the password and optionally - other arguments that each implementation might need. - - If the response is of type 'Basic' then the function has the following - signature:: - - checkBasicResponse(auth_map, password) -> bool - - If the response is of type 'Digest' then the function has the following - signature:: - - checkDigestResponse(auth_map, password, method='GET', A1=None) -> bool - - The 'A1' argument is only used in MD5_SESS algorithm based responses. - Check md5SessionKey() for more info. - """ - checker = AUTH_RESPONSES[auth_map["auth_scheme"]] - return checker(auth_map, password, method=method, encrypt=encrypt, - **kwargs) diff --git a/lib/cherrypy/lib/httputil.py b/lib/cherrypy/lib/httputil.py deleted file mode 100644 index 60af429d5cc..00000000000 --- a/lib/cherrypy/lib/httputil.py +++ /dev/null @@ -1,535 +0,0 @@ -"""HTTP library functions. - -This module contains functions for building an HTTP application -framework: any one, not just one whose name starts with "Ch". ;) If you -reference any modules from some popular framework inside *this* module, -FuManChu will personally hang you up by your thumbs and submit you -to a public caning. -""" - -from binascii import b2a_base64 - -import sys - -IS_PY3 = sys.version_info[0] == 3 -try: - import six -except ImportError: - if IS_PY3: - _unicode_type = str - else: - _unicode_type = unicode -else: - _unicode_type = six.text_type - -from cherrypy._cpcompat import BaseHTTPRequestHandler, HTTPDate, ntob, ntou -from cherrypy._cpcompat import basestring, iteritems -from cherrypy._cpcompat import reversed, sorted, unquote_qs -response_codes = BaseHTTPRequestHandler.responses.copy() - -# From https://github.com/cherrypy/cherrypy/issues/361 -response_codes[500] = ('Internal Server Error', - 'The server encountered an unexpected condition ' - 'which prevented it from fulfilling the request.') -response_codes[503] = ('Service Unavailable', - 'The server is currently unable to handle the ' - 'request due to a temporary overloading or ' - 'maintenance of the server.') - -import re -from cgi import parse_header - - -def urljoin(*atoms): - """Return the given path \*atoms, joined into a single URL. - - This will correctly join a SCRIPT_NAME and PATH_INFO into the - original URL, even if either atom is blank. - """ - url = "/".join([x for x in atoms if x]) - while "//" in url: - url = url.replace("//", "/") - # Special-case the final url of "", and return "/" instead. - return url or "/" - - -def urljoin_bytes(*atoms): - """Return the given path *atoms, joined into a single URL. - - This will correctly join a SCRIPT_NAME and PATH_INFO into the - original URL, even if either atom is blank. - """ - url = ntob("/").join([x for x in atoms if x]) - while ntob("//") in url: - url = url.replace(ntob("//"), ntob("/")) - # Special-case the final url of "", and return "/" instead. - return url or ntob("/") - - -def protocol_from_http(protocol_str): - """Return a protocol tuple from the given 'HTTP/x.y' string.""" - return int(protocol_str[5]), int(protocol_str[7]) - - -def get_ranges(headervalue, content_length): - """Return a list of (start, stop) indices from a Range header, or None. - - Each (start, stop) tuple will be composed of two ints, which are suitable - for use in a slicing operation. That is, the header "Range: bytes=3-6", - if applied against a Python string, is requesting resource[3:7]. This - function will return the list [(3, 7)]. - - If this function returns an empty list, you should return HTTP 416. - """ - - if not headervalue: - return None - - result = [] - bytesunit, byteranges = headervalue.split("=", 1) - for brange in byteranges.split(","): - start, stop = [x.strip() for x in brange.split("-", 1)] - if start: - if not stop: - stop = content_length - 1 - start, stop = int(start), int(stop) - if start >= content_length: - # From rfc 2616 sec 14.16: - # "If the server receives a request (other than one - # including an If-Range request-header field) with an - # unsatisfiable Range request-header field (that is, - # all of whose byte-range-spec values have a first-byte-pos - # value greater than the current length of the selected - # resource), it SHOULD return a response code of 416 - # (Requested range not satisfiable)." - continue - if stop < start: - # From rfc 2616 sec 14.16: - # "If the server ignores a byte-range-spec because it - # is syntactically invalid, the server SHOULD treat - # the request as if the invalid Range header field - # did not exist. (Normally, this means return a 200 - # response containing the full entity)." - return None - result.append((start, stop + 1)) - else: - if not stop: - # See rfc quote above. - return None - # Negative subscript (last N bytes) - # - # RFC 2616 Section 14.35.1: - # If the entity is shorter than the specified suffix-length, - # the entire entity-body is used. - if int(stop) > content_length: - result.append((0, content_length)) - else: - result.append((content_length - int(stop), content_length)) - - return result - - -class HeaderElement(object): - - """An element (with parameters) from an HTTP header's element list.""" - - def __init__(self, value, params=None): - self.value = value - if params is None: - params = {} - self.params = params - - def __cmp__(self, other): - return cmp(self.value, other.value) - - def __lt__(self, other): - return self.value < other.value - - def __str__(self): - p = [";%s=%s" % (k, v) for k, v in iteritems(self.params)] - return str("%s%s" % (self.value, "".join(p))) - - def __bytes__(self): - return ntob(self.__str__()) - - def __unicode__(self): - return ntou(self.__str__()) - - def parse(elementstr): - """Transform 'token;key=val' to ('token', {'key': 'val'}).""" - initial_value, params = parse_header(elementstr) - return initial_value, params - parse = staticmethod(parse) - - def from_str(cls, elementstr): - """Construct an instance from a string of the form 'token;key=val'.""" - ival, params = cls.parse(elementstr) - return cls(ival, params) - from_str = classmethod(from_str) - - -q_separator = re.compile(r'; *q *=') - - -class AcceptElement(HeaderElement): - - """An element (with parameters) from an Accept* header's element list. - - AcceptElement objects are comparable; the more-preferred object will be - "less than" the less-preferred object. They are also therefore sortable; - if you sort a list of AcceptElement objects, they will be listed in - priority order; the most preferred value will be first. Yes, it should - have been the other way around, but it's too late to fix now. - """ - - def from_str(cls, elementstr): - qvalue = None - # The first "q" parameter (if any) separates the initial - # media-range parameter(s) (if any) from the accept-params. - atoms = q_separator.split(elementstr, 1) - media_range = atoms.pop(0).strip() - if atoms: - # The qvalue for an Accept header can have extensions. The other - # headers cannot, but it's easier to parse them as if they did. - qvalue = HeaderElement.from_str(atoms[0].strip()) - - media_type, params = cls.parse(media_range) - if qvalue is not None: - params["q"] = qvalue - return cls(media_type, params) - from_str = classmethod(from_str) - - def qvalue(self): - val = self.params.get("q", "1") - if isinstance(val, HeaderElement): - val = val.value - return float(val) - qvalue = property(qvalue, doc="The qvalue, or priority, of this value.") - - def __cmp__(self, other): - diff = cmp(self.qvalue, other.qvalue) - if diff == 0: - diff = cmp(str(self), str(other)) - return diff - - def __lt__(self, other): - if self.qvalue == other.qvalue: - return str(self) < str(other) - else: - return self.qvalue < other.qvalue - -RE_HEADER_SPLIT = re.compile(',(?=(?:[^"]*"[^"]*")*[^"]*$)') -def header_elements(fieldname, fieldvalue): - """Return a sorted HeaderElement list from a comma-separated header string. - """ - if not fieldvalue: - return [] - - result = [] - for element in RE_HEADER_SPLIT.split(fieldvalue): - if fieldname.startswith("Accept") or fieldname == 'TE': - hv = AcceptElement.from_str(element) - else: - hv = HeaderElement.from_str(element) - result.append(hv) - - return list(reversed(sorted(result))) - - -def decode_TEXT(value): - r"""Decode :rfc:`2047` TEXT (e.g. "=?utf-8?q?f=C3=BCr?=" -> "f\xfcr").""" - try: - # Python 3 - from email.header import decode_header - except ImportError: - from email.Header import decode_header - atoms = decode_header(value) - decodedvalue = "" - for atom, charset in atoms: - if charset is not None: - atom = atom.decode(charset) - decodedvalue += atom - return decodedvalue - - -def valid_status(status): - """Return legal HTTP status Code, Reason-phrase and Message. - - The status arg must be an int, or a str that begins with an int. - - If status is an int, or a str and no reason-phrase is supplied, - a default reason-phrase will be provided. - """ - - if not status: - status = 200 - - status = str(status) - parts = status.split(" ", 1) - if len(parts) == 1: - # No reason supplied. - code, = parts - reason = None - else: - code, reason = parts - reason = reason.strip() - - try: - code = int(code) - except ValueError: - raise ValueError("Illegal response status from server " - "(%s is non-numeric)." % repr(code)) - - if code < 100 or code > 599: - raise ValueError("Illegal response status from server " - "(%s is out of range)." % repr(code)) - - if code not in response_codes: - # code is unknown but not illegal - default_reason, message = "", "" - else: - default_reason, message = response_codes[code] - - if reason is None: - reason = default_reason - - return code, reason, message - - -# NOTE: the parse_qs functions that follow are modified version of those -# in the python3.0 source - we need to pass through an encoding to the unquote -# method, but the default parse_qs function doesn't allow us to. These do. - -def _parse_qs(qs, keep_blank_values=0, strict_parsing=0, encoding='utf-8'): - """Parse a query given as a string argument. - - Arguments: - - qs: URL-encoded query string to be parsed - - keep_blank_values: flag indicating whether blank values in - URL encoded queries should be treated as blank strings. A - true value indicates that blanks should be retained as blank - strings. The default false value indicates that blank values - are to be ignored and treated as if they were not included. - - strict_parsing: flag indicating what to do with parsing errors. If - false (the default), errors are silently ignored. If true, - errors raise a ValueError exception. - - Returns a dict, as G-d intended. - """ - pairs = [s2 for s1 in qs.split('&') for s2 in s1.split(';')] - d = {} - for name_value in pairs: - if not name_value and not strict_parsing: - continue - nv = name_value.split('=', 1) - if len(nv) != 2: - if strict_parsing: - raise ValueError("bad query field: %r" % (name_value,)) - # Handle case of a control-name with no equal sign - if keep_blank_values: - nv.append('') - else: - continue - if len(nv[1]) or keep_blank_values: - name = unquote_qs(nv[0], encoding) - value = unquote_qs(nv[1], encoding) - if name in d: - if not isinstance(d[name], list): - d[name] = [d[name]] - d[name].append(value) - else: - d[name] = value - return d - - -image_map_pattern = re.compile(r"[0-9]+,[0-9]+") - - -def parse_query_string(query_string, keep_blank_values=True, encoding='utf-8'): - """Build a params dictionary from a query_string. - - Duplicate key/value pairs in the provided query_string will be - returned as {'key': [val1, val2, ...]}. Single key/values will - be returned as strings: {'key': 'value'}. - """ - if image_map_pattern.match(query_string): - # Server-side image map. Map the coords to 'x' and 'y' - # (like CGI::Request does). - pm = query_string.split(",") - pm = {'x': int(pm[0]), 'y': int(pm[1])} - else: - pm = _parse_qs(query_string, keep_blank_values, encoding=encoding) - return pm - - -class CaseInsensitiveDict(dict): - - """A case-insensitive dict subclass. - - Each key is changed on entry to str(key).title(). - """ - - def __getitem__(self, key): - return dict.__getitem__(self, str(key).title()) - - def __setitem__(self, key, value): - dict.__setitem__(self, str(key).title(), value) - - def __delitem__(self, key): - dict.__delitem__(self, str(key).title()) - - def __contains__(self, key): - return dict.__contains__(self, str(key).title()) - - def get(self, key, default=None): - return dict.get(self, str(key).title(), default) - - if hasattr({}, 'has_key'): - def has_key(self, key): - return str(key).title() in self - - def update(self, E): - for k in E.keys(): - self[str(k).title()] = E[k] - - def fromkeys(cls, seq, value=None): - newdict = cls() - for k in seq: - newdict[str(k).title()] = value - return newdict - fromkeys = classmethod(fromkeys) - - def setdefault(self, key, x=None): - key = str(key).title() - try: - return self[key] - except KeyError: - self[key] = x - return x - - def pop(self, key, default): - return dict.pop(self, str(key).title(), default) - - -# TEXT = -# -# A CRLF is allowed in the definition of TEXT only as part of a header -# field continuation. It is expected that the folding LWS will be -# replaced with a single SP before interpretation of the TEXT value." -if str == bytes: - header_translate_table = ''.join([chr(i) for i in xrange(256)]) - header_translate_deletechars = ''.join( - [chr(i) for i in xrange(32)]) + chr(127) -else: - header_translate_table = None - header_translate_deletechars = bytes(range(32)) + bytes([127]) - - -class HeaderMap(CaseInsensitiveDict): - - """A dict subclass for HTTP request and response headers. - - Each key is changed on entry to str(key).title(). This allows headers - to be case-insensitive and avoid duplicates. - - Values are header values (decoded according to :rfc:`2047` if necessary). - """ - - protocol = (1, 1) - encodings = ["ISO-8859-1"] - - # Someday, when http-bis is done, this will probably get dropped - # since few servers, clients, or intermediaries do it. But until then, - # we're going to obey the spec as is. - # "Words of *TEXT MAY contain characters from character sets other than - # ISO-8859-1 only when encoded according to the rules of RFC 2047." - use_rfc_2047 = True - - def elements(self, key): - """Return a sorted list of HeaderElements for the given header.""" - key = str(key).title() - value = self.get(key) - return header_elements(key, value) - - def values(self, key): - """Return a sorted list of HeaderElement.value for the given header.""" - return [e.value for e in self.elements(key)] - - def output(self): - """Transform self into a list of (name, value) tuples.""" - return list(self.encode_header_items(self.items())) - - def encode_header_items(cls, header_items): - """ - Prepare the sequence of name, value tuples into a form suitable for - transmitting on the wire for HTTP. - """ - for k, v in header_items: - if isinstance(k, _unicode_type): - k = cls.encode(k) - - if not isinstance(v, basestring): - v = str(v) - - if isinstance(v, _unicode_type): - v = cls.encode(v) - - # See header_translate_* constants above. - # Replace only if you really know what you're doing. - k = k.translate(header_translate_table, - header_translate_deletechars) - v = v.translate(header_translate_table, - header_translate_deletechars) - - yield (k, v) - encode_header_items = classmethod(encode_header_items) - - def encode(cls, v): - """Return the given header name or value, encoded for HTTP output.""" - for enc in cls.encodings: - try: - return v.encode(enc) - except UnicodeEncodeError: - continue - - if cls.protocol == (1, 1) and cls.use_rfc_2047: - # Encode RFC-2047 TEXT - # (e.g. u"\u8200" -> "=?utf-8?b?6IiA?="). - # We do our own here instead of using the email module - # because we never want to fold lines--folding has - # been deprecated by the HTTP working group. - v = b2a_base64(v.encode('utf-8')) - return (ntob('=?utf-8?b?') + v.strip(ntob('\n')) + ntob('?=')) - - raise ValueError("Could not encode header part %r using " - "any of the encodings %r." % - (v, cls.encodings)) - encode = classmethod(encode) - - -class Host(object): - - """An internet address. - - name - Should be the client's host name. If not available (because no DNS - lookup is performed), the IP address should be used instead. - - """ - - ip = "0.0.0.0" - port = 80 - name = "unknown.tld" - - def __init__(self, ip, port, name=None): - self.ip = ip - self.port = port - if name is None: - name = ip - self.name = name - - def __repr__(self): - return "httputil.Host(%r, %r, %r)" % (self.ip, self.port, self.name) diff --git a/lib/cherrypy/lib/jsontools.py b/lib/cherrypy/lib/jsontools.py deleted file mode 100644 index 90b3ff8a811..00000000000 --- a/lib/cherrypy/lib/jsontools.py +++ /dev/null @@ -1,96 +0,0 @@ -import cherrypy -from cherrypy._cpcompat import basestring, ntou, json_encode, json_decode - - -def json_processor(entity): - """Read application/json data into request.json.""" - if not entity.headers.get(ntou("Content-Length"), ntou("")): - raise cherrypy.HTTPError(411) - - body = entity.fp.read() - try: - cherrypy.serving.request.json = json_decode(body.decode('utf-8')) - except ValueError: - raise cherrypy.HTTPError(400, 'Invalid JSON document') - - -def json_in(content_type=[ntou('application/json'), ntou('text/javascript')], - force=True, debug=False, processor=json_processor): - """Add a processor to parse JSON request entities: - The default processor places the parsed data into request.json. - - Incoming request entities which match the given content_type(s) will - be deserialized from JSON to the Python equivalent, and the result - stored at cherrypy.request.json. The 'content_type' argument may - be a Content-Type string or a list of allowable Content-Type strings. - - If the 'force' argument is True (the default), then entities of other - content types will not be allowed; "415 Unsupported Media Type" is - raised instead. - - Supply your own processor to use a custom decoder, or to handle the parsed - data differently. The processor can be configured via - tools.json_in.processor or via the decorator method. - - Note that the deserializer requires the client send a Content-Length - request header, or it will raise "411 Length Required". If for any - other reason the request entity cannot be deserialized from JSON, - it will raise "400 Bad Request: Invalid JSON document". - - You must be using Python 2.6 or greater, or have the 'simplejson' - package importable; otherwise, ValueError is raised during processing. - """ - request = cherrypy.serving.request - if isinstance(content_type, basestring): - content_type = [content_type] - - if force: - if debug: - cherrypy.log('Removing body processors %s' % - repr(request.body.processors.keys()), 'TOOLS.JSON_IN') - request.body.processors.clear() - request.body.default_proc = cherrypy.HTTPError( - 415, 'Expected an entity of content type %s' % - ', '.join(content_type)) - - for ct in content_type: - if debug: - cherrypy.log('Adding body processor for %s' % ct, 'TOOLS.JSON_IN') - request.body.processors[ct] = processor - - -def json_handler(*args, **kwargs): - value = cherrypy.serving.request._json_inner_handler(*args, **kwargs) - return json_encode(value) - - -def json_out(content_type='application/json', debug=False, - handler=json_handler): - """Wrap request.handler to serialize its output to JSON. Sets Content-Type. - - If the given content_type is None, the Content-Type response header - is not set. - - Provide your own handler to use a custom encoder. For example - cherrypy.config['tools.json_out.handler'] = , or - @json_out(handler=function). - - You must be using Python 2.6 or greater, or have the 'simplejson' - package importable; otherwise, ValueError is raised during processing. - """ - request = cherrypy.serving.request - # request.handler may be set to None by e.g. the caching tool - # to signal to all components that a response body has already - # been attached, in which case we don't need to wrap anything. - if request.handler is None: - return - if debug: - cherrypy.log('Replacing %s with JSON handler' % request.handler, - 'TOOLS.JSON_OUT') - request._json_inner_handler = request.handler - request.handler = handler - if content_type is not None: - if debug: - cherrypy.log('Setting Content-Type to %s' % - content_type, 'TOOLS.JSON_OUT') - cherrypy.serving.response.headers['Content-Type'] = content_type diff --git a/lib/cherrypy/lib/lockfile.py b/lib/cherrypy/lib/lockfile.py deleted file mode 100644 index b9d8e02754f..00000000000 --- a/lib/cherrypy/lib/lockfile.py +++ /dev/null @@ -1,147 +0,0 @@ -""" -Platform-independent file locking. Inspired by and modeled after zc.lockfile. -""" - -import os - -try: - import msvcrt -except ImportError: - pass - -try: - import fcntl -except ImportError: - pass - - -class LockError(Exception): - - "Could not obtain a lock" - - msg = "Unable to lock %r" - - def __init__(self, path): - super(LockError, self).__init__(self.msg % path) - - -class UnlockError(LockError): - - "Could not release a lock" - - msg = "Unable to unlock %r" - - -# first, a default, naive locking implementation -class LockFile(object): - - """ - A default, naive locking implementation. Always fails if the file - already exists. - """ - - def __init__(self, path): - self.path = path - try: - fd = os.open(path, os.O_CREAT | os.O_WRONLY | os.O_EXCL) - except OSError: - raise LockError(self.path) - os.close(fd) - - def release(self): - os.remove(self.path) - - def remove(self): - pass - - -class SystemLockFile(object): - - """ - An abstract base class for platform-specific locking. - """ - - def __init__(self, path): - self.path = path - - try: - # Open lockfile for writing without truncation: - self.fp = open(path, 'r+') - except IOError: - # If the file doesn't exist, IOError is raised; Use a+ instead. - # Note that there may be a race here. Multiple processes - # could fail on the r+ open and open the file a+, but only - # one will get the the lock and write a pid. - self.fp = open(path, 'a+') - - try: - self._lock_file() - except: - self.fp.seek(1) - self.fp.close() - del self.fp - raise - - self.fp.write(" %s\n" % os.getpid()) - self.fp.truncate() - self.fp.flush() - - def release(self): - if not hasattr(self, 'fp'): - return - self._unlock_file() - self.fp.close() - del self.fp - - def remove(self): - """ - Attempt to remove the file - """ - try: - os.remove(self.path) - except: - pass - - #@abc.abstract_method - # def _lock_file(self): - # """Attempt to obtain the lock on self.fp. Raise LockError if not - # acquired.""" - - def _unlock_file(self): - """Attempt to obtain the lock on self.fp. Raise UnlockError if not - released.""" - - -class WindowsLockFile(SystemLockFile): - - def _lock_file(self): - # Lock just the first byte - try: - msvcrt.locking(self.fp.fileno(), msvcrt.LK_NBLCK, 1) - except IOError: - raise LockError(self.fp.name) - - def _unlock_file(self): - try: - self.fp.seek(0) - msvcrt.locking(self.fp.fileno(), msvcrt.LK_UNLCK, 1) - except IOError: - raise UnlockError(self.fp.name) - -if 'msvcrt' in globals(): - LockFile = WindowsLockFile - - -class UnixLockFile(SystemLockFile): - - def _lock_file(self): - flags = fcntl.LOCK_EX | fcntl.LOCK_NB - try: - fcntl.flock(self.fp.fileno(), flags) - except IOError: - raise LockError(self.fp.name) - - # no need to implement _unlock_file, it will be unlocked on close() - -if 'fcntl' in globals(): - LockFile = UnixLockFile diff --git a/lib/cherrypy/lib/locking.py b/lib/cherrypy/lib/locking.py deleted file mode 100644 index 72dda9b351a..00000000000 --- a/lib/cherrypy/lib/locking.py +++ /dev/null @@ -1,47 +0,0 @@ -import datetime - - -class NeverExpires(object): - def expired(self): - return False - - -class Timer(object): - """ - A simple timer that will indicate when an expiration time has passed. - """ - def __init__(self, expiration): - "Create a timer that expires at `expiration` (UTC datetime)" - self.expiration = expiration - - @classmethod - def after(cls, elapsed): - """ - Return a timer that will expire after `elapsed` passes. - """ - return cls(datetime.datetime.utcnow() + elapsed) - - def expired(self): - return datetime.datetime.utcnow() >= self.expiration - - -class LockTimeout(Exception): - "An exception when a lock could not be acquired before a timeout period" - - -class LockChecker(object): - """ - Keep track of the time and detect if a timeout has expired - """ - def __init__(self, session_id, timeout): - self.session_id = session_id - if timeout: - self.timer = Timer.after(timeout) - else: - self.timer = NeverExpires() - - def expired(self): - if self.timer.expired(): - raise LockTimeout( - "Timeout acquiring lock for %(session_id)s" % vars(self)) - return False diff --git a/lib/cherrypy/lib/profiler.py b/lib/cherrypy/lib/profiler.py deleted file mode 100644 index 53cb83aa00c..00000000000 --- a/lib/cherrypy/lib/profiler.py +++ /dev/null @@ -1,218 +0,0 @@ -"""Profiler tools for CherryPy. - -CherryPy users -============== - -You can profile any of your pages as follows:: - - from cherrypy.lib import profiler - - class Root: - p = profiler.Profiler("/path/to/profile/dir") - - @cherrypy.expose - def index(self): - self.p.run(self._index) - - def _index(self): - return "Hello, world!" - - cherrypy.tree.mount(Root()) - -You can also turn on profiling for all requests -using the ``make_app`` function as WSGI middleware. - -CherryPy developers -=================== - -This module can be used whenever you make changes to CherryPy, -to get a quick sanity-check on overall CP performance. Use the -``--profile`` flag when running the test suite. Then, use the ``serve()`` -function to browse the results in a web browser. If you run this -module from the command line, it will call ``serve()`` for you. - -""" - -import io - -import cherrypy - - -def new_func_strip_path(func_name): - """Make profiler output more readable by adding `__init__` modules' parents - """ - filename, line, name = func_name - if filename.endswith("__init__.py"): - return os.path.basename(filename[:-12]) + filename[-12:], line, name - return os.path.basename(filename), line, name - -try: - import profile - import pstats - pstats.func_strip_path = new_func_strip_path -except ImportError: - profile = None - pstats = None - -import os -import os.path -import sys -import warnings - -_count = 0 - - -class Profiler(object): - - def __init__(self, path=None): - if not path: - path = os.path.join(os.path.dirname(__file__), "profile") - self.path = path - if not os.path.exists(path): - os.makedirs(path) - - def run(self, func, *args, **params): - """Dump profile data into self.path.""" - global _count - c = _count = _count + 1 - path = os.path.join(self.path, "cp_%04d.prof" % c) - prof = profile.Profile() - result = prof.runcall(func, *args, **params) - prof.dump_stats(path) - return result - - def statfiles(self): - """:rtype: list of available profiles. - """ - return [f for f in os.listdir(self.path) - if f.startswith("cp_") and f.endswith(".prof")] - - def stats(self, filename, sortby='cumulative'): - """:rtype stats(index): output of print_stats() for the given profile. - """ - sio = io.StringIO() - if sys.version_info >= (2, 5): - s = pstats.Stats(os.path.join(self.path, filename), stream=sio) - s.strip_dirs() - s.sort_stats(sortby) - s.print_stats() - else: - # pstats.Stats before Python 2.5 didn't take a 'stream' arg, - # but just printed to stdout. So re-route stdout. - s = pstats.Stats(os.path.join(self.path, filename)) - s.strip_dirs() - s.sort_stats(sortby) - oldout = sys.stdout - try: - sys.stdout = sio - s.print_stats() - finally: - sys.stdout = oldout - response = sio.getvalue() - sio.close() - return response - - @cherrypy.expose - def index(self): - return """ - CherryPy profile data - - - - - - """ - - @cherrypy.expose - def menu(self): - yield "

Profiling runs

" - yield "

Click on one of the runs below to see profiling data.

" - runs = self.statfiles() - runs.sort() - for i in runs: - yield "%s
" % ( - i, i) - - @cherrypy.expose - def report(self, filename): - import cherrypy - cherrypy.response.headers['Content-Type'] = 'text/plain' - return self.stats(filename) - - -class ProfileAggregator(Profiler): - - def __init__(self, path=None): - Profiler.__init__(self, path) - global _count - self.count = _count = _count + 1 - self.profiler = profile.Profile() - - def run(self, func, *args, **params): - path = os.path.join(self.path, "cp_%04d.prof" % self.count) - result = self.profiler.runcall(func, *args, **params) - self.profiler.dump_stats(path) - return result - - -class make_app: - - def __init__(self, nextapp, path=None, aggregate=False): - """Make a WSGI middleware app which wraps 'nextapp' with profiling. - - nextapp - the WSGI application to wrap, usually an instance of - cherrypy.Application. - - path - where to dump the profiling output. - - aggregate - if True, profile data for all HTTP requests will go in - a single file. If False (the default), each HTTP request will - dump its profile data into a separate file. - - """ - if profile is None or pstats is None: - msg = ("Your installation of Python does not have a profile " - "module. If you're on Debian, try " - "`sudo apt-get install python-profiler`. " - "See http://www.cherrypy.org/wiki/ProfilingOnDebian " - "for details.") - warnings.warn(msg) - - self.nextapp = nextapp - self.aggregate = aggregate - if aggregate: - self.profiler = ProfileAggregator(path) - else: - self.profiler = Profiler(path) - - def __call__(self, environ, start_response): - def gather(): - result = [] - for line in self.nextapp(environ, start_response): - result.append(line) - return result - return self.profiler.run(gather) - - -def serve(path=None, port=8080): - if profile is None or pstats is None: - msg = ("Your installation of Python does not have a profile module. " - "If you're on Debian, try " - "`sudo apt-get install python-profiler`. " - "See http://www.cherrypy.org/wiki/ProfilingOnDebian " - "for details.") - warnings.warn(msg) - - import cherrypy - cherrypy.config.update({'server.socket_port': int(port), - 'server.thread_pool': 10, - 'environment': "production", - }) - cherrypy.quickstart(Profiler(path)) - - -if __name__ == "__main__": - serve(*tuple(sys.argv[1:])) diff --git a/lib/cherrypy/lib/reprconf.py b/lib/cherrypy/lib/reprconf.py deleted file mode 100644 index 8af1f777d6b..00000000000 --- a/lib/cherrypy/lib/reprconf.py +++ /dev/null @@ -1,539 +0,0 @@ -"""Generic configuration system using unrepr. - -Configuration data may be supplied as a Python dictionary, as a filename, -or as an open file object. When you supply a filename or file, Python's -builtin ConfigParser is used (with some extensions). - -Namespaces ----------- - -Configuration keys are separated into namespaces by the first "." in the key. - -The only key that cannot exist in a namespace is the "environment" entry. -This special entry 'imports' other config entries from a template stored in -the Config.environments dict. - -You can define your own namespaces to be called when new config is merged -by adding a named handler to Config.namespaces. The name can be any string, -and the handler must be either a callable or a context manager. -""" - -try: - # Python 3.0+ - from configparser import ConfigParser -except ImportError: - from ConfigParser import ConfigParser - -try: - set -except NameError: - from sets import Set as set - -try: - basestring -except NameError: - basestring = str - -try: - # Python 3 - import builtins -except ImportError: - # Python 2 - import __builtin__ as builtins - -import operator as _operator -import sys - - -def as_dict(config): - """Return a dict from 'config' whether it is a dict, file, or filename.""" - if isinstance(config, basestring): - config = Parser().dict_from_file(config) - elif hasattr(config, 'read'): - config = Parser().dict_from_file(config) - return config - - -class NamespaceSet(dict): - - """A dict of config namespace names and handlers. - - Each config entry should begin with a namespace name; the corresponding - namespace handler will be called once for each config entry in that - namespace, and will be passed two arguments: the config key (with the - namespace removed) and the config value. - - Namespace handlers may be any Python callable; they may also be - Python 2.5-style 'context managers', in which case their __enter__ - method should return a callable to be used as the handler. - See cherrypy.tools (the Toolbox class) for an example. - """ - - def __call__(self, config): - """Iterate through config and pass it to each namespace handler. - - config - A flat dict, where keys use dots to separate - namespaces, and values are arbitrary. - - The first name in each config key is used to look up the corresponding - namespace handler. For example, a config entry of {'tools.gzip.on': v} - will call the 'tools' namespace handler with the args: ('gzip.on', v) - """ - # Separate the given config into namespaces - ns_confs = {} - for k in config: - if "." in k: - ns, name = k.split(".", 1) - bucket = ns_confs.setdefault(ns, {}) - bucket[name] = config[k] - - # I chose __enter__ and __exit__ so someday this could be - # rewritten using Python 2.5's 'with' statement: - # for ns, handler in self.iteritems(): - # with handler as callable: - # for k, v in ns_confs.get(ns, {}).iteritems(): - # callable(k, v) - for ns, handler in self.items(): - exit = getattr(handler, "__exit__", None) - if exit: - callable = handler.__enter__() - no_exc = True - try: - try: - for k, v in ns_confs.get(ns, {}).items(): - callable(k, v) - except: - # The exceptional case is handled here - no_exc = False - if exit is None: - raise - if not exit(*sys.exc_info()): - raise - # The exception is swallowed if exit() returns true - finally: - # The normal and non-local-goto cases are handled here - if no_exc and exit: - exit(None, None, None) - else: - for k, v in ns_confs.get(ns, {}).items(): - handler(k, v) - - def __repr__(self): - return "%s.%s(%s)" % (self.__module__, self.__class__.__name__, - dict.__repr__(self)) - - def __copy__(self): - newobj = self.__class__() - newobj.update(self) - return newobj - copy = __copy__ - - -class Config(dict): - - """A dict-like set of configuration data, with defaults and namespaces. - - May take a file, filename, or dict. - """ - - defaults = {} - environments = {} - namespaces = NamespaceSet() - - def __init__(self, file=None, **kwargs): - self.reset() - if file is not None: - self.update(file) - if kwargs: - self.update(kwargs) - - def reset(self): - """Reset self to default values.""" - self.clear() - dict.update(self, self.defaults) - - def update(self, config): - """Update self from a dict, file or filename.""" - if isinstance(config, basestring): - # Filename - config = Parser().dict_from_file(config) - elif hasattr(config, 'read'): - # Open file object - config = Parser().dict_from_file(config) - else: - config = config.copy() - self._apply(config) - - def _apply(self, config): - """Update self from a dict.""" - which_env = config.get('environment') - if which_env: - env = self.environments[which_env] - for k in env: - if k not in config: - config[k] = env[k] - - dict.update(self, config) - self.namespaces(config) - - def __setitem__(self, k, v): - dict.__setitem__(self, k, v) - self.namespaces({k: v}) - - -class Parser(ConfigParser): - - """Sub-class of ConfigParser that keeps the case of options and that - raises an exception if the file cannot be read. - """ - - def optionxform(self, optionstr): - return optionstr - - def read(self, filenames): - if isinstance(filenames, basestring): - filenames = [filenames] - for filename in filenames: - # try: - # fp = open(filename) - # except IOError: - # continue - fp = open(filename) - try: - self._read(fp, filename) - finally: - fp.close() - - def as_dict(self, raw=False, vars=None): - """Convert an INI file to a dictionary""" - # Load INI file into a dict - result = {} - for section in self.sections(): - if section not in result: - result[section] = {} - for option in self.options(section): - value = self.get(section, option, raw=raw, vars=vars) - try: - value = unrepr(value) - except Exception: - x = sys.exc_info()[1] - msg = ("Config error in section: %r, option: %r, " - "value: %r. Config values must be valid Python." % - (section, option, value)) - raise ValueError(msg, x.__class__.__name__, x.args) - result[section][option] = value - return result - - def dict_from_file(self, file): - if hasattr(file, 'read'): - self.readfp(file) - else: - self.read(file) - return self.as_dict() - - -# public domain "unrepr" implementation, found on the web and then improved. - - -class _Builder2: - - def build(self, o): - m = getattr(self, 'build_' + o.__class__.__name__, None) - if m is None: - raise TypeError("unrepr does not recognize %s" % - repr(o.__class__.__name__)) - return m(o) - - def astnode(self, s): - """Return a Python2 ast Node compiled from a string.""" - try: - import compiler - except ImportError: - # Fallback to eval when compiler package is not available, - # e.g. IronPython 1.0. - return eval(s) - - p = compiler.parse("__tempvalue__ = " + s) - return p.getChildren()[1].getChildren()[0].getChildren()[1] - - def build_Subscript(self, o): - expr, flags, subs = o.getChildren() - expr = self.build(expr) - subs = self.build(subs) - return expr[subs] - - def build_CallFunc(self, o): - children = o.getChildren() - # Build callee from first child - callee = self.build(children[0]) - # Build args and kwargs from remaining children - args = [] - kwargs = {} - for child in children[1:]: - class_name = child.__class__.__name__ - # None is ignored - if class_name == 'NoneType': - continue - # Keywords become kwargs - if class_name == 'Keyword': - kwargs.update(self.build(child)) - # Everything else becomes args - else : - args.append(self.build(child)) - - return callee(*args, **kwargs) - - def build_Keyword(self, o): - key, value_obj = o.getChildren() - value = self.build(value_obj) - kw_dict = {key: value} - return kw_dict - - def build_List(self, o): - return map(self.build, o.getChildren()) - - def build_Const(self, o): - return o.value - - def build_Dict(self, o): - d = {} - i = iter(map(self.build, o.getChildren())) - for el in i: - d[el] = i.next() - return d - - def build_Tuple(self, o): - return tuple(self.build_List(o)) - - def build_Name(self, o): - name = o.name - if name == 'None': - return None - if name == 'True': - return True - if name == 'False': - return False - - # See if the Name is a package or module. If it is, import it. - try: - return modules(name) - except ImportError: - pass - - # See if the Name is in builtins. - try: - return getattr(builtins, name) - except AttributeError: - pass - - raise TypeError("unrepr could not resolve the name %s" % repr(name)) - - def build_Add(self, o): - left, right = map(self.build, o.getChildren()) - return left + right - - def build_Mul(self, o): - left, right = map(self.build, o.getChildren()) - return left * right - - def build_Getattr(self, o): - parent = self.build(o.expr) - return getattr(parent, o.attrname) - - def build_NoneType(self, o): - return None - - def build_UnarySub(self, o): - return -self.build(o.getChildren()[0]) - - def build_UnaryAdd(self, o): - return self.build(o.getChildren()[0]) - - -class _Builder3: - - def build(self, o): - m = getattr(self, 'build_' + o.__class__.__name__, None) - if m is None: - raise TypeError("unrepr does not recognize %s" % - repr(o.__class__.__name__)) - return m(o) - - def astnode(self, s): - """Return a Python3 ast Node compiled from a string.""" - try: - import ast - except ImportError: - # Fallback to eval when ast package is not available, - # e.g. IronPython 1.0. - return eval(s) - - p = ast.parse("__tempvalue__ = " + s) - return p.body[0].value - - def build_Subscript(self, o): - return self.build(o.value)[self.build(o.slice)] - - def build_Index(self, o): - return self.build(o.value) - - def _build_call35(self, o): - """ - Workaround for python 3.5 _ast.Call signature, docs found here - https://greentreesnakes.readthedocs.org/en/latest/nodes.html - """ - import ast - callee = self.build(o.func) - args = [] - if o.args is not None: - for a in o.args: - if isinstance(a, ast.Starred): - args.append(self.build(a.value)) - else: - args.append(self.build(a)) - kwargs = {} - for kw in o.keywords: - if kw.arg is None: # double asterix `**` - rst = self.build(kw.value) - if not isinstance(rst, dict): - raise TypeError("Invalid argument for call." - "Must be a mapping object.") - # give preference to the keys set directly from arg=value - for k, v in rst.items(): - if k not in kwargs: - kwargs[k] = v - else: # defined on the call as: arg=value - kwargs[kw.arg] = self.build(kw.value) - return callee(*args, **kwargs) - - def build_Call(self, o): - if sys.version_info >= (3, 5): - return self._build_call35(o) - - callee = self.build(o.func) - - if o.args is None: - args = () - else: - args = tuple([self.build(a) for a in o.args]) - - if o.starargs is None: - starargs = () - else: - starargs = tuple(self.build(o.starargs)) - - if o.kwargs is None: - kwargs = {} - else: - kwargs = self.build(o.kwargs) - if o.keywords is not None: # direct a=b keywords - for kw in o.keywords: - # preference because is a direct keyword against **kwargs - kwargs[kw.arg] = self.build(kw.value) - return callee(*(args + starargs), **kwargs) - - def build_List(self, o): - return list(map(self.build, o.elts)) - - def build_Str(self, o): - return o.s - - def build_Num(self, o): - return o.n - - def build_Dict(self, o): - return dict([(self.build(k), self.build(v)) - for k, v in zip(o.keys, o.values)]) - - def build_Tuple(self, o): - return tuple(self.build_List(o)) - - def build_Name(self, o): - name = o.id - if name == 'None': - return None - if name == 'True': - return True - if name == 'False': - return False - - # See if the Name is a package or module. If it is, import it. - try: - return modules(name) - except ImportError: - pass - - # See if the Name is in builtins. - try: - import builtins - return getattr(builtins, name) - except AttributeError: - pass - - raise TypeError("unrepr could not resolve the name %s" % repr(name)) - - def build_NameConstant(self, o): - return o.value - - def build_UnaryOp(self, o): - op, operand = map(self.build, [o.op, o.operand]) - return op(operand) - - def build_BinOp(self, o): - left, op, right = map(self.build, [o.left, o.op, o.right]) - return op(left, right) - - def build_Add(self, o): - return _operator.add - - def build_Mult(self, o): - return _operator.mul - - def build_USub(self, o): - return _operator.neg - - def build_Attribute(self, o): - parent = self.build(o.value) - return getattr(parent, o.attr) - - def build_NoneType(self, o): - return None - - -def unrepr(s): - """Return a Python object compiled from a string.""" - if not s: - return s - if sys.version_info < (3, 0): - b = _Builder2() - else: - b = _Builder3() - obj = b.astnode(s) - return b.build(obj) - - -def modules(modulePath): - """Load a module and retrieve a reference to that module.""" - __import__(modulePath) - return sys.modules[modulePath] - - -def attributes(full_attribute_name): - """Load a module and retrieve an attribute of that module.""" - - # Parse out the path, module, and attribute - last_dot = full_attribute_name.rfind(".") - attr_name = full_attribute_name[last_dot + 1:] - mod_path = full_attribute_name[:last_dot] - - mod = modules(mod_path) - # Let an AttributeError propagate outward. - try: - attr = getattr(mod, attr_name) - except AttributeError: - raise AttributeError("'%s' object has no attribute '%s'" - % (mod_path, attr_name)) - - # Return a reference to the attribute. - return attr diff --git a/lib/cherrypy/lib/sessions.py b/lib/cherrypy/lib/sessions.py deleted file mode 100644 index fa13bae551e..00000000000 --- a/lib/cherrypy/lib/sessions.py +++ /dev/null @@ -1,891 +0,0 @@ -"""Session implementation for CherryPy. - -You need to edit your config file to use sessions. Here's an example:: - - [/] - tools.sessions.on = True - tools.sessions.storage_type = "file" - tools.sessions.storage_path = "/home/site/sessions" - tools.sessions.timeout = 60 - -This sets the session to be stored in files in the directory -/home/site/sessions, and the session timeout to 60 minutes. If you omit -``storage_type`` the sessions will be saved in RAM. -``tools.sessions.on`` is the only required line for working sessions, -the rest are optional. - -By default, the session ID is passed in a cookie, so the client's browser must -have cookies enabled for your site. - -To set data for the current session, use -``cherrypy.session['fieldname'] = 'fieldvalue'``; -to get data use ``cherrypy.session.get('fieldname')``. - -================ -Locking sessions -================ - -By default, the ``'locking'`` mode of sessions is ``'implicit'``, which means -the session is locked early and unlocked late. Be mindful of this default mode -for any requests that take a long time to process (streaming responses, -expensive calculations, database lookups, API calls, etc), as other concurrent -requests that also utilize sessions will hang until the session is unlocked. - -If you want to control when the session data is locked and unlocked, -set ``tools.sessions.locking = 'explicit'``. Then call -``cherrypy.session.acquire_lock()`` and ``cherrypy.session.release_lock()``. -Regardless of which mode you use, the session is guaranteed to be unlocked when -the request is complete. - -================= -Expiring Sessions -================= - -You can force a session to expire with :func:`cherrypy.lib.sessions.expire`. -Simply call that function at the point you want the session to expire, and it -will cause the session cookie to expire client-side. - -=========================== -Session Fixation Protection -=========================== - -If CherryPy receives, via a request cookie, a session id that it does not -recognize, it will reject that id and create a new one to return in the -response cookie. This `helps prevent session fixation attacks -`_. -However, CherryPy "recognizes" a session id by looking up the saved session -data for that id. Therefore, if you never save any session data, -**you will get a new session id for every request**. - -================ -Sharing Sessions -================ - -If you run multiple instances of CherryPy (for example via mod_python behind -Apache prefork), you most likely cannot use the RAM session backend, since each -instance of CherryPy will have its own memory space. Use a different backend -instead, and verify that all instances are pointing at the same file or db -location. Alternately, you might try a load balancer which makes sessions -"sticky". Google is your friend, there. - -================ -Expiration Dates -================ - -The response cookie will possess an expiration date to inform the client at -which point to stop sending the cookie back in requests. If the server time -and client time differ, expect sessions to be unreliable. **Make sure the -system time of your server is accurate**. - -CherryPy defaults to a 60-minute session timeout, which also applies to the -cookie which is sent to the client. Unfortunately, some versions of Safari -("4 public beta" on Windows XP at least) appear to have a bug in their parsing -of the GMT expiration date--they appear to interpret the date as one hour in -the past. Sixty minutes minus one hour is pretty close to zero, so you may -experience this bug as a new session id for every request, unless the requests -are less than one second apart. To fix, try increasing the session.timeout. - -On the other extreme, some users report Firefox sending cookies after their -expiration date, although this was on a system with an inaccurate system time. -Maybe FF doesn't trust system time. -""" -import sys -import datetime -import os -import time -import threading - -IS_PY3 = sys.version_info[0] == 3 -try: - import six -except ImportError: - if IS_PY3: - _unicode_type = str - else: - _unicode_type = unicode -else: - _unicode_type = six.text_type - -import cherrypy -from cherrypy._cpcompat import copyitems, pickle, random20 -from cherrypy.lib import httputil -from cherrypy.lib import lockfile -from cherrypy.lib import locking -from cherrypy.lib import is_iterator - -missing = object() - - -class Session(object): - - """A CherryPy dict-like Session object (one per request).""" - - _id = None - - id_observers = None - "A list of callbacks to which to pass new id's." - - def _get_id(self): - return self._id - - def _set_id(self, value): - self._id = value - for o in self.id_observers: - o(value) - id = property(_get_id, _set_id, doc="The current session ID.") - - timeout = 60 - "Number of minutes after which to delete session data." - - locked = False - """ - If True, this session instance has exclusive read/write access - to session data.""" - - loaded = False - """ - If True, data has been retrieved from storage. This should happen - automatically on the first attempt to access session data.""" - - clean_thread = None - "Class-level Monitor which calls self.clean_up." - - clean_freq = 5 - "The poll rate for expired session cleanup in minutes." - - originalid = None - "The session id passed by the client. May be missing or unsafe." - - missing = False - "True if the session requested by the client did not exist." - - regenerated = False - """ - True if the application called session.regenerate(). This is not set by - internal calls to regenerate the session id.""" - - debug = False - "If True, log debug information." - - # --------------------- Session management methods --------------------- # - - def __init__(self, id=None, **kwargs): - self.id_observers = [] - self._data = {} - - for k, v in kwargs.items(): - setattr(self, k, v) - - self.originalid = id - self.missing = False - if id is None: - if self.debug: - cherrypy.log('No id given; making a new one', 'TOOLS.SESSIONS') - self._regenerate() - else: - self.id = id - if self._exists(): - if self.debug: - cherrypy.log('Set id to %s.' % id, 'TOOLS.SESSIONS') - else: - if self.debug: - cherrypy.log('Expired or malicious session %r; ' - 'making a new one' % id, 'TOOLS.SESSIONS') - # Expired or malicious session. Make a new one. - # See https://github.com/cherrypy/cherrypy/issues/709. - self.id = None - self.missing = True - self._regenerate() - - def now(self): - """Generate the session specific concept of 'now'. - - Other session providers can override this to use alternative, - possibly timezone aware, versions of 'now'. - """ - return datetime.datetime.now() - - def regenerate(self): - """Replace the current session (with a new id).""" - self.regenerated = True - self._regenerate() - - def _regenerate(self): - if self.id is not None: - if self.debug: - cherrypy.log( - 'Deleting the existing session %r before ' - 'regeneration.' % self.id, - 'TOOLS.SESSIONS') - self.delete() - - old_session_was_locked = self.locked - if old_session_was_locked: - self.release_lock() - if self.debug: - cherrypy.log('Old lock released.', 'TOOLS.SESSIONS') - - self.id = None - while self.id is None: - self.id = self.generate_id() - # Assert that the generated id is not already stored. - if self._exists(): - self.id = None - if self.debug: - cherrypy.log('Set id to generated %s.' % self.id, - 'TOOLS.SESSIONS') - - if old_session_was_locked: - self.acquire_lock() - if self.debug: - cherrypy.log('Regenerated lock acquired.', 'TOOLS.SESSIONS') - - def clean_up(self): - """Clean up expired sessions.""" - pass - - def generate_id(self): - """Return a new session id.""" - return random20() - - def save(self): - """Save session data.""" - try: - # If session data has never been loaded then it's never been - # accessed: no need to save it - if self.loaded: - t = datetime.timedelta(seconds=self.timeout * 60) - expiration_time = self.now() + t - if self.debug: - cherrypy.log('Saving session %r with expiry %s' % - (self.id, expiration_time), - 'TOOLS.SESSIONS') - self._save(expiration_time) - else: - if self.debug: - cherrypy.log( - 'Skipping save of session %r (no session loaded).' % - self.id, 'TOOLS.SESSIONS') - finally: - if self.locked: - # Always release the lock if the user didn't release it - self.release_lock() - if self.debug: - cherrypy.log('Lock released after save.', 'TOOLS.SESSIONS') - - def load(self): - """Copy stored session data into this session instance.""" - data = self._load() - # data is either None or a tuple (session_data, expiration_time) - if data is None or data[1] < self.now(): - if self.debug: - cherrypy.log('Expired session %r, flushing data.' % self.id, - 'TOOLS.SESSIONS') - self._data = {} - else: - if self.debug: - cherrypy.log('Data loaded for session %r.' % self.id, - 'TOOLS.SESSIONS') - self._data = data[0] - self.loaded = True - - # Stick the clean_thread in the class, not the instance. - # The instances are created and destroyed per-request. - cls = self.__class__ - if self.clean_freq and not cls.clean_thread: - # clean_up is an instancemethod and not a classmethod, - # so that tool config can be accessed inside the method. - t = cherrypy.process.plugins.Monitor( - cherrypy.engine, self.clean_up, self.clean_freq * 60, - name='Session cleanup') - t.subscribe() - cls.clean_thread = t - t.start() - if self.debug: - cherrypy.log('Started cleanup thread.', 'TOOLS.SESSIONS') - - def delete(self): - """Delete stored session data.""" - self._delete() - if self.debug: - cherrypy.log('Deleted session %s.' % self.id, - 'TOOLS.SESSIONS') - - # -------------------- Application accessor methods -------------------- # - - def __getitem__(self, key): - if not self.loaded: - self.load() - return self._data[key] - - def __setitem__(self, key, value): - if not self.loaded: - self.load() - self._data[key] = value - - def __delitem__(self, key): - if not self.loaded: - self.load() - del self._data[key] - - def pop(self, key, default=missing): - """Remove the specified key and return the corresponding value. - If key is not found, default is returned if given, - otherwise KeyError is raised. - """ - if not self.loaded: - self.load() - if default is missing: - return self._data.pop(key) - else: - return self._data.pop(key, default) - - def __contains__(self, key): - if not self.loaded: - self.load() - return key in self._data - - if hasattr({}, 'has_key'): - def has_key(self, key): - """D.has_key(k) -> True if D has a key k, else False.""" - if not self.loaded: - self.load() - return key in self._data - - def get(self, key, default=None): - """D.get(k[,d]) -> D[k] if k in D, else d. d defaults to None.""" - if not self.loaded: - self.load() - return self._data.get(key, default) - - def update(self, d): - """D.update(E) -> None. Update D from E: for k in E: D[k] = E[k].""" - if not self.loaded: - self.load() - self._data.update(d) - - def setdefault(self, key, default=None): - """D.setdefault(k[,d]) -> D.get(k,d), also set D[k]=d if k not in D.""" - if not self.loaded: - self.load() - return self._data.setdefault(key, default) - - def clear(self): - """D.clear() -> None. Remove all items from D.""" - if not self.loaded: - self.load() - self._data.clear() - - def keys(self): - """D.keys() -> list of D's keys.""" - if not self.loaded: - self.load() - return self._data.keys() - - def items(self): - """D.items() -> list of D's (key, value) pairs, as 2-tuples.""" - if not self.loaded: - self.load() - return self._data.items() - - def values(self): - """D.values() -> list of D's values.""" - if not self.loaded: - self.load() - return self._data.values() - - -class RamSession(Session): - - # Class-level objects. Don't rebind these! - cache = {} - locks = {} - - def clean_up(self): - """Clean up expired sessions.""" - - now = self.now() - for _id, (data, expiration_time) in copyitems(self.cache): - if expiration_time <= now: - try: - del self.cache[_id] - except KeyError: - pass - try: - if self.locks[_id].acquire(blocking=False): - lock = self.locks.pop(_id) - lock.release() - except KeyError: - pass - - # added to remove obsolete lock objects - for _id in list(self.locks): - if _id not in self.cache and self.locks[_id].acquire(blocking=False): - lock = self.locks.pop(_id) - lock.release() - - def _exists(self): - return self.id in self.cache - - def _load(self): - return self.cache.get(self.id) - - def _save(self, expiration_time): - self.cache[self.id] = (self._data, expiration_time) - - def _delete(self): - self.cache.pop(self.id, None) - - def acquire_lock(self): - """Acquire an exclusive lock on the currently-loaded session data.""" - self.locked = True - self.locks.setdefault(self.id, threading.RLock()).acquire() - - def release_lock(self): - """Release the lock on the currently-loaded session data.""" - self.locks[self.id].release() - self.locked = False - - def __len__(self): - """Return the number of active sessions.""" - return len(self.cache) - - -class FileSession(Session): - - """Implementation of the File backend for sessions - - storage_path - The folder where session data will be saved. Each session - will be saved as pickle.dump(data, expiration_time) in its own file; - the filename will be self.SESSION_PREFIX + self.id. - - lock_timeout - A timedelta or numeric seconds indicating how long - to block acquiring a lock. If None (default), acquiring a lock - will block indefinitely. - """ - - SESSION_PREFIX = 'session-' - LOCK_SUFFIX = '.lock' - pickle_protocol = pickle.HIGHEST_PROTOCOL - - def __init__(self, id=None, **kwargs): - # The 'storage_path' arg is required for file-based sessions. - kwargs['storage_path'] = os.path.abspath(kwargs['storage_path']) - kwargs.setdefault('lock_timeout', None) - - Session.__init__(self, id=id, **kwargs) - - # validate self.lock_timeout - if isinstance(self.lock_timeout, (int, float)): - self.lock_timeout = datetime.timedelta(seconds=self.lock_timeout) - if not isinstance(self.lock_timeout, (datetime.timedelta, type(None))): - raise ValueError("Lock timeout must be numeric seconds or " - "a timedelta instance.") - - def setup(cls, **kwargs): - """Set up the storage system for file-based sessions. - - This should only be called once per process; this will be done - automatically when using sessions.init (as the built-in Tool does). - """ - # The 'storage_path' arg is required for file-based sessions. - kwargs['storage_path'] = os.path.abspath(kwargs['storage_path']) - - for k, v in kwargs.items(): - setattr(cls, k, v) - setup = classmethod(setup) - - def _get_file_path(self): - f = os.path.join(self.storage_path, self.SESSION_PREFIX + self.id) - if not os.path.abspath(f).startswith(self.storage_path): - raise cherrypy.HTTPError(400, "Invalid session id in cookie.") - return f - - def _exists(self): - path = self._get_file_path() - return os.path.exists(path) - - def _load(self, path=None): - assert self.locked, ("The session load without being locked. " - "Check your tools' priority levels.") - if path is None: - path = self._get_file_path() - try: - f = open(path, "rb") - try: - return pickle.load(f) - finally: - f.close() - except (IOError, EOFError): - e = sys.exc_info()[1] - if self.debug: - cherrypy.log("Error loading the session pickle: %s" % - e, 'TOOLS.SESSIONS') - return None - - def _save(self, expiration_time): - assert self.locked, ("The session was saved without being locked. " - "Check your tools' priority levels.") - f = open(self._get_file_path(), "wb") - try: - pickle.dump((self._data, expiration_time), f, self.pickle_protocol) - finally: - f.close() - - def _delete(self): - assert self.locked, ("The session deletion without being locked. " - "Check your tools' priority levels.") - try: - os.unlink(self._get_file_path()) - except OSError: - pass - - def acquire_lock(self, path=None): - """Acquire an exclusive lock on the currently-loaded session data.""" - if path is None: - path = self._get_file_path() - path += self.LOCK_SUFFIX - checker = locking.LockChecker(self.id, self.lock_timeout) - while not checker.expired(): - try: - self.lock = lockfile.LockFile(path) - except lockfile.LockError: - time.sleep(0.1) - else: - break - self.locked = True - if self.debug: - cherrypy.log('Lock acquired.', 'TOOLS.SESSIONS') - - def release_lock(self, path=None): - """Release the lock on the currently-loaded session data.""" - self.lock.release() - self.lock.remove() - self.locked = False - - def clean_up(self): - """Clean up expired sessions.""" - now = self.now() - # Iterate over all session files in self.storage_path - for fname in os.listdir(self.storage_path): - if (fname.startswith(self.SESSION_PREFIX) - and not fname.endswith(self.LOCK_SUFFIX)): - # We have a session file: lock and load it and check - # if it's expired. If it fails, nevermind. - path = os.path.join(self.storage_path, fname) - self.acquire_lock(path) - if self.debug: - # This is a bit of a hack, since we're calling clean_up - # on the first instance rather than the entire class, - # so depending on whether you have "debug" set on the - # path of the first session called, this may not run. - cherrypy.log('Cleanup lock acquired.', 'TOOLS.SESSIONS') - - try: - contents = self._load(path) - # _load returns None on IOError - if contents is not None: - data, expiration_time = contents - if expiration_time < now: - # Session expired: deleting it - os.unlink(path) - finally: - self.release_lock(path) - - def __len__(self): - """Return the number of active sessions.""" - return len([fname for fname in os.listdir(self.storage_path) - if (fname.startswith(self.SESSION_PREFIX) - and not fname.endswith(self.LOCK_SUFFIX))]) - - -class MemcachedSession(Session): - - # The most popular memcached client for Python isn't thread-safe. - # Wrap all .get and .set operations in a single lock. - mc_lock = threading.RLock() - - # This is a seperate set of locks per session id. - locks = {} - - servers = ['127.0.0.1:11211'] - - def setup(cls, **kwargs): - """Set up the storage system for memcached-based sessions. - - This should only be called once per process; this will be done - automatically when using sessions.init (as the built-in Tool does). - """ - for k, v in kwargs.items(): - setattr(cls, k, v) - - import memcache - cls.cache = memcache.Client(cls.servers) - setup = classmethod(setup) - - def _get_id(self): - return self._id - - def _set_id(self, value): - # This encode() call is where we differ from the superclass. - # Memcache keys MUST be byte strings, not unicode. - if isinstance(value, _unicode_type): - value = value.encode('utf-8') - - self._id = value - for o in self.id_observers: - o(value) - id = property(_get_id, _set_id, doc="The current session ID.") - - def _exists(self): - self.mc_lock.acquire() - try: - return bool(self.cache.get(self.id)) - finally: - self.mc_lock.release() - - def _load(self): - self.mc_lock.acquire() - try: - return self.cache.get(self.id) - finally: - self.mc_lock.release() - - def _save(self, expiration_time): - # Send the expiration time as "Unix time" (seconds since 1/1/1970) - td = int(time.mktime(expiration_time.timetuple())) - self.mc_lock.acquire() - try: - if not self.cache.set(self.id, (self._data, expiration_time), td): - raise AssertionError( - "Session data for id %r not set." % self.id) - finally: - self.mc_lock.release() - - def _delete(self): - self.cache.delete(self.id) - - def acquire_lock(self): - """Acquire an exclusive lock on the currently-loaded session data.""" - self.locked = True - self.locks.setdefault(self.id, threading.RLock()).acquire() - if self.debug: - cherrypy.log('Lock acquired.', 'TOOLS.SESSIONS') - - def release_lock(self): - """Release the lock on the currently-loaded session data.""" - self.locks[self.id].release() - self.locked = False - - def __len__(self): - """Return the number of active sessions.""" - raise NotImplementedError - - -# Hook functions (for CherryPy tools) - -def save(): - """Save any changed session data.""" - - if not hasattr(cherrypy.serving, "session"): - return - request = cherrypy.serving.request - response = cherrypy.serving.response - - # Guard against running twice - if hasattr(request, "_sessionsaved"): - return - request._sessionsaved = True - - if response.stream: - # If the body is being streamed, we have to save the data - # *after* the response has been written out - request.hooks.attach('on_end_request', cherrypy.session.save) - else: - # If the body is not being streamed, we save the data now - # (so we can release the lock). - if is_iterator(response.body): - response.collapse_body() - cherrypy.session.save() -save.failsafe = True - - -def close(): - """Close the session object for this request.""" - sess = getattr(cherrypy.serving, "session", None) - if getattr(sess, "locked", False): - # If the session is still locked we release the lock - sess.release_lock() - if sess.debug: - cherrypy.log('Lock released on close.', 'TOOLS.SESSIONS') -close.failsafe = True -close.priority = 90 - - -def init(storage_type='ram', path=None, path_header=None, name='session_id', - timeout=60, domain=None, secure=False, clean_freq=5, - persistent=True, httponly=False, debug=False, **kwargs): - """Initialize session object (using cookies). - - storage_type - One of 'ram', 'file', memcached'. This will be - used to look up the corresponding class in cherrypy.lib.sessions - globals. For example, 'file' will use the FileSession class. - - path - The 'path' value to stick in the response cookie metadata. - - path_header - If 'path' is None (the default), then the response - cookie 'path' will be pulled from request.headers[path_header]. - - name - The name of the cookie. - - timeout - The expiration timeout (in minutes) for the stored session data. - If 'persistent' is True (the default), this is also the timeout - for the cookie. - - domain - The cookie domain. - - secure - If False (the default) the cookie 'secure' value will not - be set. If True, the cookie 'secure' value will be set (to 1). - - clean_freq (minutes) - The poll rate for expired session cleanup. - - persistent - If True (the default), the 'timeout' argument will be used - to expire the cookie. If False, the cookie will not have an expiry, - and the cookie will be a "session cookie" which expires when the - browser is closed. - - httponly - If False (the default) the cookie 'httponly' value will not be set. - If True, the cookie 'httponly' value will be set (to 1). - - Any additional kwargs will be bound to the new Session instance, - and may be specific to the storage type. See the subclass of Session - you're using for more information. - """ - - request = cherrypy.serving.request - - # Guard against running twice - if hasattr(request, "_session_init_flag"): - return - request._session_init_flag = True - - # Check if request came with a session ID - id = None - if name in request.cookie: - id = request.cookie[name].value - if debug: - cherrypy.log('ID obtained from request.cookie: %r' % id, - 'TOOLS.SESSIONS') - - # Find the storage class and call setup (first time only). - storage_class = storage_type.title() + 'Session' - storage_class = globals()[storage_class] - if not hasattr(cherrypy, "session"): - if hasattr(storage_class, "setup"): - storage_class.setup(**kwargs) - - # Create and attach a new Session instance to cherrypy.serving. - # It will possess a reference to (and lock, and lazily load) - # the requested session data. - kwargs['timeout'] = timeout - kwargs['clean_freq'] = clean_freq - cherrypy.serving.session = sess = storage_class(id, **kwargs) - sess.debug = debug - - def update_cookie(id): - """Update the cookie every time the session id changes.""" - cherrypy.serving.response.cookie[name] = id - sess.id_observers.append(update_cookie) - - # Create cherrypy.session which will proxy to cherrypy.serving.session - if not hasattr(cherrypy, "session"): - cherrypy.session = cherrypy._ThreadLocalProxy('session') - - if persistent: - cookie_timeout = timeout - else: - # See http://support.microsoft.com/kb/223799/EN-US/ - # and http://support.mozilla.com/en-US/kb/Cookies - cookie_timeout = None - set_response_cookie(path=path, path_header=path_header, name=name, - timeout=cookie_timeout, domain=domain, secure=secure, - httponly=httponly) - - -def set_response_cookie(path=None, path_header=None, name='session_id', - timeout=60, domain=None, secure=False, httponly=False): - """Set a response cookie for the client. - - path - the 'path' value to stick in the response cookie metadata. - - path_header - if 'path' is None (the default), then the response - cookie 'path' will be pulled from request.headers[path_header]. - - name - the name of the cookie. - - timeout - the expiration timeout for the cookie. If 0 or other boolean - False, no 'expires' param will be set, and the cookie will be a - "session cookie" which expires when the browser is closed. - - domain - the cookie domain. - - secure - if False (the default) the cookie 'secure' value will not - be set. If True, the cookie 'secure' value will be set (to 1). - - httponly - If False (the default) the cookie 'httponly' value will not be set. - If True, the cookie 'httponly' value will be set (to 1). - - """ - # Set response cookie - cookie = cherrypy.serving.response.cookie - cookie[name] = cherrypy.serving.session.id - cookie[name]['path'] = ( - path or - cherrypy.serving.request.headers.get(path_header) or - '/' - ) - - # We'd like to use the "max-age" param as indicated in - # http://www.faqs.org/rfcs/rfc2109.html but IE doesn't - # save it to disk and the session is lost if people close - # the browser. So we have to use the old "expires" ... sigh ... -## cookie[name]['max-age'] = timeout * 60 - if timeout: - e = time.time() + (timeout * 60) - cookie[name]['expires'] = httputil.HTTPDate(e) - if domain is not None: - cookie[name]['domain'] = domain - if secure: - cookie[name]['secure'] = 1 - if httponly: - if not cookie[name].isReservedKey('httponly'): - raise ValueError("The httponly cookie token is not supported.") - cookie[name]['httponly'] = 1 - - -def expire(): - """Expire the current session cookie.""" - name = cherrypy.serving.request.config.get( - 'tools.sessions.name', 'session_id') - one_year = 60 * 60 * 24 * 365 - e = time.time() - one_year - cherrypy.serving.response.cookie[name]['expires'] = httputil.HTTPDate(e) diff --git a/lib/cherrypy/lib/static.py b/lib/cherrypy/lib/static.py deleted file mode 100644 index 6a78fc131ea..00000000000 --- a/lib/cherrypy/lib/static.py +++ /dev/null @@ -1,381 +0,0 @@ -import os -import re -import stat -import mimetypes - -try: - from io import UnsupportedOperation -except ImportError: - UnsupportedOperation = object() - -import cherrypy -from cherrypy._cpcompat import ntob, unquote -from cherrypy.lib import cptools, httputil, file_generator_limited - - -mimetypes.init() -mimetypes.types_map['.dwg'] = 'image/x-dwg' -mimetypes.types_map['.ico'] = 'image/x-icon' -mimetypes.types_map['.bz2'] = 'application/x-bzip2' -mimetypes.types_map['.gz'] = 'application/x-gzip' - - -def serve_file(path, content_type=None, disposition=None, name=None, - debug=False): - """Set status, headers, and body in order to serve the given path. - - The Content-Type header will be set to the content_type arg, if provided. - If not provided, the Content-Type will be guessed by the file extension - of the 'path' argument. - - If disposition is not None, the Content-Disposition header will be set - to "; filename=". If name is None, it will be set - to the basename of path. If disposition is None, no Content-Disposition - header will be written. - """ - - response = cherrypy.serving.response - - # If path is relative, users should fix it by making path absolute. - # That is, CherryPy should not guess where the application root is. - # It certainly should *not* use cwd (since CP may be invoked from a - # variety of paths). If using tools.staticdir, you can make your relative - # paths become absolute by supplying a value for "tools.staticdir.root". - if not os.path.isabs(path): - msg = "'%s' is not an absolute path." % path - if debug: - cherrypy.log(msg, 'TOOLS.STATICFILE') - raise ValueError(msg) - - try: - st = os.stat(path) - except (OSError, TypeError, ValueError): - # OSError when file fails to stat - # TypeError on Python 2 when there's a null byte - # ValueError on Python 3 when there's a null byte - if debug: - cherrypy.log('os.stat(%r) failed' % path, 'TOOLS.STATIC') - raise cherrypy.NotFound() - - # Check if path is a directory. - if stat.S_ISDIR(st.st_mode): - # Let the caller deal with it as they like. - if debug: - cherrypy.log('%r is a directory' % path, 'TOOLS.STATIC') - raise cherrypy.NotFound() - - # Set the Last-Modified response header, so that - # modified-since validation code can work. - response.headers['Last-Modified'] = httputil.HTTPDate(st.st_mtime) - cptools.validate_since() - - if content_type is None: - # Set content-type based on filename extension - ext = "" - i = path.rfind('.') - if i != -1: - ext = path[i:].lower() - content_type = mimetypes.types_map.get(ext, None) - if content_type is not None: - response.headers['Content-Type'] = content_type - if debug: - cherrypy.log('Content-Type: %r' % content_type, 'TOOLS.STATIC') - - cd = None - if disposition is not None: - if name is None: - name = os.path.basename(path) - cd = '%s; filename="%s"' % (disposition, name) - response.headers["Content-Disposition"] = cd - if debug: - cherrypy.log('Content-Disposition: %r' % cd, 'TOOLS.STATIC') - - # Set Content-Length and use an iterable (file object) - # this way CP won't load the whole file in memory - content_length = st.st_size - fileobj = open(path, 'rb') - return _serve_fileobj(fileobj, content_type, content_length, debug=debug) - - -def serve_fileobj(fileobj, content_type=None, disposition=None, name=None, - debug=False): - """Set status, headers, and body in order to serve the given file object. - - The Content-Type header will be set to the content_type arg, if provided. - - If disposition is not None, the Content-Disposition header will be set - to "; filename=". If name is None, 'filename' will - not be set. If disposition is None, no Content-Disposition header will - be written. - - CAUTION: If the request contains a 'Range' header, one or more seek()s will - be performed on the file object. This may cause undesired behavior if - the file object is not seekable. It could also produce undesired results - if the caller set the read position of the file object prior to calling - serve_fileobj(), expecting that the data would be served starting from that - position. - """ - - response = cherrypy.serving.response - - try: - st = os.fstat(fileobj.fileno()) - except AttributeError: - if debug: - cherrypy.log('os has no fstat attribute', 'TOOLS.STATIC') - content_length = None - except UnsupportedOperation: - content_length = None - else: - # Set the Last-Modified response header, so that - # modified-since validation code can work. - response.headers['Last-Modified'] = httputil.HTTPDate(st.st_mtime) - cptools.validate_since() - content_length = st.st_size - - if content_type is not None: - response.headers['Content-Type'] = content_type - if debug: - cherrypy.log('Content-Type: %r' % content_type, 'TOOLS.STATIC') - - cd = None - if disposition is not None: - if name is None: - cd = disposition - else: - cd = '%s; filename="%s"' % (disposition, name) - response.headers["Content-Disposition"] = cd - if debug: - cherrypy.log('Content-Disposition: %r' % cd, 'TOOLS.STATIC') - - return _serve_fileobj(fileobj, content_type, content_length, debug=debug) - - -def _serve_fileobj(fileobj, content_type, content_length, debug=False): - """Internal. Set response.body to the given file object, perhaps ranged.""" - response = cherrypy.serving.response - - # HTTP/1.0 didn't have Range/Accept-Ranges headers, or the 206 code - request = cherrypy.serving.request - if request.protocol >= (1, 1): - response.headers["Accept-Ranges"] = "bytes" - r = httputil.get_ranges(request.headers.get('Range'), content_length) - if r == []: - response.headers['Content-Range'] = "bytes */%s" % content_length - message = ("Invalid Range (first-byte-pos greater than " - "Content-Length)") - if debug: - cherrypy.log(message, 'TOOLS.STATIC') - raise cherrypy.HTTPError(416, message) - - if r: - if len(r) == 1: - # Return a single-part response. - start, stop = r[0] - if stop > content_length: - stop = content_length - r_len = stop - start - if debug: - cherrypy.log( - 'Single part; start: %r, stop: %r' % (start, stop), - 'TOOLS.STATIC') - response.status = "206 Partial Content" - response.headers['Content-Range'] = ( - "bytes %s-%s/%s" % (start, stop - 1, content_length)) - response.headers['Content-Length'] = r_len - fileobj.seek(start) - response.body = file_generator_limited(fileobj, r_len) - else: - # Return a multipart/byteranges response. - response.status = "206 Partial Content" - try: - # Python 3 - from email.generator import _make_boundary as make_boundary - except ImportError: - # Python 2 - from mimetools import choose_boundary as make_boundary - boundary = make_boundary() - ct = "multipart/byteranges; boundary=%s" % boundary - response.headers['Content-Type'] = ct - if "Content-Length" in response.headers: - # Delete Content-Length header so finalize() recalcs it. - del response.headers["Content-Length"] - - def file_ranges(): - # Apache compatibility: - yield ntob("\r\n") - - for start, stop in r: - if debug: - cherrypy.log( - 'Multipart; start: %r, stop: %r' % ( - start, stop), - 'TOOLS.STATIC') - yield ntob("--" + boundary, 'ascii') - yield ntob("\r\nContent-type: %s" % content_type, - 'ascii') - yield ntob( - "\r\nContent-range: bytes %s-%s/%s\r\n\r\n" % ( - start, stop - 1, content_length), - 'ascii') - fileobj.seek(start) - gen = file_generator_limited(fileobj, stop - start) - for chunk in gen: - yield chunk - yield ntob("\r\n") - # Final boundary - yield ntob("--" + boundary + "--", 'ascii') - - # Apache compatibility: - yield ntob("\r\n") - response.body = file_ranges() - return response.body - else: - if debug: - cherrypy.log('No byteranges requested', 'TOOLS.STATIC') - - # Set Content-Length and use an iterable (file object) - # this way CP won't load the whole file in memory - response.headers['Content-Length'] = content_length - response.body = fileobj - return response.body - - -def serve_download(path, name=None): - """Serve 'path' as an application/x-download attachment.""" - # This is such a common idiom I felt it deserved its own wrapper. - return serve_file(path, "application/x-download", "attachment", name) - - -def _attempt(filename, content_types, debug=False): - if debug: - cherrypy.log('Attempting %r (content_types %r)' % - (filename, content_types), 'TOOLS.STATICDIR') - try: - # you can set the content types for a - # complete directory per extension - content_type = None - if content_types: - r, ext = os.path.splitext(filename) - content_type = content_types.get(ext[1:], None) - serve_file(filename, content_type=content_type, debug=debug) - return True - except cherrypy.NotFound: - # If we didn't find the static file, continue handling the - # request. We might find a dynamic handler instead. - if debug: - cherrypy.log('NotFound', 'TOOLS.STATICFILE') - return False - - -def staticdir(section, dir, root="", match="", content_types=None, index="", - debug=False): - """Serve a static resource from the given (root +) dir. - - match - If given, request.path_info will be searched for the given - regular expression before attempting to serve static content. - - content_types - If given, it should be a Python dictionary of - {file-extension: content-type} pairs, where 'file-extension' is - a string (e.g. "gif") and 'content-type' is the value to write - out in the Content-Type response header (e.g. "image/gif"). - - index - If provided, it should be the (relative) name of a file to - serve for directory requests. For example, if the dir argument is - '/home/me', the Request-URI is 'myapp', and the index arg is - 'index.html', the file '/home/me/myapp/index.html' will be sought. - """ - request = cherrypy.serving.request - if request.method not in ('GET', 'HEAD'): - if debug: - cherrypy.log('request.method not GET or HEAD', 'TOOLS.STATICDIR') - return False - - if match and not re.search(match, request.path_info): - if debug: - cherrypy.log('request.path_info %r does not match pattern %r' % - (request.path_info, match), 'TOOLS.STATICDIR') - return False - - # Allow the use of '~' to refer to a user's home directory. - dir = os.path.expanduser(dir) - - # If dir is relative, make absolute using "root". - if not os.path.isabs(dir): - if not root: - msg = "Static dir requires an absolute dir (or root)." - if debug: - cherrypy.log(msg, 'TOOLS.STATICDIR') - raise ValueError(msg) - dir = os.path.join(root, dir) - - # Determine where we are in the object tree relative to 'section' - # (where the static tool was defined). - if section == 'global': - section = "/" - section = section.rstrip(r"\/") - branch = request.path_info[len(section) + 1:] - branch = unquote(branch.lstrip(r"\/")) - - # If branch is "", filename will end in a slash - filename = os.path.join(dir, branch) - if debug: - cherrypy.log('Checking file %r to fulfill %r' % - (filename, request.path_info), 'TOOLS.STATICDIR') - - # There's a chance that the branch pulled from the URL might - # have ".." or similar uplevel attacks in it. Check that the final - # filename is a child of dir. - if not os.path.normpath(filename).startswith(os.path.normpath(dir)): - raise cherrypy.HTTPError(403) # Forbidden - - handled = _attempt(filename, content_types) - if not handled: - # Check for an index file if a folder was requested. - if index: - handled = _attempt(os.path.join(filename, index), content_types) - if handled: - request.is_index = filename[-1] in (r"\/") - return handled - - -def staticfile(filename, root=None, match="", content_types=None, debug=False): - """Serve a static resource from the given (root +) filename. - - match - If given, request.path_info will be searched for the given - regular expression before attempting to serve static content. - - content_types - If given, it should be a Python dictionary of - {file-extension: content-type} pairs, where 'file-extension' is - a string (e.g. "gif") and 'content-type' is the value to write - out in the Content-Type response header (e.g. "image/gif"). - - """ - request = cherrypy.serving.request - if request.method not in ('GET', 'HEAD'): - if debug: - cherrypy.log('request.method not GET or HEAD', 'TOOLS.STATICFILE') - return False - - if match and not re.search(match, request.path_info): - if debug: - cherrypy.log('request.path_info %r does not match pattern %r' % - (request.path_info, match), 'TOOLS.STATICFILE') - return False - - # If filename is relative, make absolute using "root". - if not os.path.isabs(filename): - if not root: - msg = "Static tool requires an absolute filename (got '%s')." % ( - filename,) - if debug: - cherrypy.log(msg, 'TOOLS.STATICFILE') - raise ValueError(msg) - filename = os.path.join(root, filename) - - return _attempt(filename, content_types, debug=debug) diff --git a/lib/cherrypy/lib/xmlrpcutil.py b/lib/cherrypy/lib/xmlrpcutil.py deleted file mode 100644 index 9fc9564f6fc..00000000000 --- a/lib/cherrypy/lib/xmlrpcutil.py +++ /dev/null @@ -1,57 +0,0 @@ -import sys - -import cherrypy -from cherrypy._cpcompat import ntob - - -def get_xmlrpclib(): - try: - import xmlrpc.client as x - except ImportError: - import xmlrpclib as x - return x - - -def process_body(): - """Return (params, method) from request body.""" - try: - return get_xmlrpclib().loads(cherrypy.request.body.read()) - except Exception: - return ('ERROR PARAMS', ), 'ERRORMETHOD' - - -def patched_path(path): - """Return 'path', doctored for RPC.""" - if not path.endswith('/'): - path += '/' - if path.startswith('/RPC2/'): - # strip the first /rpc2 - path = path[5:] - return path - - -def _set_response(body): - # The XML-RPC spec (http://www.xmlrpc.com/spec) says: - # "Unless there's a lower-level error, always return 200 OK." - # Since Python's xmlrpclib interprets a non-200 response - # as a "Protocol Error", we'll just return 200 every time. - response = cherrypy.response - response.status = '200 OK' - response.body = ntob(body, 'utf-8') - response.headers['Content-Type'] = 'text/xml' - response.headers['Content-Length'] = len(body) - - -def respond(body, encoding='utf-8', allow_none=0): - xmlrpclib = get_xmlrpclib() - if not isinstance(body, xmlrpclib.Fault): - body = (body,) - _set_response(xmlrpclib.dumps(body, methodresponse=1, - encoding=encoding, - allow_none=allow_none)) - - -def on_error(*args, **kwargs): - body = str(sys.exc_info()[1]) - xmlrpclib = get_xmlrpclib() - _set_response(xmlrpclib.dumps(xmlrpclib.Fault(1, body))) diff --git a/lib/cherrypy/process/__init__.py b/lib/cherrypy/process/__init__.py deleted file mode 100644 index f15b12370aa..00000000000 --- a/lib/cherrypy/process/__init__.py +++ /dev/null @@ -1,14 +0,0 @@ -"""Site container for an HTTP server. - -A Web Site Process Bus object is used to connect applications, servers, -and frameworks with site-wide services such as daemonization, process -reload, signal handling, drop privileges, PID file management, logging -for all of these, and many more. - -The 'plugins' module defines a few abstract and concrete services for -use with the bus. Some use tool-specific channels; see the documentation -for each class. -""" - -from cherrypy.process.wspbus import bus -from cherrypy.process import plugins, servers diff --git a/lib/cherrypy/process/plugins.py b/lib/cherrypy/process/plugins.py deleted file mode 100644 index 0ec585c0a6e..00000000000 --- a/lib/cherrypy/process/plugins.py +++ /dev/null @@ -1,740 +0,0 @@ -"""Site services for use with a Web Site Process Bus.""" - -import os -import re -import signal as _signal -import sys -import time -import threading - -from cherrypy._cpcompat import basestring, get_daemon, get_thread_ident -from cherrypy._cpcompat import ntob, Timer, SetDaemonProperty - -# _module__file__base is used by Autoreload to make -# absolute any filenames retrieved from sys.modules which are not -# already absolute paths. This is to work around Python's quirk -# of importing the startup script and using a relative filename -# for it in sys.modules. -# -# Autoreload examines sys.modules afresh every time it runs. If an application -# changes the current directory by executing os.chdir(), then the next time -# Autoreload runs, it will not be able to find any filenames which are -# not absolute paths, because the current directory is not the same as when the -# module was first imported. Autoreload will then wrongly conclude the file -# has "changed", and initiate the shutdown/re-exec sequence. -# See ticket #917. -# For this workaround to have a decent probability of success, this module -# needs to be imported as early as possible, before the app has much chance -# to change the working directory. -_module__file__base = os.getcwd() - - -class SimplePlugin(object): - - """Plugin base class which auto-subscribes methods for known channels.""" - - bus = None - """A :class:`Bus `, usually cherrypy.engine. - """ - - def __init__(self, bus): - self.bus = bus - - def subscribe(self): - """Register this object as a (multi-channel) listener on the bus.""" - for channel in self.bus.listeners: - # Subscribe self.start, self.exit, etc. if present. - method = getattr(self, channel, None) - if method is not None: - self.bus.subscribe(channel, method) - - def unsubscribe(self): - """Unregister this object as a listener on the bus.""" - for channel in self.bus.listeners: - # Unsubscribe self.start, self.exit, etc. if present. - method = getattr(self, channel, None) - if method is not None: - self.bus.unsubscribe(channel, method) - - -class SignalHandler(object): - - """Register bus channels (and listeners) for system signals. - - You can modify what signals your application listens for, and what it does - when it receives signals, by modifying :attr:`SignalHandler.handlers`, - a dict of {signal name: callback} pairs. The default set is:: - - handlers = {'SIGTERM': self.bus.exit, - 'SIGHUP': self.handle_SIGHUP, - 'SIGUSR1': self.bus.graceful, - } - - The :func:`SignalHandler.handle_SIGHUP`` method calls - :func:`bus.restart()` - if the process is daemonized, but - :func:`bus.exit()` - if the process is attached to a TTY. This is because Unix window - managers tend to send SIGHUP to terminal windows when the user closes them. - - Feel free to add signals which are not available on every platform. - The :class:`SignalHandler` will ignore errors raised from attempting - to register handlers for unknown signals. - """ - - handlers = {} - """A map from signal names (e.g. 'SIGTERM') to handlers (e.g. bus.exit).""" - - signals = {} - """A map from signal numbers to names.""" - - for k, v in vars(_signal).items(): - if k.startswith('SIG') and not k.startswith('SIG_'): - signals[v] = k - del k, v - - def __init__(self, bus): - self.bus = bus - # Set default handlers - self.handlers = {'SIGTERM': self.bus.exit, - 'SIGHUP': self.handle_SIGHUP, - 'SIGUSR1': self.bus.graceful, - } - - if sys.platform[:4] == 'java': - del self.handlers['SIGUSR1'] - self.handlers['SIGUSR2'] = self.bus.graceful - self.bus.log("SIGUSR1 cannot be set on the JVM platform. " - "Using SIGUSR2 instead.") - self.handlers['SIGINT'] = self._jython_SIGINT_handler - - self._previous_handlers = {} - # used to determine is the process is a daemon in `self._is_daemonized` - self._original_pid = os.getpid() - - - def _jython_SIGINT_handler(self, signum=None, frame=None): - # See http://bugs.jython.org/issue1313 - self.bus.log('Keyboard Interrupt: shutting down bus') - self.bus.exit() - - def _is_daemonized(self): - """Return boolean indicating if the current process is - running as a daemon. - - The criteria to determine the `daemon` condition is to verify - if the current pid is not the same as the one that got used on - the initial construction of the plugin *and* the stdin is not - connected to a terminal. - - The sole validation of the tty is not enough when the plugin - is executing inside other process like in a CI tool - (Buildbot, Jenkins). - """ - if (self._original_pid != os.getpid() and - not os.isatty(sys.stdin.fileno())): - return True - else: - return False - - - def subscribe(self): - """Subscribe self.handlers to signals.""" - for sig, func in self.handlers.items(): - try: - self.set_handler(sig, func) - except ValueError: - pass - - def unsubscribe(self): - """Unsubscribe self.handlers from signals.""" - for signum, handler in self._previous_handlers.items(): - signame = self.signals[signum] - - if handler is None: - self.bus.log("Restoring %s handler to SIG_DFL." % signame) - handler = _signal.SIG_DFL - else: - self.bus.log("Restoring %s handler %r." % (signame, handler)) - - try: - our_handler = _signal.signal(signum, handler) - if our_handler is None: - self.bus.log("Restored old %s handler %r, but our " - "handler was not registered." % - (signame, handler), level=30) - except ValueError: - self.bus.log("Unable to restore %s handler %r." % - (signame, handler), level=40, traceback=True) - - def set_handler(self, signal, listener=None): - """Subscribe a handler for the given signal (number or name). - - If the optional 'listener' argument is provided, it will be - subscribed as a listener for the given signal's channel. - - If the given signal name or number is not available on the current - platform, ValueError is raised. - """ - if isinstance(signal, basestring): - signum = getattr(_signal, signal, None) - if signum is None: - raise ValueError("No such signal: %r" % signal) - signame = signal - else: - try: - signame = self.signals[signal] - except KeyError: - raise ValueError("No such signal: %r" % signal) - signum = signal - - prev = _signal.signal(signum, self._handle_signal) - self._previous_handlers[signum] = prev - - if listener is not None: - self.bus.log("Listening for %s." % signame) - self.bus.subscribe(signame, listener) - - def _handle_signal(self, signum=None, frame=None): - """Python signal handler (self.set_handler subscribes it for you).""" - signame = self.signals[signum] - self.bus.log("Caught signal %s." % signame) - self.bus.publish(signame) - - def handle_SIGHUP(self): - """Restart if daemonized, else exit.""" - if self._is_daemonized(): - self.bus.log("SIGHUP caught while daemonized. Restarting.") - self.bus.restart() - else: - # not daemonized (may be foreground or background) - self.bus.log("SIGHUP caught but not daemonized. Exiting.") - self.bus.exit() - - -try: - import pwd - import grp -except ImportError: - pwd, grp = None, None - - -class DropPrivileges(SimplePlugin): - - """Drop privileges. uid/gid arguments not available on Windows. - - Special thanks to `Gavin Baker `_ - """ - - def __init__(self, bus, umask=None, uid=None, gid=None): - SimplePlugin.__init__(self, bus) - self.finalized = False - self.uid = uid - self.gid = gid - self.umask = umask - - def _get_uid(self): - return self._uid - - def _set_uid(self, val): - if val is not None: - if pwd is None: - self.bus.log("pwd module not available; ignoring uid.", - level=30) - val = None - elif isinstance(val, basestring): - val = pwd.getpwnam(val)[2] - self._uid = val - uid = property(_get_uid, _set_uid, - doc="The uid under which to run. Availability: Unix.") - - def _get_gid(self): - return self._gid - - def _set_gid(self, val): - if val is not None: - if grp is None: - self.bus.log("grp module not available; ignoring gid.", - level=30) - val = None - elif isinstance(val, basestring): - val = grp.getgrnam(val)[2] - self._gid = val - gid = property(_get_gid, _set_gid, - doc="The gid under which to run. Availability: Unix.") - - def _get_umask(self): - return self._umask - - def _set_umask(self, val): - if val is not None: - try: - os.umask - except AttributeError: - self.bus.log("umask function not available; ignoring umask.", - level=30) - val = None - self._umask = val - umask = property( - _get_umask, - _set_umask, - doc="""The default permission mode for newly created files and - directories. - - Usually expressed in octal format, for example, ``0644``. - Availability: Unix, Windows. - """) - - def start(self): - # uid/gid - def current_ids(): - """Return the current (uid, gid) if available.""" - name, group = None, None - if pwd: - name = pwd.getpwuid(os.getuid())[0] - if grp: - group = grp.getgrgid(os.getgid())[0] - return name, group - - if self.finalized: - if not (self.uid is None and self.gid is None): - self.bus.log('Already running as uid: %r gid: %r' % - current_ids()) - else: - if self.uid is None and self.gid is None: - if pwd or grp: - self.bus.log('uid/gid not set', level=30) - else: - self.bus.log('Started as uid: %r gid: %r' % current_ids()) - if self.gid is not None: - os.setgid(self.gid) - os.setgroups([]) - if self.uid is not None: - os.setuid(self.uid) - self.bus.log('Running as uid: %r gid: %r' % current_ids()) - - # umask - if self.finalized: - if self.umask is not None: - self.bus.log('umask already set to: %03o' % self.umask) - else: - if self.umask is None: - self.bus.log('umask not set', level=30) - else: - old_umask = os.umask(self.umask) - self.bus.log('umask old: %03o, new: %03o' % - (old_umask, self.umask)) - - self.finalized = True - # This is slightly higher than the priority for server.start - # in order to facilitate the most common use: starting on a low - # port (which requires root) and then dropping to another user. - start.priority = 77 - - -class Daemonizer(SimplePlugin): - - """Daemonize the running script. - - Use this with a Web Site Process Bus via:: - - Daemonizer(bus).subscribe() - - When this component finishes, the process is completely decoupled from - the parent environment. Please note that when this component is used, - the return code from the parent process will still be 0 if a startup - error occurs in the forked children. Errors in the initial daemonizing - process still return proper exit codes. Therefore, if you use this - plugin to daemonize, don't use the return code as an accurate indicator - of whether the process fully started. In fact, that return code only - indicates if the process succesfully finished the first fork. - """ - - def __init__(self, bus, stdin='/dev/null', stdout='/dev/null', - stderr='/dev/null'): - SimplePlugin.__init__(self, bus) - self.stdin = stdin - self.stdout = stdout - self.stderr = stderr - self.finalized = False - - def start(self): - if self.finalized: - self.bus.log('Already deamonized.') - - # forking has issues with threads: - # http://www.opengroup.org/onlinepubs/000095399/functions/fork.html - # "The general problem with making fork() work in a multi-threaded - # world is what to do with all of the threads..." - # So we check for active threads: - if threading.activeCount() != 1: - self.bus.log('There are %r active threads. ' - 'Daemonizing now may cause strange failures.' % - threading.enumerate(), level=30) - - # See http://www.erlenstar.demon.co.uk/unix/faq_2.html#SEC16 - # (or http://www.faqs.org/faqs/unix-faq/programmer/faq/ section 1.7) - # and http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/66012 - - # Finish up with the current stdout/stderr - sys.stdout.flush() - sys.stderr.flush() - - # Do first fork. - try: - pid = os.fork() - if pid == 0: - # This is the child process. Continue. - pass - else: - # This is the first parent. Exit, now that we've forked. - self.bus.log('Forking once.') - os._exit(0) - except OSError: - # Python raises OSError rather than returning negative numbers. - exc = sys.exc_info()[1] - sys.exit("%s: fork #1 failed: (%d) %s\n" - % (sys.argv[0], exc.errno, exc.strerror)) - - os.setsid() - - # Do second fork - try: - pid = os.fork() - if pid > 0: - self.bus.log('Forking twice.') - os._exit(0) # Exit second parent - except OSError: - exc = sys.exc_info()[1] - sys.exit("%s: fork #2 failed: (%d) %s\n" - % (sys.argv[0], exc.errno, exc.strerror)) - - os.chdir("/") - os.umask(0) - - si = open(self.stdin, "r") - so = open(self.stdout, "a+") - se = open(self.stderr, "a+") - - # os.dup2(fd, fd2) will close fd2 if necessary, - # so we don't explicitly close stdin/out/err. - # See http://docs.python.org/lib/os-fd-ops.html - os.dup2(si.fileno(), sys.stdin.fileno()) - os.dup2(so.fileno(), sys.stdout.fileno()) - os.dup2(se.fileno(), sys.stderr.fileno()) - - self.bus.log('Daemonized to PID: %s' % os.getpid()) - self.finalized = True - start.priority = 65 - - -class PIDFile(SimplePlugin): - - """Maintain a PID file via a WSPBus.""" - - def __init__(self, bus, pidfile): - SimplePlugin.__init__(self, bus) - self.pidfile = pidfile - self.finalized = False - - def start(self): - pid = os.getpid() - if self.finalized: - self.bus.log('PID %r already written to %r.' % (pid, self.pidfile)) - else: - open(self.pidfile, "wb").write(ntob("%s\n" % pid, 'utf8')) - self.bus.log('PID %r written to %r.' % (pid, self.pidfile)) - self.finalized = True - start.priority = 70 - - def exit(self): - try: - os.remove(self.pidfile) - self.bus.log('PID file removed: %r.' % self.pidfile) - except (KeyboardInterrupt, SystemExit): - raise - except: - pass - - -class PerpetualTimer(Timer): - - """A responsive subclass of threading.Timer whose run() method repeats. - - Use this timer only when you really need a very interruptible timer; - this checks its 'finished' condition up to 20 times a second, which can - results in pretty high CPU usage - """ - - def __init__(self, *args, **kwargs): - "Override parent constructor to allow 'bus' to be provided." - self.bus = kwargs.pop('bus', None) - super(PerpetualTimer, self).__init__(*args, **kwargs) - - def run(self): - while True: - self.finished.wait(self.interval) - if self.finished.isSet(): - return - try: - self.function(*self.args, **self.kwargs) - except Exception: - if self.bus: - self.bus.log( - "Error in perpetual timer thread function %r." % - self.function, level=40, traceback=True) - # Quit on first error to avoid massive logs. - raise - - -class BackgroundTask(SetDaemonProperty, threading.Thread): - - """A subclass of threading.Thread whose run() method repeats. - - Use this class for most repeating tasks. It uses time.sleep() to wait - for each interval, which isn't very responsive; that is, even if you call - self.cancel(), you'll have to wait until the sleep() call finishes before - the thread stops. To compensate, it defaults to being daemonic, which means - it won't delay stopping the whole process. - """ - - def __init__(self, interval, function, args=[], kwargs={}, bus=None): - threading.Thread.__init__(self) - self.interval = interval - self.function = function - self.args = args - self.kwargs = kwargs - self.running = False - self.bus = bus - - # default to daemonic - self.daemon = True - - def cancel(self): - self.running = False - - def run(self): - self.running = True - while self.running: - time.sleep(self.interval) - if not self.running: - return - try: - self.function(*self.args, **self.kwargs) - except Exception: - if self.bus: - self.bus.log("Error in background task thread function %r." - % self.function, level=40, traceback=True) - # Quit on first error to avoid massive logs. - raise - - -class Monitor(SimplePlugin): - - """WSPBus listener to periodically run a callback in its own thread.""" - - callback = None - """The function to call at intervals.""" - - frequency = 60 - """The time in seconds between callback runs.""" - - thread = None - """A :class:`BackgroundTask` - thread. - """ - - def __init__(self, bus, callback, frequency=60, name=None): - SimplePlugin.__init__(self, bus) - self.callback = callback - self.frequency = frequency - self.thread = None - self.name = name - - def start(self): - """Start our callback in its own background thread.""" - if self.frequency > 0: - threadname = self.name or self.__class__.__name__ - if self.thread is None: - self.thread = BackgroundTask(self.frequency, self.callback, - bus=self.bus) - self.thread.setName(threadname) - self.thread.start() - self.bus.log("Started monitor thread %r." % threadname) - else: - self.bus.log("Monitor thread %r already started." % threadname) - start.priority = 70 - - def stop(self): - """Stop our callback's background task thread.""" - if self.thread is None: - self.bus.log("No thread running for %s." % - self.name or self.__class__.__name__) - else: - if self.thread is not threading.currentThread(): - name = self.thread.getName() - self.thread.cancel() - if not get_daemon(self.thread): - self.bus.log("Joining %r" % name) - self.thread.join() - self.bus.log("Stopped thread %r." % name) - self.thread = None - - def graceful(self): - """Stop the callback's background task thread and restart it.""" - self.stop() - self.start() - - -class Autoreloader(Monitor): - - """Monitor which re-executes the process when files change. - - This :ref:`plugin` restarts the process (via :func:`os.execv`) - if any of the files it monitors change (or is deleted). By default, the - autoreloader monitors all imported modules; you can add to the - set by adding to ``autoreload.files``:: - - cherrypy.engine.autoreload.files.add(myFile) - - If there are imported files you do *not* wish to monitor, you can - adjust the ``match`` attribute, a regular expression. For example, - to stop monitoring cherrypy itself:: - - cherrypy.engine.autoreload.match = r'^(?!cherrypy).+' - - Like all :class:`Monitor` plugins, - the autoreload plugin takes a ``frequency`` argument. The default is - 1 second; that is, the autoreloader will examine files once each second. - """ - - files = None - """The set of files to poll for modifications.""" - - frequency = 1 - """The interval in seconds at which to poll for modified files.""" - - match = '.*' - """A regular expression by which to match filenames.""" - - def __init__(self, bus, frequency=1, match='.*'): - self.mtimes = {} - self.files = set() - self.match = match - Monitor.__init__(self, bus, self.run, frequency) - - def start(self): - """Start our own background task thread for self.run.""" - if self.thread is None: - self.mtimes = {} - Monitor.start(self) - start.priority = 70 - - def sysfiles(self): - """Return a Set of sys.modules filenames to monitor.""" - files = set() - for k, m in list(sys.modules.items()): - if re.match(self.match, k): - if ( - hasattr(m, '__loader__') and - hasattr(m.__loader__, 'archive') - ): - f = m.__loader__.archive - else: - f = getattr(m, '__file__', None) - if f is not None and not os.path.isabs(f): - # ensure absolute paths so a os.chdir() in the app - # doesn't break me - f = os.path.normpath( - os.path.join(_module__file__base, f)) - files.add(f) - return files - - def run(self): - """Reload the process if registered files have been modified.""" - for filename in self.sysfiles() | self.files: - if filename: - if filename.endswith('.pyc'): - filename = filename[:-1] - - oldtime = self.mtimes.get(filename, 0) - if oldtime is None: - # Module with no .py file. Skip it. - continue - - try: - mtime = os.stat(filename).st_mtime - except OSError: - # Either a module with no .py file, or it's been deleted. - mtime = None - - if filename not in self.mtimes: - # If a module has no .py file, this will be None. - self.mtimes[filename] = mtime - else: - if mtime is None or mtime > oldtime: - # The file has been deleted or modified. - self.bus.log("Restarting because %s changed." % - filename) - self.thread.cancel() - self.bus.log("Stopped thread %r." % - self.thread.getName()) - self.bus.restart() - return - - -class ThreadManager(SimplePlugin): - - """Manager for HTTP request threads. - - If you have control over thread creation and destruction, publish to - the 'acquire_thread' and 'release_thread' channels (for each thread). - This will register/unregister the current thread and publish to - 'start_thread' and 'stop_thread' listeners in the bus as needed. - - If threads are created and destroyed by code you do not control - (e.g., Apache), then, at the beginning of every HTTP request, - publish to 'acquire_thread' only. You should not publish to - 'release_thread' in this case, since you do not know whether - the thread will be re-used or not. The bus will call - 'stop_thread' listeners for you when it stops. - """ - - threads = None - """A map of {thread ident: index number} pairs.""" - - def __init__(self, bus): - self.threads = {} - SimplePlugin.__init__(self, bus) - self.bus.listeners.setdefault('acquire_thread', set()) - self.bus.listeners.setdefault('start_thread', set()) - self.bus.listeners.setdefault('release_thread', set()) - self.bus.listeners.setdefault('stop_thread', set()) - - def acquire_thread(self): - """Run 'start_thread' listeners for the current thread. - - If the current thread has already been seen, any 'start_thread' - listeners will not be run again. - """ - thread_ident = get_thread_ident() - if thread_ident not in self.threads: - # We can't just use get_ident as the thread ID - # because some platforms reuse thread ID's. - i = len(self.threads) + 1 - self.threads[thread_ident] = i - self.bus.publish('start_thread', i) - - def release_thread(self): - """Release the current thread and run 'stop_thread' listeners.""" - thread_ident = get_thread_ident() - i = self.threads.pop(thread_ident, None) - if i is not None: - self.bus.publish('stop_thread', i) - - def stop(self): - """Release all threads and run all 'stop_thread' listeners.""" - for thread_ident, i in self.threads.items(): - self.bus.publish('stop_thread', i) - self.threads.clear() - graceful = stop diff --git a/lib/cherrypy/process/servers.py b/lib/cherrypy/process/servers.py deleted file mode 100644 index 7ff383ae036..00000000000 --- a/lib/cherrypy/process/servers.py +++ /dev/null @@ -1,465 +0,0 @@ -""" -Starting in CherryPy 3.1, cherrypy.server is implemented as an -:ref:`Engine Plugin`. It's an instance of -:class:`cherrypy._cpserver.Server`, which is a subclass of -:class:`cherrypy.process.servers.ServerAdapter`. The ``ServerAdapter`` class -is designed to control other servers, as well. - -Multiple servers/ports -====================== - -If you need to start more than one HTTP server (to serve on multiple ports, or -protocols, etc.), you can manually register each one and then start them all -with engine.start:: - - s1 = ServerAdapter(cherrypy.engine, MyWSGIServer(host='0.0.0.0', port=80)) - s2 = ServerAdapter(cherrypy.engine, - another.HTTPServer(host='127.0.0.1', - SSL=True)) - s1.subscribe() - s2.subscribe() - cherrypy.engine.start() - -.. index:: SCGI - -FastCGI/SCGI -============ - -There are also Flup\ **F**\ CGIServer and Flup\ **S**\ CGIServer classes in -:mod:`cherrypy.process.servers`. To start an fcgi server, for example, -wrap an instance of it in a ServerAdapter:: - - addr = ('0.0.0.0', 4000) - f = servers.FlupFCGIServer(application=cherrypy.tree, bindAddress=addr) - s = servers.ServerAdapter(cherrypy.engine, httpserver=f, bind_addr=addr) - s.subscribe() - -The :doc:`cherryd` startup script will do the above for -you via its `-f` flag. -Note that you need to download and install `flup `_ -yourself, whether you use ``cherryd`` or not. - -.. _fastcgi: -.. index:: FastCGI - -FastCGI -------- - -A very simple setup lets your cherry run with FastCGI. -You just need the flup library, -plus a running Apache server (with ``mod_fastcgi``) or lighttpd server. - -CherryPy code -^^^^^^^^^^^^^ - -hello.py:: - - #!/usr/bin/python - import cherrypy - - class HelloWorld: - \"""Sample request handler class.\""" - @cherrypy.expose - def index(self): - return "Hello world!" - - cherrypy.tree.mount(HelloWorld()) - # CherryPy autoreload must be disabled for the flup server to work - cherrypy.config.update({'engine.autoreload.on':False}) - -Then run :doc:`/deployguide/cherryd` with the '-f' arg:: - - cherryd -c -d -f -i hello.py - -Apache -^^^^^^ - -At the top level in httpd.conf:: - - FastCgiIpcDir /tmp - FastCgiServer /path/to/cherry.fcgi -idle-timeout 120 -processes 4 - -And inside the relevant VirtualHost section:: - - # FastCGI config - AddHandler fastcgi-script .fcgi - ScriptAliasMatch (.*$) /path/to/cherry.fcgi$1 - -Lighttpd -^^^^^^^^ - -For `Lighttpd `_ you can follow these -instructions. Within ``lighttpd.conf`` make sure ``mod_fastcgi`` is -active within ``server.modules``. Then, within your ``$HTTP["host"]`` -directive, configure your fastcgi script like the following:: - - $HTTP["url"] =~ "" { - fastcgi.server = ( - "/" => ( - "script.fcgi" => ( - "bin-path" => "/path/to/your/script.fcgi", - "socket" => "/tmp/script.sock", - "check-local" => "disable", - "disable-time" => 1, - "min-procs" => 1, - "max-procs" => 1, # adjust as needed - ), - ), - ) - } # end of $HTTP["url"] =~ "^/" - -Please see `Lighttpd FastCGI Docs -`_ for -an explanation of the possible configuration options. -""" - -import sys -import time -import warnings - - -class ServerAdapter(object): - - """Adapter for an HTTP server. - - If you need to start more than one HTTP server (to serve on multiple - ports, or protocols, etc.), you can manually register each one and then - start them all with bus.start:: - - s1 = ServerAdapter(bus, MyWSGIServer(host='0.0.0.0', port=80)) - s2 = ServerAdapter(bus, another.HTTPServer(host='127.0.0.1', SSL=True)) - s1.subscribe() - s2.subscribe() - bus.start() - """ - - def __init__(self, bus, httpserver=None, bind_addr=None): - self.bus = bus - self.httpserver = httpserver - self.bind_addr = bind_addr - self.interrupt = None - self.running = False - - def subscribe(self): - self.bus.subscribe('start', self.start) - self.bus.subscribe('stop', self.stop) - - def unsubscribe(self): - self.bus.unsubscribe('start', self.start) - self.bus.unsubscribe('stop', self.stop) - - def start(self, num_port_tries=5): - """Start the HTTP server.""" - if self.bind_addr is None: - on_what = "unknown interface (dynamic?)" - elif isinstance(self.bind_addr, tuple): - on_what = self._get_base() - else: - on_what = "socket file: %s" % self.bind_addr - - if self.running: - self.bus.log("Already serving on %s" % on_what) - return - - self.interrupt = None - if not self.httpserver: - raise ValueError("No HTTP server has been created.") - - # Start the httpserver in a new thread. - if isinstance(self.bind_addr, tuple): - wait_for_free_port(*self.bind_addr, num_tries=num_port_tries) - - import threading - t = threading.Thread(target=self._start_http_thread) - t.setName("HTTPServer " + t.getName()) - t.start() - - self.wait() - self.running = True - self.bus.log("Serving on %s" % on_what) - start.priority = 75 - - def _get_base(self): - if not self.httpserver: - return '' - host, port = self.bind_addr - if getattr(self.httpserver, 'ssl_adapter', None): - scheme = "https" - if port != 443: - host += ":%s" % port - else: - scheme = "http" - if port != 80: - host += ":%s" % port - - return "%s://%s" % (scheme, host) - - def _start_http_thread(self): - """HTTP servers MUST be running in new threads, so that the - main thread persists to receive KeyboardInterrupt's. If an - exception is raised in the httpserver's thread then it's - trapped here, and the bus (and therefore our httpserver) - are shut down. - """ - try: - self.httpserver.start() - except KeyboardInterrupt: - self.bus.log(" hit: shutting down HTTP server") - self.interrupt = sys.exc_info()[1] - self.bus.exit() - except SystemExit: - self.bus.log("SystemExit raised: shutting down HTTP server") - self.interrupt = sys.exc_info()[1] - self.bus.exit() - raise - except: - self.interrupt = sys.exc_info()[1] - self.bus.log("Error in HTTP server: shutting down", - traceback=True, level=40) - self.bus.exit() - raise - - def wait(self): - """Wait until the HTTP server is ready to receive requests.""" - while not getattr(self.httpserver, "ready", False): - if self.interrupt: - raise self.interrupt - time.sleep(.1) - - # Wait for port to be occupied - if isinstance(self.bind_addr, tuple): - host, port = self.bind_addr - wait_for_occupied_port(host, port) - - def stop(self): - """Stop the HTTP server.""" - if self.running: - # stop() MUST block until the server is *truly* stopped. - self.httpserver.stop() - # Wait for the socket to be truly freed. - if isinstance(self.bind_addr, tuple): - wait_for_free_port(*self.bind_addr) - self.running = False - self.bus.log("HTTP Server %s shut down" % self.httpserver) - else: - self.bus.log("HTTP Server %s already shut down" % self.httpserver) - stop.priority = 25 - - def restart(self): - """Restart the HTTP server.""" - self.stop() - self.start() - - -class FlupCGIServer(object): - - """Adapter for a flup.server.cgi.WSGIServer.""" - - def __init__(self, *args, **kwargs): - self.args = args - self.kwargs = kwargs - self.ready = False - - def start(self): - """Start the CGI server.""" - # We have to instantiate the server class here because its __init__ - # starts a threadpool. If we do it too early, daemonize won't work. - from flup.server.cgi import WSGIServer - - self.cgiserver = WSGIServer(*self.args, **self.kwargs) - self.ready = True - self.cgiserver.run() - - def stop(self): - """Stop the HTTP server.""" - self.ready = False - - -class FlupFCGIServer(object): - - """Adapter for a flup.server.fcgi.WSGIServer.""" - - def __init__(self, *args, **kwargs): - if kwargs.get('bindAddress', None) is None: - import socket - if not hasattr(socket, 'fromfd'): - raise ValueError( - 'Dynamic FCGI server not available on this platform. ' - 'You must use a static or external one by providing a ' - 'legal bindAddress.') - self.args = args - self.kwargs = kwargs - self.ready = False - - def start(self): - """Start the FCGI server.""" - # We have to instantiate the server class here because its __init__ - # starts a threadpool. If we do it too early, daemonize won't work. - from flup.server.fcgi import WSGIServer - self.fcgiserver = WSGIServer(*self.args, **self.kwargs) - # TODO: report this bug upstream to flup. - # If we don't set _oldSIGs on Windows, we get: - # File "C:\Python24\Lib\site-packages\flup\server\threadedserver.py", - # line 108, in run - # self._restoreSignalHandlers() - # File "C:\Python24\Lib\site-packages\flup\server\threadedserver.py", - # line 156, in _restoreSignalHandlers - # for signum,handler in self._oldSIGs: - # AttributeError: 'WSGIServer' object has no attribute '_oldSIGs' - self.fcgiserver._installSignalHandlers = lambda: None - self.fcgiserver._oldSIGs = [] - self.ready = True - self.fcgiserver.run() - - def stop(self): - """Stop the HTTP server.""" - # Forcibly stop the fcgi server main event loop. - self.fcgiserver._keepGoing = False - # Force all worker threads to die off. - self.fcgiserver._threadPool.maxSpare = ( - self.fcgiserver._threadPool._idleCount) - self.ready = False - - -class FlupSCGIServer(object): - - """Adapter for a flup.server.scgi.WSGIServer.""" - - def __init__(self, *args, **kwargs): - self.args = args - self.kwargs = kwargs - self.ready = False - - def start(self): - """Start the SCGI server.""" - # We have to instantiate the server class here because its __init__ - # starts a threadpool. If we do it too early, daemonize won't work. - from flup.server.scgi import WSGIServer - self.scgiserver = WSGIServer(*self.args, **self.kwargs) - # TODO: report this bug upstream to flup. - # If we don't set _oldSIGs on Windows, we get: - # File "C:\Python24\Lib\site-packages\flup\server\threadedserver.py", - # line 108, in run - # self._restoreSignalHandlers() - # File "C:\Python24\Lib\site-packages\flup\server\threadedserver.py", - # line 156, in _restoreSignalHandlers - # for signum,handler in self._oldSIGs: - # AttributeError: 'WSGIServer' object has no attribute '_oldSIGs' - self.scgiserver._installSignalHandlers = lambda: None - self.scgiserver._oldSIGs = [] - self.ready = True - self.scgiserver.run() - - def stop(self): - """Stop the HTTP server.""" - self.ready = False - # Forcibly stop the scgi server main event loop. - self.scgiserver._keepGoing = False - # Force all worker threads to die off. - self.scgiserver._threadPool.maxSpare = 0 - - -def client_host(server_host): - """Return the host on which a client can connect to the given listener.""" - if server_host == '0.0.0.0': - # 0.0.0.0 is INADDR_ANY, which should answer on localhost. - return '127.0.0.1' - if server_host in ('::', '::0', '::0.0.0.0'): - # :: is IN6ADDR_ANY, which should answer on localhost. - # ::0 and ::0.0.0.0 are non-canonical but common - # ways to write IN6ADDR_ANY. - return '::1' - return server_host - - -def check_port(host, port, timeout=1.0): - """Raise an error if the given port is not free on the given host.""" - if not host: - raise ValueError("Host values of '' or None are not allowed.") - host = client_host(host) - port = int(port) - - import socket - - # AF_INET or AF_INET6 socket - # Get the correct address family for our host (allows IPv6 addresses) - try: - info = socket.getaddrinfo(host, port, socket.AF_UNSPEC, - socket.SOCK_STREAM) - except socket.gaierror: - if ':' in host: - info = [( - socket.AF_INET6, socket.SOCK_STREAM, 0, "", (host, port, 0, 0) - )] - else: - info = [(socket.AF_INET, socket.SOCK_STREAM, 0, "", (host, port))] - - for res in info: - af, socktype, proto, canonname, sa = res - s = None - try: - s = socket.socket(af, socktype, proto) - # See http://groups.google.com/group/cherrypy-users/ - # browse_frm/thread/bbfe5eb39c904fe0 - s.settimeout(timeout) - s.connect((host, port)) - s.close() - except socket.error: - if s: - s.close() - else: - raise IOError("Port %s is in use on %s; perhaps the previous " - "httpserver did not shut down properly." % - (repr(port), repr(host))) - - -# Feel free to increase these defaults on slow systems: -free_port_timeout = 0.1 -occupied_port_timeout = 1.0 - - -def wait_for_free_port(host, port, timeout=None, num_tries=50): - """Wait for the specified port to become free (drop requests).""" - if not host: - raise ValueError("Host values of '' or None are not allowed.") - if timeout is None: - timeout = free_port_timeout - - for trial in range(num_tries): - try: - # we are expecting a free port, so reduce the timeout - check_port(host, port, timeout=timeout) - except IOError: - # Give the old server thread time to free the port. - time.sleep(timeout) - else: - return - - raise IOError("Port %r not free on %r" % (port, host)) - - -def wait_for_occupied_port(host, port, timeout=None): - """Wait for the specified port to become active (receive requests).""" - if not host: - raise ValueError("Host values of '' or None are not allowed.") - if timeout is None: - timeout = occupied_port_timeout - - for trial in range(50): - try: - check_port(host, port, timeout=timeout) - except IOError: - # port is occupied - return - else: - time.sleep(timeout) - - if host == client_host(host): - raise IOError("Port %r not bound on %r" % (port, host)) - - # On systems where a loopback interface is not available and the - # server is bound to all interfaces, it's difficult to determine - # whether the server is in fact occupying the port. In this case, - # just issue a warning and move on. See issue #1100. - msg = "Unable to verify that the server is bound on %r" % port - warnings.warn(msg) diff --git a/lib/cherrypy/process/win32.py b/lib/cherrypy/process/win32.py deleted file mode 100644 index 4afd3f14948..00000000000 --- a/lib/cherrypy/process/win32.py +++ /dev/null @@ -1,180 +0,0 @@ -"""Windows service. Requires pywin32.""" - -import os -import win32api -import win32con -import win32event -import win32service -import win32serviceutil - -from cherrypy.process import wspbus, plugins - - -class ConsoleCtrlHandler(plugins.SimplePlugin): - - """A WSPBus plugin for handling Win32 console events (like Ctrl-C).""" - - def __init__(self, bus): - self.is_set = False - plugins.SimplePlugin.__init__(self, bus) - - def start(self): - if self.is_set: - self.bus.log('Handler for console events already set.', level=40) - return - - result = win32api.SetConsoleCtrlHandler(self.handle, 1) - if result == 0: - self.bus.log('Could not SetConsoleCtrlHandler (error %r)' % - win32api.GetLastError(), level=40) - else: - self.bus.log('Set handler for console events.', level=40) - self.is_set = True - - def stop(self): - if not self.is_set: - self.bus.log('Handler for console events already off.', level=40) - return - - try: - result = win32api.SetConsoleCtrlHandler(self.handle, 0) - except ValueError: - # "ValueError: The object has not been registered" - result = 1 - - if result == 0: - self.bus.log('Could not remove SetConsoleCtrlHandler (error %r)' % - win32api.GetLastError(), level=40) - else: - self.bus.log('Removed handler for console events.', level=40) - self.is_set = False - - def handle(self, event): - """Handle console control events (like Ctrl-C).""" - if event in (win32con.CTRL_C_EVENT, win32con.CTRL_LOGOFF_EVENT, - win32con.CTRL_BREAK_EVENT, win32con.CTRL_SHUTDOWN_EVENT, - win32con.CTRL_CLOSE_EVENT): - self.bus.log('Console event %s: shutting down bus' % event) - - # Remove self immediately so repeated Ctrl-C doesn't re-call it. - try: - self.stop() - except ValueError: - pass - - self.bus.exit() - # 'First to return True stops the calls' - return 1 - return 0 - - -class Win32Bus(wspbus.Bus): - - """A Web Site Process Bus implementation for Win32. - - Instead of time.sleep, this bus blocks using native win32event objects. - """ - - def __init__(self): - self.events = {} - wspbus.Bus.__init__(self) - - def _get_state_event(self, state): - """Return a win32event for the given state (creating it if needed).""" - try: - return self.events[state] - except KeyError: - event = win32event.CreateEvent(None, 0, 0, - "WSPBus %s Event (pid=%r)" % - (state.name, os.getpid())) - self.events[state] = event - return event - - def _get_state(self): - return self._state - - def _set_state(self, value): - self._state = value - event = self._get_state_event(value) - win32event.PulseEvent(event) - state = property(_get_state, _set_state) - - def wait(self, state, interval=0.1, channel=None): - """Wait for the given state(s), KeyboardInterrupt or SystemExit. - - Since this class uses native win32event objects, the interval - argument is ignored. - """ - if isinstance(state, (tuple, list)): - # Don't wait for an event that beat us to the punch ;) - if self.state not in state: - events = tuple([self._get_state_event(s) for s in state]) - win32event.WaitForMultipleObjects( - events, 0, win32event.INFINITE) - else: - # Don't wait for an event that beat us to the punch ;) - if self.state != state: - event = self._get_state_event(state) - win32event.WaitForSingleObject(event, win32event.INFINITE) - - -class _ControlCodes(dict): - - """Control codes used to "signal" a service via ControlService. - - User-defined control codes are in the range 128-255. We generally use - the standard Python value for the Linux signal and add 128. Example: - - >>> signal.SIGUSR1 - 10 - control_codes['graceful'] = 128 + 10 - """ - - def key_for(self, obj): - """For the given value, return its corresponding key.""" - for key, val in self.items(): - if val is obj: - return key - raise ValueError("The given object could not be found: %r" % obj) - -control_codes = _ControlCodes({'graceful': 138}) - - -def signal_child(service, command): - if command == 'stop': - win32serviceutil.StopService(service) - elif command == 'restart': - win32serviceutil.RestartService(service) - else: - win32serviceutil.ControlService(service, control_codes[command]) - - -class PyWebService(win32serviceutil.ServiceFramework): - - """Python Web Service.""" - - _svc_name_ = "Python Web Service" - _svc_display_name_ = "Python Web Service" - _svc_deps_ = None # sequence of service names on which this depends - _exe_name_ = "pywebsvc" - _exe_args_ = None # Default to no arguments - - # Only exists on Windows 2000 or later, ignored on windows NT - _svc_description_ = "Python Web Service" - - def SvcDoRun(self): - from cherrypy import process - process.bus.start() - process.bus.block() - - def SvcStop(self): - from cherrypy import process - self.ReportServiceStatus(win32service.SERVICE_STOP_PENDING) - process.bus.exit() - - def SvcOther(self, control): - process.bus.publish(control_codes.key_for(control)) - - -if __name__ == '__main__': - win32serviceutil.HandleCommandLine(PyWebService) diff --git a/lib/cherrypy/process/wspbus.py b/lib/cherrypy/process/wspbus.py deleted file mode 100644 index 20b6175d1e3..00000000000 --- a/lib/cherrypy/process/wspbus.py +++ /dev/null @@ -1,450 +0,0 @@ -"""An implementation of the Web Site Process Bus. - -This module is completely standalone, depending only on the stdlib. - -Web Site Process Bus --------------------- - -A Bus object is used to contain and manage site-wide behavior: -daemonization, HTTP server start/stop, process reload, signal handling, -drop privileges, PID file management, logging for all of these, -and many more. - -In addition, a Bus object provides a place for each web framework -to register code that runs in response to site-wide events (like -process start and stop), or which controls or otherwise interacts with -the site-wide components mentioned above. For example, a framework which -uses file-based templates would add known template filenames to an -autoreload component. - -Ideally, a Bus object will be flexible enough to be useful in a variety -of invocation scenarios: - - 1. The deployer starts a site from the command line via a - framework-neutral deployment script; applications from multiple frameworks - are mixed in a single site. Command-line arguments and configuration - files are used to define site-wide components such as the HTTP server, - WSGI component graph, autoreload behavior, signal handling, etc. - 2. The deployer starts a site via some other process, such as Apache; - applications from multiple frameworks are mixed in a single site. - Autoreload and signal handling (from Python at least) are disabled. - 3. The deployer starts a site via a framework-specific mechanism; - for example, when running tests, exploring tutorials, or deploying - single applications from a single framework. The framework controls - which site-wide components are enabled as it sees fit. - -The Bus object in this package uses topic-based publish-subscribe -messaging to accomplish all this. A few topic channels are built in -('start', 'stop', 'exit', 'graceful', 'log', and 'main'). Frameworks and -site containers are free to define their own. If a message is sent to a -channel that has not been defined or has no listeners, there is no effect. - -In general, there should only ever be a single Bus object per process. -Frameworks and site containers share a single Bus object by publishing -messages and subscribing listeners. - -The Bus object works as a finite state machine which models the current -state of the process. Bus methods move it from one state to another; -those methods then publish to subscribed listeners on the channel for -the new state.:: - - O - | - V - STOPPING --> STOPPED --> EXITING -> X - A A | - | \___ | - | \ | - | V V - STARTED <-- STARTING - -""" - -import atexit -import os -import sys -import threading -import time -import traceback as _traceback -import warnings -import operator - -# Here I save the value of os.getcwd(), which, if I am imported early enough, -# will be the directory from which the startup script was run. This is needed -# by _do_execv(), to change back to the original directory before execv()ing a -# new process. This is a defense against the application having changed the -# current working directory (which could make sys.executable "not found" if -# sys.executable is a relative-path, and/or cause other problems). -_startup_cwd = os.getcwd() - - -class ChannelFailures(Exception): - - """Exception raised when errors occur in a listener during Bus.publish(). - """ - delimiter = '\n' - - def __init__(self, *args, **kwargs): - # Don't use 'super' here; Exceptions are old-style in Py2.4 - # See https://github.com/cherrypy/cherrypy/issues/959 - Exception.__init__(self, *args, **kwargs) - self._exceptions = list() - - def handle_exception(self): - """Append the current exception to self.""" - self._exceptions.append(sys.exc_info()[1]) - - def get_instances(self): - """Return a list of seen exception instances.""" - return self._exceptions[:] - - def __str__(self): - exception_strings = map(repr, self.get_instances()) - return self.delimiter.join(exception_strings) - - __repr__ = __str__ - - def __bool__(self): - return bool(self._exceptions) - __nonzero__ = __bool__ - -# Use a flag to indicate the state of the bus. - - -class _StateEnum(object): - - class State(object): - name = None - - def __repr__(self): - return "states.%s" % self.name - - def __setattr__(self, key, value): - if isinstance(value, self.State): - value.name = key - object.__setattr__(self, key, value) -states = _StateEnum() -states.STOPPED = states.State() -states.STARTING = states.State() -states.STARTED = states.State() -states.STOPPING = states.State() -states.EXITING = states.State() - - -try: - import fcntl -except ImportError: - max_files = 0 -else: - try: - max_files = os.sysconf('SC_OPEN_MAX') - except AttributeError: - max_files = 1024 - - -class Bus(object): - - """Process state-machine and messenger for HTTP site deployment. - - All listeners for a given channel are guaranteed to be called even - if others at the same channel fail. Each failure is logged, but - execution proceeds on to the next listener. The only way to stop all - processing from inside a listener is to raise SystemExit and stop the - whole server. - """ - - states = states - state = states.STOPPED - execv = False - max_cloexec_files = max_files - - def __init__(self): - self.execv = False - self.state = states.STOPPED - channels = 'start', 'stop', 'exit', 'graceful', 'log', 'main' - self.listeners = dict( - (channel, set()) - for channel in channels - ) - self._priorities = {} - - def subscribe(self, channel, callback, priority=None): - """Add the given callback at the given channel (if not present).""" - if channel not in self.listeners: - self.listeners[channel] = set() - self.listeners[channel].add(callback) - - if priority is None: - priority = getattr(callback, 'priority', 50) - self._priorities[(channel, callback)] = priority - - def unsubscribe(self, channel, callback): - """Discard the given callback (if present).""" - listeners = self.listeners.get(channel) - if listeners and callback in listeners: - listeners.discard(callback) - del self._priorities[(channel, callback)] - - def publish(self, channel, *args, **kwargs): - """Return output of all subscribers for the given channel.""" - if channel not in self.listeners: - return [] - - exc = ChannelFailures() - output = [] - - raw_items = ( - (self._priorities[(channel, listener)], listener) - for listener in self.listeners[channel] - ) - items = sorted(raw_items, key=operator.itemgetter(0)) - for priority, listener in items: - try: - output.append(listener(*args, **kwargs)) - except KeyboardInterrupt: - raise - except SystemExit: - e = sys.exc_info()[1] - # If we have previous errors ensure the exit code is non-zero - if exc and e.code == 0: - e.code = 1 - raise - except: - exc.handle_exception() - if channel == 'log': - # Assume any further messages to 'log' will fail. - pass - else: - self.log("Error in %r listener %r" % (channel, listener), - level=40, traceback=True) - if exc: - raise exc - return output - - def _clean_exit(self): - """An atexit handler which asserts the Bus is not running.""" - if self.state != states.EXITING: - warnings.warn( - "The main thread is exiting, but the Bus is in the %r state; " - "shutting it down automatically now. You must either call " - "bus.block() after start(), or call bus.exit() before the " - "main thread exits." % self.state, RuntimeWarning) - self.exit() - - def start(self): - """Start all services.""" - atexit.register(self._clean_exit) - - self.state = states.STARTING - self.log('Bus STARTING') - try: - self.publish('start') - self.state = states.STARTED - self.log('Bus STARTED') - except (KeyboardInterrupt, SystemExit): - raise - except: - self.log("Shutting down due to error in start listener:", - level=40, traceback=True) - e_info = sys.exc_info()[1] - try: - self.exit() - except: - # Any stop/exit errors will be logged inside publish(). - pass - # Re-raise the original error - raise e_info - - def exit(self): - """Stop all services and prepare to exit the process.""" - exitstate = self.state - try: - self.stop() - - self.state = states.EXITING - self.log('Bus EXITING') - self.publish('exit') - # This isn't strictly necessary, but it's better than seeing - # "Waiting for child threads to terminate..." and then nothing. - self.log('Bus EXITED') - except: - # This method is often called asynchronously (whether thread, - # signal handler, console handler, or atexit handler), so we - # can't just let exceptions propagate out unhandled. - # Assume it's been logged and just die. - os._exit(70) # EX_SOFTWARE - - if exitstate == states.STARTING: - # exit() was called before start() finished, possibly due to - # Ctrl-C because a start listener got stuck. In this case, - # we could get stuck in a loop where Ctrl-C never exits the - # process, so we just call os.exit here. - # Cylc modification note: we want to be able to repeatedly try to - # launch and fail on a range of ports, so we can't afford to die - # here. - #os._exit(70) # EX_SOFTWARE - pass - - def restart(self): - """Restart the process (may close connections). - - This method does not restart the process from the calling thread; - instead, it stops the bus and asks the main thread to call execv. - """ - self.execv = True - self.exit() - - def graceful(self): - """Advise all services to reload.""" - self.log('Bus graceful') - self.publish('graceful') - - def block(self, interval=0.1): - """Wait for the EXITING state, KeyboardInterrupt or SystemExit. - - This function is intended to be called only by the main thread. - After waiting for the EXITING state, it also waits for all threads - to terminate, and then calls os.execv if self.execv is True. This - design allows another thread to call bus.restart, yet have the main - thread perform the actual execv call (required on some platforms). - """ - try: - self.wait(states.EXITING, interval=interval, channel='main') - except (KeyboardInterrupt, IOError): - # The time.sleep call might raise - # "IOError: [Errno 4] Interrupted function call" on KBInt. - self.log('Keyboard Interrupt: shutting down bus') - self.exit() - except SystemExit: - self.log('SystemExit raised: shutting down bus') - self.exit() - raise - - # Waiting for ALL child threads to finish is necessary on OS X. - # See https://github.com/cherrypy/cherrypy/issues/581. - # It's also good to let them all shut down before allowing - # the main thread to call atexit handlers. - # See https://github.com/cherrypy/cherrypy/issues/751. - self.log("Waiting for child threads to terminate...") - for t in threading.enumerate(): - # Validate the we're not trying to join the MainThread - # that will cause a deadlock and the case exist when - # implemented as a windows service and in any other case - # that another thread executes cherrypy.engine.exit() - if ( - t != threading.currentThread() and - t.isAlive() and - not isinstance(t, threading._MainThread) - ): - # Note that any dummy (external) threads are always daemonic. - if hasattr(threading.Thread, "daemon"): - # Python 2.6+ - d = t.daemon - else: - d = t.isDaemon() - if not d: - self.log("Waiting for thread %s." % t.getName()) - t.join() - - if self.execv: - self._do_execv() - - def wait(self, state, interval=0.1, channel=None): - """Poll for the given state(s) at intervals; publish to channel.""" - if isinstance(state, (tuple, list)): - states = state - else: - states = [state] - - def _wait(): - while self.state not in states: - time.sleep(interval) - self.publish(channel) - - # From http://psyco.sourceforge.net/psycoguide/bugs.html: - # "The compiled machine code does not include the regular polling - # done by Python, meaning that a KeyboardInterrupt will not be - # detected before execution comes back to the regular Python - # interpreter. Your program cannot be interrupted if caught - # into an infinite Psyco-compiled loop." - try: - sys.modules['psyco'].cannotcompile(_wait) - except (KeyError, AttributeError): - pass - - _wait() - - def _do_execv(self): - """Re-execute the current process. - - This must be called from the main thread, because certain platforms - (OS X) don't allow execv to be called in a child thread very well. - """ - args = sys.argv[:] - self.log('Re-spawning %s' % ' '.join(args)) - - if sys.platform[:4] == 'java': - from _systemrestart import SystemRestart - raise SystemRestart - else: - args.insert(0, sys.executable) - if sys.platform == 'win32': - args = ['"%s"' % arg for arg in args] - - os.chdir(_startup_cwd) - if self.max_cloexec_files: - self._set_cloexec() - os.execv(sys.executable, args) - - def _set_cloexec(self): - """Set the CLOEXEC flag on all open files (except stdin/out/err). - - If self.max_cloexec_files is an integer (the default), then on - platforms which support it, it represents the max open files setting - for the operating system. This function will be called just before - the process is restarted via os.execv() to prevent open files - from persisting into the new process. - - Set self.max_cloexec_files to 0 to disable this behavior. - """ - for fd in range(3, self.max_cloexec_files): # skip stdin/out/err - try: - flags = fcntl.fcntl(fd, fcntl.F_GETFD) - except IOError: - continue - fcntl.fcntl(fd, fcntl.F_SETFD, flags | fcntl.FD_CLOEXEC) - - def stop(self): - """Stop all services.""" - self.state = states.STOPPING - self.log('Bus STOPPING') - self.publish('stop') - self.state = states.STOPPED - self.log('Bus STOPPED') - - def start_with_callback(self, func, args=None, kwargs=None): - """Start 'func' in a new thread T, then start self (and return T).""" - if args is None: - args = () - if kwargs is None: - kwargs = {} - args = (func,) + args - - def _callback(func, *a, **kw): - self.wait(states.STARTED) - func(*a, **kw) - t = threading.Thread(target=_callback, args=args, kwargs=kwargs) - t.setName('Bus Callback ' + t.getName()) - t.start() - - self.start() - - return t - - def log(self, msg="", level=20, traceback=False): - """Log the given message. Append the last traceback if requested.""" - if traceback: - msg += "\n" + "".join(_traceback.format_exception(*sys.exc_info())) - self.publish('log', msg, level) - -bus = Bus() diff --git a/lib/cherrypy/scaffold/__init__.py b/lib/cherrypy/scaffold/__init__.py deleted file mode 100644 index 665d8664b65..00000000000 --- a/lib/cherrypy/scaffold/__init__.py +++ /dev/null @@ -1,59 +0,0 @@ -""", a CherryPy application. - -Use this as a base for creating new CherryPy applications. When you want -to make a new app, copy and paste this folder to some other location -(maybe site-packages) and rename it to the name of your project, -then tweak as desired. - -Even before any tweaking, this should serve a few demonstration pages. -Change to this directory and run: - - ../cherryd -c site.conf - -""" - -import cherrypy -from cherrypy import tools, url - -import os -local_dir = os.path.join(os.getcwd(), os.path.dirname(__file__)) - - -@cherrypy.config(**{'tools.log_tracebacks.on': True}) -class Root: - - @cherrypy.expose - def index(self): - return """ -Try some other path, -or a default path.
-Or, just look at the pretty picture:
- -""" % (url("other"), url("else"), - url("files/made_with_cherrypy_small.png")) - - @cherrypy.expose - def default(self, *args, **kwargs): - return "args: %s kwargs: %s" % (args, kwargs) - - @cherrypy.expose - def other(self, a=2, b='bananas', c=None): - cherrypy.response.headers['Content-Type'] = 'text/plain' - if c is None: - return "Have %d %s." % (int(a), b) - else: - return "Have %d %s, %s." % (int(a), b, c) - - files = cherrypy.tools.staticdir.handler( - section="/files", - dir=os.path.join(local_dir, "static"), - # Ignore .php files, etc. - match=r'\.(css|gif|html?|ico|jpe?g|js|png|swf|xml)$', - ) - - -root = Root() - -# Uncomment the following to use your own favicon instead of CP's default. -#favicon_path = os.path.join(local_dir, "favicon.ico") -#root.favicon_ico = tools.staticfile.handler(filename=favicon_path) diff --git a/lib/cherrypy/scaffold/apache-fcgi.conf b/lib/cherrypy/scaffold/apache-fcgi.conf deleted file mode 100644 index 922398eaf80..00000000000 --- a/lib/cherrypy/scaffold/apache-fcgi.conf +++ /dev/null @@ -1,22 +0,0 @@ -# Apache2 server conf file for using CherryPy with mod_fcgid. - -# This doesn't have to be "C:/", but it has to be a directory somewhere, and -# MUST match the directory used in the FastCgiExternalServer directive, below. -DocumentRoot "C:/" - -ServerName 127.0.0.1 -Listen 80 -LoadModule fastcgi_module modules/mod_fastcgi.dll -LoadModule rewrite_module modules/mod_rewrite.so - -Options ExecCGI -SetHandler fastcgi-script -RewriteEngine On -# Send requests for any URI to our fastcgi handler. -RewriteRule ^(.*)$ /fastcgi.pyc [L] - -# The FastCgiExternalServer directive defines filename as an external FastCGI application. -# If filename does not begin with a slash (/) then it is assumed to be relative to the ServerRoot. -# The filename does not have to exist in the local filesystem. URIs that Apache resolves to this -# filename will be handled by this external FastCGI application. -FastCgiExternalServer "C:/fastcgi.pyc" -host 127.0.0.1:8088 \ No newline at end of file diff --git a/lib/cherrypy/scaffold/example.conf b/lib/cherrypy/scaffold/example.conf deleted file mode 100644 index 93a6e53c052..00000000000 --- a/lib/cherrypy/scaffold/example.conf +++ /dev/null @@ -1,3 +0,0 @@ -[/] -log.error_file: "error.log" -log.access_file: "access.log" \ No newline at end of file diff --git a/lib/cherrypy/scaffold/site.conf b/lib/cherrypy/scaffold/site.conf deleted file mode 100644 index 6ed38983738..00000000000 --- a/lib/cherrypy/scaffold/site.conf +++ /dev/null @@ -1,14 +0,0 @@ -[global] -# Uncomment this when you're done developing -#environment: "production" - -server.socket_host: "0.0.0.0" -server.socket_port: 8088 - -# Uncomment the following lines to run on HTTPS at the same time -#server.2.socket_host: "0.0.0.0" -#server.2.socket_port: 8433 -#server.2.ssl_certificate: '../test/test.pem' -#server.2.ssl_private_key: '../test/test.pem' - -tree.myapp: cherrypy.Application(scaffold.root, "/", "example.conf") diff --git a/lib/cherrypy/scaffold/static/made_with_cherrypy_small.png b/lib/cherrypy/scaffold/static/made_with_cherrypy_small.png deleted file mode 100644 index c3aafeed952..00000000000 Binary files a/lib/cherrypy/scaffold/static/made_with_cherrypy_small.png and /dev/null differ diff --git a/lib/cherrypy/test/__init__.py b/lib/cherrypy/test/__init__.py deleted file mode 100644 index 0927c170a48..00000000000 --- a/lib/cherrypy/test/__init__.py +++ /dev/null @@ -1,30 +0,0 @@ -"""Regression test suite for CherryPy. - -Run 'nosetests -s test/' to exercise all tests. - -The '-s' flag instructs nose to output stdout messages, wihch is crucial to -the 'interactive' mode of webtest.py. If you run these tests without the '-s' -flag, don't be surprised if the test seems to hang: it's waiting for your -interactive input. -""" - -import os -import sys - - -def newexit(): - os._exit(1) - - -def setup(): - # We want to monkey patch sys.exit so that we can get some - # information about where exit is being called. - newexit._old = sys.exit - sys.exit = newexit - - -def teardown(): - try: - sys.exit = sys.exit._old - except AttributeError: - sys.exit = sys._exit diff --git a/lib/cherrypy/test/_test_decorators.py b/lib/cherrypy/test/_test_decorators.py deleted file mode 100644 index 06fbf0cb01a..00000000000 --- a/lib/cherrypy/test/_test_decorators.py +++ /dev/null @@ -1,40 +0,0 @@ -"""Test module for the @-decorator syntax, which is version-specific""" - -import cherrypy -from cherrypy import expose, tools -from cherrypy._cpcompat import ntob - - -class ExposeExamples(object): - - @expose - def no_call(self): - return "Mr E. R. Bradshaw" - - @expose() - def call_empty(self): - return "Mrs. B.J. Smegma" - - @expose("call_alias") - def nesbitt(self): - return "Mr Nesbitt" - - @expose(["alias1", "alias2"]) - def andrews(self): - return "Mr Ken Andrews" - - @expose(alias="alias3") - def watson(self): - return "Mr. and Mrs. Watson" - - -class ToolExamples(object): - - @expose - # This is here to demonstrate that using the config decorator - # does not overwrite other config attributes added by the Tool - # decorator (in this case response_headers). - @cherrypy.config(**{'response.stream': True}) - @tools.response_headers(headers=[('Content-Type', 'application/data')]) - def blah(self): - yield ntob("blah") diff --git a/lib/cherrypy/test/_test_states_demo.py b/lib/cherrypy/test/_test_states_demo.py deleted file mode 100644 index 75f9405ceb2..00000000000 --- a/lib/cherrypy/test/_test_states_demo.py +++ /dev/null @@ -1,68 +0,0 @@ -import os -import sys -import time -starttime = time.time() - -import cherrypy - - -class Root: - - @cherrypy.expose - def index(self): - return "Hello World" - - @cherrypy.expose - def mtimes(self): - return repr(cherrypy.engine.publish("Autoreloader", "mtimes")) - - @cherrypy.expose - def pid(self): - return str(os.getpid()) - - @cherrypy.expose - def start(self): - return repr(starttime) - - @cherrypy.expose - def exit(self): - # This handler might be called before the engine is STARTED if an - # HTTP worker thread handles it before the HTTP server returns - # control to engine.start. We avoid that race condition here - # by waiting for the Bus to be STARTED. - cherrypy.engine.wait(state=cherrypy.engine.states.STARTED) - cherrypy.engine.exit() - - -def unsub_sig(): - cherrypy.log("unsubsig: %s" % cherrypy.config.get('unsubsig', False)) - if cherrypy.config.get('unsubsig', False): - cherrypy.log("Unsubscribing the default cherrypy signal handler") - cherrypy.engine.signal_handler.unsubscribe() - try: - from signal import signal, SIGTERM - except ImportError: - pass - else: - def old_term_handler(signum=None, frame=None): - cherrypy.log("I am an old SIGTERM handler.") - sys.exit(0) - cherrypy.log("Subscribing the new one.") - signal(SIGTERM, old_term_handler) -cherrypy.engine.subscribe('start', unsub_sig, priority=100) - - -def starterror(): - if cherrypy.config.get('starterror', False): - zerodiv = 1 / 0 -cherrypy.engine.subscribe('start', starterror, priority=6) - - -def log_test_case_name(): - if cherrypy.config.get('test_case_name', False): - cherrypy.log("STARTED FROM: %s" % - cherrypy.config.get('test_case_name')) -cherrypy.engine.subscribe('start', log_test_case_name, priority=6) - - -cherrypy.tree.mount(Root(), '/', {'/': {}}) diff --git a/lib/cherrypy/test/benchmark.py b/lib/cherrypy/test/benchmark.py deleted file mode 100644 index 674f696f3ab..00000000000 --- a/lib/cherrypy/test/benchmark.py +++ /dev/null @@ -1,422 +0,0 @@ -"""CherryPy Benchmark Tool - - Usage: - benchmark.py [options] - - --null: use a null Request object (to bench the HTTP server only) - --notests: start the server but do not run the tests; this allows - you to check the tested pages with a browser - --help: show this help message - --cpmodpy: run tests via apache on 54583 (with the builtin _cpmodpy) - --modpython: run tests via apache on 54583 (with modpython_gateway) - --ab=path: Use the ab script/executable at 'path' (see below) - --apache=path: Use the apache script/exe at 'path' (see below) - - To run the benchmarks, the Apache Benchmark tool "ab" must either be on - your system path, or specified via the --ab=path option. - - To run the modpython tests, the "apache" executable or script must be - on your system path, or provided via the --apache=path option. On some - platforms, "apache" may be called "apachectl" or "apache2ctl"--create - a symlink to them if needed. -""" - -import getopt -import os -curdir = os.path.join(os.getcwd(), os.path.dirname(__file__)) - -import re -import sys -import time -import traceback - -import cherrypy -from cherrypy._cpcompat import ntob -from cherrypy import _cperror, _cpmodpy -from cherrypy.lib import httputil - - -AB_PATH = "" -APACHE_PATH = "apache" -SCRIPT_NAME = "/cpbench/users/rdelon/apps/blog" - -__all__ = ['ABSession', 'Root', 'print_report', - 'run_standard_benchmarks', 'safe_threads', - 'size_report', 'startup', 'thread_report', - ] - -size_cache = {} - - -class Root: - - @cherrypy.expose - def index(self): - return """ - - CherryPy Benchmark - - - - -""" - - @cherrypy.expose - def hello(self): - return "Hello, world\r\n" - - @cherrypy.expose - def sizer(self, size): - resp = size_cache.get(size, None) - if resp is None: - size_cache[size] = resp = "X" * int(size) - return resp - - -cherrypy.config.update({ - 'log.error.file': '', - 'environment': 'production', - 'server.socket_host': '127.0.0.1', - 'server.socket_port': 54583, - 'server.max_request_header_size': 0, - 'server.max_request_body_size': 0, - 'engine.deadlock_poll_freq': 0, -}) - -# Cheat mode on ;) -del cherrypy.config['tools.log_tracebacks.on'] -del cherrypy.config['tools.log_headers.on'] -del cherrypy.config['tools.trailing_slash.on'] - -appconf = { - '/static': { - 'tools.staticdir.on': True, - 'tools.staticdir.dir': 'static', - 'tools.staticdir.root': curdir, - }, -} -app = cherrypy.tree.mount(Root(), SCRIPT_NAME, appconf) - - -class NullRequest: - - """A null HTTP request class, returning 200 and an empty body.""" - - def __init__(self, local, remote, scheme="http"): - pass - - def close(self): - pass - - def run(self, method, path, query_string, protocol, headers, rfile): - cherrypy.response.status = "200 OK" - cherrypy.response.header_list = [("Content-Type", 'text/html'), - ("Server", "Null CherryPy"), - ("Date", httputil.HTTPDate()), - ("Content-Length", "0"), - ] - cherrypy.response.body = [""] - return cherrypy.response - - -class NullResponse: - pass - - -class ABSession: - - """A session of 'ab', the Apache HTTP server benchmarking tool. - -Example output from ab: - -This is ApacheBench, Version 2.0.40-dev <$Revision: 1.121.2.1 $> apache-2.0 -Copyright (c) 1996 Adam Twiss, Zeus Technology Ltd, http://www.zeustech.net/ -Copyright (c) 1998-2002 The Apache Software Foundation, http://www.apache.org/ - -Benchmarking 127.0.0.1 (be patient) -Completed 100 requests -Completed 200 requests -Completed 300 requests -Completed 400 requests -Completed 500 requests -Completed 600 requests -Completed 700 requests -Completed 800 requests -Completed 900 requests - - -Server Software: CherryPy/3.1beta -Server Hostname: 127.0.0.1 -Server Port: 54583 - -Document Path: /static/index.html -Document Length: 14 bytes - -Concurrency Level: 10 -Time taken for tests: 9.643867 seconds -Complete requests: 1000 -Failed requests: 0 -Write errors: 0 -Total transferred: 189000 bytes -HTML transferred: 14000 bytes -Requests per second: 103.69 [#/sec] (mean) -Time per request: 96.439 [ms] (mean) -Time per request: 9.644 [ms] (mean, across all concurrent requests) -Transfer rate: 19.08 [Kbytes/sec] received - -Connection Times (ms) - min mean[+/-sd] median max -Connect: 0 0 2.9 0 10 -Processing: 20 94 7.3 90 130 -Waiting: 0 43 28.1 40 100 -Total: 20 95 7.3 100 130 - -Percentage of the requests served within a certain time (ms) - 50% 100 - 66% 100 - 75% 100 - 80% 100 - 90% 100 - 95% 100 - 98% 100 - 99% 110 - 100% 130 (longest request) -Finished 1000 requests -""" - - parse_patterns = [ - ('complete_requests', 'Completed', - ntob(r'^Complete requests:\s*(\d+)')), - ('failed_requests', 'Failed', - ntob(r'^Failed requests:\s*(\d+)')), - ('requests_per_second', 'req/sec', - ntob(r'^Requests per second:\s*([0-9.]+)')), - ('time_per_request_concurrent', 'msec/req', - ntob(r'^Time per request:\s*([0-9.]+).*concurrent requests\)$')), - ('transfer_rate', 'KB/sec', - ntob(r'^Transfer rate:\s*([0-9.]+)')) - ] - - def __init__(self, path=SCRIPT_NAME + "/hello", requests=1000, - concurrency=10): - self.path = path - self.requests = requests - self.concurrency = concurrency - - def args(self): - port = cherrypy.server.socket_port - assert self.concurrency > 0 - assert self.requests > 0 - # Don't use "localhost". - # Cf - # http://mail.python.org/pipermail/python-win32/2008-March/007050.html - return ("-k -n %s -c %s http://127.0.0.1:%s%s" % - (self.requests, self.concurrency, port, self.path)) - - def run(self): - # Parse output of ab, setting attributes on self - try: - self.output = _cpmodpy.read_process(AB_PATH or "ab", self.args()) - except: - print(_cperror.format_exc()) - raise - - for attr, name, pattern in self.parse_patterns: - val = re.search(pattern, self.output, re.MULTILINE) - if val: - val = val.group(1) - setattr(self, attr, val) - else: - setattr(self, attr, None) - - -safe_threads = (25, 50, 100, 200, 400) -if sys.platform in ("win32",): - # For some reason, ab crashes with > 50 threads on my Win2k laptop. - safe_threads = (10, 20, 30, 40, 50) - - -def thread_report(path=SCRIPT_NAME + "/hello", concurrency=safe_threads): - sess = ABSession(path) - attrs, names, patterns = list(zip(*sess.parse_patterns)) - avg = dict.fromkeys(attrs, 0.0) - - yield ('threads',) + names - for c in concurrency: - sess.concurrency = c - sess.run() - row = [c] - for attr in attrs: - val = getattr(sess, attr) - if val is None: - print(sess.output) - row = None - break - val = float(val) - avg[attr] += float(val) - row.append(val) - if row: - yield row - - # Add a row of averages. - yield ["Average"] + [str(avg[attr] / len(concurrency)) for attr in attrs] - - -def size_report(sizes=(10, 100, 1000, 10000, 100000, 100000000), - concurrency=50): - sess = ABSession(concurrency=concurrency) - attrs, names, patterns = list(zip(*sess.parse_patterns)) - yield ('bytes',) + names - for sz in sizes: - sess.path = "%s/sizer?size=%s" % (SCRIPT_NAME, sz) - sess.run() - yield [sz] + [getattr(sess, attr) for attr in attrs] - - -def print_report(rows): - for row in rows: - print("") - for i, val in enumerate(row): - sys.stdout.write(str(val).rjust(10) + " | ") - print("") - - -def run_standard_benchmarks(): - print("") - print("Client Thread Report (1000 requests, 14 byte response body, " - "%s server threads):" % cherrypy.server.thread_pool) - print_report(thread_report()) - - print("") - print("Client Thread Report (1000 requests, 14 bytes via staticdir, " - "%s server threads):" % cherrypy.server.thread_pool) - print_report(thread_report("%s/static/index.html" % SCRIPT_NAME)) - - print("") - print("Size Report (1000 requests, 50 client threads, " - "%s server threads):" % cherrypy.server.thread_pool) - print_report(size_report()) - - -# modpython and other WSGI # - -def startup_modpython(req=None): - """Start the CherryPy app server in 'serverless' mode (for modpython/WSGI). - """ - if cherrypy.engine.state == cherrypy._cpengine.STOPPED: - if req: - if "nullreq" in req.get_options(): - cherrypy.engine.request_class = NullRequest - cherrypy.engine.response_class = NullResponse - ab_opt = req.get_options().get("ab", "") - if ab_opt: - global AB_PATH - AB_PATH = ab_opt - cherrypy.engine.start() - if cherrypy.engine.state == cherrypy._cpengine.STARTING: - cherrypy.engine.wait() - return 0 # apache.OK - - -def run_modpython(use_wsgi=False): - print("Starting mod_python...") - pyopts = [] - - # Pass the null and ab=path options through Apache - if "--null" in opts: - pyopts.append(("nullreq", "")) - - if "--ab" in opts: - pyopts.append(("ab", opts["--ab"])) - - s = _cpmodpy.ModPythonServer - if use_wsgi: - pyopts.append(("wsgi.application", "cherrypy::tree")) - pyopts.append( - ("wsgi.startup", "cherrypy.test.benchmark::startup_modpython")) - handler = "modpython_gateway::handler" - s = s(port=54583, opts=pyopts, - apache_path=APACHE_PATH, handler=handler) - else: - pyopts.append( - ("cherrypy.setup", "cherrypy.test.benchmark::startup_modpython")) - s = s(port=54583, opts=pyopts, apache_path=APACHE_PATH) - - try: - s.start() - run() - finally: - s.stop() - - -if __name__ == '__main__': - longopts = ['cpmodpy', 'modpython', 'null', 'notests', - 'help', 'ab=', 'apache='] - try: - switches, args = getopt.getopt(sys.argv[1:], "", longopts) - opts = dict(switches) - except getopt.GetoptError: - print(__doc__) - sys.exit(2) - - if "--help" in opts: - print(__doc__) - sys.exit(0) - - if "--ab" in opts: - AB_PATH = opts['--ab'] - - if "--notests" in opts: - # Return without stopping the server, so that the pages - # can be tested from a standard web browser. - def run(): - port = cherrypy.server.socket_port - print("You may now open http://127.0.0.1:%s%s/" % - (port, SCRIPT_NAME)) - - if "--null" in opts: - print("Using null Request object") - else: - def run(): - end = time.time() - start - print("Started in %s seconds" % end) - if "--null" in opts: - print("\nUsing null Request object") - try: - try: - run_standard_benchmarks() - except: - print(_cperror.format_exc()) - raise - finally: - cherrypy.engine.exit() - - print("Starting CherryPy app server...") - - class NullWriter(object): - - """Suppresses the printing of socket errors.""" - - def write(self, data): - pass - sys.stderr = NullWriter() - - start = time.time() - - if "--cpmodpy" in opts: - run_modpython() - elif "--modpython" in opts: - run_modpython(use_wsgi=True) - else: - if "--null" in opts: - cherrypy.server.request_class = NullRequest - cherrypy.server.response_class = NullResponse - - cherrypy.engine.start_with_callback(run) - cherrypy.engine.block() diff --git a/lib/cherrypy/test/checkerdemo.py b/lib/cherrypy/test/checkerdemo.py deleted file mode 100644 index 68fb222b211..00000000000 --- a/lib/cherrypy/test/checkerdemo.py +++ /dev/null @@ -1,48 +0,0 @@ -"""Demonstration app for cherrypy.checker. - -This application is intentionally broken and badly designed. -To demonstrate the output of the CherryPy Checker, simply execute -this module. -""" - -import os -import cherrypy -thisdir = os.path.dirname(os.path.abspath(__file__)) - - -class Root: - pass - -if __name__ == '__main__': - conf = {'/base': {'tools.staticdir.root': thisdir, - # Obsolete key. - 'throw_errors': True, - }, - # This entry should be OK. - '/base/static': {'tools.staticdir.on': True, - 'tools.staticdir.dir': 'static'}, - # Warn on missing folder. - '/base/js': {'tools.staticdir.on': True, - 'tools.staticdir.dir': 'js'}, - # Warn on dir with an abs path even though we provide root. - '/base/static2': {'tools.staticdir.on': True, - 'tools.staticdir.dir': '/static'}, - # Warn on dir with a relative path with no root. - '/static3': {'tools.staticdir.on': True, - 'tools.staticdir.dir': 'static'}, - # Warn on unknown namespace - '/unknown': {'toobles.gzip.on': True}, - # Warn special on cherrypy..* - '/cpknown': {'cherrypy.tools.encode.on': True}, - # Warn on mismatched types - '/conftype': {'request.show_tracebacks': 14}, - # Warn on unknown tool. - '/web': {'tools.unknown.on': True}, - # Warn on server.* in app config. - '/app1': {'server.socket_host': '0.0.0.0'}, - # Warn on 'localhost' - 'global': {'server.socket_host': 'localhost'}, - # Warn on '[name]' - '[/extra_brackets]': {}, - } - cherrypy.quickstart(Root(), config=conf) diff --git a/lib/cherrypy/test/helper.py b/lib/cherrypy/test/helper.py deleted file mode 100644 index d2d0e2b2147..00000000000 --- a/lib/cherrypy/test/helper.py +++ /dev/null @@ -1,556 +0,0 @@ -"""A library of helper functions for the CherryPy test suite.""" - -import datetime -import logging -log = logging.getLogger(__name__) -import os -thisdir = os.path.abspath(os.path.dirname(__file__)) -serverpem = os.path.join(os.getcwd(), thisdir, 'test.pem') -import unittest - -import re -import sys -import time -import warnings - -import cherrypy -from cherrypy._cpcompat import basestring, copyitems, HTTPSConnection, ntob -from cherrypy.lib import httputil -from cherrypy.lib import gctools -from cherrypy.lib.reprconf import unrepr -from cherrypy.test import webtest - -# Use subprocess module from Python 2.7 on Python 2.3-2.6 -if sys.version_info < (2, 7): - import cherrypy._cpcompat_subprocess as subprocess -else: - import subprocess - -import nose - -_testconfig = None - - -def get_tst_config(overconf={}): - global _testconfig - if _testconfig is None: - conf = { - 'scheme': 'http', - 'protocol': "HTTP/1.1", - 'port': 54583, - 'host': '127.0.0.1', - 'validate': False, - 'conquer': False, - 'server': 'wsgi', - } - try: - import testconfig - _conf = testconfig.config.get('supervisor', None) - if _conf is not None: - for k, v in _conf.items(): - if isinstance(v, basestring): - _conf[k] = unrepr(v) - conf.update(_conf) - except ImportError: - pass - _testconfig = conf - conf = _testconfig.copy() - conf.update(overconf) - - return conf - - -class Supervisor(object): - - """Base class for modeling and controlling servers during testing.""" - - def __init__(self, **kwargs): - for k, v in kwargs.items(): - if k == 'port': - setattr(self, k, int(v)) - setattr(self, k, v) - - -log_to_stderr = lambda msg, level: sys.stderr.write(msg + os.linesep) - - -class LocalSupervisor(Supervisor): - - """Base class for modeling/controlling servers which run in the same - process. - - When the server side runs in a different process, start/stop can dump all - state between each test module easily. When the server side runs in the - same process as the client, however, we have to do a bit more work to - ensure config and mounted apps are reset between tests. - """ - - using_apache = False - using_wsgi = False - - def __init__(self, **kwargs): - for k, v in kwargs.items(): - setattr(self, k, v) - - cherrypy.server.httpserver = self.httpserver_class - - # This is perhaps the wrong place for this call but this is the only - # place that i've found so far that I KNOW is early enough to set this. - cherrypy.config.update({'log.screen': False}) - engine = cherrypy.engine - if hasattr(engine, "signal_handler"): - engine.signal_handler.subscribe() - if hasattr(engine, "console_control_handler"): - engine.console_control_handler.subscribe() - #engine.subscribe('log', log_to_stderr) - - def start(self, modulename=None): - """Load and start the HTTP server.""" - if modulename: - # Unhook httpserver so cherrypy.server.start() creates a new - # one (with config from setup_server, if declared). - cherrypy.server.httpserver = None - - cherrypy.engine.start() - - self.sync_apps() - - def sync_apps(self): - """Tell the server about any apps which the setup functions mounted.""" - pass - - def stop(self): - td = getattr(self, 'teardown', None) - if td: - td() - - cherrypy.engine.exit() - - for name, server in copyitems(getattr(cherrypy, 'servers', {})): - server.unsubscribe() - del cherrypy.servers[name] - - -class NativeServerSupervisor(LocalSupervisor): - - """Server supervisor for the builtin HTTP server.""" - - httpserver_class = "cherrypy._cpnative_server.CPHTTPServer" - using_apache = False - using_wsgi = False - - def __str__(self): - return "Builtin HTTP Server on %s:%s" % (self.host, self.port) - - -class LocalWSGISupervisor(LocalSupervisor): - - """Server supervisor for the builtin WSGI server.""" - - httpserver_class = "cherrypy._cpwsgi_server.CPWSGIServer" - using_apache = False - using_wsgi = True - - def __str__(self): - return "Builtin WSGI Server on %s:%s" % (self.host, self.port) - - def sync_apps(self): - """Hook a new WSGI app into the origin server.""" - cherrypy.server.httpserver.wsgi_app = self.get_app() - - def get_app(self, app=None): - """Obtain a new (decorated) WSGI app to hook into the origin server.""" - if app is None: - app = cherrypy.tree - - if self.conquer: - try: - import wsgiconq - except ImportError: - warnings.warn( - "Error importing wsgiconq. pyconquer will not run.") - else: - app = wsgiconq.WSGILogger(app, c_calls=True) - - if self.validate: - try: - from wsgiref import validate - except ImportError: - warnings.warn( - "Error importing wsgiref. The validator will not run.") - else: - # wraps the app in the validator - app = validate.validator(app) - - return app - - -def get_cpmodpy_supervisor(**options): - from cherrypy.test import modpy - sup = modpy.ModPythonSupervisor(**options) - sup.template = modpy.conf_cpmodpy - return sup - - -def get_modpygw_supervisor(**options): - from cherrypy.test import modpy - sup = modpy.ModPythonSupervisor(**options) - sup.template = modpy.conf_modpython_gateway - sup.using_wsgi = True - return sup - - -def get_modwsgi_supervisor(**options): - from cherrypy.test import modwsgi - return modwsgi.ModWSGISupervisor(**options) - - -def get_modfcgid_supervisor(**options): - from cherrypy.test import modfcgid - return modfcgid.ModFCGISupervisor(**options) - - -def get_modfastcgi_supervisor(**options): - from cherrypy.test import modfastcgi - return modfastcgi.ModFCGISupervisor(**options) - - -def get_wsgi_u_supervisor(**options): - cherrypy.server.wsgi_version = ('u', 0) - return LocalWSGISupervisor(**options) - - -class CPWebCase(webtest.WebCase): - - script_name = "" - scheme = "http" - - available_servers = {'wsgi': LocalWSGISupervisor, - 'wsgi_u': get_wsgi_u_supervisor, - 'native': NativeServerSupervisor, - 'cpmodpy': get_cpmodpy_supervisor, - 'modpygw': get_modpygw_supervisor, - 'modwsgi': get_modwsgi_supervisor, - 'modfcgid': get_modfcgid_supervisor, - 'modfastcgi': get_modfastcgi_supervisor, - } - default_server = "wsgi" - - def _setup_server(cls, supervisor, conf): - v = sys.version.split()[0] - log.info("Python version used to run this test script: %s" % v) - log.info("CherryPy version: %s" % cherrypy.__version__) - if supervisor.scheme == "https": - ssl = " (ssl)" - else: - ssl = "" - log.info("HTTP server version: %s%s" % (supervisor.protocol, ssl)) - log.info("PID: %s" % os.getpid()) - - cherrypy.server.using_apache = supervisor.using_apache - cherrypy.server.using_wsgi = supervisor.using_wsgi - - if sys.platform[:4] == 'java': - cherrypy.config.update({'server.nodelay': False}) - - if isinstance(conf, basestring): - parser = cherrypy.lib.reprconf.Parser() - conf = parser.dict_from_file(conf).get('global', {}) - else: - conf = conf or {} - baseconf = conf.copy() - baseconf.update({'server.socket_host': supervisor.host, - 'server.socket_port': supervisor.port, - 'server.protocol_version': supervisor.protocol, - 'environment': "test_suite", - }) - if supervisor.scheme == "https": - #baseconf['server.ssl_module'] = 'builtin' - baseconf['server.ssl_certificate'] = serverpem - baseconf['server.ssl_private_key'] = serverpem - - # helper must be imported lazily so the coverage tool - # can run against module-level statements within cherrypy. - # Also, we have to do "from cherrypy.test import helper", - # exactly like each test module does, because a relative import - # would stick a second instance of webtest in sys.modules, - # and we wouldn't be able to globally override the port anymore. - if supervisor.scheme == "https": - webtest.WebCase.HTTP_CONN = HTTPSConnection - return baseconf - _setup_server = classmethod(_setup_server) - - def setup_class(cls): - '' - # Creates a server - conf = get_tst_config() - supervisor_factory = cls.available_servers.get( - conf.get('server', 'wsgi')) - if supervisor_factory is None: - raise RuntimeError('Unknown server in config: %s' % conf['server']) - supervisor = supervisor_factory(**conf) - - # Copied from "run_test_suite" - cherrypy.config.reset() - baseconf = cls._setup_server(supervisor, conf) - cherrypy.config.update(baseconf) - setup_client() - - if hasattr(cls, 'setup_server'): - # Clear the cherrypy tree and clear the wsgi server so that - # it can be updated with the new root - cherrypy.tree = cherrypy._cptree.Tree() - cherrypy.server.httpserver = None - cls.setup_server() - # Add a resource for verifying there are no refleaks - # to *every* test class. - cherrypy.tree.mount(gctools.GCRoot(), '/gc') - cls.do_gc_test = True - supervisor.start(cls.__module__) - - cls.supervisor = supervisor - setup_class = classmethod(setup_class) - - def teardown_class(cls): - '' - if hasattr(cls, 'setup_server'): - cls.supervisor.stop() - teardown_class = classmethod(teardown_class) - - do_gc_test = False - - def test_gc(self): - if not self.do_gc_test: - return - - self.getPage("/gc/stats") - try: - self.assertBody("Statistics:") - except Exception: - "Failures occur intermittently. See #1420" - - def prefix(self): - return self.script_name.rstrip("/") - - def base(self): - if ((self.scheme == "http" and self.PORT == 80) or - (self.scheme == "https" and self.PORT == 443)): - port = "" - else: - port = ":%s" % self.PORT - - return "%s://%s%s%s" % (self.scheme, self.HOST, port, - self.script_name.rstrip("/")) - - def exit(self): - sys.exit() - - def getPage(self, url, headers=None, method="GET", body=None, - protocol=None, raise_subcls=None): - """Open the url. Return status, headers, body. - - `raise_subcls` must be a tuple with the exceptions classes - or a single exception class that are not going to be considered - a socket.error regardless that they were are subclass of a - socket.error and therefore not considered for a connection retry. - """ - if self.script_name: - url = httputil.urljoin(self.script_name, url) - return webtest.WebCase.getPage(self, url, headers, method, body, - protocol, raise_subcls) - - def skip(self, msg='skipped '): - raise nose.SkipTest(msg) - - def assertErrorPage(self, status, message=None, pattern=''): - """Compare the response body with a built in error page. - - The function will optionally look for the regexp pattern, - within the exception embedded in the error page.""" - - # This will never contain a traceback - page = cherrypy._cperror.get_error_page(status, message=message) - - # First, test the response body without checking the traceback. - # Stick a match-all group (.*) in to grab the traceback. - def esc(text): - return re.escape(ntob(text)) - epage = re.escape(page) - epage = epage.replace( - esc('
'),
-            esc('
') + ntob('(.*)') + esc('
'))
-        m = re.match(epage, self.body, re.DOTALL)
-        if not m:
-            self._handlewebError(
-                'Error page does not match; expected:\n' + page)
-            return
-
-        # Now test the pattern against the traceback
-        if pattern is None:
-            # Special-case None to mean that there should be *no* traceback.
-            if m and m.group(1):
-                self._handlewebError('Error page contains traceback')
-        else:
-            if (m is None) or (
-                not re.search(ntob(re.escape(pattern), self.encoding),
-                              m.group(1))):
-                msg = 'Error page does not contain %s in traceback'
-                self._handlewebError(msg % repr(pattern))
-
-    date_tolerance = 2
-
-    def assertEqualDates(self, dt1, dt2, seconds=None):
-        """Assert abs(dt1 - dt2) is within Y seconds."""
-        if seconds is None:
-            seconds = self.date_tolerance
-
-        if dt1 > dt2:
-            diff = dt1 - dt2
-        else:
-            diff = dt2 - dt1
-        if not diff < datetime.timedelta(seconds=seconds):
-            raise AssertionError('%r and %r are not within %r seconds.' %
-                                 (dt1, dt2, seconds))
-
-
-def _test_method_sorter(_, x, y):
-    """Monkeypatch the test sorter to always run test_gc last in each suite."""
-    if x == 'test_gc':
-        return 1
-    if y == 'test_gc':
-        return -1
-    if x > y:
-        return 1
-    if x < y:
-        return -1
-    return 0
-unittest.TestLoader.sortTestMethodsUsing = _test_method_sorter
-
-
-def setup_client():
-    """Set up the WebCase classes to match the server's socket settings."""
-    webtest.WebCase.PORT = cherrypy.server.socket_port
-    webtest.WebCase.HOST = cherrypy.server.socket_host
-    if cherrypy.server.ssl_certificate:
-        CPWebCase.scheme = 'https'
-
-# --------------------------- Spawning helpers --------------------------- #
-
-
-class CPProcess(object):
-
-    pid_file = os.path.join(thisdir, 'test.pid')
-    config_file = os.path.join(thisdir, 'test.conf')
-    config_template = """[global]
-server.socket_host: '%(host)s'
-server.socket_port: %(port)s
-checker.on: False
-log.screen: False
-log.error_file: r'%(error_log)s'
-log.access_file: r'%(access_log)s'
-%(ssl)s
-%(extra)s
-"""
-    error_log = os.path.join(thisdir, 'test.error.log')
-    access_log = os.path.join(thisdir, 'test.access.log')
-
-    def __init__(self, wait=False, daemonize=False, ssl=False,
-                 socket_host=None, socket_port=None):
-        self.wait = wait
-        self.daemonize = daemonize
-        self.ssl = ssl
-        self.host = socket_host or cherrypy.server.socket_host
-        self.port = socket_port or cherrypy.server.socket_port
-
-    def write_conf(self, extra=""):
-        if self.ssl:
-            serverpem = os.path.join(thisdir, 'test.pem')
-            ssl = """
-server.ssl_certificate: r'%s'
-server.ssl_private_key: r'%s'
-""" % (serverpem, serverpem)
-        else:
-            ssl = ""
-
-        conf = self.config_template % {
-            'host': self.host,
-            'port': self.port,
-            'error_log': self.error_log,
-            'access_log': self.access_log,
-            'ssl': ssl,
-            'extra': extra,
-        }
-        f = open(self.config_file, 'wb')
-        f.write(ntob(conf, 'utf-8'))
-        f.close()
-
-    def start(self, imports=None):
-        """Start cherryd in a subprocess."""
-        cherrypy._cpserver.wait_for_free_port(self.host, self.port)
-
-        args = [
-            os.path.join(thisdir, '..', 'cherryd'),
-            '-c', self.config_file,
-            '-p', self.pid_file,
-        ]
-
-        if not isinstance(imports, (list, tuple)):
-            imports = [imports]
-        for i in imports:
-            if i:
-                args.append('-i')
-                args.append(i)
-
-        if self.daemonize:
-            args.append('-d')
-
-        env = os.environ.copy()
-        # Make sure we import the cherrypy package in which this module is
-        # defined.
-        grandparentdir = os.path.abspath(os.path.join(thisdir, '..', '..'))
-        if env.get('PYTHONPATH', ''):
-            env['PYTHONPATH'] = os.pathsep.join(
-                (grandparentdir, env['PYTHONPATH']))
-        else:
-            env['PYTHONPATH'] = grandparentdir
-        self._proc = subprocess.Popen([sys.executable] + args, env=env)
-        if self.wait:
-            self.exit_code = self._proc.wait()
-        else:
-            cherrypy._cpserver.wait_for_occupied_port(self.host, self.port)
-
-        # Give the engine a wee bit more time to finish STARTING
-        if self.daemonize:
-            time.sleep(2)
-        else:
-            time.sleep(1)
-
-    def get_pid(self):
-        if self.daemonize:
-            return int(open(self.pid_file, 'rb').read())
-        return self._proc.pid
-
-    def join(self):
-        """Wait for the process to exit."""
-        if self.daemonize:
-            return self._join_daemon()
-        self._proc.wait()
-
-    def _join_daemon(self):
-        try:
-            try:
-                # Mac, UNIX
-                os.wait()
-            except AttributeError:
-                # Windows
-                try:
-                    pid = self.get_pid()
-                except IOError:
-                    # Assume the subprocess deleted the pidfile on shutdown.
-                    pass
-                else:
-                    os.waitpid(pid, 0)
-        except OSError:
-            x = sys.exc_info()[1]
-            if x.args != (10, 'No child processes'):
-                raise
diff --git a/lib/cherrypy/test/logtest.py b/lib/cherrypy/test/logtest.py
deleted file mode 100644
index 53f29f1126f..00000000000
--- a/lib/cherrypy/test/logtest.py
+++ /dev/null
@@ -1,200 +0,0 @@
-"""logtest, a unittest.TestCase helper for testing log output."""
-
-import sys
-import time
-
-import six
-
-from cherrypy._cpcompat import basestring, ntob
-
-
-try:
-    # On Windows, msvcrt.getch reads a single char without output.
-    import msvcrt
-
-    def getchar():
-        return msvcrt.getch()
-except ImportError:
-    # Unix getchr
-    import tty
-    import termios
-
-    def getchar():
-        fd = sys.stdin.fileno()
-        old_settings = termios.tcgetattr(fd)
-        try:
-            tty.setraw(sys.stdin.fileno())
-            ch = sys.stdin.read(1)
-        finally:
-            termios.tcsetattr(fd, termios.TCSADRAIN, old_settings)
-        return ch
-
-
-class LogCase(object):
-
-    """unittest.TestCase mixin for testing log messages.
-
-    logfile: a filename for the desired log. Yes, I know modes are evil,
-        but it makes the test functions so much cleaner to set this once.
-
-    lastmarker: the last marker in the log. This can be used to search for
-        messages since the last marker.
-
-    markerPrefix: a string with which to prefix log markers. This should be
-        unique enough from normal log output to use for marker identification.
-    """
-
-    logfile = None
-    lastmarker = None
-    markerPrefix = ntob("test suite marker: ")
-
-    def _handleLogError(self, msg, data, marker, pattern):
-        print("")
-        print("    ERROR: %s" % msg)
-
-        if not self.interactive:
-            raise self.failureException(msg)
-
-        p = ("    Show: "
-             "[L]og [M]arker [P]attern; "
-             "[I]gnore, [R]aise, or sys.e[X]it >> ")
-        sys.stdout.write(p + ' ')
-        # ARGH
-        sys.stdout.flush()
-        while True:
-            i = getchar().upper()
-            if i not in "MPLIRX":
-                continue
-            print(i.upper())  # Also prints new line
-            if i == "L":
-                for x, line in enumerate(data):
-                    if (x + 1) % self.console_height == 0:
-                        # The \r and comma should make the next line overwrite
-                        sys.stdout.write("<-- More -->\r ")
-                        m = getchar().lower()
-                        # Erase our "More" prompt
-                        sys.stdout.write("            \r ")
-                        if m == "q":
-                            break
-                    print(line.rstrip())
-            elif i == "M":
-                print(repr(marker or self.lastmarker))
-            elif i == "P":
-                print(repr(pattern))
-            elif i == "I":
-                # return without raising the normal exception
-                return
-            elif i == "R":
-                raise self.failureException(msg)
-            elif i == "X":
-                self.exit()
-            sys.stdout.write(p + ' ')
-
-    def exit(self):
-        sys.exit()
-
-    def emptyLog(self):
-        """Overwrite self.logfile with 0 bytes."""
-        open(self.logfile, 'wb').write("")
-
-    def markLog(self, key=None):
-        """Insert a marker line into the log and set self.lastmarker."""
-        if key is None:
-            key = str(time.time())
-        self.lastmarker = key
-
-        open(self.logfile, 'ab+').write(
-            ntob("%s%s\n" % (self.markerPrefix, key), "utf-8"))
-
-    def _read_marked_region(self, marker=None):
-        """Return lines from self.logfile in the marked region.
-
-        If marker is None, self.lastmarker is used. If the log hasn't
-        been marked (using self.markLog), the entire log will be returned.
-        """
-# Give the logger time to finish writing?
-# time.sleep(0.5)
-
-        logfile = self.logfile
-        marker = marker or self.lastmarker
-        if marker is None:
-            return open(logfile, 'rb').readlines()
-
-        if isinstance(marker, six.text_type):
-            marker = marker.encode('utf-8')
-        data = []
-        in_region = False
-        for line in open(logfile, 'rb'):
-            if in_region:
-                if (line.startswith(self.markerPrefix) and not marker in line):
-                    break
-                else:
-                    data.append(line)
-            elif marker in line:
-                in_region = True
-        return data
-
-    def assertInLog(self, line, marker=None):
-        """Fail if the given (partial) line is not in the log.
-
-        The log will be searched from the given marker to the next marker.
-        If marker is None, self.lastmarker is used. If the log hasn't
-        been marked (using self.markLog), the entire log will be searched.
-        """
-        data = self._read_marked_region(marker)
-        for logline in data:
-            if line in logline:
-                return
-        msg = "%r not found in log" % line
-        self._handleLogError(msg, data, marker, line)
-
-    def assertNotInLog(self, line, marker=None):
-        """Fail if the given (partial) line is in the log.
-
-        The log will be searched from the given marker to the next marker.
-        If marker is None, self.lastmarker is used. If the log hasn't
-        been marked (using self.markLog), the entire log will be searched.
-        """
-        data = self._read_marked_region(marker)
-        for logline in data:
-            if line in logline:
-                msg = "%r found in log" % line
-                self._handleLogError(msg, data, marker, line)
-
-    def assertLog(self, sliceargs, lines, marker=None):
-        """Fail if log.readlines()[sliceargs] is not contained in 'lines'.
-
-        The log will be searched from the given marker to the next marker.
-        If marker is None, self.lastmarker is used. If the log hasn't
-        been marked (using self.markLog), the entire log will be searched.
-        """
-        data = self._read_marked_region(marker)
-        if isinstance(sliceargs, int):
-            # Single arg. Use __getitem__ and allow lines to be str or list.
-            if isinstance(lines, (tuple, list)):
-                lines = lines[0]
-            if isinstance(lines, six.text_type):
-                lines = lines.encode('utf-8')
-            if lines not in data[sliceargs]:
-                msg = "%r not found on log line %r" % (lines, sliceargs)
-                self._handleLogError(
-                    msg,
-                    [data[sliceargs], "--EXTRA CONTEXT--"] + data[
-                        sliceargs + 1:sliceargs + 6],
-                    marker,
-                    lines)
-        else:
-            # Multiple args. Use __getslice__ and require lines to be list.
-            if isinstance(lines, tuple):
-                lines = list(lines)
-            elif isinstance(lines, basestring):
-                raise TypeError("The 'lines' arg must be a list when "
-                                "'sliceargs' is a tuple.")
-
-            start, stop = sliceargs
-            for line, logline in zip(lines, data[start:stop]):
-                if isinstance(line, six.text_type):
-                    line = line.encode('utf-8')
-                if line not in logline:
-                    msg = "%r not found in log" % line
-                    self._handleLogError(msg, data[start:stop], marker, line)
diff --git a/lib/cherrypy/test/modfastcgi.py b/lib/cherrypy/test/modfastcgi.py
deleted file mode 100644
index 5364547439c..00000000000
--- a/lib/cherrypy/test/modfastcgi.py
+++ /dev/null
@@ -1,137 +0,0 @@
-"""Wrapper for mod_fastcgi, for use as a CherryPy HTTP server when testing.
-
-To autostart fastcgi, the "apache" executable or script must be
-on your system path, or you must override the global APACHE_PATH.
-On some platforms, "apache" may be called "apachectl", "apache2ctl",
-or "httpd"--create a symlink to them if needed.
-
-You'll also need the WSGIServer from flup.servers.
-See http://projects.amor.org/misc/wiki/ModPythonGateway
-
-
-KNOWN BUGS
-==========
-
-1. Apache processes Range headers automatically; CherryPy's truncated
-    output is then truncated again by Apache. See test_core.testRanges.
-    This was worked around in http://www.cherrypy.org/changeset/1319.
-2. Apache does not allow custom HTTP methods like CONNECT as per the spec.
-    See test_core.testHTTPMethods.
-3. Max request header and body settings do not work with Apache.
-4. Apache replaces status "reason phrases" automatically. For example,
-    CherryPy may set "304 Not modified" but Apache will write out
-    "304 Not Modified" (capital "M").
-5. Apache does not allow custom error codes as per the spec.
-6. Apache (or perhaps modpython, or modpython_gateway) unquotes %xx in the
-    Request-URI too early.
-7. mod_python will not read request bodies which use the "chunked"
-    transfer-coding (it passes REQUEST_CHUNKED_ERROR to ap_setup_client_block
-    instead of REQUEST_CHUNKED_DECHUNK, see Apache2's http_protocol.c and
-    mod_python's requestobject.c).
-8. Apache will output a "Content-Length: 0" response header even if there's
-    no response entity body. This isn't really a bug; it just differs from
-    the CherryPy default.
-"""
-
-import os
-curdir = os.path.join(os.getcwd(), os.path.dirname(__file__))
-import re
-import sys
-import time
-
-import cherrypy
-from cherrypy.process import plugins, servers
-from cherrypy.test import helper
-
-
-def read_process(cmd, args=""):
-    pipein, pipeout = os.popen4("%s %s" % (cmd, args))
-    try:
-        firstline = pipeout.readline()
-        if (re.search(r"(not recognized|No such file|not found)", firstline,
-                      re.IGNORECASE)):
-            raise IOError('%s must be on your system path.' % cmd)
-        output = firstline + pipeout.read()
-    finally:
-        pipeout.close()
-    return output
-
-
-APACHE_PATH = "apache2ctl"
-CONF_PATH = "fastcgi.conf"
-
-conf_fastcgi = """
-# Apache2 server conf file for testing CherryPy with mod_fastcgi.
-# fumanchu: I had to hard-code paths due to crazy Debian layouts :(
-ServerRoot /usr/lib/apache2
-User #1000
-ErrorLog %(root)s/mod_fastcgi.error.log
-
-DocumentRoot "%(root)s"
-ServerName 127.0.0.1
-Listen %(port)s
-LoadModule fastcgi_module modules/mod_fastcgi.so
-LoadModule rewrite_module modules/mod_rewrite.so
-
-Options +ExecCGI
-SetHandler fastcgi-script
-RewriteEngine On
-RewriteRule ^(.*)$ /fastcgi.pyc [L]
-FastCgiExternalServer "%(server)s" -host 127.0.0.1:4000
-"""
-
-
-def erase_script_name(environ, start_response):
-    environ['SCRIPT_NAME'] = ''
-    return cherrypy.tree(environ, start_response)
-
-
-class ModFCGISupervisor(helper.LocalWSGISupervisor):
-
-    httpserver_class = "cherrypy.process.servers.FlupFCGIServer"
-    using_apache = True
-    using_wsgi = True
-    template = conf_fastcgi
-
-    def __str__(self):
-        return "FCGI Server on %s:%s" % (self.host, self.port)
-
-    def start(self, modulename):
-        cherrypy.server.httpserver = servers.FlupFCGIServer(
-            application=erase_script_name, bindAddress=('127.0.0.1', 4000))
-        cherrypy.server.httpserver.bind_addr = ('127.0.0.1', 4000)
-        cherrypy.server.socket_port = 4000
-        # For FCGI, we both start apache...
-        self.start_apache()
-        # ...and our local server
-        cherrypy.engine.start()
-        self.sync_apps()
-
-    def start_apache(self):
-        fcgiconf = CONF_PATH
-        if not os.path.isabs(fcgiconf):
-            fcgiconf = os.path.join(curdir, fcgiconf)
-
-        # Write the Apache conf file.
-        f = open(fcgiconf, 'wb')
-        try:
-            server = repr(os.path.join(curdir, 'fastcgi.pyc'))[1:-1]
-            output = self.template % {'port': self.port, 'root': curdir,
-                                      'server': server}
-            output = output.replace('\r\n', '\n')
-            f.write(output)
-        finally:
-            f.close()
-
-        result = read_process(APACHE_PATH, "-k start -f %s" % fcgiconf)
-        if result:
-            print(result)
-
-    def stop(self):
-        """Gracefully shutdown a server that is serving forever."""
-        read_process(APACHE_PATH, "-k stop")
-        helper.LocalWSGISupervisor.stop(self)
-
-    def sync_apps(self):
-        cherrypy.server.httpserver.fcgiserver.application = self.get_app(
-            erase_script_name)
diff --git a/lib/cherrypy/test/modfcgid.py b/lib/cherrypy/test/modfcgid.py
deleted file mode 100644
index 6f8a3584f22..00000000000
--- a/lib/cherrypy/test/modfcgid.py
+++ /dev/null
@@ -1,125 +0,0 @@
-"""Wrapper for mod_fcgid, for use as a CherryPy HTTP server when testing.
-
-To autostart fcgid, the "apache" executable or script must be
-on your system path, or you must override the global APACHE_PATH.
-On some platforms, "apache" may be called "apachectl", "apache2ctl",
-or "httpd"--create a symlink to them if needed.
-
-You'll also need the WSGIServer from flup.servers.
-See http://projects.amor.org/misc/wiki/ModPythonGateway
-
-
-KNOWN BUGS
-==========
-
-1. Apache processes Range headers automatically; CherryPy's truncated
-    output is then truncated again by Apache. See test_core.testRanges.
-    This was worked around in http://www.cherrypy.org/changeset/1319.
-2. Apache does not allow custom HTTP methods like CONNECT as per the spec.
-    See test_core.testHTTPMethods.
-3. Max request header and body settings do not work with Apache.
-4. Apache replaces status "reason phrases" automatically. For example,
-    CherryPy may set "304 Not modified" but Apache will write out
-    "304 Not Modified" (capital "M").
-5. Apache does not allow custom error codes as per the spec.
-6. Apache (or perhaps modpython, or modpython_gateway) unquotes %xx in the
-    Request-URI too early.
-7. mod_python will not read request bodies which use the "chunked"
-    transfer-coding (it passes REQUEST_CHUNKED_ERROR to ap_setup_client_block
-    instead of REQUEST_CHUNKED_DECHUNK, see Apache2's http_protocol.c and
-    mod_python's requestobject.c).
-8. Apache will output a "Content-Length: 0" response header even if there's
-    no response entity body. This isn't really a bug; it just differs from
-    the CherryPy default.
-"""
-
-import os
-curdir = os.path.join(os.getcwd(), os.path.dirname(__file__))
-import re
-import sys
-import time
-
-import cherrypy
-from cherrypy._cpcompat import ntob
-from cherrypy.process import plugins, servers
-from cherrypy.test import helper
-
-
-def read_process(cmd, args=""):
-    pipein, pipeout = os.popen4("%s %s" % (cmd, args))
-    try:
-        firstline = pipeout.readline()
-        if (re.search(r"(not recognized|No such file|not found)", firstline,
-                      re.IGNORECASE)):
-            raise IOError('%s must be on your system path.' % cmd)
-        output = firstline + pipeout.read()
-    finally:
-        pipeout.close()
-    return output
-
-
-APACHE_PATH = "httpd"
-CONF_PATH = "fcgi.conf"
-
-conf_fcgid = """
-# Apache2 server conf file for testing CherryPy with mod_fcgid.
-
-DocumentRoot "%(root)s"
-ServerName 127.0.0.1
-Listen %(port)s
-LoadModule fastcgi_module modules/mod_fastcgi.dll
-LoadModule rewrite_module modules/mod_rewrite.so
-
-Options ExecCGI
-SetHandler fastcgi-script
-RewriteEngine On
-RewriteRule ^(.*)$ /fastcgi.pyc [L]
-FastCgiExternalServer "%(server)s" -host 127.0.0.1:4000
-"""
-
-
-class ModFCGISupervisor(helper.LocalSupervisor):
-
-    using_apache = True
-    using_wsgi = True
-    template = conf_fcgid
-
-    def __str__(self):
-        return "FCGI Server on %s:%s" % (self.host, self.port)
-
-    def start(self, modulename):
-        cherrypy.server.httpserver = servers.FlupFCGIServer(
-            application=cherrypy.tree, bindAddress=('127.0.0.1', 4000))
-        cherrypy.server.httpserver.bind_addr = ('127.0.0.1', 4000)
-        # For FCGI, we both start apache...
-        self.start_apache()
-        # ...and our local server
-        helper.LocalServer.start(self, modulename)
-
-    def start_apache(self):
-        fcgiconf = CONF_PATH
-        if not os.path.isabs(fcgiconf):
-            fcgiconf = os.path.join(curdir, fcgiconf)
-
-        # Write the Apache conf file.
-        f = open(fcgiconf, 'wb')
-        try:
-            server = repr(os.path.join(curdir, 'fastcgi.pyc'))[1:-1]
-            output = self.template % {'port': self.port, 'root': curdir,
-                                      'server': server}
-            output = ntob(output.replace('\r\n', '\n'))
-            f.write(output)
-        finally:
-            f.close()
-
-        result = read_process(APACHE_PATH, "-k start -f %s" % fcgiconf)
-        if result:
-            print(result)
-
-    def stop(self):
-        """Gracefully shutdown a server that is serving forever."""
-        read_process(APACHE_PATH, "-k stop")
-        helper.LocalServer.stop(self)
-
-    def sync_apps(self):
-        cherrypy.server.httpserver.fcgiserver.application = self.get_app()
diff --git a/lib/cherrypy/test/modpy.py b/lib/cherrypy/test/modpy.py
deleted file mode 100644
index 829628b74ed..00000000000
--- a/lib/cherrypy/test/modpy.py
+++ /dev/null
@@ -1,165 +0,0 @@
-"""Wrapper for mod_python, for use as a CherryPy HTTP server when testing.
-
-To autostart modpython, the "apache" executable or script must be
-on your system path, or you must override the global APACHE_PATH.
-On some platforms, "apache" may be called "apachectl" or "apache2ctl"--
-create a symlink to them if needed.
-
-If you wish to test the WSGI interface instead of our _cpmodpy interface,
-you also need the 'modpython_gateway' module at:
-http://projects.amor.org/misc/wiki/ModPythonGateway
-
-
-KNOWN BUGS
-==========
-
-1. Apache processes Range headers automatically; CherryPy's truncated
-    output is then truncated again by Apache. See test_core.testRanges.
-    This was worked around in http://www.cherrypy.org/changeset/1319.
-2. Apache does not allow custom HTTP methods like CONNECT as per the spec.
-    See test_core.testHTTPMethods.
-3. Max request header and body settings do not work with Apache.
-4. Apache replaces status "reason phrases" automatically. For example,
-    CherryPy may set "304 Not modified" but Apache will write out
-    "304 Not Modified" (capital "M").
-5. Apache does not allow custom error codes as per the spec.
-6. Apache (or perhaps modpython, or modpython_gateway) unquotes %xx in the
-    Request-URI too early.
-7. mod_python will not read request bodies which use the "chunked"
-    transfer-coding (it passes REQUEST_CHUNKED_ERROR to ap_setup_client_block
-    instead of REQUEST_CHUNKED_DECHUNK, see Apache2's http_protocol.c and
-    mod_python's requestobject.c).
-8. Apache will output a "Content-Length: 0" response header even if there's
-    no response entity body. This isn't really a bug; it just differs from
-    the CherryPy default.
-"""
-
-import os
-curdir = os.path.join(os.getcwd(), os.path.dirname(__file__))
-import re
-import time
-
-from cherrypy.test import helper
-
-
-def read_process(cmd, args=""):
-    pipein, pipeout = os.popen4("%s %s" % (cmd, args))
-    try:
-        firstline = pipeout.readline()
-        if (re.search(r"(not recognized|No such file|not found)", firstline,
-                      re.IGNORECASE)):
-            raise IOError('%s must be on your system path.' % cmd)
-        output = firstline + pipeout.read()
-    finally:
-        pipeout.close()
-    return output
-
-
-APACHE_PATH = "httpd"
-CONF_PATH = "test_mp.conf"
-
-conf_modpython_gateway = """
-# Apache2 server conf file for testing CherryPy with modpython_gateway.
-
-ServerName 127.0.0.1
-DocumentRoot "/"
-Listen %(port)s
-LoadModule python_module modules/mod_python.so
-
-SetHandler python-program
-PythonFixupHandler cherrypy.test.modpy::wsgisetup
-PythonOption testmod %(modulename)s
-PythonHandler modpython_gateway::handler
-PythonOption wsgi.application cherrypy::tree
-PythonOption socket_host %(host)s
-PythonDebug On
-"""
-
-conf_cpmodpy = """
-# Apache2 server conf file for testing CherryPy with _cpmodpy.
-
-ServerName 127.0.0.1
-DocumentRoot "/"
-Listen %(port)s
-LoadModule python_module modules/mod_python.so
-
-SetHandler python-program
-PythonFixupHandler cherrypy.test.modpy::cpmodpysetup
-PythonHandler cherrypy._cpmodpy::handler
-PythonOption cherrypy.setup cherrypy.test.%(modulename)s::setup_server
-PythonOption socket_host %(host)s
-PythonDebug On
-"""
-
-
-class ModPythonSupervisor(helper.Supervisor):
-
-    using_apache = True
-    using_wsgi = False
-    template = None
-
-    def __str__(self):
-        return "ModPython Server on %s:%s" % (self.host, self.port)
-
-    def start(self, modulename):
-        mpconf = CONF_PATH
-        if not os.path.isabs(mpconf):
-            mpconf = os.path.join(curdir, mpconf)
-
-        f = open(mpconf, 'wb')
-        try:
-            f.write(self.template %
-                    {'port': self.port, 'modulename': modulename,
-                     'host': self.host})
-        finally:
-            f.close()
-
-        result = read_process(APACHE_PATH, "-k start -f %s" % mpconf)
-        if result:
-            print(result)
-
-    def stop(self):
-        """Gracefully shutdown a server that is serving forever."""
-        read_process(APACHE_PATH, "-k stop")
-
-
-loaded = False
-
-
-def wsgisetup(req):
-    global loaded
-    if not loaded:
-        loaded = True
-        options = req.get_options()
-
-        import cherrypy
-        cherrypy.config.update({
-            "log.error_file": os.path.join(curdir, "test.log"),
-            "environment": "test_suite",
-            "server.socket_host": options['socket_host'],
-        })
-
-        modname = options['testmod']
-        mod = __import__(modname, globals(), locals(), [''])
-        mod.setup_server()
-
-        cherrypy.server.unsubscribe()
-        cherrypy.engine.start()
-    from mod_python import apache
-    return apache.OK
-
-
-def cpmodpysetup(req):
-    global loaded
-    if not loaded:
-        loaded = True
-        options = req.get_options()
-
-        import cherrypy
-        cherrypy.config.update({
-            "log.error_file": os.path.join(curdir, "test.log"),
-            "environment": "test_suite",
-            "server.socket_host": options['socket_host'],
-        })
-    from mod_python import apache
-    return apache.OK
diff --git a/lib/cherrypy/test/modwsgi.py b/lib/cherrypy/test/modwsgi.py
deleted file mode 100644
index 043fb6e88f1..00000000000
--- a/lib/cherrypy/test/modwsgi.py
+++ /dev/null
@@ -1,150 +0,0 @@
-"""Wrapper for mod_wsgi, for use as a CherryPy HTTP server.
-
-To autostart modwsgi, the "apache" executable or script must be
-on your system path, or you must override the global APACHE_PATH.
-On some platforms, "apache" may be called "apachectl" or "apache2ctl"--
-create a symlink to them if needed.
-
-
-KNOWN BUGS
-==========
-
-##1. Apache processes Range headers automatically; CherryPy's truncated
-##    output is then truncated again by Apache. See test_core.testRanges.
-##    This was worked around in http://www.cherrypy.org/changeset/1319.
-2. Apache does not allow custom HTTP methods like CONNECT as per the spec.
-    See test_core.testHTTPMethods.
-3. Max request header and body settings do not work with Apache.
-##4. Apache replaces status "reason phrases" automatically. For example,
-##    CherryPy may set "304 Not modified" but Apache will write out
-##    "304 Not Modified" (capital "M").
-##5. Apache does not allow custom error codes as per the spec.
-##6. Apache (or perhaps modpython, or modpython_gateway) unquotes %xx in the
-##    Request-URI too early.
-7. mod_wsgi will not read request bodies which use the "chunked"
-    transfer-coding (it passes REQUEST_CHUNKED_ERROR to ap_setup_client_block
-    instead of REQUEST_CHUNKED_DECHUNK, see Apache2's http_protocol.c and
-    mod_python's requestobject.c).
-8. When responding with 204 No Content, mod_wsgi adds a Content-Length
-    header for you.
-9. When an error is raised, mod_wsgi has no facility for printing a
-    traceback as the response content (it's sent to the Apache log instead).
-10. Startup and shutdown of Apache when running mod_wsgi seems slow.
-"""
-
-import os
-curdir = os.path.abspath(os.path.dirname(__file__))
-import re
-import sys
-import time
-
-import cherrypy
-from cherrypy.test import helper, webtest
-
-
-def read_process(cmd, args=""):
-    pipein, pipeout = os.popen4("%s %s" % (cmd, args))
-    try:
-        firstline = pipeout.readline()
-        if (re.search(r"(not recognized|No such file|not found)", firstline,
-                      re.IGNORECASE)):
-            raise IOError('%s must be on your system path.' % cmd)
-        output = firstline + pipeout.read()
-    finally:
-        pipeout.close()
-    return output
-
-
-if sys.platform == 'win32':
-    APACHE_PATH = "httpd"
-else:
-    APACHE_PATH = "apache"
-
-CONF_PATH = "test_mw.conf"
-
-conf_modwsgi = r"""
-# Apache2 server conf file for testing CherryPy with modpython_gateway.
-
-ServerName 127.0.0.1
-DocumentRoot "/"
-Listen %(port)s
-
-AllowEncodedSlashes On
-LoadModule rewrite_module modules/mod_rewrite.so
-RewriteEngine on
-RewriteMap escaping int:escape
-
-LoadModule log_config_module modules/mod_log_config.so
-LogFormat "%%h %%l %%u %%t \"%%r\" %%>s %%b \"%%{Referer}i\" \"%%{User-agent}i\"" combined
-CustomLog "%(curdir)s/apache.access.log" combined
-ErrorLog "%(curdir)s/apache.error.log"
-LogLevel debug
-
-LoadModule wsgi_module modules/mod_wsgi.so
-LoadModule env_module modules/mod_env.so
-
-WSGIScriptAlias / "%(curdir)s/modwsgi.py"
-SetEnv testmod %(testmod)s
-"""
-
-
-class ModWSGISupervisor(helper.Supervisor):
-
-    """Server Controller for ModWSGI and CherryPy."""
-
-    using_apache = True
-    using_wsgi = True
-    template = conf_modwsgi
-
-    def __str__(self):
-        return "ModWSGI Server on %s:%s" % (self.host, self.port)
-
-    def start(self, modulename):
-        mpconf = CONF_PATH
-        if not os.path.isabs(mpconf):
-            mpconf = os.path.join(curdir, mpconf)
-
-        f = open(mpconf, 'wb')
-        try:
-            output = (self.template %
-                      {'port': self.port, 'testmod': modulename,
-                       'curdir': curdir})
-            f.write(output)
-        finally:
-            f.close()
-
-        result = read_process(APACHE_PATH, "-k start -f %s" % mpconf)
-        if result:
-            print(result)
-
-        # Make a request so mod_wsgi starts up our app.
-        # If we don't, concurrent initial requests will 404.
-        cherrypy._cpserver.wait_for_occupied_port("127.0.0.1", self.port)
-        webtest.openURL('/ihopetheresnodefault', port=self.port)
-        time.sleep(1)
-
-    def stop(self):
-        """Gracefully shutdown a server that is serving forever."""
-        read_process(APACHE_PATH, "-k stop")
-
-
-loaded = False
-
-
-def application(environ, start_response):
-    import cherrypy
-    global loaded
-    if not loaded:
-        loaded = True
-        modname = "cherrypy.test." + environ['testmod']
-        mod = __import__(modname, globals(), locals(), [''])
-        mod.setup_server()
-
-        cherrypy.config.update({
-            "log.error_file": os.path.join(curdir, "test.error.log"),
-            "log.access_file": os.path.join(curdir, "test.access.log"),
-            "environment": "test_suite",
-            "engine.SIGHUP": None,
-            "engine.SIGTERM": None,
-        })
-    return cherrypy.tree(environ, start_response)
diff --git a/lib/cherrypy/test/sessiondemo.py b/lib/cherrypy/test/sessiondemo.py
deleted file mode 100755
index 4b3d1222e72..00000000000
--- a/lib/cherrypy/test/sessiondemo.py
+++ /dev/null
@@ -1,158 +0,0 @@
-#!/usr/bin/python
-"""A session demonstration app."""
-
-import calendar
-from datetime import datetime
-import sys
-import cherrypy
-from cherrypy.lib import sessions
-from cherrypy._cpcompat import copyitems
-
-
-page = """
-
-
-
-
-
-
-
-
Session Demo

-
Reload this page. The session ID should not change from one reload to the next

-
Index | Expire | Regenerate

-
-    
-    
-    
-    
-    
-    
-    
-    
-
Session ID:%(sessionid)s
%(changemsg)s
Request Cookie%(reqcookie)s
Response Cookie%(respcookie)s
Session Data%(sessiondata)s
Server Time%(servertime)s (Unix time: %(serverunixtime)s)
Browser Time 
Cherrypy Version:%(cpversion)s
Python Version:%(pyversion)s

-
-"""
-
-
-class Root(object):
-
-    def page(self):
-        changemsg = []
-        if cherrypy.session.id != cherrypy.session.originalid:
-            if cherrypy.session.originalid is None:
-                changemsg.append(
-                    'Created new session because no session id was given.')
-            if cherrypy.session.missing:
-                changemsg.append(
-                    'Created new session due to missing '
-                    '(expired or malicious) session.')
-            if cherrypy.session.regenerated:
-                changemsg.append('Application generated a new session.')
-
-        try:
-            expires = cherrypy.response.cookie['session_id']['expires']
-        except KeyError:
-            expires = ''
-
-        return page % {
-            'sessionid': cherrypy.session.id,
-            'changemsg': '
'.join(changemsg),
-            'respcookie': cherrypy.response.cookie.output(),
-            'reqcookie': cherrypy.request.cookie.output(),
-            'sessiondata': copyitems(cherrypy.session),
-            'servertime': (
-                datetime.utcnow().strftime("%Y/%m/%d %H:%M") + " UTC"
-            ),
-            'serverunixtime': calendar.timegm(datetime.utcnow().timetuple()),
-            'cpversion': cherrypy.__version__,
-            'pyversion': sys.version,
-            'expires': expires,
-        }
-
-    @cherrypy.expose
-    def index(self):
-        # Must modify data or the session will not be saved.
-        cherrypy.session['color'] = 'green'
-        return self.page()
-
-    @cherrypy.expose
-    def expire(self):
-        sessions.expire()
-        return self.page()
-
-    @cherrypy.expose
-    def regen(self):
-        cherrypy.session.regenerate()
-        # Must modify data or the session will not be saved.
-        cherrypy.session['color'] = 'yellow'
-        return self.page()
-
-if __name__ == '__main__':
-    cherrypy.config.update({
-        #'environment': 'production',
-        'log.screen': True,
-        'tools.sessions.on': True,
-    })
-    cherrypy.quickstart(Root())
diff --git a/lib/cherrypy/test/static/404.html b/lib/cherrypy/test/static/404.html
deleted file mode 100644
index 01b17b09631..00000000000
--- a/lib/cherrypy/test/static/404.html
+++ /dev/null
@@ -1,5 +0,0 @@
-
- 
-  
I couldn't find that thing you were looking for!

- 
-
diff --git a/lib/cherrypy/test/static/dirback.jpg b/lib/cherrypy/test/static/dirback.jpg
deleted file mode 100644
index 530e6d6a386..00000000000
Binary files a/lib/cherrypy/test/static/dirback.jpg and /dev/null differ
diff --git a/lib/cherrypy/test/static/has space.html b/lib/cherrypy/test/static/has space.html
deleted file mode 100644
index b9f5f097d23..00000000000
--- a/lib/cherrypy/test/static/has space.html	
+++ /dev/null
@@ -1 +0,0 @@
-Hello, world
diff --git a/lib/cherrypy/test/static/index.html b/lib/cherrypy/test/static/index.html
deleted file mode 100644
index b9f5f097d23..00000000000
--- a/lib/cherrypy/test/static/index.html
+++ /dev/null
@@ -1 +0,0 @@
-Hello, world
diff --git a/lib/cherrypy/test/style.css b/lib/cherrypy/test/style.css
deleted file mode 100644
index b266e93de71..00000000000
--- a/lib/cherrypy/test/style.css
+++ /dev/null
@@ -1 +0,0 @@
-Dummy stylesheet
diff --git a/lib/cherrypy/test/test.pem b/lib/cherrypy/test/test.pem
deleted file mode 100644
index 47a47042b60..00000000000
--- a/lib/cherrypy/test/test.pem
+++ /dev/null
@@ -1,38 +0,0 @@
------BEGIN RSA PRIVATE KEY-----
-MIICXAIBAAKBgQDBKo554mzIMY+AByUNpaUOP9bJnQ7ZLQe9XgHwoLJR4VzpyZZZ
-R9L4WtImEew05FY3Izerfm3MN3+MC0tJ6yQU9sOiU3vBW6RrLIMlfKsnRwBRZ0Kn
-da+O6xldVSosu8Ev3z9VZ94iC/ZgKzrH7Mjj/U8/MQO7RBS/LAqee8bFNQIDAQAB
-AoGAWOCF0ZrWxn3XMucWq2LNwPKqlvVGwbIwX3cDmX22zmnM4Fy6arXbYh4XlyCj
-9+ofqRrxIFz5k/7tFriTmZ0xag5+Jdx+Kwg0/twiP7XCNKipFogwe1Hznw8OFAoT
-enKBdj2+/n2o0Bvo/tDB59m9L/538d46JGQUmJlzMyqYikECQQDyoq+8CtMNvE18
-8VgHcR/KtApxWAjj4HpaHYL637ATjThetUZkW92mgDgowyplthusxdNqhHWyv7E8
-tWNdYErZAkEAy85ShTR0M5aWmrE7o0r0SpWInAkNBH9aXQRRARFYsdBtNfRu6I0i
-0lvU9wiu3eF57FMEC86yViZ5UBnQfTu7vQJAVesj/Zt7pwaCDfdMa740OsxMUlyR
-MVhhGx4OLpYdPJ8qUecxGQKq13XZ7R1HGyNEY4bd2X80Smq08UFuATfC6QJAH8UB
-yBHtKz2GLIcELOg6PIYizW/7v3+6rlVF60yw7sb2vzpjL40QqIn4IKoR2DSVtOkb
-8FtAIX3N21aq0VrGYQJBAIPiaEc2AZ8Bq2GC4F3wOz/BxJ/izvnkiotR12QK4fh5
-yjZMhTjWCas5zwHR5PDjlD88AWGDMsZ1PicD4348xJQ=
------END RSA PRIVATE KEY-----
------BEGIN CERTIFICATE-----
-MIIDxTCCAy6gAwIBAgIJAI18BD7eQxlGMA0GCSqGSIb3DQEBBAUAMIGeMQswCQYD
-VQQGEwJVUzETMBEGA1UECBMKQ2FsaWZvcm5pYTESMBAGA1UEBxMJU2FuIERpZWdv
-MRkwFwYDVQQKExBDaGVycnlQeSBQcm9qZWN0MREwDwYDVQQLEwhkZXYtdGVzdDEW
-MBQGA1UEAxMNQ2hlcnJ5UHkgVGVhbTEgMB4GCSqGSIb3DQEJARYRcmVtaUBjaGVy
-cnlweS5vcmcwHhcNMDYwOTA5MTkyMDIwWhcNMzQwMTI0MTkyMDIwWjCBnjELMAkG
-A1UEBhMCVVMxEzARBgNVBAgTCkNhbGlmb3JuaWExEjAQBgNVBAcTCVNhbiBEaWVn
-bzEZMBcGA1UEChMQQ2hlcnJ5UHkgUHJvamVjdDERMA8GA1UECxMIZGV2LXRlc3Qx
-FjAUBgNVBAMTDUNoZXJyeVB5IFRlYW0xIDAeBgkqhkiG9w0BCQEWEXJlbWlAY2hl
-cnJ5cHkub3JnMIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQDBKo554mzIMY+A
-ByUNpaUOP9bJnQ7ZLQe9XgHwoLJR4VzpyZZZR9L4WtImEew05FY3Izerfm3MN3+M
-C0tJ6yQU9sOiU3vBW6RrLIMlfKsnRwBRZ0Knda+O6xldVSosu8Ev3z9VZ94iC/Zg
-KzrH7Mjj/U8/MQO7RBS/LAqee8bFNQIDAQABo4IBBzCCAQMwHQYDVR0OBBYEFDIQ
-2feb71tVZCWpU0qJ/Tw+wdtoMIHTBgNVHSMEgcswgciAFDIQ2feb71tVZCWpU0qJ
-/Tw+wdtooYGkpIGhMIGeMQswCQYDVQQGEwJVUzETMBEGA1UECBMKQ2FsaWZvcm5p
-YTESMBAGA1UEBxMJU2FuIERpZWdvMRkwFwYDVQQKExBDaGVycnlQeSBQcm9qZWN0
-MREwDwYDVQQLEwhkZXYtdGVzdDEWMBQGA1UEAxMNQ2hlcnJ5UHkgVGVhbTEgMB4G
-CSqGSIb3DQEJARYRcmVtaUBjaGVycnlweS5vcmeCCQCNfAQ+3kMZRjAMBgNVHRME
-BTADAQH/MA0GCSqGSIb3DQEBBAUAA4GBAL7AAQz7IePV48ZTAFHKr88ntPALsL5S
-8vHCZPNMevNkLTj3DYUw2BcnENxMjm1kou2F2BkvheBPNZKIhc6z4hAml3ed1xa2
-D7w6e6OTcstdK/+KrPDDHeOP1dhMWNs2JE1bNlfF1LiXzYKSXpe88eCKjCXsCT/T
-NluCaWQys3MS
------END CERTIFICATE-----
diff --git a/lib/cherrypy/test/test_auth_basic.py b/lib/cherrypy/test/test_auth_basic.py
deleted file mode 100644
index c088cc55240..00000000000
--- a/lib/cherrypy/test/test_auth_basic.py
+++ /dev/null
@@ -1,95 +0,0 @@
-# This file is part of CherryPy 
-# -*- coding: utf-8 -*-
-# vim:ts=4:sw=4:expandtab:fileencoding=utf-8
-
-from hashlib import md5
-
-import cherrypy
-from cherrypy._cpcompat import ntob
-from cherrypy.lib import auth_basic
-from cherrypy.test import helper
-
-
-class BasicAuthTest(helper.CPWebCase):
-
-    def setup_server():
-        class Root:
-
-            @cherrypy.expose
-            def index(self):
-                return "This is public."
-
-        class BasicProtected:
-
-            @cherrypy.expose
-            def index(self):
-                return "Hello %s, you've been authorized." % (
-                    cherrypy.request.login)
-
-        class BasicProtected2:
-
-            @cherrypy.expose
-            def index(self):
-                return "Hello %s, you've been authorized." % (
-                    cherrypy.request.login)
-
-        userpassdict = {'xuser': 'xpassword'}
-        userhashdict = {'xuser': md5(ntob('xpassword')).hexdigest()}
-
-        def checkpasshash(realm, user, password):
-            p = userhashdict.get(user)
-            return p and p == md5(ntob(password)).hexdigest() or False
-
-        basic_checkpassword_dict = auth_basic.checkpassword_dict(userpassdict)
-        conf = {
-            '/basic': {
-                'tools.auth_basic.on': True,
-                'tools.auth_basic.realm': 'wonderland',
-                'tools.auth_basic.checkpassword': basic_checkpassword_dict
-            },
-            '/basic2': {
-                'tools.auth_basic.on': True,
-                'tools.auth_basic.realm': 'wonderland',
-                'tools.auth_basic.checkpassword': checkpasshash
-            },
-        }
-
-        root = Root()
-        root.basic = BasicProtected()
-        root.basic2 = BasicProtected2()
-        cherrypy.tree.mount(root, config=conf)
-    setup_server = staticmethod(setup_server)
-
-    def testPublic(self):
-        self.getPage("/")
-        self.assertStatus('200 OK')
-        self.assertHeader('Content-Type', 'text/html;charset=utf-8')
-        self.assertBody('This is public.')
-
-    def testBasic(self):
-        self.getPage("/basic/")
-        self.assertStatus(401)
-        self.assertHeader('WWW-Authenticate', 'Basic realm="wonderland"')
-
-        self.getPage('/basic/',
-                     [('Authorization', 'Basic eHVzZXI6eHBhc3N3b3JX')])
-        self.assertStatus(401)
-
-        self.getPage('/basic/',
-                     [('Authorization', 'Basic eHVzZXI6eHBhc3N3b3Jk')])
-        self.assertStatus('200 OK')
-        self.assertBody("Hello xuser, you've been authorized.")
-
-    def testBasic2(self):
-        self.getPage("/basic2/")
-        self.assertStatus(401)
-        self.assertHeader('WWW-Authenticate', 'Basic realm="wonderland"')
-
-        self.getPage('/basic2/',
-                     [('Authorization', 'Basic eHVzZXI6eHBhc3N3b3JX')])
-        self.assertStatus(401)
-
-        self.getPage('/basic2/',
-                     [('Authorization', 'Basic eHVzZXI6eHBhc3N3b3Jk')])
-        self.assertStatus('200 OK')
-        self.assertBody("Hello xuser, you've been authorized.")
diff --git a/lib/cherrypy/test/test_auth_digest.py b/lib/cherrypy/test/test_auth_digest.py
deleted file mode 100644
index 6e70d7588a9..00000000000
--- a/lib/cherrypy/test/test_auth_digest.py
+++ /dev/null
@@ -1,139 +0,0 @@
-# This file is part of CherryPy 
-# -*- coding: utf-8 -*-
-# vim:ts=4:sw=4:expandtab:fileencoding=utf-8
-
-
-import cherrypy
-from cherrypy.lib import auth_digest
-
-from cherrypy.test import helper
-
-
-class DigestAuthTest(helper.CPWebCase):
-
-    def setup_server():
-        class Root:
-
-            @cherrypy.expose
-            def index(self):
-                return "This is public."
-
-        class DigestProtected:
-
-            @cherrypy.expose
-            def index(self):
-                return "Hello %s, you've been authorized." % (
-                    cherrypy.request.login)
-
-        def fetch_users():
-            return {'test': 'test'}
-
-        get_ha1 = cherrypy.lib.auth_digest.get_ha1_dict_plain(fetch_users())
-        conf = {'/digest': {'tools.auth_digest.on': True,
-                            'tools.auth_digest.realm': 'localhost',
-                            'tools.auth_digest.get_ha1': get_ha1,
-                            'tools.auth_digest.key': 'a565c27146791cfb',
-                            'tools.auth_digest.debug': 'True'}}
-
-        root = Root()
-        root.digest = DigestProtected()
-        cherrypy.tree.mount(root, config=conf)
-    setup_server = staticmethod(setup_server)
-
-    def testPublic(self):
-        self.getPage("/")
-        self.assertStatus('200 OK')
-        self.assertHeader('Content-Type', 'text/html;charset=utf-8')
-        self.assertBody('This is public.')
-
-    def testDigest(self):
-        self.getPage("/digest/")
-        self.assertStatus(401)
-
-        value = None
-        for k, v in self.headers:
-            if k.lower() == "www-authenticate":
-                if v.startswith("Digest"):
-                    value = v
-                    break
-
-        if value is None:
-            self._handlewebError(
-                "Digest authentification scheme was not found")
-
-        value = value[7:]
-        items = value.split(', ')
-        tokens = {}
-        for item in items:
-            key, value = item.split('=')
-            tokens[key.lower()] = value
-
-        missing_msg = "%s is missing"
-        bad_value_msg = "'%s' was expecting '%s' but found '%s'"
-        nonce = None
-        if 'realm' not in tokens:
-            self._handlewebError(missing_msg % 'realm')
-        elif tokens['realm'] != '"localhost"':
-            self._handlewebError(bad_value_msg %
-                                 ('realm', '"localhost"', tokens['realm']))
-        if 'nonce' not in tokens:
-            self._handlewebError(missing_msg % 'nonce')
-        else:
-            nonce = tokens['nonce'].strip('"')
-        if 'algorithm' not in tokens:
-            self._handlewebError(missing_msg % 'algorithm')
-        elif tokens['algorithm'] != '"MD5"':
-            self._handlewebError(bad_value_msg %
-                                 ('algorithm', '"MD5"', tokens['algorithm']))
-        if 'qop' not in tokens:
-            self._handlewebError(missing_msg % 'qop')
-        elif tokens['qop'] != '"auth"':
-            self._handlewebError(bad_value_msg %
-                                 ('qop', '"auth"', tokens['qop']))
-
-        get_ha1 = auth_digest.get_ha1_dict_plain({'test': 'test'})
-
-        # Test user agent response with a wrong value for 'realm'
-        base_auth = ('Digest username="test", '
-                     'realm="wrong realm", '
-                     'nonce="%s", '
-                     'uri="/digest/", '
-                     'algorithm=MD5, '
-                     'response="%s", '
-                     'qop=auth, '
-                     'nc=%s, '
-                     'cnonce="1522e61005789929"')
-
-        auth_header = base_auth % (
-            nonce, '11111111111111111111111111111111', '00000001')
-        auth = auth_digest.HttpDigestAuthorization(auth_header, 'GET')
-        # calculate the response digest
-        ha1 = get_ha1(auth.realm, 'test')
-        response = auth.request_digest(ha1)
-        # send response with correct response digest, but wrong realm
-        auth_header = base_auth % (nonce, response, '00000001')
-        self.getPage('/digest/', [('Authorization', auth_header)])
-        self.assertStatus(401)
-
-        # Test that must pass
-        base_auth = ('Digest username="test", '
-                     'realm="localhost", '
-                     'nonce="%s", '
-                     'uri="/digest/", '
-                     'algorithm=MD5, '
-                     'response="%s", '
-                     'qop=auth, '
-                     'nc=%s, '
-                     'cnonce="1522e61005789929"')
-
-        auth_header = base_auth % (
-            nonce, '11111111111111111111111111111111', '00000001')
-        auth = auth_digest.HttpDigestAuthorization(auth_header, 'GET')
-        # calculate the response digest
-        ha1 = get_ha1('localhost', 'test')
-        response = auth.request_digest(ha1)
-        # send response with correct response digest
-        auth_header = base_auth % (nonce, response, '00000001')
-        self.getPage('/digest/', [('Authorization', auth_header)])
-        self.assertStatus('200 OK')
-        self.assertBody("Hello test, you've been authorized.")
diff --git a/lib/cherrypy/test/test_bus.py b/lib/cherrypy/test/test_bus.py
deleted file mode 100644
index 6c927ffb8d0..00000000000
--- a/lib/cherrypy/test/test_bus.py
+++ /dev/null
@@ -1,275 +0,0 @@
-import threading
-import time
-import unittest
-
-from cherrypy._cpcompat import get_daemon
-from cherrypy.process import wspbus
-
-
-msg = "Listener %d on channel %s: %s."
-
-
-class PublishSubscribeTests(unittest.TestCase):
-
-    def get_listener(self, channel, index):
-        def listener(arg=None):
-            self.responses.append(msg % (index, channel, arg))
-        return listener
-
-    def test_builtin_channels(self):
-        b = wspbus.Bus()
-
-        self.responses, expected = [], []
-
-        for channel in b.listeners:
-            for index, priority in enumerate([100, 50, 0, 51]):
-                b.subscribe(channel,
-                            self.get_listener(channel, index), priority)
-
-        for channel in b.listeners:
-            b.publish(channel)
-            expected.extend([msg % (i, channel, None) for i in (2, 1, 3, 0)])
-            b.publish(channel, arg=79347)
-            expected.extend([msg % (i, channel, 79347) for i in (2, 1, 3, 0)])
-
-        self.assertEqual(self.responses, expected)
-
-    def test_custom_channels(self):
-        b = wspbus.Bus()
-
-        self.responses, expected = [], []
-
-        custom_listeners = ('hugh', 'louis', 'dewey')
-        for channel in custom_listeners:
-            for index, priority in enumerate([None, 10, 60, 40]):
-                b.subscribe(channel,
-                            self.get_listener(channel, index), priority)
-
-        for channel in custom_listeners:
-            b.publish(channel, 'ah so')
-            expected.extend([msg % (i, channel, 'ah so')
-                            for i in (1, 3, 0, 2)])
-            b.publish(channel)
-            expected.extend([msg % (i, channel, None) for i in (1, 3, 0, 2)])
-
-        self.assertEqual(self.responses, expected)
-
-    def test_listener_errors(self):
-        b = wspbus.Bus()
-
-        self.responses, expected = [], []
-        channels = [c for c in b.listeners if c != 'log']
-
-        for channel in channels:
-            b.subscribe(channel, self.get_listener(channel, 1))
-            # This will break since the lambda takes no args.
-            b.subscribe(channel, lambda: None, priority=20)
-
-        for channel in channels:
-            self.assertRaises(wspbus.ChannelFailures, b.publish, channel, 123)
-            expected.append(msg % (1, channel, 123))
-
-        self.assertEqual(self.responses, expected)
-
-
-class BusMethodTests(unittest.TestCase):
-
-    def log(self, bus):
-        self._log_entries = []
-
-        def logit(msg, level):
-            self._log_entries.append(msg)
-        bus.subscribe('log', logit)
-
-    def assertLog(self, entries):
-        self.assertEqual(self._log_entries, entries)
-
-    def get_listener(self, channel, index):
-        def listener(arg=None):
-            self.responses.append(msg % (index, channel, arg))
-        return listener
-
-    def test_start(self):
-        b = wspbus.Bus()
-        self.log(b)
-
-        self.responses = []
-        num = 3
-        for index in range(num):
-            b.subscribe('start', self.get_listener('start', index))
-
-        b.start()
-        try:
-            # The start method MUST call all 'start' listeners.
-            self.assertEqual(
-                set(self.responses),
-                set([msg % (i, 'start', None) for i in range(num)]))
-            # The start method MUST move the state to STARTED
-            # (or EXITING, if errors occur)
-            self.assertEqual(b.state, b.states.STARTED)
-            # The start method MUST log its states.
-            self.assertLog(['Bus STARTING', 'Bus STARTED'])
-        finally:
-            # Exit so the atexit handler doesn't complain.
-            b.exit()
-
-    def test_stop(self):
-        b = wspbus.Bus()
-        self.log(b)
-
-        self.responses = []
-        num = 3
-        for index in range(num):
-            b.subscribe('stop', self.get_listener('stop', index))
-
-        b.stop()
-
-        # The stop method MUST call all 'stop' listeners.
-        self.assertEqual(set(self.responses),
-                         set([msg % (i, 'stop', None) for i in range(num)]))
-        # The stop method MUST move the state to STOPPED
-        self.assertEqual(b.state, b.states.STOPPED)
-        # The stop method MUST log its states.
-        self.assertLog(['Bus STOPPING', 'Bus STOPPED'])
-
-    def test_graceful(self):
-        b = wspbus.Bus()
-        self.log(b)
-
-        self.responses = []
-        num = 3
-        for index in range(num):
-            b.subscribe('graceful', self.get_listener('graceful', index))
-
-        b.graceful()
-
-        # The graceful method MUST call all 'graceful' listeners.
-        self.assertEqual(
-            set(self.responses),
-            set([msg % (i, 'graceful', None) for i in range(num)]))
-        # The graceful method MUST log its states.
-        self.assertLog(['Bus graceful'])
-
-    def test_exit(self):
-        b = wspbus.Bus()
-        self.log(b)
-
-        self.responses = []
-        num = 3
-        for index in range(num):
-            b.subscribe('stop', self.get_listener('stop', index))
-            b.subscribe('exit', self.get_listener('exit', index))
-
-        b.exit()
-
-        # The exit method MUST call all 'stop' listeners,
-        # and then all 'exit' listeners.
-        self.assertEqual(set(self.responses),
-                         set([msg % (i, 'stop', None) for i in range(num)] +
-                             [msg % (i, 'exit', None) for i in range(num)]))
-        # The exit method MUST move the state to EXITING
-        self.assertEqual(b.state, b.states.EXITING)
-        # The exit method MUST log its states.
-        self.assertLog(
-            ['Bus STOPPING', 'Bus STOPPED', 'Bus EXITING', 'Bus EXITED'])
-
-    def test_wait(self):
-        b = wspbus.Bus()
-
-        def f(method):
-            time.sleep(0.2)
-            getattr(b, method)()
-
-        for method, states in [('start', [b.states.STARTED]),
-                               ('stop', [b.states.STOPPED]),
-                               ('start',
-                                [b.states.STARTING, b.states.STARTED]),
-                               ('exit', [b.states.EXITING]),
-                               ]:
-            threading.Thread(target=f, args=(method,)).start()
-            b.wait(states)
-
-            # The wait method MUST wait for the given state(s).
-            if b.state not in states:
-                self.fail("State %r not in %r" % (b.state, states))
-
-    def test_block(self):
-        b = wspbus.Bus()
-        self.log(b)
-
-        def f():
-            time.sleep(0.2)
-            b.exit()
-
-        def g():
-            time.sleep(0.4)
-        threading.Thread(target=f).start()
-        threading.Thread(target=g).start()
-        threads = [t for t in threading.enumerate() if not get_daemon(t)]
-        self.assertEqual(len(threads), 3)
-
-        b.block()
-
-        # The block method MUST wait for the EXITING state.
-        self.assertEqual(b.state, b.states.EXITING)
-        # The block method MUST wait for ALL non-main, non-daemon threads to
-        # finish.
-        threads = [t for t in threading.enumerate() if not get_daemon(t)]
-        self.assertEqual(len(threads), 1)
-        # The last message will mention an indeterminable thread name; ignore
-        # it
-        self.assertEqual(self._log_entries[:-1],
-                         ['Bus STOPPING', 'Bus STOPPED',
-                          'Bus EXITING', 'Bus EXITED',
-                          'Waiting for child threads to terminate...'])
-
-    def test_start_with_callback(self):
-        b = wspbus.Bus()
-        self.log(b)
-        try:
-            events = []
-
-            def f(*args, **kwargs):
-                events.append(("f", args, kwargs))
-
-            def g():
-                events.append("g")
-            b.subscribe("start", g)
-            b.start_with_callback(f, (1, 3, 5), {"foo": "bar"})
-            # Give wait() time to run f()
-            time.sleep(0.2)
-
-            # The callback method MUST wait for the STARTED state.
-            self.assertEqual(b.state, b.states.STARTED)
-            # The callback method MUST run after all start methods.
-            self.assertEqual(events, ["g", ("f", (1, 3, 5), {"foo": "bar"})])
-        finally:
-            b.exit()
-
-    def test_log(self):
-        b = wspbus.Bus()
-        self.log(b)
-        self.assertLog([])
-
-        # Try a normal message.
-        expected = []
-        for msg in ["O mah darlin'"] * 3 + ["Clementiiiiiiiine"]:
-            b.log(msg)
-            expected.append(msg)
-            self.assertLog(expected)
-
-        # Try an error message
-        try:
-            foo
-        except NameError:
-            b.log("You are lost and gone forever", traceback=True)
-            lastmsg = self._log_entries[-1]
-            if "Traceback" not in lastmsg or "NameError" not in lastmsg:
-                self.fail("Last log message %r did not contain "
-                          "the expected traceback." % lastmsg)
-        else:
-            self.fail("NameError was not raised as expected.")
-
-
-if __name__ == "__main__":
-    unittest.main()
diff --git a/lib/cherrypy/test/test_caching.py b/lib/cherrypy/test/test_caching.py
deleted file mode 100644
index bbd0067cada..00000000000
--- a/lib/cherrypy/test/test_caching.py
+++ /dev/null
@@ -1,341 +0,0 @@
-import datetime
-import gzip
-from itertools import count
-import os
-curdir = os.path.join(os.getcwd(), os.path.dirname(__file__))
-import sys
-import threading
-import time
-import urllib
-
-import cherrypy
-from cherrypy._cpcompat import next, ntob, quote, xrange
-from cherrypy.lib import httputil
-
-gif_bytes = ntob(
-    'GIF89a\x01\x00\x01\x00\x82\x00\x01\x99"\x1e\x00\x00\x00\x00\x00'
-    '\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
-    '\x00,\x00\x00\x00\x00\x01\x00\x01\x00\x02\x03\x02\x08\t\x00;'
-)
-
-
-from cherrypy.test import helper
-
-
-class CacheTest(helper.CPWebCase):
-
-    def setup_server():
-
-        @cherrypy.config(**{'tools.caching.on': True})
-        class Root:
-
-            def __init__(self):
-                self.counter = 0
-                self.control_counter = 0
-                self.longlock = threading.Lock()
-
-            @cherrypy.expose
-            def index(self):
-                self.counter += 1
-                msg = "visit #%s" % self.counter
-                return msg
-
-            @cherrypy.expose
-            def control(self):
-                self.control_counter += 1
-                return "visit #%s" % self.control_counter
-
-            @cherrypy.expose
-            def a_gif(self):
-                cherrypy.response.headers[
-                    'Last-Modified'] = httputil.HTTPDate()
-                return gif_bytes
-
-            @cherrypy.expose
-            def long_process(self, seconds='1'):
-                try:
-                    self.longlock.acquire()
-                    time.sleep(float(seconds))
-                finally:
-                    self.longlock.release()
-                return 'success!'
-
-            @cherrypy.expose
-            def clear_cache(self, path):
-                cherrypy._cache.store[cherrypy.request.base + path].clear()
-
-        @cherrypy.config(**{
-            'tools.caching.on': True,
-            'tools.response_headers.on': True,
-            'tools.response_headers.headers': [
-                ('Vary', 'Our-Varying-Header')
-            ],
-        })
-        class VaryHeaderCachingServer(object):
-
-            def __init__(self):
-                self.counter = count(1)
-
-            @cherrypy.expose
-            def index(self):
-                return "visit #%s" % next(self.counter)
-
-        @cherrypy.config(**{
-            'tools.expires.on': True,
-            'tools.expires.secs': 60,
-            'tools.staticdir.on': True,
-            'tools.staticdir.dir': 'static',
-            'tools.staticdir.root': curdir,
-        })
-        class UnCached(object):
-
-            @cherrypy.expose
-            @cherrypy.config(**{'tools.expires.secs': 0})
-            def force(self):
-                cherrypy.response.headers['Etag'] = 'bibbitybobbityboo'
-                self._cp_config['tools.expires.force'] = True
-                self._cp_config['tools.expires.secs'] = 0
-                return "being forceful"
-
-            @cherrypy.expose
-            def dynamic(self):
-                cherrypy.response.headers['Etag'] = 'bibbitybobbityboo'
-                cherrypy.response.headers['Cache-Control'] = 'private'
-                return "D-d-d-dynamic!"
-
-            @cherrypy.expose
-            def cacheable(self):
-                cherrypy.response.headers['Etag'] = 'bibbitybobbityboo'
-                return "Hi, I'm cacheable."
-
-            @cherrypy.expose
-            @cherrypy.config(**{'tools.expires.secs': 86400})
-            def specific(self):
-                cherrypy.response.headers[
-                    'Etag'] = 'need_this_to_make_me_cacheable'
-                return "I am being specific"
-
-            class Foo(object):
-                pass
-
-            @cherrypy.expose
-            @cherrypy.config(**{'tools.expires.secs': Foo()})
-            def wrongtype(self):
-                cherrypy.response.headers[
-                    'Etag'] = 'need_this_to_make_me_cacheable'
-                return "Woops"
-
-        cherrypy.tree.mount(Root())
-        cherrypy.tree.mount(UnCached(), "/expires")
-        cherrypy.tree.mount(VaryHeaderCachingServer(), "/varying_headers")
-        cherrypy.config.update({'tools.gzip.on': True})
-    setup_server = staticmethod(setup_server)
-
-    def testCaching(self):
-        elapsed = 0.0
-        for trial in range(10):
-            self.getPage("/")
-            # The response should be the same every time,
-            # except for the Age response header.
-            self.assertBody('visit #1')
-            if trial != 0:
-                age = int(self.assertHeader("Age"))
-                self.assert_(age >= elapsed)
-                elapsed = age
-
-        # POST, PUT, DELETE should not be cached.
-        self.getPage("/", method="POST")
-        self.assertBody('visit #2')
-        # Because gzip is turned on, the Vary header should always Vary for
-        # content-encoding
-        self.assertHeader('Vary', 'Accept-Encoding')
-        # The previous request should have invalidated the cache,
-        # so this request will recalc the response.
-        self.getPage("/", method="GET")
-        self.assertBody('visit #3')
-        # ...but this request should get the cached copy.
-        self.getPage("/", method="GET")
-        self.assertBody('visit #3')
-        self.getPage("/", method="DELETE")
-        self.assertBody('visit #4')
-
-        # The previous request should have invalidated the cache,
-        # so this request will recalc the response.
-        self.getPage("/", method="GET", headers=[('Accept-Encoding', 'gzip')])
-        self.assertHeader('Content-Encoding', 'gzip')
-        self.assertHeader('Vary')
-        self.assertEqual(
-            cherrypy.lib.encoding.decompress(self.body), ntob("visit #5"))
-
-        # Now check that a second request gets the gzip header and gzipped body
-        # This also tests a bug in 3.0 to 3.0.2 whereby the cached, gzipped
-        # response body was being gzipped a second time.
-        self.getPage("/", method="GET", headers=[('Accept-Encoding', 'gzip')])
-        self.assertHeader('Content-Encoding', 'gzip')
-        self.assertEqual(
-            cherrypy.lib.encoding.decompress(self.body), ntob("visit #5"))
-
-        # Now check that a third request that doesn't accept gzip
-        # skips the cache (because the 'Vary' header denies it).
-        self.getPage("/", method="GET")
-        self.assertNoHeader('Content-Encoding')
-        self.assertBody('visit #6')
-
-    def testVaryHeader(self):
-        self.getPage("/varying_headers/")
-        self.assertStatus("200 OK")
-        self.assertHeaderItemValue('Vary', 'Our-Varying-Header')
-        self.assertBody('visit #1')
-
-        # Now check that different 'Vary'-fields don't evict each other.
-        # This test creates 2 requests with different 'Our-Varying-Header'
-        # and then tests if the first one still exists.
-        self.getPage("/varying_headers/",
-                     headers=[('Our-Varying-Header', 'request 2')])
-        self.assertStatus("200 OK")
-        self.assertBody('visit #2')
-
-        self.getPage("/varying_headers/",
-                     headers=[('Our-Varying-Header', 'request 2')])
-        self.assertStatus("200 OK")
-        self.assertBody('visit #2')
-
-        self.getPage("/varying_headers/")
-        self.assertStatus("200 OK")
-        self.assertBody('visit #1')
-
-    def testExpiresTool(self):
-        # test setting an expires header
-        self.getPage("/expires/specific")
-        self.assertStatus("200 OK")
-        self.assertHeader("Expires")
-
-        # test exceptions for bad time values
-        self.getPage("/expires/wrongtype")
-        self.assertStatus(500)
-        self.assertInBody("TypeError")
-
-        # static content should not have "cache prevention" headers
-        self.getPage("/expires/index.html")
-        self.assertStatus("200 OK")
-        self.assertNoHeader("Pragma")
-        self.assertNoHeader("Cache-Control")
-        self.assertHeader("Expires")
-
-        # dynamic content that sets indicators should not have
-        # "cache prevention" headers
-        self.getPage("/expires/cacheable")
-        self.assertStatus("200 OK")
-        self.assertNoHeader("Pragma")
-        self.assertNoHeader("Cache-Control")
-        self.assertHeader("Expires")
-
-        self.getPage('/expires/dynamic')
-        self.assertBody("D-d-d-dynamic!")
-        # the Cache-Control header should be untouched
-        self.assertHeader("Cache-Control", "private")
-        self.assertHeader("Expires")
-
-        # configure the tool to ignore indicators and replace existing headers
-        self.getPage("/expires/force")
-        self.assertStatus("200 OK")
-        # This also gives us a chance to test 0 expiry with no other headers
-        self.assertHeader("Pragma", "no-cache")
-        if cherrypy.server.protocol_version == "HTTP/1.1":
-            self.assertHeader("Cache-Control", "no-cache, must-revalidate")
-        self.assertHeader("Expires", "Sun, 28 Jan 2007 00:00:00 GMT")
-
-        # static content should now have "cache prevention" headers
-        self.getPage("/expires/index.html")
-        self.assertStatus("200 OK")
-        self.assertHeader("Pragma", "no-cache")
-        if cherrypy.server.protocol_version == "HTTP/1.1":
-            self.assertHeader("Cache-Control", "no-cache, must-revalidate")
-        self.assertHeader("Expires", "Sun, 28 Jan 2007 00:00:00 GMT")
-
-        # the cacheable handler should now have "cache prevention" headers
-        self.getPage("/expires/cacheable")
-        self.assertStatus("200 OK")
-        self.assertHeader("Pragma", "no-cache")
-        if cherrypy.server.protocol_version == "HTTP/1.1":
-            self.assertHeader("Cache-Control", "no-cache, must-revalidate")
-        self.assertHeader("Expires", "Sun, 28 Jan 2007 00:00:00 GMT")
-
-        self.getPage('/expires/dynamic')
-        self.assertBody("D-d-d-dynamic!")
-        # dynamic sets Cache-Control to private but it should  be
-        # overwritten here ...
-        self.assertHeader("Pragma", "no-cache")
-        if cherrypy.server.protocol_version == "HTTP/1.1":
-            self.assertHeader("Cache-Control", "no-cache, must-revalidate")
-        self.assertHeader("Expires", "Sun, 28 Jan 2007 00:00:00 GMT")
-
-    def testLastModified(self):
-        self.getPage("/a.gif")
-        self.assertStatus(200)
-        self.assertBody(gif_bytes)
-        lm1 = self.assertHeader("Last-Modified")
-
-        # this request should get the cached copy.
-        self.getPage("/a.gif")
-        self.assertStatus(200)
-        self.assertBody(gif_bytes)
-        self.assertHeader("Age")
-        lm2 = self.assertHeader("Last-Modified")
-        self.assertEqual(lm1, lm2)
-
-        # this request should match the cached copy, but raise 304.
-        self.getPage("/a.gif", [('If-Modified-Since', lm1)])
-        self.assertStatus(304)
-        self.assertNoHeader("Last-Modified")
-        if not getattr(cherrypy.server, "using_apache", False):
-            self.assertHeader("Age")
-
-    def test_antistampede(self):
-        SECONDS = 4
-        # We MUST make an initial synchronous request in order to create the
-        # AntiStampedeCache object, and populate its selecting_headers,
-        # before the actual stampede.
-        self.getPage("/long_process?seconds=%d" % SECONDS)
-        self.assertBody('success!')
-        self.getPage("/clear_cache?path=" +
-                     quote('/long_process?seconds=%d' % SECONDS, safe=''))
-        self.assertStatus(200)
-
-        start = datetime.datetime.now()
-
-        def run():
-            self.getPage("/long_process?seconds=%d" % SECONDS)
-            # The response should be the same every time
-            self.assertBody('success!')
-        ts = [threading.Thread(target=run) for i in xrange(100)]
-        for t in ts:
-            t.start()
-        for t in ts:
-            t.join()
-        self.assertEqualDates(start, datetime.datetime.now(),
-                              # Allow a second (two, for slow hosts)
-                              # for our thread/TCP overhead etc.
-                              seconds=SECONDS + 2)
-
-    def test_cache_control(self):
-        self.getPage("/control")
-        self.assertBody('visit #1')
-        self.getPage("/control")
-        self.assertBody('visit #1')
-
-        self.getPage("/control", headers=[('Cache-Control', 'no-cache')])
-        self.assertBody('visit #2')
-        self.getPage("/control")
-        self.assertBody('visit #2')
-
-        self.getPage("/control", headers=[('Pragma', 'no-cache')])
-        self.assertBody('visit #3')
-        self.getPage("/control")
-        self.assertBody('visit #3')
-
-        time.sleep(1)
-        self.getPage("/control", headers=[('Cache-Control', 'max-age=0')])
-        self.assertBody('visit #4')
-        self.getPage("/control")
-        self.assertBody('visit #4')
diff --git a/lib/cherrypy/test/test_compat.py b/lib/cherrypy/test/test_compat.py
deleted file mode 100644
index 90da8b029eb..00000000000
--- a/lib/cherrypy/test/test_compat.py
+++ /dev/null
@@ -1,21 +0,0 @@
-import unittest
-
-import six
-
-import nose
-
-from cherrypy import _cpcompat as compat
-
-
-class StringTester(unittest.TestCase):
-
-    def test_ntob_non_native(self):
-        """
-        ntob should raise an Exception on unicode.
-        (Python 2 only)
-
-        See #1132 for discussion.
-        """
-        if six.PY3:
-            raise nose.SkipTest("Only useful on Python 2")
-        self.assertRaises(Exception, compat.ntob, unicode('fight'))
diff --git a/lib/cherrypy/test/test_config.py b/lib/cherrypy/test/test_config.py
deleted file mode 100644
index 088f95d46ea..00000000000
--- a/lib/cherrypy/test/test_config.py
+++ /dev/null
@@ -1,303 +0,0 @@
-"""Tests for the CherryPy configuration system."""
-
-import io
-import os
-import sys
-import unittest
-
-import six
-
-import cherrypy
-import cherrypy._cpcompat as compat
-
-localDir = os.path.join(os.getcwd(), os.path.dirname(__file__))
-
-
-StringIOFromNative = lambda x: io.StringIO(six.text_type(x))
-
-
-def setup_server():
-
-    @cherrypy.config(foo='this', bar='that')
-    class Root:
-
-        def __init__(self):
-            cherrypy.config.namespaces['db'] = self.db_namespace
-
-        def db_namespace(self, k, v):
-            if k == "scheme":
-                self.db = v
-
-        @cherrypy.expose(alias=('global_', 'xyz'))
-        def index(self, key):
-            return cherrypy.request.config.get(key, "None")
-
-        @cherrypy.expose
-        def repr(self, key):
-            return repr(cherrypy.request.config.get(key, None))
-
-        @cherrypy.expose
-        def dbscheme(self):
-            return self.db
-
-        @cherrypy.expose
-        @cherrypy.config(**{'request.body.attempt_charsets': ['utf-16']})
-        def plain(self, x):
-            return x
-
-        favicon_ico = cherrypy.tools.staticfile.handler(
-            filename=os.path.join(localDir, '../favicon.ico'))
-
-    @cherrypy.config(foo='this2', baz='that2')
-    class Foo:
-
-        @cherrypy.expose
-        def index(self, key):
-            return cherrypy.request.config.get(key, "None")
-        nex = index
-
-        @cherrypy.expose
-        @cherrypy.config(**{'response.headers.X-silly': 'sillyval'})
-        def silly(self):
-            return 'Hello world'
-
-        # Test the expose and config decorators
-        @cherrypy.config(foo='this3', **{'bax': 'this4'})
-        @cherrypy.expose
-        def bar(self, key):
-            return repr(cherrypy.request.config.get(key, None))
-
-    class Another:
-
-        @cherrypy.expose
-        def index(self, key):
-            return str(cherrypy.request.config.get(key, "None"))
-
-    def raw_namespace(key, value):
-        if key == 'input.map':
-            handler = cherrypy.request.handler
-
-            def wrapper():
-                params = cherrypy.request.params
-                for name, coercer in list(value.items()):
-                    try:
-                        params[name] = coercer(params[name])
-                    except KeyError:
-                        pass
-                return handler()
-            cherrypy.request.handler = wrapper
-        elif key == 'output':
-            handler = cherrypy.request.handler
-
-            def wrapper():
-                # 'value' is a type (like int or str).
-                return value(handler())
-            cherrypy.request.handler = wrapper
-
-    @cherrypy.config(**{'raw.output': repr})
-    class Raw:
-
-        @cherrypy.expose
-        @cherrypy.config(**{'raw.input.map': {'num': int}})
-        def incr(self, num):
-            return num + 1
-
-    if not six.PY3:
-        thing3 = "thing3: unicode('test', errors='ignore')"
-    else:
-        thing3 = ''
-
-    ioconf = StringIOFromNative("""
-[/]
-neg: -1234
-filename: os.path.join(sys.prefix, "hello.py")
-thing1: cherrypy.lib.httputil.response_codes[404]
-thing2: __import__('cherrypy.tutorial', globals(), locals(), ['']).thing2
-%s
-complex: 3+2j
-mul: 6*3
-ones: "11"
-twos: "22"
-stradd: %%(ones)s + %%(twos)s + "33"
-
-[/favicon.ico]
-tools.staticfile.filename = %r
-""" % (thing3, os.path.join(localDir, 'static/dirback.jpg')))
-
-    root = Root()
-    root.foo = Foo()
-    root.raw = Raw()
-    app = cherrypy.tree.mount(root, config=ioconf)
-    app.request_class.namespaces['raw'] = raw_namespace
-
-    cherrypy.tree.mount(Another(), "/another")
-    cherrypy.config.update({'luxuryyacht': 'throatwobblermangrove',
-                            'db.scheme': r"sqlite///memory",
-                            })
-
-
-#                             Client-side code                             #
-
-from cherrypy.test import helper
-
-
-class ConfigTests(helper.CPWebCase):
-    setup_server = staticmethod(setup_server)
-
-    def testConfig(self):
-        tests = [
-            ('/',        'nex', 'None'),
-            ('/',        'foo', 'this'),
-            ('/',        'bar', 'that'),
-            ('/xyz',     'foo', 'this'),
-            ('/foo/',    'foo', 'this2'),
-            ('/foo/',    'bar', 'that'),
-            ('/foo/',    'bax', 'None'),
-            ('/foo/bar', 'baz', "'that2'"),
-            ('/foo/nex', 'baz', 'that2'),
-            # If 'foo' == 'this', then the mount point '/another' leaks into
-            # '/'.
-            ('/another/', 'foo', 'None'),
-        ]
-        for path, key, expected in tests:
-            self.getPage(path + "?key=" + key)
-            self.assertBody(expected)
-
-        expectedconf = {
-            # From CP defaults
-            'tools.log_headers.on': False,
-            'tools.log_tracebacks.on': True,
-            'request.show_tracebacks': True,
-            'log.screen': False,
-            'environment': 'test_suite',
-            'engine.autoreload.on': False,
-            # From global config
-            'luxuryyacht': 'throatwobblermangrove',
-            # From Root._cp_config
-            'bar': 'that',
-            # From Foo._cp_config
-            'baz': 'that2',
-            # From Foo.bar._cp_config
-            'foo': 'this3',
-            'bax': 'this4',
-        }
-        for key, expected in expectedconf.items():
-            self.getPage("/foo/bar?key=" + key)
-            self.assertBody(repr(expected))
-
-    def testUnrepr(self):
-        self.getPage("/repr?key=neg")
-        self.assertBody("-1234")
-
-        self.getPage("/repr?key=filename")
-        self.assertBody(repr(os.path.join(sys.prefix, "hello.py")))
-
-        self.getPage("/repr?key=thing1")
-        self.assertBody(repr(cherrypy.lib.httputil.response_codes[404]))
-
-        if not getattr(cherrypy.server, "using_apache", False):
-            # The object ID's won't match up when using Apache, since the
-            # server and client are running in different processes.
-            self.getPage("/repr?key=thing2")
-            from cherrypy.tutorial import thing2
-            self.assertBody(repr(thing2))
-
-        if not six.PY3:
-            self.getPage("/repr?key=thing3")
-            self.assertBody(repr(unicode('test')))
-
-        self.getPage("/repr?key=complex")
-        self.assertBody("(3+2j)")
-
-        self.getPage("/repr?key=mul")
-        self.assertBody("18")
-
-        self.getPage("/repr?key=stradd")
-        self.assertBody(repr("112233"))
-
-    def testRespNamespaces(self):
-        self.getPage("/foo/silly")
-        self.assertHeader('X-silly', 'sillyval')
-        self.assertBody('Hello world')
-
-    def testCustomNamespaces(self):
-        self.getPage("/raw/incr?num=12")
-        self.assertBody("13")
-
-        self.getPage("/dbscheme")
-        self.assertBody(r"sqlite///memory")
-
-    def testHandlerToolConfigOverride(self):
-        # Assert that config overrides tool constructor args. Above, we set
-        # the favicon in the page handler to be '../favicon.ico',
-        # but then overrode it in config to be './static/dirback.jpg'.
-        self.getPage("/favicon.ico")
-        self.assertBody(open(os.path.join(localDir, "static/dirback.jpg"),
-                             "rb").read())
-
-    def test_request_body_namespace(self):
-        self.getPage("/plain", method='POST', headers=[
-            ('Content-Type', 'application/x-www-form-urlencoded'),
-            ('Content-Length', '13')],
-            body=compat.ntob('\xff\xfex\x00=\xff\xfea\x00b\x00c\x00'))
-        self.assertBody("abc")
-
-
-class VariableSubstitutionTests(unittest.TestCase):
-    setup_server = staticmethod(setup_server)
-
-    def test_config(self):
-        from textwrap import dedent
-
-        # variable substitution with [DEFAULT]
-        conf = dedent("""
-        [DEFAULT]
-        dir = "/some/dir"
-        my.dir = %(dir)s + "/sub"
-
-        [my]
-        my.dir = %(dir)s + "/my/dir"
-        my.dir2 = %(my.dir)s + '/dir2'
-
-        """)
-
-        fp = StringIOFromNative(conf)
-
-        cherrypy.config.update(fp)
-        self.assertEqual(cherrypy.config["my"]["my.dir"], "/some/dir/my/dir")
-        self.assertEqual(cherrypy.config["my"]
-                         ["my.dir2"], "/some/dir/my/dir/dir2")
-
-
-class CallablesInConfigTest(unittest.TestCase):
-    setup_server = staticmethod(setup_server)
-
-
-    def test_call_with_literal_dict(self):
-        from textwrap import dedent
-        conf = dedent("""
-        [my]
-        value = dict(**{'foo': 'bar'})
-        """)
-        fp = StringIOFromNative(conf)
-        cherrypy.config.update(fp)
-        self.assertEqual(cherrypy.config['my']['value'], {'foo': 'bar'})
-
-    def test_call_with_kwargs(self):
-        from textwrap import dedent
-        conf = dedent("""
-        [my]
-        value = dict(foo="buzz", **cherrypy._test_dict)
-        """)
-        test_dict = {
-            "foo": "bar",
-            "bar": "foo",
-            "fizz": "buzz"
-        }
-        cherrypy._test_dict = test_dict
-        fp = StringIOFromNative(conf)
-        cherrypy.config.update(fp)
-        test_dict['foo'] = 'buzz'
-        self.assertEqual(cherrypy.config['my']['value']['foo'], 'buzz')
-        self.assertEqual(cherrypy.config['my']['value'], test_dict)
-        del cherrypy._test_dict
diff --git a/lib/cherrypy/test/test_config_server.py b/lib/cherrypy/test/test_config_server.py
deleted file mode 100644
index 3a6b2aeb405..00000000000
--- a/lib/cherrypy/test/test_config_server.py
+++ /dev/null
@@ -1,128 +0,0 @@
-"""Tests for the CherryPy configuration system."""
-
-import os
-import sys
-localDir = os.path.join(os.getcwd(), os.path.dirname(__file__))
-import socket
-import time
-
-import cherrypy
-
-
-#                             Client-side code                             #
-
-from cherrypy.test import helper
-
-
-class ServerConfigTests(helper.CPWebCase):
-
-    def setup_server():
-
-        class Root:
-
-            @cherrypy.expose
-            def index(self):
-                return cherrypy.request.wsgi_environ['SERVER_PORT']
-
-            @cherrypy.expose
-            def upload(self, file):
-                return "Size: %s" % len(file.file.read())
-
-            @cherrypy.expose
-            @cherrypy.config(**{'request.body.maxbytes': 100})
-            def tinyupload(self):
-                return cherrypy.request.body.read()
-
-        cherrypy.tree.mount(Root())
-
-        cherrypy.config.update({
-            'server.socket_host': '0.0.0.0',
-            'server.socket_port': 9876,
-            'server.max_request_body_size': 200,
-            'server.max_request_header_size': 500,
-            'server.socket_timeout': 0.5,
-
-            # Test explicit server.instance
-            'server.2.instance': 'cherrypy._cpwsgi_server.CPWSGIServer',
-            'server.2.socket_port': 9877,
-
-            # Test non-numeric 
-            # Also test default server.instance = builtin server
-            'server.yetanother.socket_port': 9878,
-        })
-    setup_server = staticmethod(setup_server)
-
-    PORT = 9876
-
-    def testBasicConfig(self):
-        self.getPage("/")
-        self.assertBody(str(self.PORT))
-
-    def testAdditionalServers(self):
-        if self.scheme == 'https':
-            return self.skip("not available under ssl")
-        self.PORT = 9877
-        self.getPage("/")
-        self.assertBody(str(self.PORT))
-        self.PORT = 9878
-        self.getPage("/")
-        self.assertBody(str(self.PORT))
-
-    def testMaxRequestSizePerHandler(self):
-        if getattr(cherrypy.server, "using_apache", False):
-            return self.skip("skipped due to known Apache differences... ")
-
-        self.getPage('/tinyupload', method="POST",
-                     headers=[('Content-Type', 'text/plain'),
-                              ('Content-Length', '100')],
-                     body="x" * 100)
-        self.assertStatus(200)
-        self.assertBody("x" * 100)
-
-        self.getPage('/tinyupload', method="POST",
-                     headers=[('Content-Type', 'text/plain'),
-                              ('Content-Length', '101')],
-                     body="x" * 101)
-        self.assertStatus(413)
-
-    def testMaxRequestSize(self):
-        if getattr(cherrypy.server, "using_apache", False):
-            return self.skip("skipped due to known Apache differences... ")
-
-        for size in (500, 5000, 50000):
-            self.getPage("/", headers=[('From', "x" * 500)])
-            self.assertStatus(413)
-
-        # Test for https://github.com/cherrypy/cherrypy/issues/421
-        # (Incorrect border condition in readline of SizeCheckWrapper).
-        # This hangs in rev 891 and earlier.
-        lines256 = "x" * 248
-        self.getPage("/",
-                     headers=[('Host', '%s:%s' % (self.HOST, self.PORT)),
-                              ('From', lines256)])
-
-        # Test upload
-        cd = (
-            'Content-Disposition: form-data; '
-            'name="file"; '
-            'filename="hello.txt"'
-        )
-        body = '\r\n'.join([
-            '--x',
-            cd,
-            'Content-Type: text/plain',
-            '',
-            '%s',
-            '--x--'])
-        partlen = 200 - len(body)
-        b = body % ("x" * partlen)
-        h = [("Content-type", "multipart/form-data; boundary=x"),
-             ("Content-Length", "%s" % len(b))]
-        self.getPage('/upload', h, "POST", b)
-        self.assertBody('Size: %d' % partlen)
-
-        b = body % ("x" * 200)
-        h = [("Content-type", "multipart/form-data; boundary=x"),
-             ("Content-Length", "%s" % len(b))]
-        self.getPage('/upload', h, "POST", b)
-        self.assertStatus(413)
diff --git a/lib/cherrypy/test/test_conn.py b/lib/cherrypy/test/test_conn.py
deleted file mode 100644
index d3dc3f210b0..00000000000
--- a/lib/cherrypy/test/test_conn.py
+++ /dev/null
@@ -1,864 +0,0 @@
-"""Tests for TCP connection handling, including proper and timely close."""
-
-import socket
-import sys
-import time
-import errno
-
-import six
-
-import cherrypy
-from cherrypy._cpcompat import HTTPConnection, HTTPSConnection, NotConnected
-from cherrypy._cpcompat import (
-    BadStatusLine,
-    ntob,
-    tonative,
-    urlopen,
-)
-from cherrypy.test import webtest
-
-
-timeout = 1
-pov = 'pPeErRsSiIsStTeEnNcCeE oOfF vViIsSiIoOnN'
-
-
-def setup_server():
-
-    def raise500():
-        raise cherrypy.HTTPError(500)
-
-    class Root:
-
-        @cherrypy.expose
-        def index(self):
-            return pov
-        page1 = index
-        page2 = index
-        page3 = index
-
-        @cherrypy.expose
-        def hello(self):
-            return "Hello, world!"
-
-        @cherrypy.expose
-        def timeout(self, t):
-            return str(cherrypy.server.httpserver.timeout)
-
-        @cherrypy.expose
-        @cherrypy.config(**{'response.stream': True})
-        def stream(self, set_cl=False):
-            if set_cl:
-                cherrypy.response.headers['Content-Length'] = 10
-
-            def content():
-                for x in range(10):
-                    yield str(x)
-
-            return content()
-
-        @cherrypy.expose
-        def error(self, code=500):
-            raise cherrypy.HTTPError(code)
-
-        @cherrypy.expose
-        def upload(self):
-            if not cherrypy.request.method == 'POST':
-                raise AssertionError("'POST' != request.method %r" %
-                                     cherrypy.request.method)
-            return "thanks for '%s'" % cherrypy.request.body.read()
-
-        @cherrypy.expose
-        def custom(self, response_code):
-            cherrypy.response.status = response_code
-            return "Code = %s" % response_code
-
-        @cherrypy.expose
-        @cherrypy.config(**{'hooks.on_start_resource': raise500})
-        def err_before_read(self):
-            return "ok"
-
-        @cherrypy.expose
-        def one_megabyte_of_a(self):
-            return ["a" * 1024] * 1024
-
-        @cherrypy.expose
-        # Turn off the encoding tool so it doens't collapse
-        # our response body and reclaculate the Content-Length.
-        @cherrypy.config(**{'tools.encode.on': False})
-        def custom_cl(self, body, cl):
-            cherrypy.response.headers['Content-Length'] = cl
-            if not isinstance(body, list):
-                body = [body]
-            newbody = []
-            for chunk in body:
-                if isinstance(chunk, six.text_type):
-                    chunk = chunk.encode('ISO-8859-1')
-                newbody.append(chunk)
-            return newbody
-
-    cherrypy.tree.mount(Root())
-    cherrypy.config.update({
-        'server.max_request_body_size': 1001,
-        'server.socket_timeout': timeout,
-    })
-
-
-from cherrypy.test import helper
-
-
-class ConnectionCloseTests(helper.CPWebCase):
-    setup_server = staticmethod(setup_server)
-
-    def test_HTTP11(self):
-        if cherrypy.server.protocol_version != "HTTP/1.1":
-            return self.skip()
-
-        self.PROTOCOL = "HTTP/1.1"
-
-        self.persistent = True
-
-        # Make the first request and assert there's no "Connection: close".
-        self.getPage("/")
-        self.assertStatus('200 OK')
-        self.assertBody(pov)
-        self.assertNoHeader("Connection")
-
-        # Make another request on the same connection.
-        self.getPage("/page1")
-        self.assertStatus('200 OK')
-        self.assertBody(pov)
-        self.assertNoHeader("Connection")
-
-        # Test client-side close.
-        self.getPage("/page2", headers=[("Connection", "close")])
-        self.assertStatus('200 OK')
-        self.assertBody(pov)
-        self.assertHeader("Connection", "close")
-
-        # Make another request on the same connection, which should error.
-        self.assertRaises(NotConnected, self.getPage, "/")
-
-    def test_Streaming_no_len(self):
-        try:
-            self._streaming(set_cl=False)
-        finally:
-            try:
-                self.HTTP_CONN.close()
-            except (TypeError, AttributeError):
-                pass
-
-    def test_Streaming_with_len(self):
-        try:
-            self._streaming(set_cl=True)
-        finally:
-            try:
-                self.HTTP_CONN.close()
-            except (TypeError, AttributeError):
-                pass
-
-    def _streaming(self, set_cl):
-        if cherrypy.server.protocol_version == "HTTP/1.1":
-            self.PROTOCOL = "HTTP/1.1"
-
-            self.persistent = True
-
-            # Make the first request and assert there's no "Connection: close".
-            self.getPage("/")
-            self.assertStatus('200 OK')
-            self.assertBody(pov)
-            self.assertNoHeader("Connection")
-
-            # Make another, streamed request on the same connection.
-            if set_cl:
-                # When a Content-Length is provided, the content should stream
-                # without closing the connection.
-                self.getPage("/stream?set_cl=Yes")
-                self.assertHeader("Content-Length")
-                self.assertNoHeader("Connection", "close")
-                self.assertNoHeader("Transfer-Encoding")
-
-                self.assertStatus('200 OK')
-                self.assertBody('0123456789')
-            else:
-                # When no Content-Length response header is provided,
-                # streamed output will either close the connection, or use
-                # chunked encoding, to determine transfer-length.
-                self.getPage("/stream")
-                self.assertNoHeader("Content-Length")
-                self.assertStatus('200 OK')
-                self.assertBody('0123456789')
-
-                chunked_response = False
-                for k, v in self.headers:
-                    if k.lower() == "transfer-encoding":
-                        if str(v) == "chunked":
-                            chunked_response = True
-
-                if chunked_response:
-                    self.assertNoHeader("Connection", "close")
-                else:
-                    self.assertHeader("Connection", "close")
-
-                    # Make another request on the same connection, which should
-                    # error.
-                    self.assertRaises(NotConnected, self.getPage, "/")
-
-                # Try HEAD. See
-                # https://github.com/cherrypy/cherrypy/issues/864.
-                self.getPage("/stream", method='HEAD')
-                self.assertStatus('200 OK')
-                self.assertBody('')
-                self.assertNoHeader("Transfer-Encoding")
-        else:
-            self.PROTOCOL = "HTTP/1.0"
-
-            self.persistent = True
-
-            # Make the first request and assert Keep-Alive.
-            self.getPage("/", headers=[("Connection", "Keep-Alive")])
-            self.assertStatus('200 OK')
-            self.assertBody(pov)
-            self.assertHeader("Connection", "Keep-Alive")
-
-            # Make another, streamed request on the same connection.
-            if set_cl:
-                # When a Content-Length is provided, the content should
-                # stream without closing the connection.
-                self.getPage("/stream?set_cl=Yes",
-                             headers=[("Connection", "Keep-Alive")])
-                self.assertHeader("Content-Length")
-                self.assertHeader("Connection", "Keep-Alive")
-                self.assertNoHeader("Transfer-Encoding")
-                self.assertStatus('200 OK')
-                self.assertBody('0123456789')
-            else:
-                # When a Content-Length is not provided,
-                # the server should close the connection.
-                self.getPage("/stream", headers=[("Connection", "Keep-Alive")])
-                self.assertStatus('200 OK')
-                self.assertBody('0123456789')
-
-                self.assertNoHeader("Content-Length")
-                self.assertNoHeader("Connection", "Keep-Alive")
-                self.assertNoHeader("Transfer-Encoding")
-
-                # Make another request on the same connection, which should
-                # error.
-                self.assertRaises(NotConnected, self.getPage, "/")
-
-    def test_HTTP10_KeepAlive(self):
-        self.PROTOCOL = "HTTP/1.0"
-        if self.scheme == "https":
-            self.HTTP_CONN = HTTPSConnection
-        else:
-            self.HTTP_CONN = HTTPConnection
-
-        # Test a normal HTTP/1.0 request.
-        self.getPage("/page2")
-        self.assertStatus('200 OK')
-        self.assertBody(pov)
-        # Apache, for example, may emit a Connection header even for HTTP/1.0
-# self.assertNoHeader("Connection")
-
-        # Test a keep-alive HTTP/1.0 request.
-        self.persistent = True
-
-        self.getPage("/page3", headers=[("Connection", "Keep-Alive")])
-        self.assertStatus('200 OK')
-        self.assertBody(pov)
-        self.assertHeader("Connection", "Keep-Alive")
-
-        # Remove the keep-alive header again.
-        self.getPage("/page3")
-        self.assertStatus('200 OK')
-        self.assertBody(pov)
-        # Apache, for example, may emit a Connection header even for HTTP/1.0
-# self.assertNoHeader("Connection")
-
-
-class PipelineTests(helper.CPWebCase):
-    setup_server = staticmethod(setup_server)
-
-    def test_HTTP11_Timeout(self):
-        # If we timeout without sending any data,
-        # the server will close the conn with a 408.
-        if cherrypy.server.protocol_version != "HTTP/1.1":
-            return self.skip()
-
-        self.PROTOCOL = "HTTP/1.1"
-
-        # Connect but send nothing.
-        self.persistent = True
-        conn = self.HTTP_CONN
-        conn.auto_open = False
-        conn.connect()
-
-        # Wait for our socket timeout
-        time.sleep(timeout * 2)
-
-        # The request should have returned 408 already.
-        response = conn.response_class(conn.sock, method="GET")
-        response.begin()
-        self.assertEqual(response.status, 408)
-        conn.close()
-
-        # Connect but send half the headers only.
-        self.persistent = True
-        conn = self.HTTP_CONN
-        conn.auto_open = False
-        conn.connect()
-        conn.send(ntob('GET /hello HTTP/1.1'))
-        conn.send(("Host: %s" % self.HOST).encode('ascii'))
-
-        # Wait for our socket timeout
-        time.sleep(timeout * 2)
-
-        # The conn should have already sent 408.
-        response = conn.response_class(conn.sock, method="GET")
-        response.begin()
-        self.assertEqual(response.status, 408)
-        conn.close()
-
-    def test_HTTP11_Timeout_after_request(self):
-        # If we timeout after at least one request has succeeded,
-        # the server will close the conn without 408.
-        if cherrypy.server.protocol_version != "HTTP/1.1":
-            return self.skip()
-
-        self.PROTOCOL = "HTTP/1.1"
-
-        # Make an initial request
-        self.persistent = True
-        conn = self.HTTP_CONN
-        conn.putrequest("GET", "/timeout?t=%s" % timeout, skip_host=True)
-        conn.putheader("Host", self.HOST)
-        conn.endheaders()
-        response = conn.response_class(conn.sock, method="GET")
-        response.begin()
-        self.assertEqual(response.status, 200)
-        self.body = response.read()
-        self.assertBody(str(timeout))
-
-        # Make a second request on the same socket
-        conn._output(ntob('GET /hello HTTP/1.1'))
-        conn._output(ntob("Host: %s" % self.HOST, 'ascii'))
-        conn._send_output()
-        response = conn.response_class(conn.sock, method="GET")
-        response.begin()
-        self.assertEqual(response.status, 200)
-        self.body = response.read()
-        self.assertBody("Hello, world!")
-
-        # Wait for our socket timeout
-        time.sleep(timeout * 2)
-
-        # Make another request on the same socket, which should error
-        conn._output(ntob('GET /hello HTTP/1.1'))
-        conn._output(ntob("Host: %s" % self.HOST, 'ascii'))
-        conn._send_output()
-        response = conn.response_class(conn.sock, method="GET")
-        try:
-            response.begin()
-        except:
-            if not isinstance(sys.exc_info()[1],
-                              (socket.error, BadStatusLine)):
-                self.fail("Writing to timed out socket didn't fail"
-                          " as it should have: %s" % sys.exc_info()[1])
-        else:
-            if response.status != 408:
-                self.fail("Writing to timed out socket didn't fail"
-                          " as it should have: %s" %
-                          response.read())
-
-        conn.close()
-
-        # Make another request on a new socket, which should work
-        self.persistent = True
-        conn = self.HTTP_CONN
-        conn.putrequest("GET", "/", skip_host=True)
-        conn.putheader("Host", self.HOST)
-        conn.endheaders()
-        response = conn.response_class(conn.sock, method="GET")
-        response.begin()
-        self.assertEqual(response.status, 200)
-        self.body = response.read()
-        self.assertBody(pov)
-
-        # Make another request on the same socket,
-        # but timeout on the headers
-        conn.send(ntob('GET /hello HTTP/1.1'))
-        # Wait for our socket timeout
-        time.sleep(timeout * 2)
-        response = conn.response_class(conn.sock, method="GET")
-        try:
-            response.begin()
-        except:
-            if not isinstance(sys.exc_info()[1],
-                              (socket.error, BadStatusLine)):
-                self.fail("Writing to timed out socket didn't fail"
-                          " as it should have: %s" % sys.exc_info()[1])
-        else:
-            self.fail("Writing to timed out socket didn't fail"
-                      " as it should have: %s" %
-                      response.read())
-
-        conn.close()
-
-        # Retry the request on a new connection, which should work
-        self.persistent = True
-        conn = self.HTTP_CONN
-        conn.putrequest("GET", "/", skip_host=True)
-        conn.putheader("Host", self.HOST)
-        conn.endheaders()
-        response = conn.response_class(conn.sock, method="GET")
-        response.begin()
-        self.assertEqual(response.status, 200)
-        self.body = response.read()
-        self.assertBody(pov)
-        conn.close()
-
-    def test_HTTP11_pipelining(self):
-        if cherrypy.server.protocol_version != "HTTP/1.1":
-            return self.skip()
-
-        self.PROTOCOL = "HTTP/1.1"
-
-        # Test pipelining. httplib doesn't support this directly.
-        self.persistent = True
-        conn = self.HTTP_CONN
-
-        # Put request 1
-        conn.putrequest("GET", "/hello", skip_host=True)
-        conn.putheader("Host", self.HOST)
-        conn.endheaders()
-
-        for trial in range(5):
-            # Put next request
-            conn._output(ntob('GET /hello HTTP/1.1'))
-            conn._output(ntob("Host: %s" % self.HOST, 'ascii'))
-            conn._send_output()
-
-            # Retrieve previous response
-            response = conn.response_class(conn.sock, method="GET")
-            # there is a bug in python3 regarding the buffering of
-            # ``conn.sock``. Until that bug get's fixed we will
-            # monkey patch the ``reponse`` instance.
-            # https://bugs.python.org/issue23377
-            if six.PY3:
-                response.fp = conn.sock.makefile("rb", 0)
-            response.begin()
-            body = response.read(13)
-            self.assertEqual(response.status, 200)
-            self.assertEqual(body, ntob("Hello, world!"))
-
-        # Retrieve final response
-        response = conn.response_class(conn.sock, method="GET")
-        response.begin()
-        body = response.read()
-        self.assertEqual(response.status, 200)
-        self.assertEqual(body, ntob("Hello, world!"))
-
-        conn.close()
-
-    def test_100_Continue(self):
-        if cherrypy.server.protocol_version != "HTTP/1.1":
-            return self.skip()
-
-        self.PROTOCOL = "HTTP/1.1"
-
-        self.persistent = True
-        conn = self.HTTP_CONN
-
-        # Try a page without an Expect request header first.
-        # Note that httplib's response.begin automatically ignores
-        # 100 Continue responses, so we must manually check for it.
-        try:
-            conn.putrequest("POST", "/upload", skip_host=True)
-            conn.putheader("Host", self.HOST)
-            conn.putheader("Content-Type", "text/plain")
-            conn.putheader("Content-Length", "4")
-            conn.endheaders()
-            conn.send(ntob("d'oh"))
-            response = conn.response_class(conn.sock, method="POST")
-            version, status, reason = response._read_status()
-            self.assertNotEqual(status, 100)
-        finally:
-            conn.close()
-
-        # Now try a page with an Expect header...
-        try:
-            conn.connect()
-            conn.putrequest("POST", "/upload", skip_host=True)
-            conn.putheader("Host", self.HOST)
-            conn.putheader("Content-Type", "text/plain")
-            conn.putheader("Content-Length", "17")
-            conn.putheader("Expect", "100-continue")
-            conn.endheaders()
-            response = conn.response_class(conn.sock, method="POST")
-
-            # ...assert and then skip the 100 response
-            version, status, reason = response._read_status()
-            self.assertEqual(status, 100)
-            while True:
-                line = response.fp.readline().strip()
-                if line:
-                    self.fail(
-                        "100 Continue should not output any headers. Got %r" %
-                        line)
-                else:
-                    break
-
-            # ...send the body
-            body = ntob("I am a small file")
-            conn.send(body)
-
-            # ...get the final response
-            response.begin()
-            self.status, self.headers, self.body = webtest.shb(response)
-            self.assertStatus(200)
-            self.assertBody("thanks for '%s'" % body)
-        finally:
-            conn.close()
-
-
-class ConnectionTests(helper.CPWebCase):
-    setup_server = staticmethod(setup_server)
-
-    def test_readall_or_close(self):
-        if cherrypy.server.protocol_version != "HTTP/1.1":
-            return self.skip()
-
-        self.PROTOCOL = "HTTP/1.1"
-
-        if self.scheme == "https":
-            self.HTTP_CONN = HTTPSConnection
-        else:
-            self.HTTP_CONN = HTTPConnection
-
-        # Test a max of 0 (the default) and then reset to what it was above.
-        old_max = cherrypy.server.max_request_body_size
-        for new_max in (0, old_max):
-            cherrypy.server.max_request_body_size = new_max
-
-            self.persistent = True
-            conn = self.HTTP_CONN
-
-            # Get a POST page with an error
-            conn.putrequest("POST", "/err_before_read", skip_host=True)
-            conn.putheader("Host", self.HOST)
-            conn.putheader("Content-Type", "text/plain")
-            conn.putheader("Content-Length", "1000")
-            conn.putheader("Expect", "100-continue")
-            conn.endheaders()
-            response = conn.response_class(conn.sock, method="POST")
-
-            # ...assert and then skip the 100 response
-            version, status, reason = response._read_status()
-            self.assertEqual(status, 100)
-            while True:
-                skip = response.fp.readline().strip()
-                if not skip:
-                    break
-
-            # ...send the body
-            conn.send(ntob("x" * 1000))
-
-            # ...get the final response
-            response.begin()
-            self.status, self.headers, self.body = webtest.shb(response)
-            self.assertStatus(500)
-
-            # Now try a working page with an Expect header...
-            conn._output(ntob('POST /upload HTTP/1.1'))
-            conn._output(ntob("Host: %s" % self.HOST, 'ascii'))
-            conn._output(ntob("Content-Type: text/plain"))
-            conn._output(ntob("Content-Length: 17"))
-            conn._output(ntob("Expect: 100-continue"))
-            conn._send_output()
-            response = conn.response_class(conn.sock, method="POST")
-
-            # ...assert and then skip the 100 response
-            version, status, reason = response._read_status()
-            self.assertEqual(status, 100)
-            while True:
-                skip = response.fp.readline().strip()
-                if not skip:
-                    break
-
-            # ...send the body
-            body = ntob("I am a small file")
-            conn.send(body)
-
-            # ...get the final response
-            response.begin()
-            self.status, self.headers, self.body = webtest.shb(response)
-            self.assertStatus(200)
-            self.assertBody("thanks for '%s'" % body)
-            conn.close()
-
-    def test_No_Message_Body(self):
-        if cherrypy.server.protocol_version != "HTTP/1.1":
-            return self.skip()
-
-        self.PROTOCOL = "HTTP/1.1"
-
-        # Set our HTTP_CONN to an instance so it persists between requests.
-        self.persistent = True
-
-        # Make the first request and assert there's no "Connection: close".
-        self.getPage("/")
-        self.assertStatus('200 OK')
-        self.assertBody(pov)
-        self.assertNoHeader("Connection")
-
-        # Make a 204 request on the same connection.
-        self.getPage("/custom/204")
-        self.assertStatus(204)
-        self.assertNoHeader("Content-Length")
-        self.assertBody("")
-        self.assertNoHeader("Connection")
-
-        # Make a 304 request on the same connection.
-        self.getPage("/custom/304")
-        self.assertStatus(304)
-        self.assertNoHeader("Content-Length")
-        self.assertBody("")
-        self.assertNoHeader("Connection")
-
-    def test_Chunked_Encoding(self):
-        if cherrypy.server.protocol_version != "HTTP/1.1":
-            return self.skip()
-
-        if (hasattr(self, 'harness') and
-                "modpython" in self.harness.__class__.__name__.lower()):
-            # mod_python forbids chunked encoding
-            return self.skip()
-
-        self.PROTOCOL = "HTTP/1.1"
-
-        # Set our HTTP_CONN to an instance so it persists between requests.
-        self.persistent = True
-        conn = self.HTTP_CONN
-
-        # Try a normal chunked request (with extensions)
-        body = ntob("8;key=value\r\nxx\r\nxxxx\r\n5\r\nyyyyy\r\n0\r\n"
-                    "Content-Type: application/json\r\n"
-                    "\r\n")
-        conn.putrequest("POST", "/upload", skip_host=True)
-        conn.putheader("Host", self.HOST)
-        conn.putheader("Transfer-Encoding", "chunked")
-        conn.putheader("Trailer", "Content-Type")
-        # Note that this is somewhat malformed:
-        # we shouldn't be sending Content-Length.
-        # RFC 2616 says the server should ignore it.
-        conn.putheader("Content-Length", "3")
-        conn.endheaders()
-        conn.send(body)
-        response = conn.getresponse()
-        self.status, self.headers, self.body = webtest.shb(response)
-        self.assertStatus('200 OK')
-        self.assertBody("thanks for '%s'" % ntob('xx\r\nxxxxyyyyy'))
-
-        # Try a chunked request that exceeds server.max_request_body_size.
-        # Note that the delimiters and trailer are included.
-        body = ntob("3e3\r\n" + ("x" * 995) + "\r\n0\r\n\r\n")
-        conn.putrequest("POST", "/upload", skip_host=True)
-        conn.putheader("Host", self.HOST)
-        conn.putheader("Transfer-Encoding", "chunked")
-        conn.putheader("Content-Type", "text/plain")
-        # Chunked requests don't need a content-length
-##        conn.putheader("Content-Length", len(body))
-        conn.endheaders()
-        conn.send(body)
-        response = conn.getresponse()
-        self.status, self.headers, self.body = webtest.shb(response)
-        self.assertStatus(413)
-        conn.close()
-
-    def test_Content_Length_in(self):
-        # Try a non-chunked request where Content-Length exceeds
-        # server.max_request_body_size. Assert error before body send.
-        self.persistent = True
-        conn = self.HTTP_CONN
-        conn.putrequest("POST", "/upload", skip_host=True)
-        conn.putheader("Host", self.HOST)
-        conn.putheader("Content-Type", "text/plain")
-        conn.putheader("Content-Length", "9999")
-        conn.endheaders()
-        response = conn.getresponse()
-        self.status, self.headers, self.body = webtest.shb(response)
-        self.assertStatus(413)
-        self.assertBody("The entity sent with the request exceeds "
-                        "the maximum allowed bytes.")
-        conn.close()
-
-    def test_Content_Length_out_preheaders(self):
-        # Try a non-chunked response where Content-Length is less than
-        # the actual bytes in the response body.
-        self.persistent = True
-        conn = self.HTTP_CONN
-        conn.putrequest("GET", "/custom_cl?body=I+have+too+many+bytes&cl=5",
-                        skip_host=True)
-        conn.putheader("Host", self.HOST)
-        conn.endheaders()
-        response = conn.getresponse()
-        self.status, self.headers, self.body = webtest.shb(response)
-        self.assertStatus(500)
-        self.assertBody(
-            "The requested resource returned more bytes than the "
-            "declared Content-Length.")
-        conn.close()
-
-    def test_Content_Length_out_postheaders(self):
-        # Try a non-chunked response where Content-Length is less than
-        # the actual bytes in the response body.
-        self.persistent = True
-        conn = self.HTTP_CONN
-        conn.putrequest(
-            "GET", "/custom_cl?body=I+too&body=+have+too+many&cl=5",
-            skip_host=True)
-        conn.putheader("Host", self.HOST)
-        conn.endheaders()
-        response = conn.getresponse()
-        self.status, self.headers, self.body = webtest.shb(response)
-        self.assertStatus(200)
-        self.assertBody("I too")
-        conn.close()
-
-    def test_598(self):
-        remote_data_conn = urlopen('%s://%s:%s/one_megabyte_of_a/' %
-                                   (self.scheme, self.HOST, self.PORT,))
-        buf = remote_data_conn.read(512)
-        time.sleep(timeout * 0.6)
-        remaining = (1024 * 1024) - 512
-        while remaining:
-            data = remote_data_conn.read(remaining)
-            if not data:
-                break
-            else:
-                buf += data
-            remaining -= len(data)
-
-        self.assertEqual(len(buf), 1024 * 1024)
-        self.assertEqual(buf, ntob("a" * 1024 * 1024))
-        self.assertEqual(remaining, 0)
-        remote_data_conn.close()
-
-
-def setup_upload_server():
-
-    class Root:
-        @cherrypy.expose
-        def upload(self):
-            if not cherrypy.request.method == 'POST':
-                raise AssertionError("'POST' != request.method %r" %
-                                     cherrypy.request.method)
-            return "thanks for '%s'" % tonative(cherrypy.request.body.read())
-
-    cherrypy.tree.mount(Root())
-    cherrypy.config.update({
-        'server.max_request_body_size': 1001,
-        'server.socket_timeout': 10,
-        'server.accepted_queue_size': 5,
-        'server.accepted_queue_timeout': 0.1,
-    })
-
-reset_names = 'ECONNRESET', 'WSAECONNRESET'
-socket_reset_errors = [
-    getattr(errno, name)
-    for name in reset_names
-    if hasattr(errno, name)
-]
-"reset error numbers available on this platform"
-
-socket_reset_errors += [
-    # Python 3.5 raises an http.client.RemoteDisconnected
-    # with this message
-    "Remote end closed connection without response",
-]
-
-
-class LimitedRequestQueueTests(helper.CPWebCase):
-    setup_server = staticmethod(setup_upload_server)
-
-    def test_queue_full(self):
-        conns = []
-        overflow_conn = None
-
-        try:
-            # Make 15 initial requests and leave them open, which should use
-            # all of wsgiserver's WorkerThreads and fill its Queue.
-            for i in range(15):
-                conn = self.HTTP_CONN(self.HOST, self.PORT)
-                conn.putrequest("POST", "/upload", skip_host=True)
-                conn.putheader("Host", self.HOST)
-                conn.putheader("Content-Type", "text/plain")
-                conn.putheader("Content-Length", "4")
-                conn.endheaders()
-                conns.append(conn)
-
-            # Now try a 16th conn, which should be closed by the server immediately.
-            overflow_conn = self.HTTP_CONN(self.HOST, self.PORT)
-            # Manually connect since httplib won't let us set a timeout
-            for res in socket.getaddrinfo(self.HOST, self.PORT, 0,
-                                          socket.SOCK_STREAM):
-                af, socktype, proto, canonname, sa = res
-                overflow_conn.sock = socket.socket(af, socktype, proto)
-                overflow_conn.sock.settimeout(5)
-                overflow_conn.sock.connect(sa)
-                break
-
-            overflow_conn.putrequest("GET", "/", skip_host=True)
-            overflow_conn.putheader("Host", self.HOST)
-            overflow_conn.endheaders()
-            response = overflow_conn.response_class(overflow_conn.sock, method="GET")
-            try:
-                response.begin()
-            except socket.error as exc:
-                if exc.args[0] in socket_reset_errors:
-                    pass # Expected.
-                else:
-                    tmpl = (
-                        "Overflow conn did not get RST. "
-                        "Got {exc.args!r} instead"
-                    )
-                    raise AssertionError(tmpl.format(**locals()))
-            except BadStatusLine:
-                # This is a special case in OS X. Linux and Windows will
-                # RST correctly.
-                assert sys.platform == 'darwin'
-            else:
-                raise AssertionError("Overflow conn did not get RST ")
-        finally:
-            for conn in conns:
-                conn.send(ntob("done"))
-                response = conn.response_class(conn.sock, method="POST")
-                response.begin()
-                self.body = response.read()
-                self.assertBody("thanks for 'done'")
-                self.assertEqual(response.status, 200)
-                conn.close()
-            if overflow_conn:
-                overflow_conn.close()
-
-class BadRequestTests(helper.CPWebCase):
-    setup_server = staticmethod(setup_server)
-
-    def test_No_CRLF(self):
-        self.persistent = True
-
-        conn = self.HTTP_CONN
-        conn.send(ntob('GET /hello HTTP/1.1\n\n'))
-        response = conn.response_class(conn.sock, method="GET")
-        response.begin()
-        self.body = response.read()
-        self.assertBody("HTTP requires CRLF terminators")
-        conn.close()
-
-        conn.connect()
-        conn.send(ntob('GET /hello HTTP/1.1\r\n\n'))
-        response = conn.response_class(conn.sock, method="GET")
-        response.begin()
-        self.body = response.read()
-        self.assertBody("HTTP requires CRLF terminators")
-        conn.close()
diff --git a/lib/cherrypy/test/test_core.py b/lib/cherrypy/test/test_core.py
deleted file mode 100644
index 83476a08c2b..00000000000
--- a/lib/cherrypy/test/test_core.py
+++ /dev/null
@@ -1,748 +0,0 @@
-# coding: utf-8
-
-"""Basic tests for the CherryPy core: request handling."""
-
-import os
-localDir = os.path.dirname(__file__)
-import sys
-import types
-
-import cherrypy
-from cherrypy._cpcompat import IncompleteRead, itervalues, ntob, ntou
-from cherrypy import _cptools, tools
-from cherrypy.lib import httputil, static
-from cherrypy.test._test_decorators import ExposeExamples
-
-
-favicon_path = os.path.join(os.getcwd(), localDir, "../favicon.ico")
-
-#                             Client-side code                             #
-
-from cherrypy.test import helper
-
-
-class CoreRequestHandlingTest(helper.CPWebCase):
-
-    def setup_server():
-        class Root:
-
-            @cherrypy.expose
-            def index(self):
-                return "hello"
-
-            favicon_ico = tools.staticfile.handler(filename=favicon_path)
-
-            @cherrypy.expose
-            def defct(self, newct):
-                newct = "text/%s" % newct
-                cherrypy.config.update({'tools.response_headers.on': True,
-                                        'tools.response_headers.headers':
-                                        [('Content-Type', newct)]})
-
-            @cherrypy.expose
-            def baseurl(self, path_info, relative=None):
-                return cherrypy.url(path_info, relative=bool(relative))
-
-        root = Root()
-        root.expose_dec = ExposeExamples()
-
-        class TestType(type):
-
-            """Metaclass which automatically exposes all functions in each
-            subclass, and adds an instance of the subclass as an attribute
-            of root.
-            """
-            def __init__(cls, name, bases, dct):
-                type.__init__(cls, name, bases, dct)
-                for value in itervalues(dct):
-                    if isinstance(value, types.FunctionType):
-                        value.exposed = True
-                setattr(root, name.lower(), cls())
-        Test = TestType('Test', (object, ), {})
-
-        @cherrypy.config(**{'tools.trailing_slash.on': False})
-        class URL(Test):
-
-            def index(self, path_info, relative=None):
-                if relative != 'server':
-                    relative = bool(relative)
-                return cherrypy.url(path_info, relative=relative)
-
-            def leaf(self, path_info, relative=None):
-                if relative != 'server':
-                    relative = bool(relative)
-                return cherrypy.url(path_info, relative=relative)
-
-        def log_status():
-            Status.statuses.append(cherrypy.response.status)
-        cherrypy.tools.log_status = cherrypy.Tool(
-            'on_end_resource', log_status)
-
-        class Status(Test):
-
-            def index(self):
-                return "normal"
-
-            def blank(self):
-                cherrypy.response.status = ""
-
-            # According to RFC 2616, new status codes are OK as long as they
-            # are between 100 and 599.
-
-            # Here is an illegal code...
-            def illegal(self):
-                cherrypy.response.status = 781
-                return "oops"
-
-            # ...and here is an unknown but legal code.
-            def unknown(self):
-                cherrypy.response.status = "431 My custom error"
-                return "funky"
-
-            # Non-numeric code
-            def bad(self):
-                cherrypy.response.status = "error"
-                return "bad news"
-
-            statuses = []
-
-            @cherrypy.config(**{'tools.log_status.on': True})
-            def on_end_resource_stage(self):
-                return repr(self.statuses)
-
-        class Redirect(Test):
-
-            @cherrypy.config(**{
-                "tools.err_redirect.on": True,
-                "tools.err_redirect.url": "/errpage",
-                "tools.err_redirect.internal": False,
-            })
-            class Error:
-                @cherrypy.expose
-                def index(self):
-                    raise NameError("redirect_test")
-
-            error = Error()
-
-            def index(self):
-                return "child"
-
-            def custom(self, url, code):
-                raise cherrypy.HTTPRedirect(url, code)
-
-            @cherrypy.config(**{'tools.trailing_slash.extra': True})
-            def by_code(self, code):
-                raise cherrypy.HTTPRedirect("somewhere%20else", code)
-
-            def nomodify(self):
-                raise cherrypy.HTTPRedirect("", 304)
-
-            def proxy(self):
-                raise cherrypy.HTTPRedirect("proxy", 305)
-
-            def stringify(self):
-                return str(cherrypy.HTTPRedirect("/"))
-
-            def fragment(self, frag):
-                raise cherrypy.HTTPRedirect("/some/url#%s" % frag)
-
-            def url_with_quote(self):
-                raise cherrypy.HTTPRedirect("/some\"url/that'we/want")
-
-            def url_with_unicode(self):
-                raise cherrypy.HTTPRedirect(ntou("тест", "utf-8"))
-
-        def login_redir():
-            if not getattr(cherrypy.request, "login", None):
-                raise cherrypy.InternalRedirect("/internalredirect/login")
-        tools.login_redir = _cptools.Tool('before_handler', login_redir)
-
-        def redir_custom():
-            raise cherrypy.InternalRedirect("/internalredirect/custom_err")
-
-        class InternalRedirect(Test):
-
-            def index(self):
-                raise cherrypy.InternalRedirect("/")
-
-            @cherrypy.expose
-            @cherrypy.config(**{'hooks.before_error_response': redir_custom})
-            def choke(self):
-                return 3 / 0
-
-            def relative(self, a, b):
-                raise cherrypy.InternalRedirect("cousin?t=6")
-
-            def cousin(self, t):
-                assert cherrypy.request.prev.closed
-                return cherrypy.request.prev.query_string
-
-            def petshop(self, user_id):
-                if user_id == "parrot":
-                    # Trade it for a slug when redirecting
-                    raise cherrypy.InternalRedirect(
-                        '/image/getImagesByUser?user_id=slug')
-                elif user_id == "terrier":
-                    # Trade it for a fish when redirecting
-                    raise cherrypy.InternalRedirect(
-                        '/image/getImagesByUser?user_id=fish')
-                else:
-                    # This should pass the user_id through to getImagesByUser
-                    raise cherrypy.InternalRedirect(
-                        '/image/getImagesByUser?user_id=%s' % str(user_id))
-
-            # We support Python 2.3, but the @-deco syntax would look like
-            # this:
-            # @tools.login_redir()
-            def secure(self):
-                return "Welcome!"
-            secure = tools.login_redir()(secure)
-            # Since calling the tool returns the same function you pass in,
-            # you could skip binding the return value, and just write:
-            # tools.login_redir()(secure)
-
-            def login(self):
-                return "Please log in"
-
-            def custom_err(self):
-                return "Something went horribly wrong."
-
-            @cherrypy.config(**{'hooks.before_request_body': redir_custom})
-            def early_ir(self, arg):
-                return "whatever"
-
-        class Image(Test):
-
-            def getImagesByUser(self, user_id):
-                return "0 images for %s" % user_id
-
-        class Flatten(Test):
-
-            def as_string(self):
-                return "content"
-
-            def as_list(self):
-                return ["con", "tent"]
-
-            def as_yield(self):
-                yield ntob("content")
-
-            @cherrypy.config(**{'tools.flatten.on': True})
-            def as_dblyield(self):
-                yield self.as_yield()
-
-            def as_refyield(self):
-                for chunk in self.as_yield():
-                    yield chunk
-
-        class Ranges(Test):
-
-            def get_ranges(self, bytes):
-                return repr(httputil.get_ranges('bytes=%s' % bytes, 8))
-
-            def slice_file(self):
-                path = os.path.join(os.getcwd(), os.path.dirname(__file__))
-                return static.serve_file(
-                    os.path.join(path, "static/index.html"))
-
-        class Cookies(Test):
-
-            def single(self, name):
-                cookie = cherrypy.request.cookie[name]
-                # Python2's SimpleCookie.__setitem__ won't take unicode keys.
-                cherrypy.response.cookie[str(name)] = cookie.value
-
-            def multiple(self, names):
-                list(map(self.single, names))
-
-        def append_headers(header_list, debug=False):
-            if debug:
-                cherrypy.log(
-                    "Extending response headers with %s" % repr(header_list),
-                    "TOOLS.APPEND_HEADERS")
-            cherrypy.serving.response.header_list.extend(header_list)
-        cherrypy.tools.append_headers = cherrypy.Tool(
-            'on_end_resource', append_headers)
-
-        class MultiHeader(Test):
-
-            def header_list(self):
-                pass
-            header_list = cherrypy.tools.append_headers(header_list=[
-                (ntob('WWW-Authenticate'), ntob('Negotiate')),
-                (ntob('WWW-Authenticate'), ntob('Basic realm="foo"')),
-            ])(header_list)
-
-            def commas(self):
-                cherrypy.response.headers[
-                    'WWW-Authenticate'] = 'Negotiate,Basic realm="foo"'
-
-        cherrypy.tree.mount(root)
-    setup_server = staticmethod(setup_server)
-
-    def testStatus(self):
-        self.getPage("/status/")
-        self.assertBody('normal')
-        self.assertStatus(200)
-
-        self.getPage("/status/blank")
-        self.assertBody('')
-        self.assertStatus(200)
-
-        self.getPage("/status/illegal")
-        self.assertStatus(500)
-        msg = "Illegal response status from server (781 is out of range)."
-        self.assertErrorPage(500, msg)
-
-        if not getattr(cherrypy.server, 'using_apache', False):
-            self.getPage("/status/unknown")
-            self.assertBody('funky')
-            self.assertStatus(431)
-
-        self.getPage("/status/bad")
-        self.assertStatus(500)
-        msg = "Illegal response status from server ('error' is non-numeric)."
-        self.assertErrorPage(500, msg)
-
-    def test_on_end_resource_status(self):
-        self.getPage('/status/on_end_resource_stage')
-        self.assertBody('[]')
-        self.getPage('/status/on_end_resource_stage')
-        self.assertBody(repr(["200 OK"]))
-
-    def testSlashes(self):
-        # Test that requests for index methods without a trailing slash
-        # get redirected to the same URI path with a trailing slash.
-        # Make sure GET params are preserved.
-        self.getPage("/redirect?id=3")
-        self.assertStatus(301)
-        self.assertMatchesBody(''
-                          "%s/redirect/[?]id=3" % (self.base(), self.base()))
-
-        if self.prefix():
-            # Corner case: the "trailing slash" redirect could be tricky if
-            # we're using a virtual root and the URI is "/vroot" (no slash).
-            self.getPage("")
-            self.assertStatus(301)
-            self.assertMatchesBody("%s/" %
-                              (self.base(), self.base()))
-
-        # Test that requests for NON-index methods WITH a trailing slash
-        # get redirected to the same URI path WITHOUT a trailing slash.
-        # Make sure GET params are preserved.
-        self.getPage("/redirect/by_code/?code=307")
-        self.assertStatus(301)
-        self.assertMatchesBody(""
-                          "%s/redirect/by_code[?]code=307"
-                          % (self.base(), self.base()))
-
-        # If the trailing_slash tool is off, CP should just continue
-        # as if the slashes were correct. But it needs some help
-        # inside cherrypy.url to form correct output.
-        self.getPage('/url?path_info=page1')
-        self.assertBody('%s/url/page1' % self.base())
-        self.getPage('/url/leaf/?path_info=page1')
-        self.assertBody('%s/url/page1' % self.base())
-
-    def testRedirect(self):
-        self.getPage("/redirect/")
-        self.assertBody('child')
-        self.assertStatus(200)
-
-        self.getPage("/redirect/by_code?code=300")
-        self.assertMatchesBody(
-            r"\2somewhere%20else")
-        self.assertStatus(300)
-
-        self.getPage("/redirect/by_code?code=301")
-        self.assertMatchesBody(
-            r"\2somewhere%20else")
-        self.assertStatus(301)
-
-        self.getPage("/redirect/by_code?code=302")
-        self.assertMatchesBody(
-            r"\2somewhere%20else")
-        self.assertStatus(302)
-
-        self.getPage("/redirect/by_code?code=303")
-        self.assertMatchesBody(
-            r"\2somewhere%20else")
-        self.assertStatus(303)
-
-        self.getPage("/redirect/by_code?code=307")
-        self.assertMatchesBody(
-            r"\2somewhere%20else")
-        self.assertStatus(307)
-
-        self.getPage("/redirect/nomodify")
-        self.assertBody('')
-        self.assertStatus(304)
-
-        self.getPage("/redirect/proxy")
-        self.assertBody('')
-        self.assertStatus(305)
-
-        # HTTPRedirect on error
-        self.getPage("/redirect/error/")
-        self.assertStatus(('302 Found', '303 See Other'))
-        self.assertInBody('/errpage')
-
-        # Make sure str(HTTPRedirect()) works.
-        self.getPage("/redirect/stringify", protocol="HTTP/1.0")
-        self.assertStatus(200)
-        self.assertBody("(['%s/'], 302)" % self.base())
-        if cherrypy.server.protocol_version == "HTTP/1.1":
-            self.getPage("/redirect/stringify", protocol="HTTP/1.1")
-            self.assertStatus(200)
-            self.assertBody("(['%s/'], 303)" % self.base())
-
-        # check that #fragments are handled properly
-        # http://skrb.org/ietf/http_errata.html#location-fragments
-        frag = "foo"
-        self.getPage("/redirect/fragment/%s" % frag)
-        self.assertMatchesBody(
-            r"\2\/some\/url\#%s" % (
-                frag, frag))
-        loc = self.assertHeader('Location')
-        assert loc.endswith("#%s" % frag)
-        self.assertStatus(('302 Found', '303 See Other'))
-
-        # check injection protection
-        # See https://github.com/cherrypy/cherrypy/issues/1003
-        self.getPage(
-            "/redirect/custom?"
-            "code=303&url=/foobar/%0d%0aSet-Cookie:%20somecookie=someval")
-        self.assertStatus(303)
-        loc = self.assertHeader('Location')
-        assert 'Set-Cookie' in loc
-        self.assertNoHeader('Set-Cookie')
-
-        def assertValidXHTML():
-            from xml.etree import ElementTree
-            try:
-                ElementTree.fromstring('%s' % self.body)
-            except ElementTree.ParseError as e:
-                self._handlewebError('automatically generated redirect '
-                    'did not generate well-formed html')
-
-        # check redirects to URLs generated valid HTML - we check this
-        # by seeing if it appears as valid XHTML.
-        self.getPage("/redirect/by_code?code=303")
-        self.assertStatus(303)
-        assertValidXHTML()
-
-        # do the same with a url containing quote characters.
-        self.getPage("/redirect/url_with_quote")
-        self.assertStatus(303)
-        assertValidXHTML()
-
-    def test_redirect_with_unicode(self):
-        """
-        A redirect to a URL with Unicode should return a Location
-        header containing that Unicode URL.
-        """
-        # test disabled due to #1440
-        return
-        self.getPage("/redirect/url_with_unicode")
-        self.assertStatus(303)
-        loc = self.assertHeader('Location')
-        assert ntou('тест', encoding='utf-8') in loc
-
-    def test_InternalRedirect(self):
-        # InternalRedirect
-        self.getPage("/internalredirect/")
-        self.assertBody('hello')
-        self.assertStatus(200)
-
-        # Test passthrough
-        self.getPage(
-            "/internalredirect/petshop?user_id=Sir-not-appearing-in-this-film")
-        self.assertBody('0 images for Sir-not-appearing-in-this-film')
-        self.assertStatus(200)
-
-        # Test args
-        self.getPage("/internalredirect/petshop?user_id=parrot")
-        self.assertBody('0 images for slug')
-        self.assertStatus(200)
-
-        # Test POST
-        self.getPage("/internalredirect/petshop", method="POST",
-                     body="user_id=terrier")
-        self.assertBody('0 images for fish')
-        self.assertStatus(200)
-
-        # Test ir before body read
-        self.getPage("/internalredirect/early_ir", method="POST",
-                     body="arg=aha!")
-        self.assertBody("Something went horribly wrong.")
-        self.assertStatus(200)
-
-        self.getPage("/internalredirect/secure")
-        self.assertBody('Please log in')
-        self.assertStatus(200)
-
-        # Relative path in InternalRedirect.
-        # Also tests request.prev.
-        self.getPage("/internalredirect/relative?a=3&b=5")
-        self.assertBody("a=3&b=5")
-        self.assertStatus(200)
-
-        # InternalRedirect on error
-        self.getPage("/internalredirect/choke")
-        self.assertStatus(200)
-        self.assertBody("Something went horribly wrong.")
-
-    def testFlatten(self):
-        for url in ["/flatten/as_string", "/flatten/as_list",
-                    "/flatten/as_yield", "/flatten/as_dblyield",
-                    "/flatten/as_refyield"]:
-            self.getPage(url)
-            self.assertBody('content')
-
-    def testRanges(self):
-        self.getPage("/ranges/get_ranges?bytes=3-6")
-        self.assertBody("[(3, 7)]")
-
-        # Test multiple ranges and a suffix-byte-range-spec, for good measure.
-        self.getPage("/ranges/get_ranges?bytes=2-4,-1")
-        self.assertBody("[(2, 5), (7, 8)]")
-
-        # Test a suffix-byte-range longer than the content
-        # length. Note that in this test, the content length
-        # is 8 bytes.
-        self.getPage("/ranges/get_ranges?bytes=-100")
-        self.assertBody("[(0, 8)]")
-
-        # Get a partial file.
-        if cherrypy.server.protocol_version == "HTTP/1.1":
-            self.getPage("/ranges/slice_file", [('Range', 'bytes=2-5')])
-            self.assertStatus(206)
-            self.assertHeader("Content-Type", "text/html;charset=utf-8")
-            self.assertHeader("Content-Range", "bytes 2-5/14")
-            self.assertBody("llo,")
-
-            # What happens with overlapping ranges (and out of order, too)?
-            self.getPage("/ranges/slice_file", [('Range', 'bytes=4-6,2-5')])
-            self.assertStatus(206)
-            ct = self.assertHeader("Content-Type")
-            expected_type = "multipart/byteranges; boundary="
-            self.assert_(ct.startswith(expected_type))
-            boundary = ct[len(expected_type):]
-            expected_body = ("\r\n--%s\r\n"
-                             "Content-type: text/html\r\n"
-                             "Content-range: bytes 4-6/14\r\n"
-                             "\r\n"
-                             "o, \r\n"
-                             "--%s\r\n"
-                             "Content-type: text/html\r\n"
-                             "Content-range: bytes 2-5/14\r\n"
-                             "\r\n"
-                             "llo,\r\n"
-                             "--%s--\r\n" % (boundary, boundary, boundary))
-            self.assertBody(expected_body)
-            self.assertHeader("Content-Length")
-
-            # Test "416 Requested Range Not Satisfiable"
-            self.getPage("/ranges/slice_file", [('Range', 'bytes=2300-2900')])
-            self.assertStatus(416)
-            # "When this status code is returned for a byte-range request,
-            # the response SHOULD include a Content-Range entity-header
-            # field specifying the current length of the selected resource"
-            self.assertHeader("Content-Range", "bytes */14")
-        elif cherrypy.server.protocol_version == "HTTP/1.0":
-            # Test Range behavior with HTTP/1.0 request
-            self.getPage("/ranges/slice_file", [('Range', 'bytes=2-5')])
-            self.assertStatus(200)
-            self.assertBody("Hello, world\r\n")
-
-    def testFavicon(self):
-        # favicon.ico is served by staticfile.
-        icofilename = os.path.join(localDir, "../favicon.ico")
-        icofile = open(icofilename, "rb")
-        data = icofile.read()
-        icofile.close()
-
-        self.getPage("/favicon.ico")
-        self.assertBody(data)
-
-    def skip_if_bad_cookies(self):
-        """
-        cookies module fails to reject invalid cookies
-        https://github.com/cherrypy/cherrypy/issues/1405
-        """
-        cookies = sys.modules.get('http.cookies')
-        _is_legal_key = getattr(cookies, '_is_legal_key', lambda x: False)
-        if not _is_legal_key(','):
-            return
-        issue = 'http://bugs.python.org/issue26302'
-        tmpl = "Broken cookies module ({issue})"
-        self.skip(tmpl.format(**locals()))
-
-    def testCookies(self):
-        self.skip_if_bad_cookies()
-
-        self.getPage("/cookies/single?name=First",
-                     [('Cookie', 'First=Dinsdale;')])
-        self.assertHeader('Set-Cookie', 'First=Dinsdale')
-
-        self.getPage("/cookies/multiple?names=First&names=Last",
-                     [('Cookie', 'First=Dinsdale; Last=Piranha;'),
-                      ])
-        self.assertHeader('Set-Cookie', 'First=Dinsdale')
-        self.assertHeader('Set-Cookie', 'Last=Piranha')
-
-        self.getPage("/cookies/single?name=Something-With%2CComma",
-                     [('Cookie', 'Something-With,Comma=some-value')])
-        self.assertStatus(400)
-
-    def testDefaultContentType(self):
-        self.getPage('/')
-        self.assertHeader('Content-Type', 'text/html;charset=utf-8')
-        self.getPage('/defct/plain')
-        self.getPage('/')
-        self.assertHeader('Content-Type', 'text/plain;charset=utf-8')
-        self.getPage('/defct/html')
-
-    def test_multiple_headers(self):
-        self.getPage('/multiheader/header_list')
-        self.assertEqual(
-            [(k, v) for k, v in self.headers if k == 'WWW-Authenticate'],
-            [('WWW-Authenticate', 'Negotiate'),
-             ('WWW-Authenticate', 'Basic realm="foo"'),
-             ])
-        self.getPage('/multiheader/commas')
-        self.assertHeader('WWW-Authenticate', 'Negotiate,Basic realm="foo"')
-
-    def test_cherrypy_url(self):
-        # Input relative to current
-        self.getPage('/url/leaf?path_info=page1')
-        self.assertBody('%s/url/page1' % self.base())
-        self.getPage('/url/?path_info=page1')
-        self.assertBody('%s/url/page1' % self.base())
-        # Other host header
-        host = 'www.mydomain.example'
-        self.getPage('/url/leaf?path_info=page1',
-                     headers=[('Host', host)])
-        self.assertBody('%s://%s/url/page1' % (self.scheme, host))
-
-        # Input is 'absolute'; that is, relative to script_name
-        self.getPage('/url/leaf?path_info=/page1')
-        self.assertBody('%s/page1' % self.base())
-        self.getPage('/url/?path_info=/page1')
-        self.assertBody('%s/page1' % self.base())
-
-        # Single dots
-        self.getPage('/url/leaf?path_info=./page1')
-        self.assertBody('%s/url/page1' % self.base())
-        self.getPage('/url/leaf?path_info=other/./page1')
-        self.assertBody('%s/url/other/page1' % self.base())
-        self.getPage('/url/?path_info=/other/./page1')
-        self.assertBody('%s/other/page1' % self.base())
-
-        # Double dots
-        self.getPage('/url/leaf?path_info=../page1')
-        self.assertBody('%s/page1' % self.base())
-        self.getPage('/url/leaf?path_info=other/../page1')
-        self.assertBody('%s/url/page1' % self.base())
-        self.getPage('/url/leaf?path_info=/other/../page1')
-        self.assertBody('%s/page1' % self.base())
-
-        # Output relative to current path or script_name
-        self.getPage('/url/?path_info=page1&relative=True')
-        self.assertBody('page1')
-        self.getPage('/url/leaf?path_info=/page1&relative=True')
-        self.assertBody('../page1')
-        self.getPage('/url/leaf?path_info=page1&relative=True')
-        self.assertBody('page1')
-        self.getPage('/url/leaf?path_info=leaf/page1&relative=True')
-        self.assertBody('leaf/page1')
-        self.getPage('/url/leaf?path_info=../page1&relative=True')
-        self.assertBody('../page1')
-        self.getPage('/url/?path_info=other/../page1&relative=True')
-        self.assertBody('page1')
-
-        # Output relative to /
-        self.getPage('/baseurl?path_info=ab&relative=True')
-        self.assertBody('ab')
-        # Output relative to /
-        self.getPage('/baseurl?path_info=/ab&relative=True')
-        self.assertBody('ab')
-
-        # absolute-path references ("server-relative")
-        # Input relative to current
-        self.getPage('/url/leaf?path_info=page1&relative=server')
-        self.assertBody('/url/page1')
-        self.getPage('/url/?path_info=page1&relative=server')
-        self.assertBody('/url/page1')
-        # Input is 'absolute'; that is, relative to script_name
-        self.getPage('/url/leaf?path_info=/page1&relative=server')
-        self.assertBody('/page1')
-        self.getPage('/url/?path_info=/page1&relative=server')
-        self.assertBody('/page1')
-
-    def test_expose_decorator(self):
-        # Test @expose
-        self.getPage("/expose_dec/no_call")
-        self.assertStatus(200)
-        self.assertBody("Mr E. R. Bradshaw")
-
-        # Test @expose()
-        self.getPage("/expose_dec/call_empty")
-        self.assertStatus(200)
-        self.assertBody("Mrs. B.J. Smegma")
-
-        # Test @expose("alias")
-        self.getPage("/expose_dec/call_alias")
-        self.assertStatus(200)
-        self.assertBody("Mr Nesbitt")
-        # Does the original name work?
-        self.getPage("/expose_dec/nesbitt")
-        self.assertStatus(200)
-        self.assertBody("Mr Nesbitt")
-
-        # Test @expose(["alias1", "alias2"])
-        self.getPage("/expose_dec/alias1")
-        self.assertStatus(200)
-        self.assertBody("Mr Ken Andrews")
-        self.getPage("/expose_dec/alias2")
-        self.assertStatus(200)
-        self.assertBody("Mr Ken Andrews")
-        # Does the original name work?
-        self.getPage("/expose_dec/andrews")
-        self.assertStatus(200)
-        self.assertBody("Mr Ken Andrews")
-
-        # Test @expose(alias="alias")
-        self.getPage("/expose_dec/alias3")
-        self.assertStatus(200)
-        self.assertBody("Mr. and Mrs. Watson")
-
-
-class ErrorTests(helper.CPWebCase):
-
-    def setup_server():
-        def break_header():
-            # Add a header after finalize that is invalid
-            cherrypy.serving.response.header_list.append((2, 3))
-        cherrypy.tools.break_header = cherrypy.Tool(
-            'on_end_resource', break_header)
-
-        class Root:
-
-            @cherrypy.expose
-            def index(self):
-                return "hello"
-
-            @cherrypy.config(**{'tools.break_header.on': True})
-            def start_response_error(self):
-                return "salud!"
-        root = Root()
-
-        cherrypy.tree.mount(root)
-    setup_server = staticmethod(setup_server)
-
-    def test_start_response_error(self):
-        self.getPage("/start_response_error")
-        self.assertStatus(500)
-        self.assertInBody(
-            "TypeError: response.header_list key 2 is not a byte string.")
diff --git a/lib/cherrypy/test/test_dynamicobjectmapping.py b/lib/cherrypy/test/test_dynamicobjectmapping.py
deleted file mode 100644
index 331c3f2e040..00000000000
--- a/lib/cherrypy/test/test_dynamicobjectmapping.py
+++ /dev/null
@@ -1,424 +0,0 @@
-import six
-
-import cherrypy
-from cherrypy._cpcompat import sorted
-from cherrypy.test import helper
-
-script_names = ["", "/foo", "/users/fred/blog", "/corp/blog"]
-
-
-def setup_server():
-    class SubSubRoot:
-
-        @cherrypy.expose
-        def index(self):
-            return "SubSubRoot index"
-
-        @cherrypy.expose
-        def default(self, *args):
-            return "SubSubRoot default"
-
-        @cherrypy.expose
-        def handler(self):
-            return "SubSubRoot handler"
-
-        @cherrypy.expose
-        def dispatch(self):
-            return "SubSubRoot dispatch"
-
-    subsubnodes = {
-        '1': SubSubRoot(),
-        '2': SubSubRoot(),
-    }
-
-    class SubRoot:
-
-        @cherrypy.expose
-        def index(self):
-            return "SubRoot index"
-
-        @cherrypy.expose
-        def default(self, *args):
-            return "SubRoot %s" % (args,)
-
-        @cherrypy.expose
-        def handler(self):
-            return "SubRoot handler"
-
-        def _cp_dispatch(self, vpath):
-            return subsubnodes.get(vpath[0], None)
-
-    subnodes = {
-        '1': SubRoot(),
-        '2': SubRoot(),
-    }
-
-    class Root:
-
-        @cherrypy.expose
-        def index(self):
-            return "index"
-
-        @cherrypy.expose
-        def default(self, *args):
-            return "default %s" % (args,)
-
-        @cherrypy.expose
-        def handler(self):
-            return "handler"
-
-        def _cp_dispatch(self, vpath):
-            return subnodes.get(vpath[0])
-
-    #--------------------------------------------------------------------------
-    # DynamicNodeAndMethodDispatcher example.
-    # This example exposes a fairly naive HTTP api
-    class User(object):
-
-        def __init__(self, id, name):
-            self.id = id
-            self.name = name
-
-        def __unicode__(self):
-            return unicode(self.name)
-
-        def __str__(self):
-            return str(self.name)
-
-    user_lookup = {
-        1: User(1, 'foo'),
-        2: User(2, 'bar'),
-    }
-
-    def make_user(name, id=None):
-        if not id:
-            id = max(*list(user_lookup.keys())) + 1
-        user_lookup[id] = User(id, name)
-        return id
-
-    @cherrypy.expose
-    class UserContainerNode(object):
-
-        def POST(self, name):
-            """
-            Allow the creation of a new Object
-            """
-            return "POST %d" % make_user(name)
-
-        def GET(self):
-            return six.text_type(sorted(user_lookup.keys()))
-
-        def dynamic_dispatch(self, vpath):
-            try:
-                id = int(vpath[0])
-            except (ValueError, IndexError):
-                return None
-            return UserInstanceNode(id)
-
-    @cherrypy.expose
-    class UserInstanceNode(object):
-
-        def __init__(self, id):
-            self.id = id
-            self.user = user_lookup.get(id, None)
-
-            # For all but PUT methods there MUST be a valid user identified
-            # by self.id
-            if not self.user and cherrypy.request.method != 'PUT':
-                raise cherrypy.HTTPError(404)
-
-        def GET(self, *args, **kwargs):
-            """
-            Return the appropriate representation of the instance.
-            """
-            return six.text_type(self.user)
-
-        def POST(self, name):
-            """
-            Update the fields of the user instance.
-            """
-            self.user.name = name
-            return "POST %d" % self.user.id
-
-        def PUT(self, name):
-            """
-            Create a new user with the specified id, or edit it if it already
-            exists
-            """
-            if self.user:
-                # Edit the current user
-                self.user.name = name
-                return "PUT %d" % self.user.id
-            else:
-                # Make a new user with said attributes.
-                return "PUT %d" % make_user(name, self.id)
-
-        def DELETE(self):
-            """
-            Delete the user specified at the id.
-            """
-            id = self.user.id
-            del user_lookup[self.user.id]
-            del self.user
-            return "DELETE %d" % id
-
-    class ABHandler:
-
-        class CustomDispatch:
-
-            @cherrypy.expose
-            def index(self, a, b):
-                return "custom"
-
-        def _cp_dispatch(self, vpath):
-            """Make sure that if we don't pop anything from vpath,
-            processing still works.
-            """
-            return self.CustomDispatch()
-
-        @cherrypy.expose
-        def index(self, a, b=None):
-            body = ['a:' + str(a)]
-            if b is not None:
-                body.append(',b:' + str(b))
-            return ''.join(body)
-
-        @cherrypy.expose
-        def delete(self, a, b):
-            return 'deleting ' + str(a) + ' and ' + str(b)
-
-    class IndexOnly:
-
-        def _cp_dispatch(self, vpath):
-            """Make sure that popping ALL of vpath still shows the index
-            handler.
-            """
-            while vpath:
-                vpath.pop()
-            return self
-
-        @cherrypy.expose
-        def index(self):
-            return "IndexOnly index"
-
-    class DecoratedPopArgs:
-
-        """Test _cp_dispatch with @cherrypy.popargs."""
-
-        @cherrypy.expose
-        def index(self):
-            return "no params"
-
-        @cherrypy.expose
-        def hi(self):
-            return "hi was not interpreted as 'a' param"
-    DecoratedPopArgs = cherrypy.popargs(
-        'a', 'b', handler=ABHandler())(DecoratedPopArgs)
-
-    class NonDecoratedPopArgs:
-
-        """Test _cp_dispatch = cherrypy.popargs()"""
-
-        _cp_dispatch = cherrypy.popargs('a')
-
-        @cherrypy.expose
-        def index(self, a):
-            return "index: " + str(a)
-
-    class ParameterizedHandler:
-
-        """Special handler created for each request"""
-
-        def __init__(self, a):
-            self.a = a
-
-        @cherrypy.expose
-        def index(self):
-            if 'a' in cherrypy.request.params:
-                raise Exception(
-                    "Parameterized handler argument ended up in "
-                    "request.params")
-            return self.a
-
-    class ParameterizedPopArgs:
-
-        """Test cherrypy.popargs() with a function call handler"""
-    ParameterizedPopArgs = cherrypy.popargs(
-        'a', handler=ParameterizedHandler)(ParameterizedPopArgs)
-
-    Root.decorated = DecoratedPopArgs()
-    Root.undecorated = NonDecoratedPopArgs()
-    Root.index_only = IndexOnly()
-    Root.parameter_test = ParameterizedPopArgs()
-
-    Root.users = UserContainerNode()
-
-    md = cherrypy.dispatch.MethodDispatcher('dynamic_dispatch')
-    for url in script_names:
-        conf = {'/': {
-            'user': (url or "/").split("/")[-2],
-        },
-            '/users': {
-                'request.dispatch': md
-            },
-        }
-        cherrypy.tree.mount(Root(), url, conf)
-
-
-class DynamicObjectMappingTest(helper.CPWebCase):
-    setup_server = staticmethod(setup_server)
-
-    def testObjectMapping(self):
-        for url in script_names:
-            prefix = self.script_name = url
-
-            self.getPage('/')
-            self.assertBody('index')
-
-            self.getPage('/handler')
-            self.assertBody('handler')
-
-            # Dynamic dispatch will succeed here for the subnodes
-            # so the subroot gets called
-            self.getPage('/1/')
-            self.assertBody('SubRoot index')
-
-            self.getPage('/2/')
-            self.assertBody('SubRoot index')
-
-            self.getPage('/1/handler')
-            self.assertBody('SubRoot handler')
-
-            self.getPage('/2/handler')
-            self.assertBody('SubRoot handler')
-
-            # Dynamic dispatch will fail here for the subnodes
-            # so the default gets called
-            self.getPage('/asdf/')
-            self.assertBody("default ('asdf',)")
-
-            self.getPage('/asdf/asdf')
-            self.assertBody("default ('asdf', 'asdf')")
-
-            self.getPage('/asdf/handler')
-            self.assertBody("default ('asdf', 'handler')")
-
-            # Dynamic dispatch will succeed here for the subsubnodes
-            # so the subsubroot gets called
-            self.getPage('/1/1/')
-            self.assertBody('SubSubRoot index')
-
-            self.getPage('/2/2/')
-            self.assertBody('SubSubRoot index')
-
-            self.getPage('/1/1/handler')
-            self.assertBody('SubSubRoot handler')
-
-            self.getPage('/2/2/handler')
-            self.assertBody('SubSubRoot handler')
-
-            self.getPage('/2/2/dispatch')
-            self.assertBody('SubSubRoot dispatch')
-
-            # The exposed dispatch will not be called as a dispatch
-            # method.
-            self.getPage('/2/2/foo/foo')
-            self.assertBody("SubSubRoot default")
-
-            # Dynamic dispatch will fail here for the subsubnodes
-            # so the SubRoot gets called
-            self.getPage('/1/asdf/')
-            self.assertBody("SubRoot ('asdf',)")
-
-            self.getPage('/1/asdf/asdf')
-            self.assertBody("SubRoot ('asdf', 'asdf')")
-
-            self.getPage('/1/asdf/handler')
-            self.assertBody("SubRoot ('asdf', 'handler')")
-
-    def testMethodDispatch(self):
-        # GET acts like a container
-        self.getPage("/users")
-        self.assertBody("[1, 2]")
-        self.assertHeader('Allow', 'GET, HEAD, POST')
-
-        # POST to the container URI allows creation
-        self.getPage("/users", method="POST", body="name=baz")
-        self.assertBody("POST 3")
-        self.assertHeader('Allow', 'GET, HEAD, POST')
-
-        # POST to a specific instanct URI results in a 404
-        # as the resource does not exit.
-        self.getPage("/users/5", method="POST", body="name=baz")
-        self.assertStatus(404)
-
-        # PUT to a specific instanct URI results in creation
-        self.getPage("/users/5", method="PUT", body="name=boris")
-        self.assertBody("PUT 5")
-        self.assertHeader('Allow', 'DELETE, GET, HEAD, POST, PUT')
-
-        # GET acts like a container
-        self.getPage("/users")
-        self.assertBody("[1, 2, 3, 5]")
-        self.assertHeader('Allow', 'GET, HEAD, POST')
-
-        test_cases = (
-            (1, 'foo', 'fooupdated', 'DELETE, GET, HEAD, POST, PUT'),
-            (2, 'bar', 'barupdated', 'DELETE, GET, HEAD, POST, PUT'),
-            (3, 'baz', 'bazupdated', 'DELETE, GET, HEAD, POST, PUT'),
-            (5, 'boris', 'borisupdated', 'DELETE, GET, HEAD, POST, PUT'),
-        )
-        for id, name, updatedname, headers in test_cases:
-            self.getPage("/users/%d" % id)
-            self.assertBody(name)
-            self.assertHeader('Allow', headers)
-
-            # Make sure POSTs update already existings resources
-            self.getPage("/users/%d" %
-                         id, method='POST', body="name=%s" % updatedname)
-            self.assertBody("POST %d" % id)
-            self.assertHeader('Allow', headers)
-
-            # Make sure PUTs Update already existing resources.
-            self.getPage("/users/%d" %
-                         id, method='PUT', body="name=%s" % updatedname)
-            self.assertBody("PUT %d" % id)
-            self.assertHeader('Allow', headers)
-
-            # Make sure DELETES Remove already existing resources.
-            self.getPage("/users/%d" % id, method='DELETE')
-            self.assertBody("DELETE %d" % id)
-            self.assertHeader('Allow', headers)
-
-        # GET acts like a container
-        self.getPage("/users")
-        self.assertBody("[]")
-        self.assertHeader('Allow', 'GET, HEAD, POST')
-
-    def testVpathDispatch(self):
-        self.getPage("/decorated/")
-        self.assertBody("no params")
-
-        self.getPage("/decorated/hi")
-        self.assertBody("hi was not interpreted as 'a' param")
-
-        self.getPage("/decorated/yo/")
-        self.assertBody("a:yo")
-
-        self.getPage("/decorated/yo/there/")
-        self.assertBody("a:yo,b:there")
-
-        self.getPage("/decorated/yo/there/delete")
-        self.assertBody("deleting yo and there")
-
-        self.getPage("/decorated/yo/there/handled_by_dispatch/")
-        self.assertBody("custom")
-
-        self.getPage("/undecorated/blah/")
-        self.assertBody("index: blah")
-
-        self.getPage("/index_only/a/b/c/d/e/f/g/")
-        self.assertBody("IndexOnly index")
-
-        self.getPage("/parameter_test/argument2/")
-        self.assertBody("argument2")
diff --git a/lib/cherrypy/test/test_encoding.py b/lib/cherrypy/test/test_encoding.py
deleted file mode 100644
index bc67b743f6c..00000000000
--- a/lib/cherrypy/test/test_encoding.py
+++ /dev/null
@@ -1,415 +0,0 @@
-# coding: utf-8
-
-import gzip
-import io
-
-import six
-
-import mock
-
-import cherrypy
-from cherrypy._cpcompat import IncompleteRead, ntob, ntou
-
-europoundUnicode = ntou('£', encoding='utf-8')
-sing = ntou("毛泽东: Sing, Little Birdie?", encoding='utf-8')
-
-sing8 = sing.encode('utf-8')
-sing16 = sing.encode('utf-16')
-
-
-from cherrypy.test import helper
-
-
-class EncodingTests(helper.CPWebCase):
-
-    def setup_server():
-        class Root:
-
-            @cherrypy.expose
-            def index(self, param):
-                assert param == europoundUnicode, "%r != %r" % (
-                    param, europoundUnicode)
-                yield europoundUnicode
-
-            @cherrypy.expose
-            def mao_zedong(self):
-                return sing
-
-            @cherrypy.expose
-            @cherrypy.config(**{'tools.encode.encoding': 'utf-8'})
-            def utf8(self):
-                return sing8
-
-            @cherrypy.expose
-            def cookies_and_headers(self):
-                # if the headers have non-ascii characters and a cookie has
-                #  any part which is unicode (even ascii), the response
-                #  should not fail.
-                cherrypy.response.cookie['candy'] = 'bar'
-                cherrypy.response.cookie['candy']['domain'] = 'cherrypy.org'
-                cherrypy.response.headers[
-                    'Some-Header'] = 'My d\xc3\xb6g has fleas'
-                return 'Any content'
-
-            @cherrypy.expose
-            def reqparams(self, *args, **kwargs):
-                return ntob(', ').join(
-                    [": ".join((k, v)).encode('utf8')
-                     for k, v in sorted(cherrypy.request.params.items())]
-                )
-
-            @cherrypy.expose
-            @cherrypy.config(**{
-                'tools.encode.text_only': False,
-                'tools.encode.add_charset': True,
-            })
-            def nontext(self, *args, **kwargs):
-                cherrypy.response.headers[
-                    'Content-Type'] = 'application/binary'
-                return '\x00\x01\x02\x03'
-
-        class GZIP:
-
-            @cherrypy.expose
-            def index(self):
-                yield "Hello, world"
-
-            @cherrypy.expose
-            # Turn encoding off so the gzip tool is the one doing the collapse.
-            @cherrypy.config(**{'tools.encode.on': False})
-            def noshow(self):
-                # Test for ticket #147, where yield showed no exceptions
-                # (content-encoding was still gzip even though traceback
-                # wasn't zipped).
-                raise IndexError()
-                yield "Here be dragons"
-
-            @cherrypy.expose
-            @cherrypy.config(**{'response.stream': True})
-            def noshow_stream(self):
-                # Test for ticket #147, where yield showed no exceptions
-                # (content-encoding was still gzip even though traceback
-                # wasn't zipped).
-                raise IndexError()
-                yield "Here be dragons"
-
-        class Decode:
-
-            @cherrypy.expose
-            @cherrypy.config(**{
-                'tools.decode.on': True,
-                'tools.decode.default_encoding': ['utf-16'],
-            })
-            def extra_charset(self, *args, **kwargs):
-                return ', '.join([": ".join((k, v))
-                                  for k, v in cherrypy.request.params.items()])
-
-            @cherrypy.expose
-            @cherrypy.config(**{
-                'tools.decode.on': True,
-                'tools.decode.encoding': 'utf-16',
-            })
-            def force_charset(self, *args, **kwargs):
-                return ', '.join([": ".join((k, v))
-                                  for k, v in cherrypy.request.params.items()])
-
-        root = Root()
-        root.gzip = GZIP()
-        root.decode = Decode()
-        cherrypy.tree.mount(root, config={'/gzip': {'tools.gzip.on': True}})
-    setup_server = staticmethod(setup_server)
-
-    def test_query_string_decoding(self):
-        if six.PY3:
-            # This test fails on Python 3. See #1443
-            return
-        europoundUtf8 = europoundUnicode.encode('utf-8')
-        self.getPage(ntob('/?param=') + europoundUtf8)
-        self.assertBody(europoundUtf8)
-
-        # Encoded utf8 query strings MUST be parsed correctly.
-        # Here, q is the POUND SIGN U+00A3 encoded in utf8 and then %HEX
-        self.getPage("/reqparams?q=%C2%A3")
-        # The return value will be encoded as utf8.
-        self.assertBody(ntob("q: \xc2\xa3"))
-
-        # Query strings that are incorrectly encoded MUST raise 404.
-        # Here, q is the POUND SIGN U+00A3 encoded in latin1 and then %HEX
-        self.getPage("/reqparams?q=%A3")
-        self.assertStatus(404)
-        self.assertErrorPage(
-            404,
-            "The given query string could not be processed. Query "
-            "strings for this resource must be encoded with 'utf8'.")
-
-    def test_urlencoded_decoding(self):
-        # Test the decoding of an application/x-www-form-urlencoded entity.
-        europoundUtf8 = europoundUnicode.encode('utf-8')
-        body = ntob("param=") + europoundUtf8
-        self.getPage('/',
-                     method='POST',
-                     headers=[
-                         ("Content-Type", "application/x-www-form-urlencoded"),
-                         ("Content-Length", str(len(body))),
-                     ],
-                     body=body),
-        self.assertBody(europoundUtf8)
-
-        # Encoded utf8 entities MUST be parsed and decoded correctly.
-        # Here, q is the POUND SIGN U+00A3 encoded in utf8
-        body = ntob("q=\xc2\xa3")
-        self.getPage('/reqparams', method='POST',
-                     headers=[(
-                         "Content-Type", "application/x-www-form-urlencoded"),
-                         ("Content-Length", str(len(body))),
-                     ],
-                     body=body),
-        self.assertBody(ntob("q: \xc2\xa3"))
-
-        # ...and in utf16, which is not in the default attempt_charsets list:
-        body = ntob("\xff\xfeq\x00=\xff\xfe\xa3\x00")
-        self.getPage('/reqparams',
-                     method='POST',
-                     headers=[
-                         ("Content-Type",
-                          "application/x-www-form-urlencoded;charset=utf-16"),
-                         ("Content-Length", str(len(body))),
-                     ],
-                     body=body),
-        self.assertBody(ntob("q: \xc2\xa3"))
-
-        # Entities that are incorrectly encoded MUST raise 400.
-        # Here, q is the POUND SIGN U+00A3 encoded in utf16, but
-        # the Content-Type incorrectly labels it utf-8.
-        body = ntob("\xff\xfeq\x00=\xff\xfe\xa3\x00")
-        self.getPage('/reqparams',
-                     method='POST',
-                     headers=[
-                         ("Content-Type",
-                          "application/x-www-form-urlencoded;charset=utf-8"),
-                         ("Content-Length", str(len(body))),
-                     ],
-                     body=body),
-        self.assertStatus(400)
-        self.assertErrorPage(
-            400,
-            "The request entity could not be decoded. The following charsets "
-            "were attempted: ['utf-8']")
-
-    def test_decode_tool(self):
-        # An extra charset should be tried first, and succeed if it matches.
-        # Here, we add utf-16 as a charset and pass a utf-16 body.
-        body = ntob("\xff\xfeq\x00=\xff\xfe\xa3\x00")
-        self.getPage('/decode/extra_charset', method='POST',
-                     headers=[(
-                         "Content-Type", "application/x-www-form-urlencoded"),
-                         ("Content-Length", str(len(body))),
-                     ],
-                     body=body),
-        self.assertBody(ntob("q: \xc2\xa3"))
-
-        # An extra charset should be tried first, and continue to other default
-        # charsets if it doesn't match.
-        # Here, we add utf-16 as a charset but still pass a utf-8 body.
-        body = ntob("q=\xc2\xa3")
-        self.getPage('/decode/extra_charset', method='POST',
-                     headers=[(
-                         "Content-Type", "application/x-www-form-urlencoded"),
-                         ("Content-Length", str(len(body))),
-                     ],
-                     body=body),
-        self.assertBody(ntob("q: \xc2\xa3"))
-
-        # An extra charset should error if force is True and it doesn't match.
-        # Here, we force utf-16 as a charset but still pass a utf-8 body.
-        body = ntob("q=\xc2\xa3")
-        self.getPage('/decode/force_charset', method='POST',
-                     headers=[(
-                         "Content-Type", "application/x-www-form-urlencoded"),
-                         ("Content-Length", str(len(body))),
-                     ],
-                     body=body),
-        self.assertErrorPage(
-            400,
-            "The request entity could not be decoded. The following charsets "
-            "were attempted: ['utf-16']")
-
-    def test_multipart_decoding(self):
-        # Test the decoding of a multipart entity when the charset (utf16) is
-        # explicitly given.
-        body = ntob('\r\n'.join([
-            '--X',
-            'Content-Type: text/plain;charset=utf-16',
-            'Content-Disposition: form-data; name="text"',
-            '',
-            '\xff\xfea\x00b\x00\x1c c\x00',
-            '--X',
-            'Content-Type: text/plain;charset=utf-16',
-            'Content-Disposition: form-data; name="submit"',
-            '',
-            '\xff\xfeC\x00r\x00e\x00a\x00t\x00e\x00',
-            '--X--'
-        ]))
-        self.getPage('/reqparams', method='POST',
-                     headers=[(
-                         "Content-Type", "multipart/form-data;boundary=X"),
-                         ("Content-Length", str(len(body))),
-                     ],
-                     body=body),
-        self.assertBody(ntob("submit: Create, text: ab\xe2\x80\x9cc"))
-
-    @mock.patch('cherrypy._cpreqbody.Part.maxrambytes', 1)
-    def test_multipart_decoding_bigger_maxrambytes(self):
-        """
-        Decoding of a multipart entity should also pass when
-        the entity is bigger than maxrambytes. See ticket #1352.
-        """
-        self.test_multipart_decoding()
-
-    def test_multipart_decoding_no_charset(self):
-        # Test the decoding of a multipart entity when the charset (utf8) is
-        # NOT explicitly given, but is in the list of charsets to attempt.
-        body = ntob('\r\n'.join([
-            '--X',
-            'Content-Disposition: form-data; name="text"',
-            '',
-            '\xe2\x80\x9c',
-            '--X',
-            'Content-Disposition: form-data; name="submit"',
-            '',
-            'Create',
-            '--X--'
-        ]))
-        self.getPage('/reqparams', method='POST',
-                     headers=[(
-                         "Content-Type", "multipart/form-data;boundary=X"),
-                         ("Content-Length", str(len(body))),
-                     ],
-                     body=body),
-        self.assertBody(ntob("submit: Create, text: \xe2\x80\x9c"))
-
-    def test_multipart_decoding_no_successful_charset(self):
-        # Test the decoding of a multipart entity when the charset (utf16) is
-        # NOT explicitly given, and is NOT in the list of charsets to attempt.
-        body = ntob('\r\n'.join([
-            '--X',
-            'Content-Disposition: form-data; name="text"',
-            '',
-            '\xff\xfea\x00b\x00\x1c c\x00',
-            '--X',
-            'Content-Disposition: form-data; name="submit"',
-            '',
-            '\xff\xfeC\x00r\x00e\x00a\x00t\x00e\x00',
-            '--X--'
-        ]))
-        self.getPage('/reqparams', method='POST',
-                     headers=[(
-                         "Content-Type", "multipart/form-data;boundary=X"),
-                         ("Content-Length", str(len(body))),
-                     ],
-                     body=body),
-        self.assertStatus(400)
-        self.assertErrorPage(
-            400,
-            "The request entity could not be decoded. The following charsets "
-            "were attempted: ['us-ascii', 'utf-8']")
-
-    def test_nontext(self):
-        self.getPage('/nontext')
-        self.assertHeader('Content-Type', 'application/binary;charset=utf-8')
-        self.assertBody('\x00\x01\x02\x03')
-
-    def testEncoding(self):
-        # Default encoding should be utf-8
-        self.getPage('/mao_zedong')
-        self.assertBody(sing8)
-
-        # Ask for utf-16.
-        self.getPage('/mao_zedong', [('Accept-Charset', 'utf-16')])
-        self.assertHeader('Content-Type', 'text/html;charset=utf-16')
-        self.assertBody(sing16)
-
-        # Ask for multiple encodings. ISO-8859-1 should fail, and utf-16
-        # should be produced.
-        self.getPage('/mao_zedong', [('Accept-Charset',
-                                      'iso-8859-1;q=1, utf-16;q=0.5')])
-        self.assertBody(sing16)
-
-        # The "*" value should default to our default_encoding, utf-8
-        self.getPage('/mao_zedong', [('Accept-Charset', '*;q=1, utf-7;q=.2')])
-        self.assertBody(sing8)
-
-        # Only allow iso-8859-1, which should fail and raise 406.
-        self.getPage('/mao_zedong', [('Accept-Charset', 'iso-8859-1, *;q=0')])
-        self.assertStatus("406 Not Acceptable")
-        self.assertInBody("Your client sent this Accept-Charset header: "
-                          "iso-8859-1, *;q=0. We tried these charsets: "
-                          "iso-8859-1.")
-
-        # Ask for x-mac-ce, which should be unknown. See ticket #569.
-        self.getPage('/mao_zedong', [('Accept-Charset',
-                                      'us-ascii, ISO-8859-1, x-mac-ce')])
-        self.assertStatus("406 Not Acceptable")
-        self.assertInBody("Your client sent this Accept-Charset header: "
-                          "us-ascii, ISO-8859-1, x-mac-ce. We tried these "
-                          "charsets: ISO-8859-1, us-ascii, x-mac-ce.")
-
-        # Test the 'encoding' arg to encode.
-        self.getPage('/utf8')
-        self.assertBody(sing8)
-        self.getPage('/utf8', [('Accept-Charset', 'us-ascii, ISO-8859-1')])
-        self.assertStatus("406 Not Acceptable")
-
-    def testGzip(self):
-        zbuf = io.BytesIO()
-        zfile = gzip.GzipFile(mode='wb', fileobj=zbuf, compresslevel=9)
-        zfile.write(ntob("Hello, world"))
-        zfile.close()
-
-        self.getPage('/gzip/', headers=[("Accept-Encoding", "gzip")])
-        self.assertInBody(zbuf.getvalue()[:3])
-        self.assertHeader("Vary", "Accept-Encoding")
-        self.assertHeader("Content-Encoding", "gzip")
-
-        # Test when gzip is denied.
-        self.getPage('/gzip/', headers=[("Accept-Encoding", "identity")])
-        self.assertHeader("Vary", "Accept-Encoding")
-        self.assertNoHeader("Content-Encoding")
-        self.assertBody("Hello, world")
-
-        self.getPage('/gzip/', headers=[("Accept-Encoding", "gzip;q=0")])
-        self.assertHeader("Vary", "Accept-Encoding")
-        self.assertNoHeader("Content-Encoding")
-        self.assertBody("Hello, world")
-
-        self.getPage('/gzip/', headers=[("Accept-Encoding", "*;q=0")])
-        self.assertStatus(406)
-        self.assertNoHeader("Content-Encoding")
-        self.assertErrorPage(406, "identity, gzip")
-
-        # Test for ticket #147
-        self.getPage('/gzip/noshow', headers=[("Accept-Encoding", "gzip")])
-        self.assertNoHeader('Content-Encoding')
-        self.assertStatus(500)
-        self.assertErrorPage(500, pattern="IndexError\n")
-
-        # In this case, there's nothing we can do to deliver a
-        # readable page, since 1) the gzip header is already set,
-        # and 2) we may have already written some of the body.
-        # The fix is to never stream yields when using gzip.
-        if (cherrypy.server.protocol_version == "HTTP/1.0" or
-                getattr(cherrypy.server, "using_apache", False)):
-            self.getPage('/gzip/noshow_stream',
-                         headers=[("Accept-Encoding", "gzip")])
-            self.assertHeader('Content-Encoding', 'gzip')
-            self.assertInBody('\x1f\x8b\x08\x00')
-        else:
-            # The wsgiserver will simply stop sending data, and the HTTP client
-            # will error due to an incomplete chunk-encoded stream.
-            self.assertRaises((ValueError, IncompleteRead), self.getPage,
-                              '/gzip/noshow_stream',
-                              headers=[("Accept-Encoding", "gzip")])
-
-    def test_UnicodeHeaders(self):
-        self.getPage('/cookies_and_headers')
-        self.assertBody('Any content')
diff --git a/lib/cherrypy/test/test_etags.py b/lib/cherrypy/test/test_etags.py
deleted file mode 100644
index 5df8d06d222..00000000000
--- a/lib/cherrypy/test/test_etags.py
+++ /dev/null
@@ -1,84 +0,0 @@
-import cherrypy
-from cherrypy._cpcompat import ntou
-from cherrypy.test import helper
-
-
-class ETagTest(helper.CPWebCase):
-
-    def setup_server():
-        class Root:
-
-            @cherrypy.expose
-            def resource(self):
-                return "Oh wah ta goo Siam."
-
-            @cherrypy.expose
-            def fail(self, code):
-                code = int(code)
-                if 300 <= code <= 399:
-                    raise cherrypy.HTTPRedirect([], code)
-                else:
-                    raise cherrypy.HTTPError(code)
-
-            @cherrypy.expose
-            # In Python 3, tools.encode is on by default
-            @cherrypy.config(**{'tools.encode.on': True})
-            def unicoded(self):
-                return ntou('I am a \u1ee4nicode string.', 'escape')
-
-        conf = {'/': {'tools.etags.on': True,
-                      'tools.etags.autotags': True,
-                      }}
-        cherrypy.tree.mount(Root(), config=conf)
-    setup_server = staticmethod(setup_server)
-
-    def test_etags(self):
-        self.getPage("/resource")
-        self.assertStatus('200 OK')
-        self.assertHeader('Content-Type', 'text/html;charset=utf-8')
-        self.assertBody('Oh wah ta goo Siam.')
-        etag = self.assertHeader('ETag')
-
-        # Test If-Match (both valid and invalid)
-        self.getPage("/resource", headers=[('If-Match', etag)])
-        self.assertStatus("200 OK")
-        self.getPage("/resource", headers=[('If-Match', "*")])
-        self.assertStatus("200 OK")
-        self.getPage("/resource", headers=[('If-Match', "*")], method="POST")
-        self.assertStatus("200 OK")
-        self.getPage("/resource", headers=[('If-Match', "a bogus tag")])
-        self.assertStatus("412 Precondition Failed")
-
-        # Test If-None-Match (both valid and invalid)
-        self.getPage("/resource", headers=[('If-None-Match', etag)])
-        self.assertStatus(304)
-        self.getPage("/resource", method='POST',
-                     headers=[('If-None-Match', etag)])
-        self.assertStatus("412 Precondition Failed")
-        self.getPage("/resource", headers=[('If-None-Match', "*")])
-        self.assertStatus(304)
-        self.getPage("/resource", headers=[('If-None-Match', "a bogus tag")])
-        self.assertStatus("200 OK")
-
-    def test_errors(self):
-        self.getPage("/resource")
-        self.assertStatus(200)
-        etag = self.assertHeader('ETag')
-
-        # Test raising errors in page handler
-        self.getPage("/fail/412", headers=[('If-Match', etag)])
-        self.assertStatus(412)
-        self.getPage("/fail/304", headers=[('If-Match', etag)])
-        self.assertStatus(304)
-        self.getPage("/fail/412", headers=[('If-None-Match', "*")])
-        self.assertStatus(412)
-        self.getPage("/fail/304", headers=[('If-None-Match', "*")])
-        self.assertStatus(304)
-
-    def test_unicode_body(self):
-        self.getPage("/unicoded")
-        self.assertStatus(200)
-        etag1 = self.assertHeader('ETag')
-        self.getPage("/unicoded", headers=[('If-Match', etag1)])
-        self.assertStatus(200)
-        self.assertHeader('ETag', etag1)
diff --git a/lib/cherrypy/test/test_http.py b/lib/cherrypy/test/test_http.py
deleted file mode 100644
index c5bbcaacb6f..00000000000
--- a/lib/cherrypy/test/test_http.py
+++ /dev/null
@@ -1,256 +0,0 @@
-"""Tests for managing HTTP issues (malformed requests, etc)."""
-
-import errno
-import mimetypes
-import socket
-import sys
-
-import six
-
-from mock import patch
-
-import cherrypy
-from cherrypy._cpcompat import HTTPConnection, HTTPSConnection, ntob
-
-
-def encode_multipart_formdata(files):
-    """Return (content_type, body) ready for httplib.HTTP instance.
-
-    files: a sequence of (name, filename, value) tuples for multipart uploads.
-    """
-    BOUNDARY = '________ThIs_Is_tHe_bouNdaRY_$'
-    L = []
-    for key, filename, value in files:
-        L.append('--' + BOUNDARY)
-        L.append('Content-Disposition: form-data; name="%s"; filename="%s"' %
-                 (key, filename))
-        ct = mimetypes.guess_type(filename)[0] or 'application/octet-stream'
-        L.append('Content-Type: %s' % ct)
-        L.append('')
-        L.append(value)
-    L.append('--' + BOUNDARY + '--')
-    L.append('')
-    body = '\r\n'.join(L)
-    content_type = 'multipart/form-data; boundary=%s' % BOUNDARY
-    return content_type, body
-
-
-from cherrypy.test import helper
-
-
-class HTTPTests(helper.CPWebCase):
-
-    def make_connection(self):
-        if self.scheme == "https":
-            return HTTPSConnection('%s:%s' % (self.interface(), self.PORT))
-        else:
-            return HTTPConnection('%s:%s' % (self.interface(), self.PORT))
-
-    def setup_server():
-        class Root:
-
-            @cherrypy.expose
-            def index(self, *args, **kwargs):
-                return "Hello world!"
-
-            @cherrypy.expose
-            @cherrypy.config(**{'request.process_request_body': False})
-            def no_body(self, *args, **kwargs):
-                return "Hello world!"
-
-            @cherrypy.expose
-            def post_multipart(self, file):
-                """Return a summary ("a * 65536\nb * 65536") of the uploaded
-                file.
-                """
-                contents = file.file.read()
-                summary = []
-                curchar = None
-                count = 0
-                for c in contents:
-                    if c == curchar:
-                        count += 1
-                    else:
-                        if count:
-                            if six.PY3:
-                                curchar = chr(curchar)
-                            summary.append("%s * %d" % (curchar, count))
-                        count = 1
-                        curchar = c
-                if count:
-                    if six.PY3:
-                        curchar = chr(curchar)
-                    summary.append("%s * %d" % (curchar, count))
-                return ", ".join(summary)
-
-            @cherrypy.expose
-            def post_filename(self, myfile):
-                '''Return the name of the file which was uploaded.'''
-                return myfile.filename
-
-        cherrypy.tree.mount(Root())
-        cherrypy.config.update({'server.max_request_body_size': 30000000})
-    setup_server = staticmethod(setup_server)
-
-    def test_no_content_length(self):
-        # "The presence of a message-body in a request is signaled by the
-        # inclusion of a Content-Length or Transfer-Encoding header field in
-        # the request's message-headers."
-        #
-        # Send a message with neither header and no body. Even though
-        # the request is of method POST, this should be OK because we set
-        # request.process_request_body to False for our handler.
-        c = self.make_connection()
-        c.request("POST", "/no_body")
-        response = c.getresponse()
-        self.body = response.fp.read()
-        self.status = str(response.status)
-        self.assertStatus(200)
-        self.assertBody(ntob('Hello world!'))
-
-        # Now send a message that has no Content-Length, but does send a body.
-        # Verify that CP times out the socket and responds
-        # with 411 Length Required.
-        if self.scheme == "https":
-            c = HTTPSConnection('%s:%s' % (self.interface(), self.PORT))
-        else:
-            c = HTTPConnection('%s:%s' % (self.interface(), self.PORT))
-        if hasattr(c, '_set_content_length'): # python 2.6 doesn't have it
-            with patch.object(c, '_set_content_length'):
-                c.request("POST", "/")
-        else:
-            c.request("POST", "/")
-        response = c.getresponse()
-        self.body = response.fp.read()
-        self.status = str(response.status)
-        self.assertStatus(411)
-
-    def test_post_multipart(self):
-        alphabet = "abcdefghijklmnopqrstuvwxyz"
-        # generate file contents for a large post
-        contents = "".join([c * 65536 for c in alphabet])
-
-        # encode as multipart form data
-        files = [('file', 'file.txt', contents)]
-        content_type, body = encode_multipart_formdata(files)
-        body = body.encode('Latin-1')
-
-        # post file
-        c = self.make_connection()
-        c.putrequest('POST', '/post_multipart')
-        c.putheader('Content-Type', content_type)
-        c.putheader('Content-Length', str(len(body)))
-        c.endheaders()
-        c.send(body)
-
-        response = c.getresponse()
-        self.body = response.fp.read()
-        self.status = str(response.status)
-        self.assertStatus(200)
-        self.assertBody(", ".join(["%s * 65536" % c for c in alphabet]))
-
-    def test_post_filename_with_special_characters(self):
-        '''Testing that we can handle filenames with special characters. This
-        was reported as a bug in:
-           https://github.com/cherrypy/cherrypy/issues/1146/
-           https://github.com/cherrypy/cherrypy/issues/1397'''
-        # We'll upload a bunch of files with differing names.
-        fnames = ['boop.csv', 'foo, bar.csv', 'bar, xxxx.csv', 'file"name.csv',
-                'file;name.csv', 'file; name.csv']
-        for fname in fnames:
-            files = [('myfile', fname, 'yunyeenyunyue')]
-            content_type, body = encode_multipart_formdata(files)
-            body = body.encode('Latin-1')
-
-            # post file
-            c = self.make_connection()
-            c.putrequest('POST', '/post_filename')
-            c.putheader('Content-Type', content_type)
-            c.putheader('Content-Length', str(len(body)))
-            c.endheaders()
-            c.send(body)
-
-            response = c.getresponse()
-            self.body = response.fp.read()
-            self.status = str(response.status)
-            self.assertStatus(200)
-            self.assertBody(fname)
-
-    def test_malformed_request_line(self):
-        if getattr(cherrypy.server, "using_apache", False):
-            return self.skip("skipped due to known Apache differences...")
-
-        # Test missing version in Request-Line
-        c = self.make_connection()
-        c._output(ntob('GET /'))
-        c._send_output()
-        if hasattr(c, 'strict'):
-            response = c.response_class(c.sock, strict=c.strict, method='GET')
-        else:
-            # Python 3.2 removed the 'strict' feature, saying:
-            # "http.client now always assumes HTTP/1.x compliant servers."
-            response = c.response_class(c.sock, method='GET')
-        response.begin()
-        self.assertEqual(response.status, 400)
-        self.assertEqual(response.fp.read(22), ntob("Malformed Request-Line"))
-        c.close()
-
-    def test_request_line_split_issue_1220(self):
-        Request_URI = "/index?intervenant-entreprise-evenement_classaction=evenement-mailremerciements&_path=intervenant-entreprise-evenement&intervenant-entreprise-evenement_action-id=19404&intervenant-entreprise-evenement_id=19404&intervenant-entreprise_id=28092"
-        self.assertEqual(len("GET %s HTTP/1.1\r\n" % Request_URI), 256)
-        self.getPage(Request_URI)
-        self.assertBody("Hello world!")
-
-    def test_malformed_header(self):
-        c = self.make_connection()
-        c.putrequest('GET', '/')
-        c.putheader('Content-Type', 'text/plain')
-        # See https://github.com/cherrypy/cherrypy/issues/941
-        c._output(ntob('Re, 1.2.3.4#015#012'))
-        c.endheaders()
-
-        response = c.getresponse()
-        self.status = str(response.status)
-        self.assertStatus(400)
-        self.body = response.fp.read(20)
-        self.assertBody("Illegal header line.")
-
-    def test_http_over_https(self):
-        if self.scheme != 'https':
-            return self.skip("skipped (not running HTTPS)... ")
-
-        # Try connecting without SSL.
-        conn = HTTPConnection('%s:%s' % (self.interface(), self.PORT))
-        conn.putrequest("GET", "/", skip_host=True)
-        conn.putheader("Host", self.HOST)
-        conn.endheaders()
-        response = conn.response_class(conn.sock, method="GET")
-        try:
-            response.begin()
-            self.assertEqual(response.status, 400)
-            self.body = response.read()
-            self.assertBody("The client sent a plain HTTP request, but this "
-                            "server only speaks HTTPS on this port.")
-        except socket.error:
-            e = sys.exc_info()[1]
-            # "Connection reset by peer" is also acceptable.
-            if e.errno != errno.ECONNRESET:
-                raise
-
-    def test_garbage_in(self):
-        # Connect without SSL regardless of server.scheme
-        c = HTTPConnection('%s:%s' % (self.interface(), self.PORT))
-        c._output(ntob('gjkgjklsgjklsgjkljklsg'))
-        c._send_output()
-        response = c.response_class(c.sock, method="GET")
-        try:
-            response.begin()
-            self.assertEqual(response.status, 400)
-            self.assertEqual(response.fp.read(22),
-                             ntob("Malformed Request-Line"))
-            c.close()
-        except socket.error:
-            e = sys.exc_info()[1]
-            # "Connection reset by peer" is also acceptable.
-            if e.errno != errno.ECONNRESET:
-                raise
diff --git a/lib/cherrypy/test/test_httpauth.py b/lib/cherrypy/test/test_httpauth.py
deleted file mode 100644
index 0dcdb72680c..00000000000
--- a/lib/cherrypy/test/test_httpauth.py
+++ /dev/null
@@ -1,195 +0,0 @@
-from hashlib import md5, sha1
-
-import cherrypy
-from cherrypy._cpcompat import ntob
-from cherrypy.lib import httpauth
-
-from cherrypy.test import helper
-
-
-class HTTPAuthTest(helper.CPWebCase):
-
-    def setup_server():
-        class Root:
-
-            @cherrypy.expose
-            def index(self):
-                return "This is public."
-
-        class DigestProtected:
-
-            @cherrypy.expose
-            def index(self):
-                return "Hello %s, you've been authorized." % (
-                    cherrypy.request.login)
-
-        class BasicProtected:
-
-            @cherrypy.expose
-            def index(self):
-                return "Hello %s, you've been authorized." % (
-                    cherrypy.request.login)
-
-        class BasicProtected2:
-
-            @cherrypy.expose
-            def index(self):
-                return "Hello %s, you've been authorized." % (
-                    cherrypy.request.login)
-
-        def fetch_users():
-            return {'test': 'test'}
-
-        def sha_password_encrypter(password):
-            return sha1(ntob(password)).hexdigest()
-
-        def fetch_password(username):
-            return sha1(ntob('test')).hexdigest()
-
-        conf = {
-            '/digest': {
-                'tools.digest_auth.on': True,
-                'tools.digest_auth.realm': 'localhost',
-                'tools.digest_auth.users': fetch_users
-            },
-            '/basic': {
-                'tools.basic_auth.on': True,
-                'tools.basic_auth.realm': 'localhost',
-                'tools.basic_auth.users': {
-                    'test': md5(ntob('test')).hexdigest()
-                }
-            },
-            '/basic2': {
-                'tools.basic_auth.on': True,
-                'tools.basic_auth.realm': 'localhost',
-                'tools.basic_auth.users': fetch_password,
-                'tools.basic_auth.encrypt': sha_password_encrypter
-            }
-        }
-
-        root = Root()
-        root.digest = DigestProtected()
-        root.basic = BasicProtected()
-        root.basic2 = BasicProtected2()
-        cherrypy.tree.mount(root, config=conf)
-    setup_server = staticmethod(setup_server)
-
-    def testPublic(self):
-        self.getPage("/")
-        self.assertStatus('200 OK')
-        self.assertHeader('Content-Type', 'text/html;charset=utf-8')
-        self.assertBody('This is public.')
-
-    def testBasic(self):
-        self.getPage("/basic/")
-        self.assertStatus(401)
-        self.assertHeader('WWW-Authenticate', 'Basic realm="localhost"')
-
-        self.getPage('/basic/', [('Authorization', 'Basic dGVzdDp0ZX60')])
-        self.assertStatus(401)
-
-        self.getPage('/basic/', [('Authorization', 'Basic dGVzdDp0ZXN0')])
-        self.assertStatus('200 OK')
-        self.assertBody("Hello test, you've been authorized.")
-
-    def testBasic2(self):
-        self.getPage("/basic2/")
-        self.assertStatus(401)
-        self.assertHeader('WWW-Authenticate', 'Basic realm="localhost"')
-
-        self.getPage('/basic2/', [('Authorization', 'Basic dGVzdDp0ZX60')])
-        self.assertStatus(401)
-
-        self.getPage('/basic2/', [('Authorization', 'Basic dGVzdDp0ZXN0')])
-        self.assertStatus('200 OK')
-        self.assertBody("Hello test, you've been authorized.")
-
-    def testDigest(self):
-        self.getPage("/digest/")
-        self.assertStatus(401)
-
-        value = None
-        for k, v in self.headers:
-            if k.lower() == "www-authenticate":
-                if v.startswith("Digest"):
-                    value = v
-                    break
-
-        if value is None:
-            self._handlewebError(
-                "Digest authentification scheme was not found")
-
-        value = value[7:]
-        items = value.split(', ')
-        tokens = {}
-        for item in items:
-            key, value = item.split('=')
-            tokens[key.lower()] = value
-
-        missing_msg = "%s is missing"
-        bad_value_msg = "'%s' was expecting '%s' but found '%s'"
-        nonce = None
-        if 'realm' not in tokens:
-            self._handlewebError(missing_msg % 'realm')
-        elif tokens['realm'] != '"localhost"':
-            self._handlewebError(bad_value_msg %
-                                 ('realm', '"localhost"', tokens['realm']))
-        if 'nonce' not in tokens:
-            self._handlewebError(missing_msg % 'nonce')
-        else:
-            nonce = tokens['nonce'].strip('"')
-        if 'algorithm' not in tokens:
-            self._handlewebError(missing_msg % 'algorithm')
-        elif tokens['algorithm'] != '"MD5"':
-            self._handlewebError(bad_value_msg %
-                                 ('algorithm', '"MD5"', tokens['algorithm']))
-        if 'qop' not in tokens:
-            self._handlewebError(missing_msg % 'qop')
-        elif tokens['qop'] != '"auth"':
-            self._handlewebError(bad_value_msg %
-                                 ('qop', '"auth"', tokens['qop']))
-
-        # Test a wrong 'realm' value
-        base_auth = (
-            'Digest '
-            'username="test", '
-            'realm="wrong realm", '
-            'nonce="%s", '
-            'uri="/digest/", '
-            'algorithm=MD5, '
-            'response="%s", '
-            'qop=auth, '
-            'nc=%s, '
-            'cnonce="1522e61005789929"'
-        )
-
-        auth = base_auth % (nonce, '', '00000001')
-        params = httpauth.parseAuthorization(auth)
-        response = httpauth._computeDigestResponse(params, 'test')
-
-        auth = base_auth % (nonce, response, '00000001')
-        self.getPage('/digest/', [('Authorization', auth)])
-        self.assertStatus(401)
-
-        # Test that must pass
-        base_auth = (
-            'Digest '
-            'username="test", '
-            'realm="localhost", '
-            'nonce="%s", '
-            'uri="/digest/", '
-            'algorithm=MD5, '
-            'response="%s", '
-            'qop=auth, '
-            'nc=%s, '
-            'cnonce="1522e61005789929"'
-        )
-
-        auth = base_auth % (nonce, '', '00000001')
-        params = httpauth.parseAuthorization(auth)
-        response = httpauth._computeDigestResponse(params, 'test')
-
-        auth = base_auth % (nonce, response, '00000001')
-        self.getPage('/digest/', [('Authorization', auth)])
-        self.assertStatus('200 OK')
-        self.assertBody("Hello test, you've been authorized.")
diff --git a/lib/cherrypy/test/test_httplib.py b/lib/cherrypy/test/test_httplib.py
deleted file mode 100644
index 76ca7641d3e..00000000000
--- a/lib/cherrypy/test/test_httplib.py
+++ /dev/null
@@ -1,29 +0,0 @@
-"""Tests for cherrypy/lib/httputil.py."""
-
-import unittest
-from cherrypy.lib import httputil
-
-
-class UtilityTests(unittest.TestCase):
-
-    def test_urljoin(self):
-        # Test all slash+atom combinations for SCRIPT_NAME and PATH_INFO
-        self.assertEqual(httputil.urljoin("/sn/", "/pi/"), "/sn/pi/")
-        self.assertEqual(httputil.urljoin("/sn/", "/pi"), "/sn/pi")
-        self.assertEqual(httputil.urljoin("/sn/", "/"), "/sn/")
-        self.assertEqual(httputil.urljoin("/sn/", ""), "/sn/")
-        self.assertEqual(httputil.urljoin("/sn", "/pi/"), "/sn/pi/")
-        self.assertEqual(httputil.urljoin("/sn", "/pi"), "/sn/pi")
-        self.assertEqual(httputil.urljoin("/sn", "/"), "/sn/")
-        self.assertEqual(httputil.urljoin("/sn", ""), "/sn")
-        self.assertEqual(httputil.urljoin("/", "/pi/"), "/pi/")
-        self.assertEqual(httputil.urljoin("/", "/pi"), "/pi")
-        self.assertEqual(httputil.urljoin("/", "/"), "/")
-        self.assertEqual(httputil.urljoin("/", ""), "/")
-        self.assertEqual(httputil.urljoin("", "/pi/"), "/pi/")
-        self.assertEqual(httputil.urljoin("", "/pi"), "/pi")
-        self.assertEqual(httputil.urljoin("", "/"), "/")
-        self.assertEqual(httputil.urljoin("", ""), "/")
-
-if __name__ == '__main__':
-    unittest.main()
diff --git a/lib/cherrypy/test/test_iterator.py b/lib/cherrypy/test/test_iterator.py
deleted file mode 100644
index 28d5049e7e1..00000000000
--- a/lib/cherrypy/test/test_iterator.py
+++ /dev/null
@@ -1,189 +0,0 @@
-import six
-
-import cherrypy
-
-
-class IteratorBase(object):
-
-    created = 0
-    datachunk = 'butternut squash' * 256
-
-    @classmethod
-    def incr(cls):
-        cls.created += 1
-
-    @classmethod
-    def decr(cls):
-        cls.created -= 1
-
-class OurGenerator(IteratorBase):
-
-    def __iter__(self):
-        self.incr()
-        try:
-            for i in range(1024):
-                yield self.datachunk
-        finally:
-            self.decr()
-
-class OurIterator(IteratorBase):
-
-    started = False
-    closed_off = False
-    count = 0
-
-    def increment(self):
-        self.incr()
-
-    def decrement(self):
-        if not self.closed_off:
-            self.closed_off = True
-            self.decr()
-
-    def __iter__(self):
-        return self
-
-    def __next__(self):
-        if not self.started:
-            self.started = True
-            self.increment()
-        self.count += 1
-        if self.count > 1024:
-            raise StopIteration
-        return self.datachunk
-
-    next = __next__
-
-    def __del__(self):
-        self.decrement()
-
-class OurClosableIterator(OurIterator):
-
-    def close(self):
-        self.decrement()
-
-class OurNotClosableIterator(OurIterator):
-
-    # We can't close something which requires an additional argument.
-    def close(self, somearg):
-        self.decrement()
-
-class OurUnclosableIterator(OurIterator):
-    close = 'close' # not callable!
-
-from cherrypy.test import helper
-class IteratorTest(helper.CPWebCase):
-
-    @staticmethod
-    def setup_server():
-
-        class Root(object):
-
-            @cherrypy.expose
-            def count(self, clsname):
-                cherrypy.response.headers['Content-Type'] = 'text/plain'
-                return six.text_type(globals()[clsname].created)
-
-            @cherrypy.expose
-            def getall(self, clsname):
-                cherrypy.response.headers['Content-Type'] = 'text/plain'
-                return globals()[clsname]()
-
-            @cherrypy.expose
-            @cherrypy.config(**{'response.stream': True})
-            def stream(self, clsname):
-                return self.getall(clsname)
-
-        cherrypy.tree.mount(Root())
-
-    def test_iterator(self):
-        try:
-            self._test_iterator()
-        except Exception:
-            "Test fails intermittently. See #1419"
-
-    def _test_iterator(self):
-        if cherrypy.server.protocol_version != "HTTP/1.1":
-            return self.skip()
-
-        self.PROTOCOL = "HTTP/1.1"
-
-        # Check the counts of all the classes, they should be zero.
-        closables = ['OurClosableIterator', 'OurGenerator']
-        unclosables = ['OurUnclosableIterator', 'OurNotClosableIterator']
-        all_classes = closables + unclosables
-
-        import random
-        random.shuffle(all_classes)
-
-        for clsname in all_classes:
-            self.getPage("/count/" + clsname)
-            self.assertStatus(200)
-            self.assertBody('0')
-
-        # We should also be able to read the entire content body
-        # successfully, though we don't need to, we just want to
-        # check the header.
-        for clsname in all_classes:
-            itr_conn = self.get_conn()
-            itr_conn.putrequest("GET", "/getall/" + clsname)
-            itr_conn.endheaders()
-            response = itr_conn.getresponse()
-            self.assertEqual(response.status, 200)
-            headers = response.getheaders()
-            for header_name, header_value in headers:
-                if header_name.lower() == 'content-length':
-                    assert header_value == six.text_type(1024 * 16 * 256), header_value
-                    break
-            else:
-                raise AssertionError('No Content-Length header found')
-
-            # As the response should be fully consumed by CherryPy
-            # before sending back, the count should still be at zero
-            # by the time the response has been sent.
-            self.getPage("/count/" + clsname)
-            self.assertStatus(200)
-            self.assertBody('0')
-
-        # Now we do the same check with streaming - some classes will
-        # be automatically closed, while others cannot.
-        stream_counts = {}
-        for clsname in all_classes:
-            itr_conn = self.get_conn()
-            itr_conn.putrequest("GET", "/stream/" + clsname)
-            itr_conn.endheaders()
-            response = itr_conn.getresponse()
-            self.assertEqual(response.status, 200)
-            response.fp.read(65536)
-
-            # Let's check the count - this should always be one.
-            self.getPage("/count/" + clsname)
-            self.assertBody('1')
-
-            # Now if we close the connection, the count should go back
-            # to zero.
-            itr_conn.close()
-            self.getPage("/count/" + clsname)
-
-            # If this is a response which should be easily closed, then
-            # we will test to see if the value has gone back down to
-            # zero.
-            if clsname in closables:
-
-                # Sometimes we try to get the answer too quickly - we
-                # will wait for 100 ms before asking again if we didn't
-                # get the answer we wanted.
-                if self.body != '0':
-                    import time
-                    time.sleep(0.1)
-                    self.getPage("/count/" + clsname)
-
-            stream_counts[clsname] = int(self.body)
-
-        # Check that we closed off the classes which should provide
-        # easy mechanisms for doing so.
-        for clsname in closables:
-            assert stream_counts[clsname] == 0, (
-                'did not close off stream response correctly, expected '
-                'count of zero for %s: %s' % (clsname, stream_counts)
-            )
diff --git a/lib/cherrypy/test/test_json.py b/lib/cherrypy/test/test_json.py
deleted file mode 100644
index 642cc087219..00000000000
--- a/lib/cherrypy/test/test_json.py
+++ /dev/null
@@ -1,101 +0,0 @@
-import cherrypy
-from cherrypy.test import helper
-
-from cherrypy._cpcompat import json
-
-
-json_out = cherrypy.config(**{'tools.json_out.on': True})
-json_in = cherrypy.config(**{'tools.json_in.on': True})
-
-class JsonTest(helper.CPWebCase):
-
-    def setup_server():
-        class Root(object):
-
-            @cherrypy.expose
-            def plain(self):
-                return 'hello'
-
-            @cherrypy.expose
-            @json_out
-            def json_string(self):
-                return 'hello'
-
-            @cherrypy.expose
-            @json_out
-            def json_list(self):
-                return ['a', 'b', 42]
-
-            @cherrypy.expose
-            @json_out
-            def json_dict(self):
-                return {'answer': 42}
-
-            @cherrypy.expose
-            @json_in
-            def json_post(self):
-                if cherrypy.request.json == [13, 'c']:
-                    return 'ok'
-                else:
-                    return 'nok'
-
-            @cherrypy.expose
-            @json_out
-            @cherrypy.config(**{'tools.caching.on': True})
-            def json_cached(self):
-                return 'hello there'
-
-        root = Root()
-        cherrypy.tree.mount(root)
-    setup_server = staticmethod(setup_server)
-
-    def test_json_output(self):
-        if json is None:
-            self.skip("json not found ")
-            return
-
-        self.getPage("/plain")
-        self.assertBody("hello")
-
-        self.getPage("/json_string")
-        self.assertBody('"hello"')
-
-        self.getPage("/json_list")
-        self.assertBody('["a", "b", 42]')
-
-        self.getPage("/json_dict")
-        self.assertBody('{"answer": 42}')
-
-    def test_json_input(self):
-        if json is None:
-            self.skip("json not found ")
-            return
-
-        body = '[13, "c"]'
-        headers = [('Content-Type', 'application/json'),
-                   ('Content-Length', str(len(body)))]
-        self.getPage("/json_post", method="POST", headers=headers, body=body)
-        self.assertBody('ok')
-
-        body = '[13, "c"]'
-        headers = [('Content-Type', 'text/plain'),
-                   ('Content-Length', str(len(body)))]
-        self.getPage("/json_post", method="POST", headers=headers, body=body)
-        self.assertStatus(415, 'Expected an application/json content type')
-
-        body = '[13, -]'
-        headers = [('Content-Type', 'application/json'),
-                   ('Content-Length', str(len(body)))]
-        self.getPage("/json_post", method="POST", headers=headers, body=body)
-        self.assertStatus(400, 'Invalid JSON document')
-
-    def test_cached(self):
-        if json is None:
-            self.skip("json not found ")
-            return
-
-        self.getPage("/json_cached")
-        self.assertStatus(200, '"hello"')
-
-        self.getPage("/json_cached")  # 2'nd time to hit cache
-        self.assertStatus(200, '"hello"')
diff --git a/lib/cherrypy/test/test_logging.py b/lib/cherrypy/test/test_logging.py
deleted file mode 100644
index ebaced89523..00000000000
--- a/lib/cherrypy/test/test_logging.py
+++ /dev/null
@@ -1,179 +0,0 @@
-"""Basic tests for the CherryPy core: request handling."""
-
-import os
-localDir = os.path.dirname(__file__)
-
-import six
-
-import cherrypy
-from cherrypy._cpcompat import ntob, ntou
-
-access_log = os.path.join(localDir, "access.log")
-error_log = os.path.join(localDir, "error.log")
-
-# Some unicode strings.
-tartaros = ntou('\u03a4\u1f71\u03c1\u03c4\u03b1\u03c1\u03bf\u03c2', 'escape')
-erebos = ntou('\u0388\u03c1\u03b5\u03b2\u03bf\u03c2.com', 'escape')
-
-
-def setup_server():
-    class Root:
-
-        @cherrypy.expose
-        def index(self):
-            return "hello"
-
-        @cherrypy.expose
-        def uni_code(self):
-            cherrypy.request.login = tartaros
-            cherrypy.request.remote.name = erebos
-
-        @cherrypy.expose
-        def slashes(self):
-            cherrypy.request.request_line = r'GET /slashed\path HTTP/1.1'
-
-        @cherrypy.expose
-        def whitespace(self):
-            # User-Agent = "User-Agent" ":" 1*( product | comment )
-            # comment    = "(" *( ctext | quoted-pair | comment ) ")"
-            # ctext      = 
-            # TEXT       = 
-            # LWS        = [CRLF] 1*( SP | HT )
-            cherrypy.request.headers['User-Agent'] = 'Browzuh (1.0\r\n\t\t.3)'
-
-        @cherrypy.expose
-        def as_string(self):
-            return "content"
-
-        @cherrypy.expose
-        def as_yield(self):
-            yield "content"
-
-        @cherrypy.expose
-        @cherrypy.config(**{'tools.log_tracebacks.on': True})
-        def error(self):
-            raise ValueError()
-
-    root = Root()
-
-    cherrypy.config.update({
-        'log.error_file': error_log,
-        'log.access_file': access_log,
-    })
-    cherrypy.tree.mount(root)
-
-
-from cherrypy.test import helper, logtest
-
-
-class AccessLogTests(helper.CPWebCase, logtest.LogCase):
-    setup_server = staticmethod(setup_server)
-
-    logfile = access_log
-
-    def testNormalReturn(self):
-        self.markLog()
-        self.getPage("/as_string",
-                     headers=[('Referer', 'http://www.cherrypy.org/'),
-                              ('User-Agent', 'Mozilla/5.0')])
-        self.assertBody('content')
-        self.assertStatus(200)
-
-        intro = '%s - - [' % self.interface()
-
-        self.assertLog(-1, intro)
-
-        if [k for k, v in self.headers if k.lower() == 'content-length']:
-            self.assertLog(-1, '] "GET %s/as_string HTTP/1.1" 200 7 '
-                           '"http://www.cherrypy.org/" "Mozilla/5.0"'
-                           % self.prefix())
-        else:
-            self.assertLog(-1, '] "GET %s/as_string HTTP/1.1" 200 - '
-                           '"http://www.cherrypy.org/" "Mozilla/5.0"'
-                           % self.prefix())
-
-    def testNormalYield(self):
-        self.markLog()
-        self.getPage("/as_yield")
-        self.assertBody('content')
-        self.assertStatus(200)
-
-        intro = '%s - - [' % self.interface()
-
-        self.assertLog(-1, intro)
-        if [k for k, v in self.headers if k.lower() == 'content-length']:
-            self.assertLog(-1, '] "GET %s/as_yield HTTP/1.1" 200 7 "" ""' %
-                           self.prefix())
-        else:
-            self.assertLog(-1, '] "GET %s/as_yield HTTP/1.1" 200 - "" ""'
-                           % self.prefix())
-
-    def testCustomLogFormat(self):
-        '''Test a customized access_log_format string,
-           which is a feature of _cplogging.LogManager.access() '''
-
-        original_logformat = cherrypy._cplogging.LogManager.access_log_format
-        cherrypy._cplogging.LogManager.access_log_format = \
-          '{h} {l} {u} {t} "{r}" {s} {b} "{f}" "{a}" {o}' \
-          if six.PY3 else \
-          '%(h)s %(l)s %(u)s %(t)s "%(r)s" %(s)s %(b)s "%(f)s" "%(a)s" %(o)s'
-
-        self.markLog()
-        self.getPage("/as_string", headers=[('Referer', 'REFERER'),
-                                            ('User-Agent', 'USERAGENT'),
-                                            ('Host', 'HOST')])
-        self.assertLog(-1, '%s - - [' % self.interface())
-        self.assertLog(-1, '] "GET /as_string HTTP/1.1" '
-                           '200 7 "REFERER" "USERAGENT" HOST')
-
-        cherrypy._cplogging.LogManager.access_log_format = original_logformat
-
-    def testEscapedOutput(self):
-        # Test unicode in access log pieces.
-        self.markLog()
-        self.getPage("/uni_code")
-        self.assertStatus(200)
-        if six.PY3:
-            # The repr of a bytestring in six.PY3 includes a b'' prefix
-            self.assertLog(-1, repr(tartaros.encode('utf8'))[2:-1])
-        else:
-            self.assertLog(-1, repr(tartaros.encode('utf8'))[1:-1])
-        # Test the erebos value. Included inline for your enlightenment.
-        # Note the 'r' prefix--those backslashes are literals.
-        self.assertLog(-1, r'\xce\x88\xcf\x81\xce\xb5\xce\xb2\xce\xbf\xcf\x82')
-
-        # Test backslashes in output.
-        self.markLog()
-        self.getPage("/slashes")
-        self.assertStatus(200)
-        if six.PY3:
-            self.assertLog(-1, ntob('"GET /slashed\\path HTTP/1.1"'))
-        else:
-            self.assertLog(-1, r'"GET /slashed\\path HTTP/1.1"')
-
-        # Test whitespace in output.
-        self.markLog()
-        self.getPage("/whitespace")
-        self.assertStatus(200)
-        # Again, note the 'r' prefix.
-        self.assertLog(-1, r'"Browzuh (1.0\r\n\t\t.3)"')
-
-
-class ErrorLogTests(helper.CPWebCase, logtest.LogCase):
-    setup_server = staticmethod(setup_server)
-
-    logfile = error_log
-
-    def testTracebacks(self):
-        # Test that tracebacks get written to the error log.
-        self.markLog()
-        ignore = helper.webtest.ignored_exceptions
-        ignore.append(ValueError)
-        try:
-            self.getPage("/error")
-            self.assertInBody("raise ValueError()")
-            self.assertLog(0, 'HTTP')
-            self.assertLog(1, 'Traceback (most recent call last):')
-            self.assertLog(-2, 'raise ValueError()')
-        finally:
-            ignore.pop()
diff --git a/lib/cherrypy/test/test_mime.py b/lib/cherrypy/test/test_mime.py
deleted file mode 100644
index 7826fd56a63..00000000000
--- a/lib/cherrypy/test/test_mime.py
+++ /dev/null
@@ -1,135 +0,0 @@
-"""Tests for various MIME issues, including the safe_multipart Tool."""
-
-import cherrypy
-from cherrypy._cpcompat import ntob, ntou, sorted
-
-
-def setup_server():
-
-    class Root:
-
-        @cherrypy.expose
-        def multipart(self, parts):
-            return repr(parts)
-
-        @cherrypy.expose
-        def multipart_form_data(self, **kwargs):
-            return repr(list(sorted(kwargs.items())))
-
-        @cherrypy.expose
-        def flashupload(self, Filedata, Upload, Filename):
-            return ("Upload: %s, Filename: %s, Filedata: %r" %
-                    (Upload, Filename, Filedata.file.read()))
-
-    cherrypy.config.update({'server.max_request_body_size': 0})
-    cherrypy.tree.mount(Root())
-
-
-#                             Client-side code                             #
-
-from cherrypy.test import helper
-
-
-class MultipartTest(helper.CPWebCase):
-    setup_server = staticmethod(setup_server)
-
-    def test_multipart(self):
-        text_part = ntou("This is the text version")
-        html_part = ntou(
-            """
-
-
- 
-
-
-
-This is the HTML version
-
-
-""")
-        body = '\r\n'.join([
-            "--123456789",
-            "Content-Type: text/plain; charset='ISO-8859-1'",
-            "Content-Transfer-Encoding: 7bit",
-            "",
-            text_part,
-            "--123456789",
-            "Content-Type: text/html; charset='ISO-8859-1'",
-            "",
-            html_part,
-            "--123456789--"])
-        headers = [
-            ('Content-Type', 'multipart/mixed; boundary=123456789'),
-            ('Content-Length', str(len(body))),
-        ]
-        self.getPage('/multipart', headers, "POST", body)
-        self.assertBody(repr([text_part, html_part]))
-
-    def test_multipart_form_data(self):
-        body = '\r\n'.join([
-            '--X',
-            'Content-Disposition: form-data; name="foo"',
-            '',
-            'bar',
-            '--X',
-            # Test a param with more than one value.
-            # See
-            # https://github.com/cherrypy/cherrypy/issues/1028
-            'Content-Disposition: form-data; name="baz"',
-            '',
-            '111',
-            '--X',
-            'Content-Disposition: form-data; name="baz"',
-            '',
-            '333',
-            '--X--'
-        ])
-        self.getPage('/multipart_form_data', method='POST',
-                     headers=[(
-                         "Content-Type", "multipart/form-data;boundary=X"),
-                         ("Content-Length", str(len(body))),
-                     ],
-                     body=body),
-        self.assertBody(
-            repr([('baz', [ntou('111'), ntou('333')]), ('foo', ntou('bar'))]))
-
-
-class SafeMultipartHandlingTest(helper.CPWebCase):
-    setup_server = staticmethod(setup_server)
-
-    def test_Flash_Upload(self):
-        headers = [
-            ('Accept', 'text/*'),
-            ('Content-Type', 'multipart/form-data; '
-             'boundary=----------KM7Ij5cH2KM7Ef1gL6ae0ae0cH2gL6'),
-            ('User-Agent', 'Shockwave Flash'),
-            ('Host', 'www.example.com:54583'),
-            ('Content-Length', '499'),
-            ('Connection', 'Keep-Alive'),
-            ('Cache-Control', 'no-cache'),
-        ]
-        filedata = ntob('\r\n'
-                        '\r\n'
-                        '\r\n')
-        body = (ntob(
-            '------------KM7Ij5cH2KM7Ef1gL6ae0ae0cH2gL6\r\n'
-            'Content-Disposition: form-data; name="Filename"\r\n'
-            '\r\n'
-            '.project\r\n'
-            '------------KM7Ij5cH2KM7Ef1gL6ae0ae0cH2gL6\r\n'
-            'Content-Disposition: form-data; '
-                'name="Filedata"; filename=".project"\r\n'
-            'Content-Type: application/octet-stream\r\n'
-            '\r\n')
-            + filedata +
-            ntob('\r\n'
-                 '------------KM7Ij5cH2KM7Ef1gL6ae0ae0cH2gL6\r\n'
-                 'Content-Disposition: form-data; name="Upload"\r\n'
-                 '\r\n'
-                 'Submit Query\r\n'
-                 # Flash apps omit the trailing \r\n on the last line:
-                 '------------KM7Ij5cH2KM7Ef1gL6ae0ae0cH2gL6--'
-                 ))
-        self.getPage('/flashupload', headers, "POST", body)
-        self.assertBody("Upload: Submit Query, Filename: .project, "
-                        "Filedata: %r" % filedata)
diff --git a/lib/cherrypy/test/test_misc_tools.py b/lib/cherrypy/test/test_misc_tools.py
deleted file mode 100644
index 4e38cd1cca1..00000000000
--- a/lib/cherrypy/test/test_misc_tools.py
+++ /dev/null
@@ -1,217 +0,0 @@
-import os
-localDir = os.path.dirname(__file__)
-logfile = os.path.join(localDir, "test_misc_tools.log")
-
-import cherrypy
-from cherrypy import tools
-
-
-def setup_server():
-    class Root:
-
-        @cherrypy.expose
-        def index(self):
-            yield "Hello, world"
-        h = [("Content-Language", "en-GB"), ('Content-Type', 'text/plain')]
-        tools.response_headers(headers=h)(index)
-
-        @cherrypy.expose
-        @cherrypy.config(**{
-            'tools.response_headers.on': True,
-            'tools.response_headers.headers': [
-                ("Content-Language", "fr"),
-                ('Content-Type', 'text/plain'),
-            ],
-            'tools.log_hooks.on': True,
-        })
-        def other(self):
-            return "salut"
-
-    @cherrypy.config(**{'tools.accept.on': True})
-    class Accept:
-
-        @cherrypy.expose
-        def index(self):
-            return 'Atom feed'
-
-        # In Python 2.4+, we could use a decorator instead:
-        # @tools.accept('application/atom+xml')
-        @cherrypy.expose
-        @cherrypy.config(**{'tools.accept.media': 'application/atom+xml'})
-        def feed(self):
-            return """
-
-    Unknown Blog
-"""
-
-        @cherrypy.expose
-        def select(self):
-            # We could also write this: mtype = cherrypy.lib.accept.accept(...)
-            mtype = tools.accept.callable(['text/html', 'text/plain'])
-            if mtype == 'text/html':
-                return "
Page Title
"
-            else:
-                return "PAGE TITLE"
-
-    class Referer:
-
-        @cherrypy.expose
-        def accept(self):
-            return "Accepted!"
-        reject = accept
-
-    class AutoVary:
-
-        @cherrypy.expose
-        def index(self):
-            # Read a header directly with 'get'
-            ae = cherrypy.request.headers.get('Accept-Encoding')
-            # Read a header directly with '__getitem__'
-            cl = cherrypy.request.headers['Host']
-            # Read a header directly with '__contains__'
-            hasif = 'If-Modified-Since' in cherrypy.request.headers
-            # Read a header directly with 'has_key'
-            if hasattr(dict, 'has_key'):
-                # Python 2
-                has = cherrypy.request.headers.has_key('Range')
-            else:
-                # Python 3
-                has = 'Range' in cherrypy.request.headers
-            # Call a lib function
-            mtype = tools.accept.callable(['text/html', 'text/plain'])
-            return "Hello, world!"
-
-    conf = {'/referer': {'tools.referer.on': True,
-                         'tools.referer.pattern': r'http://[^/]*example\.com',
-                         },
-            '/referer/reject': {'tools.referer.accept': False,
-                                'tools.referer.accept_missing': True,
-                                },
-            '/autovary': {'tools.autovary.on': True},
-            }
-
-    root = Root()
-    root.referer = Referer()
-    root.accept = Accept()
-    root.autovary = AutoVary()
-    cherrypy.tree.mount(root, config=conf)
-    cherrypy.config.update({'log.error_file': logfile})
-
-
-from cherrypy.test import helper
-
-
-class ResponseHeadersTest(helper.CPWebCase):
-    setup_server = staticmethod(setup_server)
-
-    def testResponseHeadersDecorator(self):
-        self.getPage('/')
-        self.assertHeader("Content-Language", "en-GB")
-        self.assertHeader('Content-Type', 'text/plain;charset=utf-8')
-
-    def testResponseHeaders(self):
-        self.getPage('/other')
-        self.assertHeader("Content-Language", "fr")
-        self.assertHeader('Content-Type', 'text/plain;charset=utf-8')
-
-
-class RefererTest(helper.CPWebCase):
-    setup_server = staticmethod(setup_server)
-
-    def testReferer(self):
-        self.getPage('/referer/accept')
-        self.assertErrorPage(403, 'Forbidden Referer header.')
-
-        self.getPage('/referer/accept',
-                     headers=[('Referer', 'http://www.example.com/')])
-        self.assertStatus(200)
-        self.assertBody('Accepted!')
-
-        # Reject
-        self.getPage('/referer/reject')
-        self.assertStatus(200)
-        self.assertBody('Accepted!')
-
-        self.getPage('/referer/reject',
-                     headers=[('Referer', 'http://www.example.com/')])
-        self.assertErrorPage(403, 'Forbidden Referer header.')
-
-
-class AcceptTest(helper.CPWebCase):
-    setup_server = staticmethod(setup_server)
-
-    def test_Accept_Tool(self):
-        # Test with no header provided
-        self.getPage('/accept/feed')
-        self.assertStatus(200)
-        self.assertInBody('Unknown Blog')
-
-        # Specify exact media type
-        self.getPage('/accept/feed',
-                     headers=[('Accept', 'application/atom+xml')])
-        self.assertStatus(200)
-        self.assertInBody('Unknown Blog')
-
-        # Specify matching media range
-        self.getPage('/accept/feed', headers=[('Accept', 'application/*')])
-        self.assertStatus(200)
-        self.assertInBody('Unknown Blog')
-
-        # Specify all media ranges
-        self.getPage('/accept/feed', headers=[('Accept', '*/*')])
-        self.assertStatus(200)
-        self.assertInBody('Unknown Blog')
-
-        # Specify unacceptable media types
-        self.getPage('/accept/feed', headers=[('Accept', 'text/html')])
-        self.assertErrorPage(406,
-                             "Your client sent this Accept header: text/html. "
-                             "But this resource only emits these media types: "
-                             "application/atom+xml.")
-
-        # Test resource where tool is 'on' but media is None (not set).
-        self.getPage('/accept/')
-        self.assertStatus(200)
-        self.assertBody('Atom feed')
-
-    def test_accept_selection(self):
-        # Try both our expected media types
-        self.getPage('/accept/select', [('Accept', 'text/html')])
-        self.assertStatus(200)
-        self.assertBody('
Page Title
')
-        self.getPage('/accept/select', [('Accept', 'text/plain')])
-        self.assertStatus(200)
-        self.assertBody('PAGE TITLE')
-        self.getPage('/accept/select',
-                     [('Accept', 'text/plain, text/*;q=0.5')])
-        self.assertStatus(200)
-        self.assertBody('PAGE TITLE')
-
-        # text/* and */* should prefer text/html since it comes first
-        # in our 'media' argument to tools.accept
-        self.getPage('/accept/select', [('Accept', 'text/*')])
-        self.assertStatus(200)
-        self.assertBody('
Page Title
')
-        self.getPage('/accept/select', [('Accept', '*/*')])
-        self.assertStatus(200)
-        self.assertBody('
Page Title
')
-
-        # Try unacceptable media types
-        self.getPage('/accept/select', [('Accept', 'application/xml')])
-        self.assertErrorPage(
-            406,
-            "Your client sent this Accept header: application/xml. "
-            "But this resource only emits these media types: "
-            "text/html, text/plain.")
-
-
-class AutoVaryTest(helper.CPWebCase):
-    setup_server = staticmethod(setup_server)
-
-    def testAutoVary(self):
-        self.getPage('/autovary/')
-        self.assertHeader(
-            "Vary",
-            'Accept, Accept-Charset, Accept-Encoding, '
-            'Host, If-Modified-Since, Range'
-        )
diff --git a/lib/cherrypy/test/test_objectmapping.py b/lib/cherrypy/test/test_objectmapping.py
deleted file mode 100644
index 7f9c7c28bd4..00000000000
--- a/lib/cherrypy/test/test_objectmapping.py
+++ /dev/null
@@ -1,430 +0,0 @@
-import sys
-import cherrypy
-from cherrypy._cpcompat import ntou
-from cherrypy._cptree import Application
-from cherrypy.test import helper
-
-script_names = ["", "/foo", "/users/fred/blog", "/corp/blog"]
-
-
-class ObjectMappingTest(helper.CPWebCase):
-
-    def setup_server():
-        class Root:
-
-            @cherrypy.expose
-            def index(self, name="world"):
-                return name
-
-            @cherrypy.expose
-            def foobar(self):
-                return "bar"
-
-            @cherrypy.expose
-            def default(self, *params, **kwargs):
-                return "default:" + repr(params)
-
-            @cherrypy.expose
-            def other(self):
-                return "other"
-
-            @cherrypy.expose
-            def extra(self, *p):
-                return repr(p)
-
-            @cherrypy.expose
-            def redirect(self):
-                raise cherrypy.HTTPRedirect('dir1/', 302)
-
-            def notExposed(self):
-                return "not exposed"
-
-            @cherrypy.expose
-            def confvalue(self):
-                return cherrypy.request.config.get("user")
-
-            @cherrypy.expose
-            def redirect_via_url(self, path):
-                raise cherrypy.HTTPRedirect(cherrypy.url(path))
-
-            @cherrypy.expose
-            def translate_html(self):
-                return "OK"
-
-        @cherrypy.expose
-        def mapped_func(self, ID=None):
-            return "ID is %s" % ID
-        setattr(Root, "Von B\xfclow", mapped_func)
-
-        class Exposing:
-
-            @cherrypy.expose
-            def base(self):
-                return "expose works!"
-            cherrypy.expose(base, "1")
-            cherrypy.expose(base, "2")
-
-        class ExposingNewStyle(object):
-
-            @cherrypy.expose
-            def base(self):
-                return "expose works!"
-            cherrypy.expose(base, "1")
-            cherrypy.expose(base, "2")
-
-        class Dir1:
-
-            @cherrypy.expose
-            def index(self):
-                return "index for dir1"
-
-            @cherrypy.expose
-            @cherrypy.config(**{'tools.trailing_slash.extra': True})
-            def myMethod(self):
-                return "myMethod from dir1, path_info is:" + repr(
-                    cherrypy.request.path_info)
-
-            @cherrypy.expose
-            def default(self, *params):
-                return "default for dir1, param is:" + repr(params)
-
-        class Dir2:
-
-            @cherrypy.expose
-            def index(self):
-                return "index for dir2, path is:" + cherrypy.request.path_info
-
-            @cherrypy.expose
-            def script_name(self):
-                return cherrypy.tree.script_name()
-
-            @cherrypy.expose
-            def cherrypy_url(self):
-                return cherrypy.url("/extra")
-
-            @cherrypy.expose
-            def posparam(self, *vpath):
-                return "/".join(vpath)
-
-        class Dir3:
-
-            def default(self):
-                return "default for dir3, not exposed"
-
-        class Dir4:
-
-            def index(self):
-                return "index for dir4, not exposed"
-
-        class DefNoIndex:
-
-            @cherrypy.expose
-            def default(self, *args):
-                raise cherrypy.HTTPRedirect("contact")
-
-        # MethodDispatcher code
-        @cherrypy.expose
-        class ByMethod:
-
-            def __init__(self, *things):
-                self.things = list(things)
-
-            def GET(self):
-                return repr(self.things)
-
-            def POST(self, thing):
-                self.things.append(thing)
-
-        class Collection:
-            default = ByMethod('a', 'bit')
-
-        Root.exposing = Exposing()
-        Root.exposingnew = ExposingNewStyle()
-        Root.dir1 = Dir1()
-        Root.dir1.dir2 = Dir2()
-        Root.dir1.dir2.dir3 = Dir3()
-        Root.dir1.dir2.dir3.dir4 = Dir4()
-        Root.defnoindex = DefNoIndex()
-        Root.bymethod = ByMethod('another')
-        Root.collection = Collection()
-
-        d = cherrypy.dispatch.MethodDispatcher()
-        for url in script_names:
-            conf = {'/': {'user': (url or "/").split("/")[-2]},
-                    '/bymethod': {'request.dispatch': d},
-                    '/collection': {'request.dispatch': d},
-                    }
-            cherrypy.tree.mount(Root(), url, conf)
-
-        class Isolated:
-
-            @cherrypy.expose
-            def index(self):
-                return "made it!"
-
-        cherrypy.tree.mount(Isolated(), "/isolated")
-
-        @cherrypy.expose
-        class AnotherApp:
-
-            def GET(self):
-                return "milk"
-
-        cherrypy.tree.mount(AnotherApp(), "/app",
-                            {'/': {'request.dispatch': d}})
-    setup_server = staticmethod(setup_server)
-
-    def testObjectMapping(self):
-        for url in script_names:
-            prefix = self.script_name = url
-
-            self.getPage('/')
-            self.assertBody('world')
-
-            self.getPage("/dir1/myMethod")
-            self.assertBody(
-                "myMethod from dir1, path_info is:'/dir1/myMethod'")
-
-            self.getPage("/this/method/does/not/exist")
-            self.assertBody(
-                "default:('this', 'method', 'does', 'not', 'exist')")
-
-            self.getPage("/extra/too/much")
-            self.assertBody("('too', 'much')")
-
-            self.getPage("/other")
-            self.assertBody('other')
-
-            self.getPage("/notExposed")
-            self.assertBody("default:('notExposed',)")
-
-            self.getPage("/dir1/dir2/")
-            self.assertBody('index for dir2, path is:/dir1/dir2/')
-
-            # Test omitted trailing slash (should be redirected by default).
-            self.getPage("/dir1/dir2")
-            self.assertStatus(301)
-            self.assertHeader('Location', '%s/dir1/dir2/' % self.base())
-
-            # Test extra trailing slash (should be redirected if configured).
-            self.getPage("/dir1/myMethod/")
-            self.assertStatus(301)
-            self.assertHeader('Location', '%s/dir1/myMethod' % self.base())
-
-            # Test that default method must be exposed in order to match.
-            self.getPage("/dir1/dir2/dir3/dir4/index")
-            self.assertBody(
-                "default for dir1, param is:('dir2', 'dir3', 'dir4', 'index')")
-
-            # Test *vpath when default() is defined but not index()
-            # This also tests HTTPRedirect with default.
-            self.getPage("/defnoindex")
-            self.assertStatus((302, 303))
-            self.assertHeader('Location', '%s/contact' % self.base())
-            self.getPage("/defnoindex/")
-            self.assertStatus((302, 303))
-            self.assertHeader('Location', '%s/defnoindex/contact' %
-                              self.base())
-            self.getPage("/defnoindex/page")
-            self.assertStatus((302, 303))
-            self.assertHeader('Location', '%s/defnoindex/contact' %
-                              self.base())
-
-            self.getPage("/redirect")
-            self.assertStatus('302 Found')
-            self.assertHeader('Location', '%s/dir1/' % self.base())
-
-            if not getattr(cherrypy.server, "using_apache", False):
-                # Test that we can use URL's which aren't all valid Python
-                # identifiers
-                # This should also test the %XX-unquoting of URL's.
-                self.getPage("/Von%20B%fclow?ID=14")
-                self.assertBody("ID is 14")
-
-                # Test that %2F in the path doesn't get unquoted too early;
-                # that is, it should not be used to separate path components.
-                # See ticket #393.
-                self.getPage("/page%2Fname")
-                self.assertBody("default:('page/name',)")
-
-            self.getPage("/dir1/dir2/script_name")
-            self.assertBody(url)
-            self.getPage("/dir1/dir2/cherrypy_url")
-            self.assertBody("%s/extra" % self.base())
-
-            # Test that configs don't overwrite each other from diferent apps
-            self.getPage("/confvalue")
-            self.assertBody((url or "/").split("/")[-2])
-
-        self.script_name = ""
-
-        # Test absoluteURI's in the Request-Line
-        self.getPage('http://%s:%s/' % (self.interface(), self.PORT))
-        self.assertBody('world')
-
-        self.getPage('http://%s:%s/abs/?service=http://192.168.0.1/x/y/z' %
-                     (self.interface(), self.PORT))
-        self.assertBody("default:('abs',)")
-
-        self.getPage('/rel/?service=http://192.168.120.121:8000/x/y/z')
-        self.assertBody("default:('rel',)")
-
-        # Test that the "isolated" app doesn't leak url's into the root app.
-        # If it did leak, Root.default() would answer with
-        #   "default:('isolated', 'doesnt', 'exist')".
-        self.getPage("/isolated/")
-        self.assertStatus("200 OK")
-        self.assertBody("made it!")
-        self.getPage("/isolated/doesnt/exist")
-        self.assertStatus("404 Not Found")
-
-        # Make sure /foobar maps to Root.foobar and not to the app
-        # mounted at /foo. See
-        # https://github.com/cherrypy/cherrypy/issues/573
-        self.getPage("/foobar")
-        self.assertBody("bar")
-
-    def test_translate(self):
-        self.getPage("/translate_html")
-        self.assertStatus("200 OK")
-        self.assertBody("OK")
-
-        self.getPage("/translate.html")
-        self.assertStatus("200 OK")
-        self.assertBody("OK")
-
-        self.getPage("/translate-html")
-        self.assertStatus("200 OK")
-        self.assertBody("OK")
-
-    def test_redir_using_url(self):
-        for url in script_names:
-            prefix = self.script_name = url
-
-            # Test the absolute path to the parent (leading slash)
-            self.getPage('/redirect_via_url?path=./')
-            self.assertStatus(('302 Found', '303 See Other'))
-            self.assertHeader('Location', '%s/' % self.base())
-
-            # Test the relative path to the parent (no leading slash)
-            self.getPage('/redirect_via_url?path=./')
-            self.assertStatus(('302 Found', '303 See Other'))
-            self.assertHeader('Location', '%s/' % self.base())
-
-            # Test the absolute path to the parent (leading slash)
-            self.getPage('/redirect_via_url/?path=./')
-            self.assertStatus(('302 Found', '303 See Other'))
-            self.assertHeader('Location', '%s/' % self.base())
-
-            # Test the relative path to the parent (no leading slash)
-            self.getPage('/redirect_via_url/?path=./')
-            self.assertStatus(('302 Found', '303 See Other'))
-            self.assertHeader('Location', '%s/' % self.base())
-
-    def testPositionalParams(self):
-        self.getPage("/dir1/dir2/posparam/18/24/hut/hike")
-        self.assertBody("18/24/hut/hike")
-
-        # intermediate index methods should not receive posparams;
-        # only the "final" index method should do so.
-        self.getPage("/dir1/dir2/5/3/sir")
-        self.assertBody("default for dir1, param is:('dir2', '5', '3', 'sir')")
-
-        # test that extra positional args raises an 404 Not Found
-        # See https://github.com/cherrypy/cherrypy/issues/733.
-        self.getPage("/dir1/dir2/script_name/extra/stuff")
-        self.assertStatus(404)
-
-    def testExpose(self):
-        # Test the cherrypy.expose function/decorator
-        self.getPage("/exposing/base")
-        self.assertBody("expose works!")
-
-        self.getPage("/exposing/1")
-        self.assertBody("expose works!")
-
-        self.getPage("/exposing/2")
-        self.assertBody("expose works!")
-
-        self.getPage("/exposingnew/base")
-        self.assertBody("expose works!")
-
-        self.getPage("/exposingnew/1")
-        self.assertBody("expose works!")
-
-        self.getPage("/exposingnew/2")
-        self.assertBody("expose works!")
-
-    def testMethodDispatch(self):
-        self.getPage("/bymethod")
-        self.assertBody("['another']")
-        self.assertHeader('Allow', 'GET, HEAD, POST')
-
-        self.getPage("/bymethod", method="HEAD")
-        self.assertBody("")
-        self.assertHeader('Allow', 'GET, HEAD, POST')
-
-        self.getPage("/bymethod", method="POST", body="thing=one")
-        self.assertBody("")
-        self.assertHeader('Allow', 'GET, HEAD, POST')
-
-        self.getPage("/bymethod")
-        self.assertBody(repr(['another', ntou('one')]))
-        self.assertHeader('Allow', 'GET, HEAD, POST')
-
-        self.getPage("/bymethod", method="PUT")
-        self.assertErrorPage(405)
-        self.assertHeader('Allow', 'GET, HEAD, POST')
-
-        # Test default with posparams
-        self.getPage("/collection/silly", method="POST")
-        self.getPage("/collection", method="GET")
-        self.assertBody("['a', 'bit', 'silly']")
-
-        # Test custom dispatcher set on app root (see #737).
-        self.getPage("/app")
-        self.assertBody("milk")
-
-    def testTreeMounting(self):
-        class Root(object):
-
-            @cherrypy.expose
-            def hello(self):
-                return "Hello world!"
-
-        # When mounting an application instance,
-        # we can't specify a different script name in the call to mount.
-        a = Application(Root(), '/somewhere')
-        self.assertRaises(ValueError, cherrypy.tree.mount, a, '/somewhereelse')
-
-        # When mounting an application instance...
-        a = Application(Root(), '/somewhere')
-        # ...we MUST allow in identical script name in the call to mount...
-        cherrypy.tree.mount(a, '/somewhere')
-        self.getPage('/somewhere/hello')
-        self.assertStatus(200)
-        # ...and MUST allow a missing script_name.
-        del cherrypy.tree.apps['/somewhere']
-        cherrypy.tree.mount(a)
-        self.getPage('/somewhere/hello')
-        self.assertStatus(200)
-
-        # In addition, we MUST be able to create an Application using
-        # script_name == None for access to the wsgi_environ.
-        a = Application(Root(), script_name=None)
-        # However, this does not apply to tree.mount
-        self.assertRaises(TypeError, cherrypy.tree.mount, a, None)
-
-    def testKeywords(self):
-        if sys.version_info < (3,):
-            return self.skip("skipped (Python 3 only)")
-        exec("""class Root(object):
-    @cherrypy.expose
-    def hello(self, *, name='world'):
-        return 'Hello %s!' % name
-cherrypy.tree.mount(Application(Root(), '/keywords'))""")
-
-        self.getPage('/keywords/hello')
-        self.assertStatus(200)
-        self.getPage('/keywords/hello/extra')
-        self.assertStatus(404)
diff --git a/lib/cherrypy/test/test_proxy.py b/lib/cherrypy/test/test_proxy.py
deleted file mode 100644
index e6dcf5d300e..00000000000
--- a/lib/cherrypy/test/test_proxy.py
+++ /dev/null
@@ -1,138 +0,0 @@
-import cherrypy
-from cherrypy.test import helper
-
-script_names = ["", "/path/to/myapp"]
-
-
-class ProxyTest(helper.CPWebCase):
-
-    def setup_server():
-
-        # Set up site
-        cherrypy.config.update({
-            'tools.proxy.on': True,
-            'tools.proxy.base': 'www.mydomain.test',
-        })
-
-        # Set up application
-
-        class Root:
-
-            def __init__(self, sn):
-                # Calculate a URL outside of any requests.
-                self.thisnewpage = cherrypy.url(
-                    "/this/new/page", script_name=sn)
-
-            @cherrypy.expose
-            def pageurl(self):
-                return self.thisnewpage
-
-            @cherrypy.expose
-            def index(self):
-                raise cherrypy.HTTPRedirect('dummy')
-
-            @cherrypy.expose
-            def remoteip(self):
-                return cherrypy.request.remote.ip
-
-            @cherrypy.expose
-            @cherrypy.config(**{
-                'tools.proxy.local': 'X-Host',
-                'tools.trailing_slash.extra': True,
-            })
-            def xhost(self):
-                raise cherrypy.HTTPRedirect('blah')
-
-            @cherrypy.expose
-            def base(self):
-                return cherrypy.request.base
-
-            @cherrypy.expose
-            @cherrypy.config(**{'tools.proxy.scheme': 'X-Forwarded-Ssl'})
-            def ssl(self):
-                return cherrypy.request.base
-
-            @cherrypy.expose
-            def newurl(self):
-                return ("Browse to this page."
-                        % cherrypy.url("/this/new/page"))
-
-        for sn in script_names:
-            cherrypy.tree.mount(Root(sn), sn)
-    setup_server = staticmethod(setup_server)
-
-    def testProxy(self):
-        self.getPage("/")
-        self.assertHeader('Location',
-                          "%s://www.mydomain.test%s/dummy" %
-                          (self.scheme, self.prefix()))
-
-        # Test X-Forwarded-Host (Apache 1.3.33+ and Apache 2)
-        self.getPage(
-            "/", headers=[('X-Forwarded-Host', 'http://www.example.test')])
-        self.assertHeader('Location', "http://www.example.test/dummy")
-        self.getPage("/", headers=[('X-Forwarded-Host', 'www.example.test')])
-        self.assertHeader('Location', "%s://www.example.test/dummy" %
-                          self.scheme)
-        # Test multiple X-Forwarded-Host headers
-        self.getPage("/", headers=[
-            ('X-Forwarded-Host', 'http://www.example.test, www.cherrypy.test'),
-        ])
-        self.assertHeader('Location', "http://www.example.test/dummy")
-
-        # Test X-Forwarded-For (Apache2)
-        self.getPage("/remoteip",
-                     headers=[('X-Forwarded-For', '192.168.0.20')])
-        self.assertBody("192.168.0.20")
-        #Fix bug #1268
-        self.getPage("/remoteip",
-                     headers=[
-                         ('X-Forwarded-For', '67.15.36.43, 192.168.0.20')
-                     ])
-        self.assertBody("67.15.36.43")
-
-        # Test X-Host (lighttpd; see https://trac.lighttpd.net/trac/ticket/418)
-        self.getPage("/xhost", headers=[('X-Host', 'www.example.test')])
-        self.assertHeader('Location', "%s://www.example.test/blah" %
-                          self.scheme)
-
-        # Test X-Forwarded-Proto (lighttpd)
-        self.getPage("/base", headers=[('X-Forwarded-Proto', 'https')])
-        self.assertBody("https://www.mydomain.test")
-
-        # Test X-Forwarded-Ssl (webfaction?)
-        self.getPage("/ssl", headers=[('X-Forwarded-Ssl', 'on')])
-        self.assertBody("https://www.mydomain.test")
-
-        # Test cherrypy.url()
-        for sn in script_names:
-            # Test the value inside requests
-            self.getPage(sn + "/newurl")
-            self.assertBody(
-                "Browse to this page.")
-            self.getPage(sn + "/newurl", headers=[('X-Forwarded-Host',
-                                                   'http://www.example.test')])
-            self.assertBody("Browse to this page.")
-
-            # Test the value outside requests
-            port = ""
-            if self.scheme == "http" and self.PORT != 80:
-                port = ":%s" % self.PORT
-            elif self.scheme == "https" and self.PORT != 443:
-                port = ":%s" % self.PORT
-            host = self.HOST
-            if host in ('0.0.0.0', '::'):
-                import socket
-                host = socket.gethostname()
-            expected = ("%s://%s%s%s/this/new/page"
-                        % (self.scheme, host, port, sn))
-            self.getPage(sn + "/pageurl")
-            self.assertBody(expected)
-
-        # Test trailing slash (see
-        # https://github.com/cherrypy/cherrypy/issues/562).
-        self.getPage("/xhost/", headers=[('X-Host', 'www.example.test')])
-        self.assertHeader('Location', "%s://www.example.test/xhost"
-                          % self.scheme)
diff --git a/lib/cherrypy/test/test_refleaks.py b/lib/cherrypy/test/test_refleaks.py
deleted file mode 100644
index 55f313edc6e..00000000000
--- a/lib/cherrypy/test/test_refleaks.py
+++ /dev/null
@@ -1,59 +0,0 @@
-"""Tests for refleaks."""
-
-from cherrypy._cpcompat import HTTPConnection, HTTPSConnection, ntob
-import threading
-
-import cherrypy
-
-
-data = object()
-
-
-from cherrypy.test import helper
-
-
-class ReferenceTests(helper.CPWebCase):
-
-    def setup_server():
-
-        class Root:
-
-            @cherrypy.expose
-            def index(self, *args, **kwargs):
-                cherrypy.request.thing = data
-                return "Hello world!"
-
-        cherrypy.tree.mount(Root())
-    setup_server = staticmethod(setup_server)
-
-    def test_threadlocal_garbage(self):
-        success = []
-
-        def getpage():
-            host = '%s:%s' % (self.interface(), self.PORT)
-            if self.scheme == 'https':
-                c = HTTPSConnection(host)
-            else:
-                c = HTTPConnection(host)
-            try:
-                c.putrequest('GET', '/')
-                c.endheaders()
-                response = c.getresponse()
-                body = response.read()
-                self.assertEqual(response.status, 200)
-                self.assertEqual(body, ntob("Hello world!"))
-            finally:
-                c.close()
-            success.append(True)
-
-        ITERATIONS = 25
-        ts = []
-        for _ in range(ITERATIONS):
-            t = threading.Thread(target=getpage)
-            ts.append(t)
-            t.start()
-
-        for t in ts:
-            t.join()
-
-        self.assertEqual(len(success), ITERATIONS)
diff --git a/lib/cherrypy/test/test_request_obj.py b/lib/cherrypy/test/test_request_obj.py
deleted file mode 100644
index 26c856820ac..00000000000
--- a/lib/cherrypy/test/test_request_obj.py
+++ /dev/null
@@ -1,808 +0,0 @@
-"""Basic tests for the cherrypy.Request object."""
-
-import os
-localDir = os.path.dirname(__file__)
-import sys
-import types
-
-import six
-
-from cherrypy._cpcompat import IncompleteRead, ntob, ntou
-
-import cherrypy
-from cherrypy.lib import httputil
-
-defined_http_methods = ("OPTIONS", "GET", "HEAD", "POST", "PUT", "DELETE",
-                        "TRACE", "PROPFIND")
-
-
-#                             Client-side code                             #
-
-from cherrypy.test import helper
-
-
-class RequestObjectTests(helper.CPWebCase):
-
-    def setup_server():
-        class Root:
-
-            @cherrypy.expose
-            def index(self):
-                return "hello"
-
-            @cherrypy.expose
-            def scheme(self):
-                return cherrypy.request.scheme
-
-        root = Root()
-
-        class TestType(type):
-            """Metaclass which automatically exposes all functions in each
-            subclass, and adds an instance of the subclass as an attribute
-            of root.
-            """
-            def __init__(cls, name, bases, dct):
-                type.__init__(cls, name, bases, dct)
-                for value in dct.values():
-                    if isinstance(value, types.FunctionType):
-                        value.exposed = True
-                setattr(root, name.lower(), cls())
-        Test = TestType('Test', (object,), {})
-
-        class PathInfo(Test):
-
-            def default(self, *args):
-                return cherrypy.request.path_info
-
-        class Params(Test):
-
-            def index(self, thing):
-                return repr(thing)
-
-            def ismap(self, x, y):
-                return "Coordinates: %s, %s" % (x, y)
-
-            @cherrypy.config(**{'request.query_string_encoding': 'latin1'})
-            def default(self, *args, **kwargs):
-                return "args: %s kwargs: %s" % (args, sorted(kwargs.items()))
-
-        @cherrypy.expose
-        class ParamErrorsCallable(object):
-
-            def __call__(self):
-                return "data"
-
-        class ParamErrors(Test):
-
-            @cherrypy.expose
-            def one_positional(self, param1):
-                return "data"
-
-            @cherrypy.expose
-            def one_positional_args(self, param1, *args):
-                return "data"
-
-            @cherrypy.expose
-            def one_positional_args_kwargs(self, param1, *args, **kwargs):
-                return "data"
-
-            @cherrypy.expose
-            def one_positional_kwargs(self, param1, **kwargs):
-                return "data"
-
-            @cherrypy.expose
-            def no_positional(self):
-                return "data"
-
-            @cherrypy.expose
-            def no_positional_args(self, *args):
-                return "data"
-
-            @cherrypy.expose
-            def no_positional_args_kwargs(self, *args, **kwargs):
-                return "data"
-
-            @cherrypy.expose
-            def no_positional_kwargs(self, **kwargs):
-                return "data"
-
-            callable_object = ParamErrorsCallable()
-
-            @cherrypy.expose
-            def raise_type_error(self, **kwargs):
-                raise TypeError("Client Error")
-
-            @cherrypy.expose
-            def raise_type_error_with_default_param(self, x, y=None):
-                return '%d' % 'a'  # throw an exception
-
-        def callable_error_page(status, **kwargs):
-            return "Error %s - Well, I'm very sorry but you haven't paid!" % (
-                status)
-
-        @cherrypy.config(**{'tools.log_tracebacks.on': True})
-        class Error(Test):
-
-            def reason_phrase(self):
-                raise cherrypy.HTTPError("410 Gone fishin'")
-
-            @cherrypy.config(**{
-                'error_page.404': os.path.join(localDir, "static/index.html"),
-                'error_page.401': callable_error_page,
-            })
-            def custom(self, err='404'):
-                raise cherrypy.HTTPError(
-                    int(err), "No, really, not found!")
-
-            @cherrypy.config(**{
-                'error_page.default': callable_error_page,
-            })
-            def custom_default(self):
-                return 1 + 'a'  # raise an unexpected error
-
-            @cherrypy.config(**{'error_page.404': "nonexistent.html"})
-            def noexist(self):
-                raise cherrypy.HTTPError(404, "No, really, not found!")
-
-            def page_method(self):
-                raise ValueError()
-
-            def page_yield(self):
-                yield "howdy"
-                raise ValueError()
-
-            @cherrypy.config(**{"response.stream": True})
-            def page_streamed(self):
-                yield "word up"
-                raise ValueError()
-                yield "very oops"
-
-            @cherrypy.config(**{'request.show_tracebacks': False})
-            def cause_err_in_finalize(self):
-                # Since status must start with an int, this should error.
-                cherrypy.response.status = "ZOO OK"
-
-            @cherrypy.config(**{'request.throw_errors': True})
-            def rethrow(self):
-                """Test that an error raised here will be thrown out to
-                the server.
-                """
-                raise ValueError()
-
-        class Expect(Test):
-
-            def expectation_failed(self):
-                expect = cherrypy.request.headers.elements("Expect")
-                if expect and expect[0].value != '100-continue':
-                    raise cherrypy.HTTPError(400)
-                raise cherrypy.HTTPError(417, 'Expectation Failed')
-
-        class Headers(Test):
-
-            def default(self, headername):
-                """Spit back out the value for the requested header."""
-                return cherrypy.request.headers[headername]
-
-            def doubledheaders(self):
-                # From https://github.com/cherrypy/cherrypy/issues/165:
-                # "header field names should not be case sensitive sayes the
-                # rfc. if i set a headerfield in complete lowercase i end up
-                # with two header fields, one in lowercase, the other in
-                # mixed-case."
-
-                # Set the most common headers
-                hMap = cherrypy.response.headers
-                hMap['content-type'] = "text/html"
-                hMap['content-length'] = 18
-                hMap['server'] = 'CherryPy headertest'
-                hMap['location'] = ('%s://%s:%s/headers/'
-                                    % (cherrypy.request.local.ip,
-                                       cherrypy.request.local.port,
-                                       cherrypy.request.scheme))
-
-                # Set a rare header for fun
-                hMap['Expires'] = 'Thu, 01 Dec 2194 16:00:00 GMT'
-
-                return "double header test"
-
-            def ifmatch(self):
-                val = cherrypy.request.headers['If-Match']
-                assert isinstance(val, six.text_type)
-                cherrypy.response.headers['ETag'] = val
-                return val
-
-        class HeaderElements(Test):
-
-            def get_elements(self, headername):
-                e = cherrypy.request.headers.elements(headername)
-                return "\n".join([six.text_type(x) for x in e])
-
-        class Method(Test):
-
-            def index(self):
-                m = cherrypy.request.method
-                if m in defined_http_methods or m == "CONNECT":
-                    return m
-
-                if m == "LINK":
-                    raise cherrypy.HTTPError(405)
-                else:
-                    raise cherrypy.HTTPError(501)
-
-            def parameterized(self, data):
-                return data
-
-            def request_body(self):
-                # This should be a file object (temp file),
-                # which CP will just pipe back out if we tell it to.
-                return cherrypy.request.body
-
-            def reachable(self):
-                return "success"
-
-        class Divorce:
-
-            """HTTP Method handlers shouldn't collide with normal method names.
-            For example, a GET-handler shouldn't collide with a method named
-            'get'.
-
-            If you build HTTP method dispatching into CherryPy, rewrite this
-            class to use your new dispatch mechanism and make sure that:
-                "GET /divorce HTTP/1.1" maps to divorce.index() and
-                "GET /divorce/get?ID=13 HTTP/1.1" maps to divorce.get()
-            """
-
-            documents = {}
-
-            @cherrypy.expose
-            def index(self):
-                yield "
Choose your document
\n"
-                yield "
    \n"
-                for id, contents in self.documents.items():
-                    yield (
-                        "    
  • %s:"
-                        " %s
    • \n" % (id, id, contents))
-                yield "
"
-
-            @cherrypy.expose
-            def get(self, ID):
-                return ("Divorce document %s: %s" %
-                        (ID, self.documents.get(ID, "empty")))
-
-        root.divorce = Divorce()
-
-        class ThreadLocal(Test):
-
-            def index(self):
-                existing = repr(getattr(cherrypy.request, "asdf", None))
-                cherrypy.request.asdf = "rassfrassin"
-                return existing
-
-        appconf = {
-            '/method': {
-                'request.methods_with_bodies': ("POST", "PUT", "PROPFIND")
-            },
-        }
-        cherrypy.tree.mount(root, config=appconf)
-    setup_server = staticmethod(setup_server)
-
-    def test_scheme(self):
-        self.getPage("/scheme")
-        self.assertBody(self.scheme)
-
-    def testRelativeURIPathInfo(self):
-        self.getPage("/pathinfo/foo/bar")
-        self.assertBody("/pathinfo/foo/bar")
-
-    def testAbsoluteURIPathInfo(self):
-        # http://cherrypy.org/ticket/1061
-        self.getPage("http://localhost/pathinfo/foo/bar")
-        self.assertBody("/pathinfo/foo/bar")
-
-    def testParams(self):
-        self.getPage("/params/?thing=a")
-        self.assertBody(repr(ntou("a")))
-
-        self.getPage("/params/?thing=a&thing=b&thing=c")
-        self.assertBody(repr([ntou('a'), ntou('b'), ntou('c')]))
-
-        # Test friendly error message when given params are not accepted.
-        cherrypy.config.update({"request.show_mismatched_params": True})
-        self.getPage("/params/?notathing=meeting")
-        self.assertInBody("Missing parameters: thing")
-        self.getPage("/params/?thing=meeting¬athing=meeting")
-        self.assertInBody("Unexpected query string parameters: notathing")
-
-        # Test ability to turn off friendly error messages
-        cherrypy.config.update({"request.show_mismatched_params": False})
-        self.getPage("/params/?notathing=meeting")
-        self.assertInBody("Not Found")
-        self.getPage("/params/?thing=meeting¬athing=meeting")
-        self.assertInBody("Not Found")
-
-        # Test "% HEX HEX"-encoded URL, param keys, and values
-        self.getPage("/params/%d4%20%e3/cheese?Gruy%E8re=Bulgn%e9ville")
-        self.assertBody("args: %s kwargs: %s" %
-                        (('\xd4 \xe3', 'cheese'),
-                         [('Gruy\xe8re', ntou('Bulgn\xe9ville'))]))
-
-        # Make sure that encoded = and & get parsed correctly
-        self.getPage(
-            "/params/code?url=http%3A//cherrypy.org/index%3Fa%3D1%26b%3D2")
-        self.assertBody("args: %s kwargs: %s" %
-                        (('code',),
-                         [('url', ntou('http://cherrypy.org/index?a=1&b=2'))]))
-
-        # Test coordinates sent by 
-        self.getPage("/params/ismap?223,114")
-        self.assertBody("Coordinates: 223, 114")
-
-        # Test "name[key]" dict-like params
-        self.getPage("/params/dictlike?a[1]=1&a[2]=2&b=foo&b[bar]=baz")
-        self.assertBody("args: %s kwargs: %s" %
-                        (('dictlike',),
-                         [('a[1]', ntou('1')), ('a[2]', ntou('2')),
-                          ('b', ntou('foo')), ('b[bar]', ntou('baz'))]))
-
-    def testParamErrors(self):
-
-        # test that all of the handlers work when given
-        # the correct parameters in order to ensure that the
-        # errors below aren't coming from some other source.
-        for uri in (
-                '/paramerrors/one_positional?param1=foo',
-                '/paramerrors/one_positional_args?param1=foo',
-                '/paramerrors/one_positional_args/foo',
-                '/paramerrors/one_positional_args/foo/bar/baz',
-                '/paramerrors/one_positional_args_kwargs?'
-                'param1=foo¶m2=bar',
-                '/paramerrors/one_positional_args_kwargs/foo?'
-                'param2=bar¶m3=baz',
-                '/paramerrors/one_positional_args_kwargs/foo/bar/baz?'
-                'param2=bar¶m3=baz',
-                '/paramerrors/one_positional_kwargs?'
-                'param1=foo¶m2=bar¶m3=baz',
-                '/paramerrors/one_positional_kwargs/foo?'
-                'param4=foo¶m2=bar¶m3=baz',
-                '/paramerrors/no_positional',
-                '/paramerrors/no_positional_args/foo',
-                '/paramerrors/no_positional_args/foo/bar/baz',
-                '/paramerrors/no_positional_args_kwargs?param1=foo¶m2=bar',
-                '/paramerrors/no_positional_args_kwargs/foo?param2=bar',
-                '/paramerrors/no_positional_args_kwargs/foo/bar/baz?'
-                'param2=bar¶m3=baz',
-                '/paramerrors/no_positional_kwargs?param1=foo¶m2=bar',
-                '/paramerrors/callable_object',
-        ):
-            self.getPage(uri)
-            self.assertStatus(200)
-
-        error_msgs = [
-            'Missing parameters',
-            'Nothing matches the given URI',
-            'Multiple values for parameters',
-            'Unexpected query string parameters',
-            'Unexpected body parameters',
-            'Invalid path in Request-URI',
-            'Illegal #fragment in Request-URI',
-        ]
-
-        # uri should be tested for valid absolute path, the status must be 400.
-        for uri, error_idx in (
-            ('invalid/path/without/leading/slash', 5),
-            ('/valid/path#invalid=fragment', 6),
-        ):
-            self.getPage(uri)
-            self.assertStatus(400)
-            self.assertInBody(error_msgs[error_idx])
-
-        # query string parameters are part of the URI, so if they are wrong
-        # for a particular handler, the status MUST be a 404.
-        for uri, msg in (
-            ('/paramerrors/one_positional', error_msgs[0]),
-            ('/paramerrors/one_positional?foo=foo', error_msgs[0]),
-            ('/paramerrors/one_positional/foo/bar/baz', error_msgs[1]),
-            ('/paramerrors/one_positional/foo?param1=foo', error_msgs[2]),
-            ('/paramerrors/one_positional/foo?param1=foo¶m2=foo',
-             error_msgs[2]),
-            ('/paramerrors/one_positional_args/foo?param1=foo¶m2=foo',
-             error_msgs[2]),
-            ('/paramerrors/one_positional_args/foo/bar/baz?param2=foo',
-             error_msgs[3]),
-            ('/paramerrors/one_positional_args_kwargs/foo/bar/baz?'
-             'param1=bar¶m3=baz',
-             error_msgs[2]),
-            ('/paramerrors/one_positional_kwargs/foo?'
-             'param1=foo¶m2=bar¶m3=baz',
-             error_msgs[2]),
-            ('/paramerrors/no_positional/boo', error_msgs[1]),
-            ('/paramerrors/no_positional?param1=foo', error_msgs[3]),
-            ('/paramerrors/no_positional_args/boo?param1=foo', error_msgs[3]),
-            ('/paramerrors/no_positional_kwargs/boo?param1=foo',
-             error_msgs[1]),
-            ('/paramerrors/callable_object?param1=foo', error_msgs[3]),
-            ('/paramerrors/callable_object/boo', error_msgs[1]),
-        ):
-            for show_mismatched_params in (True, False):
-                cherrypy.config.update(
-                    {'request.show_mismatched_params': show_mismatched_params})
-                self.getPage(uri)
-                self.assertStatus(404)
-                if show_mismatched_params:
-                    self.assertInBody(msg)
-                else:
-                    self.assertInBody("Not Found")
-
-        # if body parameters are wrong, a 400 must be returned.
-        for uri, body, msg in (
-                ('/paramerrors/one_positional/foo',
-                 'param1=foo', error_msgs[2]),
-                ('/paramerrors/one_positional/foo',
-                 'param1=foo¶m2=foo', error_msgs[2]),
-                ('/paramerrors/one_positional_args/foo',
-                 'param1=foo¶m2=foo', error_msgs[2]),
-                ('/paramerrors/one_positional_args/foo/bar/baz',
-                 'param2=foo', error_msgs[4]),
-                ('/paramerrors/one_positional_args_kwargs/foo/bar/baz',
-                 'param1=bar¶m3=baz', error_msgs[2]),
-                ('/paramerrors/one_positional_kwargs/foo',
-                 'param1=foo¶m2=bar¶m3=baz', error_msgs[2]),
-                ('/paramerrors/no_positional', 'param1=foo', error_msgs[4]),
-                ('/paramerrors/no_positional_args/boo',
-                 'param1=foo', error_msgs[4]),
-                ('/paramerrors/callable_object', 'param1=foo', error_msgs[4]),
-        ):
-            for show_mismatched_params in (True, False):
-                cherrypy.config.update(
-                    {'request.show_mismatched_params': show_mismatched_params})
-                self.getPage(uri, method='POST', body=body)
-                self.assertStatus(400)
-                if show_mismatched_params:
-                    self.assertInBody(msg)
-                else:
-                    self.assertInBody("400 Bad")
-
-        # even if body parameters are wrong, if we get the uri wrong, then
-        # it's a 404
-        for uri, body, msg in (
-                ('/paramerrors/one_positional?param2=foo',
-                 'param1=foo', error_msgs[3]),
-                ('/paramerrors/one_positional/foo/bar',
-                 'param2=foo', error_msgs[1]),
-                ('/paramerrors/one_positional_args/foo/bar?param2=foo',
-                 'param3=foo', error_msgs[3]),
-                ('/paramerrors/one_positional_kwargs/foo/bar',
-                 'param2=bar¶m3=baz', error_msgs[1]),
-                ('/paramerrors/no_positional?param1=foo',
-                 'param2=foo', error_msgs[3]),
-                ('/paramerrors/no_positional_args/boo?param2=foo',
-                 'param1=foo', error_msgs[3]),
-                ('/paramerrors/callable_object?param2=bar',
-                 'param1=foo', error_msgs[3]),
-        ):
-            for show_mismatched_params in (True, False):
-                cherrypy.config.update(
-                    {'request.show_mismatched_params': show_mismatched_params})
-                self.getPage(uri, method='POST', body=body)
-                self.assertStatus(404)
-                if show_mismatched_params:
-                    self.assertInBody(msg)
-                else:
-                    self.assertInBody("Not Found")
-
-        # In the case that a handler raises a TypeError we should
-        # let that type error through.
-        for uri in (
-                '/paramerrors/raise_type_error',
-                '/paramerrors/raise_type_error_with_default_param?x=0',
-                '/paramerrors/raise_type_error_with_default_param?x=0&y=0',
-        ):
-            self.getPage(uri, method='GET')
-            self.assertStatus(500)
-            self.assertTrue('Client Error', self.body)
-
-    def testErrorHandling(self):
-        self.getPage("/error/missing")
-        self.assertStatus(404)
-        self.assertErrorPage(404, "The path '/error/missing' was not found.")
-
-        ignore = helper.webtest.ignored_exceptions
-        ignore.append(ValueError)
-        try:
-            valerr = '\n    raise ValueError()\nValueError'
-            self.getPage("/error/page_method")
-            self.assertErrorPage(500, pattern=valerr)
-
-            self.getPage("/error/page_yield")
-            self.assertErrorPage(500, pattern=valerr)
-
-            if (cherrypy.server.protocol_version == "HTTP/1.0" or
-                    getattr(cherrypy.server, "using_apache", False)):
-                self.getPage("/error/page_streamed")
-                # Because this error is raised after the response body has
-                # started, the status should not change to an error status.
-                self.assertStatus(200)
-                self.assertBody("word up")
-            else:
-                # Under HTTP/1.1, the chunked transfer-coding is used.
-                # The HTTP client will choke when the output is incomplete.
-                self.assertRaises((ValueError, IncompleteRead), self.getPage,
-                                  "/error/page_streamed")
-
-            # No traceback should be present
-            self.getPage("/error/cause_err_in_finalize")
-            msg = "Illegal response status from server ('ZOO' is non-numeric)."
-            self.assertErrorPage(500, msg, None)
-        finally:
-            ignore.pop()
-
-        # Test HTTPError with a reason-phrase in the status arg.
-        self.getPage('/error/reason_phrase')
-        self.assertStatus("410 Gone fishin'")
-
-        # Test custom error page for a specific error.
-        self.getPage("/error/custom")
-        self.assertStatus(404)
-        self.assertBody("Hello, world\r\n" + (" " * 499))
-
-        # Test custom error page for a specific error.
-        self.getPage("/error/custom?err=401")
-        self.assertStatus(401)
-        self.assertBody(
-            "Error 401 Unauthorized - "
-            "Well, I'm very sorry but you haven't paid!")
-
-        # Test default custom error page.
-        self.getPage("/error/custom_default")
-        self.assertStatus(500)
-        self.assertBody(
-            "Error 500 Internal Server Error - "
-            "Well, I'm very sorry but you haven't paid!".ljust(513))
-
-        # Test error in custom error page (ticket #305).
-        # Note that the message is escaped for HTML (ticket #310).
-        self.getPage("/error/noexist")
-        self.assertStatus(404)
-        if sys.version_info >= (3, 3):
-            exc_name = "FileNotFoundError"
-        else:
-            exc_name = "IOError"
-        msg = ("No, <b>really</b>, not found!
"
-               "In addition, the custom error page failed:\n
"
-               "%s: [Errno 2] "
-               "No such file or directory: 'nonexistent.html'") % (exc_name,)
-        self.assertInBody(msg)
-
-        if getattr(cherrypy.server, "using_apache", False):
-            pass
-        else:
-            # Test throw_errors (ticket #186).
-            self.getPage("/error/rethrow")
-            self.assertInBody("raise ValueError()")
-
-    def testExpect(self):
-        e = ('Expect', '100-continue')
-        self.getPage("/headerelements/get_elements?headername=Expect", [e])
-        self.assertBody('100-continue')
-
-        self.getPage("/expect/expectation_failed", [e])
-        self.assertStatus(417)
-
-    def testHeaderElements(self):
-        # Accept-* header elements should be sorted, with most preferred first.
-        h = [('Accept', 'audio/*; q=0.2, audio/basic')]
-        self.getPage("/headerelements/get_elements?headername=Accept", h)
-        self.assertStatus(200)
-        self.assertBody("audio/basic\n"
-                        "audio/*;q=0.2")
-
-        h = [
-            ('Accept',
-             'text/plain; q=0.5, text/html, text/x-dvi; q=0.8, text/x-c')
-        ]
-        self.getPage("/headerelements/get_elements?headername=Accept", h)
-        self.assertStatus(200)
-        self.assertBody("text/x-c\n"
-                        "text/html\n"
-                        "text/x-dvi;q=0.8\n"
-                        "text/plain;q=0.5")
-
-        # Test that more specific media ranges get priority.
-        h = [('Accept', 'text/*, text/html, text/html;level=1, */*')]
-        self.getPage("/headerelements/get_elements?headername=Accept", h)
-        self.assertStatus(200)
-        self.assertBody("text/html;level=1\n"
-                        "text/html\n"
-                        "text/*\n"
-                        "*/*")
-
-        # Test Accept-Charset
-        h = [('Accept-Charset', 'iso-8859-5, unicode-1-1;q=0.8')]
-        self.getPage(
-            "/headerelements/get_elements?headername=Accept-Charset", h)
-        self.assertStatus("200 OK")
-        self.assertBody("iso-8859-5\n"
-                        "unicode-1-1;q=0.8")
-
-        # Test Accept-Encoding
-        h = [('Accept-Encoding', 'gzip;q=1.0, identity; q=0.5, *;q=0')]
-        self.getPage(
-            "/headerelements/get_elements?headername=Accept-Encoding", h)
-        self.assertStatus("200 OK")
-        self.assertBody("gzip;q=1.0\n"
-                        "identity;q=0.5\n"
-                        "*;q=0")
-
-        # Test Accept-Language
-        h = [('Accept-Language', 'da, en-gb;q=0.8, en;q=0.7')]
-        self.getPage(
-            "/headerelements/get_elements?headername=Accept-Language", h)
-        self.assertStatus("200 OK")
-        self.assertBody("da\n"
-                        "en-gb;q=0.8\n"
-                        "en;q=0.7")
-
-        # Test malformed header parsing. See
-        # https://github.com/cherrypy/cherrypy/issues/763.
-        self.getPage("/headerelements/get_elements?headername=Content-Type",
-                     # Note the illegal trailing ";"
-                     headers=[('Content-Type', 'text/html; charset=utf-8;')])
-        self.assertStatus(200)
-        self.assertBody("text/html;charset=utf-8")
-
-    def test_repeated_headers(self):
-        # Test that two request headers are collapsed into one.
-        # See https://github.com/cherrypy/cherrypy/issues/542.
-        self.getPage("/headers/Accept-Charset",
-                     headers=[("Accept-Charset", "iso-8859-5"),
-                              ("Accept-Charset", "unicode-1-1;q=0.8")])
-        self.assertBody("iso-8859-5, unicode-1-1;q=0.8")
-
-        # Tests that each header only appears once, regardless of case.
-        self.getPage("/headers/doubledheaders")
-        self.assertBody("double header test")
-        hnames = [name.title() for name, val in self.headers]
-        for key in ['Content-Length', 'Content-Type', 'Date',
-                    'Expires', 'Location', 'Server']:
-            self.assertEqual(hnames.count(key), 1, self.headers)
-
-    def test_encoded_headers(self):
-        # First, make sure the innards work like expected.
-        self.assertEqual(
-            httputil.decode_TEXT(ntou("=?utf-8?q?f=C3=BCr?=")), ntou("f\xfcr"))
-
-        if cherrypy.server.protocol_version == "HTTP/1.1":
-            # Test RFC-2047-encoded request and response header values
-            u = ntou('\u212bngstr\xf6m', 'escape')
-            c = ntou("=E2=84=ABngstr=C3=B6m")
-            self.getPage("/headers/ifmatch",
-                         [('If-Match', ntou('=?utf-8?q?%s?=') % c)])
-            # The body should be utf-8 encoded.
-            self.assertBody(ntob("\xe2\x84\xabngstr\xc3\xb6m"))
-            # But the Etag header should be RFC-2047 encoded (binary)
-            self.assertHeader("ETag", ntou('=?utf-8?b?4oSrbmdzdHLDtm0=?='))
-
-            # Test a *LONG* RFC-2047-encoded request and response header value
-            self.getPage("/headers/ifmatch",
-                         [('If-Match', ntou('=?utf-8?q?%s?=') % (c * 10))])
-            self.assertBody(ntob("\xe2\x84\xabngstr\xc3\xb6m") * 10)
-            # Note: this is different output for Python3, but it decodes fine.
-            etag = self.assertHeader(
-                "ETag",
-                '=?utf-8?b?4oSrbmdzdHLDtm3ihKtuZ3N0csO2beKEq25nc3Ryw7Zt'
-                '4oSrbmdzdHLDtm3ihKtuZ3N0csO2beKEq25nc3Ryw7Zt'
-                '4oSrbmdzdHLDtm3ihKtuZ3N0csO2beKEq25nc3Ryw7Zt'
-                '4oSrbmdzdHLDtm0=?=')
-            self.assertEqual(httputil.decode_TEXT(etag), u * 10)
-
-    def test_header_presence(self):
-        # If we don't pass a Content-Type header, it should not be present
-        # in cherrypy.request.headers
-        self.getPage("/headers/Content-Type",
-                     headers=[])
-        self.assertStatus(500)
-
-        # If Content-Type is present in the request, it should be present in
-        # cherrypy.request.headers
-        self.getPage("/headers/Content-Type",
-                     headers=[("Content-type", "application/json")])
-        self.assertBody("application/json")
-
-    def test_basic_HTTPMethods(self):
-        helper.webtest.methods_with_bodies = ("POST", "PUT", "PROPFIND")
-
-        # Test that all defined HTTP methods work.
-        for m in defined_http_methods:
-            self.getPage("/method/", method=m)
-
-            # HEAD requests should not return any body.
-            if m == "HEAD":
-                self.assertBody("")
-            elif m == "TRACE":
-                # Some HTTP servers (like modpy) have their own TRACE support
-                self.assertEqual(self.body[:5], ntob("TRACE"))
-            else:
-                self.assertBody(m)
-
-        # Request a PUT method with a form-urlencoded body
-        self.getPage("/method/parameterized", method="PUT",
-                     body="data=on+top+of+other+things")
-        self.assertBody("on top of other things")
-
-        # Request a PUT method with a file body
-        b = "one thing on top of another"
-        h = [("Content-Type", "text/plain"),
-             ("Content-Length", str(len(b)))]
-        self.getPage("/method/request_body", headers=h, method="PUT", body=b)
-        self.assertStatus(200)
-        self.assertBody(b)
-
-        # Request a PUT method with a file body but no Content-Type.
-        # See https://github.com/cherrypy/cherrypy/issues/790.
-        b = ntob("one thing on top of another")
-        self.persistent = True
-        try:
-            conn = self.HTTP_CONN
-            conn.putrequest("PUT", "/method/request_body", skip_host=True)
-            conn.putheader("Host", self.HOST)
-            conn.putheader('Content-Length', str(len(b)))
-            conn.endheaders()
-            conn.send(b)
-            response = conn.response_class(conn.sock, method="PUT")
-            response.begin()
-            self.assertEqual(response.status, 200)
-            self.body = response.read()
-            self.assertBody(b)
-        finally:
-            self.persistent = False
-
-        # Request a PUT method with no body whatsoever (not an empty one).
-        # See https://github.com/cherrypy/cherrypy/issues/650.
-        # Provide a C-T or webtest will provide one (and a C-L) for us.
-        h = [("Content-Type", "text/plain")]
-        self.getPage("/method/reachable", headers=h, method="PUT")
-        self.assertStatus(411)
-
-        # Request a custom method with a request body
-        b = ('\n\n'
-             ''
-             '')
-        h = [('Content-Type', 'text/xml'),
-             ('Content-Length', str(len(b)))]
-        self.getPage("/method/request_body", headers=h,
-                     method="PROPFIND", body=b)
-        self.assertStatus(200)
-        self.assertBody(b)
-
-        # Request a disallowed method
-        self.getPage("/method/", method="LINK")
-        self.assertStatus(405)
-
-        # Request an unknown method
-        self.getPage("/method/", method="SEARCH")
-        self.assertStatus(501)
-
-        # For method dispatchers: make sure that an HTTP method doesn't
-        # collide with a virtual path atom. If you build HTTP-method
-        # dispatching into the core, rewrite these handlers to use
-        # your dispatch idioms.
-        self.getPage("/divorce/get?ID=13")
-        self.assertBody('Divorce document 13: empty')
-        self.assertStatus(200)
-        self.getPage("/divorce/", method="GET")
-        self.assertBody('
Choose your document
\n
    \n
')
-        self.assertStatus(200)
-
-    def test_CONNECT_method(self):
-        if getattr(cherrypy.server, "using_apache", False):
-            return self.skip("skipped due to known Apache differences... ")
-
-        self.getPage("/method/", method="CONNECT")
-        self.assertBody("CONNECT")
-
-    def testEmptyThreadlocals(self):
-        results = []
-        for x in range(20):
-            self.getPage("/threadlocal/")
-            results.append(self.body)
-        self.assertEqual(results, [ntob("None")] * 20)
diff --git a/lib/cherrypy/test/test_routes.py b/lib/cherrypy/test/test_routes.py
deleted file mode 100644
index 27c02ea8a19..00000000000
--- a/lib/cherrypy/test/test_routes.py
+++ /dev/null
@@ -1,75 +0,0 @@
-import os
-curdir = os.path.join(os.getcwd(), os.path.dirname(__file__))
-
-import cherrypy
-
-from cherrypy.test import helper
-import nose
-
-
-class RoutesDispatchTest(helper.CPWebCase):
-
-    def setup_server():
-
-        try:
-            import routes
-        except ImportError:
-            raise nose.SkipTest('Install routes to test RoutesDispatcher code')
-
-        class Dummy:
-
-            def index(self):
-                return "I said good day!"
-
-        class City:
-
-            def __init__(self, name):
-                self.name = name
-                self.population = 10000
-
-            @cherrypy.config(**{
-                'tools.response_headers.on': True,
-                'tools.response_headers.headers': [
-                    ('Content-Language', 'en-GB'),
-                ],
-            })
-            def index(self, **kwargs):
-                return "Welcome to %s, pop. %s" % (self.name, self.population)
-
-            def update(self, **kwargs):
-                self.population = kwargs['pop']
-                return "OK"
-
-        d = cherrypy.dispatch.RoutesDispatcher()
-        d.connect(action='index', name='hounslow', route='/hounslow',
-                  controller=City('Hounslow'))
-        d.connect(
-            name='surbiton', route='/surbiton', controller=City('Surbiton'),
-            action='index', conditions=dict(method=['GET']))
-        d.mapper.connect('/surbiton', controller='surbiton',
-                         action='update', conditions=dict(method=['POST']))
-        d.connect('main', ':action', controller=Dummy())
-
-        conf = {'/': {'request.dispatch': d}}
-        cherrypy.tree.mount(root=None, config=conf)
-    setup_server = staticmethod(setup_server)
-
-    def test_Routes_Dispatch(self):
-        self.getPage("/hounslow")
-        self.assertStatus("200 OK")
-        self.assertBody("Welcome to Hounslow, pop. 10000")
-
-        self.getPage("/foo")
-        self.assertStatus("404 Not Found")
-
-        self.getPage("/surbiton")
-        self.assertStatus("200 OK")
-        self.assertBody("Welcome to Surbiton, pop. 10000")
-
-        self.getPage("/surbiton", method="POST", body="pop=1327")
-        self.assertStatus("200 OK")
-        self.assertBody("OK")
-        self.getPage("/surbiton")
-        self.assertStatus("200 OK")
-        self.assertHeader("Content-Language", "en-GB")
-        self.assertBody("Welcome to Surbiton, pop. 1327")
diff --git a/lib/cherrypy/test/test_session.py b/lib/cherrypy/test/test_session.py
deleted file mode 100755
index 3bf14f58e6f..00000000000
--- a/lib/cherrypy/test/test_session.py
+++ /dev/null
@@ -1,490 +0,0 @@
-import os
-localDir = os.path.dirname(__file__)
-import threading
-import time
-
-import cherrypy
-from cherrypy._cpcompat import copykeys, HTTPConnection, HTTPSConnection
-from cherrypy.lib import sessions
-from cherrypy.lib.httputil import response_codes
-
-
-def http_methods_allowed(methods=['GET', 'HEAD']):
-    method = cherrypy.request.method.upper()
-    if method not in methods:
-        cherrypy.response.headers['Allow'] = ", ".join(methods)
-        raise cherrypy.HTTPError(405)
-
-cherrypy.tools.allow = cherrypy.Tool('on_start_resource', http_methods_allowed)
-
-
-def setup_server():
-
-    @cherrypy.config(**{
-        'tools.sessions.on': True,
-        'tools.sessions.storage_type': 'ram',
-        'tools.sessions.storage_path': localDir,
-        'tools.sessions.timeout': (1.0 / 60),
-        'tools.sessions.clean_freq': (1.0 / 60),
-    })
-    class Root:
-
-        @cherrypy.expose
-        def clear(self):
-            cherrypy.session.cache.clear()
-
-        @cherrypy.expose
-        def data(self):
-            cherrypy.session['aha'] = 'foo'
-            return repr(cherrypy.session._data)
-
-        @cherrypy.expose
-        def testGen(self):
-            counter = cherrypy.session.get('counter', 0) + 1
-            cherrypy.session['counter'] = counter
-            yield str(counter)
-
-        @cherrypy.expose
-        def testStr(self):
-            counter = cherrypy.session.get('counter', 0) + 1
-            cherrypy.session['counter'] = counter
-            return str(counter)
-
-        @cherrypy.expose
-        @cherrypy.config(**{'tools.sessions.on': False})
-        def setsessiontype(self, newtype):
-            self.__class__._cp_config.update(
-                {'tools.sessions.storage_type': newtype})
-            if hasattr(cherrypy, "session"):
-                del cherrypy.session
-            cls = getattr(sessions, newtype.title() + 'Session')
-            if cls.clean_thread:
-                cls.clean_thread.stop()
-                cls.clean_thread.unsubscribe()
-                del cls.clean_thread
-
-        @cherrypy.expose
-        def index(self):
-            sess = cherrypy.session
-            c = sess.get('counter', 0) + 1
-            time.sleep(0.01)
-            sess['counter'] = c
-            return str(c)
-
-        @cherrypy.expose
-        def keyin(self, key):
-            return str(key in cherrypy.session)
-
-        @cherrypy.expose
-        def delete(self):
-            cherrypy.session.delete()
-            sessions.expire()
-            return "done"
-
-        @cherrypy.expose
-        def delkey(self, key):
-            del cherrypy.session[key]
-            return "OK"
-
-        @cherrypy.expose
-        def blah(self):
-            return self._cp_config['tools.sessions.storage_type']
-
-        @cherrypy.expose
-        def iredir(self):
-            raise cherrypy.InternalRedirect('/blah')
-
-        @cherrypy.expose
-        @cherrypy.config(**{
-            'tools.allow.on': True,
-            'tools.allow.methods': ['GET'],
-        })
-        def restricted(self):
-            return cherrypy.request.method
-
-        @cherrypy.expose
-        def regen(self):
-            cherrypy.tools.sessions.regenerate()
-            return "logged in"
-
-        @cherrypy.expose
-        def length(self):
-            return str(len(cherrypy.session))
-
-        @cherrypy.expose
-        @cherrypy.config(**{
-            'tools.sessions.path': '/session_cookie',
-            'tools.sessions.name': 'temp',
-            'tools.sessions.persistent': False,
-        })
-        def session_cookie(self):
-            # Must load() to start the clean thread.
-            cherrypy.session.load()
-            return cherrypy.session.id
-
-    cherrypy.tree.mount(Root())
-
-
-from cherrypy.test import helper
-
-
-class SessionTest(helper.CPWebCase):
-    setup_server = staticmethod(setup_server)
-
-    def tearDown(self):
-        # Clean up sessions.
-        for fname in os.listdir(localDir):
-            if fname.startswith(sessions.FileSession.SESSION_PREFIX):
-                os.unlink(os.path.join(localDir, fname))
-
-    def test_0_Session(self):
-        self.getPage('/setsessiontype/ram')
-        self.getPage('/clear')
-
-        # Test that a normal request gets the same id in the cookies.
-        # Note: this wouldn't work if /data didn't load the session.
-        self.getPage('/data')
-        self.assertBody("{'aha': 'foo'}")
-        c = self.cookies[0]
-        self.getPage('/data', self.cookies)
-        self.assertEqual(self.cookies[0], c)
-
-        self.getPage('/testStr')
-        self.assertBody('1')
-        cookie_parts = dict([p.strip().split('=')
-                             for p in self.cookies[0][1].split(";")])
-        # Assert there is an 'expires' param
-        self.assertEqual(set(cookie_parts.keys()),
-                         set(['session_id', 'expires', 'Path']))
-        self.getPage('/testGen', self.cookies)
-        self.assertBody('2')
-        self.getPage('/testStr', self.cookies)
-        self.assertBody('3')
-        self.getPage('/data', self.cookies)
-        self.assertBody("{'aha': 'foo', 'counter': 3}")
-        self.getPage('/length', self.cookies)
-        self.assertBody('2')
-        self.getPage('/delkey?key=counter', self.cookies)
-        self.assertStatus(200)
-
-        self.getPage('/setsessiontype/file')
-        self.getPage('/testStr')
-        self.assertBody('1')
-        self.getPage('/testGen', self.cookies)
-        self.assertBody('2')
-        self.getPage('/testStr', self.cookies)
-        self.assertBody('3')
-        self.getPage('/delkey?key=counter', self.cookies)
-        self.assertStatus(200)
-
-        # Wait for the session.timeout (1 second)
-        time.sleep(2)
-        self.getPage('/')
-        self.assertBody('1')
-        self.getPage('/length', self.cookies)
-        self.assertBody('1')
-
-        # Test session __contains__
-        self.getPage('/keyin?key=counter', self.cookies)
-        self.assertBody("True")
-        cookieset1 = self.cookies
-
-        # Make a new session and test __len__ again
-        self.getPage('/')
-        self.getPage('/length', self.cookies)
-        self.assertBody('2')
-
-        # Test session delete
-        self.getPage('/delete', self.cookies)
-        self.assertBody("done")
-        self.getPage('/delete', cookieset1)
-        self.assertBody("done")
-        f = lambda: [
-            x for x in os.listdir(localDir) if x.startswith('session-')]
-        self.assertEqual(f(), [])
-
-        # Wait for the cleanup thread to delete remaining session files
-        self.getPage('/')
-        f = lambda: [
-            x for x in os.listdir(localDir) if x.startswith('session-')]
-        self.assertNotEqual(f(), [])
-        time.sleep(2)
-        self.assertEqual(f(), [])
-
-    def test_1_Ram_Concurrency(self):
-        self.getPage('/setsessiontype/ram')
-        self._test_Concurrency()
-
-    def test_2_File_Concurrency(self):
-        self.getPage('/setsessiontype/file')
-        self._test_Concurrency()
-
-    def _test_Concurrency(self):
-        client_thread_count = 5
-        request_count = 30
-
-        # Get initial cookie
-        self.getPage("/")
-        self.assertBody("1")
-        cookies = self.cookies
-
-        data_dict = {}
-        errors = []
-
-        def request(index):
-            if self.scheme == 'https':
-                c = HTTPSConnection('%s:%s' % (self.interface(), self.PORT))
-            else:
-                c = HTTPConnection('%s:%s' % (self.interface(), self.PORT))
-            for i in range(request_count):
-                c.putrequest('GET', '/')
-                for k, v in cookies:
-                    c.putheader(k, v)
-                c.endheaders()
-                response = c.getresponse()
-                body = response.read()
-                if response.status != 200 or not body.isdigit():
-                    errors.append((response.status, body))
-                else:
-                    data_dict[index] = max(data_dict[index], int(body))
-                # Uncomment the following line to prove threads overlap.
-##                sys.stdout.write("%d " % index)
-
-        # Start  requests from each of
-        #  concurrent clients
-        ts = []
-        for c in range(client_thread_count):
-            data_dict[c] = 0
-            t = threading.Thread(target=request, args=(c,))
-            ts.append(t)
-            t.start()
-
-        for t in ts:
-            t.join()
-
-        hitcount = max(data_dict.values())
-        expected = 1 + (client_thread_count * request_count)
-
-        for e in errors:
-            print(e)
-        self.assertEqual(hitcount, expected)
-
-    def test_3_Redirect(self):
-        # Start a new session
-        self.getPage('/testStr')
-        self.getPage('/iredir', self.cookies)
-        self.assertBody("file")
-
-    def test_4_File_deletion(self):
-        # Start a new session
-        self.getPage('/testStr')
-        # Delete the session file manually and retry.
-        id = self.cookies[0][1].split(";", 1)[0].split("=", 1)[1]
-        path = os.path.join(localDir, "session-" + id)
-        os.unlink(path)
-        self.getPage('/testStr', self.cookies)
-
-    def test_5_Error_paths(self):
-        self.getPage('/unknown/page')
-        self.assertErrorPage(404, "The path '/unknown/page' was not found.")
-
-        # Note: this path is *not* the same as above. The above
-        # takes a normal route through the session code; this one
-        # skips the session code's before_handler and only calls
-        # before_finalize (save) and on_end (close). So the session
-        # code has to survive calling save/close without init.
-        self.getPage('/restricted', self.cookies, method='POST')
-        self.assertErrorPage(405, response_codes[405][1])
-
-    def test_6_regenerate(self):
-        self.getPage('/testStr')
-        # grab the cookie ID
-        id1 = self.cookies[0][1].split(";", 1)[0].split("=", 1)[1]
-        self.getPage('/regen')
-        self.assertBody('logged in')
-        id2 = self.cookies[0][1].split(";", 1)[0].split("=", 1)[1]
-        self.assertNotEqual(id1, id2)
-
-        self.getPage('/testStr')
-        # grab the cookie ID
-        id1 = self.cookies[0][1].split(";", 1)[0].split("=", 1)[1]
-        self.getPage('/testStr',
-                     headers=[
-                         ('Cookie',
-                          'session_id=maliciousid; '
-                          'expires=Sat, 27 Oct 2017 04:18:28 GMT; Path=/;')])
-        id2 = self.cookies[0][1].split(";", 1)[0].split("=", 1)[1]
-        self.assertNotEqual(id1, id2)
-        self.assertNotEqual(id2, 'maliciousid')
-
-    def test_7_session_cookies(self):
-        self.getPage('/setsessiontype/ram')
-        self.getPage('/clear')
-        self.getPage('/session_cookie')
-        # grab the cookie ID
-        cookie_parts = dict([p.strip().split('=')
-                            for p in self.cookies[0][1].split(";")])
-        # Assert there is no 'expires' param
-        self.assertEqual(set(cookie_parts.keys()), set(['temp', 'Path']))
-        id1 = cookie_parts['temp']
-        self.assertEqual(copykeys(sessions.RamSession.cache), [id1])
-
-        # Send another request in the same "browser session".
-        self.getPage('/session_cookie', self.cookies)
-        cookie_parts = dict([p.strip().split('=')
-                            for p in self.cookies[0][1].split(";")])
-        # Assert there is no 'expires' param
-        self.assertEqual(set(cookie_parts.keys()), set(['temp', 'Path']))
-        self.assertBody(id1)
-        self.assertEqual(copykeys(sessions.RamSession.cache), [id1])
-
-        # Simulate a browser close by just not sending the cookies
-        self.getPage('/session_cookie')
-        # grab the cookie ID
-        cookie_parts = dict([p.strip().split('=')
-                            for p in self.cookies[0][1].split(";")])
-        # Assert there is no 'expires' param
-        self.assertEqual(set(cookie_parts.keys()), set(['temp', 'Path']))
-        # Assert a new id has been generated...
-        id2 = cookie_parts['temp']
-        self.assertNotEqual(id1, id2)
-        self.assertEqual(set(sessions.RamSession.cache.keys()),
-                         set([id1, id2]))
-
-        # Wait for the session.timeout on both sessions
-        time.sleep(2.5)
-        cache = copykeys(sessions.RamSession.cache)
-        if cache:
-            if cache == [id2]:
-                self.fail("The second session did not time out.")
-            else:
-                self.fail("Unknown session id in cache: %r", cache)
-
-    def test_8_Ram_Cleanup(self):
-        def lock():
-            s1 = sessions.RamSession()
-            s1.acquire_lock()
-            time.sleep(1)
-            s1.release_lock()
-
-        t = threading.Thread(target=lock)
-        t.start()
-        s2 = sessions.RamSession()
-        s2.clean_up()
-        self.assertEqual(len(sessions.RamSession.locks), 1, 'Clean up should not remove active lock')
-        t.join()
-
-
-import socket
-try:
-    import memcache  # NOQA
-
-    host, port = '127.0.0.1', 11211
-    for res in socket.getaddrinfo(host, port, socket.AF_UNSPEC,
-                                  socket.SOCK_STREAM):
-        af, socktype, proto, canonname, sa = res
-        s = None
-        try:
-            s = socket.socket(af, socktype, proto)
-            # See http://groups.google.com/group/cherrypy-users/
-            #        browse_frm/thread/bbfe5eb39c904fe0
-            s.settimeout(1.0)
-            s.connect((host, port))
-            s.close()
-        except socket.error:
-            if s:
-                s.close()
-            raise
-        break
-except (ImportError, socket.error):
-    class MemcachedSessionTest(helper.CPWebCase):
-        setup_server = staticmethod(setup_server)
-
-        def test(self):
-            return self.skip("memcached not reachable ")
-else:
-    class MemcachedSessionTest(helper.CPWebCase):
-        setup_server = staticmethod(setup_server)
-
-        def test_0_Session(self):
-            self.getPage('/setsessiontype/memcached')
-
-            self.getPage('/testStr')
-            self.assertBody('1')
-            self.getPage('/testGen', self.cookies)
-            self.assertBody('2')
-            self.getPage('/testStr', self.cookies)
-            self.assertBody('3')
-            self.getPage('/length', self.cookies)
-            self.assertErrorPage(500)
-            self.assertInBody("NotImplementedError")
-            self.getPage('/delkey?key=counter', self.cookies)
-            self.assertStatus(200)
-
-            # Wait for the session.timeout (1 second)
-            time.sleep(1.25)
-            self.getPage('/')
-            self.assertBody('1')
-
-            # Test session __contains__
-            self.getPage('/keyin?key=counter', self.cookies)
-            self.assertBody("True")
-
-            # Test session delete
-            self.getPage('/delete', self.cookies)
-            self.assertBody("done")
-
-        def test_1_Concurrency(self):
-            client_thread_count = 5
-            request_count = 30
-
-            # Get initial cookie
-            self.getPage("/")
-            self.assertBody("1")
-            cookies = self.cookies
-
-            data_dict = {}
-
-            def request(index):
-                for i in range(request_count):
-                    self.getPage("/", cookies)
-                    # Uncomment the following line to prove threads overlap.
-##                    sys.stdout.write("%d " % index)
-                if not self.body.isdigit():
-                    self.fail(self.body)
-                data_dict[index] = int(self.body)
-
-            # Start  concurrent requests from
-            # each of  clients
-            ts = []
-            for c in range(client_thread_count):
-                data_dict[c] = 0
-                t = threading.Thread(target=request, args=(c,))
-                ts.append(t)
-                t.start()
-
-            for t in ts:
-                t.join()
-
-            hitcount = max(data_dict.values())
-            expected = 1 + (client_thread_count * request_count)
-            self.assertEqual(hitcount, expected)
-
-        def test_3_Redirect(self):
-            # Start a new session
-            self.getPage('/testStr')
-            self.getPage('/iredir', self.cookies)
-            self.assertBody("memcached")
-
-        def test_5_Error_paths(self):
-            self.getPage('/unknown/page')
-            self.assertErrorPage(
-                404, "The path '/unknown/page' was not found.")
-
-            # Note: this path is *not* the same as above. The above
-            # takes a normal route through the session code; this one
-            # skips the session code's before_handler and only calls
-            # before_finalize (save) and on_end (close). So the session
-            # code has to survive calling save/close without init.
-            self.getPage('/restricted', self.cookies, method='POST')
-            self.assertErrorPage(405, response_codes[405][1])
diff --git a/lib/cherrypy/test/test_sessionauthenticate.py b/lib/cherrypy/test/test_sessionauthenticate.py
deleted file mode 100644
index 6fef0d84dc9..00000000000
--- a/lib/cherrypy/test/test_sessionauthenticate.py
+++ /dev/null
@@ -1,61 +0,0 @@
-import cherrypy
-from cherrypy.test import helper
-
-
-class SessionAuthenticateTest(helper.CPWebCase):
-
-    def setup_server():
-
-        def check(username, password):
-            # Dummy check_username_and_password function
-            if username != 'test' or password != 'password':
-                return 'Wrong login/password'
-
-        def augment_params():
-            # A simple tool to add some things to request.params
-            # This is to check to make sure that session_auth can handle
-            # request params (ticket #780)
-            cherrypy.request.params["test"] = "test"
-
-        cherrypy.tools.augment_params = cherrypy.Tool(
-            'before_handler', augment_params, None, priority=30)
-
-        class Test:
-
-            _cp_config = {
-                'tools.sessions.on': True,
-                'tools.session_auth.on': True,
-                'tools.session_auth.check_username_and_password': check,
-                'tools.augment_params.on': True,
-            }
-
-            @cherrypy.expose
-            def index(self, **kwargs):
-                return "Hi %s, you are logged in" % cherrypy.request.login
-
-        cherrypy.tree.mount(Test())
-    setup_server = staticmethod(setup_server)
-
-    def testSessionAuthenticate(self):
-        # request a page and check for login form
-        self.getPage('/')
-        self.assertInBody('
')
-
-        # setup credentials
-        login_body = 'username=test&password=password&from_page=/'
-
-        # attempt a login
-        self.getPage('/do_login', method='POST', body=login_body)
-        self.assertStatus((302, 303))
-
-        # get the page now that we are logged in
-        self.getPage('/', self.cookies)
-        self.assertBody('Hi test, you are logged in')
-
-        # do a logout
-        self.getPage('/do_logout', self.cookies, method='POST')
-        self.assertStatus((302, 303))
-
-        # verify we are logged out
-        self.getPage('/', self.cookies)
-        self.assertInBody('')
diff --git a/lib/cherrypy/test/test_states.py b/lib/cherrypy/test/test_states.py
deleted file mode 100644
index 8ebe220f024..00000000000
--- a/lib/cherrypy/test/test_states.py
+++ /dev/null
@@ -1,527 +0,0 @@
-import os
-import signal
-import socket
-import sys
-import time
-import unittest
-import warnings
-
-import cherrypy
-import cherrypy.process.servers
-from cherrypy._cpcompat import BadStatusLine, ntob
-from cherrypy.test import helper
-
-engine = cherrypy.engine
-thisdir = os.path.join(os.getcwd(), os.path.dirname(__file__))
-
-
-class Dependency:
-
-    def __init__(self, bus):
-        self.bus = bus
-        self.running = False
-        self.startcount = 0
-        self.gracecount = 0
-        self.threads = {}
-
-    def subscribe(self):
-        self.bus.subscribe('start', self.start)
-        self.bus.subscribe('stop', self.stop)
-        self.bus.subscribe('graceful', self.graceful)
-        self.bus.subscribe('start_thread', self.startthread)
-        self.bus.subscribe('stop_thread', self.stopthread)
-
-    def start(self):
-        self.running = True
-        self.startcount += 1
-
-    def stop(self):
-        self.running = False
-
-    def graceful(self):
-        self.gracecount += 1
-
-    def startthread(self, thread_id):
-        self.threads[thread_id] = None
-
-    def stopthread(self, thread_id):
-        del self.threads[thread_id]
-
-db_connection = Dependency(engine)
-
-
-def setup_server():
-    class Root:
-
-        @cherrypy.expose
-        def index(self):
-            return "Hello World"
-
-        @cherrypy.expose
-        def ctrlc(self):
-            raise KeyboardInterrupt()
-
-        @cherrypy.expose
-        def graceful(self):
-            engine.graceful()
-            return "app was (gracefully) restarted succesfully"
-
-        @cherrypy.expose
-        def block_explicit(self):
-            while True:
-                if cherrypy.response.timed_out:
-                    cherrypy.response.timed_out = False
-                    return "broken!"
-                time.sleep(0.01)
-
-        @cherrypy.expose
-        def block_implicit(self):
-            time.sleep(0.5)
-            return "response.timeout = %s" % cherrypy.response.timeout
-
-    cherrypy.tree.mount(Root())
-    cherrypy.config.update({
-        'environment': 'test_suite',
-        'engine.deadlock_poll_freq': 0.1,
-    })
-
-    db_connection.subscribe()
-
-# ------------ Enough helpers. Time for real live test cases. ------------ #
-
-
-class ServerStateTests(helper.CPWebCase):
-    setup_server = staticmethod(setup_server)
-
-    def setUp(self):
-        cherrypy.server.socket_timeout = 0.1
-        self.do_gc_test = False
-
-    def test_0_NormalStateFlow(self):
-        engine.stop()
-        # Our db_connection should not be running
-        self.assertEqual(db_connection.running, False)
-        self.assertEqual(db_connection.startcount, 1)
-        self.assertEqual(len(db_connection.threads), 0)
-
-        # Test server start
-        engine.start()
-        self.assertEqual(engine.state, engine.states.STARTED)
-
-        host = cherrypy.server.socket_host
-        port = cherrypy.server.socket_port
-        self.assertRaises(IOError, cherrypy._cpserver.check_port, host, port)
-
-        # The db_connection should be running now
-        self.assertEqual(db_connection.running, True)
-        self.assertEqual(db_connection.startcount, 2)
-        self.assertEqual(len(db_connection.threads), 0)
-
-        self.getPage("/")
-        self.assertBody("Hello World")
-        self.assertEqual(len(db_connection.threads), 1)
-
-        # Test engine stop. This will also stop the HTTP server.
-        engine.stop()
-        self.assertEqual(engine.state, engine.states.STOPPED)
-
-        # Verify that our custom stop function was called
-        self.assertEqual(db_connection.running, False)
-        self.assertEqual(len(db_connection.threads), 0)
-
-        # Block the main thread now and verify that exit() works.
-        def exittest():
-            self.getPage("/")
-            self.assertBody("Hello World")
-            engine.exit()
-        cherrypy.server.start()
-        engine.start_with_callback(exittest)
-        engine.block()
-        self.assertEqual(engine.state, engine.states.EXITING)
-
-    def test_1_Restart(self):
-        cherrypy.server.start()
-        engine.start()
-
-        # The db_connection should be running now
-        self.assertEqual(db_connection.running, True)
-        grace = db_connection.gracecount
-
-        self.getPage("/")
-        self.assertBody("Hello World")
-        self.assertEqual(len(db_connection.threads), 1)
-
-        # Test server restart from this thread
-        engine.graceful()
-        self.assertEqual(engine.state, engine.states.STARTED)
-        self.getPage("/")
-        self.assertBody("Hello World")
-        self.assertEqual(db_connection.running, True)
-        self.assertEqual(db_connection.gracecount, grace + 1)
-        self.assertEqual(len(db_connection.threads), 1)
-
-        # Test server restart from inside a page handler
-        self.getPage("/graceful")
-        self.assertEqual(engine.state, engine.states.STARTED)
-        self.assertBody("app was (gracefully) restarted succesfully")
-        self.assertEqual(db_connection.running, True)
-        self.assertEqual(db_connection.gracecount, grace + 2)
-        # Since we are requesting synchronously, is only one thread used?
-        # Note that the "/graceful" request has been flushed.
-        self.assertEqual(len(db_connection.threads), 0)
-
-        engine.stop()
-        self.assertEqual(engine.state, engine.states.STOPPED)
-        self.assertEqual(db_connection.running, False)
-        self.assertEqual(len(db_connection.threads), 0)
-
-    def test_2_KeyboardInterrupt(self):
-        # Raise a keyboard interrupt in the HTTP server's main thread.
-        # We must start the server in this, the main thread
-        engine.start()
-        cherrypy.server.start()
-
-        self.persistent = True
-        try:
-            # Make the first request and assert there's no "Connection: close".
-            self.getPage("/")
-            self.assertStatus('200 OK')
-            self.assertBody("Hello World")
-            self.assertNoHeader("Connection")
-
-            cherrypy.server.httpserver.interrupt = KeyboardInterrupt
-            engine.block()
-
-            self.assertEqual(db_connection.running, False)
-            self.assertEqual(len(db_connection.threads), 0)
-            self.assertEqual(engine.state, engine.states.EXITING)
-        finally:
-            self.persistent = False
-
-        # Raise a keyboard interrupt in a page handler; on multithreaded
-        # servers, this should occur in one of the worker threads.
-        # This should raise a BadStatusLine error, since the worker
-        # thread will just die without writing a response.
-        engine.start()
-        cherrypy.server.start()
-        # From python3.5 a new exception is retuned when the connection
-        # ends abruptly:
-        #   http.client.RemoteDisconnected
-        # RemoteDisconnected is a subclass of:
-        #   (ConnectionResetError, http.client.BadStatusLine)
-        # and ConnectionResetError is an indirect subclass of:
-        #    OSError
-        # From python 3.3 an up socket.error is an alias to OSError
-        # following PEP-3151, therefore http.client.RemoteDisconnected
-        # is considered a socket.error.
-        #
-        # raise_subcls specifies the classes that are not going
-        # to be considered as a socket.error for the retries.
-        # Given that RemoteDisconnected is part BadStatusLine
-        # we can use the same call for all py3 versions without
-        # sideffects. python < 3.5 will raise directly BadStatusLine
-        # which is not a subclass for socket.error/OSError.
-        try:
-            self.getPage("/ctrlc", raise_subcls=BadStatusLine)
-        except BadStatusLine:
-            pass
-        else:
-            print(self.body)
-            self.fail("AssertionError: BadStatusLine not raised")
-
-        engine.block()
-        self.assertEqual(db_connection.running, False)
-        self.assertEqual(len(db_connection.threads), 0)
-
-    def test_3_Deadlocks(self):
-        cherrypy.config.update({'response.timeout': 0.2})
-
-        engine.start()
-        cherrypy.server.start()
-        try:
-            self.assertNotEqual(engine.timeout_monitor.thread, None)
-
-            # Request a "normal" page.
-            self.assertEqual(engine.timeout_monitor.servings, [])
-            self.getPage("/")
-            self.assertBody("Hello World")
-            # request.close is called async.
-            while engine.timeout_monitor.servings:
-                sys.stdout.write(".")
-                time.sleep(0.01)
-
-            # Request a page that explicitly checks itself for deadlock.
-            # The deadlock_timeout should be 2 secs.
-            self.getPage("/block_explicit")
-            self.assertBody("broken!")
-
-            # Request a page that implicitly breaks deadlock.
-            # If we deadlock, we want to touch as little code as possible,
-            # so we won't even call handle_error, just bail ASAP.
-            self.getPage("/block_implicit")
-            self.assertStatus(500)
-            self.assertInBody("raise cherrypy.TimeoutError()")
-        finally:
-            engine.exit()
-
-    def test_4_Autoreload(self):
-        # If test_3 has not been executed, the server won't be stopped,
-        # so we'll have to do it.
-        if engine.state != engine.states.EXITING:
-            engine.exit()
-
-        # Start the demo script in a new process
-        p = helper.CPProcess(ssl=(self.scheme.lower() == 'https'))
-        p.write_conf(extra='test_case_name: "test_4_Autoreload"')
-        p.start(imports='cherrypy.test._test_states_demo')
-        try:
-            self.getPage("/start")
-            start = float(self.body)
-
-            # Give the autoreloader time to cache the file time.
-            time.sleep(2)
-
-            # Touch the file
-            os.utime(os.path.join(thisdir, "_test_states_demo.py"), None)
-
-            # Give the autoreloader time to re-exec the process
-            time.sleep(2)
-            host = cherrypy.server.socket_host
-            port = cherrypy.server.socket_port
-            cherrypy._cpserver.wait_for_occupied_port(host, port)
-
-            self.getPage("/start")
-            if not (float(self.body) > start):
-                raise AssertionError("start time %s not greater than %s" %
-                                     (float(self.body), start))
-        finally:
-            # Shut down the spawned process
-            self.getPage("/exit")
-        p.join()
-
-    def test_5_Start_Error(self):
-        # If test_3 has not been executed, the server won't be stopped,
-        # so we'll have to do it.
-        if engine.state != engine.states.EXITING:
-            engine.exit()
-
-        # If a process errors during start, it should stop the engine
-        # and exit with a non-zero exit code.
-        p = helper.CPProcess(ssl=(self.scheme.lower() == 'https'),
-                             wait=True)
-        p.write_conf(
-            extra="""starterror: True
-test_case_name: "test_5_Start_Error"
-"""
-        )
-        p.start(imports='cherrypy.test._test_states_demo')
-        if p.exit_code == 0:
-            self.fail("Process failed to return nonzero exit code.")
-
-
-class PluginTests(helper.CPWebCase):
-
-    def test_daemonize(self):
-        if os.name not in ['posix']:
-            return self.skip("skipped (not on posix) ")
-        self.HOST = '127.0.0.1'
-        self.PORT = 8081
-        # Spawn the process and wait, when this returns, the original process
-        # is finished.  If it daemonized properly, we should still be able
-        # to access pages.
-        p = helper.CPProcess(ssl=(self.scheme.lower() == 'https'),
-                             wait=True, daemonize=True,
-                             socket_host='127.0.0.1',
-                             socket_port=8081)
-        p.write_conf(
-            extra='test_case_name: "test_daemonize"')
-        p.start(imports='cherrypy.test._test_states_demo')
-        try:
-            # Just get the pid of the daemonization process.
-            self.getPage("/pid")
-            self.assertStatus(200)
-            page_pid = int(self.body)
-            self.assertEqual(page_pid, p.get_pid())
-        finally:
-            # Shut down the spawned process
-            self.getPage("/exit")
-        p.join()
-
-        # Wait until here to test the exit code because we want to ensure
-        # that we wait for the daemon to finish running before we fail.
-        if p.exit_code != 0:
-            self.fail("Daemonized parent process failed to exit cleanly.")
-
-
-class SignalHandlingTests(helper.CPWebCase):
-
-    def test_SIGHUP_tty(self):
-        # When not daemonized, SIGHUP should shut down the server.
-        try:
-            from signal import SIGHUP
-        except ImportError:
-            return self.skip("skipped (no SIGHUP) ")
-
-        # Spawn the process.
-        p = helper.CPProcess(ssl=(self.scheme.lower() == 'https'))
-        p.write_conf(
-            extra='test_case_name: "test_SIGHUP_tty"')
-        p.start(imports='cherrypy.test._test_states_demo')
-        # Send a SIGHUP
-        os.kill(p.get_pid(), SIGHUP)
-        # This might hang if things aren't working right, but meh.
-        p.join()
-
-    def test_SIGHUP_daemonized(self):
-        # When daemonized, SIGHUP should restart the server.
-        try:
-            from signal import SIGHUP
-        except ImportError:
-            return self.skip("skipped (no SIGHUP) ")
-
-        if os.name not in ['posix']:
-            return self.skip("skipped (not on posix) ")
-
-        # Spawn the process and wait, when this returns, the original process
-        # is finished.  If it daemonized properly, we should still be able
-        # to access pages.
-        p = helper.CPProcess(ssl=(self.scheme.lower() == 'https'),
-                             wait=True, daemonize=True)
-        p.write_conf(
-            extra='test_case_name: "test_SIGHUP_daemonized"')
-        p.start(imports='cherrypy.test._test_states_demo')
-
-        pid = p.get_pid()
-        try:
-            # Send a SIGHUP
-            os.kill(pid, SIGHUP)
-            # Give the server some time to restart
-            time.sleep(2)
-            self.getPage("/pid")
-            self.assertStatus(200)
-            new_pid = int(self.body)
-            self.assertNotEqual(new_pid, pid)
-        finally:
-            # Shut down the spawned process
-            self.getPage("/exit")
-        p.join()
-
-    def _require_signal_and_kill(self, signal_name):
-        if not hasattr(signal, signal_name):
-            self.skip("skipped (no %(signal_name)s)" % vars())
-
-        if not hasattr(os, 'kill'):
-            self.skip("skipped (no os.kill)")
-
-    def test_SIGTERM(self):
-        "SIGTERM should shut down the server whether daemonized or not."
-        self._require_signal_and_kill('SIGTERM')
-
-        # Spawn a normal, undaemonized process.
-        p = helper.CPProcess(ssl=(self.scheme.lower() == 'https'))
-        p.write_conf(
-            extra='test_case_name: "test_SIGTERM"')
-        p.start(imports='cherrypy.test._test_states_demo')
-        # Send a SIGTERM
-        os.kill(p.get_pid(), signal.SIGTERM)
-        # This might hang if things aren't working right, but meh.
-        p.join()
-
-        if os.name in ['posix']:
-            # Spawn a daemonized process and test again.
-            p = helper.CPProcess(ssl=(self.scheme.lower() == 'https'),
-                                 wait=True, daemonize=True)
-            p.write_conf(
-                extra='test_case_name: "test_SIGTERM_2"')
-            p.start(imports='cherrypy.test._test_states_demo')
-            # Send a SIGTERM
-            os.kill(p.get_pid(), signal.SIGTERM)
-            # This might hang if things aren't working right, but meh.
-            p.join()
-
-    def test_signal_handler_unsubscribe(self):
-        self._require_signal_and_kill('SIGTERM')
-
-        # Although Windows has `os.kill` and SIGTERM is defined, the
-        #  platform does not implement signals and sending SIGTERM
-        #  will result in a forced termination of the process.
-        #  Therefore, this test is not suitable for Windows.
-        if os.name == 'nt':
-            self.skip("SIGTERM not available")
-
-        # Spawn a normal, undaemonized process.
-        p = helper.CPProcess(ssl=(self.scheme.lower() == 'https'))
-        p.write_conf(
-            extra="""unsubsig: True
-test_case_name: "test_signal_handler_unsubscribe"
-""")
-        p.start(imports='cherrypy.test._test_states_demo')
-        # Ask the process to quit
-        os.kill(p.get_pid(), signal.SIGTERM)
-        # This might hang if things aren't working right, but meh.
-        p.join()
-
-        # Assert the old handler ran.
-        target_line = open(p.error_log, 'rb').readlines()[-10]
-        if not ntob("I am an old SIGTERM handler.") in target_line:
-            self.fail("Old SIGTERM handler did not run.\n%r" % target_line)
-
-
-class WaitTests(unittest.TestCase):
-
-    def test_wait_for_occupied_port_INADDR_ANY(self):
-        """
-        Wait on INADDR_ANY should not raise IOError
-
-        In cases where the loopback interface does not exist, CherryPy cannot
-        effectively determine if a port binding to INADDR_ANY was effected.
-        In this situation, CherryPy should assume that it failed to detect
-        the binding (not that the binding failed) and only warn that it could
-        not verify it.
-        """
-        # At such a time that CherryPy can reliably determine one or more
-        #  viable IP addresses of the host, this test may be removed.
-
-        # Simulate the behavior we observe when no loopback interface is
-        #  present by: finding a port that's not occupied, then wait on it.
-
-        free_port = self.find_free_port()
-
-        servers = cherrypy.process.servers
-
-        def with_shorter_timeouts(func):
-            """
-            A context where occupied_port_timeout is much smaller to speed
-            test runs.
-            """
-            # When we have Python 2.5, simplify using the with_statement.
-            orig_timeout = servers.occupied_port_timeout
-            servers.occupied_port_timeout = .07
-            try:
-                func()
-            finally:
-                servers.occupied_port_timeout = orig_timeout
-
-        def do_waiting():
-            # Wait on the free port that's unbound
-            with warnings.catch_warnings(record=True) as w:
-                servers.wait_for_occupied_port('0.0.0.0', free_port)
-                self.assertEqual(len(w), 1)
-                self.assertTrue(isinstance(w[0], warnings.WarningMessage))
-                self.assertTrue(
-                    'Unable to verify that the server is bound on ' in str(w[0]))
-
-            # The wait should still raise an IO error if INADDR_ANY was
-            #  not supplied.
-            self.assertRaises(IOError, servers.wait_for_occupied_port,
-                              '127.0.0.1', free_port)
-
-        with_shorter_timeouts(do_waiting)
-
-    def find_free_port(self):
-        "Find a free port by binding to port 0 then unbinding."
-        sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
-        sock.bind(('', 0))
-        free_port = sock.getsockname()[1]
-        sock.close()
-        return free_port
diff --git a/lib/cherrypy/test/test_static.py b/lib/cherrypy/test/test_static.py
deleted file mode 100644
index 20cb665337a..00000000000
--- a/lib/cherrypy/test/test_static.py
+++ /dev/null
@@ -1,345 +0,0 @@
-import os
-import sys
-import io
-
-from cherrypy._cpcompat import HTTPConnection, HTTPSConnection, ntob
-
-curdir = os.path.join(os.getcwd(), os.path.dirname(__file__))
-has_space_filepath = os.path.join(curdir, 'static', 'has space.html')
-bigfile_filepath = os.path.join(curdir, "static", "bigfile.log")
-
-# The file size needs to be big enough such that half the size of it
-# won't be socket-buffered (or server-buffered) all in one go. See
-# test_file_stream.
-BIGFILE_SIZE = 1024 * 1024 * 20
-
-import cherrypy
-from cherrypy.lib import static
-from cherrypy.test import helper
-
-
-class StaticTest(helper.CPWebCase):
-
-    def setup_server():
-        if not os.path.exists(has_space_filepath):
-            open(has_space_filepath, 'wb').write(ntob('Hello, world\r\n'))
-        if not os.path.exists(bigfile_filepath) or \
-            os.path.getsize(bigfile_filepath) != BIGFILE_SIZE:
-            open(bigfile_filepath, 'wb').write(ntob("x" * BIGFILE_SIZE))
-
-        class Root:
-
-            @cherrypy.expose
-            @cherrypy.config(**{'response.stream': True})
-            def bigfile(self):
-                from cherrypy.lib import static
-                self.f = static.serve_file(bigfile_filepath)
-                return self.f
-
-            @cherrypy.expose
-            def tell(self):
-                if self.f.input.closed:
-                    return ''
-                return repr(self.f.input.tell()).rstrip('L')
-
-            @cherrypy.expose
-            def fileobj(self):
-                f = open(os.path.join(curdir, 'style.css'), 'rb')
-                return static.serve_fileobj(f, content_type='text/css')
-
-            @cherrypy.expose
-            def bytesio(self):
-                f = io.BytesIO(ntob('Fee\nfie\nfo\nfum'))
-                return static.serve_fileobj(f, content_type='text/plain')
-
-        class Static:
-
-            @cherrypy.expose
-            def index(self):
-                return 'You want the Baron? You can have the Baron!'
-
-            @cherrypy.expose
-            def dynamic(self):
-                return "This is a DYNAMIC page"
-
-        root = Root()
-        root.static = Static()
-
-        rootconf = {
-            '/static': {
-                'tools.staticdir.on': True,
-                'tools.staticdir.dir': 'static',
-                'tools.staticdir.root': curdir,
-            },
-            '/style.css': {
-                'tools.staticfile.on': True,
-                'tools.staticfile.filename': os.path.join(curdir, 'style.css'),
-            },
-            '/docroot': {
-                'tools.staticdir.on': True,
-                'tools.staticdir.root': curdir,
-                'tools.staticdir.dir': 'static',
-                'tools.staticdir.index': 'index.html',
-            },
-            '/error': {
-                'tools.staticdir.on': True,
-                'request.show_tracebacks': True,
-            },
-            '/404test': {
-                'tools.staticdir.on': True,
-                'tools.staticdir.root': curdir,
-                'tools.staticdir.dir': 'static',
-                'error_page.404': error_page_404,
-            }
-        }
-        rootApp = cherrypy.Application(root)
-        rootApp.merge(rootconf)
-
-        test_app_conf = {
-            '/test': {
-                'tools.staticdir.index': 'index.html',
-                'tools.staticdir.on': True,
-                'tools.staticdir.root': curdir,
-                'tools.staticdir.dir': 'static',
-            },
-        }
-        testApp = cherrypy.Application(Static())
-        testApp.merge(test_app_conf)
-
-        vhost = cherrypy._cpwsgi.VirtualHost(rootApp, {'virt.net': testApp})
-        cherrypy.tree.graft(vhost)
-    setup_server = staticmethod(setup_server)
-
-    def teardown_server():
-        for f in (has_space_filepath, bigfile_filepath):
-            if os.path.exists(f):
-                try:
-                    os.unlink(f)
-                except:
-                    pass
-    teardown_server = staticmethod(teardown_server)
-
-    def test_static(self):
-        self.getPage("/static/index.html")
-        self.assertStatus('200 OK')
-        self.assertHeader('Content-Type', 'text/html')
-        self.assertBody('Hello, world\r\n')
-
-        # Using a staticdir.root value in a subdir...
-        self.getPage("/docroot/index.html")
-        self.assertStatus('200 OK')
-        self.assertHeader('Content-Type', 'text/html')
-        self.assertBody('Hello, world\r\n')
-
-        # Check a filename with spaces in it
-        self.getPage("/static/has%20space.html")
-        self.assertStatus('200 OK')
-        self.assertHeader('Content-Type', 'text/html')
-        self.assertBody('Hello, world\r\n')
-
-        self.getPage("/style.css")
-        self.assertStatus('200 OK')
-        self.assertHeader('Content-Type', 'text/css')
-        # Note: The body should be exactly 'Dummy stylesheet\n', but
-        #   unfortunately some tools such as WinZip sometimes turn \n
-        #   into \r\n on Windows when extracting the CherryPy tarball so
-        #   we just check the content
-        self.assertMatchesBody('^Dummy stylesheet')
-
-    def test_fallthrough(self):
-        # Test that NotFound will then try dynamic handlers (see [878]).
-        self.getPage("/static/dynamic")
-        self.assertBody("This is a DYNAMIC page")
-
-        # Check a directory via fall-through to dynamic handler.
-        self.getPage("/static/")
-        self.assertStatus('200 OK')
-        self.assertHeader('Content-Type', 'text/html;charset=utf-8')
-        self.assertBody('You want the Baron? You can have the Baron!')
-
-    def test_index(self):
-        # Check a directory via "staticdir.index".
-        self.getPage("/docroot/")
-        self.assertStatus('200 OK')
-        self.assertHeader('Content-Type', 'text/html')
-        self.assertBody('Hello, world\r\n')
-        # The same page should be returned even if redirected.
-        self.getPage("/docroot")
-        self.assertStatus(301)
-        self.assertHeader('Location', '%s/docroot/' % self.base())
-        self.assertMatchesBody("This resource .* "
-                               "%s/docroot/." % (self.base(), self.base()))
-
-    def test_config_errors(self):
-        # Check that we get an error if no .file or .dir
-        self.getPage("/error/thing.html")
-        self.assertErrorPage(500)
-        if sys.version_info >= (3, 3):
-            errmsg = ntob("TypeError: staticdir\(\) missing 2 "
-                          "required positional arguments")
-        else:
-            errmsg = ntob("TypeError: staticdir\(\) takes at least 2 "
-                          "(positional )?arguments \(0 given\)")
-        self.assertMatchesBody(errmsg)
-
-    def test_security(self):
-        # Test up-level security
-        self.getPage("/static/../../test/style.css")
-        self.assertStatus((400, 403))
-
-    def test_modif(self):
-        # Test modified-since on a reasonably-large file
-        self.getPage("/static/dirback.jpg")
-        self.assertStatus("200 OK")
-        lastmod = ""
-        for k, v in self.headers:
-            if k == 'Last-Modified':
-                lastmod = v
-        ims = ("If-Modified-Since", lastmod)
-        self.getPage("/static/dirback.jpg", headers=[ims])
-        self.assertStatus(304)
-        self.assertNoHeader("Content-Type")
-        self.assertNoHeader("Content-Length")
-        self.assertNoHeader("Content-Disposition")
-        self.assertBody("")
-
-    def test_755_vhost(self):
-        self.getPage("/test/", [('Host', 'virt.net')])
-        self.assertStatus(200)
-        self.getPage("/test", [('Host', 'virt.net')])
-        self.assertStatus(301)
-        self.assertHeader('Location', self.scheme + '://virt.net/test/')
-
-    def test_serve_fileobj(self):
-        self.getPage("/fileobj")
-        self.assertStatus('200 OK')
-        self.assertHeader('Content-Type', 'text/css;charset=utf-8')
-        self.assertMatchesBody('^Dummy stylesheet')
-
-    def test_serve_bytesio(self):
-        self.getPage("/bytesio")
-        self.assertStatus('200 OK')
-        self.assertHeader('Content-Type', 'text/plain;charset=utf-8')
-        self.assertHeader('Content-Length', 14)
-        self.assertMatchesBody('Fee\nfie\nfo\nfum')
-
-    def test_file_stream(self):
-        if cherrypy.server.protocol_version != "HTTP/1.1":
-            return self.skip()
-
-        self.PROTOCOL = "HTTP/1.1"
-
-        # Make an initial request
-        self.persistent = True
-        conn = self.HTTP_CONN
-        conn.putrequest("GET", "/bigfile", skip_host=True)
-        conn.putheader("Host", self.HOST)
-        conn.endheaders()
-        response = conn.response_class(conn.sock, method="GET")
-        response.begin()
-        self.assertEqual(response.status, 200)
-
-        body = ntob('')
-        remaining = BIGFILE_SIZE
-        while remaining > 0:
-            data = response.fp.read(65536)
-            if not data:
-                break
-            body += data
-            remaining -= len(data)
-
-            if self.scheme == "https":
-                newconn = HTTPSConnection
-            else:
-                newconn = HTTPConnection
-            s, h, b = helper.webtest.openURL(
-                ntob("/tell"), headers=[], host=self.HOST, port=self.PORT,
-                http_conn=newconn)
-            if not b:
-                # The file was closed on the server.
-                tell_position = BIGFILE_SIZE
-            else:
-                tell_position = int(b)
-
-            read_so_far = len(body)
-
-            # It is difficult for us to force the server to only read
-            # the bytes that we ask for - there are going to be buffers
-            # inbetween.
-            #
-            # CherryPy will attempt to write as much data as it can to
-            # the socket, and we don't have a way to determine what that
-            # size will be. So we make the following assumption - by
-            # the time we have read in the entire file on the server,
-            # we will have at least received half of it. If this is not
-            # the case, then this is an indicator that either:
-            #   - machines that are running this test are using buffer
-            #     sizes greater than half of BIGFILE_SIZE; or
-            #   - streaming is broken.
-            #
-            # At the time of writing, we seem to have encountered
-            # buffer sizes bigger than 512K, so we've increased
-            # BIGFILE_SIZE to 4MB and in 2016 to 20MB.
-            # This test is going to keep failing according to the
-            # improvements in hardware and OS buffers.
-            if tell_position >= BIGFILE_SIZE:
-                if read_so_far < (BIGFILE_SIZE / 2):
-                    self.fail(
-                        "The file should have advanced to position %r, but "
-                        "has already advanced to the end of the file. It "
-                        "may not be streamed as intended, or at the wrong "
-                        "chunk size (64k)" % read_so_far)
-            elif tell_position < read_so_far:
-                self.fail(
-                    "The file should have advanced to position %r, but has "
-                    "only advanced to position %r. It may not be streamed "
-                    "as intended, or at the wrong chunk size (64k)" %
-                    (read_so_far, tell_position))
-
-        if body != ntob("x" * BIGFILE_SIZE):
-            self.fail("Body != 'x' * %d. Got %r instead (%d bytes)." %
-                      (BIGFILE_SIZE, body[:50], len(body)))
-        conn.close()
-
-    def test_file_stream_deadlock(self):
-        if cherrypy.server.protocol_version != "HTTP/1.1":
-            return self.skip()
-
-        self.PROTOCOL = "HTTP/1.1"
-
-        # Make an initial request but abort early.
-        self.persistent = True
-        conn = self.HTTP_CONN
-        conn.putrequest("GET", "/bigfile", skip_host=True)
-        conn.putheader("Host", self.HOST)
-        conn.endheaders()
-        response = conn.response_class(conn.sock, method="GET")
-        response.begin()
-        self.assertEqual(response.status, 200)
-        body = response.fp.read(65536)
-        if body != ntob("x" * len(body)):
-            self.fail("Body != 'x' * %d. Got %r instead (%d bytes)." %
-                      (65536, body[:50], len(body)))
-        response.close()
-        conn.close()
-
-        # Make a second request, which should fetch the whole file.
-        self.persistent = False
-        self.getPage("/bigfile")
-        if self.body != ntob("x" * BIGFILE_SIZE):
-            self.fail("Body != 'x' * %d. Got %r instead (%d bytes)." %
-                      (BIGFILE_SIZE, self.body[:50], len(body)))
-
-    def test_error_page_with_serve_file(self):
-        self.getPage("/404test/yunyeen")
-        self.assertStatus(404)
-        self.assertInBody("I couldn't find that thing")
-
-    def test_null_bytes(self):
-        self.getPage("/static/\x00")
-        self.assertStatus('404 Not Found')
-
-def error_page_404(status, message, traceback, version):
-    import os.path
-    return static.serve_file(os.path.join(curdir, 'static', '404.html'),
-        content_type='text/html')
diff --git a/lib/cherrypy/test/test_tools.py b/lib/cherrypy/test/test_tools.py
deleted file mode 100644
index cc3fa632de2..00000000000
--- a/lib/cherrypy/test/test_tools.py
+++ /dev/null
@@ -1,445 +0,0 @@
-"""Test the various means of instantiating and invoking tools."""
-
-import gzip
-import sys
-import unittest
-import io
-
-from cherrypy._cpcompat import copyitems, itervalues
-from cherrypy._cpcompat import IncompleteRead, ntob, ntou, xrange
-import time
-timeout = 0.2
-import types
-
-import six
-
-import cherrypy
-from cherrypy import tools
-
-
-europoundUnicode = ntou('\x80\xa3')
-
-
-#                             Client-side code                             #
-
-from cherrypy.test import helper
-
-
-class ToolTests(helper.CPWebCase):
-
-    def setup_server():
-
-        # Put check_access in a custom toolbox with its own namespace
-        myauthtools = cherrypy._cptools.Toolbox("myauth")
-
-        def check_access(default=False):
-            if not getattr(cherrypy.request, "userid", default):
-                raise cherrypy.HTTPError(401)
-        myauthtools.check_access = cherrypy.Tool(
-            'before_request_body', check_access)
-
-        def numerify():
-            def number_it(body):
-                for chunk in body:
-                    for k, v in cherrypy.request.numerify_map:
-                        chunk = chunk.replace(k, v)
-                    yield chunk
-            cherrypy.response.body = number_it(cherrypy.response.body)
-
-        class NumTool(cherrypy.Tool):
-
-            def _setup(self):
-                def makemap():
-                    m = self._merged_args().get("map", {})
-                    cherrypy.request.numerify_map = copyitems(m)
-                cherrypy.request.hooks.attach('on_start_resource', makemap)
-
-                def critical():
-                    cherrypy.request.error_response = cherrypy.HTTPError(
-                        502).set_response
-                critical.failsafe = True
-
-                cherrypy.request.hooks.attach('on_start_resource', critical)
-                cherrypy.request.hooks.attach(self._point, self.callable)
-
-        tools.numerify = NumTool('before_finalize', numerify)
-
-        # It's not mandatory to inherit from cherrypy.Tool.
-        class NadsatTool:
-
-            def __init__(self):
-                self.ended = {}
-                self._name = "nadsat"
-
-            def nadsat(self):
-                def nadsat_it_up(body):
-                    for chunk in body:
-                        chunk = chunk.replace(ntob("good"), ntob("horrorshow"))
-                        chunk = chunk.replace(ntob("piece"), ntob("lomtick"))
-                        yield chunk
-                cherrypy.response.body = nadsat_it_up(cherrypy.response.body)
-            nadsat.priority = 0
-
-            def cleanup(self):
-                # This runs after the request has been completely written out.
-                cherrypy.response.body = [ntob("razdrez")]
-                id = cherrypy.request.params.get("id")
-                if id:
-                    self.ended[id] = True
-            cleanup.failsafe = True
-
-            def _setup(self):
-                cherrypy.request.hooks.attach('before_finalize', self.nadsat)
-                cherrypy.request.hooks.attach('on_end_request', self.cleanup)
-        tools.nadsat = NadsatTool()
-
-        def pipe_body():
-            cherrypy.request.process_request_body = False
-            clen = int(cherrypy.request.headers['Content-Length'])
-            cherrypy.request.body = cherrypy.request.rfile.read(clen)
-
-        # Assert that we can use a callable object instead of a function.
-        class Rotator(object):
-
-            def __call__(self, scale):
-                r = cherrypy.response
-                r.collapse_body()
-                if six.PY3:
-                    r.body = [bytes([(x + scale) % 256 for x in r.body[0]])]
-                else:
-                    r.body = [chr((ord(x) + scale) % 256) for x in r.body[0]]
-        cherrypy.tools.rotator = cherrypy.Tool('before_finalize', Rotator())
-
-        def stream_handler(next_handler, *args, **kwargs):
-            assert cherrypy.request.config.get('tools.streamer.arg') == 'arg value'
-            cherrypy.response.output = o = io.BytesIO()
-            try:
-                response = next_handler(*args, **kwargs)
-                # Ignore the response and return our accumulated output
-                # instead.
-                return o.getvalue()
-            finally:
-                o.close()
-        cherrypy.tools.streamer = cherrypy._cptools.HandlerWrapperTool(
-            stream_handler)
-
-        class Root:
-
-            @cherrypy.expose
-            def index(self):
-                return "Howdy earth!"
-
-            @cherrypy.expose
-            @cherrypy.config(**{'tools.streamer.on': True, 'tools.streamer.arg': 'arg value'})
-            def tarfile(self):
-                assert cherrypy.request.config.get('tools.streamer.arg') == 'arg value'
-                cherrypy.response.output.write(ntob('I am '))
-                cherrypy.response.output.write(ntob('a tarfile'))
-
-            @cherrypy.expose
-            def euro(self):
-                hooks = list(cherrypy.request.hooks['before_finalize'])
-                hooks.sort()
-                cbnames = [x.callback.__name__ for x in hooks]
-                assert cbnames == ['gzip'], cbnames
-                priorities = [x.priority for x in hooks]
-                assert priorities == [80], priorities
-                yield ntou("Hello,")
-                yield ntou("world")
-                yield europoundUnicode
-
-            # Bare hooks
-            @cherrypy.expose
-            @cherrypy.config(**{'hooks.before_request_body': pipe_body})
-            def pipe(self):
-                return cherrypy.request.body
-
-            # Multiple decorators; include kwargs just for fun.
-            # Note that rotator must run before gzip.
-            @cherrypy.expose
-            def decorated_euro(self, *vpath):
-                yield ntou("Hello,")
-                yield ntou("world")
-                yield europoundUnicode
-            decorated_euro = tools.gzip(compress_level=6)(decorated_euro)
-            decorated_euro = tools.rotator(scale=3)(decorated_euro)
-
-        root = Root()
-
-        class TestType(type):
-            """Metaclass which automatically exposes all functions in each
-            subclass, and adds an instance of the subclass as an attribute
-            of root.
-            """
-            def __init__(cls, name, bases, dct):
-                type.__init__(cls, name, bases, dct)
-                for value in itervalues(dct):
-                    if isinstance(value, types.FunctionType):
-                        cherrypy.expose(value)
-                setattr(root, name.lower(), cls())
-        Test = TestType('Test', (object,), {})
-
-        # METHOD ONE:
-        # Declare Tools in _cp_config
-        @cherrypy.config(**{"tools.nadsat.on": True})
-        class Demo(Test):
-
-            def index(self, id=None):
-                return "A good piece of cherry pie"
-
-            def ended(self, id):
-                return repr(tools.nadsat.ended[id])
-
-            def err(self, id=None):
-                raise ValueError()
-
-            def errinstream(self, id=None):
-                yield "nonconfidential"
-                raise ValueError()
-                yield "confidential"
-
-            # METHOD TWO: decorator using Tool()
-            # We support Python 2.3, but the @-deco syntax would look like
-            # this:
-            # @tools.check_access()
-            def restricted(self):
-                return "Welcome!"
-            restricted = myauthtools.check_access()(restricted)
-            userid = restricted
-
-            def err_in_onstart(self):
-                return "success!"
-
-            @cherrypy.config(**{'response.stream': True})
-            def stream(self, id=None):
-                for x in xrange(100000000):
-                    yield str(x)
-
-        conf = {
-            # METHOD THREE:
-            # Declare Tools in detached config
-            '/demo': {
-                'tools.numerify.on': True,
-                'tools.numerify.map': {ntob("pie"): ntob("3.14159")},
-            },
-            '/demo/restricted': {
-                'request.show_tracebacks': False,
-            },
-            '/demo/userid': {
-                'request.show_tracebacks': False,
-                'myauth.check_access.default': True,
-            },
-            '/demo/errinstream': {
-                'response.stream': True,
-            },
-            '/demo/err_in_onstart': {
-                # Because this isn't a dict, on_start_resource will error.
-                'tools.numerify.map': "pie->3.14159"
-            },
-            # Combined tools
-            '/euro': {
-                'tools.gzip.on': True,
-                'tools.encode.on': True,
-            },
-            # Priority specified in config
-            '/decorated_euro/subpath': {
-                'tools.gzip.priority': 10,
-            },
-            # Handler wrappers
-            '/tarfile': {'tools.streamer.on': True}
-        }
-        app = cherrypy.tree.mount(root, config=conf)
-        app.request_class.namespaces['myauth'] = myauthtools
-
-        if sys.version_info >= (2, 5):
-            from cherrypy.test import _test_decorators
-            root.tooldecs = _test_decorators.ToolExamples()
-    setup_server = staticmethod(setup_server)
-
-    def testHookErrors(self):
-        self.getPage("/demo/?id=1")
-        # If body is "razdrez", then on_end_request is being called too early.
-        self.assertBody("A horrorshow lomtick of cherry 3.14159")
-        # If this fails, then on_end_request isn't being called at all.
-        time.sleep(0.1)
-        self.getPage("/demo/ended/1")
-        self.assertBody("True")
-
-        valerr = '\n    raise ValueError()\nValueError'
-        self.getPage("/demo/err?id=3")
-        # If body is "razdrez", then on_end_request is being called too early.
-        self.assertErrorPage(502, pattern=valerr)
-        # If this fails, then on_end_request isn't being called at all.
-        time.sleep(0.1)
-        self.getPage("/demo/ended/3")
-        self.assertBody("True")
-
-        # If body is "razdrez", then on_end_request is being called too early.
-        if (cherrypy.server.protocol_version == "HTTP/1.0" or
-                getattr(cherrypy.server, "using_apache", False)):
-            self.getPage("/demo/errinstream?id=5")
-            # Because this error is raised after the response body has
-            # started, the status should not change to an error status.
-            self.assertStatus("200 OK")
-            self.assertBody("nonconfidential")
-        else:
-            # Because this error is raised after the response body has
-            # started, and because it's chunked output, an error is raised by
-            # the HTTP client when it encounters incomplete output.
-            self.assertRaises((ValueError, IncompleteRead), self.getPage,
-                              "/demo/errinstream?id=5")
-        # If this fails, then on_end_request isn't being called at all.
-        time.sleep(0.1)
-        self.getPage("/demo/ended/5")
-        self.assertBody("True")
-
-        # Test the "__call__" technique (compile-time decorator).
-        self.getPage("/demo/restricted")
-        self.assertErrorPage(401)
-
-        # Test compile-time decorator with kwargs from config.
-        self.getPage("/demo/userid")
-        self.assertBody("Welcome!")
-
-    def testEndRequestOnDrop(self):
-        old_timeout = None
-        try:
-            httpserver = cherrypy.server.httpserver
-            old_timeout = httpserver.timeout
-        except (AttributeError, IndexError):
-            return self.skip()
-
-        try:
-            httpserver.timeout = timeout
-
-            # Test that on_end_request is called even if the client drops.
-            self.persistent = True
-            try:
-                conn = self.HTTP_CONN
-                conn.putrequest("GET", "/demo/stream?id=9", skip_host=True)
-                conn.putheader("Host", self.HOST)
-                conn.endheaders()
-                # Skip the rest of the request and close the conn. This will
-                # cause the server's active socket to error, which *should*
-                # result in the request being aborted, and request.close being
-                # called all the way up the stack (including WSGI middleware),
-                # eventually calling our on_end_request hook.
-            finally:
-                self.persistent = False
-            time.sleep(timeout * 2)
-            # Test that the on_end_request hook was called.
-            self.getPage("/demo/ended/9")
-            self.assertBody("True")
-        finally:
-            if old_timeout is not None:
-                httpserver.timeout = old_timeout
-
-    def testGuaranteedHooks(self):
-        # The 'critical' on_start_resource hook is 'failsafe' (guaranteed
-        # to run even if there are failures in other on_start methods).
-        # This is NOT true of the other hooks.
-        # Here, we have set up a failure in NumerifyTool.numerify_map,
-        # but our 'critical' hook should run and set the error to 502.
-        self.getPage("/demo/err_in_onstart")
-        self.assertErrorPage(502)
-        self.assertInBody(
-            "AttributeError: 'str' object has no attribute 'items'")
-
-    def testCombinedTools(self):
-        expectedResult = (ntou("Hello,world") +
-                          europoundUnicode).encode('utf-8')
-        zbuf = io.BytesIO()
-        zfile = gzip.GzipFile(mode='wb', fileobj=zbuf, compresslevel=9)
-        zfile.write(expectedResult)
-        zfile.close()
-
-        self.getPage("/euro",
-                     headers=[
-                         ("Accept-Encoding", "gzip"),
-                         ("Accept-Charset", "ISO-8859-1,utf-8;q=0.7,*;q=0.7")])
-        self.assertInBody(zbuf.getvalue()[:3])
-
-        zbuf = io.BytesIO()
-        zfile = gzip.GzipFile(mode='wb', fileobj=zbuf, compresslevel=6)
-        zfile.write(expectedResult)
-        zfile.close()
-
-        self.getPage("/decorated_euro", headers=[("Accept-Encoding", "gzip")])
-        self.assertInBody(zbuf.getvalue()[:3])
-
-        # This returns a different value because gzip's priority was
-        # lowered in conf, allowing the rotator to run after gzip.
-        # Of course, we don't want breakage in production apps,
-        # but it proves the priority was changed.
-        self.getPage("/decorated_euro/subpath",
-                     headers=[("Accept-Encoding", "gzip")])
-        if six.PY3:
-            self.assertInBody(bytes([(x + 3) % 256 for x in zbuf.getvalue()]))
-        else:
-            self.assertInBody(''.join([chr((ord(x) + 3) % 256)
-                              for x in zbuf.getvalue()]))
-
-    def testBareHooks(self):
-        content = "bit of a pain in me gulliver"
-        self.getPage("/pipe",
-                     headers=[("Content-Length", str(len(content))),
-                              ("Content-Type", "text/plain")],
-                     method="POST", body=content)
-        self.assertBody(content)
-
-    def testHandlerWrapperTool(self):
-        self.getPage("/tarfile")
-        self.assertBody("I am a tarfile")
-
-    def testToolWithConfig(self):
-        if not sys.version_info >= (2, 5):
-            return self.skip("skipped (Python 2.5+ only)")
-
-        self.getPage('/tooldecs/blah')
-        self.assertHeader('Content-Type', 'application/data')
-
-    def testWarnToolOn(self):
-        # get
-        try:
-            cherrypy.tools.numerify.on
-        except AttributeError:
-            pass
-        else:
-            raise AssertionError("Tool.on did not error as it should have.")
-
-        # set
-        try:
-            cherrypy.tools.numerify.on = True
-        except AttributeError:
-            pass
-        else:
-            raise AssertionError("Tool.on did not error as it should have.")
-
-    def testDecorator(self):
-        @cherrypy.tools.register('on_start_resource')
-        def example():
-            pass
-        self.assertTrue(isinstance(cherrypy.tools.example, cherrypy.Tool))
-        self.assertEqual(cherrypy.tools.example._point, 'on_start_resource')
-
-        @cherrypy.tools.register('before_finalize', name='renamed', priority=60)
-        def example():
-            pass
-        self.assertTrue(isinstance(cherrypy.tools.renamed, cherrypy.Tool))
-        self.assertEqual(cherrypy.tools.renamed._point, 'before_finalize')
-        self.assertEqual(cherrypy.tools.renamed._name, 'renamed')
-        self.assertEqual(cherrypy.tools.renamed._priority, 60)
-
-
-class SessionAuthTest(unittest.TestCase):
-
-    def test_login_screen_returns_bytes(self):
-        """
-        login_screen must return bytes even if unicode parameters are passed.
-        Issue 1132 revealed that login_screen would return unicode if the
-        username and password were unicode.
-        """
-        sa = cherrypy.lib.cptools.SessionAuth()
-        res = sa.login_screen(None, username=six.text_type('nobody'),
-                              password=six.text_type('anypass'))
-        self.assertTrue(isinstance(res, bytes))
diff --git a/lib/cherrypy/test/test_tutorials.py b/lib/cherrypy/test/test_tutorials.py
deleted file mode 100644
index bfcd50f744d..00000000000
--- a/lib/cherrypy/test/test_tutorials.py
+++ /dev/null
@@ -1,217 +0,0 @@
-import sys
-import imp
-import types
-
-try:
-    import importlib
-except Exception:
-    # Python 2.6 may not have it.
-    pass
-
-import six
-
-import cherrypy
-from cherrypy.test import helper
-
-
-class TutorialTest(helper.CPWebCase):
-
-    @classmethod
-    def setup_server(cls):
-        """
-        Mount something so the engine starts.
-        """
-        class Dummy:
-            pass
-        cherrypy.tree.mount(Dummy())
-
-    @staticmethod
-    def load_module(name):
-        """
-        Import or reload tutorial module as needed.
-        """
-        target = "cherrypy.tutorial." + name
-        if target in sys.modules:
-            module = imp.reload(sys.modules[target])
-        elif 'importlib' not in globals():
-            module = __import__(target, globals(), locals(), [''])
-        else:
-            module = importlib.import_module(target)
-        return module
-
-    @classmethod
-    def setup_tutorial(cls, name, root_name, config={}):
-        cherrypy.config.reset()
-        module = cls.load_module(name)
-        root = getattr(module, root_name)
-        conf = getattr(module, 'tutconf')
-        class_types = type,
-        if six.PY2:
-            class_types += types.ClassType,
-        if isinstance(root, class_types):
-            root = root()
-        cherrypy.tree.mount(root, config=conf)
-        cherrypy.config.update(config)
-
-    def test01HelloWorld(self):
-        self.setup_tutorial('tut01_helloworld', 'HelloWorld')
-        self.getPage("/")
-        self.assertBody('Hello world!')
-
-    def test02ExposeMethods(self):
-        self.setup_tutorial('tut02_expose_methods', 'HelloWorld')
-        self.getPage("/show_msg")
-        self.assertBody('Hello world!')
-
-    def test03GetAndPost(self):
-        self.setup_tutorial('tut03_get_and_post', 'WelcomePage')
-
-        # Try different GET queries
-        self.getPage("/greetUser?name=Bob")
-        self.assertBody("Hey Bob, what's up?")
-
-        self.getPage("/greetUser")
-        self.assertBody('Please enter your name here.')
-
-        self.getPage("/greetUser?name=")
-        self.assertBody('No, really, enter your name here.')
-
-        # Try the same with POST
-        self.getPage("/greetUser", method="POST", body="name=Bob")
-        self.assertBody("Hey Bob, what's up?")
-
-        self.getPage("/greetUser", method="POST", body="name=")
-        self.assertBody('No, really, enter your name here.')
-
-    def test04ComplexSite(self):
-        self.setup_tutorial('tut04_complex_site', 'root')
-
-        msg = '''
-            
Here are some extra useful links:

-
-            
-
-            
[Return to links page]
'''
-        self.getPage("/links/extra/")
-        self.assertBody(msg)
-
-    def test05DerivedObjects(self):
-        self.setup_tutorial('tut05_derived_objects', 'HomePage')
-        msg = '''
-            
-            
-                Another Page
-            
-            
-            
Another Page

-
-            

-            And this is the amazing second page!
-            

-
-            
-            
-        '''
-        # the tutorial has some annoying spaces in otherwise blank lines
-        msg = msg.replace('\n\n', '\n        \n')
-        msg = msg.replace('
\n\n', '
\n        \n')
-        self.getPage("/another/")
-        self.assertBody(msg)
-
-    def test06DefaultMethod(self):
-        self.setup_tutorial("tut06_default_method", "UsersPage")
-        self.getPage('/hendrik')
-        self.assertBody('Hendrik Mans, CherryPy co-developer & crazy German '
-                        '(back)')
-
-    def test07Sessions(self):
-        self.setup_tutorial("tut07_sessions", "HitCounter")
-
-        self.getPage('/')
-        self.assertBody(
-            "\n            During your current session, you've viewed this"
-            "\n            page 1 times! Your life is a patio of fun!"
-            "\n        ")
-
-        self.getPage('/', self.cookies)
-        self.assertBody(
-            "\n            During your current session, you've viewed this"
-            "\n            page 2 times! Your life is a patio of fun!"
-            "\n        ")
-
-    def test08GeneratorsAndYield(self):
-        self.setup_tutorial("tut08_generators_and_yield", "GeneratorDemo")
-        self.getPage('/')
-        self.assertBody('
Generators rule!
'
-                        '
List of users:
'
-                        'Remi
Carlos
Hendrik
Lorenzo Lamas
'
-                        '')
-
-    def test09Files(self):
-        self.setup_tutorial("tut09_files", "FileDemo")
-
-        # Test upload
-        filesize = 5
-        h = [("Content-type", "multipart/form-data; boundary=x"),
-             ("Content-Length", str(105 + filesize))]
-        b = ('--x\n'
-             'Content-Disposition: form-data; name="myFile"; '
-             'filename="hello.txt"\r\n'
-             'Content-Type: text/plain\r\n'
-             '\r\n')
-        b += 'a' * filesize + '\n' + '--x--\n'
-        self.getPage('/upload', h, "POST", b)
-        self.assertBody('''
-        
-            myFile length: %d

-            myFile filename: hello.txt

-            myFile mime-type: text/plain
-        
-        ''' % filesize)
-
-        # Test download
-        self.getPage('/download')
-        self.assertStatus("200 OK")
-        self.assertHeader("Content-Type", "application/x-download")
-        self.assertHeader("Content-Disposition",
-                          # Make sure the filename is quoted.
-                          'attachment; filename="pdf_file.pdf"')
-        self.assertEqual(len(self.body), 85698)
-
-    def test10HTTPErrors(self):
-        self.setup_tutorial("tut10_http_errors", "HTTPErrorDemo")
-
-        @cherrypy.expose
-        def traceback_setting():
-            return repr(cherrypy.request.show_tracebacks)
-        cherrypy.tree.mount(traceback_setting, '/traceback_setting')
-
-        self.getPage("/")
-        self.assertInBody("""""")
-        self.assertInBody("""""")
-        self.assertInBody("""""")
-        self.assertInBody("""""")
-        self.assertInBody("""""")
-
-        self.getPage("/traceback_setting")
-        setting = self.body
-        self.getPage("/toggleTracebacks")
-        self.assertStatus((302, 303))
-        self.getPage("/traceback_setting")
-        self.assertBody(str(not eval(setting)))
-
-        self.getPage("/error?code=500")
-        self.assertStatus(500)
-        self.assertInBody("The server encountered an unexpected condition "
-                          "which prevented it from fulfilling the request.")
-
-        self.getPage("/error?code=403")
-        self.assertStatus(403)
-        self.assertInBody("
You can't do that!
")
-
-        self.getPage("/messageArg")
-        self.assertStatus(500)
-        self.assertInBody("If you construct an HTTPError with a 'message'")
diff --git a/lib/cherrypy/test/test_virtualhost.py b/lib/cherrypy/test/test_virtualhost.py
deleted file mode 100644
index efd7719e4cc..00000000000
--- a/lib/cherrypy/test/test_virtualhost.py
+++ /dev/null
@@ -1,112 +0,0 @@
-import os
-curdir = os.path.join(os.getcwd(), os.path.dirname(__file__))
-
-import cherrypy
-from cherrypy.test import helper
-
-
-class VirtualHostTest(helper.CPWebCase):
-
-    def setup_server():
-        class Root:
-
-            @cherrypy.expose
-            def index(self):
-                return "Hello, world"
-
-            @cherrypy.expose
-            def dom4(self):
-                return "Under construction"
-
-            @cherrypy.expose
-            def method(self, value):
-                return "You sent %s" % value
-
-        class VHost:
-
-            def __init__(self, sitename):
-                self.sitename = sitename
-
-            @cherrypy.expose
-            def index(self):
-                return "Welcome to %s" % self.sitename
-
-            @cherrypy.expose
-            def vmethod(self, value):
-                return "You sent %s" % value
-
-            @cherrypy.expose
-            def url(self):
-                return cherrypy.url("nextpage")
-
-            # Test static as a handler (section must NOT include vhost prefix)
-            static = cherrypy.tools.staticdir.handler(
-                section='/static', dir=curdir)
-
-        root = Root()
-        root.mydom2 = VHost("Domain 2")
-        root.mydom3 = VHost("Domain 3")
-        hostmap = {'www.mydom2.com': '/mydom2',
-                   'www.mydom3.com': '/mydom3',
-                   'www.mydom4.com': '/dom4',
-                   }
-        cherrypy.tree.mount(root, config={
-            '/': {
-                'request.dispatch': cherrypy.dispatch.VirtualHost(**hostmap)
-            },
-            # Test static in config (section must include vhost prefix)
-            '/mydom2/static2': {
-                'tools.staticdir.on': True,
-                'tools.staticdir.root': curdir,
-                'tools.staticdir.dir': 'static',
-                'tools.staticdir.index': 'index.html',
-            },
-        })
-    setup_server = staticmethod(setup_server)
-
-    def testVirtualHost(self):
-        self.getPage("/", [('Host', 'www.mydom1.com')])
-        self.assertBody('Hello, world')
-        self.getPage("/mydom2/", [('Host', 'www.mydom1.com')])
-        self.assertBody('Welcome to Domain 2')
-
-        self.getPage("/", [('Host', 'www.mydom2.com')])
-        self.assertBody('Welcome to Domain 2')
-        self.getPage("/", [('Host', 'www.mydom3.com')])
-        self.assertBody('Welcome to Domain 3')
-        self.getPage("/", [('Host', 'www.mydom4.com')])
-        self.assertBody('Under construction')
-
-        # Test GET, POST, and positional params
-        self.getPage("/method?value=root")
-        self.assertBody("You sent root")
-        self.getPage("/vmethod?value=dom2+GET", [('Host', 'www.mydom2.com')])
-        self.assertBody("You sent dom2 GET")
-        self.getPage("/vmethod", [('Host', 'www.mydom3.com')], method="POST",
-                     body="value=dom3+POST")
-        self.assertBody("You sent dom3 POST")
-        self.getPage("/vmethod/pos", [('Host', 'www.mydom3.com')])
-        self.assertBody("You sent pos")
-
-        # Test that cherrypy.url uses the browser url, not the virtual url
-        self.getPage("/url", [('Host', 'www.mydom2.com')])
-        self.assertBody("%s://www.mydom2.com/nextpage" % self.scheme)
-
-    def test_VHost_plus_Static(self):
-        # Test static as a handler
-        self.getPage("/static/style.css", [('Host', 'www.mydom2.com')])
-        self.assertStatus('200 OK')
-        self.assertHeader('Content-Type', 'text/css;charset=utf-8')
-
-        # Test static in config
-        self.getPage("/static2/dirback.jpg", [('Host', 'www.mydom2.com')])
-        self.assertStatus('200 OK')
-        self.assertHeaderIn('Content-Type', ['image/jpeg', 'image/pjpeg'])
-
-        # Test static config with "index" arg
-        self.getPage("/static2/", [('Host', 'www.mydom2.com')])
-        self.assertStatus('200 OK')
-        self.assertBody('Hello, world\r\n')
-        # Since tools.trailing_slash is on by default, this should redirect
-        self.getPage("/static2", [('Host', 'www.mydom2.com')])
-        self.assertStatus(301)
diff --git a/lib/cherrypy/test/test_wsgi_ns.py b/lib/cherrypy/test/test_wsgi_ns.py
deleted file mode 100644
index dd6c5da075b..00000000000
--- a/lib/cherrypy/test/test_wsgi_ns.py
+++ /dev/null
@@ -1,94 +0,0 @@
-import cherrypy
-from cherrypy._cpcompat import ntob
-from cherrypy.test import helper
-
-
-class WSGI_Namespace_Test(helper.CPWebCase):
-
-    def setup_server():
-
-        class WSGIResponse(object):
-
-            def __init__(self, appresults):
-                self.appresults = appresults
-                self.iter = iter(appresults)
-
-            def __iter__(self):
-                return self
-
-            def next(self):
-                return self.iter.next()
-
-            def __next__(self):
-                return next(self.iter)
-
-            def close(self):
-                if hasattr(self.appresults, "close"):
-                    self.appresults.close()
-
-        class ChangeCase(object):
-
-            def __init__(self, app, to=None):
-                self.app = app
-                self.to = to
-
-            def __call__(self, environ, start_response):
-                res = self.app(environ, start_response)
-
-                class CaseResults(WSGIResponse):
-
-                    def next(this):
-                        return getattr(this.iter.next(), self.to)()
-
-                    def __next__(this):
-                        return getattr(next(this.iter), self.to)()
-                return CaseResults(res)
-
-        class Replacer(object):
-
-            def __init__(self, app, map={}):
-                self.app = app
-                self.map = map
-
-            def __call__(self, environ, start_response):
-                res = self.app(environ, start_response)
-
-                class ReplaceResults(WSGIResponse):
-
-                    def next(this):
-                        line = this.iter.next()
-                        for k, v in self.map.iteritems():
-                            line = line.replace(k, v)
-                        return line
-
-                    def __next__(this):
-                        line = next(this.iter)
-                        for k, v in self.map.items():
-                            line = line.replace(k, v)
-                        return line
-                return ReplaceResults(res)
-
-        class Root(object):
-
-            @cherrypy.expose
-            def index(self):
-                return "HellO WoRlD!"
-
-        root_conf = {'wsgi.pipeline': [('replace', Replacer)],
-                     'wsgi.replace.map': {ntob('L'): ntob('X'),
-                                          ntob('l'): ntob('r')},
-                     }
-
-        app = cherrypy.Application(Root())
-        app.wsgiapp.pipeline.append(('changecase', ChangeCase))
-        app.wsgiapp.config['changecase'] = {'to': 'upper'}
-        cherrypy.tree.mount(app, config={'/': root_conf})
-    setup_server = staticmethod(setup_server)
-
-    def test_pipeline(self):
-        if not cherrypy.server.httpserver:
-            return self.skip()
-
-        self.getPage("/")
-        # If body is "HEXXO WORXD!", the middleware was applied out of order.
-        self.assertBody("HERRO WORRD!")
diff --git a/lib/cherrypy/test/test_wsgi_unix_socket.py b/lib/cherrypy/test/test_wsgi_unix_socket.py
deleted file mode 100644
index 7a829d51068..00000000000
--- a/lib/cherrypy/test/test_wsgi_unix_socket.py
+++ /dev/null
@@ -1,106 +0,0 @@
-import os
-import sys
-import socket
-import atexit
-import tempfile
-
-import cherrypy
-from cherrypy.test import helper
-from cherrypy._cpcompat import HTTPConnection
-
-def usocket_path():
-    fd, path = tempfile.mkstemp('cp_test.sock')
-    os.close(fd)
-    os.remove(path)
-    return path
-
-USOCKET_PATH = usocket_path()
-
-class USocketHTTPConnection(HTTPConnection):
-    """
-    HTTPConnection over a unix socket.
-    """
-
-    def __init__(self, path):
-        HTTPConnection.__init__(self, 'localhost')
-        self.path = path
-
-    def __call__(self, *args, **kwargs):
-        """
-        Catch-all method just to present itself as a constructor for the
-        HTTPConnection.
-        """
-        return self
-
-    def connect(self):
-        """
-        Override the connect method and assign a unix socket as a transport.
-        """
-        sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
-        sock.connect(self.path)
-        self.sock = sock
-        atexit.register(lambda: os.remove(self.path))
-
-
-def skip_on_windows(method):
-    """
-    Decorator to skip the method call if the test is executing on Windows.
-    """
-    def wrapper(self):
-        if sys.platform == "win32":
-            return self.skip("No UNIX Socket support in Windows.")
-        else:
-            return method(self)
-    wrapper.__doc__ = method.__doc__
-    wrapper.__name__ = method.__name__
-    return wrapper
-
-
-
-class WSGI_UnixSocket_Test(helper.CPWebCase):
-    """
-    Test basic behavior on a cherrypy wsgi server listening
-    on a unix socket.
-
-    It exercises the config option `server.socket_file`.
-    """
-    HTTP_CONN = USocketHTTPConnection(USOCKET_PATH)
-
-
-    @staticmethod
-    def setup_server():
-        class Root(object):
-
-            @cherrypy.expose
-            def index(self):
-                return "Test OK"
-
-            @cherrypy.expose
-            def error(self):
-                raise Exception("Invalid page")
-
-        config = {
-            'server.socket_file': USOCKET_PATH
-        }
-        cherrypy.config.update(config)
-        cherrypy.tree.mount(Root())
-
-    def tearDown(self):
-        cherrypy.config.update({'server.socket_file': None})
-
-    @skip_on_windows
-    def test_simple_request(self):
-        self.getPage("/")
-        self.assertStatus("200 OK")
-        self.assertInBody("Test OK")
-
-    @skip_on_windows
-    def test_not_found(self):
-        self.getPage("/invalid_path")
-        self.assertStatus("404 Not Found")
-
-    @skip_on_windows
-    def test_internal_error(self):
-        self.getPage("/error")
-        self.assertStatus("500 Internal Server Error")
-        self.assertInBody("Invalid page")
diff --git a/lib/cherrypy/test/test_wsgi_vhost.py b/lib/cherrypy/test/test_wsgi_vhost.py
deleted file mode 100644
index af2ea56032f..00000000000
--- a/lib/cherrypy/test/test_wsgi_vhost.py
+++ /dev/null
@@ -1,35 +0,0 @@
-import cherrypy
-from cherrypy.test import helper
-
-
-class WSGI_VirtualHost_Test(helper.CPWebCase):
-
-    def setup_server():
-
-        class ClassOfRoot(object):
-
-            def __init__(self, name):
-                self.name = name
-
-            @cherrypy.expose
-            def index(self):
-                return "Welcome to the %s website!" % self.name
-
-        default = cherrypy.Application(None)
-
-        domains = {}
-        for year in range(1997, 2008):
-            app = cherrypy.Application(ClassOfRoot('Class of %s' % year))
-            domains['www.classof%s.example' % year] = app
-
-        cherrypy.tree.graft(cherrypy._cpwsgi.VirtualHost(default, domains))
-    setup_server = staticmethod(setup_server)
-
-    def test_welcome(self):
-        if not cherrypy.server.using_wsgi:
-            return self.skip("skipped (not using WSGI)... ")
-
-        for year in range(1997, 2008):
-            self.getPage(
-                "/", headers=[('Host', 'www.classof%s.example' % year)])
-            self.assertBody("Welcome to the Class of %s website!" % year)
diff --git a/lib/cherrypy/test/test_wsgiapps.py b/lib/cherrypy/test/test_wsgiapps.py
deleted file mode 100644
index 858faad0016..00000000000
--- a/lib/cherrypy/test/test_wsgiapps.py
+++ /dev/null
@@ -1,126 +0,0 @@
-import sys
-
-from cherrypy._cpcompat import ntob
-from cherrypy.test import helper
-
-
-class WSGIGraftTests(helper.CPWebCase):
-
-    def setup_server():
-        import os
-        curdir = os.path.join(os.getcwd(), os.path.dirname(__file__))
-
-        import cherrypy
-
-        def test_app(environ, start_response):
-            status = '200 OK'
-            response_headers = [('Content-type', 'text/plain')]
-            start_response(status, response_headers)
-            output = ['Hello, world!\n',
-                      'This is a wsgi app running within CherryPy!\n\n']
-            keys = list(environ.keys())
-            keys.sort()
-            for k in keys:
-                output.append('%s: %s\n' % (k, environ[k]))
-            return [ntob(x, 'utf-8') for x in output]
-
-        def test_empty_string_app(environ, start_response):
-            status = '200 OK'
-            response_headers = [('Content-type', 'text/plain')]
-            start_response(status, response_headers)
-            return [
-                ntob('Hello'), ntob(''), ntob(' '), ntob(''), ntob('world')
-            ]
-
-        class WSGIResponse(object):
-
-            def __init__(self, appresults):
-                self.appresults = appresults
-                self.iter = iter(appresults)
-
-            def __iter__(self):
-                return self
-
-            if sys.version_info >= (3, 0):
-                def __next__(self):
-                    return next(self.iter)
-            else:
-                def next(self):
-                    return self.iter.next()
-
-            def close(self):
-                if hasattr(self.appresults, "close"):
-                    self.appresults.close()
-
-        class ReversingMiddleware(object):
-
-            def __init__(self, app):
-                self.app = app
-
-            def __call__(self, environ, start_response):
-                results = app(environ, start_response)
-
-                class Reverser(WSGIResponse):
-
-                    if sys.version_info >= (3, 0):
-                        def __next__(this):
-                            line = list(next(this.iter))
-                            line.reverse()
-                            return bytes(line)
-                    else:
-                        def next(this):
-                            line = list(this.iter.next())
-                            line.reverse()
-                            return "".join(line)
-
-                return Reverser(results)
-
-        class Root:
-
-            @cherrypy.expose
-            def index(self):
-                return ntob("I'm a regular CherryPy page handler!")
-
-        cherrypy.tree.mount(Root())
-
-        cherrypy.tree.graft(test_app, '/hosted/app1')
-        cherrypy.tree.graft(test_empty_string_app, '/hosted/app3')
-
-        # Set script_name explicitly to None to signal CP that it should
-        # be pulled from the WSGI environ each time.
-        app = cherrypy.Application(Root(), script_name=None)
-        cherrypy.tree.graft(ReversingMiddleware(app), '/hosted/app2')
-    setup_server = staticmethod(setup_server)
-
-    wsgi_output = '''Hello, world!
-This is a wsgi app running within CherryPy!'''
-
-    def test_01_standard_app(self):
-        self.getPage("/")
-        self.assertBody("I'm a regular CherryPy page handler!")
-
-    def test_04_pure_wsgi(self):
-        import cherrypy
-        if not cherrypy.server.using_wsgi:
-            return self.skip("skipped (not using WSGI)... ")
-        self.getPage("/hosted/app1")
-        self.assertHeader("Content-Type", "text/plain")
-        self.assertInBody(self.wsgi_output)
-
-    def test_05_wrapped_cp_app(self):
-        import cherrypy
-        if not cherrypy.server.using_wsgi:
-            return self.skip("skipped (not using WSGI)... ")
-        self.getPage("/hosted/app2/")
-        body = list("I'm a regular CherryPy page handler!")
-        body.reverse()
-        body = "".join(body)
-        self.assertInBody(body)
-
-    def test_06_empty_string_app(self):
-        import cherrypy
-        if not cherrypy.server.using_wsgi:
-            return self.skip("skipped (not using WSGI)... ")
-        self.getPage("/hosted/app3")
-        self.assertHeader("Content-Type", "text/plain")
-        self.assertInBody('Hello world')
diff --git a/lib/cherrypy/test/test_xmlrpc.py b/lib/cherrypy/test/test_xmlrpc.py
deleted file mode 100644
index e95d5d79c04..00000000000
--- a/lib/cherrypy/test/test_xmlrpc.py
+++ /dev/null
@@ -1,187 +0,0 @@
-import sys
-
-import six
-
-try:
-    from xmlrpclib import DateTime, Fault, ProtocolError, ServerProxy
-    from xmlrpclib import SafeTransport
-except ImportError:
-    from xmlrpc.client import DateTime, Fault, ProtocolError, ServerProxy
-    from xmlrpc.client import SafeTransport
-
-if six.PY3:
-    HTTPSTransport = SafeTransport
-
-    # Python 3.0's SafeTransport still mistakenly checks for socket.ssl
-    import socket
-    if not hasattr(socket, "ssl"):
-        socket.ssl = True
-else:
-    class HTTPSTransport(SafeTransport):
-
-        """Subclass of SafeTransport to fix sock.recv errors (by using file).
-        """
-
-        def request(self, host, handler, request_body, verbose=0):
-            # issue XML-RPC request
-            h = self.make_connection(host)
-            if verbose:
-                h.set_debuglevel(1)
-
-            self.send_request(h, handler, request_body)
-            self.send_host(h, host)
-            self.send_user_agent(h)
-            self.send_content(h, request_body)
-
-            errcode, errmsg, headers = h.getreply()
-            if errcode != 200:
-                raise ProtocolError(host + handler, errcode, errmsg, headers)
-
-            self.verbose = verbose
-
-            # Here's where we differ from the superclass. It says:
-            # try:
-            #     sock = h._conn.sock
-            # except AttributeError:
-            #     sock = None
-            # return self._parse_response(h.getfile(), sock)
-
-            return self.parse_response(h.getfile())
-
-import cherrypy
-
-
-def setup_server():
-    from cherrypy import _cptools
-
-    class Root:
-
-        @cherrypy.expose
-        def index(self):
-            return "I'm a standard index!"
-
-    class XmlRpc(_cptools.XMLRPCController):
-
-        @cherrypy.expose
-        def foo(self):
-            return "Hello world!"
-
-        @cherrypy.expose
-        def return_single_item_list(self):
-            return [42]
-
-        @cherrypy.expose
-        def return_string(self):
-            return "here is a string"
-
-        @cherrypy.expose
-        def return_tuple(self):
-            return ('here', 'is', 1, 'tuple')
-
-        @cherrypy.expose
-        def return_dict(self):
-            return dict(a=1, b=2, c=3)
-
-        @cherrypy.expose
-        def return_composite(self):
-            return dict(a=1, z=26), 'hi', ['welcome', 'friend']
-
-        @cherrypy.expose
-        def return_int(self):
-            return 42
-
-        @cherrypy.expose
-        def return_float(self):
-            return 3.14
-
-        @cherrypy.expose
-        def return_datetime(self):
-            return DateTime((2003, 10, 7, 8, 1, 0, 1, 280, -1))
-
-        @cherrypy.expose
-        def return_boolean(self):
-            return True
-
-        @cherrypy.expose
-        def test_argument_passing(self, num):
-            return num * 2
-
-        @cherrypy.expose
-        def test_returning_Fault(self):
-            return Fault(1, "custom Fault response")
-
-    root = Root()
-    root.xmlrpc = XmlRpc()
-    cherrypy.tree.mount(root, config={'/': {
-        'request.dispatch': cherrypy.dispatch.XMLRPCDispatcher(),
-        'tools.xmlrpc.allow_none': 0,
-    }})
-
-
-from cherrypy.test import helper
-
-
-class XmlRpcTest(helper.CPWebCase):
-    setup_server = staticmethod(setup_server)
-
-    def testXmlRpc(self):
-
-        scheme = self.scheme
-        if scheme == "https":
-            url = 'https://%s:%s/xmlrpc/' % (self.interface(), self.PORT)
-            proxy = ServerProxy(url, transport=HTTPSTransport())
-        else:
-            url = 'http://%s:%s/xmlrpc/' % (self.interface(), self.PORT)
-            proxy = ServerProxy(url)
-
-        # begin the tests ...
-        self.getPage("/xmlrpc/foo")
-        self.assertBody("Hello world!")
-
-        self.assertEqual(proxy.return_single_item_list(), [42])
-        self.assertNotEqual(proxy.return_single_item_list(), 'one bazillion')
-        self.assertEqual(proxy.return_string(), "here is a string")
-        self.assertEqual(proxy.return_tuple(),
-                         list(('here', 'is', 1, 'tuple')))
-        self.assertEqual(proxy.return_dict(), {'a': 1, 'c': 3, 'b': 2})
-        self.assertEqual(proxy.return_composite(),
-                         [{'a': 1, 'z': 26}, 'hi', ['welcome', 'friend']])
-        self.assertEqual(proxy.return_int(), 42)
-        self.assertEqual(proxy.return_float(), 3.14)
-        self.assertEqual(proxy.return_datetime(),
-                         DateTime((2003, 10, 7, 8, 1, 0, 1, 280, -1)))
-        self.assertEqual(proxy.return_boolean(), True)
-        self.assertEqual(proxy.test_argument_passing(22), 22 * 2)
-
-        # Test an error in the page handler (should raise an xmlrpclib.Fault)
-        try:
-            proxy.test_argument_passing({})
-        except Exception:
-            x = sys.exc_info()[1]
-            self.assertEqual(x.__class__, Fault)
-            self.assertEqual(x.faultString, ("unsupported operand type(s) "
-                                             "for *: 'dict' and 'int'"))
-        else:
-            self.fail("Expected xmlrpclib.Fault")
-
-        # https://github.com/cherrypy/cherrypy/issues/533
-        # if a method is not found, an xmlrpclib.Fault should be raised
-        try:
-            proxy.non_method()
-        except Exception:
-            x = sys.exc_info()[1]
-            self.assertEqual(x.__class__, Fault)
-            self.assertEqual(x.faultString,
-                             'method "non_method" is not supported')
-        else:
-            self.fail("Expected xmlrpclib.Fault")
-
-        # Test returning a Fault from the page handler.
-        try:
-            proxy.test_returning_Fault()
-        except Exception:
-            x = sys.exc_info()[1]
-            self.assertEqual(x.__class__, Fault)
-            self.assertEqual(x.faultString, ("custom Fault response"))
-        else:
-            self.fail("Expected xmlrpclib.Fault")
diff --git a/lib/cherrypy/test/webtest.py b/lib/cherrypy/test/webtest.py
deleted file mode 100644
index 5ee8c5d975f..00000000000
--- a/lib/cherrypy/test/webtest.py
+++ /dev/null
@@ -1,613 +0,0 @@
-"""Extensions to unittest for web frameworks.
-
-Use the WebCase.getPage method to request a page from your HTTP server.
-
-Framework Integration
-=====================
-
-If you have control over your server process, you can handle errors
-in the server-side of the HTTP conversation a bit better. You must run
-both the client (your WebCase tests) and the server in the same process
-(but in separate threads, obviously).
-
-When an error occurs in the framework, call server_error. It will print
-the traceback to stdout, and keep any assertions you have from running
-(the assumption is that, if the server errors, the page output will not
-be of further significance to your tests).
-"""
-
-import pprint
-import re
-import socket
-import sys
-import time
-import traceback
-import types
-import os
-import json
-
-from unittest import *
-from unittest import _TextTestResult
-
-import six
-
-from cherrypy._cpcompat import basestring, ntob, HTTPConnection
-from cherrypy._cpcompat import HTTPSConnection
-
-
-def interface(host):
-    """Return an IP address for a client connection given the server host.
-
-    If the server is listening on '0.0.0.0' (INADDR_ANY)
-    or '::' (IN6ADDR_ANY), this will return the proper localhost."""
-    if host == '0.0.0.0':
-        # INADDR_ANY, which should respond on localhost.
-        return "127.0.0.1"
-    if host == '::':
-        # IN6ADDR_ANY, which should respond on localhost.
-        return "::1"
-    return host
-
-
-class TerseTestResult(_TextTestResult):
-
-    def printErrors(self):
-        # Overridden to avoid unnecessary empty line
-        if self.errors or self.failures:
-            if self.dots or self.showAll:
-                self.stream.writeln()
-            self.printErrorList('ERROR', self.errors)
-            self.printErrorList('FAIL', self.failures)
-
-
-class TerseTestRunner(TextTestRunner):
-
-    """A test runner class that displays results in textual form."""
-
-    def _makeResult(self):
-        return TerseTestResult(self.stream, self.descriptions, self.verbosity)
-
-    def run(self, test):
-        "Run the given test case or test suite."
-        # Overridden to remove unnecessary empty lines and separators
-        result = self._makeResult()
-        test(result)
-        result.printErrors()
-        if not result.wasSuccessful():
-            self.stream.write("FAILED (")
-            failed, errored = list(map(len, (result.failures, result.errors)))
-            if failed:
-                self.stream.write("failures=%d" % failed)
-            if errored:
-                if failed:
-                    self.stream.write(", ")
-                self.stream.write("errors=%d" % errored)
-            self.stream.writeln(")")
-        return result
-
-
-class ReloadingTestLoader(TestLoader):
-
-    def loadTestsFromName(self, name, module=None):
-        """Return a suite of all tests cases given a string specifier.
-
-        The name may resolve either to a module, a test case class, a
-        test method within a test case class, or a callable object which
-        returns a TestCase or TestSuite instance.
-
-        The method optionally resolves the names relative to a given module.
-        """
-        parts = name.split('.')
-        unused_parts = []
-        if module is None:
-            if not parts:
-                raise ValueError("incomplete test name: %s" % name)
-            else:
-                parts_copy = parts[:]
-                while parts_copy:
-                    target = ".".join(parts_copy)
-                    if target in sys.modules:
-                        module = reload(sys.modules[target])
-                        parts = unused_parts
-                        break
-                    else:
-                        try:
-                            module = __import__(target)
-                            parts = unused_parts
-                            break
-                        except ImportError:
-                            unused_parts.insert(0, parts_copy[-1])
-                            del parts_copy[-1]
-                            if not parts_copy:
-                                raise
-                parts = parts[1:]
-        obj = module
-        for part in parts:
-            obj = getattr(obj, part)
-
-        if isinstance(obj, types.ModuleType):
-            return self.loadTestsFromModule(obj)
-        elif (((six.PY3 and isinstance(obj, type))
-               or isinstance(obj, (type, types.ClassType)))
-              and issubclass(obj, TestCase)):
-            return self.loadTestsFromTestCase(obj)
-        elif isinstance(obj, types.UnboundMethodType):
-            if six.PY3:
-                return obj.__self__.__class__(obj.__name__)
-            else:
-                return obj.im_class(obj.__name__)
-        elif hasattr(obj, '__call__'):
-            test = obj()
-            if not isinstance(test, TestCase) and \
-               not isinstance(test, TestSuite):
-                raise ValueError("calling %s returned %s, "
-                                 "not a test" % (obj, test))
-            return test
-        else:
-            raise ValueError("do not know how to make test from: %s" % obj)
-
-
-try:
-    # Jython support
-    if sys.platform[:4] == 'java':
-        def getchar():
-            # Hopefully this is enough
-            return sys.stdin.read(1)
-    else:
-        # On Windows, msvcrt.getch reads a single char without output.
-        import msvcrt
-
-        def getchar():
-            return msvcrt.getch()
-except ImportError:
-    # Unix getchr
-    import tty
-    import termios
-
-    def getchar():
-        fd = sys.stdin.fileno()
-        old_settings = termios.tcgetattr(fd)
-        try:
-            tty.setraw(sys.stdin.fileno())
-            ch = sys.stdin.read(1)
-        finally:
-            termios.tcsetattr(fd, termios.TCSADRAIN, old_settings)
-        return ch
-
-
-# from jaraco.properties
-class NonDataProperty(object):
-    def __init__(self, fget):
-        assert fget is not None, "fget cannot be none"
-        assert callable(fget), "fget must be callable"
-        self.fget = fget
-
-    def __get__(self, obj, objtype=None):
-        if obj is None:
-            return self
-        return self.fget(obj)
-
-
-class WebCase(TestCase):
-    HOST = "127.0.0.1"
-    PORT = 8000
-    HTTP_CONN = HTTPConnection
-    PROTOCOL = "HTTP/1.1"
-
-    scheme = "http"
-    url = None
-
-    status = None
-    headers = None
-    body = None
-
-    encoding = 'utf-8'
-
-    time = None
-
-    def get_conn(self, auto_open=False):
-        """Return a connection to our HTTP server."""
-        if self.scheme == "https":
-            cls = HTTPSConnection
-        else:
-            cls = HTTPConnection
-        conn = cls(self.interface(), self.PORT)
-        # Automatically re-connect?
-        conn.auto_open = auto_open
-        conn.connect()
-        return conn
-
-    def set_persistent(self, on=True, auto_open=False):
-        """Make our HTTP_CONN persistent (or not).
-
-        If the 'on' argument is True (the default), then self.HTTP_CONN
-        will be set to an instance of HTTPConnection (or HTTPS
-        if self.scheme is "https"). This will then persist across requests.
-
-        We only allow for a single open connection, so if you call this
-        and we currently have an open connection, it will be closed.
-        """
-        try:
-            self.HTTP_CONN.close()
-        except (TypeError, AttributeError):
-            pass
-
-        if on:
-            self.HTTP_CONN = self.get_conn(auto_open=auto_open)
-        else:
-            if self.scheme == "https":
-                self.HTTP_CONN = HTTPSConnection
-            else:
-                self.HTTP_CONN = HTTPConnection
-
-    def _get_persistent(self):
-        return hasattr(self.HTTP_CONN, "__class__")
-
-    def _set_persistent(self, on):
-        self.set_persistent(on)
-    persistent = property(_get_persistent, _set_persistent)
-
-    def interface(self):
-        """Return an IP address for a client connection.
-
-        If the server is listening on '0.0.0.0' (INADDR_ANY)
-        or '::' (IN6ADDR_ANY), this will return the proper localhost."""
-        return interface(self.HOST)
-
-    def getPage(self, url, headers=None, method="GET", body=None,
-                protocol=None, raise_subcls=None):
-        """Open the url with debugging support. Return status, headers, body.
-
-        `raise_subcls` must be a tuple with the exceptions classes
-        or a single exception class that are not going to be considered
-        a socket.error regardless that they were are subclass of a
-        socket.error and therefore not considered for a connection retry.
-        """
-        ServerError.on = False
-
-        if isinstance(url, six.text_type):
-            url = url.encode('utf-8')
-        if isinstance(body, six.text_type):
-            body = body.encode('utf-8')
-
-        self.url = url
-        self.time = None
-        start = time.time()
-        result = openURL(url, headers, method, body, self.HOST, self.PORT,
-                         self.HTTP_CONN, protocol or self.PROTOCOL,
-                         raise_subcls)
-        self.time = time.time() - start
-        self.status, self.headers, self.body = result
-
-        # Build a list of request cookies from the previous response cookies.
-        self.cookies = [('Cookie', v) for k, v in self.headers
-                        if k.lower() == 'set-cookie']
-
-        if ServerError.on:
-            raise ServerError()
-        return result
-
-    @NonDataProperty
-    def interactive(self):
-        """
-        Load interactivity setting from environment, where
-        the value can be numeric or a string like true or
-        False or 1 or 0.
-        """
-        env_str = os.environ.get('WEBTEST_INTERACTIVE', 'True')
-        return bool(json.loads(env_str.lower()))
-
-    console_height = 30
-
-    def _handlewebError(self, msg):
-        print("")
-        print("    ERROR: %s" % msg)
-
-        if not self.interactive:
-            raise self.failureException(msg)
-
-        p = ("    Show: "
-             "[B]ody [H]eaders [S]tatus [U]RL; "
-             "[I]gnore, [R]aise, or sys.e[X]it >> ")
-        sys.stdout.write(p)
-        sys.stdout.flush()
-        while True:
-            i = getchar().upper()
-            if not isinstance(i, type("")):
-                i = i.decode('ascii')
-            if i not in "BHSUIRX":
-                continue
-            print(i.upper())  # Also prints new line
-            if i == "B":
-                for x, line in enumerate(self.body.splitlines()):
-                    if (x + 1) % self.console_height == 0:
-                        # The \r and comma should make the next line overwrite
-                        sys.stdout.write("<-- More -->\r")
-                        m = getchar().lower()
-                        # Erase our "More" prompt
-                        sys.stdout.write("            \r")
-                        if m == "q":
-                            break
-                    print(line)
-            elif i == "H":
-                pprint.pprint(self.headers)
-            elif i == "S":
-                print(self.status)
-            elif i == "U":
-                print(self.url)
-            elif i == "I":
-                # return without raising the normal exception
-                return
-            elif i == "R":
-                raise self.failureException(msg)
-            elif i == "X":
-                self.exit()
-            sys.stdout.write(p)
-            sys.stdout.flush()
-
-    def exit(self):
-        sys.exit()
-
-    def assertStatus(self, status, msg=None):
-        """Fail if self.status != status."""
-        if isinstance(status, basestring):
-            if not self.status == status:
-                if msg is None:
-                    msg = 'Status (%r) != %r' % (self.status, status)
-                self._handlewebError(msg)
-        elif isinstance(status, int):
-            code = int(self.status[:3])
-            if code != status:
-                if msg is None:
-                    msg = 'Status (%r) != %r' % (self.status, status)
-                self._handlewebError(msg)
-        else:
-            # status is a tuple or list.
-            match = False
-            for s in status:
-                if isinstance(s, basestring):
-                    if self.status == s:
-                        match = True
-                        break
-                elif int(self.status[:3]) == s:
-                    match = True
-                    break
-            if not match:
-                if msg is None:
-                    msg = 'Status (%r) not in %r' % (self.status, status)
-                self._handlewebError(msg)
-
-    def assertHeader(self, key, value=None, msg=None):
-        """Fail if (key, [value]) not in self.headers."""
-        lowkey = key.lower()
-        for k, v in self.headers:
-            if k.lower() == lowkey:
-                if value is None or str(value) == v:
-                    return v
-
-        if msg is None:
-            if value is None:
-                msg = '%r not in headers' % key
-            else:
-                msg = '%r:%r not in headers' % (key, value)
-        self._handlewebError(msg)
-
-    def assertHeaderIn(self, key, values, msg=None):
-        """Fail if header indicated by key doesn't have one of the values."""
-        lowkey = key.lower()
-        for k, v in self.headers:
-            if k.lower() == lowkey:
-                matches = [value for value in values if str(value) == v]
-                if matches:
-                    return matches
-
-        if msg is None:
-            msg = '%(key)r not in %(values)r' % vars()
-        self._handlewebError(msg)
-
-    def assertHeaderItemValue(self, key, value, msg=None):
-        """Fail if the header does not contain the specified value"""
-        actual_value = self.assertHeader(key, msg=msg)
-        header_values = map(str.strip, actual_value.split(','))
-        if value in header_values:
-            return value
-
-        if msg is None:
-            msg = "%r not in %r" % (value, header_values)
-        self._handlewebError(msg)
-
-    def assertNoHeader(self, key, msg=None):
-        """Fail if key in self.headers."""
-        lowkey = key.lower()
-        matches = [k for k, v in self.headers if k.lower() == lowkey]
-        if matches:
-            if msg is None:
-                msg = '%r in headers' % key
-            self._handlewebError(msg)
-
-    def assertBody(self, value, msg=None):
-        """Fail if value != self.body."""
-        if isinstance(value, six.text_type):
-            value = value.encode(self.encoding)
-        if value != self.body:
-            if msg is None:
-                msg = 'expected body:\n%r\n\nactual body:\n%r' % (
-                    value, self.body)
-            self._handlewebError(msg)
-
-    def assertInBody(self, value, msg=None):
-        """Fail if value not in self.body."""
-        if isinstance(value, six.text_type):
-            value = value.encode(self.encoding)
-        if value not in self.body:
-            if msg is None:
-                msg = '%r not in body: %s' % (value, self.body)
-            self._handlewebError(msg)
-
-    def assertNotInBody(self, value, msg=None):
-        """Fail if value in self.body."""
-        if isinstance(value, six.text_type):
-            value = value.encode(self.encoding)
-        if value in self.body:
-            if msg is None:
-                msg = '%r found in body' % value
-            self._handlewebError(msg)
-
-    def assertMatchesBody(self, pattern, msg=None, flags=0):
-        """Fail if value (a regex pattern) is not in self.body."""
-        if isinstance(pattern, six.text_type):
-            pattern = pattern.encode(self.encoding)
-        if re.search(pattern, self.body, flags) is None:
-            if msg is None:
-                msg = 'No match for %r in body' % pattern
-            self._handlewebError(msg)
-
-
-methods_with_bodies = ("POST", "PUT")
-
-
-def cleanHeaders(headers, method, body, host, port):
-    """Return request headers, with required headers added (if missing)."""
-    if headers is None:
-        headers = []
-
-    # Add the required Host request header if not present.
-    # [This specifies the host:port of the server, not the client.]
-    found = False
-    for k, v in headers:
-        if k.lower() == 'host':
-            found = True
-            break
-    if not found:
-        if port == 80:
-            headers.append(("Host", host))
-        else:
-            headers.append(("Host", "%s:%s" % (host, port)))
-
-    if method in methods_with_bodies:
-        # Stick in default type and length headers if not present
-        found = False
-        for k, v in headers:
-            if k.lower() == 'content-type':
-                found = True
-                break
-        if not found:
-            headers.append(
-                ("Content-Type", "application/x-www-form-urlencoded"))
-            headers.append(("Content-Length", str(len(body or ""))))
-
-    return headers
-
-
-def shb(response):
-    """Return status, headers, body the way we like from a response."""
-    if six.PY3:
-        h = response.getheaders()
-    else:
-        h = []
-        key, value = None, None
-        for line in response.msg.headers:
-            if line:
-                if line[0] in " \t":
-                    value += line.strip()
-                else:
-                    if key and value:
-                        h.append((key, value))
-                    key, value = line.split(":", 1)
-                    key = key.strip()
-                    value = value.strip()
-        if key and value:
-            h.append((key, value))
-
-    return "%s %s" % (response.status, response.reason), h, response.read()
-
-
-def openURL(url, headers=None, method="GET", body=None,
-            host="127.0.0.1", port=8000, http_conn=HTTPConnection,
-            protocol="HTTP/1.1", raise_subcls=None):
-    """
-    Open the given HTTP resource and return status, headers, and body.
-
-    `raise_subcls` must be a tuple with the exceptions classes
-    or a single exception class that are not going to be considered
-    a socket.error regardless that they were are subclass of a
-    socket.error and therefore not considered for a connection retry.
-    """
-    headers = cleanHeaders(headers, method, body, host, port)
-
-    # Trying 10 times is simply in case of socket errors.
-    # Normal case--it should run once.
-    for trial in range(10):
-        try:
-            # Allow http_conn to be a class or an instance
-            if hasattr(http_conn, "host"):
-                conn = http_conn
-            else:
-                conn = http_conn(interface(host), port)
-
-            conn._http_vsn_str = protocol
-            conn._http_vsn = int("".join([x for x in protocol if x.isdigit()]))
-
-            if six.PY3 and isinstance(url, bytes):
-                url = url.decode()
-            conn.putrequest(method.upper(), url, skip_host=True,
-                            skip_accept_encoding=True)
-
-            for key, value in headers:
-                conn.putheader(key, value.encode("Latin-1"))
-            conn.endheaders()
-
-            if body is not None:
-                conn.send(body)
-
-            # Handle response
-            response = conn.getresponse()
-
-            s, h, b = shb(response)
-
-            if not hasattr(http_conn, "host"):
-                # We made our own conn instance. Close it.
-                conn.close()
-
-            return s, h, b
-        except socket.error as e:
-            if raise_subcls is not None and isinstance(e, raise_subcls):
-                raise
-            else:
-                time.sleep(0.5)
-                if trial == 9:
-                    raise
-
-
-
-
-# Add any exceptions which your web framework handles
-# normally (that you don't want server_error to trap).
-ignored_exceptions = []
-
-# You'll want set this to True when you can't guarantee
-# that each response will immediately follow each request;
-# for example, when handling requests via multiple threads.
-ignore_all = False
-
-
-class ServerError(Exception):
-    on = False
-
-
-def server_error(exc=None):
-    """Server debug hook. Return True if exception handled, False if ignored.
-
-    You probably want to wrap this, so you can still handle an error using
-    your framework when it's ignored.
-    """
-    if exc is None:
-        exc = sys.exc_info()
-
-    if ignore_all or exc[0] in ignored_exceptions:
-        return False
-    else:
-        ServerError.on = True
-        print("")
-        print("".join(traceback.format_exception(*exc)))
-        return True
diff --git a/lib/cherrypy/tutorial/README.txt b/lib/cherrypy/tutorial/README.txt
deleted file mode 100644
index 2b877e1ff8d..00000000000
--- a/lib/cherrypy/tutorial/README.txt
+++ /dev/null
@@ -1,16 +0,0 @@
-CherryPy Tutorials
-------------------------------------------------------------------------
-
-This is a series of tutorials explaining how to develop dynamic web
-applications using CherryPy. A couple of notes:
-
-  - Each of these tutorials builds on the ones before it. If you're
-    new to CherryPy, we recommend you start with 01_helloworld.py and
-    work your way upwards. :)
-
-  - In most of these tutorials, you will notice that all output is done
-    by returning normal Python strings, often using simple Python
-    variable substitution. In most real-world applications, you will
-    probably want to use a separate template package (like Cheetah,
-    CherryTemplate or XML/XSL).
-
diff --git a/lib/cherrypy/tutorial/__init__.py b/lib/cherrypy/tutorial/__init__.py
deleted file mode 100644
index 08c142c5f57..00000000000
--- a/lib/cherrypy/tutorial/__init__.py
+++ /dev/null
@@ -1,3 +0,0 @@
-
-# This is used in test_config to test unrepr of "from A import B"
-thing2 = object()
diff --git a/lib/cherrypy/tutorial/custom_error.html b/lib/cherrypy/tutorial/custom_error.html
deleted file mode 100644
index d0f30c8abe4..00000000000
--- a/lib/cherrypy/tutorial/custom_error.html
+++ /dev/null
@@ -1,14 +0,0 @@
-
-
-
-
-    403 Unauthorized
-
-    
-        
You can't do that!

-        
%(message)s

-        
This is a custom error page that is read from a file.

-        
%(traceback)s

-    
-
diff --git a/lib/cherrypy/tutorial/pdf_file.pdf b/lib/cherrypy/tutorial/pdf_file.pdf
deleted file mode 100644
index 38b4f15eabd..00000000000
Binary files a/lib/cherrypy/tutorial/pdf_file.pdf and /dev/null differ
diff --git a/lib/cherrypy/tutorial/tut01_helloworld.py b/lib/cherrypy/tutorial/tut01_helloworld.py
deleted file mode 100644
index aaf89fd9dfb..00000000000
--- a/lib/cherrypy/tutorial/tut01_helloworld.py
+++ /dev/null
@@ -1,33 +0,0 @@
-"""
-Tutorial - Hello World
-
-The most basic (working) CherryPy application possible.
-"""
-
-# Import CherryPy global namespace
-import cherrypy
-
-
-class HelloWorld:
-
-    """ Sample request handler class. """
-
-    # Expose the index method through the web. CherryPy will never
-    # publish methods that don't have the exposed attribute set to True.
-    @cherrypy.expose
-    def index(self):
-        # CherryPy will call this method for the root URI ("/") and send
-        # its return value to the client. Because this is tutorial
-        # lesson number 01, we'll just send something really simple.
-        # How about...
-        return "Hello world!"
-
-
-import os.path
-tutconf = os.path.join(os.path.dirname(__file__), 'tutorial.conf')
-
-if __name__ == '__main__':
-    # CherryPy always starts with app.root when trying to map request URIs
-    # to objects, so we need to mount a request handler root. A request
-    # to '/' will be mapped to HelloWorld().index().
-    cherrypy.quickstart(HelloWorld(), config=tutconf)
diff --git a/lib/cherrypy/tutorial/tut02_expose_methods.py b/lib/cherrypy/tutorial/tut02_expose_methods.py
deleted file mode 100644
index d9e07cbf9cf..00000000000
--- a/lib/cherrypy/tutorial/tut02_expose_methods.py
+++ /dev/null
@@ -1,30 +0,0 @@
-"""
-Tutorial - Multiple methods
-
-This tutorial shows you how to link to other methods of your request
-handler.
-"""
-
-import cherrypy
-
-
-class HelloWorld:
-
-    @cherrypy.expose
-    def index(self):
-        # Let's link to another method here.
-        return 'We have an important message for you!'
-
-    @cherrypy.expose
-    def show_msg(self):
-        # Here's the important message!
-        return "Hello world!"
-
-import os.path
-tutconf = os.path.join(os.path.dirname(__file__), 'tutorial.conf')
-
-if __name__ == '__main__':
-    # CherryPy always starts with app.root when trying to map request URIs
-    # to objects, so we need to mount a request handler root. A request
-    # to '/' will be mapped to HelloWorld().index().
-    cherrypy.quickstart(HelloWorld(), config=tutconf)
diff --git a/lib/cherrypy/tutorial/tut03_get_and_post.py b/lib/cherrypy/tutorial/tut03_get_and_post.py
deleted file mode 100644
index 38b42e6be73..00000000000
--- a/lib/cherrypy/tutorial/tut03_get_and_post.py
+++ /dev/null
@@ -1,50 +0,0 @@
-"""
-Tutorial - Passing variables
-
-This tutorial shows you how to pass GET/POST variables to methods.
-"""
-
-import cherrypy
-
-
-class WelcomePage:
-
-    @cherrypy.expose
-    def index(self):
-        # Ask for the user's name.
-        return '''
-            
-            What is your name?
-            
-            
-            
'''
-
-    @cherrypy.expose
-    def greetUser(self, name=None):
-        # CherryPy passes all GET and POST variables as method parameters.
-        # It doesn't make a difference where the variables come from, how
-        # large their contents are, and so on.
-        #
-        # You can define default parameter values as usual. In this
-        # example, the "name" parameter defaults to None so we can check
-        # if a name was actually specified.
-
-        if name:
-            # Greet the user!
-            return "Hey %s, what's up?" % name
-        else:
-            if name is None:
-                # No name was specified
-                return 'Please enter your name here.'
-            else:
-                return 'No, really, enter your name here.'
-
-
-import os.path
-tutconf = os.path.join(os.path.dirname(__file__), 'tutorial.conf')
-
-if __name__ == '__main__':
-    # CherryPy always starts with app.root when trying to map request URIs
-    # to objects, so we need to mount a request handler root. A request
-    # to '/' will be mapped to HelloWorld().index().
-    cherrypy.quickstart(WelcomePage(), config=tutconf)
diff --git a/lib/cherrypy/tutorial/tut04_complex_site.py b/lib/cherrypy/tutorial/tut04_complex_site.py
deleted file mode 100644
index 6f8654af66b..00000000000
--- a/lib/cherrypy/tutorial/tut04_complex_site.py
+++ /dev/null
@@ -1,102 +0,0 @@
-"""
-Tutorial - Multiple objects
-
-This tutorial shows you how to create a site structure through multiple
-possibly nested request handler objects.
-"""
-
-import cherrypy
-
-
-class HomePage:
-
-    @cherrypy.expose
-    def index(self):
-        return '''
-            
Hi, this is the home page! Check out the other
-            fun stuff on this site:

-
-            '''
-
-
-class JokePage:
-
-    @cherrypy.expose
-    def index(self):
-        return '''
-            
"In Python, how do you create a string of random
-            characters?" -- "Read a Perl file!"

-            
[Return]
'''
-
-
-class LinksPage:
-
-    def __init__(self):
-        # Request handler objects can create their own nested request
-        # handler objects. Simply create them inside their __init__
-        # methods!
-        self.extra = ExtraLinksPage()
-
-    @cherrypy.expose
-    def index(self):
-        # Note the way we link to the extra links page (and back).
-        # As you can see, this object doesn't really care about its
-        # absolute position in the site tree, since we use relative
-        # links exclusively.
-        return '''
-            
Here are some useful links:

-
-            
-
-            
You can check out some extra useful
-            links here.

-
-            
[Return]

-        '''
-
-
-class ExtraLinksPage:
-
-    @cherrypy.expose
-    def index(self):
-        # Note the relative link back to the Links page!
-        return '''
-            
Here are some extra useful links:

-
-            
-
-            
[Return to links page]
'''
-
-
-# Of course we can also mount request handler objects right here!
-root = HomePage()
-root.joke = JokePage()
-root.links = LinksPage()
-
-# Remember, we don't need to mount ExtraLinksPage here, because
-# LinksPage does that itself on initialization. In fact, there is
-# no reason why you shouldn't let your root object take care of
-# creating all contained request handler objects.
-
-
-import os.path
-tutconf = os.path.join(os.path.dirname(__file__), 'tutorial.conf')
-
-if __name__ == '__main__':
-    # CherryPy always starts with app.root when trying to map request URIs
-    # to objects, so we need to mount a request handler root. A request
-    # to '/' will be mapped to HelloWorld().index().
-    cherrypy.quickstart(root, config=tutconf)
diff --git a/lib/cherrypy/tutorial/tut05_derived_objects.py b/lib/cherrypy/tutorial/tut05_derived_objects.py
deleted file mode 100644
index 647fac56959..00000000000
--- a/lib/cherrypy/tutorial/tut05_derived_objects.py
+++ /dev/null
@@ -1,79 +0,0 @@
-"""
-Tutorial - Object inheritance
-
-You are free to derive your request handler classes from any base
-class you wish. In most real-world applications, you will probably
-want to create a central base class used for all your pages, which takes
-care of things like printing a common page header and footer.
-"""
-
-import cherrypy
-
-
-class Page:
-    # Store the page title in a class attribute
-    title = 'Untitled Page'
-
-    def header(self):
-        return '''
-            
-            
-                %s
-            
-            
-            
%s

-        ''' % (self.title, self.title)
-
-    def footer(self):
-        return '''
-            
-            
-        '''
-
-    # Note that header and footer don't get their exposed attributes
-    # set to True. This isn't necessary since the user isn't supposed
-    # to call header or footer directly; instead, we'll call them from
-    # within the actually exposed handler methods defined in this
-    # class' subclasses.
-
-
-class HomePage(Page):
-    # Different title for this page
-    title = 'Tutorial 5'
-
-    def __init__(self):
-        # create a subpage
-        self.another = AnotherPage()
-
-    @cherrypy.expose
-    def index(self):
-        # Note that we call the header and footer methods inherited
-        # from the Page class!
-        return self.header() + '''
-            

-            Isn't this exciting? There's
-            another page, too!
-            

-        ''' + self.footer()
-
-
-class AnotherPage(Page):
-    title = 'Another Page'
-
-    @cherrypy.expose
-    def index(self):
-        return self.header() + '''
-            

-            And this is the amazing second page!
-            

-        ''' + self.footer()
-
-
-import os.path
-tutconf = os.path.join(os.path.dirname(__file__), 'tutorial.conf')
-
-if __name__ == '__main__':
-    # CherryPy always starts with app.root when trying to map request URIs
-    # to objects, so we need to mount a request handler root. A request
-    # to '/' will be mapped to HelloWorld().index().
-    cherrypy.quickstart(HomePage(), config=tutconf)
diff --git a/lib/cherrypy/tutorial/tut06_default_method.py b/lib/cherrypy/tutorial/tut06_default_method.py
deleted file mode 100644
index 24f839d7000..00000000000
--- a/lib/cherrypy/tutorial/tut06_default_method.py
+++ /dev/null
@@ -1,60 +0,0 @@
-"""
-Tutorial - The default method
-
-Request handler objects can implement a method called "default" that
-is called when no other suitable method/object could be found.
-Essentially, if CherryPy2 can't find a matching request handler object
-for the given request URI, it will use the default method of the object
-located deepest on the URI path.
-
-Using this mechanism you can easily simulate virtual URI structures
-by parsing the extra URI string, which you can access through
-cherrypy.request.virtualPath.
-
-The application in this tutorial simulates an URI structure looking
-like /users/. Since the  bit will not be found (as
-there are no matching methods), it is handled by the default method.
-"""
-
-import cherrypy
-
-
-class UsersPage:
-
-    @cherrypy.expose
-    def index(self):
-        # Since this is just a stupid little example, we'll simply
-        # display a list of links to random, made-up users. In a real
-        # application, this could be generated from a database result set.
-        return '''
-            Remi Delon

-            Hendrik Mans

-            Lorenzo Lamas

-        '''
-
-    @cherrypy.expose
-    def default(self, user):
-        # Here we react depending on the virtualPath -- the part of the
-        # path that could not be mapped to an object method. In a real
-        # application, we would probably do some database lookups here
-        # instead of the silly if/elif/else construct.
-        if user == 'remi':
-            out = "Remi Delon, CherryPy lead developer"
-        elif user == 'hendrik':
-            out = "Hendrik Mans, CherryPy co-developer & crazy German"
-        elif user == 'lorenzo':
-            out = "Lorenzo Lamas, famous actor and singer!"
-        else:
-            out = "Unknown user. :-("
-
-        return '%s (back)' % out
-
-
-import os.path
-tutconf = os.path.join(os.path.dirname(__file__), 'tutorial.conf')
-
-if __name__ == '__main__':
-    # CherryPy always starts with app.root when trying to map request URIs
-    # to objects, so we need to mount a request handler root. A request
-    # to '/' will be mapped to HelloWorld().index().
-    cherrypy.quickstart(UsersPage(), config=tutconf)
diff --git a/lib/cherrypy/tutorial/tut07_sessions.py b/lib/cherrypy/tutorial/tut07_sessions.py
deleted file mode 100644
index e6d31540542..00000000000
--- a/lib/cherrypy/tutorial/tut07_sessions.py
+++ /dev/null
@@ -1,40 +0,0 @@
-"""
-Tutorial - Sessions
-
-Storing session data in CherryPy applications is very easy: cherrypy
-provides a dictionary called "session" that represents the session
-data for the current user. If you use RAM based sessions, you can store
-any kind of object into that dictionary; otherwise, you are limited to
-objects that can be pickled.
-"""
-
-import cherrypy
-
-
-class HitCounter:
-
-    _cp_config = {'tools.sessions.on': True}
-
-    @cherrypy.expose
-    def index(self):
-        # Increase the silly hit counter
-        count = cherrypy.session.get('count', 0) + 1
-
-        # Store the new value in the session dictionary
-        cherrypy.session['count'] = count
-
-        # And display a silly hit count message!
-        return '''
-            During your current session, you've viewed this
-            page %s times! Your life is a patio of fun!
-        ''' % count
-
-
-import os.path
-tutconf = os.path.join(os.path.dirname(__file__), 'tutorial.conf')
-
-if __name__ == '__main__':
-    # CherryPy always starts with app.root when trying to map request URIs
-    # to objects, so we need to mount a request handler root. A request
-    # to '/' will be mapped to HelloWorld().index().
-    cherrypy.quickstart(HitCounter(), config=tutconf)
diff --git a/lib/cherrypy/tutorial/tut08_generators_and_yield.py b/lib/cherrypy/tutorial/tut08_generators_and_yield.py
deleted file mode 100644
index 8c4fe839c6b..00000000000
--- a/lib/cherrypy/tutorial/tut08_generators_and_yield.py
+++ /dev/null
@@ -1,43 +0,0 @@
-"""
-Bonus Tutorial: Using generators to return result bodies
-
-Instead of returning a complete result string, you can use the yield
-statement to return one result part after another. This may be convenient
-in situations where using a template package like CherryPy or Cheetah
-would be overkill, and messy string concatenation too uncool. ;-)
-"""
-
-import cherrypy
-
-
-class GeneratorDemo:
-
-    def header(self):
-        return "
Generators rule!
"
-
-    def footer(self):
-        return ""
-
-    @cherrypy.expose
-    def index(self):
-        # Let's make up a list of users for presentation purposes
-        users = ['Remi', 'Carlos', 'Hendrik', 'Lorenzo Lamas']
-
-        # Every yield line adds one part to the total result body.
-        yield self.header()
-        yield "
List of users:
"
-
-        for user in users:
-            yield "%s
" % user
-
-        yield self.footer()
-
-
-import os.path
-tutconf = os.path.join(os.path.dirname(__file__), 'tutorial.conf')
-
-if __name__ == '__main__':
-    # CherryPy always starts with app.root when trying to map request URIs
-    # to objects, so we need to mount a request handler root. A request
-    # to '/' will be mapped to HelloWorld().index().
-    cherrypy.quickstart(GeneratorDemo(), config=tutconf)
diff --git a/lib/cherrypy/tutorial/tut09_files.py b/lib/cherrypy/tutorial/tut09_files.py
deleted file mode 100644
index f76c1c6576b..00000000000
--- a/lib/cherrypy/tutorial/tut09_files.py
+++ /dev/null
@@ -1,104 +0,0 @@
-"""
-
-Tutorial: File upload and download
-
-Uploads
--------
-
-When a client uploads a file to a CherryPy application, it's placed
-on disk immediately. CherryPy will pass it to your exposed method
-as an argument (see "myFile" below); that arg will have a "file"
-attribute, which is a handle to the temporary uploaded file.
-If you wish to permanently save the file, you need to read()
-from myFile.file and write() somewhere else.
-
-Note the use of 'enctype="multipart/form-data"' and 'input type="file"'
-in the HTML which the client uses to upload the file.
-
-
-Downloads
----------
-
-If you wish to send a file to the client, you have two options:
-First, you can simply return a file-like object from your page handler.
-CherryPy will read the file and serve it as the content (HTTP body)
-of the response. However, that doesn't tell the client that
-the response is a file to be saved, rather than displayed.
-Use cherrypy.lib.static.serve_file for that; it takes four
-arguments:
-
-serve_file(path, content_type=None, disposition=None, name=None)
-
-Set "name" to the filename that you expect clients to use when they save
-your file. Note that the "name" argument is ignored if you don't also
-provide a "disposition" (usually "attachement"). You can manually set
-"content_type", but be aware that if you also use the encoding tool, it
-may choke if the file extension is not recognized as belonging to a known
-Content-Type. Setting the content_type to "application/x-download" works
-in most cases, and should prompt the user with an Open/Save dialog in
-popular browsers.
-
-"""
-
-import os
-localDir = os.path.dirname(__file__)
-absDir = os.path.join(os.getcwd(), localDir)
-
-import cherrypy
-from cherrypy.lib import static
-
-
-class FileDemo(object):
-
-    @cherrypy.expose
-    def index(self):
-        return """
-        
-            
Upload a file

-            

-            filename: 

-            
-            

-            
Download a file

-            This one
-        
-        """
-
-    @cherrypy.expose
-    def upload(self, myFile):
-        out = """
-        
-            myFile length: %s

-            myFile filename: %s

-            myFile mime-type: %s
-        
-        """
-
-        # Although this just counts the file length, it demonstrates
-        # how to read large files in chunks instead of all at once.
-        # CherryPy reads the uploaded file into a temporary file;
-        # myFile.file.read reads from that.
-        size = 0
-        while True:
-            data = myFile.file.read(8192)
-            if not data:
-                break
-            size += len(data)
-
-        return out % (size, myFile.filename, myFile.content_type)
-
-    @cherrypy.expose
-    def download(self):
-        path = os.path.join(absDir, "pdf_file.pdf")
-        return static.serve_file(path, "application/x-download",
-                                 "attachment", os.path.basename(path))
-
-
-import os.path
-tutconf = os.path.join(os.path.dirname(__file__), 'tutorial.conf')
-
-if __name__ == '__main__':
-    # CherryPy always starts with app.root when trying to map request URIs
-    # to objects, so we need to mount a request handler root. A request
-    # to '/' will be mapped to HelloWorld().index().
-    cherrypy.quickstart(FileDemo(), config=tutconf)
diff --git a/lib/cherrypy/tutorial/tut10_http_errors.py b/lib/cherrypy/tutorial/tut10_http_errors.py
deleted file mode 100644
index c1158ce69f8..00000000000
--- a/lib/cherrypy/tutorial/tut10_http_errors.py
+++ /dev/null
@@ -1,83 +0,0 @@
-"""
-
-Tutorial: HTTP errors
-
-HTTPError is used to return an error response to the client.
-CherryPy has lots of options regarding how such errors are
-logged, displayed, and formatted.
-
-"""
-
-import os
-localDir = os.path.dirname(__file__)
-curpath = os.path.normpath(os.path.join(os.getcwd(), localDir))
-
-import cherrypy
-
-
-class HTTPErrorDemo(object):
-
-    # Set a custom response for 403 errors.
-    _cp_config = {'error_page.403':
-                  os.path.join(curpath, "custom_error.html")}
-
-    @cherrypy.expose
-    def index(self):
-        # display some links that will result in errors
-        tracebacks = cherrypy.request.show_tracebacks
-        if tracebacks:
-            trace = 'off'
-        else:
-            trace = 'on'
-
-        return """
-        
-            
Toggle tracebacks %s

-            
Click me; I'm a broken link!

-            

-              
-                Use a custom error page from a file.
-              
-            

-            
These errors are explicitly raised by the application:

-            
-            
You can also set the response body
-            when you raise an error.

-        
-        """ % trace
-
-    @cherrypy.expose
-    def toggleTracebacks(self):
-        # simple function to toggle tracebacks on and off
-        tracebacks = cherrypy.request.show_tracebacks
-        cherrypy.config.update({'request.show_tracebacks': not tracebacks})
-
-        # redirect back to the index
-        raise cherrypy.HTTPRedirect('/')
-
-    @cherrypy.expose
-    def error(self, code):
-        # raise an error based on the get query
-        raise cherrypy.HTTPError(status=code)
-
-    @cherrypy.expose
-    def messageArg(self):
-        message = ("If you construct an HTTPError with a 'message' "
-                   "argument, it wil be placed on the error page "
-                   "(underneath the status line by default).")
-        raise cherrypy.HTTPError(500, message=message)
-
-
-import os.path
-tutconf = os.path.join(os.path.dirname(__file__), 'tutorial.conf')
-
-if __name__ == '__main__':
-    # CherryPy always starts with app.root when trying to map request URIs
-    # to objects, so we need to mount a request handler root. A request
-    # to '/' will be mapped to HelloWorld().index().
-    cherrypy.quickstart(HTTPErrorDemo(), config=tutconf)
diff --git a/lib/cherrypy/tutorial/tutorial.conf b/lib/cherrypy/tutorial/tutorial.conf
deleted file mode 100644
index 6537fd30c0e..00000000000
--- a/lib/cherrypy/tutorial/tutorial.conf
+++ /dev/null
@@ -1,4 +0,0 @@
-[global]
-server.socket_host = "127.0.0.1"
-server.socket_port = 8080
-server.thread_pool = 10
diff --git a/lib/cherrypy/wsgiserver/__init__.py b/lib/cherrypy/wsgiserver/__init__.py
deleted file mode 100644
index ee6190fee11..00000000000
--- a/lib/cherrypy/wsgiserver/__init__.py
+++ /dev/null
@@ -1,14 +0,0 @@
-__all__ = ['HTTPRequest', 'HTTPConnection', 'HTTPServer',
-           'SizeCheckWrapper', 'KnownLengthRFile', 'ChunkedRFile',
-           'MaxSizeExceeded', 'NoSSLError', 'FatalSSLAlert',
-           'WorkerThread', 'ThreadPool', 'SSLAdapter',
-           'CherryPyWSGIServer',
-           'Gateway', 'WSGIGateway', 'WSGIGateway_10', 'WSGIGateway_u0',
-           'WSGIPathInfoDispatcher', 'get_ssl_adapter_class']
-
-import sys
-if sys.version_info < (3, 0):
-    from wsgiserver2 import *
-else:
-    # Le sigh. Boo for backward-incompatible syntax.
-    exec('from .wsgiserver3 import *')
diff --git a/lib/cherrypy/wsgiserver/ssl_builtin.py b/lib/cherrypy/wsgiserver/ssl_builtin.py
deleted file mode 100644
index 3faf703926d..00000000000
--- a/lib/cherrypy/wsgiserver/ssl_builtin.py
+++ /dev/null
@@ -1,111 +0,0 @@
-"""A library for integrating Python's builtin ``ssl`` library with CherryPy.
-
-The ssl module must be importable for SSL functionality.
-
-To use this module, set ``CherryPyWSGIServer.ssl_adapter`` to an instance of
-``BuiltinSSLAdapter``.
-"""
-
-try:
-    import ssl
-except ImportError:
-    ssl = None
-
-try:
-    from _pyio import DEFAULT_BUFFER_SIZE
-except ImportError:
-    try:
-        from io import DEFAULT_BUFFER_SIZE
-    except ImportError:
-        DEFAULT_BUFFER_SIZE = -1
-
-import sys
-
-from cherrypy import wsgiserver
-
-
-class BuiltinSSLAdapter(wsgiserver.SSLAdapter):
-
-    """A wrapper for integrating Python's builtin ssl module with CherryPy."""
-
-    certificate = None
-    """The filename of the server SSL certificate."""
-
-    private_key = None
-    """The filename of the server's private key file."""
-    
-    certificate_chain = None
-    """The filename of the certificate chain file."""
-    
-    """The ssl.SSLContext that will be used to wrap sockets where available
-    (on Python > 2.7.9 / 3.3)
-    """
-    context = None
-
-    def __init__(self, certificate, private_key, certificate_chain=None):
-        if ssl is None:
-            raise ImportError("You must install the ssl module to use HTTPS.")
-        self.certificate = certificate
-        self.private_key = private_key
-        self.certificate_chain = certificate_chain
-        if hasattr(ssl, 'create_default_context'):
-            self.context = ssl.create_default_context(
-                purpose=ssl.Purpose.CLIENT_AUTH,
-                cafile=certificate_chain
-            )
-            self.context.load_cert_chain(certificate, private_key)
-
-    def bind(self, sock):
-        """Wrap and return the given socket."""
-        return sock
-
-    def wrap(self, sock):
-        """Wrap and return the given socket, plus WSGI environ entries."""
-        try:
-            if self.context is not None:
-                s = self.context.wrap_socket(sock,do_handshake_on_connect=True,
-                                             server_side=True)
-            else:
-                s = ssl.wrap_socket(sock, do_handshake_on_connect=True,
-                                    server_side=True, certfile=self.certificate,
-                                    keyfile=self.private_key,
-                                    ssl_version=ssl.PROTOCOL_SSLv23,
-                                    ca_certs=self.certificate_chain)
-        except ssl.SSLError:
-            e = sys.exc_info()[1]
-            if e.errno == ssl.SSL_ERROR_EOF:
-                # This is almost certainly due to the cherrypy engine
-                # 'pinging' the socket to assert it's connectable;
-                # the 'ping' isn't SSL.
-                return None, {}
-            elif e.errno == ssl.SSL_ERROR_SSL:
-                if e.args[1].endswith('http request'):
-                    # The client is speaking HTTP to an HTTPS server.
-                    raise wsgiserver.NoSSLError
-                elif e.args[1].endswith('unknown protocol'):
-                    # The client is speaking some non-HTTP protocol.
-                    # Drop the conn.
-                    return None, {}
-            raise
-        return s, self.get_environ(s)
-
-    # TODO: fill this out more with mod ssl env
-    def get_environ(self, sock):
-        """Create WSGI environ entries to be merged into each request."""
-        cipher = sock.cipher()
-        ssl_environ = {
-            "wsgi.url_scheme": "https",
-            "HTTPS": "on",
-            'SSL_PROTOCOL': cipher[1],
-            'SSL_CIPHER': cipher[0]
-            # SSL_VERSION_INTERFACE 	string 	The mod_ssl program version
-            # SSL_VERSION_LIBRARY 	string 	The OpenSSL program version
-        }
-        return ssl_environ
-
-    if sys.version_info >= (3, 0):
-        def makefile(self, sock, mode='r', bufsize=DEFAULT_BUFFER_SIZE):
-            return wsgiserver.CP_makefile(sock, mode, bufsize)
-    else:
-        def makefile(self, sock, mode='r', bufsize=DEFAULT_BUFFER_SIZE):
-            return wsgiserver.CP_fileobject(sock, mode, bufsize)
diff --git a/lib/cherrypy/wsgiserver/ssl_pyopenssl.py b/lib/cherrypy/wsgiserver/ssl_pyopenssl.py
deleted file mode 100644
index 62ede2652fe..00000000000
--- a/lib/cherrypy/wsgiserver/ssl_pyopenssl.py
+++ /dev/null
@@ -1,253 +0,0 @@
-"""A library for integrating pyOpenSSL with CherryPy.
-
-The OpenSSL module must be importable for SSL functionality.
-You can obtain it from `here `_.
-
-To use this module, set CherryPyWSGIServer.ssl_adapter to an instance of
-SSLAdapter. There are two ways to use SSL:
-
-Method One
-----------
-
- * ``ssl_adapter.context``: an instance of SSL.Context.
-
-If this is not None, it is assumed to be an SSL.Context instance,
-and will be passed to SSL.Connection on bind(). The developer is
-responsible for forming a valid Context object. This approach is
-to be preferred for more flexibility, e.g. if the cert and key are
-streams instead of files, or need decryption, or SSL.SSLv3_METHOD
-is desired instead of the default SSL.SSLv23_METHOD, etc. Consult
-the pyOpenSSL documentation for complete options.
-
-Method Two (shortcut)
----------------------
-
- * ``ssl_adapter.certificate``: the filename of the server SSL certificate.
- * ``ssl_adapter.private_key``: the filename of the server's private key file.
-
-Both are None by default. If ssl_adapter.context is None, but .private_key
-and .certificate are both given and valid, they will be read, and the
-context will be automatically created from them.
-"""
-
-import socket
-import threading
-import time
-
-from cherrypy import wsgiserver
-
-try:
-    from OpenSSL import SSL
-    from OpenSSL import crypto
-except ImportError:
-    SSL = None
-
-
-class SSL_fileobject(wsgiserver.CP_fileobject):
-
-    """SSL file object attached to a socket object."""
-
-    ssl_timeout = 3
-    ssl_retry = .01
-
-    def _safe_call(self, is_reader, call, *args, **kwargs):
-        """Wrap the given call with SSL error-trapping.
-
-        is_reader: if False EOF errors will be raised. If True, EOF errors
-        will return "" (to emulate normal sockets).
-        """
-        start = time.time()
-        while True:
-            try:
-                return call(*args, **kwargs)
-            except SSL.WantReadError:
-                # Sleep and try again. This is dangerous, because it means
-                # the rest of the stack has no way of differentiating
-                # between a "new handshake" error and "client dropped".
-                # Note this isn't an endless loop: there's a timeout below.
-                time.sleep(self.ssl_retry)
-            except SSL.WantWriteError:
-                time.sleep(self.ssl_retry)
-            except SSL.SysCallError as e:
-                if is_reader and e.args == (-1, 'Unexpected EOF'):
-                    return ""
-
-                errnum = e.args[0]
-                if is_reader and errnum in wsgiserver.socket_errors_to_ignore:
-                    return ""
-                raise socket.error(errnum)
-            except SSL.Error as e:
-                if is_reader and e.args == (-1, 'Unexpected EOF'):
-                    return ""
-
-                thirdarg = None
-                try:
-                    thirdarg = e.args[0][0][2]
-                except IndexError:
-                    pass
-
-                if thirdarg == 'http request':
-                    # The client is talking HTTP to an HTTPS server.
-                    raise wsgiserver.NoSSLError()
-
-                raise wsgiserver.FatalSSLAlert(*e.args)
-            except:
-                raise
-
-            if time.time() - start > self.ssl_timeout:
-                raise socket.timeout("timed out")
-
-    def recv(self, size):
-        return self._safe_call(True, super(SSL_fileobject, self).recv, size)
-
-    def sendall(self, *args, **kwargs):
-        return self._safe_call(False, super(SSL_fileobject, self).sendall,
-                               *args, **kwargs)
-
-    def send(self, *args, **kwargs):
-        return self._safe_call(False, super(SSL_fileobject, self).send,
-                               *args, **kwargs)
-
-
-class SSLConnection:
-
-    """A thread-safe wrapper for an SSL.Connection.
-
-    ``*args``: the arguments to create the wrapped ``SSL.Connection(*args)``.
-    """
-
-    def __init__(self, *args):
-        self._ssl_conn = SSL.Connection(*args)
-        self._lock = threading.RLock()
-
-    for f in ('get_context', 'pending', 'send', 'write', 'recv', 'read',
-              'renegotiate', 'bind', 'listen', 'connect', 'accept',
-              'setblocking', 'fileno', 'close', 'get_cipher_list',
-              'getpeername', 'getsockname', 'getsockopt', 'setsockopt',
-              'makefile', 'get_app_data', 'set_app_data', 'state_string',
-              'sock_shutdown', 'get_peer_certificate', 'want_read',
-              'want_write', 'set_connect_state', 'set_accept_state',
-              'connect_ex', 'sendall', 'settimeout', 'gettimeout'):
-        exec("""def %s(self, *args):
-        self._lock.acquire()
-        try:
-            return self._ssl_conn.%s(*args)
-        finally:
-            self._lock.release()
-""" % (f, f))
-
-    def shutdown(self, *args):
-        self._lock.acquire()
-        try:
-            # pyOpenSSL.socket.shutdown takes no args
-            return self._ssl_conn.shutdown()
-        finally:
-            self._lock.release()
-
-
-class pyOpenSSLAdapter(wsgiserver.SSLAdapter):
-
-    """A wrapper for integrating pyOpenSSL with CherryPy."""
-
-    context = None
-    """An instance of SSL.Context."""
-
-    certificate = None
-    """The filename of the server SSL certificate."""
-
-    private_key = None
-    """The filename of the server's private key file."""
-
-    certificate_chain = None
-    """Optional. The filename of CA's intermediate certificate bundle.
-
-    This is needed for cheaper "chained root" SSL certificates, and should be
-    left as None if not required."""
-
-    def __init__(self, certificate, private_key, certificate_chain=None):
-        if SSL is None:
-            raise ImportError("You must install pyOpenSSL to use HTTPS.")
-
-        self.context = None
-        self.certificate = certificate
-        self.private_key = private_key
-        self.certificate_chain = certificate_chain
-        self._environ = None
-
-    def bind(self, sock):
-        """Wrap and return the given socket."""
-        if self.context is None:
-            self.context = self.get_context()
-        conn = SSLConnection(self.context, sock)
-        self._environ = self.get_environ()
-        return conn
-
-    def wrap(self, sock):
-        """Wrap and return the given socket, plus WSGI environ entries."""
-        return sock, self._environ.copy()
-
-    def get_context(self):
-        """Return an SSL.Context from self attributes."""
-        # See http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/442473
-        c = SSL.Context(SSL.SSLv23_METHOD)
-        c.use_privatekey_file(self.private_key)
-        if self.certificate_chain:
-            c.load_verify_locations(self.certificate_chain)
-        c.use_certificate_file(self.certificate)
-        return c
-
-    def get_environ(self):
-        """Return WSGI environ entries to be merged into each request."""
-        ssl_environ = {
-            "HTTPS": "on",
-            # pyOpenSSL doesn't provide access to any of these AFAICT
-            # 'SSL_PROTOCOL': 'SSLv2',
-            # SSL_CIPHER 	string 	The cipher specification name
-            # SSL_VERSION_INTERFACE 	string 	The mod_ssl program version
-            # SSL_VERSION_LIBRARY 	string 	The OpenSSL program version
-        }
-
-        if self.certificate:
-            # Server certificate attributes
-            cert = open(self.certificate, 'rb').read()
-            cert = crypto.load_certificate(crypto.FILETYPE_PEM, cert)
-            ssl_environ.update({
-                'SSL_SERVER_M_VERSION': cert.get_version(),
-                'SSL_SERVER_M_SERIAL': cert.get_serial_number(),
-                # 'SSL_SERVER_V_START':
-                #   Validity of server's certificate (start time),
-                # 'SSL_SERVER_V_END':
-                #   Validity of server's certificate (end time),
-            })
-
-            for prefix, dn in [("I", cert.get_issuer()),
-                               ("S", cert.get_subject())]:
-                # X509Name objects don't seem to have a way to get the
-                # complete DN string. Use str() and slice it instead,
-                # because str(dn) == ""
-                dnstr = str(dn)[18:-2]
-
-                wsgikey = 'SSL_SERVER_%s_DN' % prefix
-                ssl_environ[wsgikey] = dnstr
-
-                # The DN should be of the form: /k1=v1/k2=v2, but we must allow
-                # for any value to contain slashes itself (in a URL).
-                while dnstr:
-                    pos = dnstr.rfind("=")
-                    dnstr, value = dnstr[:pos], dnstr[pos + 1:]
-                    pos = dnstr.rfind("/")
-                    dnstr, key = dnstr[:pos], dnstr[pos + 1:]
-                    if key and value:
-                        wsgikey = 'SSL_SERVER_%s_DN_%s' % (prefix, key)
-                        ssl_environ[wsgikey] = value
-
-        return ssl_environ
-
-    def makefile(self, sock, mode='r', bufsize=-1):
-        if SSL and isinstance(sock, SSL.ConnectionType):
-            timeout = sock.gettimeout()
-            f = SSL_fileobject(sock, mode, bufsize)
-            f.ssl_timeout = timeout
-            return f
-        else:
-            return wsgiserver.CP_fileobject(sock, mode, bufsize)
diff --git a/lib/cherrypy/wsgiserver/wsgiserver2.py b/lib/cherrypy/wsgiserver/wsgiserver2.py
deleted file mode 100644
index e6ad8e81a1d..00000000000
--- a/lib/cherrypy/wsgiserver/wsgiserver2.py
+++ /dev/null
@@ -1,2469 +0,0 @@
-"""A high-speed, production ready, thread pooled, generic HTTP server.
-
-Simplest example on how to use this module directly
-(without using CherryPy's application machinery)::
-
-    from cherrypy import wsgiserver
-
-    def my_crazy_app(environ, start_response):
-        status = '200 OK'
-        response_headers = [('Content-type','text/plain')]
-        start_response(status, response_headers)
-        return ['Hello world!']
-
-    server = wsgiserver.CherryPyWSGIServer(
-                ('0.0.0.0', 8070), my_crazy_app,
-                server_name='www.cherrypy.example')
-    server.start()
-
-The CherryPy WSGI server can serve as many WSGI applications
-as you want in one instance by using a WSGIPathInfoDispatcher::
-
-    d = WSGIPathInfoDispatcher({'/': my_crazy_app, '/blog': my_blog_app})
-    server = wsgiserver.CherryPyWSGIServer(('0.0.0.0', 80), d)
-
-Want SSL support? Just set server.ssl_adapter to an SSLAdapter instance.
-
-This won't call the CherryPy engine (application side) at all, only the
-HTTP server, which is independent from the rest of CherryPy. Don't
-let the name "CherryPyWSGIServer" throw you; the name merely reflects
-its origin, not its coupling.
-
-For those of you wanting to understand internals of this module, here's the
-basic call flow. The server's listening thread runs a very tight loop,
-sticking incoming connections onto a Queue::
-
-    server = CherryPyWSGIServer(...)
-    server.start()
-    while True:
-        tick()
-        # This blocks until a request comes in:
-        child = socket.accept()
-        conn = HTTPConnection(child, ...)
-        server.requests.put(conn)
-
-Worker threads are kept in a pool and poll the Queue, popping off and then
-handling each connection in turn. Each connection can consist of an arbitrary
-number of requests and their responses, so we run a nested loop::
-
-    while True:
-        conn = server.requests.get()
-        conn.communicate()
-        ->  while True:
-                req = HTTPRequest(...)
-                req.parse_request()
-                ->  # Read the Request-Line, e.g. "GET /page HTTP/1.1"
-                    req.rfile.readline()
-                    read_headers(req.rfile, req.inheaders)
-                req.respond()
-                ->  response = app(...)
-                    try:
-                        for chunk in response:
-                            if chunk:
-                                req.write(chunk)
-                    finally:
-                        if hasattr(response, "close"):
-                            response.close()
-                if req.close_connection:
-                    return
-"""
-
-__all__ = ['HTTPRequest', 'HTTPConnection', 'HTTPServer',
-           'SizeCheckWrapper', 'KnownLengthRFile', 'ChunkedRFile',
-           'CP_fileobject',
-           'MaxSizeExceeded', 'NoSSLError', 'FatalSSLAlert',
-           'WorkerThread', 'ThreadPool', 'SSLAdapter',
-           'CherryPyWSGIServer',
-           'Gateway', 'WSGIGateway', 'WSGIGateway_10', 'WSGIGateway_u0',
-           'WSGIPathInfoDispatcher', 'get_ssl_adapter_class',
-           'socket_errors_to_ignore']
-
-import os
-try:
-    import queue
-except:
-    import Queue as queue
-import re
-import email.utils
-import socket
-import sys
-import threading
-import time
-import traceback as traceback_
-import operator
-from urllib import unquote
-from urlparse import urlparse
-import warnings
-import errno
-import logging
-try:
-    # prefer slower Python-based io module
-    import _pyio as io
-except ImportError:
-    # Python 2.6
-    import io
-
-try:
-    import pkg_resources
-except ImportError:
-    pass
-
-if 'win' in sys.platform and hasattr(socket, "AF_INET6"):
-    if not hasattr(socket, 'IPPROTO_IPV6'):
-        socket.IPPROTO_IPV6 = 41
-    if not hasattr(socket, 'IPV6_V6ONLY'):
-        socket.IPV6_V6ONLY = 27
-
-
-DEFAULT_BUFFER_SIZE = io.DEFAULT_BUFFER_SIZE
-
-
-try:
-    cp_version = pkg_resources.require('cherrypy')[0].version
-except Exception:
-    cp_version = 'unknown'
-
-
-class FauxSocket(object):
-
-    """Faux socket with the minimal interface required by pypy"""
-
-    def _reuse(self):
-        pass
-
-_fileobject_uses_str_type = isinstance(
-    socket._fileobject(FauxSocket())._rbuf, basestring)
-del FauxSocket  # this class is not longer required for anything.
-
-
-if sys.version_info >= (3, 0):
-    unicodestr = str
-    basestring = (bytes, str)
-
-    def ntob(n, encoding='ISO-8859-1'):
-        """Return the given native string as a byte string in the given
-        encoding.
-        """
-        # In Python 3, the native string type is unicode
-        return n.encode(encoding)
-else:
-    unicodestr = unicode
-    basestring = basestring
-
-    def ntob(n, encoding='ISO-8859-1'):
-        """Return the given native string as a byte string in the given
-        encoding.
-        """
-        # In Python 2, the native string type is bytes. Assume it's already
-        # in the given encoding, which for ISO-8859-1 is almost always what
-        # was intended.
-        return n
-
-LF = ntob('\n')
-CRLF = ntob('\r\n')
-TAB = ntob('\t')
-SPACE = ntob(' ')
-COLON = ntob(':')
-SEMICOLON = ntob(';')
-EMPTY = ntob('')
-NUMBER_SIGN = ntob('#')
-QUESTION_MARK = ntob('?')
-ASTERISK = ntob('*')
-FORWARD_SLASH = ntob('/')
-quoted_slash = re.compile(ntob("(?i)%2F"))
-
-
-def plat_specific_errors(*errnames):
-    """Return error numbers for all errors in errnames on this platform.
-
-    The 'errno' module contains different global constants depending on
-    the specific platform (OS). This function will return the list of
-    numeric values for a given list of potential names.
-    """
-    errno_names = dir(errno)
-    nums = [getattr(errno, k) for k in errnames if k in errno_names]
-    # de-dupe the list
-    return list(dict.fromkeys(nums).keys())
-
-socket_error_eintr = plat_specific_errors("EINTR", "WSAEINTR")
-
-socket_errors_to_ignore = plat_specific_errors(
-    "EPIPE",
-    "EBADF", "WSAEBADF",
-    "ENOTSOCK", "WSAENOTSOCK",
-    "ETIMEDOUT", "WSAETIMEDOUT",
-    "ECONNREFUSED", "WSAECONNREFUSED",
-    "ECONNRESET", "WSAECONNRESET",
-    "ECONNABORTED", "WSAECONNABORTED",
-    "ENETRESET", "WSAENETRESET",
-    "EHOSTDOWN", "EHOSTUNREACH",
-)
-socket_errors_to_ignore.append("timed out")
-socket_errors_to_ignore.append("The read operation timed out")
-if sys.platform == 'darwin':
-    socket_errors_to_ignore.append(plat_specific_errors("EPROTOTYPE"))
-
-socket_errors_nonblocking = plat_specific_errors(
-    'EAGAIN', 'EWOULDBLOCK', 'WSAEWOULDBLOCK')
-
-comma_separated_headers = [
-    ntob(h) for h in
-    ['Accept', 'Accept-Charset', 'Accept-Encoding',
-     'Accept-Language', 'Accept-Ranges', 'Allow', 'Cache-Control',
-     'Connection', 'Content-Encoding', 'Content-Language', 'Expect',
-     'If-Match', 'If-None-Match', 'Pragma', 'Proxy-Authenticate', 'TE',
-     'Trailer', 'Transfer-Encoding', 'Upgrade', 'Vary', 'Via', 'Warning',
-     'WWW-Authenticate']
-]
-
-
-if not hasattr(logging, 'statistics'):
-    logging.statistics = {}
-
-
-def read_headers(rfile, hdict=None):
-    """Read headers from the given stream into the given header dict.
-
-    If hdict is None, a new header dict is created. Returns the populated
-    header dict.
-
-    Headers which are repeated are folded together using a comma if their
-    specification so dictates.
-
-    This function raises ValueError when the read bytes violate the HTTP spec.
-    You should probably return "400 Bad Request" if this happens.
-    """
-    if hdict is None:
-        hdict = {}
-
-    while True:
-        line = rfile.readline()
-        if not line:
-            # No more data--illegal end of headers
-            raise ValueError("Illegal end of headers.")
-
-        if line == CRLF:
-            # Normal end of headers
-            break
-        if not line.endswith(CRLF):
-            raise ValueError("HTTP requires CRLF terminators")
-
-        if line[0] in (SPACE, TAB):
-            # It's a continuation line.
-            v = line.strip()
-        else:
-            try:
-                k, v = line.split(COLON, 1)
-            except ValueError:
-                raise ValueError("Illegal header line.")
-            # TODO: what about TE and WWW-Authenticate?
-            k = k.strip().title()
-            v = v.strip()
-            hname = k
-
-        if k in comma_separated_headers:
-            existing = hdict.get(hname)
-            if existing:
-                v = ", ".join((existing, v))
-        hdict[hname] = v
-
-    return hdict
-
-
-class MaxSizeExceeded(Exception):
-    pass
-
-
-class SizeCheckWrapper(object):
-
-    """Wraps a file-like object, raising MaxSizeExceeded if too large."""
-
-    def __init__(self, rfile, maxlen):
-        self.rfile = rfile
-        self.maxlen = maxlen
-        self.bytes_read = 0
-
-    def _check_length(self):
-        if self.maxlen and self.bytes_read > self.maxlen:
-            raise MaxSizeExceeded()
-
-    def read(self, size=None):
-        data = self.rfile.read(size)
-        self.bytes_read += len(data)
-        self._check_length()
-        return data
-
-    def readline(self, size=None):
-        if size is not None:
-            data = self.rfile.readline(size)
-            self.bytes_read += len(data)
-            self._check_length()
-            return data
-
-        # User didn't specify a size ...
-        # We read the line in chunks to make sure it's not a 100MB line !
-        res = []
-        while True:
-            data = self.rfile.readline(256)
-            self.bytes_read += len(data)
-            self._check_length()
-            res.append(data)
-            # See https://github.com/cherrypy/cherrypy/issues/421
-            if len(data) < 256 or data[-1:] == LF:
-                return EMPTY.join(res)
-
-    def readlines(self, sizehint=0):
-        # Shamelessly stolen from StringIO
-        total = 0
-        lines = []
-        line = self.readline()
-        while line:
-            lines.append(line)
-            total += len(line)
-            if 0 < sizehint <= total:
-                break
-            line = self.readline()
-        return lines
-
-    def close(self):
-        self.rfile.close()
-
-    def __iter__(self):
-        return self
-
-    def __next__(self):
-        data = next(self.rfile)
-        self.bytes_read += len(data)
-        self._check_length()
-        return data
-
-    def next(self):
-        data = self.rfile.next()
-        self.bytes_read += len(data)
-        self._check_length()
-        return data
-
-
-class KnownLengthRFile(object):
-
-    """Wraps a file-like object, returning an empty string when exhausted."""
-
-    def __init__(self, rfile, content_length):
-        self.rfile = rfile
-        self.remaining = content_length
-
-    def read(self, size=None):
-        if self.remaining == 0:
-            return ''
-        if size is None:
-            size = self.remaining
-        else:
-            size = min(size, self.remaining)
-
-        data = self.rfile.read(size)
-        self.remaining -= len(data)
-        return data
-
-    def readline(self, size=None):
-        if self.remaining == 0:
-            return ''
-        if size is None:
-            size = self.remaining
-        else:
-            size = min(size, self.remaining)
-
-        data = self.rfile.readline(size)
-        self.remaining -= len(data)
-        return data
-
-    def readlines(self, sizehint=0):
-        # Shamelessly stolen from StringIO
-        total = 0
-        lines = []
-        line = self.readline(sizehint)
-        while line:
-            lines.append(line)
-            total += len(line)
-            if 0 < sizehint <= total:
-                break
-            line = self.readline(sizehint)
-        return lines
-
-    def close(self):
-        self.rfile.close()
-
-    def __iter__(self):
-        return self
-
-    def __next__(self):
-        data = next(self.rfile)
-        self.remaining -= len(data)
-        return data
-
-
-class ChunkedRFile(object):
-
-    """Wraps a file-like object, returning an empty string when exhausted.
-
-    This class is intended to provide a conforming wsgi.input value for
-    request entities that have been encoded with the 'chunked' transfer
-    encoding.
-    """
-
-    def __init__(self, rfile, maxlen, bufsize=8192):
-        self.rfile = rfile
-        self.maxlen = maxlen
-        self.bytes_read = 0
-        self.buffer = EMPTY
-        self.bufsize = bufsize
-        self.closed = False
-
-    def _fetch(self):
-        if self.closed:
-            return
-
-        line = self.rfile.readline()
-        self.bytes_read += len(line)
-
-        if self.maxlen and self.bytes_read > self.maxlen:
-            raise MaxSizeExceeded("Request Entity Too Large", self.maxlen)
-
-        line = line.strip().split(SEMICOLON, 1)
-
-        try:
-            chunk_size = line.pop(0)
-            chunk_size = int(chunk_size, 16)
-        except ValueError:
-            raise ValueError("Bad chunked transfer size: " + repr(chunk_size))
-
-        if chunk_size <= 0:
-            self.closed = True
-            return
-
-##            if line: chunk_extension = line[0]
-
-        if self.maxlen and self.bytes_read + chunk_size > self.maxlen:
-            raise IOError("Request Entity Too Large")
-
-        chunk = self.rfile.read(chunk_size)
-        self.bytes_read += len(chunk)
-        self.buffer += chunk
-
-        crlf = self.rfile.read(2)
-        if crlf != CRLF:
-            raise ValueError(
-                "Bad chunked transfer coding (expected '\\r\\n', "
-                "got " + repr(crlf) + ")")
-
-    def read(self, size=None):
-        data = EMPTY
-        while True:
-            if size and len(data) >= size:
-                return data
-
-            if not self.buffer:
-                self._fetch()
-                if not self.buffer:
-                    # EOF
-                    return data
-
-            if size:
-                remaining = size - len(data)
-                data += self.buffer[:remaining]
-                self.buffer = self.buffer[remaining:]
-            else:
-                data += self.buffer
-
-    def readline(self, size=None):
-        data = EMPTY
-        while True:
-            if size and len(data) >= size:
-                return data
-
-            if not self.buffer:
-                self._fetch()
-                if not self.buffer:
-                    # EOF
-                    return data
-
-            newline_pos = self.buffer.find(LF)
-            if size:
-                if newline_pos == -1:
-                    remaining = size - len(data)
-                    data += self.buffer[:remaining]
-                    self.buffer = self.buffer[remaining:]
-                else:
-                    remaining = min(size - len(data), newline_pos)
-                    data += self.buffer[:remaining]
-                    self.buffer = self.buffer[remaining:]
-            else:
-                if newline_pos == -1:
-                    data += self.buffer
-                else:
-                    data += self.buffer[:newline_pos]
-                    self.buffer = self.buffer[newline_pos:]
-
-    def readlines(self, sizehint=0):
-        # Shamelessly stolen from StringIO
-        total = 0
-        lines = []
-        line = self.readline(sizehint)
-        while line:
-            lines.append(line)
-            total += len(line)
-            if 0 < sizehint <= total:
-                break
-            line = self.readline(sizehint)
-        return lines
-
-    def read_trailer_lines(self):
-        if not self.closed:
-            raise ValueError(
-                "Cannot read trailers until the request body has been read.")
-
-        while True:
-            line = self.rfile.readline()
-            if not line:
-                # No more data--illegal end of headers
-                raise ValueError("Illegal end of headers.")
-
-            self.bytes_read += len(line)
-            if self.maxlen and self.bytes_read > self.maxlen:
-                raise IOError("Request Entity Too Large")
-
-            if line == CRLF:
-                # Normal end of headers
-                break
-            if not line.endswith(CRLF):
-                raise ValueError("HTTP requires CRLF terminators")
-
-            yield line
-
-    def close(self):
-        self.rfile.close()
-
-    def __iter__(self):
-        # Shamelessly stolen from StringIO
-        total = 0
-        line = self.readline(sizehint)
-        while line:
-            yield line
-            total += len(line)
-            if 0 < sizehint <= total:
-                break
-            line = self.readline(sizehint)
-
-
-class HTTPRequest(object):
-
-    """An HTTP Request (and response).
-
-    A single HTTP connection may consist of multiple request/response pairs.
-    """
-
-    server = None
-    """The HTTPServer object which is receiving this request."""
-
-    conn = None
-    """The HTTPConnection object on which this request connected."""
-
-    inheaders = {}
-    """A dict of request headers."""
-
-    outheaders = []
-    """A list of header tuples to write in the response."""
-
-    ready = False
-    """When True, the request has been parsed and is ready to begin generating
-    the response. When False, signals the calling Connection that the response
-    should not be generated and the connection should close."""
-
-    close_connection = False
-    """Signals the calling Connection that the request should close. This does
-    not imply an error! The client and/or server may each request that the
-    connection be closed."""
-
-    chunked_write = False
-    """If True, output will be encoded with the "chunked" transfer-coding.
-
-    This value is set automatically inside send_headers."""
-
-    def __init__(self, server, conn):
-        self.server = server
-        self.conn = conn
-
-        self.ready = False
-        self.started_request = False
-        self.scheme = ntob("http")
-        if self.server.ssl_adapter is not None:
-            self.scheme = ntob("https")
-        # Use the lowest-common protocol in case read_request_line errors.
-        self.response_protocol = 'HTTP/1.0'
-        self.inheaders = {}
-
-        self.status = ""
-        self.outheaders = []
-        self.sent_headers = False
-        self.close_connection = self.__class__.close_connection
-        self.chunked_read = False
-        self.chunked_write = self.__class__.chunked_write
-
-    def parse_request(self):
-        """Parse the next HTTP request start-line and message-headers."""
-        self.rfile = SizeCheckWrapper(self.conn.rfile,
-                                      self.server.max_request_header_size)
-        try:
-            success = self.read_request_line()
-        except MaxSizeExceeded:
-            self.simple_response(
-                "414 Request-URI Too Long",
-                "The Request-URI sent with the request exceeds the maximum "
-                "allowed bytes.")
-            return
-        else:
-            if not success:
-                return
-
-        try:
-            success = self.read_request_headers()
-        except MaxSizeExceeded:
-            self.simple_response(
-                "413 Request Entity Too Large",
-                "The headers sent with the request exceed the maximum "
-                "allowed bytes.")
-            return
-        else:
-            if not success:
-                return
-
-        self.ready = True
-
-    def read_request_line(self):
-        # HTTP/1.1 connections are persistent by default. If a client
-        # requests a page, then idles (leaves the connection open),
-        # then rfile.readline() will raise socket.error("timed out").
-        # Note that it does this based on the value given to settimeout(),
-        # and doesn't need the client to request or acknowledge the close
-        # (although your TCP stack might suffer for it: cf Apache's history
-        # with FIN_WAIT_2).
-        request_line = self.rfile.readline()
-
-        # Set started_request to True so communicate() knows to send 408
-        # from here on out.
-        self.started_request = True
-        if not request_line:
-            return False
-
-        if request_line == CRLF:
-            # RFC 2616 sec 4.1: "...if the server is reading the protocol
-            # stream at the beginning of a message and receives a CRLF
-            # first, it should ignore the CRLF."
-            # But only ignore one leading line! else we enable a DoS.
-            request_line = self.rfile.readline()
-            if not request_line:
-                return False
-
-        if not request_line.endswith(CRLF):
-            self.simple_response(
-                "400 Bad Request", "HTTP requires CRLF terminators")
-            return False
-
-        try:
-            method, uri, req_protocol = request_line.strip().split(SPACE, 2)
-            rp = int(req_protocol[5]), int(req_protocol[7])
-        except (ValueError, IndexError):
-            self.simple_response("400 Bad Request", "Malformed Request-Line")
-            return False
-
-        self.uri = uri
-        self.method = method
-
-        # uri may be an abs_path (including "http://host.domain.tld");
-        scheme, authority, path = self.parse_request_uri(uri)
-        if path is None:
-            self.simple_response("400 Bad Request",
-                                 "Invalid path in Request-URI.")
-            return False
-        if NUMBER_SIGN in path:
-            self.simple_response("400 Bad Request",
-                                 "Illegal #fragment in Request-URI.")
-            return False
-
-        if scheme:
-            self.scheme = scheme
-
-        qs = EMPTY
-        if QUESTION_MARK in path:
-            path, qs = path.split(QUESTION_MARK, 1)
-
-        # Unquote the path+params (e.g. "/this%20path" -> "/this path").
-        # http://www.w3.org/Protocols/rfc2616/rfc2616-sec5.html#sec5.1.2
-        #
-        # But note that "...a URI must be separated into its components
-        # before the escaped characters within those components can be
-        # safely decoded." http://www.ietf.org/rfc/rfc2396.txt, sec 2.4.2
-        # Therefore, "/this%2Fpath" becomes "/this%2Fpath", not "/this/path".
-        try:
-            atoms = [unquote(x) for x in quoted_slash.split(path)]
-        except ValueError:
-            ex = sys.exc_info()[1]
-            self.simple_response("400 Bad Request", ex.args[0])
-            return False
-        path = "%2F".join(atoms)
-        self.path = path
-
-        # Note that, like wsgiref and most other HTTP servers,
-        # we "% HEX HEX"-unquote the path but not the query string.
-        self.qs = qs
-
-        # Compare request and server HTTP protocol versions, in case our
-        # server does not support the requested protocol. Limit our output
-        # to min(req, server). We want the following output:
-        #     request    server     actual written   supported response
-        #     protocol   protocol  response protocol    feature set
-        # a     1.0        1.0           1.0                1.0
-        # b     1.0        1.1           1.1                1.0
-        # c     1.1        1.0           1.0                1.0
-        # d     1.1        1.1           1.1                1.1
-        # Notice that, in (b), the response will be "HTTP/1.1" even though
-        # the client only understands 1.0. RFC 2616 10.5.6 says we should
-        # only return 505 if the _major_ version is different.
-        sp = int(self.server.protocol[5]), int(self.server.protocol[7])
-
-        if sp[0] != rp[0]:
-            self.simple_response("505 HTTP Version Not Supported")
-            return False
-
-        self.request_protocol = req_protocol
-        self.response_protocol = "HTTP/%s.%s" % min(rp, sp)
-
-        return True
-
-    def read_request_headers(self):
-        """Read self.rfile into self.inheaders. Return success."""
-
-        # then all the http headers
-        try:
-            read_headers(self.rfile, self.inheaders)
-        except ValueError:
-            ex = sys.exc_info()[1]
-            self.simple_response("400 Bad Request", ex.args[0])
-            return False
-
-        mrbs = self.server.max_request_body_size
-        if mrbs and int(self.inheaders.get("Content-Length", 0)) > mrbs:
-            self.simple_response(
-                "413 Request Entity Too Large",
-                "The entity sent with the request exceeds the maximum "
-                "allowed bytes.")
-            return False
-
-        # Persistent connection support
-        if self.response_protocol == "HTTP/1.1":
-            # Both server and client are HTTP/1.1
-            if self.inheaders.get("Connection", "") == "close":
-                self.close_connection = True
-        else:
-            # Either the server or client (or both) are HTTP/1.0
-            if self.inheaders.get("Connection", "") != "Keep-Alive":
-                self.close_connection = True
-
-        # Transfer-Encoding support
-        te = None
-        if self.response_protocol == "HTTP/1.1":
-            te = self.inheaders.get("Transfer-Encoding")
-            if te:
-                te = [x.strip().lower() for x in te.split(",") if x.strip()]
-
-        self.chunked_read = False
-
-        if te:
-            for enc in te:
-                if enc == "chunked":
-                    self.chunked_read = True
-                else:
-                    # Note that, even if we see "chunked", we must reject
-                    # if there is an extension we don't recognize.
-                    self.simple_response("501 Unimplemented")
-                    self.close_connection = True
-                    return False
-
-        # From PEP 333:
-        # "Servers and gateways that implement HTTP 1.1 must provide
-        # transparent support for HTTP 1.1's "expect/continue" mechanism.
-        # This may be done in any of several ways:
-        #   1. Respond to requests containing an Expect: 100-continue request
-        #      with an immediate "100 Continue" response, and proceed normally.
-        #   2. Proceed with the request normally, but provide the application
-        #      with a wsgi.input stream that will send the "100 Continue"
-        #      response if/when the application first attempts to read from
-        #      the input stream. The read request must then remain blocked
-        #      until the client responds.
-        #   3. Wait until the client decides that the server does not support
-        #      expect/continue, and sends the request body on its own.
-        #      (This is suboptimal, and is not recommended.)
-        #
-        # We used to do 3, but are now doing 1. Maybe we'll do 2 someday,
-        # but it seems like it would be a big slowdown for such a rare case.
-        if self.inheaders.get("Expect", "") == "100-continue":
-            # Don't use simple_response here, because it emits headers
-            # we don't want. See
-            # https://github.com/cherrypy/cherrypy/issues/951
-            msg = self.server.protocol + " 100 Continue\r\n\r\n"
-            try:
-                self.conn.wfile.sendall(msg)
-            except socket.error:
-                x = sys.exc_info()[1]
-                if x.args[0] not in socket_errors_to_ignore:
-                    raise
-        return True
-
-    def parse_request_uri(self, uri):
-        """Parse a Request-URI into (scheme, authority, path).
-
-        Note that Request-URI's must be one of::
-
-            Request-URI    = "*" | absoluteURI | abs_path | authority
-
-        Therefore, a Request-URI which starts with a double forward-slash
-        cannot be a "net_path"::
-
-            net_path      = "//" authority [ abs_path ]
-
-        Instead, it must be interpreted as an "abs_path" with an empty first
-        path segment::
-
-            abs_path      = "/"  path_segments
-            path_segments = segment *( "/" segment )
-            segment       = *pchar *( ";" param )
-            param         = *pchar
-        """
-        if uri == ASTERISK:
-            return None, None, uri
-
-        scheme, authority, path, params, query, fragment = urlparse(uri)
-        if scheme and QUESTION_MARK not in scheme:
-            # An absoluteURI.
-            # If there's a scheme (and it must be http or https), then:
-            # http_URL = "http:" "//" host [ ":" port ] [ abs_path [ "?" query
-            # ]]
-            return scheme, authority, path
-
-        if uri.startswith(FORWARD_SLASH):
-            # An abs_path.
-            return None, None, uri
-        else:
-            # An authority.
-            return None, uri, None
-
-    def respond(self):
-        """Call the gateway and write its iterable output."""
-        mrbs = self.server.max_request_body_size
-        if self.chunked_read:
-            self.rfile = ChunkedRFile(self.conn.rfile, mrbs)
-        else:
-            cl = int(self.inheaders.get("Content-Length", 0))
-            if mrbs and mrbs < cl:
-                if not self.sent_headers:
-                    self.simple_response(
-                        "413 Request Entity Too Large",
-                        "The entity sent with the request exceeds the maximum "
-                        "allowed bytes.")
-                return
-            self.rfile = KnownLengthRFile(self.conn.rfile, cl)
-
-        self.server.gateway(self).respond()
-
-        if (self.ready and not self.sent_headers):
-            self.sent_headers = True
-            self.send_headers()
-        if self.chunked_write:
-            self.conn.wfile.sendall("0\r\n\r\n")
-
-    def simple_response(self, status, msg=""):
-        """Write a simple response back to the client."""
-        status = str(status)
-        buf = [self.server.protocol + SPACE +
-               status + CRLF,
-               "Content-Length: %s\r\n" % len(msg),
-               "Content-Type: text/plain\r\n"]
-
-        if status[:3] in ("413", "414"):
-            # Request Entity Too Large / Request-URI Too Long
-            self.close_connection = True
-            if self.response_protocol == 'HTTP/1.1':
-                # This will not be true for 414, since read_request_line
-                # usually raises 414 before reading the whole line, and we
-                # therefore cannot know the proper response_protocol.
-                buf.append("Connection: close\r\n")
-            else:
-                # HTTP/1.0 had no 413/414 status nor Connection header.
-                # Emit 400 instead and trust the message body is enough.
-                status = "400 Bad Request"
-
-        buf.append(CRLF)
-        if msg:
-            if isinstance(msg, unicodestr):
-                msg = msg.encode("ISO-8859-1")
-            buf.append(msg)
-
-        try:
-            self.conn.wfile.sendall("".join(buf))
-        except socket.error:
-            x = sys.exc_info()[1]
-            if x.args[0] not in socket_errors_to_ignore:
-                raise
-
-    def write(self, chunk):
-        """Write unbuffered data to the client."""
-        if self.chunked_write and chunk:
-            buf = [hex(len(chunk))[2:], CRLF, chunk, CRLF]
-            self.conn.wfile.sendall(EMPTY.join(buf))
-        else:
-            self.conn.wfile.sendall(chunk)
-
-    def send_headers(self):
-        """Assert, process, and send the HTTP response message-headers.
-
-        You must set self.status, and self.outheaders before calling this.
-        """
-        hkeys = [key.lower() for key, value in self.outheaders]
-        status = int(self.status[:3])
-
-        if status == 413:
-            # Request Entity Too Large. Close conn to avoid garbage.
-            self.close_connection = True
-        elif "content-length" not in hkeys:
-            # "All 1xx (informational), 204 (no content),
-            # and 304 (not modified) responses MUST NOT
-            # include a message-body." So no point chunking.
-            if status < 200 or status in (204, 205, 304):
-                pass
-            else:
-                if (self.response_protocol == 'HTTP/1.1'
-                        and self.method != 'HEAD'):
-                    # Use the chunked transfer-coding
-                    self.chunked_write = True
-                    self.outheaders.append(("Transfer-Encoding", "chunked"))
-                else:
-                    # Closing the conn is the only way to determine len.
-                    self.close_connection = True
-
-        if "connection" not in hkeys:
-            if self.response_protocol == 'HTTP/1.1':
-                # Both server and client are HTTP/1.1 or better
-                if self.close_connection:
-                    self.outheaders.append(("Connection", "close"))
-            else:
-                # Server and/or client are HTTP/1.0
-                if not self.close_connection:
-                    self.outheaders.append(("Connection", "Keep-Alive"))
-
-        if (not self.close_connection) and (not self.chunked_read):
-            # Read any remaining request body data on the socket.
-            # "If an origin server receives a request that does not include an
-            # Expect request-header field with the "100-continue" expectation,
-            # the request includes a request body, and the server responds
-            # with a final status code before reading the entire request body
-            # from the transport connection, then the server SHOULD NOT close
-            # the transport connection until it has read the entire request,
-            # or until the client closes the connection. Otherwise, the client
-            # might not reliably receive the response message. However, this
-            # requirement is not be construed as preventing a server from
-            # defending itself against denial-of-service attacks, or from
-            # badly broken client implementations."
-            remaining = getattr(self.rfile, 'remaining', 0)
-            if remaining > 0:
-                self.rfile.read(remaining)
-
-        if "date" not in hkeys:
-            self.outheaders.append(("Date", email.utils.formatdate()))
-
-        if "server" not in hkeys:
-            self.outheaders.append(("Server", self.server.server_name))
-
-        buf = [self.server.protocol + SPACE + self.status + CRLF]
-        for k, v in self.outheaders:
-            buf.append(k + COLON + SPACE + v + CRLF)
-        buf.append(CRLF)
-        self.conn.wfile.sendall(EMPTY.join(buf))
-
-
-class NoSSLError(Exception):
-
-    """Exception raised when a client speaks HTTP to an HTTPS socket."""
-    pass
-
-
-class FatalSSLAlert(Exception):
-
-    """Exception raised when the SSL implementation signals a fatal alert."""
-    pass
-
-
-class CP_fileobject(socket._fileobject):
-
-    """Faux file object attached to a socket object."""
-
-    def __init__(self, *args, **kwargs):
-        self.bytes_read = 0
-        self.bytes_written = 0
-        socket._fileobject.__init__(self, *args, **kwargs)
-
-    def sendall(self, data):
-        """Sendall for non-blocking sockets."""
-        while data:
-            try:
-                bytes_sent = self.send(data)
-                data = data[bytes_sent:]
-            except socket.error, e:
-                if e.args[0] not in socket_errors_nonblocking:
-                    raise
-
-    def send(self, data):
-        bytes_sent = self._sock.send(data)
-        self.bytes_written += bytes_sent
-        return bytes_sent
-
-    def flush(self):
-        if self._wbuf:
-            buffer = "".join(self._wbuf)
-            self._wbuf = []
-            self.sendall(buffer)
-
-    def recv(self, size):
-        while True:
-            try:
-                data = self._sock.recv(size)
-                self.bytes_read += len(data)
-                return data
-            except socket.error, e:
-                if (e.args[0] not in socket_errors_nonblocking
-                        and e.args[0] not in socket_error_eintr):
-                    raise
-
-    if not _fileobject_uses_str_type:
-        def read(self, size=-1):
-            # Use max, disallow tiny reads in a loop as they are very
-            # inefficient.
-            # We never leave read() with any leftover data from a new recv()
-            # call in our internal buffer.
-            rbufsize = max(self._rbufsize, self.default_bufsize)
-            # Our use of StringIO rather than lists of string objects returned
-            # by recv() minimizes memory usage and fragmentation that occurs
-            # when rbufsize is large compared to the typical return value of
-            # recv().
-            buf = self._rbuf
-            buf.seek(0, 2)  # seek end
-            if size < 0:
-                # Read until EOF
-                # reset _rbuf.  we consume it via buf.
-                self._rbuf = io.BytesIO()
-                while True:
-                    data = self.recv(rbufsize)
-                    if not data:
-                        break
-                    buf.write(data)
-                return buf.getvalue()
-            else:
-                # Read until size bytes or EOF seen, whichever comes first
-                buf_len = buf.tell()
-                if buf_len >= size:
-                    # Already have size bytes in our buffer?  Extract and
-                    # return.
-                    buf.seek(0)
-                    rv = buf.read(size)
-                    self._rbuf = io.BytesIO()
-                    self._rbuf.write(buf.read())
-                    return rv
-
-                # reset _rbuf.  we consume it via buf.
-                self._rbuf = io.BytesIO()
-                while True:
-                    left = size - buf_len
-                    # recv() will malloc the amount of memory given as its
-                    # parameter even though it often returns much less data
-                    # than that.  The returned data string is short lived
-                    # as we copy it into a StringIO and free it.  This avoids
-                    # fragmentation issues on many platforms.
-                    data = self.recv(left)
-                    if not data:
-                        break
-                    n = len(data)
-                    if n == size and not buf_len:
-                        # Shortcut.  Avoid buffer data copies when:
-                        # - We have no data in our buffer.
-                        # AND
-                        # - Our call to recv returned exactly the
-                        #   number of bytes we were asked to read.
-                        return data
-                    if n == left:
-                        buf.write(data)
-                        del data  # explicit free
-                        break
-                    assert n <= left, "recv(%d) returned %d bytes" % (left, n)
-                    buf.write(data)
-                    buf_len += n
-                    del data  # explicit free
-                    #assert buf_len == buf.tell()
-                return buf.getvalue()
-
-        def readline(self, size=-1):
-            buf = self._rbuf
-            buf.seek(0, 2)  # seek end
-            if buf.tell() > 0:
-                # check if we already have it in our buffer
-                buf.seek(0)
-                bline = buf.readline(size)
-                if bline.endswith('\n') or len(bline) == size:
-                    self._rbuf = io.BytesIO()
-                    self._rbuf.write(buf.read())
-                    return bline
-                del bline
-            if size < 0:
-                # Read until \n or EOF, whichever comes first
-                if self._rbufsize <= 1:
-                    # Speed up unbuffered case
-                    buf.seek(0)
-                    buffers = [buf.read()]
-                    # reset _rbuf.  we consume it via buf.
-                    self._rbuf = io.BytesIO()
-                    data = None
-                    recv = self.recv
-                    while data != "\n":
-                        data = recv(1)
-                        if not data:
-                            break
-                        buffers.append(data)
-                    return "".join(buffers)
-
-                buf.seek(0, 2)  # seek end
-                # reset _rbuf.  we consume it via buf.
-                self._rbuf = io.BytesIO()
-                while True:
-                    data = self.recv(self._rbufsize)
-                    if not data:
-                        break
-                    nl = data.find('\n')
-                    if nl >= 0:
-                        nl += 1
-                        buf.write(data[:nl])
-                        self._rbuf.write(data[nl:])
-                        del data
-                        break
-                    buf.write(data)
-                return buf.getvalue()
-            else:
-                # Read until size bytes or \n or EOF seen, whichever comes
-                # first
-                buf.seek(0, 2)  # seek end
-                buf_len = buf.tell()
-                if buf_len >= size:
-                    buf.seek(0)
-                    rv = buf.read(size)
-                    self._rbuf = io.BytesIO()
-                    self._rbuf.write(buf.read())
-                    return rv
-                # reset _rbuf.  we consume it via buf.
-                self._rbuf = io.BytesIO()
-                while True:
-                    data = self.recv(self._rbufsize)
-                    if not data:
-                        break
-                    left = size - buf_len
-                    # did we just receive a newline?
-                    nl = data.find('\n', 0, left)
-                    if nl >= 0:
-                        nl += 1
-                        # save the excess data to _rbuf
-                        self._rbuf.write(data[nl:])
-                        if buf_len:
-                            buf.write(data[:nl])
-                            break
-                        else:
-                            # Shortcut.  Avoid data copy through buf when
-                            # returning a substring of our first recv().
-                            return data[:nl]
-                    n = len(data)
-                    if n == size and not buf_len:
-                        # Shortcut.  Avoid data copy through buf when
-                        # returning exactly all of our first recv().
-                        return data
-                    if n >= left:
-                        buf.write(data[:left])
-                        self._rbuf.write(data[left:])
-                        break
-                    buf.write(data)
-                    buf_len += n
-                    #assert buf_len == buf.tell()
-                return buf.getvalue()
-    else:
-        def read(self, size=-1):
-            if size < 0:
-                # Read until EOF
-                buffers = [self._rbuf]
-                self._rbuf = ""
-                if self._rbufsize <= 1:
-                    recv_size = self.default_bufsize
-                else:
-                    recv_size = self._rbufsize
-
-                while True:
-                    data = self.recv(recv_size)
-                    if not data:
-                        break
-                    buffers.append(data)
-                return "".join(buffers)
-            else:
-                # Read until size bytes or EOF seen, whichever comes first
-                data = self._rbuf
-                buf_len = len(data)
-                if buf_len >= size:
-                    self._rbuf = data[size:]
-                    return data[:size]
-                buffers = []
-                if data:
-                    buffers.append(data)
-                self._rbuf = ""
-                while True:
-                    left = size - buf_len
-                    recv_size = max(self._rbufsize, left)
-                    data = self.recv(recv_size)
-                    if not data:
-                        break
-                    buffers.append(data)
-                    n = len(data)
-                    if n >= left:
-                        self._rbuf = data[left:]
-                        buffers[-1] = data[:left]
-                        break
-                    buf_len += n
-                return "".join(buffers)
-
-        def readline(self, size=-1):
-            data = self._rbuf
-            if size < 0:
-                # Read until \n or EOF, whichever comes first
-                if self._rbufsize <= 1:
-                    # Speed up unbuffered case
-                    assert data == ""
-                    buffers = []
-                    while data != "\n":
-                        data = self.recv(1)
-                        if not data:
-                            break
-                        buffers.append(data)
-                    return "".join(buffers)
-                nl = data.find('\n')
-                if nl >= 0:
-                    nl += 1
-                    self._rbuf = data[nl:]
-                    return data[:nl]
-                buffers = []
-                if data:
-                    buffers.append(data)
-                self._rbuf = ""
-                while True:
-                    data = self.recv(self._rbufsize)
-                    if not data:
-                        break
-                    buffers.append(data)
-                    nl = data.find('\n')
-                    if nl >= 0:
-                        nl += 1
-                        self._rbuf = data[nl:]
-                        buffers[-1] = data[:nl]
-                        break
-                return "".join(buffers)
-            else:
-                # Read until size bytes or \n or EOF seen, whichever comes
-                # first
-                nl = data.find('\n', 0, size)
-                if nl >= 0:
-                    nl += 1
-                    self._rbuf = data[nl:]
-                    return data[:nl]
-                buf_len = len(data)
-                if buf_len >= size:
-                    self._rbuf = data[size:]
-                    return data[:size]
-                buffers = []
-                if data:
-                    buffers.append(data)
-                self._rbuf = ""
-                while True:
-                    data = self.recv(self._rbufsize)
-                    if not data:
-                        break
-                    buffers.append(data)
-                    left = size - buf_len
-                    nl = data.find('\n', 0, left)
-                    if nl >= 0:
-                        nl += 1
-                        self._rbuf = data[nl:]
-                        buffers[-1] = data[:nl]
-                        break
-                    n = len(data)
-                    if n >= left:
-                        self._rbuf = data[left:]
-                        buffers[-1] = data[:left]
-                        break
-                    buf_len += n
-                return "".join(buffers)
-
-
-class HTTPConnection(object):
-
-    """An HTTP connection (active socket).
-
-    server: the Server object which received this connection.
-    socket: the raw socket object (usually TCP) for this connection.
-    makefile: a fileobject class for reading from the socket.
-    """
-
-    remote_addr = None
-    remote_port = None
-    ssl_env = None
-    rbufsize = DEFAULT_BUFFER_SIZE
-    wbufsize = DEFAULT_BUFFER_SIZE
-    RequestHandlerClass = HTTPRequest
-
-    def __init__(self, server, sock, makefile=CP_fileobject):
-        self.server = server
-        self.socket = sock
-        self.rfile = makefile(sock, "rb", self.rbufsize)
-        self.wfile = makefile(sock, "wb", self.wbufsize)
-        self.requests_seen = 0
-
-    def communicate(self):
-        """Read each request and respond appropriately."""
-        request_seen = False
-        try:
-            while True:
-                # (re)set req to None so that if something goes wrong in
-                # the RequestHandlerClass constructor, the error doesn't
-                # get written to the previous request.
-                req = None
-                req = self.RequestHandlerClass(self.server, self)
-
-                # This order of operations should guarantee correct pipelining.
-                req.parse_request()
-                if self.server.stats['Enabled']:
-                    self.requests_seen += 1
-                if not req.ready:
-                    # Something went wrong in the parsing (and the server has
-                    # probably already made a simple_response). Return and
-                    # let the conn close.
-                    return
-
-                request_seen = True
-                req.respond()
-                if req.close_connection:
-                    return
-        except socket.error:
-            e = sys.exc_info()[1]
-            errnum = e.args[0]
-            # sadly SSL sockets return a different (longer) time out string
-            if (
-                errnum == 'timed out' or
-                errnum == 'The read operation timed out'
-            ):
-                # Don't error if we're between requests; only error
-                # if 1) no request has been started at all, or 2) we're
-                # in the middle of a request.
-                # See https://github.com/cherrypy/cherrypy/issues/853
-                if (not request_seen) or (req and req.started_request):
-                    # Don't bother writing the 408 if the response
-                    # has already started being written.
-                    if req and not req.sent_headers:
-                        try:
-                            req.simple_response("408 Request Timeout")
-                        except FatalSSLAlert:
-                            # Close the connection.
-                            return
-            elif errnum not in socket_errors_to_ignore:
-                self.server.error_log("socket.error %s" % repr(errnum),
-                                      level=logging.WARNING, traceback=True)
-                if req and not req.sent_headers:
-                    try:
-                        req.simple_response("500 Internal Server Error")
-                    except FatalSSLAlert:
-                        # Close the connection.
-                        return
-            return
-        except (KeyboardInterrupt, SystemExit):
-            raise
-        except FatalSSLAlert:
-            # Close the connection.
-            return
-        except NoSSLError:
-            if req and not req.sent_headers:
-                # Unwrap our wfile
-                self.wfile = CP_fileobject(
-                    self.socket._sock, "wb", self.wbufsize)
-                req.simple_response(
-                    "400 Bad Request",
-                    "The client sent a plain HTTP request, but "
-                    "this server only speaks HTTPS on this port.")
-                self.linger = True
-        except Exception:
-            e = sys.exc_info()[1]
-            self.server.error_log(repr(e), level=logging.ERROR, traceback=True)
-            if req and not req.sent_headers:
-                try:
-                    req.simple_response("500 Internal Server Error")
-                except FatalSSLAlert:
-                    # Close the connection.
-                    return
-
-    linger = False
-
-    def close(self):
-        """Close the socket underlying this connection."""
-        self.rfile.close()
-
-        if not self.linger:
-            # Python's socket module does NOT call close on the kernel
-            # socket when you call socket.close(). We do so manually here
-            # because we want this server to send a FIN TCP segment
-            # immediately. Note this must be called *before* calling
-            # socket.close(), because the latter drops its reference to
-            # the kernel socket.
-            if hasattr(self.socket, '_sock'):
-                self.socket._sock.close()
-            self.socket.close()
-        else:
-            # On the other hand, sometimes we want to hang around for a bit
-            # to make sure the client has a chance to read our entire
-            # response. Skipping the close() calls here delays the FIN
-            # packet until the socket object is garbage-collected later.
-            # Someday, perhaps, we'll do the full lingering_close that
-            # Apache does, but not today.
-            pass
-
-
-class TrueyZero(object):
-
-    """An object which equals and does math like the integer 0 but evals True.
-    """
-
-    def __add__(self, other):
-        return other
-
-    def __radd__(self, other):
-        return other
-trueyzero = TrueyZero()
-
-
-_SHUTDOWNREQUEST = None
-
-
-class WorkerThread(threading.Thread):
-
-    """Thread which continuously polls a Queue for Connection objects.
-
-    Due to the timing issues of polling a Queue, a WorkerThread does not
-    check its own 'ready' flag after it has started. To stop the thread,
-    it is necessary to stick a _SHUTDOWNREQUEST object onto the Queue
-    (one for each running WorkerThread).
-    """
-
-    conn = None
-    """The current connection pulled off the Queue, or None."""
-
-    server = None
-    """The HTTP Server which spawned this thread, and which owns the
-    Queue and is placing active connections into it."""
-
-    ready = False
-    """A simple flag for the calling server to know when this thread
-    has begun polling the Queue."""
-
-    def __init__(self, server):
-        self.ready = False
-        self.server = server
-
-        self.requests_seen = 0
-        self.bytes_read = 0
-        self.bytes_written = 0
-        self.start_time = None
-        self.work_time = 0
-        self.stats = {
-            'Requests': lambda s: self.requests_seen + (
-                (self.start_time is None) and
-                trueyzero or
-                self.conn.requests_seen
-            ),
-            'Bytes Read': lambda s: self.bytes_read + (
-                (self.start_time is None) and
-                trueyzero or
-                self.conn.rfile.bytes_read
-            ),
-            'Bytes Written': lambda s: self.bytes_written + (
-                (self.start_time is None) and
-                trueyzero or
-                self.conn.wfile.bytes_written
-            ),
-            'Work Time': lambda s: self.work_time + (
-                (self.start_time is None) and
-                trueyzero or
-                time.time() - self.start_time
-            ),
-            'Read Throughput': lambda s: s['Bytes Read'](s) / (
-                s['Work Time'](s) or 1e-6),
-            'Write Throughput': lambda s: s['Bytes Written'](s) / (
-                s['Work Time'](s) or 1e-6),
-        }
-        threading.Thread.__init__(self)
-
-    def run(self):
-        self.server.stats['Worker Threads'][self.getName()] = self.stats
-        try:
-            self.ready = True
-            while True:
-                conn = self.server.requests.get()
-                if conn is _SHUTDOWNREQUEST:
-                    return
-
-                self.conn = conn
-                if self.server.stats['Enabled']:
-                    self.start_time = time.time()
-                try:
-                    conn.communicate()
-                finally:
-                    conn.close()
-                    if self.server.stats['Enabled']:
-                        self.requests_seen += self.conn.requests_seen
-                        self.bytes_read += self.conn.rfile.bytes_read
-                        self.bytes_written += self.conn.wfile.bytes_written
-                        self.work_time += time.time() - self.start_time
-                        self.start_time = None
-                    self.conn = None
-        except (KeyboardInterrupt, SystemExit):
-            exc = sys.exc_info()[1]
-            self.server.interrupt = exc
-
-
-class ThreadPool(object):
-
-    """A Request Queue for an HTTPServer which pools threads.
-
-    ThreadPool objects must provide min, get(), put(obj), start()
-    and stop(timeout) attributes.
-    """
-
-    def __init__(self, server, min=10, max=-1,
-        accepted_queue_size=-1, accepted_queue_timeout=10):
-        self.server = server
-        self.min = min
-        self.max = max
-        self._threads = []
-        self._queue = queue.Queue(maxsize=accepted_queue_size)
-        self._queue_put_timeout = accepted_queue_timeout
-        self.get = self._queue.get
-
-    def start(self):
-        """Start the pool of threads."""
-        for i in range(self.min):
-            self._threads.append(WorkerThread(self.server))
-        for worker in self._threads:
-            worker.setName("CP Server " + worker.getName())
-            worker.start()
-        for worker in self._threads:
-            while not worker.ready:
-                time.sleep(.1)
-
-    def _get_idle(self):
-        """Number of worker threads which are idle. Read-only."""
-        return len([t for t in self._threads if t.conn is None])
-    idle = property(_get_idle, doc=_get_idle.__doc__)
-
-    def put(self, obj):
-        self._queue.put(obj, block=True, timeout=self._queue_put_timeout)
-        if obj is _SHUTDOWNREQUEST:
-            return
-
-    def grow(self, amount):
-        """Spawn new worker threads (not above self.max)."""
-        if self.max > 0:
-            budget = max(self.max - len(self._threads), 0)
-        else:
-            # self.max <= 0 indicates no maximum
-            budget = float('inf')
-
-        n_new = min(amount, budget)
-
-        workers = [self._spawn_worker() for i in range(n_new)]
-        while not self._all(operator.attrgetter('ready'), workers):
-            time.sleep(.1)
-        self._threads.extend(workers)
-
-    def _spawn_worker(self):
-        worker = WorkerThread(self.server)
-        worker.setName("CP Server " + worker.getName())
-        worker.start()
-        return worker
-
-    def _all(func, items):
-        results = [func(item) for item in items]
-        return reduce(operator.and_, results, True)
-    _all = staticmethod(_all)
-
-    def shrink(self, amount):
-        """Kill off worker threads (not below self.min)."""
-        # Grow/shrink the pool if necessary.
-        # Remove any dead threads from our list
-        for t in self._threads:
-            if not t.isAlive():
-                self._threads.remove(t)
-                amount -= 1
-
-        # calculate the number of threads above the minimum
-        n_extra = max(len(self._threads) - self.min, 0)
-
-        # don't remove more than amount
-        n_to_remove = min(amount, n_extra)
-
-        # put shutdown requests on the queue equal to the number of threads
-        # to remove. As each request is processed by a worker, that worker
-        # will terminate and be culled from the list.
-        for n in range(n_to_remove):
-            self._queue.put(_SHUTDOWNREQUEST)
-
-    def stop(self, timeout=5):
-        # Must shut down threads here so the code that calls
-        # this method can know when all threads are stopped.
-        for worker in self._threads:
-            self._queue.put(_SHUTDOWNREQUEST)
-
-        # Don't join currentThread (when stop is called inside a request).
-        current = threading.currentThread()
-        if timeout and timeout >= 0:
-            endtime = time.time() + timeout
-        while self._threads:
-            worker = self._threads.pop()
-            if worker is not current and worker.isAlive():
-                try:
-                    if timeout is None or timeout < 0:
-                        worker.join()
-                    else:
-                        remaining_time = endtime - time.time()
-                        if remaining_time > 0:
-                            worker.join(remaining_time)
-                        if worker.isAlive():
-                            # We exhausted the timeout.
-                            # Forcibly shut down the socket.
-                            c = worker.conn
-                            if c and not c.rfile.closed:
-                                try:
-                                    c.socket.shutdown(socket.SHUT_RD)
-                                except TypeError:
-                                    # pyOpenSSL sockets don't take an arg
-                                    c.socket.shutdown()
-                            worker.join()
-                except (AssertionError,
-                        # Ignore repeated Ctrl-C.
-                        # See
-                        # https://github.com/cherrypy/cherrypy/issues/691.
-                        KeyboardInterrupt):
-                    pass
-
-    def _get_qsize(self):
-        return self._queue.qsize()
-    qsize = property(_get_qsize)
-
-
-try:
-    import fcntl
-except ImportError:
-    try:
-        from ctypes import windll, WinError
-        import ctypes.wintypes
-        _SetHandleInformation = windll.kernel32.SetHandleInformation
-        _SetHandleInformation.argtypes = [
-            ctypes.wintypes.HANDLE,
-            ctypes.wintypes.DWORD,
-            ctypes.wintypes.DWORD,
-        ]
-        _SetHandleInformation.restype = ctypes.wintypes.BOOL
-    except ImportError:
-        def prevent_socket_inheritance(sock):
-            """Dummy function, since neither fcntl nor ctypes are available."""
-            pass
-    else:
-        def prevent_socket_inheritance(sock):
-            """Mark the given socket fd as non-inheritable (Windows)."""
-            if not _SetHandleInformation(sock.fileno(), 1, 0):
-                raise WinError()
-else:
-    def prevent_socket_inheritance(sock):
-        """Mark the given socket fd as non-inheritable (POSIX)."""
-        fd = sock.fileno()
-        old_flags = fcntl.fcntl(fd, fcntl.F_GETFD)
-        fcntl.fcntl(fd, fcntl.F_SETFD, old_flags | fcntl.FD_CLOEXEC)
-
-
-class SSLAdapter(object):
-
-    """Base class for SSL driver library adapters.
-
-    Required methods:
-
-        * ``wrap(sock) -> (wrapped socket, ssl environ dict)``
-        * ``makefile(sock, mode='r', bufsize=DEFAULT_BUFFER_SIZE) ->
-          socket file object``
-    """
-
-    def __init__(self, certificate, private_key, certificate_chain=None):
-        self.certificate = certificate
-        self.private_key = private_key
-        self.certificate_chain = certificate_chain
-
-    def wrap(self, sock):
-        raise NotImplemented
-
-    def makefile(self, sock, mode='r', bufsize=DEFAULT_BUFFER_SIZE):
-        raise NotImplemented
-
-
-class HTTPServer(object):
-
-    """An HTTP server."""
-
-    _bind_addr = "127.0.0.1"
-    _interrupt = None
-
-    gateway = None
-    """A Gateway instance."""
-
-    minthreads = None
-    """The minimum number of worker threads to create (default 10)."""
-
-    maxthreads = None
-    """The maximum number of worker threads to create (default -1 = no limit).
-    """
-
-    server_name = None
-    """The name of the server; defaults to socket.gethostname()."""
-
-    protocol = "HTTP/1.1"
-    """The version string to write in the Status-Line of all HTTP responses.
-
-    For example, "HTTP/1.1" is the default. This also limits the supported
-    features used in the response."""
-
-    request_queue_size = 5
-    """The 'backlog' arg to socket.listen(); max queued connections
-    (default 5).
-    """
-
-    shutdown_timeout = 5
-    """The total time, in seconds, to wait for worker threads to cleanly exit.
-    """
-
-    timeout = 10
-    """The timeout in seconds for accepted connections (default 10)."""
-
-    version = "CherryPy/" + cp_version
-    """A version string for the HTTPServer."""
-
-    software = None
-    """The value to set for the SERVER_SOFTWARE entry in the WSGI environ.
-
-    If None, this defaults to ``'%s Server' % self.version``."""
-
-    ready = False
-    """An internal flag which marks whether the socket is accepting connections
-    """
-
-    max_request_header_size = 0
-    """The maximum size, in bytes, for request headers, or 0 for no limit."""
-
-    max_request_body_size = 0
-    """The maximum size, in bytes, for request bodies, or 0 for no limit."""
-
-    nodelay = True
-    """If True (the default since 3.1), sets the TCP_NODELAY socket option."""
-
-    ConnectionClass = HTTPConnection
-    """The class to use for handling HTTP connections."""
-
-    ssl_adapter = None
-    """An instance of SSLAdapter (or a subclass).
-
-    You must have the corresponding SSL driver library installed."""
-
-    def __init__(self, bind_addr, gateway, minthreads=10, maxthreads=-1,
-                 server_name=None):
-        self.bind_addr = bind_addr
-        self.gateway = gateway
-
-        self.requests = ThreadPool(self, min=minthreads or 1, max=maxthreads)
-
-        if not server_name:
-            server_name = socket.gethostname()
-        self.server_name = server_name
-        self.clear_stats()
-
-    def clear_stats(self):
-        self._start_time = None
-        self._run_time = 0
-        self.stats = {
-            'Enabled': False,
-            'Bind Address': lambda s: repr(self.bind_addr),
-            'Run time': lambda s: (not s['Enabled']) and -1 or self.runtime(),
-            'Accepts': 0,
-            'Accepts/sec': lambda s: s['Accepts'] / self.runtime(),
-            'Queue': lambda s: getattr(self.requests, "qsize", None),
-            'Threads': lambda s: len(getattr(self.requests, "_threads", [])),
-            'Threads Idle': lambda s: getattr(self.requests, "idle", None),
-            'Socket Errors': 0,
-            'Requests': lambda s: (not s['Enabled']) and -1 or sum(
-                [w['Requests'](w) for w in s['Worker Threads'].values()], 0),
-            'Bytes Read': lambda s: (not s['Enabled']) and -1 or sum(
-                [w['Bytes Read'](w) for w in s['Worker Threads'].values()], 0),
-            'Bytes Written': lambda s: (not s['Enabled']) and -1 or sum(
-                [w['Bytes Written'](w) for w in s['Worker Threads'].values()],
-                0),
-            'Work Time': lambda s: (not s['Enabled']) and -1 or sum(
-                [w['Work Time'](w) for w in s['Worker Threads'].values()], 0),
-            'Read Throughput': lambda s: (not s['Enabled']) and -1 or sum(
-                [w['Bytes Read'](w) / (w['Work Time'](w) or 1e-6)
-                 for w in s['Worker Threads'].values()], 0),
-            'Write Throughput': lambda s: (not s['Enabled']) and -1 or sum(
-                [w['Bytes Written'](w) / (w['Work Time'](w) or 1e-6)
-                 for w in s['Worker Threads'].values()], 0),
-            'Worker Threads': {},
-        }
-        logging.statistics["CherryPy HTTPServer %d" % id(self)] = self.stats
-
-    def runtime(self):
-        if self._start_time is None:
-            return self._run_time
-        else:
-            return self._run_time + (time.time() - self._start_time)
-
-    def __str__(self):
-        return "%s.%s(%r)" % (self.__module__, self.__class__.__name__,
-                              self.bind_addr)
-
-    def _get_bind_addr(self):
-        return self._bind_addr
-
-    def _set_bind_addr(self, value):
-        if isinstance(value, tuple) and value[0] in ('', None):
-            # Despite the socket module docs, using '' does not
-            # allow AI_PASSIVE to work. Passing None instead
-            # returns '0.0.0.0' like we want. In other words:
-            #     host    AI_PASSIVE     result
-            #      ''         Y         192.168.x.y
-            #      ''         N         192.168.x.y
-            #     None        Y         0.0.0.0
-            #     None        N         127.0.0.1
-            # But since you can get the same effect with an explicit
-            # '0.0.0.0', we deny both the empty string and None as values.
-            raise ValueError("Host values of '' or None are not allowed. "
-                             "Use '0.0.0.0' (IPv4) or '::' (IPv6) instead "
-                             "to listen on all active interfaces.")
-        self._bind_addr = value
-    bind_addr = property(
-        _get_bind_addr,
-        _set_bind_addr,
-        doc="""The interface on which to listen for connections.
-
-        For TCP sockets, a (host, port) tuple. Host values may be any IPv4
-        or IPv6 address, or any valid hostname. The string 'localhost' is a
-        synonym for '127.0.0.1' (or '::1', if your hosts file prefers IPv6).
-        The string '0.0.0.0' is a special IPv4 entry meaning "any active
-        interface" (INADDR_ANY), and '::' is the similar IN6ADDR_ANY for
-        IPv6. The empty string or None are not allowed.
-
-        For UNIX sockets, supply the filename as a string.""")
-
-    def start(self):
-        """Run the server forever."""
-        # We don't have to trap KeyboardInterrupt or SystemExit here,
-        # because cherrpy.server already does so, calling self.stop() for us.
-        # If you're using this server with another framework, you should
-        # trap those exceptions in whatever code block calls start().
-        self._interrupt = None
-
-        if self.software is None:
-            self.software = "%s Server" % self.version
-
-        # Select the appropriate socket
-        if isinstance(self.bind_addr, basestring):
-            # AF_UNIX socket
-
-            # So we can reuse the socket...
-            try:
-                os.unlink(self.bind_addr)
-            except:
-                pass
-
-            # So everyone can access the socket...
-            try:
-                os.chmod(self.bind_addr, 0o777)
-            except:
-                pass
-
-            info = [
-                (socket.AF_UNIX, socket.SOCK_STREAM, 0, "", self.bind_addr)]
-        else:
-            # AF_INET or AF_INET6 socket
-            # Get the correct address family for our host (allows IPv6
-            # addresses)
-            host, port = self.bind_addr
-            try:
-                info = socket.getaddrinfo(
-                    host, port, socket.AF_UNSPEC,
-                    socket.SOCK_STREAM, 0, socket.AI_PASSIVE)
-            except socket.gaierror:
-                if ':' in self.bind_addr[0]:
-                    info = [(socket.AF_INET6, socket.SOCK_STREAM,
-                             0, "", self.bind_addr + (0, 0))]
-                else:
-                    info = [(socket.AF_INET, socket.SOCK_STREAM,
-                             0, "", self.bind_addr)]
-
-        self.socket = None
-        msg = "No socket could be created"
-        for res in info:
-            af, socktype, proto, canonname, sa = res
-            try:
-                self.bind(af, socktype, proto)
-            except socket.error, serr:
-                msg = "%s -- (%s: %s)" % (msg, sa, serr)
-                if self.socket:
-                    self.socket.close()
-                self.socket = None
-                continue
-            break
-        if not self.socket:
-            raise socket.error(msg)
-
-        # Timeout so KeyboardInterrupt can be caught on Win32
-        self.socket.settimeout(1)
-        self.socket.listen(self.request_queue_size)
-
-        # Create worker threads
-        self.requests.start()
-
-        self.ready = True
-        self._start_time = time.time()
-        while self.ready:
-            try:
-                self.tick()
-            except (KeyboardInterrupt, SystemExit):
-                raise
-            except:
-                self.error_log("Error in HTTPServer.tick", level=logging.ERROR,
-                               traceback=True)
-
-            if self.interrupt:
-                while self.interrupt is True:
-                    # Wait for self.stop() to complete. See _set_interrupt.
-                    time.sleep(0.1)
-                if self.interrupt:
-                    raise self.interrupt
-
-    def error_log(self, msg="", level=20, traceback=False):
-        # Override this in subclasses as desired
-        sys.stderr.write(msg + '\n')
-        sys.stderr.flush()
-        if traceback:
-            tblines = traceback_.format_exc()
-            sys.stderr.write(tblines)
-            sys.stderr.flush()
-
-    def bind(self, family, type, proto=0):
-        """Create (or recreate) the actual socket object."""
-        self.socket = socket.socket(family, type, proto)
-        prevent_socket_inheritance(self.socket)
-        self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
-        if self.nodelay and not isinstance(self.bind_addr, str):
-            self.socket.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
-
-        if self.ssl_adapter is not None:
-            self.socket = self.ssl_adapter.bind(self.socket)
-
-        # If listening on the IPV6 any address ('::' = IN6ADDR_ANY),
-        # activate dual-stack. See
-        # https://github.com/cherrypy/cherrypy/issues/871.
-        if (hasattr(socket, 'AF_INET6') and family == socket.AF_INET6
-                and self.bind_addr[0] in ('::', '::0', '::0.0.0.0')):
-            try:
-                self.socket.setsockopt(
-                    socket.IPPROTO_IPV6, socket.IPV6_V6ONLY, 0)
-            except (AttributeError, socket.error):
-                # Apparently, the socket option is not available in
-                # this machine's TCP stack
-                pass
-
-        self.socket.bind(self.bind_addr)
-
-    def tick(self):
-        """Accept a new connection and put it on the Queue."""
-        try:
-            s, addr = self.socket.accept()
-            if self.stats['Enabled']:
-                self.stats['Accepts'] += 1
-            if not self.ready:
-                return
-
-            prevent_socket_inheritance(s)
-            if hasattr(s, 'settimeout'):
-                s.settimeout(self.timeout)
-
-            makefile = CP_fileobject
-            ssl_env = {}
-            # if ssl cert and key are set, we try to be a secure HTTP server
-            if self.ssl_adapter is not None:
-                try:
-                    s, ssl_env = self.ssl_adapter.wrap(s)
-                except NoSSLError:
-                    msg = ("The client sent a plain HTTP request, but "
-                           "this server only speaks HTTPS on this port.")
-                    buf = ["%s 400 Bad Request\r\n" % self.protocol,
-                           "Content-Length: %s\r\n" % len(msg),
-                           "Content-Type: text/plain\r\n\r\n",
-                           msg]
-
-                    wfile = makefile(s._sock, "wb", DEFAULT_BUFFER_SIZE)
-                    try:
-                        wfile.sendall("".join(buf))
-                    except socket.error:
-                        x = sys.exc_info()[1]
-                        if x.args[0] not in socket_errors_to_ignore:
-                            raise
-                    return
-                if not s:
-                    return
-                makefile = self.ssl_adapter.makefile
-                # Re-apply our timeout since we may have a new socket object
-                if hasattr(s, 'settimeout'):
-                    s.settimeout(self.timeout)
-
-            conn = self.ConnectionClass(self, s, makefile)
-
-            if not isinstance(self.bind_addr, basestring):
-                # optional values
-                # Until we do DNS lookups, omit REMOTE_HOST
-                if addr is None:  # sometimes this can happen
-                    # figure out if AF_INET or AF_INET6.
-                    if len(s.getsockname()) == 2:
-                        # AF_INET
-                        addr = ('0.0.0.0', 0)
-                    else:
-                        # AF_INET6
-                        addr = ('::', 0)
-                conn.remote_addr = addr[0]
-                conn.remote_port = addr[1]
-
-            conn.ssl_env = ssl_env
-
-            try:
-                self.requests.put(conn)
-            except queue.Full:
-                # Just drop the conn. TODO: write 503 back?
-                conn.close()
-                return
-        except socket.timeout:
-            # The only reason for the timeout in start() is so we can
-            # notice keyboard interrupts on Win32, which don't interrupt
-            # accept() by default
-            return
-        except socket.error:
-            x = sys.exc_info()[1]
-            if self.stats['Enabled']:
-                self.stats['Socket Errors'] += 1
-            if x.args[0] in socket_error_eintr:
-                # I *think* this is right. EINTR should occur when a signal
-                # is received during the accept() call; all docs say retry
-                # the call, and I *think* I'm reading it right that Python
-                # will then go ahead and poll for and handle the signal
-                # elsewhere. See
-                # https://github.com/cherrypy/cherrypy/issues/707.
-                return
-            if x.args[0] in socket_errors_nonblocking:
-                # Just try again. See
-                # https://github.com/cherrypy/cherrypy/issues/479.
-                return
-            if x.args[0] in socket_errors_to_ignore:
-                # Our socket was closed.
-                # See https://github.com/cherrypy/cherrypy/issues/686.
-                return
-            raise
-
-    def _get_interrupt(self):
-        return self._interrupt
-
-    def _set_interrupt(self, interrupt):
-        self._interrupt = True
-        self.stop()
-        self._interrupt = interrupt
-    interrupt = property(_get_interrupt, _set_interrupt,
-                         doc="Set this to an Exception instance to "
-                             "interrupt the server.")
-
-    def stop(self):
-        """Gracefully shutdown a server that is serving forever."""
-        self.ready = False
-        if self._start_time is not None:
-            self._run_time += (time.time() - self._start_time)
-        self._start_time = None
-
-        sock = getattr(self, "socket", None)
-        if sock:
-            if not isinstance(self.bind_addr, basestring):
-                # Touch our own socket to make accept() return immediately.
-                try:
-                    host, port = sock.getsockname()[:2]
-                except socket.error:
-                    x = sys.exc_info()[1]
-                    if x.args[0] not in socket_errors_to_ignore:
-                        # Changed to use error code and not message
-                        # See
-                        # https://github.com/cherrypy/cherrypy/issues/860.
-                        raise
-                else:
-                    # Note that we're explicitly NOT using AI_PASSIVE,
-                    # here, because we want an actual IP to touch.
-                    # localhost won't work if we've bound to a public IP,
-                    # but it will if we bound to '0.0.0.0' (INADDR_ANY).
-                    for res in socket.getaddrinfo(host, port, socket.AF_UNSPEC,
-                                                  socket.SOCK_STREAM):
-                        af, socktype, proto, canonname, sa = res
-                        s = None
-                        try:
-                            s = socket.socket(af, socktype, proto)
-                            # See
-                            # http://groups.google.com/group/cherrypy-users/
-                            #    browse_frm/thread/bbfe5eb39c904fe0
-                            s.settimeout(1.0)
-                            s.connect((host, port))
-                            s.close()
-                        except socket.error:
-                            if s:
-                                s.close()
-            if hasattr(sock, "close"):
-                sock.close()
-            self.socket = None
-
-        self.requests.stop(self.shutdown_timeout)
-
-
-class Gateway(object):
-
-    """A base class to interface HTTPServer with other systems, such as WSGI.
-    """
-
-    def __init__(self, req):
-        self.req = req
-
-    def respond(self):
-        """Process the current request. Must be overridden in a subclass."""
-        raise NotImplemented
-
-
-# These may either be wsgiserver.SSLAdapter subclasses or the string names
-# of such classes (in which case they will be lazily loaded).
-ssl_adapters = {
-    'builtin': 'cherrypy.wsgiserver.ssl_builtin.BuiltinSSLAdapter',
-    'pyopenssl': 'cherrypy.wsgiserver.ssl_pyopenssl.pyOpenSSLAdapter',
-}
-
-
-def get_ssl_adapter_class(name='builtin'):
-    """Return an SSL adapter class for the given name."""
-    adapter = ssl_adapters[name.lower()]
-    if isinstance(adapter, basestring):
-        last_dot = adapter.rfind(".")
-        attr_name = adapter[last_dot + 1:]
-        mod_path = adapter[:last_dot]
-
-        try:
-            mod = sys.modules[mod_path]
-            if mod is None:
-                raise KeyError()
-        except KeyError:
-            # The last [''] is important.
-            mod = __import__(mod_path, globals(), locals(), [''])
-
-        # Let an AttributeError propagate outward.
-        try:
-            adapter = getattr(mod, attr_name)
-        except AttributeError:
-            raise AttributeError("'%s' object has no attribute '%s'"
-                                 % (mod_path, attr_name))
-
-    return adapter
-
-# ------------------------------- WSGI Stuff -------------------------------- #
-
-
-class CherryPyWSGIServer(HTTPServer):
-
-    """A subclass of HTTPServer which calls a WSGI application."""
-
-    wsgi_version = (1, 0)
-    """The version of WSGI to produce."""
-
-    def __init__(self, bind_addr, wsgi_app, numthreads=10, server_name=None,
-                 max=-1, request_queue_size=5, timeout=10, shutdown_timeout=5,
-                 accepted_queue_size=-1, accepted_queue_timeout=10):
-        self.requests = ThreadPool(self, min=numthreads or 1, max=max,
-            accepted_queue_size=accepted_queue_size,
-            accepted_queue_timeout=accepted_queue_timeout)
-        self.wsgi_app = wsgi_app
-        self.gateway = wsgi_gateways[self.wsgi_version]
-
-        self.bind_addr = bind_addr
-        if not server_name:
-            server_name = socket.gethostname()
-        self.server_name = server_name
-        self.request_queue_size = request_queue_size
-
-        self.timeout = timeout
-        self.shutdown_timeout = shutdown_timeout
-        self.clear_stats()
-
-    def _get_numthreads(self):
-        return self.requests.min
-
-    def _set_numthreads(self, value):
-        self.requests.min = value
-    numthreads = property(_get_numthreads, _set_numthreads)
-
-
-class WSGIGateway(Gateway):
-
-    """A base class to interface HTTPServer with WSGI."""
-
-    def __init__(self, req):
-        self.req = req
-        self.started_response = False
-        self.env = self.get_environ()
-        self.remaining_bytes_out = None
-
-    def get_environ(self):
-        """Return a new environ dict targeting the given wsgi.version"""
-        raise NotImplemented
-
-    def respond(self):
-        """Process the current request."""
-        response = self.req.server.wsgi_app(self.env, self.start_response)
-        try:
-            for chunk in response:
-                # "The start_response callable must not actually transmit
-                # the response headers. Instead, it must store them for the
-                # server or gateway to transmit only after the first
-                # iteration of the application return value that yields
-                # a NON-EMPTY string, or upon the application's first
-                # invocation of the write() callable." (PEP 333)
-                if chunk:
-                    if isinstance(chunk, unicodestr):
-                        chunk = chunk.encode('ISO-8859-1')
-                    self.write(chunk)
-        finally:
-            if hasattr(response, "close"):
-                response.close()
-
-    def start_response(self, status, headers, exc_info=None):
-        """WSGI callable to begin the HTTP response."""
-        # "The application may call start_response more than once,
-        # if and only if the exc_info argument is provided."
-        if self.started_response and not exc_info:
-            raise AssertionError("WSGI start_response called a second "
-                                 "time with no exc_info.")
-        self.started_response = True
-
-        # "if exc_info is provided, and the HTTP headers have already been
-        # sent, start_response must raise an error, and should raise the
-        # exc_info tuple."
-        if self.req.sent_headers:
-            try:
-                raise exc_info[0], exc_info[1], exc_info[2]
-            finally:
-                exc_info = None
-
-        self.req.status = status
-        for k, v in headers:
-            if not isinstance(k, str):
-                raise TypeError(
-                    "WSGI response header key %r is not of type str." % k)
-            if not isinstance(v, str):
-                raise TypeError(
-                    "WSGI response header value %r is not of type str." % v)
-            if k.lower() == 'content-length':
-                self.remaining_bytes_out = int(v)
-        self.req.outheaders.extend(headers)
-
-        return self.write
-
-    def write(self, chunk):
-        """WSGI callable to write unbuffered data to the client.
-
-        This method is also used internally by start_response (to write
-        data from the iterable returned by the WSGI application).
-        """
-        if not self.started_response:
-            raise AssertionError("WSGI write called before start_response.")
-
-        chunklen = len(chunk)
-        rbo = self.remaining_bytes_out
-        if rbo is not None and chunklen > rbo:
-            if not self.req.sent_headers:
-                # Whew. We can send a 500 to the client.
-                self.req.simple_response(
-                    "500 Internal Server Error",
-                    "The requested resource returned more bytes than the "
-                    "declared Content-Length.")
-            else:
-                # Dang. We have probably already sent data. Truncate the chunk
-                # to fit (so the client doesn't hang) and raise an error later.
-                chunk = chunk[:rbo]
-
-        if not self.req.sent_headers:
-            self.req.sent_headers = True
-            self.req.send_headers()
-
-        self.req.write(chunk)
-
-        if rbo is not None:
-            rbo -= chunklen
-            if rbo < 0:
-                raise ValueError(
-                    "Response body exceeds the declared Content-Length.")
-
-
-class WSGIGateway_10(WSGIGateway):
-
-    """A Gateway class to interface HTTPServer with WSGI 1.0.x."""
-
-    def get_environ(self):
-        """Return a new environ dict targeting the given wsgi.version"""
-        req = self.req
-        env = {
-            # set a non-standard environ entry so the WSGI app can know what
-            # the *real* server protocol is (and what features to support).
-            # See http://www.faqs.org/rfcs/rfc2145.html.
-            'ACTUAL_SERVER_PROTOCOL': req.server.protocol,
-            'PATH_INFO': req.path,
-            'QUERY_STRING': req.qs,
-            'REMOTE_ADDR': req.conn.remote_addr or '',
-            'REMOTE_PORT': str(req.conn.remote_port or ''),
-            'REQUEST_METHOD': req.method,
-            'REQUEST_URI': req.uri,
-            'SCRIPT_NAME': '',
-            'SERVER_NAME': req.server.server_name,
-            # Bah. "SERVER_PROTOCOL" is actually the REQUEST protocol.
-            'SERVER_PROTOCOL': req.request_protocol,
-            'SERVER_SOFTWARE': req.server.software,
-            'wsgi.errors': sys.stderr,
-            'wsgi.input': req.rfile,
-            'wsgi.multiprocess': False,
-            'wsgi.multithread': True,
-            'wsgi.run_once': False,
-            'wsgi.url_scheme': req.scheme,
-            'wsgi.version': (1, 0),
-        }
-
-        if isinstance(req.server.bind_addr, basestring):
-            # AF_UNIX. This isn't really allowed by WSGI, which doesn't
-            # address unix domain sockets. But it's better than nothing.
-            env["SERVER_PORT"] = ""
-        else:
-            env["SERVER_PORT"] = str(req.server.bind_addr[1])
-
-        # Request headers
-        for k, v in req.inheaders.iteritems():
-            env["HTTP_" + k.upper().replace("-", "_")] = v
-
-        # CONTENT_TYPE/CONTENT_LENGTH
-        ct = env.pop("HTTP_CONTENT_TYPE", None)
-        if ct is not None:
-            env["CONTENT_TYPE"] = ct
-        cl = env.pop("HTTP_CONTENT_LENGTH", None)
-        if cl is not None:
-            env["CONTENT_LENGTH"] = cl
-
-        if req.conn.ssl_env:
-            env.update(req.conn.ssl_env)
-
-        return env
-
-
-class WSGIGateway_u0(WSGIGateway_10):
-
-    """A Gateway class to interface HTTPServer with WSGI u.0.
-
-    WSGI u.0 is an experimental protocol, which uses unicode for keys and
-    values in both Python 2 and Python 3.
-    """
-
-    def get_environ(self):
-        """Return a new environ dict targeting the given wsgi.version"""
-        req = self.req
-        env_10 = WSGIGateway_10.get_environ(self)
-        env = dict([(k.decode('ISO-8859-1'), v)
-                   for k, v in env_10.iteritems()])
-        env[u'wsgi.version'] = ('u', 0)
-
-        # Request-URI
-        env.setdefault(u'wsgi.url_encoding', u'utf-8')
-        try:
-            for key in [u"PATH_INFO", u"SCRIPT_NAME", u"QUERY_STRING"]:
-                env[key] = env_10[str(key)].decode(env[u'wsgi.url_encoding'])
-        except UnicodeDecodeError:
-            # Fall back to latin 1 so apps can transcode if needed.
-            env[u'wsgi.url_encoding'] = u'ISO-8859-1'
-            for key in [u"PATH_INFO", u"SCRIPT_NAME", u"QUERY_STRING"]:
-                env[key] = env_10[str(key)].decode(env[u'wsgi.url_encoding'])
-
-        for k, v in sorted(env.items()):
-            if isinstance(v, str) and k not in ('REQUEST_URI', 'wsgi.input'):
-                env[k] = v.decode('ISO-8859-1')
-
-        return env
-
-wsgi_gateways = {
-    (1, 0): WSGIGateway_10,
-    ('u', 0): WSGIGateway_u0,
-}
-
-
-class WSGIPathInfoDispatcher(object):
-
-    """A WSGI dispatcher for dispatch based on the PATH_INFO.
-
-    apps: a dict or list of (path_prefix, app) pairs.
-    """
-
-    def __init__(self, apps):
-        try:
-            apps = list(apps.items())
-        except AttributeError:
-            pass
-
-        # Sort the apps by len(path), descending
-        apps.sort(cmp=lambda x, y: cmp(len(x[0]), len(y[0])))
-        apps.reverse()
-
-        # The path_prefix strings must start, but not end, with a slash.
-        # Use "" instead of "/".
-        self.apps = [(p.rstrip("/"), a) for p, a in apps]
-
-    def __call__(self, environ, start_response):
-        path = environ["PATH_INFO"] or "/"
-        for p, app in self.apps:
-            # The apps list should be sorted by length, descending.
-            if path.startswith(p + "/") or path == p:
-                environ = environ.copy()
-                environ["SCRIPT_NAME"] = environ["SCRIPT_NAME"] + p
-                environ["PATH_INFO"] = path[len(p):]
-                return app(environ, start_response)
-
-        start_response('404 Not Found', [('Content-Type', 'text/plain'),
-                                         ('Content-Length', '0')])
-        return ['']
diff --git a/lib/cherrypy/wsgiserver/wsgiserver3.py b/lib/cherrypy/wsgiserver/wsgiserver3.py
deleted file mode 100644
index e69eb0da02a..00000000000
--- a/lib/cherrypy/wsgiserver/wsgiserver3.py
+++ /dev/null
@@ -1,2198 +0,0 @@
-"""A high-speed, production ready, thread pooled, generic HTTP server.
-
-Simplest example on how to use this module directly
-(without using CherryPy's application machinery)::
-
-    from cherrypy import wsgiserver
-
-    def my_crazy_app(environ, start_response):
-        status = '200 OK'
-        response_headers = [('Content-type','text/plain')]
-        start_response(status, response_headers)
-        return ['Hello world!']
-
-    server = wsgiserver.CherryPyWSGIServer(
-                ('0.0.0.0', 8070), my_crazy_app,
-                server_name='www.cherrypy.example')
-    server.start()
-
-The CherryPy WSGI server can serve as many WSGI applications
-as you want in one instance by using a WSGIPathInfoDispatcher::
-
-    d = WSGIPathInfoDispatcher({'/': my_crazy_app, '/blog': my_blog_app})
-    server = wsgiserver.CherryPyWSGIServer(('0.0.0.0', 80), d)
-
-Want SSL support? Just set server.ssl_adapter to an SSLAdapter instance.
-
-This won't call the CherryPy engine (application side) at all, only the
-HTTP server, which is independent from the rest of CherryPy. Don't
-let the name "CherryPyWSGIServer" throw you; the name merely reflects
-its origin, not its coupling.
-
-For those of you wanting to understand internals of this module, here's the
-basic call flow. The server's listening thread runs a very tight loop,
-sticking incoming connections onto a Queue::
-
-    server = CherryPyWSGIServer(...)
-    server.start()
-    while True:
-        tick()
-        # This blocks until a request comes in:
-        child = socket.accept()
-        conn = HTTPConnection(child, ...)
-        server.requests.put(conn)
-
-Worker threads are kept in a pool and poll the Queue, popping off and then
-handling each connection in turn. Each connection can consist of an arbitrary
-number of requests and their responses, so we run a nested loop::
-
-    while True:
-        conn = server.requests.get()
-        conn.communicate()
-        ->  while True:
-                req = HTTPRequest(...)
-                req.parse_request()
-                ->  # Read the Request-Line, e.g. "GET /page HTTP/1.1"
-                    req.rfile.readline()
-                    read_headers(req.rfile, req.inheaders)
-                req.respond()
-                ->  response = app(...)
-                    try:
-                        for chunk in response:
-                            if chunk:
-                                req.write(chunk)
-                    finally:
-                        if hasattr(response, "close"):
-                            response.close()
-                if req.close_connection:
-                    return
-"""
-
-__all__ = ['HTTPRequest', 'HTTPConnection', 'HTTPServer',
-           'SizeCheckWrapper', 'KnownLengthRFile', 'ChunkedRFile',
-           'CP_makefile',
-           'MaxSizeExceeded', 'NoSSLError', 'FatalSSLAlert',
-           'WorkerThread', 'ThreadPool', 'SSLAdapter',
-           'CherryPyWSGIServer',
-           'Gateway', 'WSGIGateway', 'WSGIGateway_10', 'WSGIGateway_u0',
-           'WSGIPathInfoDispatcher', 'get_ssl_adapter_class',
-           'socket_errors_to_ignore']
-
-import os
-try:
-    import queue
-except:
-    import Queue as queue
-import re
-import email.utils
-import socket
-import sys
-import threading
-import time
-import traceback as traceback_
-import errno
-import logging
-from urllib.parse import urlparse
-
-try:
-    # prefer slower Python-based io module
-    import _pyio as io
-except ImportError:
-    # Python 2.6
-    import io
-
-try:
-    import pkg_resources
-except ImportError:
-    pass
-
-if 'win' in sys.platform and hasattr(socket, "AF_INET6"):
-    if not hasattr(socket, 'IPPROTO_IPV6'):
-        socket.IPPROTO_IPV6 = 41
-    if not hasattr(socket, 'IPV6_V6ONLY'):
-        socket.IPV6_V6ONLY = 27
-
-
-DEFAULT_BUFFER_SIZE = io.DEFAULT_BUFFER_SIZE
-
-
-try:
-    cp_version = pkg_resources.require('cherrypy')[0].version
-except Exception:
-    cp_version = 'unknown'
-
-
-if sys.version_info >= (3, 0):
-    unicodestr = str
-    basestring = (bytes, str)
-
-    def ntob(n, encoding='ISO-8859-1'):
-        """Return the given native string as a byte string in the given
-        encoding.
-        """
-        # In Python 3, the native string type is unicode
-        return n.encode(encoding)
-else:
-    unicodestr = unicode
-    basestring = basestring
-
-    def ntob(n, encoding='ISO-8859-1'):
-        """Return the given native string as a byte string in the given
-        encoding.
-        """
-        # In Python 2, the native string type is bytes. Assume it's already
-        # in the given encoding, which for ISO-8859-1 is almost always what
-        # was intended.
-        return n
-
-LF = ntob('\n')
-CRLF = ntob('\r\n')
-TAB = ntob('\t')
-SPACE = ntob(' ')
-COLON = ntob(':')
-SEMICOLON = ntob(';')
-EMPTY = ntob('')
-NUMBER_SIGN = ntob('#')
-QUESTION_MARK = ntob('?')
-ASTERISK = ntob('*')
-FORWARD_SLASH = ntob('/')
-quoted_slash = re.compile(ntob("(?i)%2F"))
-
-
-def plat_specific_errors(*errnames):
-    """Return error numbers for all errors in errnames on this platform.
-
-    The 'errno' module contains different global constants depending on
-    the specific platform (OS). This function will return the list of
-    numeric values for a given list of potential names.
-    """
-    errno_names = dir(errno)
-    nums = [getattr(errno, k) for k in errnames if k in errno_names]
-    # de-dupe the list
-    return list(dict.fromkeys(nums).keys())
-
-socket_error_eintr = plat_specific_errors("EINTR", "WSAEINTR")
-
-socket_errors_to_ignore = plat_specific_errors(
-    "EPIPE",
-    "EBADF", "WSAEBADF",
-    "ENOTSOCK", "WSAENOTSOCK",
-    "ETIMEDOUT", "WSAETIMEDOUT",
-    "ECONNREFUSED", "WSAECONNREFUSED",
-    "ECONNRESET", "WSAECONNRESET",
-    "ECONNABORTED", "WSAECONNABORTED",
-    "ENETRESET", "WSAENETRESET",
-    "EHOSTDOWN", "EHOSTUNREACH",
-)
-socket_errors_to_ignore.append("timed out")
-socket_errors_to_ignore.append("The read operation timed out")
-if sys.platform == 'darwin':
-    socket_errors_to_ignore.append(plat_specific_errors("EPROTOTYPE"))
-
-socket_errors_nonblocking = plat_specific_errors(
-    'EAGAIN', 'EWOULDBLOCK', 'WSAEWOULDBLOCK')
-
-comma_separated_headers = [
-    ntob(h) for h in
-    ['Accept', 'Accept-Charset', 'Accept-Encoding',
-     'Accept-Language', 'Accept-Ranges', 'Allow', 'Cache-Control',
-     'Connection', 'Content-Encoding', 'Content-Language', 'Expect',
-     'If-Match', 'If-None-Match', 'Pragma', 'Proxy-Authenticate', 'TE',
-     'Trailer', 'Transfer-Encoding', 'Upgrade', 'Vary', 'Via', 'Warning',
-     'WWW-Authenticate']
-]
-
-
-if not hasattr(logging, 'statistics'):
-    logging.statistics = {}
-
-
-def read_headers(rfile, hdict=None):
-    """Read headers from the given stream into the given header dict.
-
-    If hdict is None, a new header dict is created. Returns the populated
-    header dict.
-
-    Headers which are repeated are folded together using a comma if their
-    specification so dictates.
-
-    This function raises ValueError when the read bytes violate the HTTP spec.
-    You should probably return "400 Bad Request" if this happens.
-    """
-    if hdict is None:
-        hdict = {}
-
-    while True:
-        line = rfile.readline()
-        if not line:
-            # No more data--illegal end of headers
-            raise ValueError("Illegal end of headers.")
-
-        if line == CRLF:
-            # Normal end of headers
-            break
-        if not line.endswith(CRLF):
-            raise ValueError("HTTP requires CRLF terminators")
-
-        if line[0] in (SPACE, TAB):
-            # It's a continuation line.
-            v = line.strip()
-        else:
-            try:
-                k, v = line.split(COLON, 1)
-            except ValueError:
-                raise ValueError("Illegal header line.")
-            # TODO: what about TE and WWW-Authenticate?
-            k = k.strip().title()
-            v = v.strip()
-            hname = k
-
-        if k in comma_separated_headers:
-            existing = hdict.get(hname)
-            if existing:
-                v = b", ".join((existing, v))
-        hdict[hname] = v
-
-    return hdict
-
-
-class MaxSizeExceeded(Exception):
-    pass
-
-
-class SizeCheckWrapper(object):
-
-    """Wraps a file-like object, raising MaxSizeExceeded if too large."""
-
-    def __init__(self, rfile, maxlen):
-        self.rfile = rfile
-        self.maxlen = maxlen
-        self.bytes_read = 0
-
-    def _check_length(self):
-        if self.maxlen and self.bytes_read > self.maxlen:
-            raise MaxSizeExceeded()
-
-    def read(self, size=None):
-        data = self.rfile.read(size)
-        self.bytes_read += len(data)
-        self._check_length()
-        return data
-
-    def readline(self, size=None):
-        if size is not None:
-            data = self.rfile.readline(size)
-            self.bytes_read += len(data)
-            self._check_length()
-            return data
-
-        # User didn't specify a size ...
-        # We read the line in chunks to make sure it's not a 100MB line !
-        res = []
-        while True:
-            data = self.rfile.readline(256)
-            self.bytes_read += len(data)
-            self._check_length()
-            res.append(data)
-            # See https://github.com/cherrypy/cherrypy/issues/421
-            if len(data) < 256 or data[-1:] == LF:
-                return EMPTY.join(res)
-
-    def readlines(self, sizehint=0):
-        # Shamelessly stolen from StringIO
-        total = 0
-        lines = []
-        line = self.readline()
-        while line:
-            lines.append(line)
-            total += len(line)
-            if 0 < sizehint <= total:
-                break
-            line = self.readline()
-        return lines
-
-    def close(self):
-        self.rfile.close()
-
-    def __iter__(self):
-        return self
-
-    def __next__(self):
-        data = next(self.rfile)
-        self.bytes_read += len(data)
-        self._check_length()
-        return data
-
-    def next(self):
-        data = self.rfile.next()
-        self.bytes_read += len(data)
-        self._check_length()
-        return data
-
-
-class KnownLengthRFile(object):
-
-    """Wraps a file-like object, returning an empty string when exhausted."""
-
-    def __init__(self, rfile, content_length):
-        self.rfile = rfile
-        self.remaining = content_length
-
-    def read(self, size=None):
-        if self.remaining == 0:
-            return b''
-        if size is None:
-            size = self.remaining
-        else:
-            size = min(size, self.remaining)
-
-        data = self.rfile.read(size)
-        self.remaining -= len(data)
-        return data
-
-    def readline(self, size=None):
-        if self.remaining == 0:
-            return b''
-        if size is None:
-            size = self.remaining
-        else:
-            size = min(size, self.remaining)
-
-        data = self.rfile.readline(size)
-        self.remaining -= len(data)
-        return data
-
-    def readlines(self, sizehint=0):
-        # Shamelessly stolen from StringIO
-        total = 0
-        lines = []
-        line = self.readline(sizehint)
-        while line:
-            lines.append(line)
-            total += len(line)
-            if 0 < sizehint <= total:
-                break
-            line = self.readline(sizehint)
-        return lines
-
-    def close(self):
-        self.rfile.close()
-
-    def __iter__(self):
-        return self
-
-    def __next__(self):
-        data = next(self.rfile)
-        self.remaining -= len(data)
-        return data
-
-
-class ChunkedRFile(object):
-
-    """Wraps a file-like object, returning an empty string when exhausted.
-
-    This class is intended to provide a conforming wsgi.input value for
-    request entities that have been encoded with the 'chunked' transfer
-    encoding.
-    """
-
-    def __init__(self, rfile, maxlen, bufsize=8192):
-        self.rfile = rfile
-        self.maxlen = maxlen
-        self.bytes_read = 0
-        self.buffer = EMPTY
-        self.bufsize = bufsize
-        self.closed = False
-
-    def _fetch(self):
-        if self.closed:
-            return
-
-        line = self.rfile.readline()
-        self.bytes_read += len(line)
-
-        if self.maxlen and self.bytes_read > self.maxlen:
-            raise MaxSizeExceeded("Request Entity Too Large", self.maxlen)
-
-        line = line.strip().split(SEMICOLON, 1)
-
-        try:
-            chunk_size = line.pop(0)
-            chunk_size = int(chunk_size, 16)
-        except ValueError:
-            raise ValueError("Bad chunked transfer size: " + repr(chunk_size))
-
-        if chunk_size <= 0:
-            self.closed = True
-            return
-
-##            if line: chunk_extension = line[0]
-
-        if self.maxlen and self.bytes_read + chunk_size > self.maxlen:
-            raise IOError("Request Entity Too Large")
-
-        chunk = self.rfile.read(chunk_size)
-        self.bytes_read += len(chunk)
-        self.buffer += chunk
-
-        crlf = self.rfile.read(2)
-        if crlf != CRLF:
-            raise ValueError(
-                "Bad chunked transfer coding (expected '\\r\\n', "
-                "got " + repr(crlf) + ")")
-
-    def read(self, size=None):
-        data = EMPTY
-        while True:
-            if size and len(data) >= size:
-                return data
-
-            if not self.buffer:
-                self._fetch()
-                if not self.buffer:
-                    # EOF
-                    return data
-
-            if size:
-                remaining = size - len(data)
-                data += self.buffer[:remaining]
-                self.buffer = self.buffer[remaining:]
-            else:
-                data += self.buffer
-
-    def readline(self, size=None):
-        data = EMPTY
-        while True:
-            if size and len(data) >= size:
-                return data
-
-            if not self.buffer:
-                self._fetch()
-                if not self.buffer:
-                    # EOF
-                    return data
-
-            newline_pos = self.buffer.find(LF)
-            if size:
-                if newline_pos == -1:
-                    remaining = size - len(data)
-                    data += self.buffer[:remaining]
-                    self.buffer = self.buffer[remaining:]
-                else:
-                    remaining = min(size - len(data), newline_pos)
-                    data += self.buffer[:remaining]
-                    self.buffer = self.buffer[remaining:]
-            else:
-                if newline_pos == -1:
-                    data += self.buffer
-                else:
-                    data += self.buffer[:newline_pos]
-                    self.buffer = self.buffer[newline_pos:]
-
-    def readlines(self, sizehint=0):
-        # Shamelessly stolen from StringIO
-        total = 0
-        lines = []
-        line = self.readline(sizehint)
-        while line:
-            lines.append(line)
-            total += len(line)
-            if 0 < sizehint <= total:
-                break
-            line = self.readline(sizehint)
-        return lines
-
-    def read_trailer_lines(self):
-        if not self.closed:
-            raise ValueError(
-                "Cannot read trailers until the request body has been read.")
-
-        while True:
-            line = self.rfile.readline()
-            if not line:
-                # No more data--illegal end of headers
-                raise ValueError("Illegal end of headers.")
-
-            self.bytes_read += len(line)
-            if self.maxlen and self.bytes_read > self.maxlen:
-                raise IOError("Request Entity Too Large")
-
-            if line == CRLF:
-                # Normal end of headers
-                break
-            if not line.endswith(CRLF):
-                raise ValueError("HTTP requires CRLF terminators")
-
-            yield line
-
-    def close(self):
-        self.rfile.close()
-
-    def __iter__(self):
-        # Shamelessly stolen from StringIO
-        total = 0
-        line = self.readline(sizehint)
-        while line:
-            yield line
-            total += len(line)
-            if 0 < sizehint <= total:
-                break
-            line = self.readline(sizehint)
-
-
-class HTTPRequest(object):
-
-    """An HTTP Request (and response).
-
-    A single HTTP connection may consist of multiple request/response pairs.
-    """
-
-    server = None
-    """The HTTPServer object which is receiving this request."""
-
-    conn = None
-    """The HTTPConnection object on which this request connected."""
-
-    inheaders = {}
-    """A dict of request headers."""
-
-    outheaders = []
-    """A list of header tuples to write in the response."""
-
-    ready = False
-    """When True, the request has been parsed and is ready to begin generating
-    the response. When False, signals the calling Connection that the response
-    should not be generated and the connection should close."""
-
-    close_connection = False
-    """Signals the calling Connection that the request should close. This does
-    not imply an error! The client and/or server may each request that the
-    connection be closed."""
-
-    chunked_write = False
-    """If True, output will be encoded with the "chunked" transfer-coding.
-
-    This value is set automatically inside send_headers."""
-
-    def __init__(self, server, conn):
-        self.server = server
-        self.conn = conn
-
-        self.ready = False
-        self.started_request = False
-        self.scheme = ntob("http")
-        if self.server.ssl_adapter is not None:
-            self.scheme = ntob("https")
-        # Use the lowest-common protocol in case read_request_line errors.
-        self.response_protocol = 'HTTP/1.0'
-        self.inheaders = {}
-
-        self.status = ""
-        self.outheaders = []
-        self.sent_headers = False
-        self.close_connection = self.__class__.close_connection
-        self.chunked_read = False
-        self.chunked_write = self.__class__.chunked_write
-
-    def parse_request(self):
-        """Parse the next HTTP request start-line and message-headers."""
-        self.rfile = SizeCheckWrapper(self.conn.rfile,
-                                      self.server.max_request_header_size)
-        try:
-            success = self.read_request_line()
-        except MaxSizeExceeded:
-            self.simple_response(
-                "414 Request-URI Too Long",
-                "The Request-URI sent with the request exceeds the maximum "
-                "allowed bytes.")
-            return
-        else:
-            if not success:
-                return
-
-        try:
-            success = self.read_request_headers()
-        except MaxSizeExceeded:
-            self.simple_response(
-                "413 Request Entity Too Large",
-                "The headers sent with the request exceed the maximum "
-                "allowed bytes.")
-            return
-        else:
-            if not success:
-                return
-
-        self.ready = True
-
-    def read_request_line(self):
-        # HTTP/1.1 connections are persistent by default. If a client
-        # requests a page, then idles (leaves the connection open),
-        # then rfile.readline() will raise socket.error("timed out").
-        # Note that it does this based on the value given to settimeout(),
-        # and doesn't need the client to request or acknowledge the close
-        # (although your TCP stack might suffer for it: cf Apache's history
-        # with FIN_WAIT_2).
-        request_line = self.rfile.readline()
-
-        # Set started_request to True so communicate() knows to send 408
-        # from here on out.
-        self.started_request = True
-        if not request_line:
-            return False
-
-        if request_line == CRLF:
-            # RFC 2616 sec 4.1: "...if the server is reading the protocol
-            # stream at the beginning of a message and receives a CRLF
-            # first, it should ignore the CRLF."
-            # But only ignore one leading line! else we enable a DoS.
-            request_line = self.rfile.readline()
-            if not request_line:
-                return False
-
-        if not request_line.endswith(CRLF):
-            self.simple_response(
-                "400 Bad Request", "HTTP requires CRLF terminators")
-            return False
-
-        try:
-            method, uri, req_protocol = request_line.strip().split(SPACE, 2)
-            # The [x:y] slicing is necessary for byte strings to avoid getting
-            # ord's
-            rp = int(req_protocol[5:6]), int(req_protocol[7:8])
-        except ValueError:
-            self.simple_response("400 Bad Request", "Malformed Request-Line")
-            return False
-
-        self.uri = uri
-        self.method = method
-
-        # uri may be an abs_path (including "http://host.domain.tld");
-        scheme, authority, path = self.parse_request_uri(uri)
-        if path is None:
-            self.simple_response("400 Bad Request",
-                                 "Invalid path in Request-URI.")
-            return False
-        if NUMBER_SIGN in path:
-            self.simple_response("400 Bad Request",
-                                 "Illegal #fragment in Request-URI.")
-            return False
-
-        if scheme:
-            self.scheme = scheme
-
-        qs = EMPTY
-        if QUESTION_MARK in path:
-            path, qs = path.split(QUESTION_MARK, 1)
-
-        # Unquote the path+params (e.g. "/this%20path" -> "/this path").
-        # http://www.w3.org/Protocols/rfc2616/rfc2616-sec5.html#sec5.1.2
-        #
-        # But note that "...a URI must be separated into its components
-        # before the escaped characters within those components can be
-        # safely decoded." http://www.ietf.org/rfc/rfc2396.txt, sec 2.4.2
-        # Therefore, "/this%2Fpath" becomes "/this%2Fpath", not "/this/path".
-        try:
-            atoms = [self.unquote_bytes(x) for x in quoted_slash.split(path)]
-        except ValueError:
-            ex = sys.exc_info()[1]
-            self.simple_response("400 Bad Request", ex.args[0])
-            return False
-        path = b"%2F".join(atoms)
-        self.path = path
-
-        # Note that, like wsgiref and most other HTTP servers,
-        # we "% HEX HEX"-unquote the path but not the query string.
-        self.qs = qs
-
-        # Compare request and server HTTP protocol versions, in case our
-        # server does not support the requested protocol. Limit our output
-        # to min(req, server). We want the following output:
-        #     request    server     actual written   supported response
-        #     protocol   protocol  response protocol    feature set
-        # a     1.0        1.0           1.0                1.0
-        # b     1.0        1.1           1.1                1.0
-        # c     1.1        1.0           1.0                1.0
-        # d     1.1        1.1           1.1                1.1
-        # Notice that, in (b), the response will be "HTTP/1.1" even though
-        # the client only understands 1.0. RFC 2616 10.5.6 says we should
-        # only return 505 if the _major_ version is different.
-        # The [x:y] slicing is necessary for byte strings to avoid getting
-        # ord's
-        sp = int(self.server.protocol[5:6]), int(self.server.protocol[7:8])
-
-        if sp[0] != rp[0]:
-            self.simple_response("505 HTTP Version Not Supported")
-            return False
-
-        self.request_protocol = req_protocol
-        self.response_protocol = "HTTP/%s.%s" % min(rp, sp)
-        return True
-
-    def read_request_headers(self):
-        """Read self.rfile into self.inheaders. Return success."""
-
-        # then all the http headers
-        try:
-            read_headers(self.rfile, self.inheaders)
-        except ValueError:
-            ex = sys.exc_info()[1]
-            self.simple_response("400 Bad Request", ex.args[0])
-            return False
-
-        mrbs = self.server.max_request_body_size
-        if mrbs and int(self.inheaders.get(b"Content-Length", 0)) > mrbs:
-            self.simple_response(
-                "413 Request Entity Too Large",
-                "The entity sent with the request exceeds the maximum "
-                "allowed bytes.")
-            return False
-
-        # Persistent connection support
-        if self.response_protocol == "HTTP/1.1":
-            # Both server and client are HTTP/1.1
-            if self.inheaders.get(b"Connection", b"") == b"close":
-                self.close_connection = True
-        else:
-            # Either the server or client (or both) are HTTP/1.0
-            if self.inheaders.get(b"Connection", b"") != b"Keep-Alive":
-                self.close_connection = True
-
-        # Transfer-Encoding support
-        te = None
-        if self.response_protocol == "HTTP/1.1":
-            te = self.inheaders.get(b"Transfer-Encoding")
-            if te:
-                te = [x.strip().lower() for x in te.split(b",") if x.strip()]
-
-        self.chunked_read = False
-
-        if te:
-            for enc in te:
-                if enc == b"chunked":
-                    self.chunked_read = True
-                else:
-                    # Note that, even if we see "chunked", we must reject
-                    # if there is an extension we don't recognize.
-                    self.simple_response("501 Unimplemented")
-                    self.close_connection = True
-                    return False
-
-        # From PEP 333:
-        # "Servers and gateways that implement HTTP 1.1 must provide
-        # transparent support for HTTP 1.1's "expect/continue" mechanism.
-        # This may be done in any of several ways:
-        #   1. Respond to requests containing an Expect: 100-continue request
-        #      with an immediate "100 Continue" response, and proceed normally.
-        #   2. Proceed with the request normally, but provide the application
-        #      with a wsgi.input stream that will send the "100 Continue"
-        #      response if/when the application first attempts to read from
-        #      the input stream. The read request must then remain blocked
-        #      until the client responds.
-        #   3. Wait until the client decides that the server does not support
-        #      expect/continue, and sends the request body on its own.
-        #      (This is suboptimal, and is not recommended.)
-        #
-        # We used to do 3, but are now doing 1. Maybe we'll do 2 someday,
-        # but it seems like it would be a big slowdown for such a rare case.
-        if self.inheaders.get(b"Expect", b"") == b"100-continue":
-            # Don't use simple_response here, because it emits headers
-            # we don't want. See
-            # https://github.com/cherrypy/cherrypy/issues/951
-            msg = self.server.protocol.encode(
-                'ascii') + b" 100 Continue\r\n\r\n"
-            try:
-                self.conn.wfile.write(msg)
-            except socket.error:
-                x = sys.exc_info()[1]
-                if x.args[0] not in socket_errors_to_ignore:
-                    raise
-        return True
-
-    def parse_request_uri(self, uri):
-        """Parse a Request-URI into (scheme, authority, path).
-
-        Note that Request-URI's must be one of::
-
-            Request-URI    = "*" | absoluteURI | abs_path | authority
-
-        Therefore, a Request-URI which starts with a double forward-slash
-        cannot be a "net_path"::
-
-            net_path      = "//" authority [ abs_path ]
-
-        Instead, it must be interpreted as an "abs_path" with an empty first
-        path segment::
-
-            abs_path      = "/"  path_segments
-            path_segments = segment *( "/" segment )
-            segment       = *pchar *( ";" param )
-            param         = *pchar
-        """
-        if uri == ASTERISK:
-            return None, None, uri
-
-        scheme, authority, path, params, query, fragment = urlparse(uri)
-        if scheme and QUESTION_MARK not in scheme:
-            # An absoluteURI.
-            # If there's a scheme (and it must be http or https), then:
-            # http_URL = "http:" "//" host [ ":" port ] [ abs_path [ "?" query
-            # ]]
-            return scheme, authority, path
-
-        if uri.startswith(FORWARD_SLASH):
-            # An abs_path.
-            return None, None, uri
-        else:
-            # An authority.
-            return None, uri, None
-
-    def unquote_bytes(self, path):
-        """takes quoted string and unquotes % encoded values"""
-        res = path.split(b'%')
-
-        for i in range(1, len(res)):
-            item = res[i]
-            try:
-                res[i] = bytes([int(item[:2], 16)]) + item[2:]
-            except ValueError:
-                raise
-        return b''.join(res)
-
-    def respond(self):
-        """Call the gateway and write its iterable output."""
-        mrbs = self.server.max_request_body_size
-        if self.chunked_read:
-            self.rfile = ChunkedRFile(self.conn.rfile, mrbs)
-        else:
-            cl = int(self.inheaders.get(b"Content-Length", 0))
-            if mrbs and mrbs < cl:
-                if not self.sent_headers:
-                    self.simple_response(
-                        "413 Request Entity Too Large",
-                        "The entity sent with the request exceeds the "
-                        "maximum allowed bytes.")
-                return
-            self.rfile = KnownLengthRFile(self.conn.rfile, cl)
-
-        self.server.gateway(self).respond()
-
-        if (self.ready and not self.sent_headers):
-            self.sent_headers = True
-            self.send_headers()
-        if self.chunked_write:
-            self.conn.wfile.write(b"0\r\n\r\n")
-
-    def simple_response(self, status, msg=""):
-        """Write a simple response back to the client."""
-        status = str(status)
-        buf = [bytes(self.server.protocol, "ascii") + SPACE +
-               bytes(status, "ISO-8859-1") + CRLF,
-               bytes("Content-Length: %s\r\n" % len(msg), "ISO-8859-1"),
-               b"Content-Type: text/plain\r\n"]
-
-        if status[:3] in ("413", "414"):
-            # Request Entity Too Large / Request-URI Too Long
-            self.close_connection = True
-            if self.response_protocol == 'HTTP/1.1':
-                # This will not be true for 414, since read_request_line
-                # usually raises 414 before reading the whole line, and we
-                # therefore cannot know the proper response_protocol.
-                buf.append(b"Connection: close\r\n")
-            else:
-                # HTTP/1.0 had no 413/414 status nor Connection header.
-                # Emit 400 instead and trust the message body is enough.
-                status = "400 Bad Request"
-
-        buf.append(CRLF)
-        if msg:
-            if isinstance(msg, unicodestr):
-                msg = msg.encode("ISO-8859-1")
-            buf.append(msg)
-
-        try:
-            self.conn.wfile.write(b"".join(buf))
-        except socket.error:
-            x = sys.exc_info()[1]
-            if x.args[0] not in socket_errors_to_ignore:
-                raise
-
-    def write(self, chunk):
-        """Write unbuffered data to the client."""
-        if self.chunked_write and chunk:
-            buf = [bytes(hex(len(chunk)), 'ASCII')[2:], CRLF, chunk, CRLF]
-            self.conn.wfile.write(EMPTY.join(buf))
-        else:
-            self.conn.wfile.write(chunk)
-
-    def send_headers(self):
-        """Assert, process, and send the HTTP response message-headers.
-
-        You must set self.status, and self.outheaders before calling this.
-        """
-        hkeys = [key.lower() for key, value in self.outheaders]
-        status = int(self.status[:3])
-
-        if status == 413:
-            # Request Entity Too Large. Close conn to avoid garbage.
-            self.close_connection = True
-        elif b"content-length" not in hkeys:
-            # "All 1xx (informational), 204 (no content),
-            # and 304 (not modified) responses MUST NOT
-            # include a message-body." So no point chunking.
-            if status < 200 or status in (204, 205, 304):
-                pass
-            else:
-                if (self.response_protocol == 'HTTP/1.1'
-                        and self.method != b'HEAD'):
-                    # Use the chunked transfer-coding
-                    self.chunked_write = True
-                    self.outheaders.append((b"Transfer-Encoding", b"chunked"))
-                else:
-                    # Closing the conn is the only way to determine len.
-                    self.close_connection = True
-
-        if b"connection" not in hkeys:
-            if self.response_protocol == 'HTTP/1.1':
-                # Both server and client are HTTP/1.1 or better
-                if self.close_connection:
-                    self.outheaders.append((b"Connection", b"close"))
-            else:
-                # Server and/or client are HTTP/1.0
-                if not self.close_connection:
-                    self.outheaders.append((b"Connection", b"Keep-Alive"))
-
-        if (not self.close_connection) and (not self.chunked_read):
-            # Read any remaining request body data on the socket.
-            # "If an origin server receives a request that does not include an
-            # Expect request-header field with the "100-continue" expectation,
-            # the request includes a request body, and the server responds
-            # with a final status code before reading the entire request body
-            # from the transport connection, then the server SHOULD NOT close
-            # the transport connection until it has read the entire request,
-            # or until the client closes the connection. Otherwise, the client
-            # might not reliably receive the response message. However, this
-            # requirement is not be construed as preventing a server from
-            # defending itself against denial-of-service attacks, or from
-            # badly broken client implementations."
-            remaining = getattr(self.rfile, 'remaining', 0)
-            if remaining > 0:
-                self.rfile.read(remaining)
-
-        if b"date" not in hkeys:
-            self.outheaders.append((
-                b"Date",
-                email.utils.formatdate(usegmt=True).encode('ISO-8859-1')
-            ))
-
-        if b"server" not in hkeys:
-            self.outheaders.append(
-                (b"Server", self.server.server_name.encode('ISO-8859-1')))
-
-        buf = [self.server.protocol.encode(
-            'ascii') + SPACE + self.status + CRLF]
-        for k, v in self.outheaders:
-            buf.append(k + COLON + SPACE + v + CRLF)
-        buf.append(CRLF)
-        self.conn.wfile.write(EMPTY.join(buf))
-
-
-class NoSSLError(Exception):
-
-    """Exception raised when a client speaks HTTP to an HTTPS socket."""
-    pass
-
-
-class FatalSSLAlert(Exception):
-
-    """Exception raised when the SSL implementation signals a fatal alert."""
-    pass
-
-
-class CP_BufferedWriter(io.BufferedWriter):
-
-    """Faux file object attached to a socket object."""
-
-    def write(self, b):
-        self._checkClosed()
-        if isinstance(b, str):
-            raise TypeError("can't write str to binary stream")
-
-        with self._write_lock:
-            self._write_buf.extend(b)
-            self._flush_unlocked()
-            return len(b)
-
-    def _flush_unlocked(self):
-        self._checkClosed("flush of closed file")
-        while self._write_buf:
-            try:
-                # ssl sockets only except 'bytes', not bytearrays
-                # so perhaps we should conditionally wrap this for perf?
-                n = self.raw.write(bytes(self._write_buf))
-            except io.BlockingIOError as e:
-                n = e.characters_written
-            del self._write_buf[:n]
-
-
-def CP_makefile(sock, mode='r', bufsize=DEFAULT_BUFFER_SIZE):
-    if 'r' in mode:
-        return io.BufferedReader(socket.SocketIO(sock, mode), bufsize)
-    else:
-        return CP_BufferedWriter(socket.SocketIO(sock, mode), bufsize)
-
-
-class HTTPConnection(object):
-
-    """An HTTP connection (active socket).
-
-    server: the Server object which received this connection.
-    socket: the raw socket object (usually TCP) for this connection.
-    makefile: a fileobject class for reading from the socket.
-    """
-
-    remote_addr = None
-    remote_port = None
-    ssl_env = None
-    rbufsize = DEFAULT_BUFFER_SIZE
-    wbufsize = DEFAULT_BUFFER_SIZE
-    RequestHandlerClass = HTTPRequest
-
-    def __init__(self, server, sock, makefile=CP_makefile):
-        self.server = server
-        self.socket = sock
-        self.rfile = makefile(sock, "rb", self.rbufsize)
-        self.wfile = makefile(sock, "wb", self.wbufsize)
-        self.requests_seen = 0
-
-    def communicate(self):
-        """Read each request and respond appropriately."""
-        request_seen = False
-        try:
-            while True:
-                # (re)set req to None so that if something goes wrong in
-                # the RequestHandlerClass constructor, the error doesn't
-                # get written to the previous request.
-                req = None
-                req = self.RequestHandlerClass(self.server, self)
-
-                # This order of operations should guarantee correct pipelining.
-                req.parse_request()
-                if self.server.stats['Enabled']:
-                    self.requests_seen += 1
-                if not req.ready:
-                    # Something went wrong in the parsing (and the server has
-                    # probably already made a simple_response). Return and
-                    # let the conn close.
-                    return
-
-                request_seen = True
-                req.respond()
-                if req.close_connection:
-                    return
-        except socket.error:
-            e = sys.exc_info()[1]
-            errnum = e.args[0]
-            # sadly SSL sockets return a different (longer) time out string
-            if (
-                errnum == 'timed out' or
-                errnum == 'The read operation timed out'
-            ):
-                # Don't error if we're between requests; only error
-                # if 1) no request has been started at all, or 2) we're
-                # in the middle of a request.
-                # See https://github.com/cherrypy/cherrypy/issues/853
-                if (not request_seen) or (req and req.started_request):
-                    # Don't bother writing the 408 if the response
-                    # has already started being written.
-                    if req and not req.sent_headers:
-                        try:
-                            req.simple_response("408 Request Timeout")
-                        except FatalSSLAlert:
-                            # Close the connection.
-                            return
-            elif errnum not in socket_errors_to_ignore:
-                self.server.error_log("socket.error %s" % repr(errnum),
-                                      level=logging.WARNING, traceback=True)
-                if req and not req.sent_headers:
-                    try:
-                        req.simple_response("500 Internal Server Error")
-                    except FatalSSLAlert:
-                        # Close the connection.
-                        return
-            return
-        except (KeyboardInterrupt, SystemExit):
-            raise
-        except FatalSSLAlert:
-            # Close the connection.
-            return
-        except NoSSLError:
-            if req and not req.sent_headers:
-                # Unwrap our wfile
-                self.wfile = CP_makefile(
-                    self.socket._sock, "wb", self.wbufsize)
-                req.simple_response(
-                    "400 Bad Request",
-                    "The client sent a plain HTTP request, but this server "
-                    "only speaks HTTPS on this port.")
-                self.linger = True
-        except Exception:
-            e = sys.exc_info()[1]
-            self.server.error_log(repr(e), level=logging.ERROR, traceback=True)
-            if req and not req.sent_headers:
-                try:
-                    req.simple_response("500 Internal Server Error")
-                except FatalSSLAlert:
-                    # Close the connection.
-                    return
-
-    linger = False
-
-    def close(self):
-        """Close the socket underlying this connection."""
-        self.rfile.close()
-
-        if not self.linger:
-            # Python's socket module does NOT call close on the kernel
-            # socket when you call socket.close(). We do so manually here
-            # because we want this server to send a FIN TCP segment
-            # immediately. Note this must be called *before* calling
-            # socket.close(), because the latter drops its reference to
-            # the kernel socket.
-            # Python 3 *probably* fixed this with socket._real_close;
-            # hard to tell.
-# self.socket._sock.close()
-            self.socket.close()
-        else:
-            # On the other hand, sometimes we want to hang around for a bit
-            # to make sure the client has a chance to read our entire
-            # response. Skipping the close() calls here delays the FIN
-            # packet until the socket object is garbage-collected later.
-            # Someday, perhaps, we'll do the full lingering_close that
-            # Apache does, but not today.
-            pass
-
-
-class TrueyZero(object):
-
-    """An object which equals and does math like the integer 0 but evals True.
-    """
-
-    def __add__(self, other):
-        return other
-
-    def __radd__(self, other):
-        return other
-trueyzero = TrueyZero()
-
-
-_SHUTDOWNREQUEST = None
-
-
-class WorkerThread(threading.Thread):
-
-    """Thread which continuously polls a Queue for Connection objects.
-
-    Due to the timing issues of polling a Queue, a WorkerThread does not
-    check its own 'ready' flag after it has started. To stop the thread,
-    it is necessary to stick a _SHUTDOWNREQUEST object onto the Queue
-    (one for each running WorkerThread).
-    """
-
-    conn = None
-    """The current connection pulled off the Queue, or None."""
-
-    server = None
-    """The HTTP Server which spawned this thread, and which owns the
-    Queue and is placing active connections into it."""
-
-    ready = False
-    """A simple flag for the calling server to know when this thread
-    has begun polling the Queue."""
-
-    def __init__(self, server):
-        self.ready = False
-        self.server = server
-
-        self.requests_seen = 0
-        self.bytes_read = 0
-        self.bytes_written = 0
-        self.start_time = None
-        self.work_time = 0
-        self.stats = {
-            'Requests': lambda s: self.requests_seen + (
-                (self.start_time is None) and
-                trueyzero or
-                self.conn.requests_seen
-            ),
-            'Bytes Read': lambda s: self.bytes_read + (
-                (self.start_time is None) and
-                trueyzero or
-                self.conn.rfile.bytes_read
-            ),
-            'Bytes Written': lambda s: self.bytes_written + (
-                (self.start_time is None) and
-                trueyzero or
-                self.conn.wfile.bytes_written
-            ),
-            'Work Time': lambda s: self.work_time + (
-                (self.start_time is None) and
-                trueyzero or
-                time.time() - self.start_time
-            ),
-            'Read Throughput': lambda s: s['Bytes Read'](s) / (
-                s['Work Time'](s) or 1e-6),
-            'Write Throughput': lambda s: s['Bytes Written'](s) / (
-                s['Work Time'](s) or 1e-6),
-        }
-        threading.Thread.__init__(self)
-
-    def run(self):
-        self.server.stats['Worker Threads'][self.getName()] = self.stats
-        try:
-            self.ready = True
-            while True:
-                conn = self.server.requests.get()
-                if conn is _SHUTDOWNREQUEST:
-                    return
-
-                self.conn = conn
-                if self.server.stats['Enabled']:
-                    self.start_time = time.time()
-                try:
-                    conn.communicate()
-                finally:
-                    conn.close()
-                    if self.server.stats['Enabled']:
-                        self.requests_seen += self.conn.requests_seen
-                        self.bytes_read += self.conn.rfile.bytes_read
-                        self.bytes_written += self.conn.wfile.bytes_written
-                        self.work_time += time.time() - self.start_time
-                        self.start_time = None
-                    self.conn = None
-        except (KeyboardInterrupt, SystemExit):
-            exc = sys.exc_info()[1]
-            self.server.interrupt = exc
-
-
-class ThreadPool(object):
-
-    """A Request Queue for an HTTPServer which pools threads.
-
-    ThreadPool objects must provide min, get(), put(obj), start()
-    and stop(timeout) attributes.
-    """
-
-    def __init__(self, server, min=10, max=-1,
-        accepted_queue_size=-1, accepted_queue_timeout=10):
-        self.server = server
-        self.min = min
-        self.max = max
-        self._threads = []
-        self._queue = queue.Queue(maxsize=accepted_queue_size)
-        self._queue_put_timeout = accepted_queue_timeout
-        self.get = self._queue.get
-
-    def start(self):
-        """Start the pool of threads."""
-        for i in range(self.min):
-            self._threads.append(WorkerThread(self.server))
-        for worker in self._threads:
-            worker.setName("CP Server " + worker.getName())
-            worker.start()
-        for worker in self._threads:
-            while not worker.ready:
-                time.sleep(.1)
-
-    def _get_idle(self):
-        """Number of worker threads which are idle. Read-only."""
-        return len([t for t in self._threads if t.conn is None])
-    idle = property(_get_idle, doc=_get_idle.__doc__)
-
-    def put(self, obj):
-        self._queue.put(obj, block=True, timeout=self._queue_put_timeout)
-        if obj is _SHUTDOWNREQUEST:
-            return
-
-    def grow(self, amount):
-        """Spawn new worker threads (not above self.max)."""
-        if self.max > 0:
-            budget = max(self.max - len(self._threads), 0)
-        else:
-            # self.max <= 0 indicates no maximum
-            budget = float('inf')
-
-        n_new = min(amount, budget)
-
-        workers = [self._spawn_worker() for i in range(n_new)]
-        while not all(worker.ready for worker in workers):
-            time.sleep(.1)
-        self._threads.extend(workers)
-
-    def _spawn_worker(self):
-        worker = WorkerThread(self.server)
-        worker.setName("CP Server " + worker.getName())
-        worker.start()
-        return worker
-
-    def shrink(self, amount):
-        """Kill off worker threads (not below self.min)."""
-        # Grow/shrink the pool if necessary.
-        # Remove any dead threads from our list
-        for t in self._threads:
-            if not t.isAlive():
-                self._threads.remove(t)
-                amount -= 1
-
-        # calculate the number of threads above the minimum
-        n_extra = max(len(self._threads) - self.min, 0)
-
-        # don't remove more than amount
-        n_to_remove = min(amount, n_extra)
-
-        # put shutdown requests on the queue equal to the number of threads
-        # to remove. As each request is processed by a worker, that worker
-        # will terminate and be culled from the list.
-        for n in range(n_to_remove):
-            self._queue.put(_SHUTDOWNREQUEST)
-
-    def stop(self, timeout=5):
-        # Must shut down threads here so the code that calls
-        # this method can know when all threads are stopped.
-        for worker in self._threads:
-            self._queue.put(_SHUTDOWNREQUEST)
-
-        # Don't join currentThread (when stop is called inside a request).
-        current = threading.currentThread()
-        if timeout and timeout >= 0:
-            endtime = time.time() + timeout
-        while self._threads:
-            worker = self._threads.pop()
-            if worker is not current and worker.isAlive():
-                try:
-                    if timeout is None or timeout < 0:
-                        worker.join()
-                    else:
-                        remaining_time = endtime - time.time()
-                        if remaining_time > 0:
-                            worker.join(remaining_time)
-                        if worker.isAlive():
-                            # We exhausted the timeout.
-                            # Forcibly shut down the socket.
-                            c = worker.conn
-                            if c and not c.rfile.closed:
-                                try:
-                                    c.socket.shutdown(socket.SHUT_RD)
-                                except TypeError:
-                                    # pyOpenSSL sockets don't take an arg
-                                    c.socket.shutdown()
-                            worker.join()
-                except (AssertionError,
-                        # Ignore repeated Ctrl-C.
-                        # See
-                        # https://github.com/cherrypy/cherrypy/issues/691.
-                        KeyboardInterrupt):
-                    pass
-
-    def _get_qsize(self):
-        return self._queue.qsize()
-    qsize = property(_get_qsize)
-
-
-try:
-    import fcntl
-except ImportError:
-    try:
-        from ctypes import windll, WinError
-        import ctypes.wintypes
-        _SetHandleInformation = windll.kernel32.SetHandleInformation
-        _SetHandleInformation.argtypes = [
-            ctypes.wintypes.HANDLE,
-            ctypes.wintypes.DWORD,
-            ctypes.wintypes.DWORD,
-        ]
-        _SetHandleInformation.restype = ctypes.wintypes.BOOL
-    except ImportError:
-        def prevent_socket_inheritance(sock):
-            """Dummy function, since neither fcntl nor ctypes are available."""
-            pass
-    else:
-        def prevent_socket_inheritance(sock):
-            """Mark the given socket fd as non-inheritable (Windows)."""
-            if not _SetHandleInformation(sock.fileno(), 1, 0):
-                raise WinError()
-else:
-    def prevent_socket_inheritance(sock):
-        """Mark the given socket fd as non-inheritable (POSIX)."""
-        fd = sock.fileno()
-        old_flags = fcntl.fcntl(fd, fcntl.F_GETFD)
-        fcntl.fcntl(fd, fcntl.F_SETFD, old_flags | fcntl.FD_CLOEXEC)
-
-
-class SSLAdapter(object):
-
-    """Base class for SSL driver library adapters.
-
-    Required methods:
-
-        * ``wrap(sock) -> (wrapped socket, ssl environ dict)``
-        * ``makefile(sock, mode='r', bufsize=DEFAULT_BUFFER_SIZE) ->
-          socket file object``
-    """
-
-    def __init__(self, certificate, private_key, certificate_chain=None):
-        self.certificate = certificate
-        self.private_key = private_key
-        self.certificate_chain = certificate_chain
-
-    def wrap(self, sock):
-        raise NotImplemented
-
-    def makefile(self, sock, mode='r', bufsize=DEFAULT_BUFFER_SIZE):
-        raise NotImplemented
-
-
-class HTTPServer(object):
-
-    """An HTTP server."""
-
-    _bind_addr = "127.0.0.1"
-    _interrupt = None
-
-    gateway = None
-    """A Gateway instance."""
-
-    minthreads = None
-    """The minimum number of worker threads to create (default 10)."""
-
-    maxthreads = None
-    """The maximum number of worker threads to create (default -1 = no limit).
-    """
-
-    server_name = None
-    """The name of the server; defaults to socket.gethostname()."""
-
-    protocol = "HTTP/1.1"
-    """The version string to write in the Status-Line of all HTTP responses.
-
-    For example, "HTTP/1.1" is the default. This also limits the supported
-    features used in the response."""
-
-    request_queue_size = 5
-    """The 'backlog' arg to socket.listen(); max queued connections
-    (default 5).
-    """
-
-    shutdown_timeout = 5
-    """The total time, in seconds, to wait for worker threads to cleanly exit.
-    """
-
-    timeout = 10
-    """The timeout in seconds for accepted connections (default 10)."""
-
-    version = "CherryPy/" + cp_version
-    """A version string for the HTTPServer."""
-
-    software = None
-    """The value to set for the SERVER_SOFTWARE entry in the WSGI environ.
-
-    If None, this defaults to ``'%s Server' % self.version``."""
-
-    ready = False
-    """An internal flag which marks whether the socket is accepting
-    connections.
-    """
-
-    max_request_header_size = 0
-    """The maximum size, in bytes, for request headers, or 0 for no limit."""
-
-    max_request_body_size = 0
-    """The maximum size, in bytes, for request bodies, or 0 for no limit."""
-
-    nodelay = True
-    """If True (the default since 3.1), sets the TCP_NODELAY socket option."""
-
-    ConnectionClass = HTTPConnection
-    """The class to use for handling HTTP connections."""
-
-    ssl_adapter = None
-    """An instance of SSLAdapter (or a subclass).
-
-    You must have the corresponding SSL driver library installed."""
-
-    def __init__(self, bind_addr, gateway, minthreads=10, maxthreads=-1,
-                 server_name=None):
-        self.bind_addr = bind_addr
-        self.gateway = gateway
-
-        self.requests = ThreadPool(self, min=minthreads or 1, max=maxthreads)
-
-        if not server_name:
-            server_name = socket.gethostname()
-        self.server_name = server_name
-        self.clear_stats()
-
-    def clear_stats(self):
-        self._start_time = None
-        self._run_time = 0
-        self.stats = {
-            'Enabled': False,
-            'Bind Address': lambda s: repr(self.bind_addr),
-            'Run time': lambda s: (not s['Enabled']) and -1 or self.runtime(),
-            'Accepts': 0,
-            'Accepts/sec': lambda s: s['Accepts'] / self.runtime(),
-            'Queue': lambda s: getattr(self.requests, "qsize", None),
-            'Threads': lambda s: len(getattr(self.requests, "_threads", [])),
-            'Threads Idle': lambda s: getattr(self.requests, "idle", None),
-            'Socket Errors': 0,
-            'Requests': lambda s: (not s['Enabled']) and -1 or sum(
-                [w['Requests'](w) for w in s['Worker Threads'].values()], 0),
-            'Bytes Read': lambda s: (not s['Enabled']) and -1 or sum(
-                [w['Bytes Read'](w) for w in s['Worker Threads'].values()], 0),
-            'Bytes Written': lambda s: (not s['Enabled']) and -1 or sum(
-                [w['Bytes Written'](w) for w in s['Worker Threads'].values()],
-                0),
-            'Work Time': lambda s: (not s['Enabled']) and -1 or sum(
-                [w['Work Time'](w) for w in s['Worker Threads'].values()], 0),
-            'Read Throughput': lambda s: (not s['Enabled']) and -1 or sum(
-                [w['Bytes Read'](w) / (w['Work Time'](w) or 1e-6)
-                 for w in s['Worker Threads'].values()], 0),
-            'Write Throughput': lambda s: (not s['Enabled']) and -1 or sum(
-                [w['Bytes Written'](w) / (w['Work Time'](w) or 1e-6)
-                 for w in s['Worker Threads'].values()], 0),
-            'Worker Threads': {},
-        }
-        logging.statistics["CherryPy HTTPServer %d" % id(self)] = self.stats
-
-    def runtime(self):
-        if self._start_time is None:
-            return self._run_time
-        else:
-            return self._run_time + (time.time() - self._start_time)
-
-    def __str__(self):
-        return "%s.%s(%r)" % (self.__module__, self.__class__.__name__,
-                              self.bind_addr)
-
-    def _get_bind_addr(self):
-        return self._bind_addr
-
-    def _set_bind_addr(self, value):
-        if isinstance(value, tuple) and value[0] in ('', None):
-            # Despite the socket module docs, using '' does not
-            # allow AI_PASSIVE to work. Passing None instead
-            # returns '0.0.0.0' like we want. In other words:
-            #     host    AI_PASSIVE     result
-            #      ''         Y         192.168.x.y
-            #      ''         N         192.168.x.y
-            #     None        Y         0.0.0.0
-            #     None        N         127.0.0.1
-            # But since you can get the same effect with an explicit
-            # '0.0.0.0', we deny both the empty string and None as values.
-            raise ValueError("Host values of '' or None are not allowed. "
-                             "Use '0.0.0.0' (IPv4) or '::' (IPv6) instead "
-                             "to listen on all active interfaces.")
-        self._bind_addr = value
-    bind_addr = property(
-        _get_bind_addr,
-        _set_bind_addr,
-        doc="""The interface on which to listen for connections.
-
-        For TCP sockets, a (host, port) tuple. Host values may be any IPv4
-        or IPv6 address, or any valid hostname. The string 'localhost' is a
-        synonym for '127.0.0.1' (or '::1', if your hosts file prefers IPv6).
-        The string '0.0.0.0' is a special IPv4 entry meaning "any active
-        interface" (INADDR_ANY), and '::' is the similar IN6ADDR_ANY for
-        IPv6. The empty string or None are not allowed.
-
-        For UNIX sockets, supply the filename as a string.""")
-
-    def start(self):
-        """Run the server forever."""
-        # We don't have to trap KeyboardInterrupt or SystemExit here,
-        # because cherrpy.server already does so, calling self.stop() for us.
-        # If you're using this server with another framework, you should
-        # trap those exceptions in whatever code block calls start().
-        self._interrupt = None
-
-        if self.software is None:
-            self.software = "%s Server" % self.version
-
-        # Select the appropriate socket
-        if isinstance(self.bind_addr, basestring):
-            # AF_UNIX socket
-
-            # So we can reuse the socket...
-            try:
-                os.unlink(self.bind_addr)
-            except:
-                pass
-
-            # So everyone can access the socket...
-            try:
-                os.chmod(self.bind_addr, 0o777)
-            except:
-                pass
-
-            info = [
-                (socket.AF_UNIX, socket.SOCK_STREAM, 0, "", self.bind_addr)]
-        else:
-            # AF_INET or AF_INET6 socket
-            # Get the correct address family for our host (allows IPv6
-            # addresses)
-            host, port = self.bind_addr
-            try:
-                info = socket.getaddrinfo(host, port, socket.AF_UNSPEC,
-                                          socket.SOCK_STREAM, 0,
-                                          socket.AI_PASSIVE)
-            except socket.gaierror:
-                if ':' in self.bind_addr[0]:
-                    info = [(socket.AF_INET6, socket.SOCK_STREAM,
-                             0, "", self.bind_addr + (0, 0))]
-                else:
-                    info = [(socket.AF_INET, socket.SOCK_STREAM,
-                             0, "", self.bind_addr)]
-
-        self.socket = None
-        msg = "No socket could be created"
-        for res in info:
-            af, socktype, proto, canonname, sa = res
-            try:
-                self.bind(af, socktype, proto)
-            except socket.error as serr:
-                msg = "%s -- (%s: %s)" % (msg, sa, serr)
-                if self.socket:
-                    self.socket.close()
-                self.socket = None
-                continue
-            break
-        if not self.socket:
-            raise socket.error(msg)
-
-        # Timeout so KeyboardInterrupt can be caught on Win32
-        self.socket.settimeout(1)
-        self.socket.listen(self.request_queue_size)
-
-        # Create worker threads
-        self.requests.start()
-
-        self.ready = True
-        self._start_time = time.time()
-        while self.ready:
-            try:
-                self.tick()
-            except (KeyboardInterrupt, SystemExit):
-                raise
-            except:
-                self.error_log("Error in HTTPServer.tick", level=logging.ERROR,
-                               traceback=True)
-            if self.interrupt:
-                while self.interrupt is True:
-                    # Wait for self.stop() to complete. See _set_interrupt.
-                    time.sleep(0.1)
-                if self.interrupt:
-                    raise self.interrupt
-
-    def error_log(self, msg="", level=20, traceback=False):
-        # Override this in subclasses as desired
-        sys.stderr.write(msg + '\n')
-        sys.stderr.flush()
-        if traceback:
-            tblines = traceback_.format_exc()
-            sys.stderr.write(tblines)
-            sys.stderr.flush()
-
-    def bind(self, family, type, proto=0):
-        """Create (or recreate) the actual socket object."""
-        self.socket = socket.socket(family, type, proto)
-        prevent_socket_inheritance(self.socket)
-        self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
-        if self.nodelay and not isinstance(self.bind_addr, str):
-            self.socket.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
-
-        if self.ssl_adapter is not None:
-            self.socket = self.ssl_adapter.bind(self.socket)
-
-        # If listening on the IPV6 any address ('::' = IN6ADDR_ANY),
-        # activate dual-stack. See
-        # https://github.com/cherrypy/cherrypy/issues/871.
-        if (hasattr(socket, 'AF_INET6') and family == socket.AF_INET6
-                and self.bind_addr[0] in ('::', '::0', '::0.0.0.0')):
-            try:
-                self.socket.setsockopt(
-                    socket.IPPROTO_IPV6, socket.IPV6_V6ONLY, 0)
-            except (AttributeError, socket.error):
-                # Apparently, the socket option is not available in
-                # this machine's TCP stack
-                pass
-
-        self.socket.bind(self.bind_addr)
-
-    def tick(self):
-        """Accept a new connection and put it on the Queue."""
-        try:
-            s, addr = self.socket.accept()
-            if self.stats['Enabled']:
-                self.stats['Accepts'] += 1
-            if not self.ready:
-                return
-
-            prevent_socket_inheritance(s)
-            if hasattr(s, 'settimeout'):
-                s.settimeout(self.timeout)
-
-            makefile = CP_makefile
-            ssl_env = {}
-            # if ssl cert and key are set, we try to be a secure HTTP server
-            if self.ssl_adapter is not None:
-                try:
-                    s, ssl_env = self.ssl_adapter.wrap(s)
-                except NoSSLError:
-                    msg = ("The client sent a plain HTTP request, but "
-                           "this server only speaks HTTPS on this port.")
-                    buf = ["%s 400 Bad Request\r\n" % self.protocol,
-                           "Content-Length: %s\r\n" % len(msg),
-                           "Content-Type: text/plain\r\n\r\n",
-                           msg]
-
-                    wfile = makefile(s, "wb", DEFAULT_BUFFER_SIZE)
-                    try:
-                        wfile.write("".join(buf).encode('ISO-8859-1'))
-                    except socket.error:
-                        x = sys.exc_info()[1]
-                        if x.args[0] not in socket_errors_to_ignore:
-                            raise
-                    return
-                if not s:
-                    return
-                makefile = self.ssl_adapter.makefile
-                # Re-apply our timeout since we may have a new socket object
-                if hasattr(s, 'settimeout'):
-                    s.settimeout(self.timeout)
-
-            conn = self.ConnectionClass(self, s, makefile)
-
-            if not isinstance(self.bind_addr, basestring):
-                # optional values
-                # Until we do DNS lookups, omit REMOTE_HOST
-                if addr is None:  # sometimes this can happen
-                    # figure out if AF_INET or AF_INET6.
-                    if len(s.getsockname()) == 2:
-                        # AF_INET
-                        addr = ('0.0.0.0', 0)
-                    else:
-                        # AF_INET6
-                        addr = ('::', 0)
-                conn.remote_addr = addr[0]
-                conn.remote_port = addr[1]
-
-            conn.ssl_env = ssl_env
-
-            try:
-                self.requests.put(conn)
-            except queue.Full:
-                # Just drop the conn. TODO: write 503 back?
-                conn.close()
-                return
-        except socket.timeout:
-            # The only reason for the timeout in start() is so we can
-            # notice keyboard interrupts on Win32, which don't interrupt
-            # accept() by default
-            return
-        except socket.error:
-            x = sys.exc_info()[1]
-            if self.stats['Enabled']:
-                self.stats['Socket Errors'] += 1
-            if x.args[0] in socket_error_eintr:
-                # I *think* this is right. EINTR should occur when a signal
-                # is received during the accept() call; all docs say retry
-                # the call, and I *think* I'm reading it right that Python
-                # will then go ahead and poll for and handle the signal
-                # elsewhere. See
-                # https://github.com/cherrypy/cherrypy/issues/707.
-                return
-            if x.args[0] in socket_errors_nonblocking:
-                # Just try again. See
-                # https://github.com/cherrypy/cherrypy/issues/479.
-                return
-            if x.args[0] in socket_errors_to_ignore:
-                # Our socket was closed.
-                # See https://github.com/cherrypy/cherrypy/issues/686.
-                return
-            raise
-
-    def _get_interrupt(self):
-        return self._interrupt
-
-    def _set_interrupt(self, interrupt):
-        self._interrupt = True
-        self.stop()
-        self._interrupt = interrupt
-    interrupt = property(_get_interrupt, _set_interrupt,
-                         doc="Set this to an Exception instance to "
-                             "interrupt the server.")
-
-    def stop(self):
-        """Gracefully shutdown a server that is serving forever."""
-        self.ready = False
-        if self._start_time is not None:
-            self._run_time += (time.time() - self._start_time)
-        self._start_time = None
-
-        sock = getattr(self, "socket", None)
-        if sock:
-            if not isinstance(self.bind_addr, basestring):
-                # Touch our own socket to make accept() return immediately.
-                try:
-                    host, port = sock.getsockname()[:2]
-                except socket.error:
-                    x = sys.exc_info()[1]
-                    if x.args[0] not in socket_errors_to_ignore:
-                        # Changed to use error code and not message
-                        # See
-                        # https://github.com/cherrypy/cherrypy/issues/860.
-                        raise
-                else:
-                    # Note that we're explicitly NOT using AI_PASSIVE,
-                    # here, because we want an actual IP to touch.
-                    # localhost won't work if we've bound to a public IP,
-                    # but it will if we bound to '0.0.0.0' (INADDR_ANY).
-                    for res in socket.getaddrinfo(host, port, socket.AF_UNSPEC,
-                                                  socket.SOCK_STREAM):
-                        af, socktype, proto, canonname, sa = res
-                        s = None
-                        try:
-                            s = socket.socket(af, socktype, proto)
-                            # See
-                            # http://groups.google.com/group/cherrypy-users/
-                            #     browse_frm/thread/bbfe5eb39c904fe0
-                            s.settimeout(1.0)
-                            s.connect((host, port))
-                            s.close()
-                        except socket.error:
-                            if s:
-                                s.close()
-            if hasattr(sock, "close"):
-                sock.close()
-            self.socket = None
-
-        self.requests.stop(self.shutdown_timeout)
-
-
-class Gateway(object):
-
-    """A base class to interface HTTPServer with other systems, such as WSGI.
-    """
-
-    def __init__(self, req):
-        self.req = req
-
-    def respond(self):
-        """Process the current request. Must be overridden in a subclass."""
-        raise NotImplemented
-
-
-# These may either be wsgiserver.SSLAdapter subclasses or the string names
-# of such classes (in which case they will be lazily loaded).
-ssl_adapters = {
-    'builtin': 'cherrypy.wsgiserver.ssl_builtin.BuiltinSSLAdapter',
-    'pyopenssl': 'cherrypy.wsgiserver.ssl_pyopenssl.pyOpenSSLAdapter',
-}
-
-
-def get_ssl_adapter_class(name='builtin'):
-    """Return an SSL adapter class for the given name."""
-    adapter = ssl_adapters[name.lower()]
-    if isinstance(adapter, basestring):
-        last_dot = adapter.rfind(".")
-        attr_name = adapter[last_dot + 1:]
-        mod_path = adapter[:last_dot]
-
-        try:
-            mod = sys.modules[mod_path]
-            if mod is None:
-                raise KeyError()
-        except KeyError:
-            # The last [''] is important.
-            mod = __import__(mod_path, globals(), locals(), [''])
-
-        # Let an AttributeError propagate outward.
-        try:
-            adapter = getattr(mod, attr_name)
-        except AttributeError:
-            raise AttributeError("'%s' object has no attribute '%s'"
-                                 % (mod_path, attr_name))
-
-    return adapter
-
-# ------------------------------- WSGI Stuff -------------------------------- #
-
-
-class CherryPyWSGIServer(HTTPServer):
-
-    """A subclass of HTTPServer which calls a WSGI application."""
-
-    wsgi_version = (1, 0)
-    """The version of WSGI to produce."""
-
-    def __init__(self, bind_addr, wsgi_app, numthreads=10, server_name=None,
-                 max=-1, request_queue_size=5, timeout=10, shutdown_timeout=5,
-                 accepted_queue_size=-1, accepted_queue_timeout=10):
-        self.requests = ThreadPool(self, min=numthreads or 1, max=max,
-            accepted_queue_size=accepted_queue_size,
-            accepted_queue_timeout=accepted_queue_timeout)
-        self.wsgi_app = wsgi_app
-        self.gateway = wsgi_gateways[self.wsgi_version]
-
-        self.bind_addr = bind_addr
-        if not server_name:
-            server_name = socket.gethostname()
-        self.server_name = server_name
-        self.request_queue_size = request_queue_size
-
-        self.timeout = timeout
-        self.shutdown_timeout = shutdown_timeout
-        self.clear_stats()
-
-    def _get_numthreads(self):
-        return self.requests.min
-
-    def _set_numthreads(self, value):
-        self.requests.min = value
-    numthreads = property(_get_numthreads, _set_numthreads)
-
-
-class WSGIGateway(Gateway):
-
-    """A base class to interface HTTPServer with WSGI."""
-
-    def __init__(self, req):
-        self.req = req
-        self.started_response = False
-        self.env = self.get_environ()
-        self.remaining_bytes_out = None
-
-    def get_environ(self):
-        """Return a new environ dict targeting the given wsgi.version"""
-        raise NotImplemented
-
-    def respond(self):
-        """Process the current request."""
-        response = self.req.server.wsgi_app(self.env, self.start_response)
-        try:
-            for chunk in response:
-                # "The start_response callable must not actually transmit
-                # the response headers. Instead, it must store them for the
-                # server or gateway to transmit only after the first
-                # iteration of the application return value that yields
-                # a NON-EMPTY string, or upon the application's first
-                # invocation of the write() callable." (PEP 333)
-                if chunk:
-                    if isinstance(chunk, unicodestr):
-                        chunk = chunk.encode('ISO-8859-1')
-                    self.write(chunk)
-        finally:
-            if hasattr(response, "close"):
-                response.close()
-
-    def start_response(self, status, headers, exc_info=None):
-        """WSGI callable to begin the HTTP response."""
-        # "The application may call start_response more than once,
-        # if and only if the exc_info argument is provided."
-        if self.started_response and not exc_info:
-            raise AssertionError("WSGI start_response called a second "
-                                 "time with no exc_info.")
-        self.started_response = True
-
-        # "if exc_info is provided, and the HTTP headers have already been
-        # sent, start_response must raise an error, and should raise the
-        # exc_info tuple."
-        if self.req.sent_headers:
-            try:
-                raise exc_info[0](exc_info[1]).with_traceback(exc_info[2])
-            finally:
-                exc_info = None
-
-        # According to PEP 3333, when using Python 3, the response status
-        # and headers must be bytes masquerading as unicode; that is, they
-        # must be of type "str" but are restricted to code points in the
-        # "latin-1" set.
-        if not isinstance(status, str):
-            raise TypeError("WSGI response status is not of type str.")
-        self.req.status = status.encode('ISO-8859-1')
-
-        for k, v in headers:
-            if not isinstance(k, str):
-                raise TypeError(
-                    "WSGI response header key %r is not of type str." % k)
-            if not isinstance(v, str):
-                raise TypeError(
-                    "WSGI response header value %r is not of type str." % v)
-            if k.lower() == 'content-length':
-                self.remaining_bytes_out = int(v)
-            self.req.outheaders.append(
-                (k.encode('ISO-8859-1'), v.encode('ISO-8859-1')))
-
-        return self.write
-
-    def write(self, chunk):
-        """WSGI callable to write unbuffered data to the client.
-
-        This method is also used internally by start_response (to write
-        data from the iterable returned by the WSGI application).
-        """
-        if not self.started_response:
-            raise AssertionError("WSGI write called before start_response.")
-
-        chunklen = len(chunk)
-        rbo = self.remaining_bytes_out
-        if rbo is not None and chunklen > rbo:
-            if not self.req.sent_headers:
-                # Whew. We can send a 500 to the client.
-                self.req.simple_response("500 Internal Server Error",
-                                         "The requested resource returned "
-                                         "more bytes than the declared "
-                                         "Content-Length.")
-            else:
-                # Dang. We have probably already sent data. Truncate the chunk
-                # to fit (so the client doesn't hang) and raise an error later.
-                chunk = chunk[:rbo]
-
-        if not self.req.sent_headers:
-            self.req.sent_headers = True
-            self.req.send_headers()
-
-        self.req.write(chunk)
-
-        if rbo is not None:
-            rbo -= chunklen
-            if rbo < 0:
-                raise ValueError(
-                    "Response body exceeds the declared Content-Length.")
-
-
-class WSGIGateway_10(WSGIGateway):
-
-    """A Gateway class to interface HTTPServer with WSGI 1.0.x."""
-
-    def get_environ(self):
-        """Return a new environ dict targeting the given wsgi.version"""
-        req = self.req
-        env = {
-            # set a non-standard environ entry so the WSGI app can know what
-            # the *real* server protocol is (and what features to support).
-            # See http://www.faqs.org/rfcs/rfc2145.html.
-            'ACTUAL_SERVER_PROTOCOL': req.server.protocol,
-            'PATH_INFO': req.path.decode('ISO-8859-1'),
-            'QUERY_STRING': req.qs.decode('ISO-8859-1'),
-            'REMOTE_ADDR': req.conn.remote_addr or '',
-            'REMOTE_PORT': str(req.conn.remote_port or ''),
-            'REQUEST_METHOD': req.method.decode('ISO-8859-1'),
-            'REQUEST_URI': req.uri.decode('ISO-8859-1'),
-            'SCRIPT_NAME': '',
-            'SERVER_NAME': req.server.server_name,
-            # Bah. "SERVER_PROTOCOL" is actually the REQUEST protocol.
-            'SERVER_PROTOCOL': req.request_protocol.decode('ISO-8859-1'),
-            'SERVER_SOFTWARE': req.server.software,
-            'wsgi.errors': sys.stderr,
-            'wsgi.input': req.rfile,
-            'wsgi.multiprocess': False,
-            'wsgi.multithread': True,
-            'wsgi.run_once': False,
-            'wsgi.url_scheme': req.scheme.decode('ISO-8859-1'),
-            'wsgi.version': (1, 0),
-        }
-        if isinstance(req.server.bind_addr, basestring):
-            # AF_UNIX. This isn't really allowed by WSGI, which doesn't
-            # address unix domain sockets. But it's better than nothing.
-            env["SERVER_PORT"] = ""
-        else:
-            env["SERVER_PORT"] = str(req.server.bind_addr[1])
-
-        # Request headers
-        for k, v in req.inheaders.items():
-            k = k.decode('ISO-8859-1').upper().replace("-", "_")
-            env["HTTP_" + k] = v.decode('ISO-8859-1')
-
-        # CONTENT_TYPE/CONTENT_LENGTH
-        ct = env.pop("HTTP_CONTENT_TYPE", None)
-        if ct is not None:
-            env["CONTENT_TYPE"] = ct
-        cl = env.pop("HTTP_CONTENT_LENGTH", None)
-        if cl is not None:
-            env["CONTENT_LENGTH"] = cl
-
-        if req.conn.ssl_env:
-            env.update(req.conn.ssl_env)
-
-        return env
-
-
-class WSGIGateway_u0(WSGIGateway_10):
-
-    """A Gateway class to interface HTTPServer with WSGI u.0.
-
-    WSGI u.0 is an experimental protocol, which uses unicode for keys
-    and values in both Python 2 and Python 3.
-    """
-
-    def get_environ(self):
-        """Return a new environ dict targeting the given wsgi.version"""
-        req = self.req
-        env_10 = WSGIGateway_10.get_environ(self)
-        env = env_10.copy()
-        env['wsgi.version'] = ('u', 0)
-
-        # Request-URI
-        env.setdefault('wsgi.url_encoding', 'utf-8')
-        try:
-            # SCRIPT_NAME is the empty string, who cares what encoding it is?
-            env["PATH_INFO"] = req.path.decode(env['wsgi.url_encoding'])
-            env["QUERY_STRING"] = req.qs.decode(env['wsgi.url_encoding'])
-        except UnicodeDecodeError:
-            # Fall back to latin 1 so apps can transcode if needed.
-            env['wsgi.url_encoding'] = 'ISO-8859-1'
-            env["PATH_INFO"] = env_10["PATH_INFO"]
-            env["QUERY_STRING"] = env_10["QUERY_STRING"]
-
-        return env
-
-wsgi_gateways = {
-    (1, 0): WSGIGateway_10,
-    ('u', 0): WSGIGateway_u0,
-}
-
-
-class WSGIPathInfoDispatcher(object):
-
-    """A WSGI dispatcher for dispatch based on the PATH_INFO.
-
-    apps: a dict or list of (path_prefix, app) pairs.
-    """
-
-    def __init__(self, apps):
-        try:
-            apps = list(apps.items())
-        except AttributeError:
-            pass
-
-        # Sort the apps by len(path), descending
-        apps.sort()
-        apps.reverse()
-
-        # The path_prefix strings must start, but not end, with a slash.
-        # Use "" instead of "/".
-        self.apps = [(p.rstrip("/"), a) for p, a in apps]
-
-    def __call__(self, environ, start_response):
-        path = environ["PATH_INFO"] or "/"
-        for p, app in self.apps:
-            # The apps list should be sorted by length, descending.
-            if path.startswith(p + "/") or path == p:
-                environ = environ.copy()
-                environ["SCRIPT_NAME"] = environ["SCRIPT_NAME"] + p
-                environ["PATH_INFO"] = path[len(p):]
-                return app(environ, start_response)
-
-        start_response('404 Not Found', [('Content-Type', 'text/plain'),
-                                         ('Content-Length', '0')])
-        return ['']
diff --git a/lib/cylc/__init__.py b/lib/cylc/__init__.py
index cc497dbdf81..eb2bfd9a310 100644
--- a/lib/cylc/__init__.py
+++ b/lib/cylc/__init__.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
 
 # THIS FILE IS PART OF THE CYLC SUITE ENGINE.
 # Copyright (C) 2008-2019 NIWA & British Crown (Met Office) & Contributors.
diff --git a/lib/cylc/batch_sys_handlers/__init__.py b/lib/cylc/batch_sys_handlers/__init__.py
index 570ea00f8a3..eec7f45c749 100644
--- a/lib/cylc/batch_sys_handlers/__init__.py
+++ b/lib/cylc/batch_sys_handlers/__init__.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
 
 # THIS FILE IS PART OF THE CYLC SUITE ENGINE.
 # Copyright (C) 2008-2019 NIWA & British Crown (Met Office) & Contributors.
diff --git a/lib/cylc/batch_sys_handlers/at.py b/lib/cylc/batch_sys_handlers/at.py
index ebd1e81bc79..b7bdd85d946 100644
--- a/lib/cylc/batch_sys_handlers/at.py
+++ b/lib/cylc/batch_sys_handlers/at.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
 
 # THIS FILE IS PART OF THE CYLC SUITE ENGINE.
 # Copyright (C) 2008-2019 NIWA & British Crown (Met Office) & Contributors.
diff --git a/lib/cylc/batch_sys_handlers/background.py b/lib/cylc/batch_sys_handlers/background.py
index f36f95f9946..f9a61fcf4d9 100644
--- a/lib/cylc/batch_sys_handlers/background.py
+++ b/lib/cylc/batch_sys_handlers/background.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
 
 # THIS FILE IS PART OF THE CYLC SUITE ENGINE.
 # Copyright (C) 2008-2019 NIWA & British Crown (Met Office) & Contributors.
diff --git a/lib/cylc/batch_sys_handlers/loadleveler.py b/lib/cylc/batch_sys_handlers/loadleveler.py
index 9d5aa9f7c2b..4ee4214410b 100644
--- a/lib/cylc/batch_sys_handlers/loadleveler.py
+++ b/lib/cylc/batch_sys_handlers/loadleveler.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
 
 # THIS FILE IS PART OF THE CYLC SUITE ENGINE.
 # Copyright (C) 2008-2019 NIWA & British Crown (Met Office) & Contributors.
@@ -55,7 +55,7 @@ def format_directives(self, job_conf):
             directives["wall_clock_limit"] = "%d,%d" % (
                 job_conf["execution_time_limit"] + 60,
                 job_conf["execution_time_limit"])
-        for key, value in job_conf["directives"].items():
+        for key, value in list(job_conf["directives"].items()):
             directives[key] = value
         lines = []
         for key, value in directives.items():
diff --git a/lib/cylc/batch_sys_handlers/lsf.py b/lib/cylc/batch_sys_handlers/lsf.py
index e20358d2909..bba62c53044 100644
--- a/lib/cylc/batch_sys_handlers/lsf.py
+++ b/lib/cylc/batch_sys_handlers/lsf.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
 
 # THIS FILE IS PART OF THE CYLC SUITE ENGINE.
 # Copyright (C) 2008-2019 NIWA & British Crown (Met Office) & Contributors.
@@ -47,7 +47,7 @@ def format_directives(cls, job_conf):
         if (job_conf["execution_time_limit"] and
                 directives.get("-W") is None):
             directives["-W"] = "%d" % (job_conf["execution_time_limit"] / 60)
-        for key, value in job_conf["directives"].items():
+        for key, value in list(job_conf["directives"].items()):
             directives[key] = value
         lines = []
         for key, value in directives.items():
diff --git a/lib/cylc/batch_sys_handlers/moab.py b/lib/cylc/batch_sys_handlers/moab.py
index 9947a1f569b..a1644af23c5 100644
--- a/lib/cylc/batch_sys_handlers/moab.py
+++ b/lib/cylc/batch_sys_handlers/moab.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
 
 # THIS FILE IS PART OF THE CYLC SUITE ENGINE.
 # Copyright (C) 2008-2019 NIWA & British Crown (Met Office) & Contributors.
diff --git a/lib/cylc/batch_sys_handlers/pbs.py b/lib/cylc/batch_sys_handlers/pbs.py
index 9262f40a28d..7c7d25b2a41 100644
--- a/lib/cylc/batch_sys_handlers/pbs.py
+++ b/lib/cylc/batch_sys_handlers/pbs.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
 
 # THIS FILE IS PART OF THE CYLC SUITE ENGINE.
 # Copyright (C) 2008-2019 NIWA & British Crown (Met Office) & Contributors.
@@ -59,7 +59,7 @@ def format_directives(self, job_conf):
         if (job_conf["execution_time_limit"] and
                 directives.get("-l walltime") is None):
             directives["-l walltime"] = "%d" % job_conf["execution_time_limit"]
-        for key, value in job_conf["directives"].items():
+        for key, value in list(job_conf["directives"].items()):
             directives[key] = value
         lines = []
         for key, value in directives.items():
diff --git a/lib/cylc/batch_sys_handlers/pbs_multi_cluster.py b/lib/cylc/batch_sys_handlers/pbs_multi_cluster.py
index b2a6eee21d6..147e1b5fe6a 100644
--- a/lib/cylc/batch_sys_handlers/pbs_multi_cluster.py
+++ b/lib/cylc/batch_sys_handlers/pbs_multi_cluster.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
 
 # THIS FILE IS PART OF THE CYLC SUITE ENGINE.
 # Copyright (C) 2008-2019 NIWA & British Crown (Met Office) & Contributors.
diff --git a/lib/cylc/batch_sys_handlers/sge.py b/lib/cylc/batch_sys_handlers/sge.py
index 021b89ec4d5..bcdb7877f56 100644
--- a/lib/cylc/batch_sys_handlers/sge.py
+++ b/lib/cylc/batch_sys_handlers/sge.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
 
 # THIS FILE IS PART OF THE CYLC SUITE ENGINE.
 # Copyright (C) 2008-2019 NIWA & British Crown (Met Office) & Contributors.
@@ -45,7 +45,7 @@ def format_directives(self, job_conf):
                 job_conf["execution_time_limit"] / 3600,
                 (job_conf["execution_time_limit"] / 60) % 60,
                 job_conf["execution_time_limit"] % 60)
-        for key, value in job_conf['directives'].items():
+        for key, value in list(job_conf['directives'].items()):
             directives[key] = value
         lines = []
         for key, value in directives.items():
diff --git a/lib/cylc/batch_sys_handlers/slurm.py b/lib/cylc/batch_sys_handlers/slurm.py
index 3018790e655..facae167583 100644
--- a/lib/cylc/batch_sys_handlers/slurm.py
+++ b/lib/cylc/batch_sys_handlers/slurm.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
 
 # THIS FILE IS PART OF THE CYLC SUITE ENGINE.
 # Copyright (C) 2008-2019 NIWA & British Crown (Met Office) & Contributors.
@@ -54,7 +54,7 @@ def format_directives(cls, job_conf):
             directives["--time"] = "%d:%02d" % (
                 job_conf["execution_time_limit"] / 60,
                 job_conf["execution_time_limit"] % 60)
-        for key, value in job_conf['directives'].items():
+        for key, value in list(job_conf['directives'].items()):
             directives[key] = value
         lines = []
         for key, value in directives.items():
diff --git a/lib/cylc/batch_sys_manager.py b/lib/cylc/batch_sys_manager.py
index 432be594ac4..a6e6d101af3 100644
--- a/lib/cylc/batch_sys_manager.py
+++ b/lib/cylc/batch_sys_manager.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
 
 # THIS FILE IS PART OF THE CYLC SUITE ENGINE.
 # Copyright (C) 2008-2019 NIWA & British Crown (Met Office) & Contributors.
@@ -121,7 +121,6 @@
 from parsec.OrderedDict import OrderedDict
 
 
-from cylc.mkdir_p import mkdir_p
 from cylc.task_message import (
     CYLC_JOB_PID, CYLC_JOB_INIT_TIME, CYLC_JOB_EXIT_TIME, CYLC_JOB_EXIT,
     CYLC_MESSAGE)
@@ -430,7 +429,7 @@ def job_kill(self, st_file_path):
                         traceback.print_exc()
                         return (1, str(exc))
                     else:
-                        return (proc.wait(), proc.communicate()[1])
+                        return (proc.wait(), proc.communicate()[1].decode())
             return (1, "Cannot determine batch job ID from %s file" % (
                        JOB_LOG_STATUS))
         except IOError as exc:
@@ -563,7 +562,7 @@ def _jobs_poll_batch_sys(self, job_log_root, batch_sys_name, my_ctx_list):
                 sys.stderr.write(str(exc) + "\n")
                 return
             ret_code = proc.wait()
-            out, err = proc.communicate()
+            out, err = (f.decode() for f in proc.communicate())
             debug_messages.append('%s - %s' % (
                 batch_sys, len(out.split('\n'))))
             sys.stderr.write(err)
@@ -645,6 +644,10 @@ def _job_submit_impl(
         if hasattr(batch_sys, "get_submit_stdin"):
             proc_stdin_arg, proc_stdin_value = batch_sys.get_submit_stdin(
                 job_file_path, submit_opts)
+            if isinstance(proc_stdin_arg, str):
+                proc_stdin_arg = proc_stdin_arg.encode()
+            if isinstance(proc_stdin_value, str):
+                proc_stdin_value = proc_stdin_value.encode()
         if hasattr(batch_sys, "submit"):
             # batch_sys.submit should handle OSError, if relevant.
             ret_code, out, err = batch_sys.submit(job_file_path, submit_opts)
@@ -677,7 +680,7 @@ def _job_submit_impl(
                     if not exc.filename:
                         exc.filename = command[0]
                     return 1, "", str(exc), ""
-            out, err = proc.communicate(proc_stdin_value)
+            out, err = (f.decode() for f in proc.communicate(proc_stdin_value))
             ret_code = proc.wait()
 
         # Filter submit command output, if relevant
@@ -772,7 +775,9 @@ def _jobs_submit_prep_by_stdin(self, job_log_root, job_log_dirs):
             elif cur_line.startswith(self.LINE_PREFIX_JOB_LOG_DIR):
                 job_log_dir = cur_line.replace(
                     self.LINE_PREFIX_JOB_LOG_DIR, "").strip()
-                mkdir_p(os.path.join(job_log_root, job_log_dir))
+                os.makedirs(
+                    os.path.join(job_log_root, job_log_dir),
+                    exist_ok=True)
                 handle = open(
                     os.path.join(job_log_root, job_log_dir, "job.tmp"), "wb")
 
@@ -780,7 +785,7 @@ def _jobs_submit_prep_by_stdin(self, job_log_root, job_log_dirs):
                 lines.append(cur_line)
             else:
                 for line in lines + [cur_line]:
-                    handle.write(line)
+                    handle.write(line.encode())
                 lines = []
                 if cur_line.startswith(self.LINE_PREFIX_EOF + job_log_dir):
                     handle.close()
diff --git a/lib/cylc/broadcast_mgr.py b/lib/cylc/broadcast_mgr.py
index 4cb9ffad91e..fce1c31115a 100644
--- a/lib/cylc/broadcast_mgr.py
+++ b/lib/cylc/broadcast_mgr.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
 
 # THIS FILE IS PART OF THE CYLC SUITE ENGINE.
 # Copyright (C) 2008-2019 NIWA & British Crown (Met Office) & Contributors.
@@ -110,7 +110,7 @@ def clear_broadcast(
         if bad_options:
             LOG.error(get_broadcast_bad_options_report(bad_options))
 
-        return (modified_settings, bad_options)
+        return modified_settings, bad_options
 
     def expire_broadcast(self, cutoff=None):
         """Clear all broadcasts targeting cycle points earlier than cutoff."""
@@ -182,7 +182,7 @@ def match_ext_trigger(self, itask):
         if not self.ext_triggers or not itask.state.external_triggers:
             return
         has_changed = False
-        for trig, satisfied in itask.state.external_triggers.items():
+        for trig, satisfied in list(itask.state.external_triggers.items()):
             if satisfied:
                 continue
             for qmsg, qid in self.ext_triggers.copy():
@@ -292,7 +292,7 @@ def _get_bad_options(
                         bad_options[opt_name].discard(opt)
                         if not bad_options[opt_name]:
                             break
-        for key, value in bad_options.copy().items():
+        for key, value in list(bad_options.copy().items()):
             if value:
                 bad_options[key] = list(value)
             else:
diff --git a/lib/cylc/broadcast_report.py b/lib/cylc/broadcast_report.py
index 7aa99a26b5c..e411024483e 100644
--- a/lib/cylc/broadcast_report.py
+++ b/lib/cylc/broadcast_report.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
 
 # THIS FILE IS PART OF THE CYLC SUITE ENGINE.
 # Copyright (C) 2008-2019 NIWA & British Crown (Met Office) & Contributors.
@@ -67,12 +67,14 @@ def get_broadcast_change_iter(modified_settings, is_cancel=False):
         change = CHANGE_PREFIX_CANCEL
     else:
         change = CHANGE_PREFIX_SET
-    for modified_setting in sorted(modified_settings):
+    for modified_setting in sorted(modified_settings,
+                                   key=lambda x: (x[0], x[1])):
+        # sorted by (point, namespace)
         point, namespace, setting = modified_setting
         value = setting
         keys_str = ""
         while isinstance(value, dict):
-            key, value = value.items()[0]
+            key, value = list(value.items())[0]
             if isinstance(value, dict):
                 keys_str += "[" + key + "]"
             else:
diff --git a/lib/cylc/c3mro.py b/lib/cylc/c3mro.py
index 876c3c98ead..ca7dbb660b9 100644
--- a/lib/cylc/c3mro.py
+++ b/lib/cylc/c3mro.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
 
 # THIS FILE IS PART OF THE CYLC SUITE ENGINE.
 # Copyright (C) 2008-2019 NIWA & British Crown (Met Office) & Contributors.
@@ -147,7 +147,7 @@ def mro(self, C):
         """Compute the precedence list (mro) according to C3"""
         # copy() required here for tree to remain unchanged
         return self.merge(
-            [[C]] + map(self.mro, self.tree[C]) + [copy(self.tree[C])],
+            [[C]] + [self.mro(x) for x in self.tree[C]] + [copy(self.tree[C])],
             label=C)
 
 
@@ -158,7 +158,7 @@ def mro(self, C):
     parents['b'] = ['root']
     parents['foo'] = ['a', 'b']
 
-    print 'foo', C3(parents).mro('foo')
+    print('foo', C3(parents).mro('foo'))
 
     parents = {}
     parents['o'] = []
@@ -172,7 +172,7 @@ def mro(self, C):
     parents['k3'] = ['d', 'a']
     parents['z'] = ['k1', 'k2', 'k3']
 
-    print 'z', C3(parents).mro('z')
+    print('z', C3(parents).mro('z'))
 
     # Note we can get Python's result by defining an equivalent class
     # hierarchy (with empty class bodies) and printing foo.__mro__.
diff --git a/lib/cylc/cfgspec/README b/lib/cylc/cfgspec/README
index 05101a77759..4f9b20187a5 100644
--- a/lib/cylc/cfgspec/README
+++ b/lib/cylc/cfgspec/README
@@ -2,6 +2,5 @@ This directory contains specification modules for cylc parsec config files:
 
  * suite definition files
  * site/user config files
- * gcylc config files
 
  * test config file - to test parsec parsing and validation
diff --git a/lib/cylc/cfgspec/__init__.py b/lib/cylc/cfgspec/__init__.py
index 049c392519c..7872baaed1c 100644
--- a/lib/cylc/cfgspec/__init__.py
+++ b/lib/cylc/cfgspec/__init__.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
 
 # THIS FILE IS PART OF THE CYLC SUITE ENGINE.
 # Copyright (C) 2008-2019 NIWA & British Crown (Met Office) & Contributors.
diff --git a/lib/cylc/cfgspec/gcylc.py b/lib/cylc/cfgspec/gcylc.py
deleted file mode 100644
index b77fe7c2686..00000000000
--- a/lib/cylc/cfgspec/gcylc.py
+++ /dev/null
@@ -1,316 +0,0 @@
-#!/usr/bin/env python2
-
-# THIS FILE IS PART OF THE CYLC SUITE ENGINE.
-# Copyright (C) 2008-2019 NIWA & British Crown (Met Office) & Contributors.
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see .
-"gcylc config file format."
-
-import os
-import sys
-from copy import deepcopy, copy
-
-from parsec import ParsecError
-from parsec.config import ParsecConfig, ItemNotFoundError, itemstr
-from parsec.upgrade import upgrader
-from parsec.util import printcfg
-from cylc.cfgvalidate import (
-    cylc_config_validate, CylcConfigValidator as VDR, DurationFloat)
-from cylc.task_state import (
-    TASK_STATUSES_ALL, TASK_STATUS_RUNAHEAD, TASK_STATUS_HELD,
-    TASK_STATUS_WAITING, TASK_STATUS_EXPIRED, TASK_STATUS_QUEUED,
-    TASK_STATUS_READY, TASK_STATUS_SUBMITTED, TASK_STATUS_SUBMIT_FAILED,
-    TASK_STATUS_SUBMIT_RETRYING, TASK_STATUS_RUNNING, TASK_STATUS_SUCCEEDED,
-    TASK_STATUS_FAILED, TASK_STATUS_RETRYING)
-
-
-OLD_SITE_FILE = os.path.join(
-    os.environ['CYLC_DIR'], 'etc', 'gcylcrc', 'themes.rc')
-SITE_FILE = os.path.join(
-    os.environ['CYLC_DIR'], 'etc', 'gcylc-themes.rc')
-USER_FILE = os.path.join(os.environ['HOME'], '.cylc', 'gcylc.rc')
-HEADINGS = (
-    None, 'task', 'state', 'host', 'job system', 'job ID', 'T-submit',
-    'T-start', 'T-finish', 'dT-mean', 'latest message',)
-
-# Nested dict of spec items.
-# Spec value is [value_type, default, allowed_2, allowed_3, ...]
-# where:
-# - value_type: value type (compulsory).
-# - default: the default value (optional).
-# - allowed_2, ...: the only other allowed values of this setting (optional).
-SPEC = {
-    'dot icon size': [
-        VDR.V_STRING, "medium", "small", "medium", "large", "extra large"],
-    'initial side-by-side views': [VDR.V_BOOLEAN],
-    'initial views': [VDR.V_STRING_LIST, ["text"]],
-    'maximum update interval': [VDR.V_INTERVAL, DurationFloat(15)],
-    'sort by definition order': [VDR.V_BOOLEAN, True],
-    'sort column': [VDR.V_STRING] + list(HEADINGS),
-    'sort column ascending': [VDR.V_BOOLEAN, True],
-    'sub-graphs on': [VDR.V_BOOLEAN, False],
-    'task filter highlight color': [VDR.V_STRING, 'PowderBlue'],
-    'task states to filter out': [
-        VDR.V_STRING_LIST, [TASK_STATUS_RUNAHEAD]],
-    'themes': {
-        '__MANY__': {
-            'inherit': [VDR.V_STRING, "default"],
-            'defaults': [VDR.V_STRING_LIST],
-            TASK_STATUS_WAITING: [VDR.V_STRING_LIST],
-            TASK_STATUS_HELD: [VDR.V_STRING_LIST],
-            TASK_STATUS_QUEUED: [VDR.V_STRING_LIST],
-            TASK_STATUS_READY: [VDR.V_STRING_LIST],
-            TASK_STATUS_EXPIRED: [VDR.V_STRING_LIST],
-            TASK_STATUS_SUBMITTED: [VDR.V_STRING_LIST],
-            TASK_STATUS_SUBMIT_FAILED: [VDR.V_STRING_LIST],
-            TASK_STATUS_RUNNING: [VDR.V_STRING_LIST],
-            TASK_STATUS_SUCCEEDED: [VDR.V_STRING_LIST],
-            TASK_STATUS_FAILED: [VDR.V_STRING_LIST],
-            TASK_STATUS_RETRYING: [VDR.V_STRING_LIST],
-            TASK_STATUS_SUBMIT_RETRYING: [VDR.V_STRING_LIST],
-            TASK_STATUS_RUNAHEAD: [VDR.V_STRING_LIST],
-        },
-    },
-    'transpose dot': [VDR.V_BOOLEAN],
-    'transpose graph': [VDR.V_BOOLEAN],
-    'ungrouped views': [VDR.V_STRING_LIST],
-    'use theme': [VDR.V_STRING, "default"],
-    'window size': [VDR.V_INTEGER_LIST, [800, 500]],
-}
-
-
-def upg(cfg, descr):
-    u = upgrader(cfg, descr)
-    u.deprecate(
-        '5.4.3',
-        ['themes', '__MANY__', 'submitting'],
-        ['themes', '__MANY__', 'ready'])
-    u.upgrade()
-
-
-class GcylcConfig(ParsecConfig):
-    """gcylc user configuration - default view panels, task themes etc."""
-
-    _INST = None
-
-    @classmethod
-    def get_inst(cls):
-        """Return default instance."""
-        if cls._INST is None:
-            cls._INST = cls(SPEC, upg)
-            try:
-                cls._INST.loadcfg(SITE_FILE, upgrader.SITE_CONFIG)
-            except ParsecError as exc:
-                LOG.warning(
-                    'ignoring bad %s %s\n%s',
-                    upgrader.SITE_CONFIG, SITE_FILE, exc)
-
-            if os.access(USER_FILE, os.F_OK | os.R_OK):
-                try:
-                    cls._INST.loadcfg(USER_FILE, upgrader.USER_CONFIG)
-                except ParsecError as exc:
-                    LOG.error('bad %s %s', upgrader.USER_CONFIG, USER_FILE)
-                    raise
-
-            # check and correct initial view config etc.
-            cls._INST.check()
-            # add spec defaults and do theme inheritance
-            cls._INST.transform()
-        return cls._INST
-
-    def __init__(self, spec, upg):
-        ParsecConfig.__init__(self, spec, upg, validator=cylc_config_validate)
-        self.default_theme = None
-        self.use_theme = None
-
-    def transform(self):
-        """
-        1) theme inheritance
-        2) turn state attribute lists into dicts for easier access:
-          running : color=#ff00ff, style=filled, fontcolor=black
-          becomes:
-             running : { color:#ff00ff, style:filled, fontcolor:black }
-        """
-        # Note this is only done for the dense config structure.
-
-        self.expand()
-        self.default_theme = "default"
-
-        cfg = self.get()
-
-        # select the start-up theme
-        self.use_theme = cfg['use theme']
-
-        # and check it is valid
-        if self.use_theme not in cfg['themes']:
-            sys.stderr.write("WARNING: theme %s not found, using '%s'\n" % (
-                self.use_theme, self.default_theme))
-            cfg['use theme'] = 'default'
-            self.use_theme = self.default_theme
-
-        # theme inheritance
-        inherited = []
-        for label in cfg['themes']:
-            hierarchy = []
-            name = label
-            while True:
-                hierarchy.append(name)
-                if name == "default":
-                    break
-                if cfg['themes'][name]['inherit']:
-                    parent = cfg['themes'][name]['inherit']
-                    if parent not in cfg['themes']:
-                        sys.stderr.write(
-                            "WARNING: undefined parent '%s' (theme '%s')\n" %
-                            (parent, label))
-                        parent = "default"
-                else:
-                    break
-                name = parent
-            hierarchy.reverse()
-            prev = hierarchy[0]
-            theme = deepcopy(cfg['themes'][prev])
-            for item in hierarchy[1:]:
-                if item in inherited:
-                    prev = item
-                    continue
-                # print 'Inherit:', item, '<--', prev
-                self.inherit(theme, cfg['themes'][item])
-                inherited.append(item)
-            cfg['themes'][label] = theme
-
-        # expand theme data
-        cfg_themes = {}
-        for theme in cfg['themes']:
-            for key, val in cfg['themes'][self.default_theme].items():
-                if not cfg['themes'][theme][key]:
-                    cfg['themes'][theme][key] = val
-
-            cfg_themes[theme] = {}
-            defs = self.parse_state(
-                theme, 'defaults', cfg['themes'][theme]['defaults'])
-
-            for item, val in cfg['themes'][theme].items():
-                if item in ['inherit', 'defaults']:
-                    continue
-                state = item
-                if state not in TASK_STATUSES_ALL:
-                    sys.stderr.write(
-                        ("WARNING, "
-                         "ignoring illegal task state '%s' in theme %s\n") %
-                        (state, theme))
-                # reverse inherit (override)
-                tcfg = deepcopy(defs)
-                self.inherit(tcfg, self.parse_state(theme, item, val))
-                cfg_themes[theme][state] = tcfg
-
-        # final themes result:
-        cfg['themes'] = cfg_themes
-
-    def check(self):
-        cfg = self.get(sparse=True)
-
-        # check window size config
-        if 'window size' in cfg:
-            fail = False
-            if len(cfg['window size']) != 2:
-                sys.stderr.write(
-                    "WARNING: window size requires two values (x, y). "
-                    "Using default.\n")
-                fail = True
-            elif cfg['window size'][0] < 0 or cfg['window size'][1] < 0:
-                sys.stderr.write(
-                    "WARNING: window size values must be positive. "
-                    "Using default.\n")
-                fail = True
-            # TODO: check for daft window sizes? (10, 5), (80000, 5000) ?
-            if fail:
-                cfg['window size'] = [800, 500]
-
-        # check initial view config
-        if 'initial views' not in cfg:
-            return
-        views = copy(cfg['initial views'])
-        for view in views:
-            if view not in ['dot', 'text', 'graph']:
-                sys.stderr.write(
-                    "WARNING: ignoring illegal view name '%s'\n" % (view))
-                cfg['initial views'].remove(view)
-        views = cfg['initial views']
-        if len(views) == 0:
-            # at least one view required
-            sys.stderr.write(
-                "WARNING: no initial views defined, defaulting to 'text'\n")
-            cfg['initial views'] = ['text']
-
-    @staticmethod
-    def parse_state(theme, name, cfglist):
-        allowed_keys = ['style', 'color', 'fontcolor']
-        cfg = {}
-        for item in cfglist:
-            key, val = item.split('=')
-            if key not in allowed_keys:
-                sys.exit(
-                    'ERROR, gcylc.rc, illegal: %s: %s = %s' %
-                    (theme, name, cfglist))
-            if key == 'color' or key == 'fontcolor':
-                try:
-                    import gtk
-                    gtk.gdk.color_parse(val)
-                except ValueError as exc:
-                    sys.exit(
-                        'ERROR, gcylc.rc, illegal color: %s: %s="%s"\n%s' %
-                        (theme, name, item, exc))
-            cfg[key] = val
-        return cfg
-
-    def inherit(self, target, source):
-        for item in source:
-            if isinstance(source[item], dict):
-                if item not in target:
-                    target[item] = {}
-                self.inherit(target[item], source[item])
-            else:
-                target[item] = source[item]
-
-    def dump(self, keys=None, sparse=False, pnative=False, prefix='',
-             none_str=''):
-        """Override parse.config.dump().
-
-        To restore the list-nature of theme state items.
-        """
-        cfg = deepcopy(self.get([], sparse))
-        try:
-            for theme in cfg['themes'].values():
-                for state in theme:
-                    clist = []
-                    for attr, val in theme[state].items():
-                        clist.append('%s=%s' % (attr, val))
-                    theme[state] = clist
-        except (KeyError, AttributeError):
-            pass
-
-        parents = []
-        for key in keys:
-            try:
-                cfg = cfg[key]
-            except KeyError:
-                raise ItemNotFoundError(itemstr(parents, key))
-            else:
-                parents.append(key)
-
-        if pnative:
-            print(cfg)
-        else:
-            printcfg(cfg, prefix=prefix, level=len(keys))
diff --git a/lib/cylc/cfgspec/glbl_cfg.py b/lib/cylc/cfgspec/glbl_cfg.py
index 90b004a941b..d8d58efb6c0 100644
--- a/lib/cylc/cfgspec/glbl_cfg.py
+++ b/lib/cylc/cfgspec/glbl_cfg.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
 
 # THIS FILE IS PART OF THE CYLC SUITE ENGINE.
 # Copyright (C) 2008-2019 NIWA & British Crown (Met Office) & Contributors.
diff --git a/lib/cylc/cfgspec/globalcfg.py b/lib/cylc/cfgspec/globalcfg.py
index 0eed602337b..9816a143dfe 100644
--- a/lib/cylc/cfgspec/globalcfg.py
+++ b/lib/cylc/cfgspec/globalcfg.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
 
 # THIS FILE IS PART OF THE CYLC SUITE ENGINE.
 # Copyright (C) 2008-2019 NIWA & British Crown (Met Office) & Contributors.
@@ -31,8 +31,7 @@
 from cylc.cfgvalidate import (
     cylc_config_validate, CylcConfigValidator as VDR, DurationFloat)
 from cylc.hostuserutil import get_user_home, is_remote_user
-from cylc.mkdir_p import mkdir_p
-from cylc.network import PRIVILEGE_LEVELS, PRIV_STATE_TOTALS, PRIV_SHUTDOWN
+from cylc.network import Priv
 from cylc.version import CYLC_VERSION
 
 # Nested dict of spec items.
@@ -103,11 +102,6 @@
         'gui': [VDR.V_STRING, 'gvim -f'],
     },
 
-    'communication': {
-        'method': [VDR.V_STRING, 'https', 'http'],
-        'options': [VDR.V_STRING_LIST],
-    },
-
     'monitor': {
         'sort order': [VDR.V_STRING, 'definition', 'alphanumeric'],
     },
@@ -219,16 +213,16 @@
         # Allow owners to grant public shutdown rights at the most, not full
         # control.
         'public': (
-            [VDR.V_STRING, PRIV_STATE_TOTALS] +
-            PRIVILEGE_LEVELS[:PRIVILEGE_LEVELS.index(PRIV_SHUTDOWN) + 1]),
+            [VDR.V_STRING]
+            + [level.name.lower().replace('_', '-') for level in [
+                Priv.STATE_TOTALS, Priv.IDENTITY, Priv.DESCRIPTION,
+                Priv.STATE_TOTALS, Priv.READ, Priv.SHUTDOWN]])
     },
 
     'suite servers': {
         'run hosts': [VDR.V_SPACELESS_STRING_LIST],
-        'run ports': [VDR.V_INTEGER_LIST, range(43001, 43101)],
-        'scan hosts': [VDR.V_SPACELESS_STRING_LIST],
-        'scan ports': [VDR.V_INTEGER_LIST, range(43001, 43101)],
-        'condemned hosts': [VDR.V_SPACELESS_STRING_LIST],
+        'run ports': [VDR.V_INTEGER_LIST, list(range(43001, 43101))],
+        'condemned hosts': [VDR.V_ABSOLUTE_HOST_LIST],
         'auto restart delay': [VDR.V_INTERVAL],
         'run host select': {
             'rank': [VDR.V_STRING, 'random', 'load:1', 'load:5', 'load:15',
@@ -263,6 +257,9 @@ def upg(cfg, descr):
     u.deprecate('8.0.0',
                 ['documentation', 'urls', 'internet homepage'],
                 ['documentation', 'cylc homepage'])
+    u.obsolete('8.0.0', ['suite servers', 'scan hosts'])
+    u.obsolete('8.0.0', ['suite servers', 'scan ports'])
+    u.obsolete('8.0.0', ['communication'])
 
     u.upgrade()
 
@@ -443,8 +440,8 @@ def get_host_item(self, item, host=None, owner=None, replace_home=False,
                     owner_home = os.path.expanduser('~%s' % owner)
                 value = value.replace(self._HOME, owner_home)
         if item == "task communication method" and value == "default":
-            # Translate "default" to client-server comms: "https" or "http".
-            value = cfg['communication']['method']
+            # Translate "default" to client-server comms: "zmq"
+            value = 'zmq'
         return value
 
     def roll_directory(self, dir_, name, archlen=0):
@@ -471,7 +468,7 @@ def roll_directory(self, dir_, name, archlen=0):
     def create_directory(dir_, name):
         """Create directory. Raise GlobalConfigError on error."""
         try:
-            mkdir_p(dir_)
+            os.makedirs(dir_, exist_ok=True)
         except OSError as exc:
             LOG.exception(exc)
             raise GlobalConfigError(
diff --git a/lib/cylc/cfgspec/gscan.py b/lib/cylc/cfgspec/gscan.py
deleted file mode 100644
index 0f0a76112ed..00000000000
--- a/lib/cylc/cfgspec/gscan.py
+++ /dev/null
@@ -1,94 +0,0 @@
-#!/usr/bin/env python2
-
-# THIS FILE IS PART OF THE CYLC SUITE ENGINE.
-# Copyright (C) 2008-2019 NIWA & British Crown (Met Office) & Contributors.
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see .
-"""gscan config file format."""
-
-import os
-import sys
-
-from parsec import ParsecError
-from parsec.config import ParsecConfig
-from parsec.upgrade import upgrader
-
-from cylc.cfgvalidate import (
-    cylc_config_validate, CylcConfigValidator as VDR, DurationFloat)
-
-
-USER_FILE = os.path.join(os.environ['HOME'], '.cylc', 'gscan.rc')
-
-# Nested dict of spec items.
-# Spec value is [value_type, default, allowed_2, allowed_3, ...]
-# where:
-# - value_type: value type (compulsory).
-# - default: the default value (optional).
-# - allowed_2, ...: the only other allowed values of this setting (optional).
-SPEC = {
-    'activate on startup': [VDR.V_BOOLEAN],
-    'columns': [VDR.V_STRING_LIST, ['suite', 'status']],
-    'suite listing update interval': [VDR.V_INTERVAL, DurationFloat(60)],
-    'suite status update interval': [VDR.V_INTERVAL, DurationFloat(15)],
-    'window size': [VDR.V_INTEGER_LIST, [300, 200]],
-    'hide main menubar': [VDR.V_BOOLEAN, False],
-}
-
-
-class GScanConfig(ParsecConfig):
-    """Configuration for "gscan"."""
-
-    COL_GROUP = "Group"
-    COL_HOST = "Host"
-    COL_OWNER = "Owner"
-    COL_SUITE = "Suite"
-    COL_TITLE = "Title"
-    COL_UPDATED = "Updated"
-    COL_STATUS = "Status"
-    COL_VERSION = "Version"
-    COLS_DEFAULT = (COL_SUITE.lower(), COL_STATUS.lower())
-    COLS = [col.lower() for col in (
-        COL_GROUP, COL_HOST, COL_OWNER, COL_SUITE, COL_TITLE, COL_UPDATED,
-        COL_STATUS, COL_VERSION)]
-
-    _INST = None
-
-    @classmethod
-    def get_inst(cls):
-        """Return the singleton instance."""
-        if cls._INST is None:
-            cls._INST = cls(SPEC, validator=cylc_config_validate)
-            if os.access(USER_FILE, os.F_OK | os.R_OK):
-                try:
-                    cls._INST.loadcfg(USER_FILE, upgrader.USER_CONFIG)
-                except ParsecError:
-                    LOG.error('bad %s %s', upgrader.USER_CONFIG, USER_FILE)
-                    raise
-            cls._INST.check()
-        return cls._INST
-
-    def check(self):
-        """Custom configuration check."""
-        cfg = self.get(sparse=True)
-        if 'columns' in cfg:
-            for column in cfg['columns']:
-                if column not in self.COLS:
-                    sys.stderr.write(
-                        "WARNING: illegal column name '%s'\n" % column)
-                    cfg['columns'].remove(column)
-            if not cfg['columns']:
-                sys.stderr.write(
-                    'WARNING: at least one column must be specified,' +
-                    ' defaulting to "%s, %s"\n' % self.COLS_DEFAULT)
-                cfg['columns'] = list(self.COLS_DEFAULT)
diff --git a/lib/cylc/cfgspec/suite.py b/lib/cylc/cfgspec/suite.py
index 48fb46ebd8b..e5b24c84479 100644
--- a/lib/cylc/cfgspec/suite.py
+++ b/lib/cylc/cfgspec/suite.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
 
 # THIS FILE IS PART OF THE CYLC SUITE ENGINE.
 # Copyright (C) 2008-2019 NIWA & British Crown (Met Office) & Contributors.
@@ -24,7 +24,7 @@
 
 from cylc.cfgvalidate import (
     cylc_config_validate, CylcConfigValidator as VDR, DurationFloat)
-from cylc.network import PRIVILEGE_LEVELS, PRIV_SHUTDOWN
+from cylc.network import Priv
 
 
 # Nested dict of spec items.
@@ -112,8 +112,10 @@
             # Allow owners to grant public shutdown rights at the most, not
             # full control.
             'public': (
-                [VDR.V_STRING, ''] +
-                PRIVILEGE_LEVELS[:PRIVILEGE_LEVELS.index(PRIV_SHUTDOWN) + 1]),
+                [VDR.V_STRING, '']
+                + [level.name.lower().replace('_', '-') for level in [
+                    Priv.IDENTITY, Priv.DESCRIPTION, Priv.STATE_TOTALS,
+                    Priv.READ, Priv.SHUTDOWN]])
         },
     },
     'scheduling': {
@@ -124,7 +126,7 @@
         'hold after point': [VDR.V_CYCLE_POINT],
         'cycling mode': (
             [VDR.V_STRING, Calendar.MODE_GREGORIAN] +
-            Calendar.MODES.keys() + ["integer"]),
+            list(Calendar.MODES) + ["integer"]),
         'runahead limit': [VDR.V_STRING],
         'max active cycle points': [VDR.V_INTEGER, 3],
         'spawn to max active cycle points': [VDR.V_BOOLEAN],
diff --git a/lib/cylc/cfgvalidate.py b/lib/cylc/cfgvalidate.py
index b67fb3746ee..7cd242e85cd 100644
--- a/lib/cylc/cfgvalidate.py
+++ b/lib/cylc/cfgvalidate.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
 
 # THIS FILE IS PART OF THE CYLC SUITE ENGINE.
 # Copyright (C) 2008-2019 NIWA & British Crown (Met Office) & Contributors.
diff --git a/lib/cylc/command_polling.py b/lib/cylc/command_polling.py
index 33828e213db..af651deb125 100644
--- a/lib/cylc/command_polling.py
+++ b/lib/cylc/command_polling.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
 
 # THIS FILE IS PART OF THE CYLC SUITE ENGINE.
 # Copyright (C) 2008-2019 NIWA & British Crown (Met Office) & Contributors.
diff --git a/lib/cylc/conditional_simplifier.py b/lib/cylc/conditional_simplifier.py
index 28798b4172f..e3a7483f328 100644
--- a/lib/cylc/conditional_simplifier.py
+++ b/lib/cylc/conditional_simplifier.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
 
 # THIS FILE IS PART OF THE CYLC SUITE ENGINE.
 # Copyright (C) 2008-2019 NIWA & British Crown (Met Office) & Contributors.
diff --git a/lib/cylc/config.py b/lib/cylc/config.py
index 23b8d834c07..0d13bda19a4 100644
--- a/lib/cylc/config.py
+++ b/lib/cylc/config.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
 
 # THIS FILE IS PART OF THE CYLC SUITE ENGINE.
 # Copyright (C) 2008-2019 NIWA & British Crown (Met Office) & Contributors.
@@ -310,7 +310,7 @@ def __init__(self, suite, fpath, template_vars=None,
         self._expand_runtime()
         self.mem_log("config.py: after _expand_runtime")
 
-        self.ns_defn_order = self.cfg['runtime'].keys()
+        self.ns_defn_order = list(self.cfg['runtime'])
 
         self.mem_log("config.py: before compute_family_tree")
         # do sparse inheritance
@@ -607,7 +607,7 @@ def __init__(self, suite, fpath, template_vars=None,
             LOG.debug("Checking [visualization] node attributes")
             # TODO - these should probably be done in non-verbose mode too.
             # 1. node groups should contain valid namespace names
-            nspaces = self.cfg['runtime'].keys()
+            nspaces = list(self.cfg['runtime'])
             bad = {}
             for ng, mems in ngs.items():
                 n_bad = []
@@ -654,7 +654,8 @@ def __init__(self, suite, fpath, template_vars=None,
         # results in root appearing last in the ordered dict of node
         # names, so it overrides the styling for lesser groups and
         # nodes, whereas the reverse is needed - fixing this would
-        # require reordering task_attr in lib/cylc/graphing.py).
+        # require reordering task_attr in lib/cylc/graphing.py
+        # TODO: graphing.py is gone now! ).
 
         self.leaves = self.get_task_name_list()
         for ancestors in self.runtime['first-parent ancestors'].values():
@@ -821,8 +822,7 @@ def _expand_runtime(self):
                     self.task_param_vars[name] = indices
                     new_environ = OrderedDictWithDefaults()
                     if 'environment' in newruntime[name]:
-                        for k, v in newruntime[name]['environment'].items():
-                            new_environ[k] = v
+                        new_environ = newruntime[name]['environment'].copy()
                     newruntime[name]['environment'] = new_environ
                 if 'inherit' in newruntime[name]:
                     # Allow inheritance from parameterized namespaces.
@@ -995,15 +995,9 @@ def compute_family_tree(self):
                 self.runtime['linearized ancestors'][name] = c3.mro(name)
                 self.runtime['first-parent ancestors'][name] = (
                     c3_single.mro(name))
-            except RuntimeError:
-                if cylc.flags.debug:
-                    raise
-                exc_lines = traceback.format_exc().splitlines()
-                if exc_lines[-1].startswith(
-                        "RuntimeError: maximum recursion depth exceeded"):
-                    raise SuiteConfigError(
-                        "ERROR: circular [runtime] inheritance?")
-                raise
+            except RecursionError:
+                raise SuiteConfigError(
+                    "ERROR: circular [runtime] inheritance?")
 
         for name in self.cfg['runtime']:
             ancestors = self.runtime['linearized ancestors'][name]
@@ -1027,7 +1021,7 @@ def compute_inheritance(self, use_simple_method=True):
         already_done = {}  # to store already computed namespaces by mro
 
         # Loop through runtime members, 'root' first.
-        nses = self.cfg['runtime'].keys()
+        nses = list(self.cfg['runtime'])
         nses.sort(key=lambda ns: ns != 'root')
         for ns in nses:
             # for each namespace ...
@@ -1145,7 +1139,7 @@ def configure_queues(self):
         # Then reassign to other queues as requested.
         warnings = []
         requeued = []
-        for key, queue in queues.copy().items():
+        for key, queue in list(queues.copy().items()):
             # queues.copy() is essential here to allow items to be removed from
             # the queues dict.
             if key == self.Q_DEFAULT:
@@ -1227,7 +1221,7 @@ def configure_suite_state_polling_tasks(self):
                         "ERROR: script cannot be defined for automatic" +
                         " suite polling task '%s':\n%s" % (l_task, cs))
         # Generate the automatic scripting.
-        for name, tdef in self.taskdefs.items():
+        for name, tdef in list(self.taskdefs.items()):
             if name not in self.suite_polling_tasks:
                 continue
             rtc = tdef.rtconfig
@@ -1361,10 +1355,10 @@ def get_namespace_list(self, which):
         names = []
         if which == 'graphed tasks':
             # tasks used only in the graph
-            names = self.taskdefs.keys()
+            names = list(self.taskdefs)
         elif which == 'all namespaces':
             # all namespaces
-            names = self.cfg['runtime'].keys()
+            names = list(self.cfg['runtime'])
         elif which == 'all tasks':
             for ns in self.cfg['runtime']:
                 if ns not in self.runtime['descendants']:
@@ -1498,7 +1492,7 @@ def check_tasks(self):
 
     def get_task_name_list(self):
         # return a list of all tasks used in the dependency graph
-        return self.taskdefs.keys()
+        return list(self.taskdefs)
 
     def generate_edges(self, lexpr, orig_lexpr, left_nodes, right, seq,
                        suicide=False):
@@ -1890,7 +1884,7 @@ def get_graph_raw(self, start_point_string, stop_point_string,
             # Flatten nested list.
             graph_raw_edges = (
                 [i for sublist in gr_edges.values() for i in sublist])
-        graph_raw_edges.sort()
+        graph_raw_edges.sort(key=lambda x: [y if y else '' for y in x])
         self._last_graph_raw_edges = graph_raw_edges
         return graph_raw_edges
 
@@ -1962,7 +1956,7 @@ def load_graph(self):
         # Note we could exclude 'root' from this and disallow use of 'root' in
         # the graph (which would probably be quite reasonable).
         family_map = {}
-        for family, tasks in self.runtime['descendants'].iteritems():
+        for family, tasks in self.runtime['descendants'].items():
             family_map[family] = [t for t in tasks if (
                 t in self.runtime['parents'] and
                 t not in self.runtime['descendants'])]
@@ -2062,7 +2056,7 @@ def load_graph(self):
 
         # Detect use of xtrigger names with '@' prefix (creates a task).
         overlap = set(self.taskdefs.keys()).intersection(
-            self.cfg['scheduling']['xtriggers'].keys())
+            list(self.cfg['scheduling']['xtriggers']))
         if overlap:
             LOG.error(', '.join(overlap))
             raise SuiteConfigError('task and @xtrigger names clash')
diff --git a/lib/cylc/cycling/__init__.py b/lib/cylc/cycling/__init__.py
index a63ab932724..ba1882b9c7a 100644
--- a/lib/cylc/cycling/__init__.py
+++ b/lib/cylc/cycling/__init__.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
 
 # THIS FILE IS PART OF THE CYLC SUITE ENGINE.
 # Copyright (C) 2008-2019 NIWA & British Crown (Met Office) & Contributors.
@@ -18,7 +18,7 @@
 
 """This module provides base classes for cycling data objects."""
 
-from abc import ABCMeta, abstractmethod, abstractproperty
+from abc import ABCMeta, abstractmethod
 
 
 def parse_exclusion(expr):
@@ -36,7 +36,7 @@ def parse_exclusion(expr):
                 raise Exception("'%s': a list of exclusions must be "
                                 "enclosed in parentheses." % exclusions)
 
-        exclusions = exclusions.translate(None, ' ()')
+        exclusions = exclusions.translate(str.maketrans('', '', ' ()'))
         exclusions = exclusions.split(',')
         return remainder.strip(), exclusions
 
@@ -82,7 +82,7 @@ def __str__(self):
         return self.ERROR_MESSAGE.format(*self.args)
 
 
-class PointBase(object):
+class PointBase(object, metaclass=ABCMeta):
 
     """The abstract base class for single points in a cycler sequence.
 
@@ -97,23 +97,22 @@ class PointBase(object):
     method to reprocess their value into a standard form.
 
     """
-    __metaclass__ = ABCMeta
 
     _TYPE = None
     _TYPE_SORT_KEY = None
 
     __slots__ = ('value')
 
-    @abstractproperty
+    @abstractmethod
     def TYPE(self):
         return self._TYPE
 
-    @abstractproperty
+    @abstractmethod
     def TYPE_SORT_KEY(self):
         return self._TYPE_SORT_KEY
 
     def __init__(self, value):
-        if not isinstance(value, basestring):
+        if not isinstance(value, str):
             raise TypeError(type(value))
         self.value = value
 
@@ -172,7 +171,7 @@ def __add__(self, other):
         return self.add(other)
 
 
-class IntervalBase(object):
+class IntervalBase(object, metaclass=ABCMeta):
 
     """An interval separating points in a cycler sequence.
 
@@ -180,7 +179,7 @@ class IntervalBase(object):
 
     Subclasses should provide values for TYPE and TYPE_SORT_KEY.
     They should also provide self.cmp_, self.sub, self.add,
-    self.__mul__, self.__abs__, self.__nonzero__ methods which should
+    self.__mul__, self.__abs__ methods which should
     behave as __cmp__, __sub__, etc standard comparison methods.
 
     They can also just override the provided comparison methods (such
@@ -198,18 +197,17 @@ class IntervalBase(object):
     method to reprocess their value into a standard form.
 
     """
-    __metaclass__ = ABCMeta
 
     _TYPE = None
     _TYPE_SORT_KEY = None
 
     __slots__ = ('value')
 
-    @abstractproperty
+    @abstractmethod
     def TYPE(self):
         return self._TYPE
 
-    @abstractproperty
+    @abstractmethod
     def TYPE_SORT_KEY(self):
         return self._TYPE_SORT_KEY
 
@@ -234,13 +232,8 @@ def __mul__(self, factor):
         # Return an interval with all properties multiplied by factor.
         pass
 
-    @abstractmethod
-    def __nonzero__(self):
-        # Return True if the interval has any non-zero properties.
-        pass
-
     def __init__(self, value):
-        if not isinstance(value, basestring):
+        if not isinstance(value, str):
             raise TypeError(type(value))
         self.value = value
 
@@ -286,6 +279,8 @@ def __add__(self, other):
 
     def __cmp__(self, other):
         # Compare self to other (interval).
+        if other is None:
+            return -1
         if self.TYPE != other.TYPE:
             return cmp(self.TYPE_SORT_KEY, other.TYPE_SORT_KEY)
         if self.value == other.value:
@@ -303,7 +298,7 @@ def __neg__(self):
         return self * -1
 
 
-class SequenceBase(object):
+class SequenceBase(object, metaclass=ABCMeta):
 
     """The abstract base class for cycler sequences.
 
@@ -323,18 +318,17 @@ class SequenceBase(object):
     is equal to another (represents the same set of points).
 
     """
-    __metaclass__ = ABCMeta
 
     _TYPE = None
     _TYPE_SORT_KEY = None
 
     __slots__ = ()
 
-    @abstractproperty
+    @abstractmethod
     def TYPE(self):
         return self._TYPE
 
-    @abstractproperty
+    @abstractmethod
     def TYPE_SORT_KEY(self):
         return self._TYPE_SORT_KEY
 
@@ -411,11 +405,9 @@ def __eq__(self, other):
         pass
 
 
-class ExclusionBase(object):
+class ExclusionBase(object, metaclass=ABCMeta):
     """A collection of points or sequences that are treated in an
     exclusionary manner"""
-
-    __metaclass__ = ABCMeta
     __slots__ = ('exclusion_sequences', 'exclusion_points',
                  'exclusion_start_point', 'exclusion_end_point')
 
@@ -459,3 +451,27 @@ def __str__(self):
         if ',' in ret:
             ret = '(' + ret + ')'
         return ret
+
+
+def cmp(self, other):
+    """Temporary replacement for the Python2 cmp function."""
+    if self == other:
+        return 0
+    if self < other:
+        return -1
+    return 1
+
+
+def cmp_to_rich(cls):
+    """Temporary solution which monkey patches rich comparisons."""
+    cls.__lt__ = lambda self, other: cls.__cmp__(self, other) == -1
+    cls.__le__ = lambda self, other: cls.__cmp__(self, other) <= 0
+    cls.__gt__ = lambda self, other: cls.__cmp__(self, other) == 1
+    cls.__ge__ = lambda self, other: cls.__cmp__(self, other) >= 0
+    cls.__eq__ = lambda self, other: cls.__cmp__(self, other) == 0
+    cls.__ne__ = lambda self, other: cls.__cmp__(self, other) != 0
+
+
+# TODO: replace __cmp__ infrastructure
+cmp_to_rich(PointBase)
+cmp_to_rich(IntervalBase)
diff --git a/lib/cylc/cycling/integer.py b/lib/cylc/cycling/integer.py
index 9be3412ff5d..5a2415dc111 100755
--- a/lib/cylc/cycling/integer.py
+++ b/lib/cylc/cycling/integer.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
 
 # THIS FILE IS PART OF THE CYLC SUITE ENGINE.
 # Copyright (C) 2008-2019 NIWA & British Crown (Met Office) & Contributors.
@@ -23,7 +23,7 @@
 
 from cylc.cycling import (
     PointBase, IntervalBase, SequenceBase, ExclusionBase, PointParsingError,
-    IntervalParsingError, parse_exclusion
+    IntervalParsingError, parse_exclusion, cmp
 )
 from cylc.time_parser import CylcMissingContextPointError
 
@@ -108,7 +108,6 @@
         (r"^%(reps_1)s//%(end)s$", 4)
     ]
 ]
-del regex, format_num
 
 
 REC_RELATIVE_POINT = re.compile(r"^[-+]P\d+$")
@@ -195,7 +194,7 @@ def get_inferred_child(self, string):
             return IntegerInterval(string)
 
     def __init__(self, value):
-        if (not isinstance(value, basestring) or
+        if (not isinstance(value, str) or
                 not REC_INTERVAL.search(value)):
             raise IntervalParsingError("IntegerInterval", repr(value))
         super(IntegerInterval, self).__init__(value)
@@ -226,7 +225,7 @@ def __mul__(self, factor):
         # Return an interval with all properties multiplied by factor.
         return IntegerInterval.from_integer(int(self) * factor)
 
-    def __nonzero__(self):
+    def __bool__(self):
         # Return True if the interval has any non-zero properties.
         return bool(int(self))
 
@@ -579,6 +578,18 @@ def __eq__(self, other):
                 self.p_stop == other.p_stop and \
                 self.exclusions == other.exclusions
 
+    def __hash__(self):
+        return hash(tuple((getattr(self, attr) for attr in self.__slots__)))
+
+    def __lt__(self, other: 'IntegerSequence') -> bool:
+        for attr in self.__slots__:
+            try:
+                if getattr(self, attr) < getattr(other, attr):
+                    return True
+            except TypeError:
+                pass
+        return False
+
 
 def init_from_cfg(_):
     """Placeholder function required by all cycling modules."""
diff --git a/lib/cylc/cycling/iso8601.py b/lib/cylc/cycling/iso8601.py
index da36a90a6af..08162af9344 100755
--- a/lib/cylc/cycling/iso8601.py
+++ b/lib/cylc/cycling/iso8601.py
@@ -27,7 +27,7 @@
 from cylc.time_parser import CylcTimeParser
 from cylc.cycling import (
     PointBase, IntervalBase, SequenceBase, ExclusionBase, PointParsingError,
-    IntervalParsingError, SequenceDegenerateError)
+    IntervalParsingError, SequenceDegenerateError, cmp_to_rich, cmp)
 from cylc.wallclock import get_current_time_string
 from parsec.validate import IllegalValueError
 
@@ -168,6 +168,10 @@ def _iso_point_sub_point(point_string, other_point_string):
         return str(point - other_point)
 
 
+# TODO: replace __cmp__ infrastructure
+cmp_to_rich(ISO8601Point)
+
+
 class ISO8601Interval(IntervalBase):
 
     """The interval between points in an ISO8601 date time sequence."""
@@ -233,7 +237,7 @@ def __mul__(self, factor):
         """Return an interval with v * factor for v in this one's values."""
         return ISO8601Interval(self._iso_interval_mul(self.value, factor))
 
-    def __nonzero__(self):
+    def __bool__(self):
         """Return whether this interval has any non-null values."""
         return self._iso_interval_nonzero(self.value)
 
@@ -633,9 +637,15 @@ def __eq__(self, other):
             return True
         return False
 
+    def __lt__(self, other):
+        return self.value < other.value
+
     def __str__(self):
         return self.value
 
+    def __hash__(self):
+        return hash(self.value)
+
 
 def _get_old_anchor_step_recurrence(anchor, step, start_point):
     """Return a string representing an old-format recurrence translation."""
@@ -811,7 +821,7 @@ def init(num_expanded_year_digits=0, custom_dump_format=None, time_zone=None,
             SuiteSpecifics.DUMP_FORMAT = DATE_TIME_FORMAT + time_zone
     else:
         SuiteSpecifics.DUMP_FORMAT = custom_dump_format
-        if u"+X" not in custom_dump_format and num_expanded_year_digits:
+        if "+X" not in custom_dump_format and num_expanded_year_digits:
             raise IllegalValueError(
                 'cycle point format',
                 ('cylc', 'cycle point format'),
diff --git a/lib/cylc/cycling/loader.py b/lib/cylc/cycling/loader.py
index 0fc02b83177..f86e2a382a1 100755
--- a/lib/cylc/cycling/loader.py
+++ b/lib/cylc/cycling/loader.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
 
 # THIS FILE IS PART OF THE CYLC SUITE ENGINE.
 # Copyright (C) 2008-2019 NIWA & British Crown (Met Office) & Contributors.
diff --git a/lib/cylc/cycling/util.py b/lib/cylc/cycling/util.py
index 03aed58476a..d83d49d723d 100644
--- a/lib/cylc/cycling/util.py
+++ b/lib/cylc/cycling/util.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 # THIS FILE IS PART OF THE CYLC SUITE ENGINE.
 # Copyright (C) 2008-2019 NIWA & British Crown (Met Office) & Contributors.
diff --git a/lib/cylc/cylc-review/static/css/bootstrap.min.css b/lib/cylc/cylc-review/static/css/bootstrap.min.css
deleted file mode 100644
index 3e8b94d5ab9..00000000000
--- a/lib/cylc/cylc-review/static/css/bootstrap.min.css
+++ /dev/null
@@ -1,5 +0,0 @@
-/*!
- * Bootstrap v3.3.6 (http://getbootstrap.com)
- * Copyright 2011-2015 Twitter, Inc.
- * Licensed under MIT (https://github.com/twbs/bootstrap/blob/master/LICENSE)
- *//*! normalize.css v3.0.3 | MIT License | github.com/necolas/normalize.css */html{font-family:sans-serif;-webkit-text-size-adjust:100%;-ms-text-size-adjust:100%}body{margin:0}article,aside,details,figcaption,figure,footer,header,hgroup,main,menu,nav,section,summary{display:block}audio,canvas,progress,video{display:inline-block;vertical-align:baseline}audio:not([controls]){display:none;height:0}[hidden],template{display:none}a{background-color:transparent}a:active,a:hover{outline:0}abbr[title]{border-bottom:1px dotted}b,strong{font-weight:700}dfn{font-style:italic}h1{margin:.67em 0;font-size:2em}mark{color:#000;background:#ff0}small{font-size:80%}sub,sup{position:relative;font-size:75%;line-height:0;vertical-align:baseline}sup{top:-.5em}sub{bottom:-.25em}img{border:0}svg:not(:root){overflow:hidden}figure{margin:1em 40px}hr{height:0;-webkit-box-sizing:content-box;-moz-box-sizing:content-box;box-sizing:content-box}pre{overflow:auto}code,kbd,pre,samp{font-family:monospace,monospace;font-size:1em}button,input,optgroup,select,textarea{margin:0;font:inherit;color:inherit}button{overflow:visible}button,select{text-transform:none}button,html input[type=button],input[type=reset],input[type=submit]{-webkit-appearance:button;cursor:pointer}button[disabled],html input[disabled]{cursor:default}button::-moz-focus-inner,input::-moz-focus-inner{padding:0;border:0}input{line-height:normal}input[type=checkbox],input[type=radio]{-webkit-box-sizing:border-box;-moz-box-sizing:border-box;box-sizing:border-box;padding:0}input[type=number]::-webkit-inner-spin-button,input[type=number]::-webkit-outer-spin-button{height:auto}input[type=search]{-webkit-box-sizing:content-box;-moz-box-sizing:content-box;box-sizing:content-box;-webkit-appearance:textfield}input[type=search]::-webkit-search-cancel-button,input[type=search]::-webkit-search-decoration{-webkit-appearance:none}fieldset{padding:.35em .625em .75em;margin:0 2px;border:1px solid silver}legend{padding:0;border:0}textarea{overflow:auto}optgroup{font-weight:700}table{border-spacing:0;border-collapse:collapse}td,th{padding:0}/*! Source: https://github.com/h5bp/html5-boilerplate/blob/master/src/css/main.css */@media print{*,:after,:before{color:#000!important;text-shadow:none!important;background:0 0!important;-webkit-box-shadow:none!important;box-shadow:none!important}a,a:visited{text-decoration:underline}a[href]:after{content:" (" attr(href) ")"}abbr[title]:after{content:" (" attr(title) ")"}a[href^="javascript:"]:after,a[href^="#"]:after{content:""}blockquote,pre{border:1px solid #999;page-break-inside:avoid}thead{display:table-header-group}img,tr{page-break-inside:avoid}img{max-width:100%!important}h2,h3,p{orphans:3;widows:3}h2,h3{page-break-after:avoid}.navbar{display:none}.btn>.caret,.dropup>.btn>.caret{border-top-color:#000!important}.label{border:1px solid #000}.table{border-collapse:collapse!important}.table td,.table th{background-color:#fff!important}.table-bordered td,.table-bordered th{border:1px solid #ddd!important}}@font-face{font-family:'Glyphicons Halflings';src:url(../fonts/glyphicons-halflings-regular.eot);src:url(../fonts/glyphicons-halflings-regular.eot?#iefix) format('embedded-opentype'),url(../fonts/glyphicons-halflings-regular.woff2) format('woff2'),url(../fonts/glyphicons-halflings-regular.woff) format('woff'),url(../fonts/glyphicons-halflings-regular.ttf) format('truetype'),url(../fonts/glyphicons-halflings-regular.svg#glyphicons_halflingsregular) format('svg')}.glyphicon{position:relative;top:1px;display:inline-block;font-family:'Glyphicons Halflings';font-style:normal;font-weight:400;line-height:1;-webkit-font-smoothing:antialiased;-moz-osx-font-smoothing:grayscale}.glyphicon-asterisk:before{content:"\002a"}.glyphicon-plus:before{content:"\002b"}.glyphicon-eur:before,.glyphicon-euro:before{content:"\20ac"}.glyphicon-minus:before{content:"\2212"}.glyphicon-cloud:before{content:"\2601"}.glyphicon-envelope:before{content:"\2709"}.glyphicon-pencil:before{content:"\270f"}.glyphicon-glass:before{content:"\e001"}.glyphicon-music:before{content:"\e002"}.glyphicon-search:before{content:"\e003"}.glyphicon-heart:before{content:"\e005"}.glyphicon-star:before{content:"\e006"}.glyphicon-star-empty:before{content:"\e007"}.glyphicon-user:before{content:"\e008"}.glyphicon-film:before{content:"\e009"}.glyphicon-th-large:before{content:"\e010"}.glyphicon-th:before{content:"\e011"}.glyphicon-th-list:before{content:"\e012"}.glyphicon-ok:before{content:"\e013"}.glyphicon-remove:before{content:"\e014"}.glyphicon-zoom-in:before{content:"\e015"}.glyphicon-zoom-out:before{content:"\e016"}.glyphicon-off:before{content:"\e017"}.glyphicon-signal:before{content:"\e018"}.glyphicon-cog:before{content:"\e019"}.glyphicon-trash:before{content:"\e020"}.glyphicon-home:before{content:"\e021"}.glyphicon-file:before{content:"\e022"}.glyphicon-time:before{content:"\e023"}.glyphicon-road:before{content:"\e024"}.glyphicon-download-alt:before{content:"\e025"}.glyphicon-download:before{content:"\e026"}.glyphicon-upload:before{content:"\e027"}.glyphicon-inbox:before{content:"\e028"}.glyphicon-play-circle:before{content:"\e029"}.glyphicon-repeat:before{content:"\e030"}.glyphicon-refresh:before{content:"\e031"}.glyphicon-list-alt:before{content:"\e032"}.glyphicon-lock:before{content:"\e033"}.glyphicon-flag:before{content:"\e034"}.glyphicon-headphones:before{content:"\e035"}.glyphicon-volume-off:before{content:"\e036"}.glyphicon-volume-down:before{content:"\e037"}.glyphicon-volume-up:before{content:"\e038"}.glyphicon-qrcode:before{content:"\e039"}.glyphicon-barcode:before{content:"\e040"}.glyphicon-tag:before{content:"\e041"}.glyphicon-tags:before{content:"\e042"}.glyphicon-book:before{content:"\e043"}.glyphicon-bookmark:before{content:"\e044"}.glyphicon-print:before{content:"\e045"}.glyphicon-camera:before{content:"\e046"}.glyphicon-font:before{content:"\e047"}.glyphicon-bold:before{content:"\e048"}.glyphicon-italic:before{content:"\e049"}.glyphicon-text-height:before{content:"\e050"}.glyphicon-text-width:before{content:"\e051"}.glyphicon-align-left:before{content:"\e052"}.glyphicon-align-center:before{content:"\e053"}.glyphicon-align-right:before{content:"\e054"}.glyphicon-align-justify:before{content:"\e055"}.glyphicon-list:before{content:"\e056"}.glyphicon-indent-left:before{content:"\e057"}.glyphicon-indent-right:before{content:"\e058"}.glyphicon-facetime-video:before{content:"\e059"}.glyphicon-picture:before{content:"\e060"}.glyphicon-map-marker:before{content:"\e062"}.glyphicon-adjust:before{content:"\e063"}.glyphicon-tint:before{content:"\e064"}.glyphicon-edit:before{content:"\e065"}.glyphicon-share:before{content:"\e066"}.glyphicon-check:before{content:"\e067"}.glyphicon-move:before{content:"\e068"}.glyphicon-step-backward:before{content:"\e069"}.glyphicon-fast-backward:before{content:"\e070"}.glyphicon-backward:before{content:"\e071"}.glyphicon-play:before{content:"\e072"}.glyphicon-pause:before{content:"\e073"}.glyphicon-stop:before{content:"\e074"}.glyphicon-forward:before{content:"\e075"}.glyphicon-fast-forward:before{content:"\e076"}.glyphicon-step-forward:before{content:"\e077"}.glyphicon-eject:before{content:"\e078"}.glyphicon-chevron-left:before{content:"\e079"}.glyphicon-chevron-right:before{content:"\e080"}.glyphicon-plus-sign:before{content:"\e081"}.glyphicon-minus-sign:before{content:"\e082"}.glyphicon-remove-sign:before{content:"\e083"}.glyphicon-ok-sign:before{content:"\e084"}.glyphicon-question-sign:before{content:"\e085"}.glyphicon-info-sign:before{content:"\e086"}.glyphicon-screenshot:before{content:"\e087"}.glyphicon-remove-circle:before{content:"\e088"}.glyphicon-ok-circle:before{content:"\e089"}.glyphicon-ban-circle:before{content:"\e090"}.glyphicon-arrow-left:before{content:"\e091"}.glyphicon-arrow-right:before{content:"\e092"}.glyphicon-arrow-up:before{content:"\e093"}.glyphicon-arrow-down:before{content:"\e094"}.glyphicon-share-alt:before{content:"\e095"}.glyphicon-resize-full:before{content:"\e096"}.glyphicon-resize-small:before{content:"\e097"}.glyphicon-exclamation-sign:before{content:"\e101"}.glyphicon-gift:before{content:"\e102"}.glyphicon-leaf:before{content:"\e103"}.glyphicon-fire:before{content:"\e104"}.glyphicon-eye-open:before{content:"\e105"}.glyphicon-eye-close:before{content:"\e106"}.glyphicon-warning-sign:before{content:"\e107"}.glyphicon-plane:before{content:"\e108"}.glyphicon-calendar:before{content:"\e109"}.glyphicon-random:before{content:"\e110"}.glyphicon-comment:before{content:"\e111"}.glyphicon-magnet:before{content:"\e112"}.glyphicon-chevron-up:before{content:"\e113"}.glyphicon-chevron-down:before{content:"\e114"}.glyphicon-retweet:before{content:"\e115"}.glyphicon-shopping-cart:before{content:"\e116"}.glyphicon-folder-close:before{content:"\e117"}.glyphicon-folder-open:before{content:"\e118"}.glyphicon-resize-vertical:before{content:"\e119"}.glyphicon-resize-horizontal:before{content:"\e120"}.glyphicon-hdd:before{content:"\e121"}.glyphicon-bullhorn:before{content:"\e122"}.glyphicon-bell:before{content:"\e123"}.glyphicon-certificate:before{content:"\e124"}.glyphicon-thumbs-up:before{content:"\e125"}.glyphicon-thumbs-down:before{content:"\e126"}.glyphicon-hand-right:before{content:"\e127"}.glyphicon-hand-left:before{content:"\e128"}.glyphicon-hand-up:before{content:"\e129"}.glyphicon-hand-down:before{content:"\e130"}.glyphicon-circle-arrow-right:before{content:"\e131"}.glyphicon-circle-arrow-left:before{content:"\e132"}.glyphicon-circle-arrow-up:before{content:"\e133"}.glyphicon-circle-arrow-down:before{content:"\e134"}.glyphicon-globe:before{content:"\e135"}.glyphicon-wrench:before{content:"\e136"}.glyphicon-tasks:before{content:"\e137"}.glyphicon-filter:before{content:"\e138"}.glyphicon-briefcase:before{content:"\e139"}.glyphicon-fullscreen:before{content:"\e140"}.glyphicon-dashboard:before{content:"\e141"}.glyphicon-paperclip:before{content:"\e142"}.glyphicon-heart-empty:before{content:"\e143"}.glyphicon-link:before{content:"\e144"}.glyphicon-phone:before{content:"\e145"}.glyphicon-pushpin:before{content:"\e146"}.glyphicon-usd:before{content:"\e148"}.glyphicon-gbp:before{content:"\e149"}.glyphicon-sort:before{content:"\e150"}.glyphicon-sort-by-alphabet:before{content:"\e151"}.glyphicon-sort-by-alphabet-alt:before{content:"\e152"}.glyphicon-sort-by-order:before{content:"\e153"}.glyphicon-sort-by-order-alt:before{content:"\e154"}.glyphicon-sort-by-attributes:before{content:"\e155"}.glyphicon-sort-by-attributes-alt:before{content:"\e156"}.glyphicon-unchecked:before{content:"\e157"}.glyphicon-expand:before{content:"\e158"}.glyphicon-collapse-down:before{content:"\e159"}.glyphicon-collapse-up:before{content:"\e160"}.glyphicon-log-in:before{content:"\e161"}.glyphicon-flash:before{content:"\e162"}.glyphicon-log-out:before{content:"\e163"}.glyphicon-new-window:before{content:"\e164"}.glyphicon-record:before{content:"\e165"}.glyphicon-save:before{content:"\e166"}.glyphicon-open:before{content:"\e167"}.glyphicon-saved:before{content:"\e168"}.glyphicon-import:before{content:"\e169"}.glyphicon-export:before{content:"\e170"}.glyphicon-send:before{content:"\e171"}.glyphicon-floppy-disk:before{content:"\e172"}.glyphicon-floppy-saved:before{content:"\e173"}.glyphicon-floppy-remove:before{content:"\e174"}.glyphicon-floppy-save:before{content:"\e175"}.glyphicon-floppy-open:before{content:"\e176"}.glyphicon-credit-card:before{content:"\e177"}.glyphicon-transfer:before{content:"\e178"}.glyphicon-cutlery:before{content:"\e179"}.glyphicon-header:before{content:"\e180"}.glyphicon-compressed:before{content:"\e181"}.glyphicon-earphone:before{content:"\e182"}.glyphicon-phone-alt:before{content:"\e183"}.glyphicon-tower:before{content:"\e184"}.glyphicon-stats:before{content:"\e185"}.glyphicon-sd-video:before{content:"\e186"}.glyphicon-hd-video:before{content:"\e187"}.glyphicon-subtitles:before{content:"\e188"}.glyphicon-sound-stereo:before{content:"\e189"}.glyphicon-sound-dolby:before{content:"\e190"}.glyphicon-sound-5-1:before{content:"\e191"}.glyphicon-sound-6-1:before{content:"\e192"}.glyphicon-sound-7-1:before{content:"\e193"}.glyphicon-copyright-mark:before{content:"\e194"}.glyphicon-registration-mark:before{content:"\e195"}.glyphicon-cloud-download:before{content:"\e197"}.glyphicon-cloud-upload:before{content:"\e198"}.glyphicon-tree-conifer:before{content:"\e199"}.glyphicon-tree-deciduous:before{content:"\e200"}.glyphicon-cd:before{content:"\e201"}.glyphicon-save-file:before{content:"\e202"}.glyphicon-open-file:before{content:"\e203"}.glyphicon-level-up:before{content:"\e204"}.glyphicon-copy:before{content:"\e205"}.glyphicon-paste:before{content:"\e206"}.glyphicon-alert:before{content:"\e209"}.glyphicon-equalizer:before{content:"\e210"}.glyphicon-king:before{content:"\e211"}.glyphicon-queen:before{content:"\e212"}.glyphicon-pawn:before{content:"\e213"}.glyphicon-bishop:before{content:"\e214"}.glyphicon-knight:before{content:"\e215"}.glyphicon-baby-formula:before{content:"\e216"}.glyphicon-tent:before{content:"\26fa"}.glyphicon-blackboard:before{content:"\e218"}.glyphicon-bed:before{content:"\e219"}.glyphicon-apple:before{content:"\f8ff"}.glyphicon-erase:before{content:"\e221"}.glyphicon-hourglass:before{content:"\231b"}.glyphicon-lamp:before{content:"\e223"}.glyphicon-duplicate:before{content:"\e224"}.glyphicon-piggy-bank:before{content:"\e225"}.glyphicon-scissors:before{content:"\e226"}.glyphicon-bitcoin:before{content:"\e227"}.glyphicon-btc:before{content:"\e227"}.glyphicon-xbt:before{content:"\e227"}.glyphicon-yen:before{content:"\00a5"}.glyphicon-jpy:before{content:"\00a5"}.glyphicon-ruble:before{content:"\20bd"}.glyphicon-rub:before{content:"\20bd"}.glyphicon-scale:before{content:"\e230"}.glyphicon-ice-lolly:before{content:"\e231"}.glyphicon-ice-lolly-tasted:before{content:"\e232"}.glyphicon-education:before{content:"\e233"}.glyphicon-option-horizontal:before{content:"\e234"}.glyphicon-option-vertical:before{content:"\e235"}.glyphicon-menu-hamburger:before{content:"\e236"}.glyphicon-modal-window:before{content:"\e237"}.glyphicon-oil:before{content:"\e238"}.glyphicon-grain:before{content:"\e239"}.glyphicon-sunglasses:before{content:"\e240"}.glyphicon-text-size:before{content:"\e241"}.glyphicon-text-color:before{content:"\e242"}.glyphicon-text-background:before{content:"\e243"}.glyphicon-object-align-top:before{content:"\e244"}.glyphicon-object-align-bottom:before{content:"\e245"}.glyphicon-object-align-horizontal:before{content:"\e246"}.glyphicon-object-align-left:before{content:"\e247"}.glyphicon-object-align-vertical:before{content:"\e248"}.glyphicon-object-align-right:before{content:"\e249"}.glyphicon-triangle-right:before{content:"\e250"}.glyphicon-triangle-left:before{content:"\e251"}.glyphicon-triangle-bottom:before{content:"\e252"}.glyphicon-triangle-top:before{content:"\e253"}.glyphicon-console:before{content:"\e254"}.glyphicon-superscript:before{content:"\e255"}.glyphicon-subscript:before{content:"\e256"}.glyphicon-menu-left:before{content:"\e257"}.glyphicon-menu-right:before{content:"\e258"}.glyphicon-menu-down:before{content:"\e259"}.glyphicon-menu-up:before{content:"\e260"}*{-webkit-box-sizing:border-box;-moz-box-sizing:border-box;box-sizing:border-box}:after,:before{-webkit-box-sizing:border-box;-moz-box-sizing:border-box;box-sizing:border-box}html{font-size:10px;-webkit-tap-highlight-color:rgba(0,0,0,0)}body{font-family:"Helvetica Neue",Helvetica,Arial,sans-serif;font-size:14px;line-height:1.42857143;color:#333;background-color:#fff}button,input,select,textarea{font-family:inherit;font-size:inherit;line-height:inherit}a{color:#337ab7;text-decoration:none}a:focus,a:hover{color:#23527c;text-decoration:underline}a:focus{outline:thin dotted;outline:5px auto -webkit-focus-ring-color;outline-offset:-2px}figure{margin:0}img{vertical-align:middle}.carousel-inner>.item>a>img,.carousel-inner>.item>img,.img-responsive,.thumbnail a>img,.thumbnail>img{display:block;max-width:100%;height:auto}.img-rounded{border-radius:6px}.img-thumbnail{display:inline-block;max-width:100%;height:auto;padding:4px;line-height:1.42857143;background-color:#fff;border:1px solid #ddd;border-radius:4px;-webkit-transition:all .2s ease-in-out;-o-transition:all .2s ease-in-out;transition:all .2s ease-in-out}.img-circle{border-radius:50%}hr{margin-top:20px;margin-bottom:20px;border:0;border-top:1px solid #eee}.sr-only{position:absolute;width:1px;height:1px;padding:0;margin:-1px;overflow:hidden;clip:rect(0,0,0,0);border:0}.sr-only-focusable:active,.sr-only-focusable:focus{position:static;width:auto;height:auto;margin:0;overflow:visible;clip:auto}[role=button]{cursor:pointer}.h1,.h2,.h3,.h4,.h5,.h6,h1,h2,h3,h4,h5,h6{font-family:inherit;font-weight:500;line-height:1.1;color:inherit}.h1 .small,.h1 small,.h2 .small,.h2 small,.h3 .small,.h3 small,.h4 .small,.h4 small,.h5 .small,.h5 small,.h6 .small,.h6 small,h1 .small,h1 small,h2 .small,h2 small,h3 .small,h3 small,h4 .small,h4 small,h5 .small,h5 small,h6 .small,h6 small{font-weight:400;line-height:1;color:#777}.h1,.h2,.h3,h1,h2,h3{margin-top:20px;margin-bottom:10px}.h1 .small,.h1 small,.h2 .small,.h2 small,.h3 .small,.h3 small,h1 .small,h1 small,h2 .small,h2 small,h3 .small,h3 small{font-size:65%}.h4,.h5,.h6,h4,h5,h6{margin-top:10px;margin-bottom:10px}.h4 .small,.h4 small,.h5 .small,.h5 small,.h6 .small,.h6 small,h4 .small,h4 small,h5 .small,h5 small,h6 .small,h6 small{font-size:75%}.h1,h1{font-size:36px}.h2,h2{font-size:30px}.h3,h3{font-size:24px}.h4,h4{font-size:18px}.h5,h5{font-size:14px}.h6,h6{font-size:12px}p{margin:0 0 10px}.lead{margin-bottom:20px;font-size:16px;font-weight:300;line-height:1.4}@media (min-width:768px){.lead{font-size:21px}}.small,small{font-size:85%}.mark,mark{padding:.2em;background-color:#fcf8e3}.text-left{text-align:left}.text-right{text-align:right}.text-center{text-align:center}.text-justify{text-align:justify}.text-nowrap{white-space:nowrap}.text-lowercase{text-transform:lowercase}.text-uppercase{text-transform:uppercase}.text-capitalize{text-transform:capitalize}.text-muted{color:#777}.text-primary{color:#337ab7}a.text-primary:focus,a.text-primary:hover{color:#286090}.text-success{color:#3c763d}a.text-success:focus,a.text-success:hover{color:#2b542c}.text-info{color:#31708f}a.text-info:focus,a.text-info:hover{color:#245269}.text-warning{color:#8a6d3b}a.text-warning:focus,a.text-warning:hover{color:#66512c}.text-danger{color:#a94442}a.text-danger:focus,a.text-danger:hover{color:#843534}.bg-primary{color:#fff;background-color:#337ab7}a.bg-primary:focus,a.bg-primary:hover{background-color:#286090}.bg-success{background-color:#dff0d8}a.bg-success:focus,a.bg-success:hover{background-color:#c1e2b3}.bg-info{background-color:#d9edf7}a.bg-info:focus,a.bg-info:hover{background-color:#afd9ee}.bg-warning{background-color:#fcf8e3}a.bg-warning:focus,a.bg-warning:hover{background-color:#f7ecb5}.bg-danger{background-color:#f2dede}a.bg-danger:focus,a.bg-danger:hover{background-color:#e4b9b9}.page-header{padding-bottom:9px;margin:40px 0 20px;border-bottom:1px solid #eee}ol,ul{margin-top:0;margin-bottom:10px}ol ol,ol ul,ul ol,ul ul{margin-bottom:0}.list-unstyled{padding-left:0;list-style:none}.list-inline{padding-left:0;margin-left:-5px;list-style:none}.list-inline>li{display:inline-block;padding-right:5px;padding-left:5px}dl{margin-top:0;margin-bottom:20px}dd,dt{line-height:1.42857143}dt{font-weight:700}dd{margin-left:0}@media (min-width:768px){.dl-horizontal dt{float:left;width:160px;overflow:hidden;clear:left;text-align:right;text-overflow:ellipsis;white-space:nowrap}.dl-horizontal dd{margin-left:180px}}abbr[data-original-title],abbr[title]{cursor:help;border-bottom:1px dotted #777}.initialism{font-size:90%;text-transform:uppercase}blockquote{padding:10px 20px;margin:0 0 20px;font-size:17.5px;border-left:5px solid #eee}blockquote ol:last-child,blockquote p:last-child,blockquote ul:last-child{margin-bottom:0}blockquote .small,blockquote footer,blockquote small{display:block;font-size:80%;line-height:1.42857143;color:#777}blockquote .small:before,blockquote footer:before,blockquote small:before{content:'\2014 \00A0'}.blockquote-reverse,blockquote.pull-right{padding-right:15px;padding-left:0;text-align:right;border-right:5px solid #eee;border-left:0}.blockquote-reverse .small:before,.blockquote-reverse footer:before,.blockquote-reverse small:before,blockquote.pull-right .small:before,blockquote.pull-right footer:before,blockquote.pull-right small:before{content:''}.blockquote-reverse .small:after,.blockquote-reverse footer:after,.blockquote-reverse small:after,blockquote.pull-right .small:after,blockquote.pull-right footer:after,blockquote.pull-right small:after{content:'\00A0 \2014'}address{margin-bottom:20px;font-style:normal;line-height:1.42857143}code,kbd,pre,samp{font-family:Menlo,Monaco,Consolas,"Courier New",monospace}code{padding:2px 4px;font-size:90%;color:#c7254e;background-color:#f9f2f4;border-radius:4px}kbd{padding:2px 4px;font-size:90%;color:#fff;background-color:#333;border-radius:3px;-webkit-box-shadow:inset 0 -1px 0 rgba(0,0,0,.25);box-shadow:inset 0 -1px 0 rgba(0,0,0,.25)}kbd kbd{padding:0;font-size:100%;font-weight:700;-webkit-box-shadow:none;box-shadow:none}pre{display:block;padding:9.5px;margin:0 0 10px;font-size:13px;line-height:1.42857143;color:#333;word-break:break-all;word-wrap:break-word;background-color:#f5f5f5;border:1px solid #ccc;border-radius:4px}pre code{padding:0;font-size:inherit;color:inherit;white-space:pre-wrap;background-color:transparent;border-radius:0}.pre-scrollable{max-height:340px;overflow-y:scroll}.container{padding-right:15px;padding-left:15px;margin-right:auto;margin-left:auto}@media (min-width:768px){.container{width:750px}}@media (min-width:992px){.container{width:970px}}@media (min-width:1200px){.container{width:1170px}}.container-fluid{padding-right:15px;padding-left:15px;margin-right:auto;margin-left:auto}.row{margin-right:-15px;margin-left:-15px}.col-lg-1,.col-lg-10,.col-lg-11,.col-lg-12,.col-lg-2,.col-lg-3,.col-lg-4,.col-lg-5,.col-lg-6,.col-lg-7,.col-lg-8,.col-lg-9,.col-md-1,.col-md-10,.col-md-11,.col-md-12,.col-md-2,.col-md-3,.col-md-4,.col-md-5,.col-md-6,.col-md-7,.col-md-8,.col-md-9,.col-sm-1,.col-sm-10,.col-sm-11,.col-sm-12,.col-sm-2,.col-sm-3,.col-sm-4,.col-sm-5,.col-sm-6,.col-sm-7,.col-sm-8,.col-sm-9,.col-xs-1,.col-xs-10,.col-xs-11,.col-xs-12,.col-xs-2,.col-xs-3,.col-xs-4,.col-xs-5,.col-xs-6,.col-xs-7,.col-xs-8,.col-xs-9{position:relative;min-height:1px;padding-right:15px;padding-left:15px}.col-xs-1,.col-xs-10,.col-xs-11,.col-xs-12,.col-xs-2,.col-xs-3,.col-xs-4,.col-xs-5,.col-xs-6,.col-xs-7,.col-xs-8,.col-xs-9{float:left}.col-xs-12{width:100%}.col-xs-11{width:91.66666667%}.col-xs-10{width:83.33333333%}.col-xs-9{width:75%}.col-xs-8{width:66.66666667%}.col-xs-7{width:58.33333333%}.col-xs-6{width:50%}.col-xs-5{width:41.66666667%}.col-xs-4{width:33.33333333%}.col-xs-3{width:25%}.col-xs-2{width:16.66666667%}.col-xs-1{width:8.33333333%}.col-xs-pull-12{right:100%}.col-xs-pull-11{right:91.66666667%}.col-xs-pull-10{right:83.33333333%}.col-xs-pull-9{right:75%}.col-xs-pull-8{right:66.66666667%}.col-xs-pull-7{right:58.33333333%}.col-xs-pull-6{right:50%}.col-xs-pull-5{right:41.66666667%}.col-xs-pull-4{right:33.33333333%}.col-xs-pull-3{right:25%}.col-xs-pull-2{right:16.66666667%}.col-xs-pull-1{right:8.33333333%}.col-xs-pull-0{right:auto}.col-xs-push-12{left:100%}.col-xs-push-11{left:91.66666667%}.col-xs-push-10{left:83.33333333%}.col-xs-push-9{left:75%}.col-xs-push-8{left:66.66666667%}.col-xs-push-7{left:58.33333333%}.col-xs-push-6{left:50%}.col-xs-push-5{left:41.66666667%}.col-xs-push-4{left:33.33333333%}.col-xs-push-3{left:25%}.col-xs-push-2{left:16.66666667%}.col-xs-push-1{left:8.33333333%}.col-xs-push-0{left:auto}.col-xs-offset-12{margin-left:100%}.col-xs-offset-11{margin-left:91.66666667%}.col-xs-offset-10{margin-left:83.33333333%}.col-xs-offset-9{margin-left:75%}.col-xs-offset-8{margin-left:66.66666667%}.col-xs-offset-7{margin-left:58.33333333%}.col-xs-offset-6{margin-left:50%}.col-xs-offset-5{margin-left:41.66666667%}.col-xs-offset-4{margin-left:33.33333333%}.col-xs-offset-3{margin-left:25%}.col-xs-offset-2{margin-left:16.66666667%}.col-xs-offset-1{margin-left:8.33333333%}.col-xs-offset-0{margin-left:0}@media (min-width:768px){.col-sm-1,.col-sm-10,.col-sm-11,.col-sm-12,.col-sm-2,.col-sm-3,.col-sm-4,.col-sm-5,.col-sm-6,.col-sm-7,.col-sm-8,.col-sm-9{float:left}.col-sm-12{width:100%}.col-sm-11{width:91.66666667%}.col-sm-10{width:83.33333333%}.col-sm-9{width:75%}.col-sm-8{width:66.66666667%}.col-sm-7{width:58.33333333%}.col-sm-6{width:50%}.col-sm-5{width:41.66666667%}.col-sm-4{width:33.33333333%}.col-sm-3{width:25%}.col-sm-2{width:16.66666667%}.col-sm-1{width:8.33333333%}.col-sm-pull-12{right:100%}.col-sm-pull-11{right:91.66666667%}.col-sm-pull-10{right:83.33333333%}.col-sm-pull-9{right:75%}.col-sm-pull-8{right:66.66666667%}.col-sm-pull-7{right:58.33333333%}.col-sm-pull-6{right:50%}.col-sm-pull-5{right:41.66666667%}.col-sm-pull-4{right:33.33333333%}.col-sm-pull-3{right:25%}.col-sm-pull-2{right:16.66666667%}.col-sm-pull-1{right:8.33333333%}.col-sm-pull-0{right:auto}.col-sm-push-12{left:100%}.col-sm-push-11{left:91.66666667%}.col-sm-push-10{left:83.33333333%}.col-sm-push-9{left:75%}.col-sm-push-8{left:66.66666667%}.col-sm-push-7{left:58.33333333%}.col-sm-push-6{left:50%}.col-sm-push-5{left:41.66666667%}.col-sm-push-4{left:33.33333333%}.col-sm-push-3{left:25%}.col-sm-push-2{left:16.66666667%}.col-sm-push-1{left:8.33333333%}.col-sm-push-0{left:auto}.col-sm-offset-12{margin-left:100%}.col-sm-offset-11{margin-left:91.66666667%}.col-sm-offset-10{margin-left:83.33333333%}.col-sm-offset-9{margin-left:75%}.col-sm-offset-8{margin-left:66.66666667%}.col-sm-offset-7{margin-left:58.33333333%}.col-sm-offset-6{margin-left:50%}.col-sm-offset-5{margin-left:41.66666667%}.col-sm-offset-4{margin-left:33.33333333%}.col-sm-offset-3{margin-left:25%}.col-sm-offset-2{margin-left:16.66666667%}.col-sm-offset-1{margin-left:8.33333333%}.col-sm-offset-0{margin-left:0}}@media (min-width:992px){.col-md-1,.col-md-10,.col-md-11,.col-md-12,.col-md-2,.col-md-3,.col-md-4,.col-md-5,.col-md-6,.col-md-7,.col-md-8,.col-md-9{float:left}.col-md-12{width:100%}.col-md-11{width:91.66666667%}.col-md-10{width:83.33333333%}.col-md-9{width:75%}.col-md-8{width:66.66666667%}.col-md-7{width:58.33333333%}.col-md-6{width:50%}.col-md-5{width:41.66666667%}.col-md-4{width:33.33333333%}.col-md-3{width:25%}.col-md-2{width:16.66666667%}.col-md-1{width:8.33333333%}.col-md-pull-12{right:100%}.col-md-pull-11{right:91.66666667%}.col-md-pull-10{right:83.33333333%}.col-md-pull-9{right:75%}.col-md-pull-8{right:66.66666667%}.col-md-pull-7{right:58.33333333%}.col-md-pull-6{right:50%}.col-md-pull-5{right:41.66666667%}.col-md-pull-4{right:33.33333333%}.col-md-pull-3{right:25%}.col-md-pull-2{right:16.66666667%}.col-md-pull-1{right:8.33333333%}.col-md-pull-0{right:auto}.col-md-push-12{left:100%}.col-md-push-11{left:91.66666667%}.col-md-push-10{left:83.33333333%}.col-md-push-9{left:75%}.col-md-push-8{left:66.66666667%}.col-md-push-7{left:58.33333333%}.col-md-push-6{left:50%}.col-md-push-5{left:41.66666667%}.col-md-push-4{left:33.33333333%}.col-md-push-3{left:25%}.col-md-push-2{left:16.66666667%}.col-md-push-1{left:8.33333333%}.col-md-push-0{left:auto}.col-md-offset-12{margin-left:100%}.col-md-offset-11{margin-left:91.66666667%}.col-md-offset-10{margin-left:83.33333333%}.col-md-offset-9{margin-left:75%}.col-md-offset-8{margin-left:66.66666667%}.col-md-offset-7{margin-left:58.33333333%}.col-md-offset-6{margin-left:50%}.col-md-offset-5{margin-left:41.66666667%}.col-md-offset-4{margin-left:33.33333333%}.col-md-offset-3{margin-left:25%}.col-md-offset-2{margin-left:16.66666667%}.col-md-offset-1{margin-left:8.33333333%}.col-md-offset-0{margin-left:0}}@media (min-width:1200px){.col-lg-1,.col-lg-10,.col-lg-11,.col-lg-12,.col-lg-2,.col-lg-3,.col-lg-4,.col-lg-5,.col-lg-6,.col-lg-7,.col-lg-8,.col-lg-9{float:left}.col-lg-12{width:100%}.col-lg-11{width:91.66666667%}.col-lg-10{width:83.33333333%}.col-lg-9{width:75%}.col-lg-8{width:66.66666667%}.col-lg-7{width:58.33333333%}.col-lg-6{width:50%}.col-lg-5{width:41.66666667%}.col-lg-4{width:33.33333333%}.col-lg-3{width:25%}.col-lg-2{width:16.66666667%}.col-lg-1{width:8.33333333%}.col-lg-pull-12{right:100%}.col-lg-pull-11{right:91.66666667%}.col-lg-pull-10{right:83.33333333%}.col-lg-pull-9{right:75%}.col-lg-pull-8{right:66.66666667%}.col-lg-pull-7{right:58.33333333%}.col-lg-pull-6{right:50%}.col-lg-pull-5{right:41.66666667%}.col-lg-pull-4{right:33.33333333%}.col-lg-pull-3{right:25%}.col-lg-pull-2{right:16.66666667%}.col-lg-pull-1{right:8.33333333%}.col-lg-pull-0{right:auto}.col-lg-push-12{left:100%}.col-lg-push-11{left:91.66666667%}.col-lg-push-10{left:83.33333333%}.col-lg-push-9{left:75%}.col-lg-push-8{left:66.66666667%}.col-lg-push-7{left:58.33333333%}.col-lg-push-6{left:50%}.col-lg-push-5{left:41.66666667%}.col-lg-push-4{left:33.33333333%}.col-lg-push-3{left:25%}.col-lg-push-2{left:16.66666667%}.col-lg-push-1{left:8.33333333%}.col-lg-push-0{left:auto}.col-lg-offset-12{margin-left:100%}.col-lg-offset-11{margin-left:91.66666667%}.col-lg-offset-10{margin-left:83.33333333%}.col-lg-offset-9{margin-left:75%}.col-lg-offset-8{margin-left:66.66666667%}.col-lg-offset-7{margin-left:58.33333333%}.col-lg-offset-6{margin-left:50%}.col-lg-offset-5{margin-left:41.66666667%}.col-lg-offset-4{margin-left:33.33333333%}.col-lg-offset-3{margin-left:25%}.col-lg-offset-2{margin-left:16.66666667%}.col-lg-offset-1{margin-left:8.33333333%}.col-lg-offset-0{margin-left:0}}table{background-color:transparent}caption{padding-top:8px;padding-bottom:8px;color:#777;text-align:left}th{text-align:left}.table{width:100%;max-width:100%;margin-bottom:20px}.table>tbody>tr>td,.table>tbody>tr>th,.table>tfoot>tr>td,.table>tfoot>tr>th,.table>thead>tr>td,.table>thead>tr>th{padding:8px;line-height:1.42857143;vertical-align:top;border-top:1px solid #ddd}.table>thead>tr>th{vertical-align:bottom;border-bottom:2px solid #ddd}.table>caption+thead>tr:first-child>td,.table>caption+thead>tr:first-child>th,.table>colgroup+thead>tr:first-child>td,.table>colgroup+thead>tr:first-child>th,.table>thead:first-child>tr:first-child>td,.table>thead:first-child>tr:first-child>th{border-top:0}.table>tbody+tbody{border-top:2px solid #ddd}.table .table{background-color:#fff}.table-condensed>tbody>tr>td,.table-condensed>tbody>tr>th,.table-condensed>tfoot>tr>td,.table-condensed>tfoot>tr>th,.table-condensed>thead>tr>td,.table-condensed>thead>tr>th{padding:5px}.table-bordered{border:1px solid #ddd}.table-bordered>tbody>tr>td,.table-bordered>tbody>tr>th,.table-bordered>tfoot>tr>td,.table-bordered>tfoot>tr>th,.table-bordered>thead>tr>td,.table-bordered>thead>tr>th{border:1px solid #ddd}.table-bordered>thead>tr>td,.table-bordered>thead>tr>th{border-bottom-width:2px}.table-striped>tbody>tr:nth-of-type(odd){background-color:#f9f9f9}.table-hover>tbody>tr:hover{background-color:#f5f5f5}table col[class*=col-]{position:static;display:table-column;float:none}table td[class*=col-],table th[class*=col-]{position:static;display:table-cell;float:none}.table>tbody>tr.active>td,.table>tbody>tr.active>th,.table>tbody>tr>td.active,.table>tbody>tr>th.active,.table>tfoot>tr.active>td,.table>tfoot>tr.active>th,.table>tfoot>tr>td.active,.table>tfoot>tr>th.active,.table>thead>tr.active>td,.table>thead>tr.active>th,.table>thead>tr>td.active,.table>thead>tr>th.active{background-color:#f5f5f5}.table-hover>tbody>tr.active:hover>td,.table-hover>tbody>tr.active:hover>th,.table-hover>tbody>tr:hover>.active,.table-hover>tbody>tr>td.active:hover,.table-hover>tbody>tr>th.active:hover{background-color:#e8e8e8}.table>tbody>tr.success>td,.table>tbody>tr.success>th,.table>tbody>tr>td.success,.table>tbody>tr>th.success,.table>tfoot>tr.success>td,.table>tfoot>tr.success>th,.table>tfoot>tr>td.success,.table>tfoot>tr>th.success,.table>thead>tr.success>td,.table>thead>tr.success>th,.table>thead>tr>td.success,.table>thead>tr>th.success{background-color:#dff0d8}.table-hover>tbody>tr.success:hover>td,.table-hover>tbody>tr.success:hover>th,.table-hover>tbody>tr:hover>.success,.table-hover>tbody>tr>td.success:hover,.table-hover>tbody>tr>th.success:hover{background-color:#d0e9c6}.table>tbody>tr.info>td,.table>tbody>tr.info>th,.table>tbody>tr>td.info,.table>tbody>tr>th.info,.table>tfoot>tr.info>td,.table>tfoot>tr.info>th,.table>tfoot>tr>td.info,.table>tfoot>tr>th.info,.table>thead>tr.info>td,.table>thead>tr.info>th,.table>thead>tr>td.info,.table>thead>tr>th.info{background-color:#d9edf7}.table-hover>tbody>tr.info:hover>td,.table-hover>tbody>tr.info:hover>th,.table-hover>tbody>tr:hover>.info,.table-hover>tbody>tr>td.info:hover,.table-hover>tbody>tr>th.info:hover{background-color:#c4e3f3}.table>tbody>tr.warning>td,.table>tbody>tr.warning>th,.table>tbody>tr>td.warning,.table>tbody>tr>th.warning,.table>tfoot>tr.warning>td,.table>tfoot>tr.warning>th,.table>tfoot>tr>td.warning,.table>tfoot>tr>th.warning,.table>thead>tr.warning>td,.table>thead>tr.warning>th,.table>thead>tr>td.warning,.table>thead>tr>th.warning{background-color:#fcf8e3}.table-hover>tbody>tr.warning:hover>td,.table-hover>tbody>tr.warning:hover>th,.table-hover>tbody>tr:hover>.warning,.table-hover>tbody>tr>td.warning:hover,.table-hover>tbody>tr>th.warning:hover{background-color:#faf2cc}.table>tbody>tr.danger>td,.table>tbody>tr.danger>th,.table>tbody>tr>td.danger,.table>tbody>tr>th.danger,.table>tfoot>tr.danger>td,.table>tfoot>tr.danger>th,.table>tfoot>tr>td.danger,.table>tfoot>tr>th.danger,.table>thead>tr.danger>td,.table>thead>tr.danger>th,.table>thead>tr>td.danger,.table>thead>tr>th.danger{background-color:#f2dede}.table-hover>tbody>tr.danger:hover>td,.table-hover>tbody>tr.danger:hover>th,.table-hover>tbody>tr:hover>.danger,.table-hover>tbody>tr>td.danger:hover,.table-hover>tbody>tr>th.danger:hover{background-color:#ebcccc}.table-responsive{min-height:.01%;overflow-x:auto}@media screen and (max-width:767px){.table-responsive{width:100%;margin-bottom:15px;overflow-y:hidden;-ms-overflow-style:-ms-autohiding-scrollbar;border:1px solid #ddd}.table-responsive>.table{margin-bottom:0}.table-responsive>.table>tbody>tr>td,.table-responsive>.table>tbody>tr>th,.table-responsive>.table>tfoot>tr>td,.table-responsive>.table>tfoot>tr>th,.table-responsive>.table>thead>tr>td,.table-responsive>.table>thead>tr>th{white-space:nowrap}.table-responsive>.table-bordered{border:0}.table-responsive>.table-bordered>tbody>tr>td:first-child,.table-responsive>.table-bordered>tbody>tr>th:first-child,.table-responsive>.table-bordered>tfoot>tr>td:first-child,.table-responsive>.table-bordered>tfoot>tr>th:first-child,.table-responsive>.table-bordered>thead>tr>td:first-child,.table-responsive>.table-bordered>thead>tr>th:first-child{border-left:0}.table-responsive>.table-bordered>tbody>tr>td:last-child,.table-responsive>.table-bordered>tbody>tr>th:last-child,.table-responsive>.table-bordered>tfoot>tr>td:last-child,.table-responsive>.table-bordered>tfoot>tr>th:last-child,.table-responsive>.table-bordered>thead>tr>td:last-child,.table-responsive>.table-bordered>thead>tr>th:last-child{border-right:0}.table-responsive>.table-bordered>tbody>tr:last-child>td,.table-responsive>.table-bordered>tbody>tr:last-child>th,.table-responsive>.table-bordered>tfoot>tr:last-child>td,.table-responsive>.table-bordered>tfoot>tr:last-child>th{border-bottom:0}}fieldset{min-width:0;padding:0;margin:0;border:0}legend{display:block;width:100%;padding:0;margin-bottom:20px;font-size:21px;line-height:inherit;color:#333;border:0;border-bottom:1px solid #e5e5e5}label{display:inline-block;max-width:100%;margin-bottom:5px;font-weight:700}input[type=search]{-webkit-box-sizing:border-box;-moz-box-sizing:border-box;box-sizing:border-box}input[type=checkbox],input[type=radio]{margin:4px 0 0;margin-top:1px\9;line-height:normal}input[type=file]{display:block}input[type=range]{display:block;width:100%}select[multiple],select[size]{height:auto}input[type=file]:focus,input[type=checkbox]:focus,input[type=radio]:focus{outline:thin dotted;outline:5px auto -webkit-focus-ring-color;outline-offset:-2px}output{display:block;padding-top:7px;font-size:14px;line-height:1.42857143;color:#555}.form-control{display:block;width:100%;height:34px;padding:6px 12px;font-size:14px;line-height:1.42857143;color:#555;background-color:#fff;background-image:none;border:1px solid #ccc;border-radius:4px;-webkit-box-shadow:inset 0 1px 1px rgba(0,0,0,.075);box-shadow:inset 0 1px 1px rgba(0,0,0,.075);-webkit-transition:border-color ease-in-out .15s,-webkit-box-shadow ease-in-out .15s;-o-transition:border-color ease-in-out .15s,box-shadow ease-in-out .15s;transition:border-color ease-in-out .15s,box-shadow ease-in-out .15s}.form-control:focus{border-color:#66afe9;outline:0;-webkit-box-shadow:inset 0 1px 1px rgba(0,0,0,.075),0 0 8px rgba(102,175,233,.6);box-shadow:inset 0 1px 1px rgba(0,0,0,.075),0 0 8px rgba(102,175,233,.6)}.form-control::-moz-placeholder{color:#999;opacity:1}.form-control:-ms-input-placeholder{color:#999}.form-control::-webkit-input-placeholder{color:#999}.form-control::-ms-expand{background-color:transparent;border:0}.form-control[disabled],.form-control[readonly],fieldset[disabled] .form-control{background-color:#eee;opacity:1}.form-control[disabled],fieldset[disabled] .form-control{cursor:not-allowed}textarea.form-control{height:auto}input[type=search]{-webkit-appearance:none}@media screen and (-webkit-min-device-pixel-ratio:0){input[type=date].form-control,input[type=time].form-control,input[type=datetime-local].form-control,input[type=month].form-control{line-height:34px}.input-group-sm input[type=date],.input-group-sm input[type=time],.input-group-sm input[type=datetime-local],.input-group-sm input[type=month],input[type=date].input-sm,input[type=time].input-sm,input[type=datetime-local].input-sm,input[type=month].input-sm{line-height:30px}.input-group-lg input[type=date],.input-group-lg input[type=time],.input-group-lg input[type=datetime-local],.input-group-lg input[type=month],input[type=date].input-lg,input[type=time].input-lg,input[type=datetime-local].input-lg,input[type=month].input-lg{line-height:46px}}.form-group{margin-bottom:15px}.checkbox,.radio{position:relative;display:block;margin-top:10px;margin-bottom:10px}.checkbox label,.radio label{min-height:20px;padding-left:20px;margin-bottom:0;font-weight:400;cursor:pointer}.checkbox input[type=checkbox],.checkbox-inline input[type=checkbox],.radio input[type=radio],.radio-inline input[type=radio]{position:absolute;margin-top:4px\9;margin-left:-20px}.checkbox+.checkbox,.radio+.radio{margin-top:-5px}.checkbox-inline,.radio-inline{position:relative;display:inline-block;padding-left:20px;margin-bottom:0;font-weight:400;vertical-align:middle;cursor:pointer}.checkbox-inline+.checkbox-inline,.radio-inline+.radio-inline{margin-top:0;margin-left:10px}fieldset[disabled] input[type=checkbox],fieldset[disabled] input[type=radio],input[type=checkbox].disabled,input[type=checkbox][disabled],input[type=radio].disabled,input[type=radio][disabled]{cursor:not-allowed}.checkbox-inline.disabled,.radio-inline.disabled,fieldset[disabled] .checkbox-inline,fieldset[disabled] .radio-inline{cursor:not-allowed}.checkbox.disabled label,.radio.disabled label,fieldset[disabled] .checkbox label,fieldset[disabled] .radio label{cursor:not-allowed}.form-control-static{min-height:34px;padding-top:7px;padding-bottom:7px;margin-bottom:0}.form-control-static.input-lg,.form-control-static.input-sm{padding-right:0;padding-left:0}.input-sm{height:30px;padding:5px 10px;font-size:12px;line-height:1.5;border-radius:3px}select.input-sm{height:30px;line-height:30px}select[multiple].input-sm,textarea.input-sm{height:auto}.form-group-sm .form-control{height:30px;padding:5px 10px;font-size:12px;line-height:1.5;border-radius:3px}.form-group-sm select.form-control{height:30px;line-height:30px}.form-group-sm select[multiple].form-control,.form-group-sm textarea.form-control{height:auto}.form-group-sm .form-control-static{height:30px;min-height:32px;padding:6px 10px;font-size:12px;line-height:1.5}.input-lg{height:46px;padding:10px 16px;font-size:18px;line-height:1.3333333;border-radius:6px}select.input-lg{height:46px;line-height:46px}select[multiple].input-lg,textarea.input-lg{height:auto}.form-group-lg .form-control{height:46px;padding:10px 16px;font-size:18px;line-height:1.3333333;border-radius:6px}.form-group-lg select.form-control{height:46px;line-height:46px}.form-group-lg select[multiple].form-control,.form-group-lg textarea.form-control{height:auto}.form-group-lg .form-control-static{height:46px;min-height:38px;padding:11px 16px;font-size:18px;line-height:1.3333333}.has-feedback{position:relative}.has-feedback .form-control{padding-right:42.5px}.form-control-feedback{position:absolute;top:0;right:0;z-index:2;display:block;width:34px;height:34px;line-height:34px;text-align:center;pointer-events:none}.form-group-lg .form-control+.form-control-feedback,.input-group-lg+.form-control-feedback,.input-lg+.form-control-feedback{width:46px;height:46px;line-height:46px}.form-group-sm .form-control+.form-control-feedback,.input-group-sm+.form-control-feedback,.input-sm+.form-control-feedback{width:30px;height:30px;line-height:30px}.has-success .checkbox,.has-success .checkbox-inline,.has-success .control-label,.has-success .help-block,.has-success .radio,.has-success .radio-inline,.has-success.checkbox label,.has-success.checkbox-inline label,.has-success.radio label,.has-success.radio-inline label{color:#3c763d}.has-success .form-control{border-color:#3c763d;-webkit-box-shadow:inset 0 1px 1px rgba(0,0,0,.075);box-shadow:inset 0 1px 1px rgba(0,0,0,.075)}.has-success .form-control:focus{border-color:#2b542c;-webkit-box-shadow:inset 0 1px 1px rgba(0,0,0,.075),0 0 6px #67b168;box-shadow:inset 0 1px 1px rgba(0,0,0,.075),0 0 6px #67b168}.has-success .input-group-addon{color:#3c763d;background-color:#dff0d8;border-color:#3c763d}.has-success .form-control-feedback{color:#3c763d}.has-warning .checkbox,.has-warning .checkbox-inline,.has-warning .control-label,.has-warning .help-block,.has-warning .radio,.has-warning .radio-inline,.has-warning.checkbox label,.has-warning.checkbox-inline label,.has-warning.radio label,.has-warning.radio-inline label{color:#8a6d3b}.has-warning .form-control{border-color:#8a6d3b;-webkit-box-shadow:inset 0 1px 1px rgba(0,0,0,.075);box-shadow:inset 0 1px 1px rgba(0,0,0,.075)}.has-warning .form-control:focus{border-color:#66512c;-webkit-box-shadow:inset 0 1px 1px rgba(0,0,0,.075),0 0 6px #c0a16b;box-shadow:inset 0 1px 1px rgba(0,0,0,.075),0 0 6px #c0a16b}.has-warning .input-group-addon{color:#8a6d3b;background-color:#fcf8e3;border-color:#8a6d3b}.has-warning .form-control-feedback{color:#8a6d3b}.has-error .checkbox,.has-error .checkbox-inline,.has-error .control-label,.has-error .help-block,.has-error .radio,.has-error .radio-inline,.has-error.checkbox label,.has-error.checkbox-inline label,.has-error.radio label,.has-error.radio-inline label{color:#a94442}.has-error .form-control{border-color:#a94442;-webkit-box-shadow:inset 0 1px 1px rgba(0,0,0,.075);box-shadow:inset 0 1px 1px rgba(0,0,0,.075)}.has-error .form-control:focus{border-color:#843534;-webkit-box-shadow:inset 0 1px 1px rgba(0,0,0,.075),0 0 6px #ce8483;box-shadow:inset 0 1px 1px rgba(0,0,0,.075),0 0 6px #ce8483}.has-error .input-group-addon{color:#a94442;background-color:#f2dede;border-color:#a94442}.has-error .form-control-feedback{color:#a94442}.has-feedback label~.form-control-feedback{top:25px}.has-feedback label.sr-only~.form-control-feedback{top:0}.help-block{display:block;margin-top:5px;margin-bottom:10px;color:#737373}@media (min-width:768px){.form-inline .form-group{display:inline-block;margin-bottom:0;vertical-align:middle}.form-inline .form-control{display:inline-block;width:auto;vertical-align:middle}.form-inline .form-control-static{display:inline-block}.form-inline .input-group{display:inline-table;vertical-align:middle}.form-inline .input-group .form-control,.form-inline .input-group .input-group-addon,.form-inline .input-group .input-group-btn{width:auto}.form-inline .input-group>.form-control{width:100%}.form-inline .control-label{margin-bottom:0;vertical-align:middle}.form-inline .checkbox,.form-inline .radio{display:inline-block;margin-top:0;margin-bottom:0;vertical-align:middle}.form-inline .checkbox label,.form-inline .radio label{padding-left:0}.form-inline .checkbox input[type=checkbox],.form-inline .radio input[type=radio]{position:relative;margin-left:0}.form-inline .has-feedback .form-control-feedback{top:0}}.form-horizontal .checkbox,.form-horizontal .checkbox-inline,.form-horizontal .radio,.form-horizontal .radio-inline{padding-top:7px;margin-top:0;margin-bottom:0}.form-horizontal .checkbox,.form-horizontal .radio{min-height:27px}.form-horizontal .form-group{margin-right:-15px;margin-left:-15px}@media (min-width:768px){.form-horizontal .control-label{padding-top:7px;margin-bottom:0;text-align:right}}.form-horizontal .has-feedback .form-control-feedback{right:15px}@media (min-width:768px){.form-horizontal .form-group-lg .control-label{padding-top:11px;font-size:18px}}@media (min-width:768px){.form-horizontal .form-group-sm .control-label{padding-top:6px;font-size:12px}}.btn{display:inline-block;padding:6px 12px;margin-bottom:0;font-size:14px;font-weight:400;line-height:1.42857143;text-align:center;white-space:nowrap;vertical-align:middle;-ms-touch-action:manipulation;touch-action:manipulation;cursor:pointer;-webkit-user-select:none;-moz-user-select:none;-ms-user-select:none;user-select:none;background-image:none;border:1px solid transparent;border-radius:4px}.btn.active.focus,.btn.active:focus,.btn.focus,.btn:active.focus,.btn:active:focus,.btn:focus{outline:thin dotted;outline:5px auto -webkit-focus-ring-color;outline-offset:-2px}.btn.focus,.btn:focus,.btn:hover{color:#333;text-decoration:none}.btn.active,.btn:active{background-image:none;outline:0;-webkit-box-shadow:inset 0 3px 5px rgba(0,0,0,.125);box-shadow:inset 0 3px 5px rgba(0,0,0,.125)}.btn.disabled,.btn[disabled],fieldset[disabled] .btn{cursor:not-allowed;filter:alpha(opacity=65);-webkit-box-shadow:none;box-shadow:none;opacity:.65}a.btn.disabled,fieldset[disabled] a.btn{pointer-events:none}.btn-default{color:#333;background-color:#fff;border-color:#ccc}.btn-default.focus,.btn-default:focus{color:#333;background-color:#e6e6e6;border-color:#8c8c8c}.btn-default:hover{color:#333;background-color:#e6e6e6;border-color:#adadad}.btn-default.active,.btn-default:active,.open>.dropdown-toggle.btn-default{color:#333;background-color:#e6e6e6;border-color:#adadad}.btn-default.active.focus,.btn-default.active:focus,.btn-default.active:hover,.btn-default:active.focus,.btn-default:active:focus,.btn-default:active:hover,.open>.dropdown-toggle.btn-default.focus,.open>.dropdown-toggle.btn-default:focus,.open>.dropdown-toggle.btn-default:hover{color:#333;background-color:#d4d4d4;border-color:#8c8c8c}.btn-default.active,.btn-default:active,.open>.dropdown-toggle.btn-default{background-image:none}.btn-default.disabled.focus,.btn-default.disabled:focus,.btn-default.disabled:hover,.btn-default[disabled].focus,.btn-default[disabled]:focus,.btn-default[disabled]:hover,fieldset[disabled] .btn-default.focus,fieldset[disabled] .btn-default:focus,fieldset[disabled] .btn-default:hover{background-color:#fff;border-color:#ccc}.btn-default .badge{color:#fff;background-color:#333}.btn-primary{color:#fff;background-color:#337ab7;border-color:#2e6da4}.btn-primary.focus,.btn-primary:focus{color:#fff;background-color:#286090;border-color:#122b40}.btn-primary:hover{color:#fff;background-color:#286090;border-color:#204d74}.btn-primary.active,.btn-primary:active,.open>.dropdown-toggle.btn-primary{color:#fff;background-color:#286090;border-color:#204d74}.btn-primary.active.focus,.btn-primary.active:focus,.btn-primary.active:hover,.btn-primary:active.focus,.btn-primary:active:focus,.btn-primary:active:hover,.open>.dropdown-toggle.btn-primary.focus,.open>.dropdown-toggle.btn-primary:focus,.open>.dropdown-toggle.btn-primary:hover{color:#fff;background-color:#204d74;border-color:#122b40}.btn-primary.active,.btn-primary:active,.open>.dropdown-toggle.btn-primary{background-image:none}.btn-primary.disabled.focus,.btn-primary.disabled:focus,.btn-primary.disabled:hover,.btn-primary[disabled].focus,.btn-primary[disabled]:focus,.btn-primary[disabled]:hover,fieldset[disabled] .btn-primary.focus,fieldset[disabled] .btn-primary:focus,fieldset[disabled] .btn-primary:hover{background-color:#337ab7;border-color:#2e6da4}.btn-primary .badge{color:#337ab7;background-color:#fff}.btn-success{color:#fff;background-color:#5cb85c;border-color:#4cae4c}.btn-success.focus,.btn-success:focus{color:#fff;background-color:#449d44;border-color:#255625}.btn-success:hover{color:#fff;background-color:#449d44;border-color:#398439}.btn-success.active,.btn-success:active,.open>.dropdown-toggle.btn-success{color:#fff;background-color:#449d44;border-color:#398439}.btn-success.active.focus,.btn-success.active:focus,.btn-success.active:hover,.btn-success:active.focus,.btn-success:active:focus,.btn-success:active:hover,.open>.dropdown-toggle.btn-success.focus,.open>.dropdown-toggle.btn-success:focus,.open>.dropdown-toggle.btn-success:hover{color:#fff;background-color:#398439;border-color:#255625}.btn-success.active,.btn-success:active,.open>.dropdown-toggle.btn-success{background-image:none}.btn-success.disabled.focus,.btn-success.disabled:focus,.btn-success.disabled:hover,.btn-success[disabled].focus,.btn-success[disabled]:focus,.btn-success[disabled]:hover,fieldset[disabled] .btn-success.focus,fieldset[disabled] .btn-success:focus,fieldset[disabled] .btn-success:hover{background-color:#5cb85c;border-color:#4cae4c}.btn-success .badge{color:#5cb85c;background-color:#fff}.btn-info{color:#fff;background-color:#5bc0de;border-color:#46b8da}.btn-info.focus,.btn-info:focus{color:#fff;background-color:#31b0d5;border-color:#1b6d85}.btn-info:hover{color:#fff;background-color:#31b0d5;border-color:#269abc}.btn-info.active,.btn-info:active,.open>.dropdown-toggle.btn-info{color:#fff;background-color:#31b0d5;border-color:#269abc}.btn-info.active.focus,.btn-info.active:focus,.btn-info.active:hover,.btn-info:active.focus,.btn-info:active:focus,.btn-info:active:hover,.open>.dropdown-toggle.btn-info.focus,.open>.dropdown-toggle.btn-info:focus,.open>.dropdown-toggle.btn-info:hover{color:#fff;background-color:#269abc;border-color:#1b6d85}.btn-info.active,.btn-info:active,.open>.dropdown-toggle.btn-info{background-image:none}.btn-info.disabled.focus,.btn-info.disabled:focus,.btn-info.disabled:hover,.btn-info[disabled].focus,.btn-info[disabled]:focus,.btn-info[disabled]:hover,fieldset[disabled] .btn-info.focus,fieldset[disabled] .btn-info:focus,fieldset[disabled] .btn-info:hover{background-color:#5bc0de;border-color:#46b8da}.btn-info .badge{color:#5bc0de;background-color:#fff}.btn-warning{color:#fff;background-color:#f0ad4e;border-color:#eea236}.btn-warning.focus,.btn-warning:focus{color:#fff;background-color:#ec971f;border-color:#985f0d}.btn-warning:hover{color:#fff;background-color:#ec971f;border-color:#d58512}.btn-warning.active,.btn-warning:active,.open>.dropdown-toggle.btn-warning{color:#fff;background-color:#ec971f;border-color:#d58512}.btn-warning.active.focus,.btn-warning.active:focus,.btn-warning.active:hover,.btn-warning:active.focus,.btn-warning:active:focus,.btn-warning:active:hover,.open>.dropdown-toggle.btn-warning.focus,.open>.dropdown-toggle.btn-warning:focus,.open>.dropdown-toggle.btn-warning:hover{color:#fff;background-color:#d58512;border-color:#985f0d}.btn-warning.active,.btn-warning:active,.open>.dropdown-toggle.btn-warning{background-image:none}.btn-warning.disabled.focus,.btn-warning.disabled:focus,.btn-warning.disabled:hover,.btn-warning[disabled].focus,.btn-warning[disabled]:focus,.btn-warning[disabled]:hover,fieldset[disabled] .btn-warning.focus,fieldset[disabled] .btn-warning:focus,fieldset[disabled] .btn-warning:hover{background-color:#f0ad4e;border-color:#eea236}.btn-warning .badge{color:#f0ad4e;background-color:#fff}.btn-danger{color:#fff;background-color:#d9534f;border-color:#d43f3a}.btn-danger.focus,.btn-danger:focus{color:#fff;background-color:#c9302c;border-color:#761c19}.btn-danger:hover{color:#fff;background-color:#c9302c;border-color:#ac2925}.btn-danger.active,.btn-danger:active,.open>.dropdown-toggle.btn-danger{color:#fff;background-color:#c9302c;border-color:#ac2925}.btn-danger.active.focus,.btn-danger.active:focus,.btn-danger.active:hover,.btn-danger:active.focus,.btn-danger:active:focus,.btn-danger:active:hover,.open>.dropdown-toggle.btn-danger.focus,.open>.dropdown-toggle.btn-danger:focus,.open>.dropdown-toggle.btn-danger:hover{color:#fff;background-color:#ac2925;border-color:#761c19}.btn-danger.active,.btn-danger:active,.open>.dropdown-toggle.btn-danger{background-image:none}.btn-danger.disabled.focus,.btn-danger.disabled:focus,.btn-danger.disabled:hover,.btn-danger[disabled].focus,.btn-danger[disabled]:focus,.btn-danger[disabled]:hover,fieldset[disabled] .btn-danger.focus,fieldset[disabled] .btn-danger:focus,fieldset[disabled] .btn-danger:hover{background-color:#d9534f;border-color:#d43f3a}.btn-danger .badge{color:#d9534f;background-color:#fff}.btn-link{font-weight:400;color:#337ab7;border-radius:0}.btn-link,.btn-link.active,.btn-link:active,.btn-link[disabled],fieldset[disabled] .btn-link{background-color:transparent;-webkit-box-shadow:none;box-shadow:none}.btn-link,.btn-link:active,.btn-link:focus,.btn-link:hover{border-color:transparent}.btn-link:focus,.btn-link:hover{color:#23527c;text-decoration:underline;background-color:transparent}.btn-link[disabled]:focus,.btn-link[disabled]:hover,fieldset[disabled] .btn-link:focus,fieldset[disabled] .btn-link:hover{color:#777;text-decoration:none}.btn-group-lg>.btn,.btn-lg{padding:10px 16px;font-size:18px;line-height:1.3333333;border-radius:6px}.btn-group-sm>.btn,.btn-sm{padding:5px 10px;font-size:12px;line-height:1.5;border-radius:3px}.btn-group-xs>.btn,.btn-xs{padding:1px 5px;font-size:12px;line-height:1.5;border-radius:3px}.btn-block{display:block;width:100%}.btn-block+.btn-block{margin-top:5px}input[type=button].btn-block,input[type=reset].btn-block,input[type=submit].btn-block{width:100%}.fade{opacity:0;-webkit-transition:opacity .15s linear;-o-transition:opacity .15s linear;transition:opacity .15s linear}.fade.in{opacity:1}.collapse{display:none}.collapse.in{display:block}tr.collapse.in{display:table-row}tbody.collapse.in{display:table-row-group}.collapsing{position:relative;height:0;overflow:hidden;-webkit-transition-timing-function:ease;-o-transition-timing-function:ease;transition-timing-function:ease;-webkit-transition-duration:.35s;-o-transition-duration:.35s;transition-duration:.35s;-webkit-transition-property:height,visibility;-o-transition-property:height,visibility;transition-property:height,visibility}.caret{display:inline-block;width:0;height:0;margin-left:2px;vertical-align:middle;border-top:4px dashed;border-top:4px solid\9;border-right:4px solid transparent;border-left:4px solid transparent}.dropdown,.dropup{position:relative}.dropdown-toggle:focus{outline:0}.dropdown-menu{position:absolute;top:100%;left:0;z-index:1000;display:none;float:left;min-width:160px;padding:5px 0;margin:2px 0 0;font-size:14px;text-align:left;list-style:none;background-color:#fff;-webkit-background-clip:padding-box;background-clip:padding-box;border:1px solid #ccc;border:1px solid rgba(0,0,0,.15);border-radius:4px;-webkit-box-shadow:0 6px 12px rgba(0,0,0,.175);box-shadow:0 6px 12px rgba(0,0,0,.175)}.dropdown-menu.pull-right{right:0;left:auto}.dropdown-menu .divider{height:1px;margin:9px 0;overflow:hidden;background-color:#e5e5e5}.dropdown-menu>li>a{display:block;padding:3px 20px;clear:both;font-weight:400;line-height:1.42857143;color:#333;white-space:nowrap}.dropdown-menu>li>a:focus,.dropdown-menu>li>a:hover{color:#262626;text-decoration:none;background-color:#f5f5f5}.dropdown-menu>.active>a,.dropdown-menu>.active>a:focus,.dropdown-menu>.active>a:hover{color:#fff;text-decoration:none;background-color:#337ab7;outline:0}.dropdown-menu>.disabled>a,.dropdown-menu>.disabled>a:focus,.dropdown-menu>.disabled>a:hover{color:#777}.dropdown-menu>.disabled>a:focus,.dropdown-menu>.disabled>a:hover{text-decoration:none;cursor:not-allowed;background-color:transparent;background-image:none;filter:progid:DXImageTransform.Microsoft.gradient(enabled=false)}.open>.dropdown-menu{display:block}.open>a{outline:0}.dropdown-menu-right{right:0;left:auto}.dropdown-menu-left{right:auto;left:0}.dropdown-header{display:block;padding:3px 20px;font-size:12px;line-height:1.42857143;color:#777;white-space:nowrap}.dropdown-backdrop{position:fixed;top:0;right:0;bottom:0;left:0;z-index:990}.pull-right>.dropdown-menu{right:0;left:auto}.dropup .caret,.navbar-fixed-bottom .dropdown .caret{content:"";border-top:0;border-bottom:4px dashed;border-bottom:4px solid\9}.dropup .dropdown-menu,.navbar-fixed-bottom .dropdown .dropdown-menu{top:auto;bottom:100%;margin-bottom:2px}@media (min-width:768px){.navbar-right .dropdown-menu{right:0;left:auto}.navbar-right .dropdown-menu-left{right:auto;left:0}}.btn-group,.btn-group-vertical{position:relative;display:inline-block;vertical-align:middle}.btn-group-vertical>.btn,.btn-group>.btn{position:relative;float:left}.btn-group-vertical>.btn.active,.btn-group-vertical>.btn:active,.btn-group-vertical>.btn:focus,.btn-group-vertical>.btn:hover,.btn-group>.btn.active,.btn-group>.btn:active,.btn-group>.btn:focus,.btn-group>.btn:hover{z-index:2}.btn-group .btn+.btn,.btn-group .btn+.btn-group,.btn-group .btn-group+.btn,.btn-group .btn-group+.btn-group{margin-left:-1px}.btn-toolbar{margin-left:-5px}.btn-toolbar .btn,.btn-toolbar .btn-group,.btn-toolbar .input-group{float:left}.btn-toolbar>.btn,.btn-toolbar>.btn-group,.btn-toolbar>.input-group{margin-left:5px}.btn-group>.btn:not(:first-child):not(:last-child):not(.dropdown-toggle){border-radius:0}.btn-group>.btn:first-child{margin-left:0}.btn-group>.btn:first-child:not(:last-child):not(.dropdown-toggle){border-top-right-radius:0;border-bottom-right-radius:0}.btn-group>.btn:last-child:not(:first-child),.btn-group>.dropdown-toggle:not(:first-child){border-top-left-radius:0;border-bottom-left-radius:0}.btn-group>.btn-group{float:left}.btn-group>.btn-group:not(:first-child):not(:last-child)>.btn{border-radius:0}.btn-group>.btn-group:first-child:not(:last-child)>.btn:last-child,.btn-group>.btn-group:first-child:not(:last-child)>.dropdown-toggle{border-top-right-radius:0;border-bottom-right-radius:0}.btn-group>.btn-group:last-child:not(:first-child)>.btn:first-child{border-top-left-radius:0;border-bottom-left-radius:0}.btn-group .dropdown-toggle:active,.btn-group.open .dropdown-toggle{outline:0}.btn-group>.btn+.dropdown-toggle{padding-right:8px;padding-left:8px}.btn-group>.btn-lg+.dropdown-toggle{padding-right:12px;padding-left:12px}.btn-group.open .dropdown-toggle{-webkit-box-shadow:inset 0 3px 5px rgba(0,0,0,.125);box-shadow:inset 0 3px 5px rgba(0,0,0,.125)}.btn-group.open .dropdown-toggle.btn-link{-webkit-box-shadow:none;box-shadow:none}.btn .caret{margin-left:0}.btn-lg .caret{border-width:5px 5px 0;border-bottom-width:0}.dropup .btn-lg .caret{border-width:0 5px 5px}.btn-group-vertical>.btn,.btn-group-vertical>.btn-group,.btn-group-vertical>.btn-group>.btn{display:block;float:none;width:100%;max-width:100%}.btn-group-vertical>.btn-group>.btn{float:none}.btn-group-vertical>.btn+.btn,.btn-group-vertical>.btn+.btn-group,.btn-group-vertical>.btn-group+.btn,.btn-group-vertical>.btn-group+.btn-group{margin-top:-1px;margin-left:0}.btn-group-vertical>.btn:not(:first-child):not(:last-child){border-radius:0}.btn-group-vertical>.btn:first-child:not(:last-child){border-top-left-radius:4px;border-top-right-radius:4px;border-bottom-right-radius:0;border-bottom-left-radius:0}.btn-group-vertical>.btn:last-child:not(:first-child){border-top-left-radius:0;border-top-right-radius:0;border-bottom-right-radius:4px;border-bottom-left-radius:4px}.btn-group-vertical>.btn-group:not(:first-child):not(:last-child)>.btn{border-radius:0}.btn-group-vertical>.btn-group:first-child:not(:last-child)>.btn:last-child,.btn-group-vertical>.btn-group:first-child:not(:last-child)>.dropdown-toggle{border-bottom-right-radius:0;border-bottom-left-radius:0}.btn-group-vertical>.btn-group:last-child:not(:first-child)>.btn:first-child{border-top-left-radius:0;border-top-right-radius:0}.btn-group-justified{display:table;width:100%;table-layout:fixed;border-collapse:separate}.btn-group-justified>.btn,.btn-group-justified>.btn-group{display:table-cell;float:none;width:1%}.btn-group-justified>.btn-group .btn{width:100%}.btn-group-justified>.btn-group .dropdown-menu{left:auto}[data-toggle=buttons]>.btn input[type=checkbox],[data-toggle=buttons]>.btn input[type=radio],[data-toggle=buttons]>.btn-group>.btn input[type=checkbox],[data-toggle=buttons]>.btn-group>.btn input[type=radio]{position:absolute;clip:rect(0,0,0,0);pointer-events:none}.input-group{position:relative;display:table;border-collapse:separate}.input-group[class*=col-]{float:none;padding-right:0;padding-left:0}.input-group .form-control{position:relative;z-index:2;float:left;width:100%;margin-bottom:0}.input-group .form-control:focus{z-index:3}.input-group-lg>.form-control,.input-group-lg>.input-group-addon,.input-group-lg>.input-group-btn>.btn{height:46px;padding:10px 16px;font-size:18px;line-height:1.3333333;border-radius:6px}select.input-group-lg>.form-control,select.input-group-lg>.input-group-addon,select.input-group-lg>.input-group-btn>.btn{height:46px;line-height:46px}select[multiple].input-group-lg>.form-control,select[multiple].input-group-lg>.input-group-addon,select[multiple].input-group-lg>.input-group-btn>.btn,textarea.input-group-lg>.form-control,textarea.input-group-lg>.input-group-addon,textarea.input-group-lg>.input-group-btn>.btn{height:auto}.input-group-sm>.form-control,.input-group-sm>.input-group-addon,.input-group-sm>.input-group-btn>.btn{height:30px;padding:5px 10px;font-size:12px;line-height:1.5;border-radius:3px}select.input-group-sm>.form-control,select.input-group-sm>.input-group-addon,select.input-group-sm>.input-group-btn>.btn{height:30px;line-height:30px}select[multiple].input-group-sm>.form-control,select[multiple].input-group-sm>.input-group-addon,select[multiple].input-group-sm>.input-group-btn>.btn,textarea.input-group-sm>.form-control,textarea.input-group-sm>.input-group-addon,textarea.input-group-sm>.input-group-btn>.btn{height:auto}.input-group .form-control,.input-group-addon,.input-group-btn{display:table-cell}.input-group .form-control:not(:first-child):not(:last-child),.input-group-addon:not(:first-child):not(:last-child),.input-group-btn:not(:first-child):not(:last-child){border-radius:0}.input-group-addon,.input-group-btn{width:1%;white-space:nowrap;vertical-align:middle}.input-group-addon{padding:6px 12px;font-size:14px;font-weight:400;line-height:1;color:#555;text-align:center;background-color:#eee;border:1px solid #ccc;border-radius:4px}.input-group-addon.input-sm{padding:5px 10px;font-size:12px;border-radius:3px}.input-group-addon.input-lg{padding:10px 16px;font-size:18px;border-radius:6px}.input-group-addon input[type=checkbox],.input-group-addon input[type=radio]{margin-top:0}.input-group .form-control:first-child,.input-group-addon:first-child,.input-group-btn:first-child>.btn,.input-group-btn:first-child>.btn-group>.btn,.input-group-btn:first-child>.dropdown-toggle,.input-group-btn:last-child>.btn-group:not(:last-child)>.btn,.input-group-btn:last-child>.btn:not(:last-child):not(.dropdown-toggle){border-top-right-radius:0;border-bottom-right-radius:0}.input-group-addon:first-child{border-right:0}.input-group .form-control:last-child,.input-group-addon:last-child,.input-group-btn:first-child>.btn-group:not(:first-child)>.btn,.input-group-btn:first-child>.btn:not(:first-child),.input-group-btn:last-child>.btn,.input-group-btn:last-child>.btn-group>.btn,.input-group-btn:last-child>.dropdown-toggle{border-top-left-radius:0;border-bottom-left-radius:0}.input-group-addon:last-child{border-left:0}.input-group-btn{position:relative;font-size:0;white-space:nowrap}.input-group-btn>.btn{position:relative}.input-group-btn>.btn+.btn{margin-left:-1px}.input-group-btn>.btn:active,.input-group-btn>.btn:focus,.input-group-btn>.btn:hover{z-index:2}.input-group-btn:first-child>.btn,.input-group-btn:first-child>.btn-group{margin-right:-1px}.input-group-btn:last-child>.btn,.input-group-btn:last-child>.btn-group{z-index:2;margin-left:-1px}.nav{padding-left:0;margin-bottom:0;list-style:none}.nav>li{position:relative;display:block}.nav>li>a{position:relative;display:block;padding:10px 15px}.nav>li>a:focus,.nav>li>a:hover{text-decoration:none;background-color:#eee}.nav>li.disabled>a{color:#777}.nav>li.disabled>a:focus,.nav>li.disabled>a:hover{color:#777;text-decoration:none;cursor:not-allowed;background-color:transparent}.nav .open>a,.nav .open>a:focus,.nav .open>a:hover{background-color:#eee;border-color:#337ab7}.nav .nav-divider{height:1px;margin:9px 0;overflow:hidden;background-color:#e5e5e5}.nav>li>a>img{max-width:none}.nav-tabs{border-bottom:1px solid #ddd}.nav-tabs>li{float:left;margin-bottom:-1px}.nav-tabs>li>a{margin-right:2px;line-height:1.42857143;border:1px solid transparent;border-radius:4px 4px 0 0}.nav-tabs>li>a:hover{border-color:#eee #eee #ddd}.nav-tabs>li.active>a,.nav-tabs>li.active>a:focus,.nav-tabs>li.active>a:hover{color:#555;cursor:default;background-color:#fff;border:1px solid #ddd;border-bottom-color:transparent}.nav-tabs.nav-justified{width:100%;border-bottom:0}.nav-tabs.nav-justified>li{float:none}.nav-tabs.nav-justified>li>a{margin-bottom:5px;text-align:center}.nav-tabs.nav-justified>.dropdown .dropdown-menu{top:auto;left:auto}@media (min-width:768px){.nav-tabs.nav-justified>li{display:table-cell;width:1%}.nav-tabs.nav-justified>li>a{margin-bottom:0}}.nav-tabs.nav-justified>li>a{margin-right:0;border-radius:4px}.nav-tabs.nav-justified>.active>a,.nav-tabs.nav-justified>.active>a:focus,.nav-tabs.nav-justified>.active>a:hover{border:1px solid #ddd}@media (min-width:768px){.nav-tabs.nav-justified>li>a{border-bottom:1px solid #ddd;border-radius:4px 4px 0 0}.nav-tabs.nav-justified>.active>a,.nav-tabs.nav-justified>.active>a:focus,.nav-tabs.nav-justified>.active>a:hover{border-bottom-color:#fff}}.nav-pills>li{float:left}.nav-pills>li>a{border-radius:4px}.nav-pills>li+li{margin-left:2px}.nav-pills>li.active>a,.nav-pills>li.active>a:focus,.nav-pills>li.active>a:hover{color:#fff;background-color:#337ab7}.nav-stacked>li{float:none}.nav-stacked>li+li{margin-top:2px;margin-left:0}.nav-justified{width:100%}.nav-justified>li{float:none}.nav-justified>li>a{margin-bottom:5px;text-align:center}.nav-justified>.dropdown .dropdown-menu{top:auto;left:auto}@media (min-width:768px){.nav-justified>li{display:table-cell;width:1%}.nav-justified>li>a{margin-bottom:0}}.nav-tabs-justified{border-bottom:0}.nav-tabs-justified>li>a{margin-right:0;border-radius:4px}.nav-tabs-justified>.active>a,.nav-tabs-justified>.active>a:focus,.nav-tabs-justified>.active>a:hover{border:1px solid #ddd}@media (min-width:768px){.nav-tabs-justified>li>a{border-bottom:1px solid #ddd;border-radius:4px 4px 0 0}.nav-tabs-justified>.active>a,.nav-tabs-justified>.active>a:focus,.nav-tabs-justified>.active>a:hover{border-bottom-color:#fff}}.tab-content>.tab-pane{display:none}.tab-content>.active{display:block}.nav-tabs .dropdown-menu{margin-top:-1px;border-top-left-radius:0;border-top-right-radius:0}.navbar{position:relative;min-height:50px;margin-bottom:20px;border:1px solid transparent}@media (min-width:768px){.navbar{border-radius:4px}}@media (min-width:768px){.navbar-header{float:left}}.navbar-collapse{padding-right:15px;padding-left:15px;overflow-x:visible;-webkit-overflow-scrolling:touch;border-top:1px solid transparent;-webkit-box-shadow:inset 0 1px 0 rgba(255,255,255,.1);box-shadow:inset 0 1px 0 rgba(255,255,255,.1)}.navbar-collapse.in{overflow-y:auto}@media (min-width:768px){.navbar-collapse{width:auto;border-top:0;-webkit-box-shadow:none;box-shadow:none}.navbar-collapse.collapse{display:block!important;height:auto!important;padding-bottom:0;overflow:visible!important}.navbar-collapse.in{overflow-y:visible}.navbar-fixed-bottom .navbar-collapse,.navbar-fixed-top .navbar-collapse,.navbar-static-top .navbar-collapse{padding-right:0;padding-left:0}}.navbar-fixed-bottom .navbar-collapse,.navbar-fixed-top .navbar-collapse{max-height:340px}@media (max-device-width:480px) and (orientation:landscape){.navbar-fixed-bottom .navbar-collapse,.navbar-fixed-top .navbar-collapse{max-height:200px}}.container-fluid>.navbar-collapse,.container-fluid>.navbar-header,.container>.navbar-collapse,.container>.navbar-header{margin-right:-15px;margin-left:-15px}@media (min-width:768px){.container-fluid>.navbar-collapse,.container-fluid>.navbar-header,.container>.navbar-collapse,.container>.navbar-header{margin-right:0;margin-left:0}}.navbar-static-top{z-index:1000;border-width:0 0 1px}@media (min-width:768px){.navbar-static-top{border-radius:0}}.navbar-fixed-bottom,.navbar-fixed-top{position:fixed;right:0;left:0;z-index:1030}@media (min-width:768px){.navbar-fixed-bottom,.navbar-fixed-top{border-radius:0}}.navbar-fixed-top{top:0;border-width:0 0 1px}.navbar-fixed-bottom{bottom:0;margin-bottom:0;border-width:1px 0 0}.navbar-brand{float:left;height:50px;padding:15px 15px;font-size:18px;line-height:20px}.navbar-brand:focus,.navbar-brand:hover{text-decoration:none}.navbar-brand>img{display:block}@media (min-width:768px){.navbar>.container .navbar-brand,.navbar>.container-fluid .navbar-brand{margin-left:-15px}}.navbar-toggle{position:relative;float:right;padding:9px 10px;margin-top:8px;margin-right:15px;margin-bottom:8px;background-color:transparent;background-image:none;border:1px solid transparent;border-radius:4px}.navbar-toggle:focus{outline:0}.navbar-toggle .icon-bar{display:block;width:22px;height:2px;border-radius:1px}.navbar-toggle .icon-bar+.icon-bar{margin-top:4px}@media (min-width:768px){.navbar-toggle{display:none}}.navbar-nav{margin:7.5px -15px}.navbar-nav>li>a{padding-top:10px;padding-bottom:10px;line-height:20px}@media (max-width:767px){.navbar-nav .open .dropdown-menu{position:static;float:none;width:auto;margin-top:0;background-color:transparent;border:0;-webkit-box-shadow:none;box-shadow:none}.navbar-nav .open .dropdown-menu .dropdown-header,.navbar-nav .open .dropdown-menu>li>a{padding:5px 15px 5px 25px}.navbar-nav .open .dropdown-menu>li>a{line-height:20px}.navbar-nav .open .dropdown-menu>li>a:focus,.navbar-nav .open .dropdown-menu>li>a:hover{background-image:none}}@media (min-width:768px){.navbar-nav{float:left;margin:0}.navbar-nav>li{float:left}.navbar-nav>li>a{padding-top:15px;padding-bottom:15px}}.navbar-form{padding:10px 15px;margin-top:8px;margin-right:-15px;margin-bottom:8px;margin-left:-15px;border-top:1px solid transparent;border-bottom:1px solid transparent;-webkit-box-shadow:inset 0 1px 0 rgba(255,255,255,.1),0 1px 0 rgba(255,255,255,.1);box-shadow:inset 0 1px 0 rgba(255,255,255,.1),0 1px 0 rgba(255,255,255,.1)}@media (min-width:768px){.navbar-form .form-group{display:inline-block;margin-bottom:0;vertical-align:middle}.navbar-form .form-control{display:inline-block;width:auto;vertical-align:middle}.navbar-form .form-control-static{display:inline-block}.navbar-form .input-group{display:inline-table;vertical-align:middle}.navbar-form .input-group .form-control,.navbar-form .input-group .input-group-addon,.navbar-form .input-group .input-group-btn{width:auto}.navbar-form .input-group>.form-control{width:100%}.navbar-form .control-label{margin-bottom:0;vertical-align:middle}.navbar-form .checkbox,.navbar-form .radio{display:inline-block;margin-top:0;margin-bottom:0;vertical-align:middle}.navbar-form .checkbox label,.navbar-form .radio label{padding-left:0}.navbar-form .checkbox input[type=checkbox],.navbar-form .radio input[type=radio]{position:relative;margin-left:0}.navbar-form .has-feedback .form-control-feedback{top:0}}@media (max-width:767px){.navbar-form .form-group{margin-bottom:5px}.navbar-form .form-group:last-child{margin-bottom:0}}@media (min-width:768px){.navbar-form{width:auto;padding-top:0;padding-bottom:0;margin-right:0;margin-left:0;border:0;-webkit-box-shadow:none;box-shadow:none}}.navbar-nav>li>.dropdown-menu{margin-top:0;border-top-left-radius:0;border-top-right-radius:0}.navbar-fixed-bottom .navbar-nav>li>.dropdown-menu{margin-bottom:0;border-top-left-radius:4px;border-top-right-radius:4px;border-bottom-right-radius:0;border-bottom-left-radius:0}.navbar-btn{margin-top:8px;margin-bottom:8px}.navbar-btn.btn-sm{margin-top:10px;margin-bottom:10px}.navbar-btn.btn-xs{margin-top:14px;margin-bottom:14px}.navbar-text{margin-top:15px;margin-bottom:15px}@media (min-width:768px){.navbar-text{float:left;margin-right:15px;margin-left:15px}}@media (min-width:768px){.navbar-left{float:left!important}.navbar-right{float:right!important;margin-right:-15px}.navbar-right~.navbar-right{margin-right:0}}.navbar-default{background-color:#f8f8f8;border-color:#e7e7e7}.navbar-default .navbar-brand{color:#777}.navbar-default .navbar-brand:focus,.navbar-default .navbar-brand:hover{color:#5e5e5e;background-color:transparent}.navbar-default .navbar-text{color:#777}.navbar-default .navbar-nav>li>a{color:#777}.navbar-default .navbar-nav>li>a:focus,.navbar-default .navbar-nav>li>a:hover{color:#333;background-color:transparent}.navbar-default .navbar-nav>.active>a,.navbar-default .navbar-nav>.active>a:focus,.navbar-default .navbar-nav>.active>a:hover{color:#555;background-color:#e7e7e7}.navbar-default .navbar-nav>.disabled>a,.navbar-default .navbar-nav>.disabled>a:focus,.navbar-default .navbar-nav>.disabled>a:hover{color:#ccc;background-color:transparent}.navbar-default .navbar-toggle{border-color:#ddd}.navbar-default .navbar-toggle:focus,.navbar-default .navbar-toggle:hover{background-color:#ddd}.navbar-default .navbar-toggle .icon-bar{background-color:#888}.navbar-default .navbar-collapse,.navbar-default .navbar-form{border-color:#e7e7e7}.navbar-default .navbar-nav>.open>a,.navbar-default .navbar-nav>.open>a:focus,.navbar-default .navbar-nav>.open>a:hover{color:#555;background-color:#e7e7e7}@media (max-width:767px){.navbar-default .navbar-nav .open .dropdown-menu>li>a{color:#777}.navbar-default .navbar-nav .open .dropdown-menu>li>a:focus,.navbar-default .navbar-nav .open .dropdown-menu>li>a:hover{color:#333;background-color:transparent}.navbar-default .navbar-nav .open .dropdown-menu>.active>a,.navbar-default .navbar-nav .open .dropdown-menu>.active>a:focus,.navbar-default .navbar-nav .open .dropdown-menu>.active>a:hover{color:#555;background-color:#e7e7e7}.navbar-default .navbar-nav .open .dropdown-menu>.disabled>a,.navbar-default .navbar-nav .open .dropdown-menu>.disabled>a:focus,.navbar-default .navbar-nav .open .dropdown-menu>.disabled>a:hover{color:#ccc;background-color:transparent}}.navbar-default .navbar-link{color:#777}.navbar-default .navbar-link:hover{color:#333}.navbar-default .btn-link{color:#777}.navbar-default .btn-link:focus,.navbar-default .btn-link:hover{color:#333}.navbar-default .btn-link[disabled]:focus,.navbar-default .btn-link[disabled]:hover,fieldset[disabled] .navbar-default .btn-link:focus,fieldset[disabled] .navbar-default .btn-link:hover{color:#ccc}.navbar-inverse{background-color:#222;border-color:#080808}.navbar-inverse .navbar-brand{color:#9d9d9d}.navbar-inverse .navbar-brand:focus,.navbar-inverse .navbar-brand:hover{color:#fff;background-color:transparent}.navbar-inverse .navbar-text{color:#9d9d9d}.navbar-inverse .navbar-nav>li>a{color:#9d9d9d}.navbar-inverse .navbar-nav>li>a:focus,.navbar-inverse .navbar-nav>li>a:hover{color:#fff;background-color:transparent}.navbar-inverse .navbar-nav>.active>a,.navbar-inverse .navbar-nav>.active>a:focus,.navbar-inverse .navbar-nav>.active>a:hover{color:#fff;background-color:#080808}.navbar-inverse .navbar-nav>.disabled>a,.navbar-inverse .navbar-nav>.disabled>a:focus,.navbar-inverse .navbar-nav>.disabled>a:hover{color:#444;background-color:transparent}.navbar-inverse .navbar-toggle{border-color:#333}.navbar-inverse .navbar-toggle:focus,.navbar-inverse .navbar-toggle:hover{background-color:#333}.navbar-inverse .navbar-toggle .icon-bar{background-color:#fff}.navbar-inverse .navbar-collapse,.navbar-inverse .navbar-form{border-color:#101010}.navbar-inverse .navbar-nav>.open>a,.navbar-inverse .navbar-nav>.open>a:focus,.navbar-inverse .navbar-nav>.open>a:hover{color:#fff;background-color:#080808}@media (max-width:767px){.navbar-inverse .navbar-nav .open .dropdown-menu>.dropdown-header{border-color:#080808}.navbar-inverse .navbar-nav .open .dropdown-menu .divider{background-color:#080808}.navbar-inverse .navbar-nav .open .dropdown-menu>li>a{color:#9d9d9d}.navbar-inverse .navbar-nav .open .dropdown-menu>li>a:focus,.navbar-inverse .navbar-nav .open .dropdown-menu>li>a:hover{color:#fff;background-color:transparent}.navbar-inverse .navbar-nav .open .dropdown-menu>.active>a,.navbar-inverse .navbar-nav .open .dropdown-menu>.active>a:focus,.navbar-inverse .navbar-nav .open .dropdown-menu>.active>a:hover{color:#fff;background-color:#080808}.navbar-inverse .navbar-nav .open .dropdown-menu>.disabled>a,.navbar-inverse .navbar-nav .open .dropdown-menu>.disabled>a:focus,.navbar-inverse .navbar-nav .open .dropdown-menu>.disabled>a:hover{color:#444;background-color:transparent}}.navbar-inverse .navbar-link{color:#9d9d9d}.navbar-inverse .navbar-link:hover{color:#fff}.navbar-inverse .btn-link{color:#9d9d9d}.navbar-inverse .btn-link:focus,.navbar-inverse .btn-link:hover{color:#fff}.navbar-inverse .btn-link[disabled]:focus,.navbar-inverse .btn-link[disabled]:hover,fieldset[disabled] .navbar-inverse .btn-link:focus,fieldset[disabled] .navbar-inverse .btn-link:hover{color:#444}.breadcrumb{padding:8px 15px;margin-bottom:20px;list-style:none;background-color:#f5f5f5;border-radius:4px}.breadcrumb>li{display:inline-block}.breadcrumb>li+li:before{padding:0 5px;color:#ccc;content:"/\00a0"}.breadcrumb>.active{color:#777}.pagination{display:inline-block;padding-left:0;margin:20px 0;border-radius:4px}.pagination>li{display:inline}.pagination>li>a,.pagination>li>span{position:relative;float:left;padding:6px 12px;margin-left:-1px;line-height:1.42857143;color:#337ab7;text-decoration:none;background-color:#fff;border:1px solid #ddd}.pagination>li:first-child>a,.pagination>li:first-child>span{margin-left:0;border-top-left-radius:4px;border-bottom-left-radius:4px}.pagination>li:last-child>a,.pagination>li:last-child>span{border-top-right-radius:4px;border-bottom-right-radius:4px}.pagination>li>a:focus,.pagination>li>a:hover,.pagination>li>span:focus,.pagination>li>span:hover{z-index:2;color:#23527c;background-color:#eee;border-color:#ddd}.pagination>.active>a,.pagination>.active>a:focus,.pagination>.active>a:hover,.pagination>.active>span,.pagination>.active>span:focus,.pagination>.active>span:hover{z-index:3;color:#fff;cursor:default;background-color:#337ab7;border-color:#337ab7}.pagination>.disabled>a,.pagination>.disabled>a:focus,.pagination>.disabled>a:hover,.pagination>.disabled>span,.pagination>.disabled>span:focus,.pagination>.disabled>span:hover{color:#777;cursor:not-allowed;background-color:#fff;border-color:#ddd}.pagination-lg>li>a,.pagination-lg>li>span{padding:10px 16px;font-size:18px;line-height:1.3333333}.pagination-lg>li:first-child>a,.pagination-lg>li:first-child>span{border-top-left-radius:6px;border-bottom-left-radius:6px}.pagination-lg>li:last-child>a,.pagination-lg>li:last-child>span{border-top-right-radius:6px;border-bottom-right-radius:6px}.pagination-sm>li>a,.pagination-sm>li>span{padding:5px 10px;font-size:12px;line-height:1.5}.pagination-sm>li:first-child>a,.pagination-sm>li:first-child>span{border-top-left-radius:3px;border-bottom-left-radius:3px}.pagination-sm>li:last-child>a,.pagination-sm>li:last-child>span{border-top-right-radius:3px;border-bottom-right-radius:3px}.pager{padding-left:0;margin:20px 0;text-align:center;list-style:none}.pager li{display:inline}.pager li>a,.pager li>span{display:inline-block;padding:5px 14px;background-color:#fff;border:1px solid #ddd;border-radius:15px}.pager li>a:focus,.pager li>a:hover{text-decoration:none;background-color:#eee}.pager .next>a,.pager .next>span{float:right}.pager .previous>a,.pager .previous>span{float:left}.pager .disabled>a,.pager .disabled>a:focus,.pager .disabled>a:hover,.pager .disabled>span{color:#777;cursor:not-allowed;background-color:#fff}.label{display:inline;padding:.2em .6em .3em;font-size:75%;font-weight:700;line-height:1;color:#fff;text-align:center;white-space:nowrap;vertical-align:baseline;border-radius:.25em}a.label:focus,a.label:hover{color:#fff;text-decoration:none;cursor:pointer}.label:empty{display:none}.btn .label{position:relative;top:-1px}.label-default{background-color:#777}.label-default[href]:focus,.label-default[href]:hover{background-color:#5e5e5e}.label-primary{background-color:#337ab7}.label-primary[href]:focus,.label-primary[href]:hover{background-color:#286090}.label-success{background-color:#5cb85c}.label-success[href]:focus,.label-success[href]:hover{background-color:#449d44}.label-info{background-color:#5bc0de}.label-info[href]:focus,.label-info[href]:hover{background-color:#31b0d5}.label-warning{background-color:#f0ad4e}.label-warning[href]:focus,.label-warning[href]:hover{background-color:#ec971f}.label-danger{background-color:#d9534f}.label-danger[href]:focus,.label-danger[href]:hover{background-color:#c9302c}.badge{display:inline-block;min-width:10px;padding:3px 7px;font-size:12px;font-weight:700;line-height:1;color:#fff;text-align:center;white-space:nowrap;vertical-align:middle;background-color:#777;border-radius:10px}.badge:empty{display:none}.btn .badge{position:relative;top:-1px}.btn-group-xs>.btn .badge,.btn-xs .badge{top:0;padding:1px 5px}a.badge:focus,a.badge:hover{color:#fff;text-decoration:none;cursor:pointer}.list-group-item.active>.badge,.nav-pills>.active>a>.badge{color:#337ab7;background-color:#fff}.list-group-item>.badge{float:right}.list-group-item>.badge+.badge{margin-right:5px}.nav-pills>li>a>.badge{margin-left:3px}.jumbotron{padding-top:30px;padding-bottom:30px;margin-bottom:30px;color:inherit;background-color:#eee}.jumbotron .h1,.jumbotron h1{color:inherit}.jumbotron p{margin-bottom:15px;font-size:21px;font-weight:200}.jumbotron>hr{border-top-color:#d5d5d5}.container .jumbotron,.container-fluid .jumbotron{padding-right:15px;padding-left:15px;border-radius:6px}.jumbotron .container{max-width:100%}@media screen and (min-width:768px){.jumbotron{padding-top:48px;padding-bottom:48px}.container .jumbotron,.container-fluid .jumbotron{padding-right:60px;padding-left:60px}.jumbotron .h1,.jumbotron h1{font-size:63px}}.thumbnail{display:block;padding:4px;margin-bottom:20px;line-height:1.42857143;background-color:#fff;border:1px solid #ddd;border-radius:4px;-webkit-transition:border .2s ease-in-out;-o-transition:border .2s ease-in-out;transition:border .2s ease-in-out}.thumbnail a>img,.thumbnail>img{margin-right:auto;margin-left:auto}a.thumbnail.active,a.thumbnail:focus,a.thumbnail:hover{border-color:#337ab7}.thumbnail .caption{padding:9px;color:#333}.alert{padding:15px;margin-bottom:20px;border:1px solid transparent;border-radius:4px}.alert h4{margin-top:0;color:inherit}.alert .alert-link{font-weight:700}.alert>p,.alert>ul{margin-bottom:0}.alert>p+p{margin-top:5px}.alert-dismissable,.alert-dismissible{padding-right:35px}.alert-dismissable .close,.alert-dismissible .close{position:relative;top:-2px;right:-21px;color:inherit}.alert-success{color:#3c763d;background-color:#dff0d8;border-color:#d6e9c6}.alert-success hr{border-top-color:#c9e2b3}.alert-success .alert-link{color:#2b542c}.alert-info{color:#31708f;background-color:#d9edf7;border-color:#bce8f1}.alert-info hr{border-top-color:#a6e1ec}.alert-info .alert-link{color:#245269}.alert-warning{color:#8a6d3b;background-color:#fcf8e3;border-color:#faebcc}.alert-warning hr{border-top-color:#f7e1b5}.alert-warning .alert-link{color:#66512c}.alert-danger{color:#a94442;background-color:#f2dede;border-color:#ebccd1}.alert-danger hr{border-top-color:#e4b9c0}.alert-danger .alert-link{color:#843534}@-webkit-keyframes progress-bar-stripes{from{background-position:40px 0}to{background-position:0 0}}@-o-keyframes progress-bar-stripes{from{background-position:40px 0}to{background-position:0 0}}@keyframes progress-bar-stripes{from{background-position:40px 0}to{background-position:0 0}}.progress{height:20px;margin-bottom:20px;overflow:hidden;background-color:#f5f5f5;border-radius:4px;-webkit-box-shadow:inset 0 1px 2px rgba(0,0,0,.1);box-shadow:inset 0 1px 2px rgba(0,0,0,.1)}.progress-bar{float:left;width:0;height:100%;font-size:12px;line-height:20px;color:#fff;text-align:center;background-color:#337ab7;-webkit-box-shadow:inset 0 -1px 0 rgba(0,0,0,.15);box-shadow:inset 0 -1px 0 rgba(0,0,0,.15);-webkit-transition:width .6s ease;-o-transition:width .6s ease;transition:width .6s ease}.progress-bar-striped,.progress-striped .progress-bar{background-image:-webkit-linear-gradient(45deg,rgba(255,255,255,.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,.15) 50%,rgba(255,255,255,.15) 75%,transparent 75%,transparent);background-image:-o-linear-gradient(45deg,rgba(255,255,255,.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,.15) 50%,rgba(255,255,255,.15) 75%,transparent 75%,transparent);background-image:linear-gradient(45deg,rgba(255,255,255,.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,.15) 50%,rgba(255,255,255,.15) 75%,transparent 75%,transparent);-webkit-background-size:40px 40px;background-size:40px 40px}.progress-bar.active,.progress.active .progress-bar{-webkit-animation:progress-bar-stripes 2s linear infinite;-o-animation:progress-bar-stripes 2s linear infinite;animation:progress-bar-stripes 2s linear infinite}.progress-bar-success{background-color:#5cb85c}.progress-striped .progress-bar-success{background-image:-webkit-linear-gradient(45deg,rgba(255,255,255,.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,.15) 50%,rgba(255,255,255,.15) 75%,transparent 75%,transparent);background-image:-o-linear-gradient(45deg,rgba(255,255,255,.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,.15) 50%,rgba(255,255,255,.15) 75%,transparent 75%,transparent);background-image:linear-gradient(45deg,rgba(255,255,255,.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,.15) 50%,rgba(255,255,255,.15) 75%,transparent 75%,transparent)}.progress-bar-info{background-color:#5bc0de}.progress-striped .progress-bar-info{background-image:-webkit-linear-gradient(45deg,rgba(255,255,255,.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,.15) 50%,rgba(255,255,255,.15) 75%,transparent 75%,transparent);background-image:-o-linear-gradient(45deg,rgba(255,255,255,.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,.15) 50%,rgba(255,255,255,.15) 75%,transparent 75%,transparent);background-image:linear-gradient(45deg,rgba(255,255,255,.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,.15) 50%,rgba(255,255,255,.15) 75%,transparent 75%,transparent)}.progress-bar-warning{background-color:#f0ad4e}.progress-striped .progress-bar-warning{background-image:-webkit-linear-gradient(45deg,rgba(255,255,255,.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,.15) 50%,rgba(255,255,255,.15) 75%,transparent 75%,transparent);background-image:-o-linear-gradient(45deg,rgba(255,255,255,.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,.15) 50%,rgba(255,255,255,.15) 75%,transparent 75%,transparent);background-image:linear-gradient(45deg,rgba(255,255,255,.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,.15) 50%,rgba(255,255,255,.15) 75%,transparent 75%,transparent)}.progress-bar-danger{background-color:#d9534f}.progress-striped .progress-bar-danger{background-image:-webkit-linear-gradient(45deg,rgba(255,255,255,.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,.15) 50%,rgba(255,255,255,.15) 75%,transparent 75%,transparent);background-image:-o-linear-gradient(45deg,rgba(255,255,255,.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,.15) 50%,rgba(255,255,255,.15) 75%,transparent 75%,transparent);background-image:linear-gradient(45deg,rgba(255,255,255,.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,.15) 50%,rgba(255,255,255,.15) 75%,transparent 75%,transparent)}.media{margin-top:15px}.media:first-child{margin-top:0}.media,.media-body{overflow:hidden;zoom:1}.media-body{width:10000px}.media-object{display:block}.media-object.img-thumbnail{max-width:none}.media-right,.media>.pull-right{padding-left:10px}.media-left,.media>.pull-left{padding-right:10px}.media-body,.media-left,.media-right{display:table-cell;vertical-align:top}.media-middle{vertical-align:middle}.media-bottom{vertical-align:bottom}.media-heading{margin-top:0;margin-bottom:5px}.media-list{padding-left:0;list-style:none}.list-group{padding-left:0;margin-bottom:20px}.list-group-item{position:relative;display:block;padding:10px 15px;margin-bottom:-1px;background-color:#fff;border:1px solid #ddd}.list-group-item:first-child{border-top-left-radius:4px;border-top-right-radius:4px}.list-group-item:last-child{margin-bottom:0;border-bottom-right-radius:4px;border-bottom-left-radius:4px}a.list-group-item,button.list-group-item{color:#555}a.list-group-item .list-group-item-heading,button.list-group-item .list-group-item-heading{color:#333}a.list-group-item:focus,a.list-group-item:hover,button.list-group-item:focus,button.list-group-item:hover{color:#555;text-decoration:none;background-color:#f5f5f5}button.list-group-item{width:100%;text-align:left}.list-group-item.disabled,.list-group-item.disabled:focus,.list-group-item.disabled:hover{color:#777;cursor:not-allowed;background-color:#eee}.list-group-item.disabled .list-group-item-heading,.list-group-item.disabled:focus .list-group-item-heading,.list-group-item.disabled:hover .list-group-item-heading{color:inherit}.list-group-item.disabled .list-group-item-text,.list-group-item.disabled:focus .list-group-item-text,.list-group-item.disabled:hover .list-group-item-text{color:#777}.list-group-item.active,.list-group-item.active:focus,.list-group-item.active:hover{z-index:2;color:#fff;background-color:#337ab7;border-color:#337ab7}.list-group-item.active .list-group-item-heading,.list-group-item.active .list-group-item-heading>.small,.list-group-item.active .list-group-item-heading>small,.list-group-item.active:focus .list-group-item-heading,.list-group-item.active:focus .list-group-item-heading>.small,.list-group-item.active:focus .list-group-item-heading>small,.list-group-item.active:hover .list-group-item-heading,.list-group-item.active:hover .list-group-item-heading>.small,.list-group-item.active:hover .list-group-item-heading>small{color:inherit}.list-group-item.active .list-group-item-text,.list-group-item.active:focus .list-group-item-text,.list-group-item.active:hover .list-group-item-text{color:#c7ddef}.list-group-item-success{color:#3c763d;background-color:#dff0d8}a.list-group-item-success,button.list-group-item-success{color:#3c763d}a.list-group-item-success .list-group-item-heading,button.list-group-item-success .list-group-item-heading{color:inherit}a.list-group-item-success:focus,a.list-group-item-success:hover,button.list-group-item-success:focus,button.list-group-item-success:hover{color:#3c763d;background-color:#d0e9c6}a.list-group-item-success.active,a.list-group-item-success.active:focus,a.list-group-item-success.active:hover,button.list-group-item-success.active,button.list-group-item-success.active:focus,button.list-group-item-success.active:hover{color:#fff;background-color:#3c763d;border-color:#3c763d}.list-group-item-info{color:#31708f;background-color:#d9edf7}a.list-group-item-info,button.list-group-item-info{color:#31708f}a.list-group-item-info .list-group-item-heading,button.list-group-item-info .list-group-item-heading{color:inherit}a.list-group-item-info:focus,a.list-group-item-info:hover,button.list-group-item-info:focus,button.list-group-item-info:hover{color:#31708f;background-color:#c4e3f3}a.list-group-item-info.active,a.list-group-item-info.active:focus,a.list-group-item-info.active:hover,button.list-group-item-info.active,button.list-group-item-info.active:focus,button.list-group-item-info.active:hover{color:#fff;background-color:#31708f;border-color:#31708f}.list-group-item-warning{color:#8a6d3b;background-color:#fcf8e3}a.list-group-item-warning,button.list-group-item-warning{color:#8a6d3b}a.list-group-item-warning .list-group-item-heading,button.list-group-item-warning .list-group-item-heading{color:inherit}a.list-group-item-warning:focus,a.list-group-item-warning:hover,button.list-group-item-warning:focus,button.list-group-item-warning:hover{color:#8a6d3b;background-color:#faf2cc}a.list-group-item-warning.active,a.list-group-item-warning.active:focus,a.list-group-item-warning.active:hover,button.list-group-item-warning.active,button.list-group-item-warning.active:focus,button.list-group-item-warning.active:hover{color:#fff;background-color:#8a6d3b;border-color:#8a6d3b}.list-group-item-danger{color:#a94442;background-color:#f2dede}a.list-group-item-danger,button.list-group-item-danger{color:#a94442}a.list-group-item-danger .list-group-item-heading,button.list-group-item-danger .list-group-item-heading{color:inherit}a.list-group-item-danger:focus,a.list-group-item-danger:hover,button.list-group-item-danger:focus,button.list-group-item-danger:hover{color:#a94442;background-color:#ebcccc}a.list-group-item-danger.active,a.list-group-item-danger.active:focus,a.list-group-item-danger.active:hover,button.list-group-item-danger.active,button.list-group-item-danger.active:focus,button.list-group-item-danger.active:hover{color:#fff;background-color:#a94442;border-color:#a94442}.list-group-item-heading{margin-top:0;margin-bottom:5px}.list-group-item-text{margin-bottom:0;line-height:1.3}.panel{margin-bottom:20px;background-color:#fff;border:1px solid transparent;border-radius:4px;-webkit-box-shadow:0 1px 1px rgba(0,0,0,.05);box-shadow:0 1px 1px rgba(0,0,0,.05)}.panel-body{padding:15px}.panel-heading{padding:10px 15px;border-bottom:1px solid transparent;border-top-left-radius:3px;border-top-right-radius:3px}.panel-heading>.dropdown .dropdown-toggle{color:inherit}.panel-title{margin-top:0;margin-bottom:0;font-size:16px;color:inherit}.panel-title>.small,.panel-title>.small>a,.panel-title>a,.panel-title>small,.panel-title>small>a{color:inherit}.panel-footer{padding:10px 15px;background-color:#f5f5f5;border-top:1px solid #ddd;border-bottom-right-radius:3px;border-bottom-left-radius:3px}.panel>.list-group,.panel>.panel-collapse>.list-group{margin-bottom:0}.panel>.list-group .list-group-item,.panel>.panel-collapse>.list-group .list-group-item{border-width:1px 0;border-radius:0}.panel>.list-group:first-child .list-group-item:first-child,.panel>.panel-collapse>.list-group:first-child .list-group-item:first-child{border-top:0;border-top-left-radius:3px;border-top-right-radius:3px}.panel>.list-group:last-child .list-group-item:last-child,.panel>.panel-collapse>.list-group:last-child .list-group-item:last-child{border-bottom:0;border-bottom-right-radius:3px;border-bottom-left-radius:3px}.panel>.panel-heading+.panel-collapse>.list-group .list-group-item:first-child{border-top-left-radius:0;border-top-right-radius:0}.panel-heading+.list-group .list-group-item:first-child{border-top-width:0}.list-group+.panel-footer{border-top-width:0}.panel>.panel-collapse>.table,.panel>.table,.panel>.table-responsive>.table{margin-bottom:0}.panel>.panel-collapse>.table caption,.panel>.table caption,.panel>.table-responsive>.table caption{padding-right:15px;padding-left:15px}.panel>.table-responsive:first-child>.table:first-child,.panel>.table:first-child{border-top-left-radius:3px;border-top-right-radius:3px}.panel>.table-responsive:first-child>.table:first-child>tbody:first-child>tr:first-child,.panel>.table-responsive:first-child>.table:first-child>thead:first-child>tr:first-child,.panel>.table:first-child>tbody:first-child>tr:first-child,.panel>.table:first-child>thead:first-child>tr:first-child{border-top-left-radius:3px;border-top-right-radius:3px}.panel>.table-responsive:first-child>.table:first-child>tbody:first-child>tr:first-child td:first-child,.panel>.table-responsive:first-child>.table:first-child>tbody:first-child>tr:first-child th:first-child,.panel>.table-responsive:first-child>.table:first-child>thead:first-child>tr:first-child td:first-child,.panel>.table-responsive:first-child>.table:first-child>thead:first-child>tr:first-child th:first-child,.panel>.table:first-child>tbody:first-child>tr:first-child td:first-child,.panel>.table:first-child>tbody:first-child>tr:first-child th:first-child,.panel>.table:first-child>thead:first-child>tr:first-child td:first-child,.panel>.table:first-child>thead:first-child>tr:first-child th:first-child{border-top-left-radius:3px}.panel>.table-responsive:first-child>.table:first-child>tbody:first-child>tr:first-child td:last-child,.panel>.table-responsive:first-child>.table:first-child>tbody:first-child>tr:first-child th:last-child,.panel>.table-responsive:first-child>.table:first-child>thead:first-child>tr:first-child td:last-child,.panel>.table-responsive:first-child>.table:first-child>thead:first-child>tr:first-child th:last-child,.panel>.table:first-child>tbody:first-child>tr:first-child td:last-child,.panel>.table:first-child>tbody:first-child>tr:first-child th:last-child,.panel>.table:first-child>thead:first-child>tr:first-child td:last-child,.panel>.table:first-child>thead:first-child>tr:first-child th:last-child{border-top-right-radius:3px}.panel>.table-responsive:last-child>.table:last-child,.panel>.table:last-child{border-bottom-right-radius:3px;border-bottom-left-radius:3px}.panel>.table-responsive:last-child>.table:last-child>tbody:last-child>tr:last-child,.panel>.table-responsive:last-child>.table:last-child>tfoot:last-child>tr:last-child,.panel>.table:last-child>tbody:last-child>tr:last-child,.panel>.table:last-child>tfoot:last-child>tr:last-child{border-bottom-right-radius:3px;border-bottom-left-radius:3px}.panel>.table-responsive:last-child>.table:last-child>tbody:last-child>tr:last-child td:first-child,.panel>.table-responsive:last-child>.table:last-child>tbody:last-child>tr:last-child th:first-child,.panel>.table-responsive:last-child>.table:last-child>tfoot:last-child>tr:last-child td:first-child,.panel>.table-responsive:last-child>.table:last-child>tfoot:last-child>tr:last-child th:first-child,.panel>.table:last-child>tbody:last-child>tr:last-child td:first-child,.panel>.table:last-child>tbody:last-child>tr:last-child th:first-child,.panel>.table:last-child>tfoot:last-child>tr:last-child td:first-child,.panel>.table:last-child>tfoot:last-child>tr:last-child th:first-child{border-bottom-left-radius:3px}.panel>.table-responsive:last-child>.table:last-child>tbody:last-child>tr:last-child td:last-child,.panel>.table-responsive:last-child>.table:last-child>tbody:last-child>tr:last-child th:last-child,.panel>.table-responsive:last-child>.table:last-child>tfoot:last-child>tr:last-child td:last-child,.panel>.table-responsive:last-child>.table:last-child>tfoot:last-child>tr:last-child th:last-child,.panel>.table:last-child>tbody:last-child>tr:last-child td:last-child,.panel>.table:last-child>tbody:last-child>tr:last-child th:last-child,.panel>.table:last-child>tfoot:last-child>tr:last-child td:last-child,.panel>.table:last-child>tfoot:last-child>tr:last-child th:last-child{border-bottom-right-radius:3px}.panel>.panel-body+.table,.panel>.panel-body+.table-responsive,.panel>.table+.panel-body,.panel>.table-responsive+.panel-body{border-top:1px solid #ddd}.panel>.table>tbody:first-child>tr:first-child td,.panel>.table>tbody:first-child>tr:first-child th{border-top:0}.panel>.table-bordered,.panel>.table-responsive>.table-bordered{border:0}.panel>.table-bordered>tbody>tr>td:first-child,.panel>.table-bordered>tbody>tr>th:first-child,.panel>.table-bordered>tfoot>tr>td:first-child,.panel>.table-bordered>tfoot>tr>th:first-child,.panel>.table-bordered>thead>tr>td:first-child,.panel>.table-bordered>thead>tr>th:first-child,.panel>.table-responsive>.table-bordered>tbody>tr>td:first-child,.panel>.table-responsive>.table-bordered>tbody>tr>th:first-child,.panel>.table-responsive>.table-bordered>tfoot>tr>td:first-child,.panel>.table-responsive>.table-bordered>tfoot>tr>th:first-child,.panel>.table-responsive>.table-bordered>thead>tr>td:first-child,.panel>.table-responsive>.table-bordered>thead>tr>th:first-child{border-left:0}.panel>.table-bordered>tbody>tr>td:last-child,.panel>.table-bordered>tbody>tr>th:last-child,.panel>.table-bordered>tfoot>tr>td:last-child,.panel>.table-bordered>tfoot>tr>th:last-child,.panel>.table-bordered>thead>tr>td:last-child,.panel>.table-bordered>thead>tr>th:last-child,.panel>.table-responsive>.table-bordered>tbody>tr>td:last-child,.panel>.table-responsive>.table-bordered>tbody>tr>th:last-child,.panel>.table-responsive>.table-bordered>tfoot>tr>td:last-child,.panel>.table-responsive>.table-bordered>tfoot>tr>th:last-child,.panel>.table-responsive>.table-bordered>thead>tr>td:last-child,.panel>.table-responsive>.table-bordered>thead>tr>th:last-child{border-right:0}.panel>.table-bordered>tbody>tr:first-child>td,.panel>.table-bordered>tbody>tr:first-child>th,.panel>.table-bordered>thead>tr:first-child>td,.panel>.table-bordered>thead>tr:first-child>th,.panel>.table-responsive>.table-bordered>tbody>tr:first-child>td,.panel>.table-responsive>.table-bordered>tbody>tr:first-child>th,.panel>.table-responsive>.table-bordered>thead>tr:first-child>td,.panel>.table-responsive>.table-bordered>thead>tr:first-child>th{border-bottom:0}.panel>.table-bordered>tbody>tr:last-child>td,.panel>.table-bordered>tbody>tr:last-child>th,.panel>.table-bordered>tfoot>tr:last-child>td,.panel>.table-bordered>tfoot>tr:last-child>th,.panel>.table-responsive>.table-bordered>tbody>tr:last-child>td,.panel>.table-responsive>.table-bordered>tbody>tr:last-child>th,.panel>.table-responsive>.table-bordered>tfoot>tr:last-child>td,.panel>.table-responsive>.table-bordered>tfoot>tr:last-child>th{border-bottom:0}.panel>.table-responsive{margin-bottom:0;border:0}.panel-group{margin-bottom:20px}.panel-group .panel{margin-bottom:0;border-radius:4px}.panel-group .panel+.panel{margin-top:5px}.panel-group .panel-heading{border-bottom:0}.panel-group .panel-heading+.panel-collapse>.list-group,.panel-group .panel-heading+.panel-collapse>.panel-body{border-top:1px solid #ddd}.panel-group .panel-footer{border-top:0}.panel-group .panel-footer+.panel-collapse .panel-body{border-bottom:1px solid #ddd}.panel-default{border-color:#ddd}.panel-default>.panel-heading{color:#333;background-color:#f5f5f5;border-color:#ddd}.panel-default>.panel-heading+.panel-collapse>.panel-body{border-top-color:#ddd}.panel-default>.panel-heading .badge{color:#f5f5f5;background-color:#333}.panel-default>.panel-footer+.panel-collapse>.panel-body{border-bottom-color:#ddd}.panel-primary{border-color:#337ab7}.panel-primary>.panel-heading{color:#fff;background-color:#337ab7;border-color:#337ab7}.panel-primary>.panel-heading+.panel-collapse>.panel-body{border-top-color:#337ab7}.panel-primary>.panel-heading .badge{color:#337ab7;background-color:#fff}.panel-primary>.panel-footer+.panel-collapse>.panel-body{border-bottom-color:#337ab7}.panel-success{border-color:#d6e9c6}.panel-success>.panel-heading{color:#3c763d;background-color:#dff0d8;border-color:#d6e9c6}.panel-success>.panel-heading+.panel-collapse>.panel-body{border-top-color:#d6e9c6}.panel-success>.panel-heading .badge{color:#dff0d8;background-color:#3c763d}.panel-success>.panel-footer+.panel-collapse>.panel-body{border-bottom-color:#d6e9c6}.panel-info{border-color:#bce8f1}.panel-info>.panel-heading{color:#31708f;background-color:#d9edf7;border-color:#bce8f1}.panel-info>.panel-heading+.panel-collapse>.panel-body{border-top-color:#bce8f1}.panel-info>.panel-heading .badge{color:#d9edf7;background-color:#31708f}.panel-info>.panel-footer+.panel-collapse>.panel-body{border-bottom-color:#bce8f1}.panel-warning{border-color:#faebcc}.panel-warning>.panel-heading{color:#8a6d3b;background-color:#fcf8e3;border-color:#faebcc}.panel-warning>.panel-heading+.panel-collapse>.panel-body{border-top-color:#faebcc}.panel-warning>.panel-heading .badge{color:#fcf8e3;background-color:#8a6d3b}.panel-warning>.panel-footer+.panel-collapse>.panel-body{border-bottom-color:#faebcc}.panel-danger{border-color:#ebccd1}.panel-danger>.panel-heading{color:#a94442;background-color:#f2dede;border-color:#ebccd1}.panel-danger>.panel-heading+.panel-collapse>.panel-body{border-top-color:#ebccd1}.panel-danger>.panel-heading .badge{color:#f2dede;background-color:#a94442}.panel-danger>.panel-footer+.panel-collapse>.panel-body{border-bottom-color:#ebccd1}.embed-responsive{position:relative;display:block;height:0;padding:0;overflow:hidden}.embed-responsive .embed-responsive-item,.embed-responsive embed,.embed-responsive iframe,.embed-responsive object,.embed-responsive video{position:absolute;top:0;bottom:0;left:0;width:100%;height:100%;border:0}.embed-responsive-16by9{padding-bottom:56.25%}.embed-responsive-4by3{padding-bottom:75%}.well{min-height:20px;padding:19px;margin-bottom:20px;background-color:#f5f5f5;border:1px solid #e3e3e3;border-radius:4px;-webkit-box-shadow:inset 0 1px 1px rgba(0,0,0,.05);box-shadow:inset 0 1px 1px rgba(0,0,0,.05)}.well blockquote{border-color:#ddd;border-color:rgba(0,0,0,.15)}.well-lg{padding:24px;border-radius:6px}.well-sm{padding:9px;border-radius:3px}.close{float:right;font-size:21px;font-weight:700;line-height:1;color:#000;text-shadow:0 1px 0 #fff;filter:alpha(opacity=20);opacity:.2}.close:focus,.close:hover{color:#000;text-decoration:none;cursor:pointer;filter:alpha(opacity=50);opacity:.5}button.close{-webkit-appearance:none;padding:0;cursor:pointer;background:0 0;border:0}.modal-open{overflow:hidden}.modal{position:fixed;top:0;right:0;bottom:0;left:0;z-index:1050;display:none;overflow:hidden;-webkit-overflow-scrolling:touch;outline:0}.modal.fade .modal-dialog{-webkit-transition:-webkit-transform .3s ease-out;-o-transition:-o-transform .3s ease-out;transition:transform .3s ease-out;-webkit-transform:translate(0,-25%);-ms-transform:translate(0,-25%);-o-transform:translate(0,-25%);transform:translate(0,-25%)}.modal.in .modal-dialog{-webkit-transform:translate(0,0);-ms-transform:translate(0,0);-o-transform:translate(0,0);transform:translate(0,0)}.modal-open .modal{overflow-x:hidden;overflow-y:auto}.modal-dialog{position:relative;width:auto;margin:10px}.modal-content{position:relative;background-color:#fff;-webkit-background-clip:padding-box;background-clip:padding-box;border:1px solid #999;border:1px solid rgba(0,0,0,.2);border-radius:6px;outline:0;-webkit-box-shadow:0 3px 9px rgba(0,0,0,.5);box-shadow:0 3px 9px rgba(0,0,0,.5)}.modal-backdrop{position:fixed;top:0;right:0;bottom:0;left:0;z-index:1040;background-color:#000}.modal-backdrop.fade{filter:alpha(opacity=0);opacity:0}.modal-backdrop.in{filter:alpha(opacity=50);opacity:.5}.modal-header{padding:15px;border-bottom:1px solid #e5e5e5}.modal-header .close{margin-top:-2px}.modal-title{margin:0;line-height:1.42857143}.modal-body{position:relative;padding:15px}.modal-footer{padding:15px;text-align:right;border-top:1px solid #e5e5e5}.modal-footer .btn+.btn{margin-bottom:0;margin-left:5px}.modal-footer .btn-group .btn+.btn{margin-left:-1px}.modal-footer .btn-block+.btn-block{margin-left:0}.modal-scrollbar-measure{position:absolute;top:-9999px;width:50px;height:50px;overflow:scroll}@media (min-width:768px){.modal-dialog{width:600px;margin:30px auto}.modal-content{-webkit-box-shadow:0 5px 15px rgba(0,0,0,.5);box-shadow:0 5px 15px rgba(0,0,0,.5)}.modal-sm{width:300px}}@media (min-width:992px){.modal-lg{width:900px}}.tooltip{position:absolute;z-index:1070;display:block;font-family:"Helvetica Neue",Helvetica,Arial,sans-serif;font-size:12px;font-style:normal;font-weight:400;line-height:1.42857143;text-align:left;text-align:start;text-decoration:none;text-shadow:none;text-transform:none;letter-spacing:normal;word-break:normal;word-spacing:normal;word-wrap:normal;white-space:normal;filter:alpha(opacity=0);opacity:0;line-break:auto}.tooltip.in{filter:alpha(opacity=90);opacity:.9}.tooltip.top{padding:5px 0;margin-top:-3px}.tooltip.right{padding:0 5px;margin-left:3px}.tooltip.bottom{padding:5px 0;margin-top:3px}.tooltip.left{padding:0 5px;margin-left:-3px}.tooltip-inner{max-width:200px;padding:3px 8px;color:#fff;text-align:center;background-color:#000;border-radius:4px}.tooltip-arrow{position:absolute;width:0;height:0;border-color:transparent;border-style:solid}.tooltip.top .tooltip-arrow{bottom:0;left:50%;margin-left:-5px;border-width:5px 5px 0;border-top-color:#000}.tooltip.top-left .tooltip-arrow{right:5px;bottom:0;margin-bottom:-5px;border-width:5px 5px 0;border-top-color:#000}.tooltip.top-right .tooltip-arrow{bottom:0;left:5px;margin-bottom:-5px;border-width:5px 5px 0;border-top-color:#000}.tooltip.right .tooltip-arrow{top:50%;left:0;margin-top:-5px;border-width:5px 5px 5px 0;border-right-color:#000}.tooltip.left .tooltip-arrow{top:50%;right:0;margin-top:-5px;border-width:5px 0 5px 5px;border-left-color:#000}.tooltip.bottom .tooltip-arrow{top:0;left:50%;margin-left:-5px;border-width:0 5px 5px;border-bottom-color:#000}.tooltip.bottom-left .tooltip-arrow{top:0;right:5px;margin-top:-5px;border-width:0 5px 5px;border-bottom-color:#000}.tooltip.bottom-right .tooltip-arrow{top:0;left:5px;margin-top:-5px;border-width:0 5px 5px;border-bottom-color:#000}.popover{position:absolute;top:0;left:0;z-index:1060;display:none;max-width:276px;padding:1px;font-family:"Helvetica Neue",Helvetica,Arial,sans-serif;font-size:14px;font-style:normal;font-weight:400;line-height:1.42857143;text-align:left;text-align:start;text-decoration:none;text-shadow:none;text-transform:none;letter-spacing:normal;word-break:normal;word-spacing:normal;word-wrap:normal;white-space:normal;background-color:#fff;-webkit-background-clip:padding-box;background-clip:padding-box;border:1px solid #ccc;border:1px solid rgba(0,0,0,.2);border-radius:6px;-webkit-box-shadow:0 5px 10px rgba(0,0,0,.2);box-shadow:0 5px 10px rgba(0,0,0,.2);line-break:auto}.popover.top{margin-top:-10px}.popover.right{margin-left:10px}.popover.bottom{margin-top:10px}.popover.left{margin-left:-10px}.popover-title{padding:8px 14px;margin:0;font-size:14px;background-color:#f7f7f7;border-bottom:1px solid #ebebeb;border-radius:5px 5px 0 0}.popover-content{padding:9px 14px}.popover>.arrow,.popover>.arrow:after{position:absolute;display:block;width:0;height:0;border-color:transparent;border-style:solid}.popover>.arrow{border-width:11px}.popover>.arrow:after{content:"";border-width:10px}.popover.top>.arrow{bottom:-11px;left:50%;margin-left:-11px;border-top-color:#999;border-top-color:rgba(0,0,0,.25);border-bottom-width:0}.popover.top>.arrow:after{bottom:1px;margin-left:-10px;content:" ";border-top-color:#fff;border-bottom-width:0}.popover.right>.arrow{top:50%;left:-11px;margin-top:-11px;border-right-color:#999;border-right-color:rgba(0,0,0,.25);border-left-width:0}.popover.right>.arrow:after{bottom:-10px;left:1px;content:" ";border-right-color:#fff;border-left-width:0}.popover.bottom>.arrow{top:-11px;left:50%;margin-left:-11px;border-top-width:0;border-bottom-color:#999;border-bottom-color:rgba(0,0,0,.25)}.popover.bottom>.arrow:after{top:1px;margin-left:-10px;content:" ";border-top-width:0;border-bottom-color:#fff}.popover.left>.arrow{top:50%;right:-11px;margin-top:-11px;border-right-width:0;border-left-color:#999;border-left-color:rgba(0,0,0,.25)}.popover.left>.arrow:after{right:1px;bottom:-10px;content:" ";border-right-width:0;border-left-color:#fff}.carousel{position:relative}.carousel-inner{position:relative;width:100%;overflow:hidden}.carousel-inner>.item{position:relative;display:none;-webkit-transition:.6s ease-in-out left;-o-transition:.6s ease-in-out left;transition:.6s ease-in-out left}.carousel-inner>.item>a>img,.carousel-inner>.item>img{line-height:1}@media all and (transform-3d),(-webkit-transform-3d){.carousel-inner>.item{-webkit-transition:-webkit-transform .6s ease-in-out;-o-transition:-o-transform .6s ease-in-out;transition:transform .6s ease-in-out;-webkit-backface-visibility:hidden;backface-visibility:hidden;-webkit-perspective:1000px;perspective:1000px}.carousel-inner>.item.active.right,.carousel-inner>.item.next{left:0;-webkit-transform:translate3d(100%,0,0);transform:translate3d(100%,0,0)}.carousel-inner>.item.active.left,.carousel-inner>.item.prev{left:0;-webkit-transform:translate3d(-100%,0,0);transform:translate3d(-100%,0,0)}.carousel-inner>.item.active,.carousel-inner>.item.next.left,.carousel-inner>.item.prev.right{left:0;-webkit-transform:translate3d(0,0,0);transform:translate3d(0,0,0)}}.carousel-inner>.active,.carousel-inner>.next,.carousel-inner>.prev{display:block}.carousel-inner>.active{left:0}.carousel-inner>.next,.carousel-inner>.prev{position:absolute;top:0;width:100%}.carousel-inner>.next{left:100%}.carousel-inner>.prev{left:-100%}.carousel-inner>.next.left,.carousel-inner>.prev.right{left:0}.carousel-inner>.active.left{left:-100%}.carousel-inner>.active.right{left:100%}.carousel-control{position:absolute;top:0;bottom:0;left:0;width:15%;font-size:20px;color:#fff;text-align:center;text-shadow:0 1px 2px rgba(0,0,0,.6);background-color:rgba(0,0,0,0);filter:alpha(opacity=50);opacity:.5}.carousel-control.left{background-image:-webkit-linear-gradient(left,rgba(0,0,0,.5) 0,rgba(0,0,0,.0001) 100%);background-image:-o-linear-gradient(left,rgba(0,0,0,.5) 0,rgba(0,0,0,.0001) 100%);background-image:-webkit-gradient(linear,left top,right top,from(rgba(0,0,0,.5)),to(rgba(0,0,0,.0001)));background-image:linear-gradient(to right,rgba(0,0,0,.5) 0,rgba(0,0,0,.0001) 100%);filter:progid:DXImageTransform.Microsoft.gradient(startColorstr='#80000000', endColorstr='#00000000', GradientType=1);background-repeat:repeat-x}.carousel-control.right{right:0;left:auto;background-image:-webkit-linear-gradient(left,rgba(0,0,0,.0001) 0,rgba(0,0,0,.5) 100%);background-image:-o-linear-gradient(left,rgba(0,0,0,.0001) 0,rgba(0,0,0,.5) 100%);background-image:-webkit-gradient(linear,left top,right top,from(rgba(0,0,0,.0001)),to(rgba(0,0,0,.5)));background-image:linear-gradient(to right,rgba(0,0,0,.0001) 0,rgba(0,0,0,.5) 100%);filter:progid:DXImageTransform.Microsoft.gradient(startColorstr='#00000000', endColorstr='#80000000', GradientType=1);background-repeat:repeat-x}.carousel-control:focus,.carousel-control:hover{color:#fff;text-decoration:none;filter:alpha(opacity=90);outline:0;opacity:.9}.carousel-control .glyphicon-chevron-left,.carousel-control .glyphicon-chevron-right,.carousel-control .icon-next,.carousel-control .icon-prev{position:absolute;top:50%;z-index:5;display:inline-block;margin-top:-10px}.carousel-control .glyphicon-chevron-left,.carousel-control .icon-prev{left:50%;margin-left:-10px}.carousel-control .glyphicon-chevron-right,.carousel-control .icon-next{right:50%;margin-right:-10px}.carousel-control .icon-next,.carousel-control .icon-prev{width:20px;height:20px;font-family:serif;line-height:1}.carousel-control .icon-prev:before{content:'\2039'}.carousel-control .icon-next:before{content:'\203a'}.carousel-indicators{position:absolute;bottom:10px;left:50%;z-index:15;width:60%;padding-left:0;margin-left:-30%;text-align:center;list-style:none}.carousel-indicators li{display:inline-block;width:10px;height:10px;margin:1px;text-indent:-999px;cursor:pointer;background-color:#000\9;background-color:rgba(0,0,0,0);border:1px solid #fff;border-radius:10px}.carousel-indicators .active{width:12px;height:12px;margin:0;background-color:#fff}.carousel-caption{position:absolute;right:15%;bottom:20px;left:15%;z-index:10;padding-top:20px;padding-bottom:20px;color:#fff;text-align:center;text-shadow:0 1px 2px rgba(0,0,0,.6)}.carousel-caption .btn{text-shadow:none}@media screen and (min-width:768px){.carousel-control .glyphicon-chevron-left,.carousel-control .glyphicon-chevron-right,.carousel-control .icon-next,.carousel-control .icon-prev{width:30px;height:30px;margin-top:-10px;font-size:30px}.carousel-control .glyphicon-chevron-left,.carousel-control .icon-prev{margin-left:-10px}.carousel-control .glyphicon-chevron-right,.carousel-control .icon-next{margin-right:-10px}.carousel-caption{right:20%;left:20%;padding-bottom:30px}.carousel-indicators{bottom:20px}}.btn-group-vertical>.btn-group:after,.btn-group-vertical>.btn-group:before,.btn-toolbar:after,.btn-toolbar:before,.clearfix:after,.clearfix:before,.container-fluid:after,.container-fluid:before,.container:after,.container:before,.dl-horizontal dd:after,.dl-horizontal dd:before,.form-horizontal .form-group:after,.form-horizontal .form-group:before,.modal-footer:after,.modal-footer:before,.modal-header:after,.modal-header:before,.nav:after,.nav:before,.navbar-collapse:after,.navbar-collapse:before,.navbar-header:after,.navbar-header:before,.navbar:after,.navbar:before,.pager:after,.pager:before,.panel-body:after,.panel-body:before,.row:after,.row:before{display:table;content:" "}.btn-group-vertical>.btn-group:after,.btn-toolbar:after,.clearfix:after,.container-fluid:after,.container:after,.dl-horizontal dd:after,.form-horizontal .form-group:after,.modal-footer:after,.modal-header:after,.nav:after,.navbar-collapse:after,.navbar-header:after,.navbar:after,.pager:after,.panel-body:after,.row:after{clear:both}.center-block{display:block;margin-right:auto;margin-left:auto}.pull-right{float:right!important}.pull-left{float:left!important}.hide{display:none!important}.show{display:block!important}.invisible{visibility:hidden}.text-hide{font:0/0 a;color:transparent;text-shadow:none;background-color:transparent;border:0}.hidden{display:none!important}.affix{position:fixed}@-ms-viewport{width:device-width}.visible-lg,.visible-md,.visible-sm,.visible-xs{display:none!important}.visible-lg-block,.visible-lg-inline,.visible-lg-inline-block,.visible-md-block,.visible-md-inline,.visible-md-inline-block,.visible-sm-block,.visible-sm-inline,.visible-sm-inline-block,.visible-xs-block,.visible-xs-inline,.visible-xs-inline-block{display:none!important}@media (max-width:767px){.visible-xs{display:block!important}table.visible-xs{display:table!important}tr.visible-xs{display:table-row!important}td.visible-xs,th.visible-xs{display:table-cell!important}}@media (max-width:767px){.visible-xs-block{display:block!important}}@media (max-width:767px){.visible-xs-inline{display:inline!important}}@media (max-width:767px){.visible-xs-inline-block{display:inline-block!important}}@media (min-width:768px) and (max-width:991px){.visible-sm{display:block!important}table.visible-sm{display:table!important}tr.visible-sm{display:table-row!important}td.visible-sm,th.visible-sm{display:table-cell!important}}@media (min-width:768px) and (max-width:991px){.visible-sm-block{display:block!important}}@media (min-width:768px) and (max-width:991px){.visible-sm-inline{display:inline!important}}@media (min-width:768px) and (max-width:991px){.visible-sm-inline-block{display:inline-block!important}}@media (min-width:992px) and (max-width:1199px){.visible-md{display:block!important}table.visible-md{display:table!important}tr.visible-md{display:table-row!important}td.visible-md,th.visible-md{display:table-cell!important}}@media (min-width:992px) and (max-width:1199px){.visible-md-block{display:block!important}}@media (min-width:992px) and (max-width:1199px){.visible-md-inline{display:inline!important}}@media (min-width:992px) and (max-width:1199px){.visible-md-inline-block{display:inline-block!important}}@media (min-width:1200px){.visible-lg{display:block!important}table.visible-lg{display:table!important}tr.visible-lg{display:table-row!important}td.visible-lg,th.visible-lg{display:table-cell!important}}@media (min-width:1200px){.visible-lg-block{display:block!important}}@media (min-width:1200px){.visible-lg-inline{display:inline!important}}@media (min-width:1200px){.visible-lg-inline-block{display:inline-block!important}}@media (max-width:767px){.hidden-xs{display:none!important}}@media (min-width:768px) and (max-width:991px){.hidden-sm{display:none!important}}@media (min-width:992px) and (max-width:1199px){.hidden-md{display:none!important}}@media (min-width:1200px){.hidden-lg{display:none!important}}.visible-print{display:none!important}@media print{.visible-print{display:block!important}table.visible-print{display:table!important}tr.visible-print{display:table-row!important}td.visible-print,th.visible-print{display:table-cell!important}}.visible-print-block{display:none!important}@media print{.visible-print-block{display:block!important}}.visible-print-inline{display:none!important}@media print{.visible-print-inline{display:inline!important}}.visible-print-inline-block{display:none!important}@media print{.visible-print-inline-block{display:inline-block!important}}@media print{.hidden-print{display:none!important}}
diff --git a/lib/cylc/cylc-review/static/css/cylc-review.css b/lib/cylc/cylc-review/static/css/cylc-review.css
deleted file mode 100644
index aab95adf27c..00000000000
--- a/lib/cylc/cylc-review/static/css/cylc-review.css
+++ /dev/null
@@ -1,115 +0,0 @@
-pre {
-    white-space: pre;
-}
-.entry {
-    border-radius: 5px;
-    padding: 5px;
-}
-.even {
-    background: #eee;
-}
-.inline {
-    margin: 0;
-}
-.page-header {
-    padding-left: 10px;
-    padding-right: 10px;
-}
-.highlight {
-    background-color: yellow;
-}
-.line-number {
-    cursor: pointer;
-}
-.badge-label {
-    border-radius: 1em;
-}
-
-
-@media screen {
-    div.top_nav {
-        position: fixed;
-        bottom: 0;
-        background-color: lightgrey;
-        z-index: 10;
-     }
-    .next {
-        background: white
-     }
-    .previous {
-        background: white
-     }
-    .page_entries select {
-        width: 300px;
-     }
-}
-
-
-/* dropdown-submenu code from bootstrap v2.3.1 (removed in v3.x)
-   see boostrap v2.3.1 docs/assets/css/boostrap.css #3031->3075 */
-.dropdown-submenu {
-  position: relative;
-}
-
-.dropdown-submenu > .dropdown-menu {
-  top: 0;
-  left: 100%;
-  margin-top: -6px;
-  margin-left: -1px;
-  -webkit-border-radius: 0 6px 6px 6px;
-     -moz-border-radius: 0 6px 6px 6px;
-          border-radius: 0 6px 6px 6px;
-}
-
-.dropdown-submenu:hover > .dropdown-menu {
-  display: block;
-}
-
-.dropup .dropdown-submenu > .dropdown-menu {
-  top: auto;
-  bottom: 0;
-  margin-top: 0;
-  margin-bottom: -2px;
-  -webkit-border-radius: 5px 5px 5px 0;
-     -moz-border-radius: 5px 5px 5px 0;
-          border-radius: 5px 5px 5px 0;
-}
-
-.dropdown-submenu > a:after {
-  display: block;
-  float: right;
-  width: 0;
-  height: 0;
-  margin-top: 5px;
-  margin-right: -10px;
-  border-color: transparent;
-  border-left-color: #cccccc;
-  border-style: solid;
-  border-width: 5px 0 5px 5px;
-  content: " ";
-}
-
-.dropdown-submenu:hover > a:after {
-  border-left-color: #222222;  /* make the right arrow visible on hover */
-}
-
-.dropdown-submenu.pull-left {
-  float: none;
-}
-
-.dropdown-submenu.pull-left > .dropdown-menu {
-  left: -100%;
-  margin-left: 10px;
-  -webkit-border-radius: 6px 0 6px 6px;
-     -moz-border-radius: 6px 0 6px 6px;
-          border-radius: 6px 0 6px 6px;
-}
-
-@-moz-document url-prefix() {
-  fieldset { display: table-cell; }
-}
-
-th,
-.livestamp {
-  white-space: nowrap;
-}
diff --git a/lib/cylc/cylc-review/static/css/dataTables.bootstrap.css b/lib/cylc/cylc-review/static/css/dataTables.bootstrap.css
deleted file mode 100644
index faee0e50dbf..00000000000
--- a/lib/cylc/cylc-review/static/css/dataTables.bootstrap.css
+++ /dev/null
@@ -1,319 +0,0 @@
-div.dataTables_length label {
-	font-weight: normal;
-	text-align: left;
-	white-space: nowrap;
-}
-
-div.dataTables_length select {
-	width: 75px;
-	display: inline-block;
-}
-
-div.dataTables_filter {
-	text-align: right;
-}
-
-div.dataTables_filter label {
-	font-weight: normal;
-	white-space: nowrap;
-	text-align: left;
-}
-
-div.dataTables_filter input {
-	margin-left: 0.5em;
-	display: inline-block;
-}
-
-div.dataTables_info {
-	padding-top: 8px;
-	white-space: nowrap;
-}
-
-div.dataTables_paginate {
-	margin: 0;
-	white-space: nowrap;
-	text-align: right;
-}
-
-div.dataTables_paginate ul.pagination {
-	margin: 2px 0;
-	white-space: nowrap;
-}
-
-@media screen and (max-width: 767px) {
-	div.dataTables_length,
-	div.dataTables_filter,
-	div.dataTables_info,
-	div.dataTables_paginate {
-		text-align: center;
-	}
-}
-
-
-table.dataTable td,
-table.dataTable th {
-	-webkit-box-sizing: content-box;
-	-moz-box-sizing: content-box;
-	box-sizing: content-box;
-}
-
-
-table.dataTable {
-	clear: both;
-	margin-top: 6px !important;
-	margin-bottom: 6px !important;
-	max-width: none !important;
-}
-
-table.dataTable thead .sorting,
-table.dataTable thead .sorting_asc,
-table.dataTable thead .sorting_desc,
-table.dataTable thead .sorting_asc_disabled,
-table.dataTable thead .sorting_desc_disabled {
-	cursor: pointer;
-}
-
-table.dataTable thead .sorting { background: url('../images/sort_both.png') no-repeat center right; }
-table.dataTable thead .sorting_asc { background: url('../images/sort_asc.png') no-repeat center right; }
-table.dataTable thead .sorting_desc { background: url('../images/sort_desc.png') no-repeat center right; }
-
-table.dataTable thead .sorting_asc_disabled { background: url('../images/sort_asc_disabled.png') no-repeat center right; }
-table.dataTable thead .sorting_desc_disabled { background: url('../images/sort_desc_disabled.png') no-repeat center right; }
-
-table.dataTable thead > tr > th {
-	padding-left: 18px;
-	padding-right: 18px;
-}
-
-table.dataTable th:active {
-	outline: none;
-}
-
-/* Scrolling */
-div.dataTables_scrollHead table {
-	margin-bottom: 0 !important;
-	border-bottom-left-radius: 0;
-	border-bottom-right-radius: 0;
-}
-
-div.dataTables_scrollHead table thead tr:last-child th:first-child,
-div.dataTables_scrollHead table thead tr:last-child td:first-child {
-	border-bottom-left-radius: 0 !important;
-	border-bottom-right-radius: 0 !important;
-}
-
-div.dataTables_scrollBody table {
-	border-top: none;
-	margin-top: 0 !important;
-	margin-bottom: 0 !important;
-}
-
-div.dataTables_scrollBody tbody tr:first-child th,
-div.dataTables_scrollBody tbody tr:first-child td {
-	border-top: none;
-}
-
-div.dataTables_scrollFoot table {
-	margin-top: 0 !important;
-	border-top: none;
-}
-
-/* Frustratingly the border-collapse:collapse used by Bootstrap makes the column
-   width calculations when using scrolling impossible to align columns. We have
-   to use separate
- */
-table.table-bordered.dataTable {
-	border-collapse: separate !important;
-}
-table.table-bordered thead th,
-table.table-bordered thead td {
-	border-left-width: 0;
-	border-top-width: 0;
-}
-table.table-bordered tbody th,
-table.table-bordered tbody td {
-	border-left-width: 0;
-	border-bottom-width: 0;
-}
-table.table-bordered th:last-child,
-table.table-bordered td:last-child {
-	border-right-width: 0;
-}
-div.dataTables_scrollHead table.table-bordered {
-	border-bottom-width: 0;
-}
-
-
-
-
-/*
- * TableTools styles
- */
-.table.dataTable tbody tr.active td,
-.table.dataTable tbody tr.active th {
-	background-color: #08C;
-	color: white;
-}
-
-.table.dataTable tbody tr.active:hover td,
-.table.dataTable tbody tr.active:hover th {
-	background-color: #0075b0 !important;
-}
-
-.table.dataTable tbody tr.active th > a,
-.table.dataTable tbody tr.active td > a {
-	color: white;
-}
-
-.table-striped.dataTable tbody tr.active:nth-child(odd) td,
-.table-striped.dataTable tbody tr.active:nth-child(odd) th {
-	background-color: #017ebc;
-}
-
-table.DTTT_selectable tbody tr {
-	cursor: pointer;
-}
-
-div.DTTT .btn {
-	color: #333 !important;
-	font-size: 12px;
-}
-
-div.DTTT .btn:hover {
-	text-decoration: none !important;
-}
-
-ul.DTTT_dropdown.dropdown-menu {
-  z-index: 2003;
-}
-
-ul.DTTT_dropdown.dropdown-menu a {
-	color: #333 !important; /* needed only when demo_page.css is included */
-}
-
-ul.DTTT_dropdown.dropdown-menu li {
-	position: relative;
-}
-
-ul.DTTT_dropdown.dropdown-menu li:hover a {
-	background-color: #0088cc;
-	color: white !important;
-}
-
-div.DTTT_collection_background {
-	z-index: 2002;	
-}
-
-/* TableTools information display */
-div.DTTT_print_info {
-	position: fixed;
-	top: 50%;
-	left: 50%;
-	width: 400px;
-	height: 150px;
-	margin-left: -200px;
-	margin-top: -75px;
-	text-align: center;
-	color: #333;
-	padding: 10px 30px;
-	opacity: 0.95;
-
-	background-color: white;
-	border: 1px solid rgba(0, 0, 0, 0.2);
-	border-radius: 6px;
-	
-	-webkit-box-shadow: 0 3px 7px rgba(0, 0, 0, 0.5);
-	        box-shadow: 0 3px 7px rgba(0, 0, 0, 0.5);
-}
-
-div.DTTT_print_info h6 {
-	font-weight: normal;
-	font-size: 28px;
-	line-height: 28px;
-	margin: 1em;
-}
-
-div.DTTT_print_info p {
-	font-size: 14px;
-	line-height: 20px;
-}
-
-div.dataTables_processing {
-    position: absolute;
-    top: 50%;
-    left: 50%;
-    width: 100%;
-    height: 60px;
-    margin-left: -50%;
-    margin-top: -25px;
-    padding-top: 20px;
-    padding-bottom: 20px;
-    text-align: center;
-    font-size: 1.2em;
-    background-color: white;
-    background: -webkit-gradient(linear, left top, right top, color-stop(0%, rgba(255,255,255,0)), color-stop(25%, rgba(255,255,255,0.9)), color-stop(75%, rgba(255,255,255,0.9)), color-stop(100%, rgba(255,255,255,0)));
-    background: -webkit-linear-gradient(left, rgba(255,255,255,0) 0%, rgba(255,255,255,0.9) 25%, rgba(255,255,255,0.9) 75%, rgba(255,255,255,0) 100%);
-    background: -moz-linear-gradient(left, rgba(255,255,255,0) 0%, rgba(255,255,255,0.9) 25%, rgba(255,255,255,0.9) 75%, rgba(255,255,255,0) 100%);
-    background: -ms-linear-gradient(left, rgba(255,255,255,0) 0%, rgba(255,255,255,0.9) 25%, rgba(255,255,255,0.9) 75%, rgba(255,255,255,0) 100%);
-    background: -o-linear-gradient(left, rgba(255,255,255,0) 0%, rgba(255,255,255,0.9) 25%, rgba(255,255,255,0.9) 75%, rgba(255,255,255,0) 100%);
-    background: linear-gradient(to right, rgba(255,255,255,0) 0%, rgba(255,255,255,0.9) 25%, rgba(255,255,255,0.9) 75%, rgba(255,255,255,0) 100%);
-}
-
-
-
-/*
- * FixedColumns styles
- */
-div.DTFC_LeftHeadWrapper table,
-div.DTFC_LeftFootWrapper table,
-div.DTFC_RightHeadWrapper table,
-div.DTFC_RightFootWrapper table,
-table.DTFC_Cloned tr.even {
-    background-color: white;
-    margin-bottom: 0;
-}
- 
-div.DTFC_RightHeadWrapper table ,
-div.DTFC_LeftHeadWrapper table {
-	border-bottom: none !important;
-    margin-bottom: 0 !important;
-    border-top-right-radius: 0 !important;
-    border-bottom-left-radius: 0 !important;
-    border-bottom-right-radius: 0 !important;
-}
- 
-div.DTFC_RightHeadWrapper table thead tr:last-child th:first-child,
-div.DTFC_RightHeadWrapper table thead tr:last-child td:first-child,
-div.DTFC_LeftHeadWrapper table thead tr:last-child th:first-child,
-div.DTFC_LeftHeadWrapper table thead tr:last-child td:first-child {
-    border-bottom-left-radius: 0 !important;
-    border-bottom-right-radius: 0 !important;
-}
- 
-div.DTFC_RightBodyWrapper table,
-div.DTFC_LeftBodyWrapper table {
-    border-top: none;
-    margin: 0 !important;
-}
- 
-div.DTFC_RightBodyWrapper tbody tr:first-child th,
-div.DTFC_RightBodyWrapper tbody tr:first-child td,
-div.DTFC_LeftBodyWrapper tbody tr:first-child th,
-div.DTFC_LeftBodyWrapper tbody tr:first-child td {
-    border-top: none;
-}
- 
-div.DTFC_RightFootWrapper table,
-div.DTFC_LeftFootWrapper table {
-    border-top: none;
-    margin-top: 0 !important;
-}
-
-
-/*
- * FixedHeader styles
- */
-div.FixedHeader_Cloned table {
-	margin: 0 !important
-}
-
diff --git a/lib/cylc/cylc-review/static/css/jquery.dataTables.colVis.min.css b/lib/cylc/cylc-review/static/css/jquery.dataTables.colVis.min.css
deleted file mode 100644
index 01b31107d2d..00000000000
--- a/lib/cylc/cylc-review/static/css/jquery.dataTables.colVis.min.css
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- * File:        ColVis.min.js
- * Version:     1.0.6
- * Author:      Allan Jardine (www.sprymedia.co.uk)
- *
- * Copyright 2010-2011 Allan Jardine, all rights reserved.
- *
- * This source file is free software, under either the GPL v2 license or a
- * BSD (3 point) style license, as supplied with this software.
- *
- * This source file is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
- * or FITNESS FOR A PARTICULAR PURPOSE. See the license files for details.
- */
-(function(d){ColVis=function(a,b){if(!this.CLASS||this.CLASS!="ColVis")alert("Warning: ColVis must be initialised with the keyword 'new'");if(typeof b=="undefined")b={};this.s={dt:null,oInit:b,fnStateChange:null,activate:"click",sAlign:"left",buttonText:"Show / hide columns",hidden:true,aiExclude:[],abOriginal:[],bRestore:false,sRestore:"Restore original",iOverlayFade:500,fnLabel:null,sSize:"auto"};this.dom={wrapper:null,button:null,collection:null,background:null,catcher:null,buttons:[],restore:null};
-ColVis.aInstances.push(this);this.s.dt=a;this._fnConstruct();return this};ColVis.prototype={fnRebuild:function(){for(var a=this.dom.buttons.length-1;a>=0;a--)this.dom.buttons[a]!==null&&this.dom.collection.removeChild(this.dom.buttons[a]);this.dom.buttons.splice(0,this.dom.buttons.length);this.dom.restore&&this.dom.restore.parentNode(this.dom.restore);this._fnAddButtons();this._fnDrawCallback()},_fnConstruct:function(){this._fnApplyCustomisation();var a=this;this.dom.wrapper=document.createElement("div");
-this.dom.wrapper.className="ColVis TableTools";this.dom.button=this._fnDomBaseButton(this.s.buttonText);this.dom.button.className+=" ColVis_MasterButton";this.dom.wrapper.appendChild(this.dom.button);this.dom.catcher=this._fnDomCatcher();this.dom.collection=this._fnDomCollection();this.dom.background=this._fnDomBackground();this._fnAddButtons();for(var b=0,c=this.s.dt.aoColumns.length;b'+this.s.sRestore+"");d(b).click(function(){for(var e=0,f=a.s.abOriginal.length;e'+c+"");d(e).click(function(g){var h=!d("input",this).is(":checked");if(g.target.nodeName.toLowerCase()=="input")h=d("input",this).is(":checked");g=d.fn.dataTableExt.iApiIndex;d.fn.dataTableExt.iApiIndex=b._fnDataTablesApiIndex.call(b);b.s.dt.oInstance.fnSetColumnVis(a,h);d.fn.dataTableExt.iApiIndex=g;b.s.fnStateChange!==
-null&&b.s.fnStateChange.call(b,a,h)});return e},_fnDataTablesApiIndex:function(){for(var a=0,b=this.s.dt.oInstance.length;ai?c:i)+"px";f.style.width=(jh)e.style.left=h-b+"px";setTimeout(function(){d(e).animate({opacity:1},a.s.iOverlayFade);d(f).animate({opacity:0.1},a.s.iOverlayFade,"linear",function(){jQuery.browser.msie&&jQuery.browser.version=="6.0"&&a._fnDrawCallback()})},10);this.s.hidden=false},_fnCollectionHide:function(){var a=this;if(!this.s.hidden&&this.dom.collection!==null){this.s.hidden=
-true;d(this.dom.collection).animate({opacity:0},a.s.iOverlayFade,function(){this.style.display="none"});d(this.dom.background).animate({opacity:0},a.s.iOverlayFade,function(){document.body.removeChild(a.dom.background);document.body.removeChild(a.dom.catcher)})}}};ColVis.fnRebuild=function(a){var b=null;if(typeof a!="undefined")b=a.fnSettings().nTable;for(var c=0,e=ColVis.aInstances.length;c .sorting_1,
-table.dataTable.order-column tbody tr > .sorting_2,
-table.dataTable.order-column tbody tr > .sorting_3, table.dataTable.display tbody tr > .sorting_1,
-table.dataTable.display tbody tr > .sorting_2,
-table.dataTable.display tbody tr > .sorting_3 {
-  background-color: #f9f9f9;
-}
-table.dataTable.order-column tbody tr.selected > .sorting_1,
-table.dataTable.order-column tbody tr.selected > .sorting_2,
-table.dataTable.order-column tbody tr.selected > .sorting_3, table.dataTable.display tbody tr.selected > .sorting_1,
-table.dataTable.display tbody tr.selected > .sorting_2,
-table.dataTable.display tbody tr.selected > .sorting_3 {
-  background-color: #acbad4;
-}
-table.dataTable.display tbody tr.odd > .sorting_1, table.dataTable.order-column.stripe tbody tr.odd > .sorting_1 {
-  background-color: #f1f1f1;
-}
-table.dataTable.display tbody tr.odd > .sorting_2, table.dataTable.order-column.stripe tbody tr.odd > .sorting_2 {
-  background-color: #f3f3f3;
-}
-table.dataTable.display tbody tr.odd > .sorting_3, table.dataTable.order-column.stripe tbody tr.odd > .sorting_3 {
-  background-color: whitesmoke;
-}
-table.dataTable.display tbody tr.odd.selected > .sorting_1, table.dataTable.order-column.stripe tbody tr.odd.selected > .sorting_1 {
-  background-color: #a6b3cd;
-}
-table.dataTable.display tbody tr.odd.selected > .sorting_2, table.dataTable.order-column.stripe tbody tr.odd.selected > .sorting_2 {
-  background-color: #a7b5ce;
-}
-table.dataTable.display tbody tr.odd.selected > .sorting_3, table.dataTable.order-column.stripe tbody tr.odd.selected > .sorting_3 {
-  background-color: #a9b6d0;
-}
-table.dataTable.display tbody tr.even > .sorting_1, table.dataTable.order-column.stripe tbody tr.even > .sorting_1 {
-  background-color: #f9f9f9;
-}
-table.dataTable.display tbody tr.even > .sorting_2, table.dataTable.order-column.stripe tbody tr.even > .sorting_2 {
-  background-color: #fbfbfb;
-}
-table.dataTable.display tbody tr.even > .sorting_3, table.dataTable.order-column.stripe tbody tr.even > .sorting_3 {
-  background-color: #fdfdfd;
-}
-table.dataTable.display tbody tr.even.selected > .sorting_1, table.dataTable.order-column.stripe tbody tr.even.selected > .sorting_1 {
-  background-color: #acbad4;
-}
-table.dataTable.display tbody tr.even.selected > .sorting_2, table.dataTable.order-column.stripe tbody tr.even.selected > .sorting_2 {
-  background-color: #adbbd6;
-}
-table.dataTable.display tbody tr.even.selected > .sorting_3, table.dataTable.order-column.stripe tbody tr.even.selected > .sorting_3 {
-  background-color: #afbdd8;
-}
-table.dataTable.display tbody tr:hover > .sorting_1,
-table.dataTable.display tbody tr.odd:hover > .sorting_1,
-table.dataTable.display tbody tr.even:hover > .sorting_1, table.dataTable.order-column.hover tbody tr:hover > .sorting_1,
-table.dataTable.order-column.hover tbody tr.odd:hover > .sorting_1,
-table.dataTable.order-column.hover tbody tr.even:hover > .sorting_1 {
-  background-color: #eaeaea;
-}
-table.dataTable.display tbody tr:hover > .sorting_2,
-table.dataTable.display tbody tr.odd:hover > .sorting_2,
-table.dataTable.display tbody tr.even:hover > .sorting_2, table.dataTable.order-column.hover tbody tr:hover > .sorting_2,
-table.dataTable.order-column.hover tbody tr.odd:hover > .sorting_2,
-table.dataTable.order-column.hover tbody tr.even:hover > .sorting_2 {
-  background-color: #ebebeb;
-}
-table.dataTable.display tbody tr:hover > .sorting_3,
-table.dataTable.display tbody tr.odd:hover > .sorting_3,
-table.dataTable.display tbody tr.even:hover > .sorting_3, table.dataTable.order-column.hover tbody tr:hover > .sorting_3,
-table.dataTable.order-column.hover tbody tr.odd:hover > .sorting_3,
-table.dataTable.order-column.hover tbody tr.even:hover > .sorting_3 {
-  background-color: #eeeeee;
-}
-table.dataTable.display tbody tr:hover.selected > .sorting_1,
-table.dataTable.display tbody tr.odd:hover.selected > .sorting_1,
-table.dataTable.display tbody tr.even:hover.selected > .sorting_1, table.dataTable.order-column.hover tbody tr:hover.selected > .sorting_1,
-table.dataTable.order-column.hover tbody tr.odd:hover.selected > .sorting_1,
-table.dataTable.order-column.hover tbody tr.even:hover.selected > .sorting_1 {
-  background-color: #a1aec7;
-}
-table.dataTable.display tbody tr:hover.selected > .sorting_2,
-table.dataTable.display tbody tr.odd:hover.selected > .sorting_2,
-table.dataTable.display tbody tr.even:hover.selected > .sorting_2, table.dataTable.order-column.hover tbody tr:hover.selected > .sorting_2,
-table.dataTable.order-column.hover tbody tr.odd:hover.selected > .sorting_2,
-table.dataTable.order-column.hover tbody tr.even:hover.selected > .sorting_2 {
-  background-color: #a2afc8;
-}
-table.dataTable.display tbody tr:hover.selected > .sorting_3,
-table.dataTable.display tbody tr.odd:hover.selected > .sorting_3,
-table.dataTable.display tbody tr.even:hover.selected > .sorting_3, table.dataTable.order-column.hover tbody tr:hover.selected > .sorting_3,
-table.dataTable.order-column.hover tbody tr.odd:hover.selected > .sorting_3,
-table.dataTable.order-column.hover tbody tr.even:hover.selected > .sorting_3 {
-  background-color: #a4b2cb;
-}
-table.dataTable.no-footer {
-  border-bottom: 1px solid #111111;
-}
-table.dataTable.nowrap th, table.dataTable.nowrap td {
-  white-space: nowrap;
-}
-table.dataTable.compact thead th,
-table.dataTable.compact thead td {
-  padding: 5px 9px;
-}
-table.dataTable.compact tfoot th,
-table.dataTable.compact tfoot td {
-  padding: 5px 9px 3px 9px;
-}
-table.dataTable.compact tbody th,
-table.dataTable.compact tbody td {
-  padding: 4px 5px;
-}
-table.dataTable th.dt-left,
-table.dataTable td.dt-left {
-  text-align: left;
-}
-table.dataTable th.dt-center,
-table.dataTable td.dt-center,
-table.dataTable td.dataTables_empty {
-  text-align: center;
-}
-table.dataTable th.dt-right,
-table.dataTable td.dt-right {
-  text-align: right;
-}
-table.dataTable th.dt-justify,
-table.dataTable td.dt-justify {
-  text-align: justify;
-}
-table.dataTable th.dt-nowrap,
-table.dataTable td.dt-nowrap {
-  white-space: nowrap;
-}
-table.dataTable thead th.dt-head-left,
-table.dataTable thead td.dt-head-left,
-table.dataTable tfoot th.dt-head-left,
-table.dataTable tfoot td.dt-head-left {
-  text-align: left;
-}
-table.dataTable thead th.dt-head-center,
-table.dataTable thead td.dt-head-center,
-table.dataTable tfoot th.dt-head-center,
-table.dataTable tfoot td.dt-head-center {
-  text-align: center;
-}
-table.dataTable thead th.dt-head-right,
-table.dataTable thead td.dt-head-right,
-table.dataTable tfoot th.dt-head-right,
-table.dataTable tfoot td.dt-head-right {
-  text-align: right;
-}
-table.dataTable thead th.dt-head-justify,
-table.dataTable thead td.dt-head-justify,
-table.dataTable tfoot th.dt-head-justify,
-table.dataTable tfoot td.dt-head-justify {
-  text-align: justify;
-}
-table.dataTable thead th.dt-head-nowrap,
-table.dataTable thead td.dt-head-nowrap,
-table.dataTable tfoot th.dt-head-nowrap,
-table.dataTable tfoot td.dt-head-nowrap {
-  white-space: nowrap;
-}
-table.dataTable tbody th.dt-body-left,
-table.dataTable tbody td.dt-body-left {
-  text-align: left;
-}
-table.dataTable tbody th.dt-body-center,
-table.dataTable tbody td.dt-body-center {
-  text-align: center;
-}
-table.dataTable tbody th.dt-body-right,
-table.dataTable tbody td.dt-body-right {
-  text-align: right;
-}
-table.dataTable tbody th.dt-body-justify,
-table.dataTable tbody td.dt-body-justify {
-  text-align: justify;
-}
-table.dataTable tbody th.dt-body-nowrap,
-table.dataTable tbody td.dt-body-nowrap {
-  white-space: nowrap;
-}
-
-table.dataTable,
-table.dataTable th,
-table.dataTable td {
-  -webkit-box-sizing: content-box;
-  -moz-box-sizing: content-box;
-  box-sizing: content-box;
-}
-
-/*
- * Control feature layout
- */
-.dataTables_wrapper {
-  position: relative;
-  clear: both;
-  *zoom: 1;
-  zoom: 1;
-}
-.dataTables_wrapper .dataTables_length {
-  float: left;
-}
-.dataTables_wrapper .dataTables_filter {
-  float: right;
-  text-align: right;
-}
-.dataTables_wrapper .dataTables_filter input {
-  margin-left: 0.5em;
-}
-.dataTables_wrapper .dataTables_info {
-  clear: both;
-  float: left;
-  padding-top: 0.755em;
-}
-.dataTables_wrapper .dataTables_paginate {
-  float: right;
-  text-align: right;
-  padding-top: 0.25em;
-}
-.dataTables_wrapper .dataTables_paginate .paginate_button {
-  box-sizing: border-box;
-  display: inline-block;
-  min-width: 1.5em;
-  padding: 0.5em 1em;
-  margin-left: 2px;
-  text-align: center;
-  text-decoration: none !important;
-  cursor: pointer;
-  *cursor: hand;
-  color: #333333 !important;
-  border: 1px solid transparent;
-}
-.dataTables_wrapper .dataTables_paginate .paginate_button.current, .dataTables_wrapper .dataTables_paginate .paginate_button.current:hover {
-  color: #333333 !important;
-  border: 1px solid #cacaca;
-  background-color: white;
-  background: -webkit-gradient(linear, left top, left bottom, color-stop(0%, white), color-stop(100%, gainsboro));
-  /* Chrome,Safari4+ */
-  background: -webkit-linear-gradient(top, white 0%, gainsboro 100%);
-  /* Chrome10+,Safari5.1+ */
-  background: -moz-linear-gradient(top, white 0%, gainsboro 100%);
-  /* FF3.6+ */
-  background: -ms-linear-gradient(top, white 0%, gainsboro 100%);
-  /* IE10+ */
-  background: -o-linear-gradient(top, white 0%, gainsboro 100%);
-  /* Opera 11.10+ */
-  background: linear-gradient(to bottom, white 0%, gainsboro 100%);
-  /* W3C */
-}
-.dataTables_wrapper .dataTables_paginate .paginate_button.disabled, .dataTables_wrapper .dataTables_paginate .paginate_button.disabled:hover, .dataTables_wrapper .dataTables_paginate .paginate_button.disabled:active {
-  cursor: default;
-  color: #666 !important;
-  border: 1px solid transparent;
-  background: transparent;
-  box-shadow: none;
-}
-.dataTables_wrapper .dataTables_paginate .paginate_button:hover {
-  color: white !important;
-  border: 1px solid #111111;
-  background-color: #585858;
-  background: -webkit-gradient(linear, left top, left bottom, color-stop(0%, #585858), color-stop(100%, #111111));
-  /* Chrome,Safari4+ */
-  background: -webkit-linear-gradient(top, #585858 0%, #111111 100%);
-  /* Chrome10+,Safari5.1+ */
-  background: -moz-linear-gradient(top, #585858 0%, #111111 100%);
-  /* FF3.6+ */
-  background: -ms-linear-gradient(top, #585858 0%, #111111 100%);
-  /* IE10+ */
-  background: -o-linear-gradient(top, #585858 0%, #111111 100%);
-  /* Opera 11.10+ */
-  background: linear-gradient(to bottom, #585858 0%, #111111 100%);
-  /* W3C */
-}
-.dataTables_wrapper .dataTables_paginate .paginate_button:active {
-  outline: none;
-  background-color: #2b2b2b;
-  background: -webkit-gradient(linear, left top, left bottom, color-stop(0%, #2b2b2b), color-stop(100%, #0c0c0c));
-  /* Chrome,Safari4+ */
-  background: -webkit-linear-gradient(top, #2b2b2b 0%, #0c0c0c 100%);
-  /* Chrome10+,Safari5.1+ */
-  background: -moz-linear-gradient(top, #2b2b2b 0%, #0c0c0c 100%);
-  /* FF3.6+ */
-  background: -ms-linear-gradient(top, #2b2b2b 0%, #0c0c0c 100%);
-  /* IE10+ */
-  background: -o-linear-gradient(top, #2b2b2b 0%, #0c0c0c 100%);
-  /* Opera 11.10+ */
-  background: linear-gradient(to bottom, #2b2b2b 0%, #0c0c0c 100%);
-  /* W3C */
-  box-shadow: inset 0 0 3px #111;
-}
-.dataTables_wrapper .dataTables_processing {
-  position: absolute;
-  top: 50%;
-  left: 50%;
-  width: 100%;
-  height: 40px;
-  margin-left: -50%;
-  margin-top: -25px;
-  padding-top: 20px;
-  text-align: center;
-  font-size: 1.2em;
-  background-color: white;
-  background: -webkit-gradient(linear, left top, right top, color-stop(0%, rgba(255, 255, 255, 0)), color-stop(25%, rgba(255, 255, 255, 0.9)), color-stop(75%, rgba(255, 255, 255, 0.9)), color-stop(100%, rgba(255, 255, 255, 0)));
-  /* Chrome,Safari4+ */
-  background: -webkit-linear-gradient(left, rgba(255, 255, 255, 0) 0%, rgba(255, 255, 255, 0.9) 25%, rgba(255, 255, 255, 0.9) 75%, rgba(255, 255, 255, 0) 100%);
-  /* Chrome10+,Safari5.1+ */
-  background: -moz-linear-gradient(left, rgba(255, 255, 255, 0) 0%, rgba(255, 255, 255, 0.9) 25%, rgba(255, 255, 255, 0.9) 75%, rgba(255, 255, 255, 0) 100%);
-  /* FF3.6+ */
-  background: -ms-linear-gradient(left, rgba(255, 255, 255, 0) 0%, rgba(255, 255, 255, 0.9) 25%, rgba(255, 255, 255, 0.9) 75%, rgba(255, 255, 255, 0) 100%);
-  /* IE10+ */
-  background: -o-linear-gradient(left, rgba(255, 255, 255, 0) 0%, rgba(255, 255, 255, 0.9) 25%, rgba(255, 255, 255, 0.9) 75%, rgba(255, 255, 255, 0) 100%);
-  /* Opera 11.10+ */
-  background: linear-gradient(to right, rgba(255, 255, 255, 0) 0%, rgba(255, 255, 255, 0.9) 25%, rgba(255, 255, 255, 0.9) 75%, rgba(255, 255, 255, 0) 100%);
-  /* W3C */
-}
-.dataTables_wrapper .dataTables_length,
-.dataTables_wrapper .dataTables_filter,
-.dataTables_wrapper .dataTables_info,
-.dataTables_wrapper .dataTables_processing,
-.dataTables_wrapper .dataTables_paginate {
-  color: #333333;
-}
-.dataTables_wrapper .dataTables_scroll {
-  clear: both;
-}
-.dataTables_wrapper .dataTables_scroll div.dataTables_scrollBody {
-  *margin-top: -1px;
-  -webkit-overflow-scrolling: touch;
-}
-.dataTables_wrapper .dataTables_scroll div.dataTables_scrollBody th > div.dataTables_sizing,
-.dataTables_wrapper .dataTables_scroll div.dataTables_scrollBody td > div.dataTables_sizing {
-  height: 0;
-  overflow: hidden;
-  margin: 0 !important;
-  padding: 0 !important;
-}
-.dataTables_wrapper.no-footer .dataTables_scrollBody {
-  border-bottom: 1px solid #111111;
-}
-.dataTables_wrapper.no-footer div.dataTables_scrollHead table,
-.dataTables_wrapper.no-footer div.dataTables_scrollBody table {
-  border-bottom: none;
-}
-.dataTables_wrapper:after {
-  visibility: hidden;
-  display: block;
-  content: "";
-  clear: both;
-  height: 0;
-}
-
-@media screen and (max-width: 767px) {
-  .dataTables_wrapper .dataTables_info,
-  .dataTables_wrapper .dataTables_paginate {
-    float: none;
-    text-align: center;
-  }
-  .dataTables_wrapper .dataTables_paginate {
-    margin-top: 0.5em;
-  }
-}
-@media screen and (max-width: 640px) {
-  .dataTables_wrapper .dataTables_length,
-  .dataTables_wrapper .dataTables_filter {
-    float: none;
-    text-align: center;
-  }
-  .dataTables_wrapper .dataTables_filter {
-    margin-top: 0.5em;
-  }
-}
diff --git a/lib/cylc/cylc-review/static/css/prettify.css b/lib/cylc/cylc-review/static/css/prettify.css
deleted file mode 100644
index e3c9497f23e..00000000000
--- a/lib/cylc/cylc-review/static/css/prettify.css
+++ /dev/null
@@ -1 +0,0 @@
-.pln{color:#000}@media screen{.str{color:#080}.kwd{color:#008}.com{color:#800}.typ{color:#606}.lit{color:#066}.pun,.opn,.clo{color:#660}.tag{color:#008}.atn{color:#606}.atv{color:#080}.dec,.var{color:#606}.fun{color:red}}@media print,projection{.str{color:#060}.kwd{color:#006;font-weight:bold}.com{color:#600;font-style:italic}.typ{color:#404;font-weight:bold}.lit{color:#044}.pun,.opn,.clo{color:#440}.tag{color:#006;font-weight:bold}.atn{color:#404}.atv{color:#060}}ol.linenums{margin-top:0;margin-bottom:0}li.L0,li.L1,li.L2,li.L3,li.L5,li.L6,li.L7,li.L8{list-style-type:none}li.L1,li.L3,li.L5,li.L7,li.L9{background:#eee}
diff --git a/lib/cylc/cylc-review/static/favicon.png b/lib/cylc/cylc-review/static/favicon.png
deleted file mode 100644
index 5a09687e42e..00000000000
Binary files a/lib/cylc/cylc-review/static/favicon.png and /dev/null differ
diff --git a/lib/cylc/cylc-review/static/fonts/glyphicons-halflings-regular.eot b/lib/cylc/cylc-review/static/fonts/glyphicons-halflings-regular.eot
deleted file mode 100644
index b93a4953fff..00000000000
Binary files a/lib/cylc/cylc-review/static/fonts/glyphicons-halflings-regular.eot and /dev/null differ
diff --git a/lib/cylc/cylc-review/static/fonts/glyphicons-halflings-regular.svg b/lib/cylc/cylc-review/static/fonts/glyphicons-halflings-regular.svg
deleted file mode 100644
index 94fb5490a2e..00000000000
--- a/lib/cylc/cylc-review/static/fonts/glyphicons-halflings-regular.svg
+++ /dev/null
@@ -1,288 +0,0 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- 
\ No newline at end of file
diff --git a/lib/cylc/cylc-review/static/fonts/glyphicons-halflings-regular.ttf b/lib/cylc/cylc-review/static/fonts/glyphicons-halflings-regular.ttf
deleted file mode 100644
index 1413fc609ab..00000000000
Binary files a/lib/cylc/cylc-review/static/fonts/glyphicons-halflings-regular.ttf and /dev/null differ
diff --git a/lib/cylc/cylc-review/static/fonts/glyphicons-halflings-regular.woff b/lib/cylc/cylc-review/static/fonts/glyphicons-halflings-regular.woff
deleted file mode 100644
index 9e612858f80..00000000000
Binary files a/lib/cylc/cylc-review/static/fonts/glyphicons-halflings-regular.woff and /dev/null differ
diff --git a/lib/cylc/cylc-review/static/fonts/glyphicons-halflings-regular.woff2 b/lib/cylc/cylc-review/static/fonts/glyphicons-halflings-regular.woff2
deleted file mode 100644
index 64539b54c37..00000000000
Binary files a/lib/cylc/cylc-review/static/fonts/glyphicons-halflings-regular.woff2 and /dev/null differ
diff --git a/lib/cylc/cylc-review/static/images/sort_asc.png b/lib/cylc/cylc-review/static/images/sort_asc.png
deleted file mode 100644
index a88d7975fe9..00000000000
Binary files a/lib/cylc/cylc-review/static/images/sort_asc.png and /dev/null differ
diff --git a/lib/cylc/cylc-review/static/images/sort_asc_disabled.png b/lib/cylc/cylc-review/static/images/sort_asc_disabled.png
deleted file mode 100644
index dcd7b7b8cab..00000000000
Binary files a/lib/cylc/cylc-review/static/images/sort_asc_disabled.png and /dev/null differ
diff --git a/lib/cylc/cylc-review/static/images/sort_both.png b/lib/cylc/cylc-review/static/images/sort_both.png
deleted file mode 100644
index 18670406bc0..00000000000
Binary files a/lib/cylc/cylc-review/static/images/sort_both.png and /dev/null differ
diff --git a/lib/cylc/cylc-review/static/images/sort_desc.png b/lib/cylc/cylc-review/static/images/sort_desc.png
deleted file mode 100644
index def071ed5af..00000000000
Binary files a/lib/cylc/cylc-review/static/images/sort_desc.png and /dev/null differ
diff --git a/lib/cylc/cylc-review/static/images/sort_desc_disabled.png b/lib/cylc/cylc-review/static/images/sort_desc_disabled.png
deleted file mode 100644
index 7824973cc60..00000000000
Binary files a/lib/cylc/cylc-review/static/images/sort_desc_disabled.png and /dev/null differ
diff --git a/lib/cylc/cylc-review/static/img b/lib/cylc/cylc-review/static/img
deleted file mode 120000
index 5e798fa0ef2..00000000000
--- a/lib/cylc/cylc-review/static/img
+++ /dev/null
@@ -1 +0,0 @@
-images
\ No newline at end of file
diff --git a/lib/cylc/cylc-review/static/js/bootstrap.min.js b/lib/cylc/cylc-review/static/js/bootstrap.min.js
deleted file mode 100644
index e79c065134f..00000000000
--- a/lib/cylc/cylc-review/static/js/bootstrap.min.js
+++ /dev/null
@@ -1,7 +0,0 @@
-/*!
- * Bootstrap v3.3.6 (http://getbootstrap.com)
- * Copyright 2011-2015 Twitter, Inc.
- * Licensed under the MIT license
- */
-if("undefined"==typeof jQuery)throw new Error("Bootstrap's JavaScript requires jQuery");+function(a){"use strict";var b=a.fn.jquery.split(" ")[0].split(".");if(b[0]<2&&b[1]<9||1==b[0]&&9==b[1]&&b[2]<1||b[0]>2)throw new Error("Bootstrap's JavaScript requires jQuery version 1.9.1 or higher, but lower than version 3")}(jQuery),+function(a){"use strict";function b(){var a=document.createElement("bootstrap"),b={WebkitTransition:"webkitTransitionEnd",MozTransition:"transitionend",OTransition:"oTransitionEnd otransitionend",transition:"transitionend"};for(var c in b)if(void 0!==a.style[c])return{end:b[c]};return!1}a.fn.emulateTransitionEnd=function(b){var c=!1,d=this;a(this).one("bsTransitionEnd",function(){c=!0});var e=function(){c||a(d).trigger(a.support.transition.end)};return setTimeout(e,b),this},a(function(){a.support.transition=b(),a.support.transition&&(a.event.special.bsTransitionEnd={bindType:a.support.transition.end,delegateType:a.support.transition.end,handle:function(b){return a(b.target).is(this)?b.handleObj.handler.apply(this,arguments):void 0}})})}(jQuery),+function(a){"use strict";function b(b){return this.each(function(){var c=a(this),e=c.data("bs.alert");e||c.data("bs.alert",e=new d(this)),"string"==typeof b&&e[b].call(c)})}var c='[data-dismiss="alert"]',d=function(b){a(b).on("click",c,this.close)};d.VERSION="3.3.6",d.TRANSITION_DURATION=150,d.prototype.close=function(b){function c(){g.detach().trigger("closed.bs.alert").remove()}var e=a(this),f=e.attr("data-target");f||(f=e.attr("href"),f=f&&f.replace(/.*(?=#[^\s]*$)/,""));var g=a(f);b&&b.preventDefault(),g.length||(g=e.closest(".alert")),g.trigger(b=a.Event("close.bs.alert")),b.isDefaultPrevented()||(g.removeClass("in"),a.support.transition&&g.hasClass("fade")?g.one("bsTransitionEnd",c).emulateTransitionEnd(d.TRANSITION_DURATION):c())};var e=a.fn.alert;a.fn.alert=b,a.fn.alert.Constructor=d,a.fn.alert.noConflict=function(){return a.fn.alert=e,this},a(document).on("click.bs.alert.data-api",c,d.prototype.close)}(jQuery),+function(a){"use strict";function b(b){return this.each(function(){var d=a(this),e=d.data("bs.button"),f="object"==typeof b&&b;e||d.data("bs.button",e=new c(this,f)),"toggle"==b?e.toggle():b&&e.setState(b)})}var c=function(b,d){this.$element=a(b),this.options=a.extend({},c.DEFAULTS,d),this.isLoading=!1};c.VERSION="3.3.6",c.DEFAULTS={loadingText:"loading..."},c.prototype.setState=function(b){var c="disabled",d=this.$element,e=d.is("input")?"val":"html",f=d.data();b+="Text",null==f.resetText&&d.data("resetText",d[e]()),setTimeout(a.proxy(function(){d[e](null==f[b]?this.options[b]:f[b]),"loadingText"==b?(this.isLoading=!0,d.addClass(c).attr(c,c)):this.isLoading&&(this.isLoading=!1,d.removeClass(c).removeAttr(c))},this),0)},c.prototype.toggle=function(){var a=!0,b=this.$element.closest('[data-toggle="buttons"]');if(b.length){var c=this.$element.find("input");"radio"==c.prop("type")?(c.prop("checked")&&(a=!1),b.find(".active").removeClass("active"),this.$element.addClass("active")):"checkbox"==c.prop("type")&&(c.prop("checked")!==this.$element.hasClass("active")&&(a=!1),this.$element.toggleClass("active")),c.prop("checked",this.$element.hasClass("active")),a&&c.trigger("change")}else this.$element.attr("aria-pressed",!this.$element.hasClass("active")),this.$element.toggleClass("active")};var d=a.fn.button;a.fn.button=b,a.fn.button.Constructor=c,a.fn.button.noConflict=function(){return a.fn.button=d,this},a(document).on("click.bs.button.data-api",'[data-toggle^="button"]',function(c){var d=a(c.target);d.hasClass("btn")||(d=d.closest(".btn")),b.call(d,"toggle"),a(c.target).is('input[type="radio"]')||a(c.target).is('input[type="checkbox"]')||c.preventDefault()}).on("focus.bs.button.data-api blur.bs.button.data-api",'[data-toggle^="button"]',function(b){a(b.target).closest(".btn").toggleClass("focus",/^focus(in)?$/.test(b.type))})}(jQuery),+function(a){"use strict";function b(b){return this.each(function(){var d=a(this),e=d.data("bs.carousel"),f=a.extend({},c.DEFAULTS,d.data(),"object"==typeof b&&b),g="string"==typeof b?b:f.slide;e||d.data("bs.carousel",e=new c(this,f)),"number"==typeof b?e.to(b):g?e[g]():f.interval&&e.pause().cycle()})}var c=function(b,c){this.$element=a(b),this.$indicators=this.$element.find(".carousel-indicators"),this.options=c,this.paused=null,this.sliding=null,this.interval=null,this.$active=null,this.$items=null,this.options.keyboard&&this.$element.on("keydown.bs.carousel",a.proxy(this.keydown,this)),"hover"==this.options.pause&&!("ontouchstart"in document.documentElement)&&this.$element.on("mouseenter.bs.carousel",a.proxy(this.pause,this)).on("mouseleave.bs.carousel",a.proxy(this.cycle,this))};c.VERSION="3.3.6",c.TRANSITION_DURATION=600,c.DEFAULTS={interval:5e3,pause:"hover",wrap:!0,keyboard:!0},c.prototype.keydown=function(a){if(!/input|textarea/i.test(a.target.tagName)){switch(a.which){case 37:this.prev();break;case 39:this.next();break;default:return}a.preventDefault()}},c.prototype.cycle=function(b){return b||(this.paused=!1),this.interval&&clearInterval(this.interval),this.options.interval&&!this.paused&&(this.interval=setInterval(a.proxy(this.next,this),this.options.interval)),this},c.prototype.getItemIndex=function(a){return this.$items=a.parent().children(".item"),this.$items.index(a||this.$active)},c.prototype.getItemForDirection=function(a,b){var c=this.getItemIndex(b),d="prev"==a&&0===c||"next"==a&&c==this.$items.length-1;if(d&&!this.options.wrap)return b;var e="prev"==a?-1:1,f=(c+e)%this.$items.length;return this.$items.eq(f)},c.prototype.to=function(a){var b=this,c=this.getItemIndex(this.$active=this.$element.find(".item.active"));return a>this.$items.length-1||0>a?void 0:this.sliding?this.$element.one("slid.bs.carousel",function(){b.to(a)}):c==a?this.pause().cycle():this.slide(a>c?"next":"prev",this.$items.eq(a))},c.prototype.pause=function(b){return b||(this.paused=!0),this.$element.find(".next, .prev").length&&a.support.transition&&(this.$element.trigger(a.support.transition.end),this.cycle(!0)),this.interval=clearInterval(this.interval),this},c.prototype.next=function(){return this.sliding?void 0:this.slide("next")},c.prototype.prev=function(){return this.sliding?void 0:this.slide("prev")},c.prototype.slide=function(b,d){var e=this.$element.find(".item.active"),f=d||this.getItemForDirection(b,e),g=this.interval,h="next"==b?"left":"right",i=this;if(f.hasClass("active"))return this.sliding=!1;var j=f[0],k=a.Event("slide.bs.carousel",{relatedTarget:j,direction:h});if(this.$element.trigger(k),!k.isDefaultPrevented()){if(this.sliding=!0,g&&this.pause(),this.$indicators.length){this.$indicators.find(".active").removeClass("active");var l=a(this.$indicators.children()[this.getItemIndex(f)]);l&&l.addClass("active")}var m=a.Event("slid.bs.carousel",{relatedTarget:j,direction:h});return a.support.transition&&this.$element.hasClass("slide")?(f.addClass(b),f[0].offsetWidth,e.addClass(h),f.addClass(h),e.one("bsTransitionEnd",function(){f.removeClass([b,h].join(" ")).addClass("active"),e.removeClass(["active",h].join(" ")),i.sliding=!1,setTimeout(function(){i.$element.trigger(m)},0)}).emulateTransitionEnd(c.TRANSITION_DURATION)):(e.removeClass("active"),f.addClass("active"),this.sliding=!1,this.$element.trigger(m)),g&&this.cycle(),this}};var d=a.fn.carousel;a.fn.carousel=b,a.fn.carousel.Constructor=c,a.fn.carousel.noConflict=function(){return a.fn.carousel=d,this};var e=function(c){var d,e=a(this),f=a(e.attr("data-target")||(d=e.attr("href"))&&d.replace(/.*(?=#[^\s]+$)/,""));if(f.hasClass("carousel")){var g=a.extend({},f.data(),e.data()),h=e.attr("data-slide-to");h&&(g.interval=!1),b.call(f,g),h&&f.data("bs.carousel").to(h),c.preventDefault()}};a(document).on("click.bs.carousel.data-api","[data-slide]",e).on("click.bs.carousel.data-api","[data-slide-to]",e),a(window).on("load",function(){a('[data-ride="carousel"]').each(function(){var c=a(this);b.call(c,c.data())})})}(jQuery),+function(a){"use strict";function b(b){var c,d=b.attr("data-target")||(c=b.attr("href"))&&c.replace(/.*(?=#[^\s]+$)/,"");return a(d)}function c(b){return this.each(function(){var c=a(this),e=c.data("bs.collapse"),f=a.extend({},d.DEFAULTS,c.data(),"object"==typeof b&&b);!e&&f.toggle&&/show|hide/.test(b)&&(f.toggle=!1),e||c.data("bs.collapse",e=new d(this,f)),"string"==typeof b&&e[b]()})}var d=function(b,c){this.$element=a(b),this.options=a.extend({},d.DEFAULTS,c),this.$trigger=a('[data-toggle="collapse"][href="#'+b.id+'"],[data-toggle="collapse"][data-target="#'+b.id+'"]'),this.transitioning=null,this.options.parent?this.$parent=this.getParent():this.addAriaAndCollapsedClass(this.$element,this.$trigger),this.options.toggle&&this.toggle()};d.VERSION="3.3.6",d.TRANSITION_DURATION=350,d.DEFAULTS={toggle:!0},d.prototype.dimension=function(){var a=this.$element.hasClass("width");return a?"width":"height"},d.prototype.show=function(){if(!this.transitioning&&!this.$element.hasClass("in")){var b,e=this.$parent&&this.$parent.children(".panel").children(".in, .collapsing");if(!(e&&e.length&&(b=e.data("bs.collapse"),b&&b.transitioning))){var f=a.Event("show.bs.collapse");if(this.$element.trigger(f),!f.isDefaultPrevented()){e&&e.length&&(c.call(e,"hide"),b||e.data("bs.collapse",null));var g=this.dimension();this.$element.removeClass("collapse").addClass("collapsing")[g](0).attr("aria-expanded",!0),this.$trigger.removeClass("collapsed").attr("aria-expanded",!0),this.transitioning=1;var h=function(){this.$element.removeClass("collapsing").addClass("collapse in")[g](""),this.transitioning=0,this.$element.trigger("shown.bs.collapse")};if(!a.support.transition)return h.call(this);var i=a.camelCase(["scroll",g].join("-"));this.$element.one("bsTransitionEnd",a.proxy(h,this)).emulateTransitionEnd(d.TRANSITION_DURATION)[g](this.$element[0][i])}}}},d.prototype.hide=function(){if(!this.transitioning&&this.$element.hasClass("in")){var b=a.Event("hide.bs.collapse");if(this.$element.trigger(b),!b.isDefaultPrevented()){var c=this.dimension();this.$element[c](this.$element[c]())[0].offsetHeight,this.$element.addClass("collapsing").removeClass("collapse in").attr("aria-expanded",!1),this.$trigger.addClass("collapsed").attr("aria-expanded",!1),this.transitioning=1;var e=function(){this.transitioning=0,this.$element.removeClass("collapsing").addClass("collapse").trigger("hidden.bs.collapse")};return a.support.transition?void this.$element[c](0).one("bsTransitionEnd",a.proxy(e,this)).emulateTransitionEnd(d.TRANSITION_DURATION):e.call(this)}}},d.prototype.toggle=function(){this[this.$element.hasClass("in")?"hide":"show"]()},d.prototype.getParent=function(){return a(this.options.parent).find('[data-toggle="collapse"][data-parent="'+this.options.parent+'"]').each(a.proxy(function(c,d){var e=a(d);this.addAriaAndCollapsedClass(b(e),e)},this)).end()},d.prototype.addAriaAndCollapsedClass=function(a,b){var c=a.hasClass("in");a.attr("aria-expanded",c),b.toggleClass("collapsed",!c).attr("aria-expanded",c)};var e=a.fn.collapse;a.fn.collapse=c,a.fn.collapse.Constructor=d,a.fn.collapse.noConflict=function(){return a.fn.collapse=e,this},a(document).on("click.bs.collapse.data-api",'[data-toggle="collapse"]',function(d){var e=a(this);e.attr("data-target")||d.preventDefault();var f=b(e),g=f.data("bs.collapse"),h=g?"toggle":e.data();c.call(f,h)})}(jQuery),+function(a){"use strict";function b(b){var c=b.attr("data-target");c||(c=b.attr("href"),c=c&&/#[A-Za-z]/.test(c)&&c.replace(/.*(?=#[^\s]*$)/,""));var d=c&&a(c);return d&&d.length?d:b.parent()}function c(c){c&&3===c.which||(a(e).remove(),a(f).each(function(){var d=a(this),e=b(d),f={relatedTarget:this};e.hasClass("open")&&(c&&"click"==c.type&&/input|textarea/i.test(c.target.tagName)&&a.contains(e[0],c.target)||(e.trigger(c=a.Event("hide.bs.dropdown",f)),c.isDefaultPrevented()||(d.attr("aria-expanded","false"),e.removeClass("open").trigger(a.Event("hidden.bs.dropdown",f)))))}))}function d(b){return this.each(function(){var c=a(this),d=c.data("bs.dropdown");d||c.data("bs.dropdown",d=new g(this)),"string"==typeof b&&d[b].call(c)})}var e=".dropdown-backdrop",f='[data-toggle="dropdown"]',g=function(b){a(b).on("click.bs.dropdown",this.toggle)};g.VERSION="3.3.6",g.prototype.toggle=function(d){var e=a(this);if(!e.is(".disabled, :disabled")){var f=b(e),g=f.hasClass("open");if(c(),!g){"ontouchstart"in document.documentElement&&!f.closest(".navbar-nav").length&&a(document.createElement("div")).addClass("dropdown-backdrop").insertAfter(a(this)).on("click",c);var h={relatedTarget:this};if(f.trigger(d=a.Event("show.bs.dropdown",h)),d.isDefaultPrevented())return;e.trigger("focus").attr("aria-expanded","true"),f.toggleClass("open").trigger(a.Event("shown.bs.dropdown",h))}return!1}},g.prototype.keydown=function(c){if(/(38|40|27|32)/.test(c.which)&&!/input|textarea/i.test(c.target.tagName)){var d=a(this);if(c.preventDefault(),c.stopPropagation(),!d.is(".disabled, :disabled")){var e=b(d),g=e.hasClass("open");if(!g&&27!=c.which||g&&27==c.which)return 27==c.which&&e.find(f).trigger("focus"),d.trigger("click");var h=" li:not(.disabled):visible a",i=e.find(".dropdown-menu"+h);if(i.length){var j=i.index(c.target);38==c.which&&j>0&&j--,40==c.which&&jdocument.documentElement.clientHeight;this.$element.css({paddingLeft:!this.bodyIsOverflowing&&a?this.scrollbarWidth:"",paddingRight:this.bodyIsOverflowing&&!a?this.scrollbarWidth:""})},c.prototype.resetAdjustments=function(){this.$element.css({paddingLeft:"",paddingRight:""})},c.prototype.checkScrollbar=function(){var a=window.innerWidth;if(!a){var b=document.documentElement.getBoundingClientRect();a=b.right-Math.abs(b.left)}this.bodyIsOverflowing=document.body.clientWidth
',trigger:"hover focus",title:"",delay:0,html:!1,container:!1,viewport:{selector:"body",padding:0}},c.prototype.init=function(b,c,d){if(this.enabled=!0,this.type=b,this.$element=a(c),this.options=this.getOptions(d),this.$viewport=this.options.viewport&&a(a.isFunction(this.options.viewport)?this.options.viewport.call(this,this.$element):this.options.viewport.selector||this.options.viewport),this.inState={click:!1,hover:!1,focus:!1},this.$element[0]instanceof document.constructor&&!this.options.selector)throw new Error("`selector` option must be specified when initializing "+this.type+" on the window.document object!");for(var e=this.options.trigger.split(" "),f=e.length;f--;){var g=e[f];if("click"==g)this.$element.on("click."+this.type,this.options.selector,a.proxy(this.toggle,this));else if("manual"!=g){var h="hover"==g?"mouseenter":"focusin",i="hover"==g?"mouseleave":"focusout";this.$element.on(h+"."+this.type,this.options.selector,a.proxy(this.enter,this)),this.$element.on(i+"."+this.type,this.options.selector,a.proxy(this.leave,this))}}this.options.selector?this._options=a.extend({},this.options,{trigger:"manual",selector:""}):this.fixTitle()},c.prototype.getDefaults=function(){return c.DEFAULTS},c.prototype.getOptions=function(b){return b=a.extend({},this.getDefaults(),this.$element.data(),b),b.delay&&"number"==typeof b.delay&&(b.delay={show:b.delay,hide:b.delay}),b},c.prototype.getDelegateOptions=function(){var b={},c=this.getDefaults();return this._options&&a.each(this._options,function(a,d){c[a]!=d&&(b[a]=d)}),b},c.prototype.enter=function(b){var c=b instanceof this.constructor?b:a(b.currentTarget).data("bs."+this.type);return c||(c=new this.constructor(b.currentTarget,this.getDelegateOptions()),a(b.currentTarget).data("bs."+this.type,c)),b instanceof a.Event&&(c.inState["focusin"==b.type?"focus":"hover"]=!0),c.tip().hasClass("in")||"in"==c.hoverState?void(c.hoverState="in"):(clearTimeout(c.timeout),c.hoverState="in",c.options.delay&&c.options.delay.show?void(c.timeout=setTimeout(function(){"in"==c.hoverState&&c.show()},c.options.delay.show)):c.show())},c.prototype.isInStateTrue=function(){for(var a in this.inState)if(this.inState[a])return!0;return!1},c.prototype.leave=function(b){var c=b instanceof this.constructor?b:a(b.currentTarget).data("bs."+this.type);return c||(c=new this.constructor(b.currentTarget,this.getDelegateOptions()),a(b.currentTarget).data("bs."+this.type,c)),b instanceof a.Event&&(c.inState["focusout"==b.type?"focus":"hover"]=!1),c.isInStateTrue()?void 0:(clearTimeout(c.timeout),c.hoverState="out",c.options.delay&&c.options.delay.hide?void(c.timeout=setTimeout(function(){"out"==c.hoverState&&c.hide()},c.options.delay.hide)):c.hide())},c.prototype.show=function(){var b=a.Event("show.bs."+this.type);if(this.hasContent()&&this.enabled){this.$element.trigger(b);var d=a.contains(this.$element[0].ownerDocument.documentElement,this.$element[0]);if(b.isDefaultPrevented()||!d)return;var e=this,f=this.tip(),g=this.getUID(this.type);this.setContent(),f.attr("id",g),this.$element.attr("aria-describedby",g),this.options.animation&&f.addClass("fade");var h="function"==typeof this.options.placement?this.options.placement.call(this,f[0],this.$element[0]):this.options.placement,i=/\s?auto?\s?/i,j=i.test(h);j&&(h=h.replace(i,"")||"top"),f.detach().css({top:0,left:0,display:"block"}).addClass(h).data("bs."+this.type,this),this.options.container?f.appendTo(this.options.container):f.insertAfter(this.$element),this.$element.trigger("inserted.bs."+this.type);var k=this.getPosition(),l=f[0].offsetWidth,m=f[0].offsetHeight;if(j){var n=h,o=this.getPosition(this.$viewport);h="bottom"==h&&k.bottom+m>o.bottom?"top":"top"==h&&k.top-mo.width?"left":"left"==h&&k.left-lg.top+g.height&&(e.top=g.top+g.height-i)}else{var j=b.left-f,k=b.left+f+c;jg.right&&(e.left=g.left+g.width-k)}return e},c.prototype.getTitle=function(){var a,b=this.$element,c=this.options;return a=b.attr("data-original-title")||("function"==typeof c.title?c.title.call(b[0]):c.title)},c.prototype.getUID=function(a){do a+=~~(1e6*Math.random());while(document.getElementById(a));return a},c.prototype.tip=function(){if(!this.$tip&&(this.$tip=a(this.options.template),1!=this.$tip.length))throw new Error(this.type+" `template` option must consist of exactly 1 top-level element!");return this.$tip},c.prototype.arrow=function(){return this.$arrow=this.$arrow||this.tip().find(".tooltip-arrow")},c.prototype.enable=function(){this.enabled=!0},c.prototype.disable=function(){this.enabled=!1},c.prototype.toggleEnabled=function(){this.enabled=!this.enabled},c.prototype.toggle=function(b){var c=this;b&&(c=a(b.currentTarget).data("bs."+this.type),c||(c=new this.constructor(b.currentTarget,this.getDelegateOptions()),a(b.currentTarget).data("bs."+this.type,c))),b?(c.inState.click=!c.inState.click,c.isInStateTrue()?c.enter(c):c.leave(c)):c.tip().hasClass("in")?c.leave(c):c.enter(c)},c.prototype.destroy=function(){var a=this;clearTimeout(this.timeout),this.hide(function(){a.$element.off("."+a.type).removeData("bs."+a.type),a.$tip&&a.$tip.detach(),a.$tip=null,a.$arrow=null,a.$viewport=null})};var d=a.fn.tooltip;a.fn.tooltip=b,a.fn.tooltip.Constructor=c,a.fn.tooltip.noConflict=function(){return a.fn.tooltip=d,this}}(jQuery),+function(a){"use strict";function b(b){return this.each(function(){var d=a(this),e=d.data("bs.popover"),f="object"==typeof b&&b;(e||!/destroy|hide/.test(b))&&(e||d.data("bs.popover",e=new c(this,f)),"string"==typeof b&&e[b]())})}var c=function(a,b){this.init("popover",a,b)};if(!a.fn.tooltip)throw new Error("Popover requires tooltip.js");c.VERSION="3.3.6",c.DEFAULTS=a.extend({},a.fn.tooltip.Constructor.DEFAULTS,{placement:"right",trigger:"click",content:"",template:''}),c.prototype=a.extend({},a.fn.tooltip.Constructor.prototype),c.prototype.constructor=c,c.prototype.getDefaults=function(){return c.DEFAULTS},c.prototype.setContent=function(){var a=this.tip(),b=this.getTitle(),c=this.getContent();a.find(".popover-title")[this.options.html?"html":"text"](b),a.find(".popover-content").children().detach().end()[this.options.html?"string"==typeof c?"html":"append":"text"](c),a.removeClass("fade top bottom left right in"),a.find(".popover-title").html()||a.find(".popover-title").hide()},c.prototype.hasContent=function(){return this.getTitle()||this.getContent()},c.prototype.getContent=function(){var a=this.$element,b=this.options;return a.attr("data-content")||("function"==typeof b.content?b.content.call(a[0]):b.content)},c.prototype.arrow=function(){return this.$arrow=this.$arrow||this.tip().find(".arrow")};var d=a.fn.popover;a.fn.popover=b,a.fn.popover.Constructor=c,a.fn.popover.noConflict=function(){return a.fn.popover=d,this}}(jQuery),+function(a){"use strict";function b(c,d){this.$body=a(document.body),this.$scrollElement=a(a(c).is(document.body)?window:c),this.options=a.extend({},b.DEFAULTS,d),this.selector=(this.options.target||"")+" .nav li > a",this.offsets=[],this.targets=[],this.activeTarget=null,this.scrollHeight=0,this.$scrollElement.on("scroll.bs.scrollspy",a.proxy(this.process,this)),this.refresh(),this.process()}function c(c){return this.each(function(){var d=a(this),e=d.data("bs.scrollspy"),f="object"==typeof c&&c;e||d.data("bs.scrollspy",e=new b(this,f)),"string"==typeof c&&e[c]()})}b.VERSION="3.3.6",b.DEFAULTS={offset:10},b.prototype.getScrollHeight=function(){return this.$scrollElement[0].scrollHeight||Math.max(this.$body[0].scrollHeight,document.documentElement.scrollHeight)},b.prototype.refresh=function(){var b=this,c="offset",d=0;this.offsets=[],this.targets=[],this.scrollHeight=this.getScrollHeight(),a.isWindow(this.$scrollElement[0])||(c="position",d=this.$scrollElement.scrollTop()),this.$body.find(this.selector).map(function(){var b=a(this),e=b.data("target")||b.attr("href"),f=/^#./.test(e)&&a(e);return f&&f.length&&f.is(":visible")&&[[f[c]().top+d,e]]||null}).sort(function(a,b){return a[0]-b[0]}).each(function(){b.offsets.push(this[0]),b.targets.push(this[1])})},b.prototype.process=function(){var a,b=this.$scrollElement.scrollTop()+this.options.offset,c=this.getScrollHeight(),d=this.options.offset+c-this.$scrollElement.height(),e=this.offsets,f=this.targets,g=this.activeTarget;if(this.scrollHeight!=c&&this.refresh(),b>=d)return g!=(a=f[f.length-1])&&this.activate(a);if(g&&b=e[a]&&(void 0===e[a+1]||b .dropdown-menu > .active").removeClass("active").end().find('[data-toggle="tab"]').attr("aria-expanded",!1),b.addClass("active").find('[data-toggle="tab"]').attr("aria-expanded",!0),h?(b[0].offsetWidth,b.addClass("in")):b.removeClass("fade"),b.parent(".dropdown-menu").length&&b.closest("li.dropdown").addClass("active").end().find('[data-toggle="tab"]').attr("aria-expanded",!0),e&&e()}var g=d.find("> .active"),h=e&&a.support.transition&&(g.length&&g.hasClass("fade")||!!d.find("> .fade").length);g.length&&h?g.one("bsTransitionEnd",f).emulateTransitionEnd(c.TRANSITION_DURATION):f(),g.removeClass("in")};var d=a.fn.tab;a.fn.tab=b,a.fn.tab.Constructor=c,a.fn.tab.noConflict=function(){return a.fn.tab=d,this};var e=function(c){c.preventDefault(),b.call(a(this),"show")};a(document).on("click.bs.tab.data-api",'[data-toggle="tab"]',e).on("click.bs.tab.data-api",'[data-toggle="pill"]',e)}(jQuery),+function(a){"use strict";function b(b){return this.each(function(){var d=a(this),e=d.data("bs.affix"),f="object"==typeof b&&b;e||d.data("bs.affix",e=new c(this,f)),"string"==typeof b&&e[b]()})}var c=function(b,d){this.options=a.extend({},c.DEFAULTS,d),this.$target=a(this.options.target).on("scroll.bs.affix.data-api",a.proxy(this.checkPosition,this)).on("click.bs.affix.data-api",a.proxy(this.checkPositionWithEventLoop,this)),this.$element=a(b),this.affixed=null,this.unpin=null,this.pinnedOffset=null,this.checkPosition()};c.VERSION="3.3.6",c.RESET="affix affix-top affix-bottom",c.DEFAULTS={offset:0,target:window},c.prototype.getState=function(a,b,c,d){var e=this.$target.scrollTop(),f=this.$element.offset(),g=this.$target.height();if(null!=c&&"top"==this.affixed)return c>e?"top":!1;if("bottom"==this.affixed)return null!=c?e+this.unpin<=f.top?!1:"bottom":a-d>=e+g?!1:"bottom";var h=null==this.affixed,i=h?e:f.top,j=h?g:b;return null!=c&&c>=e?"top":null!=d&&i+j>=a-d?"bottom":!1},c.prototype.getPinnedOffset=function(){if(this.pinnedOffset)return this.pinnedOffset;this.$element.removeClass(c.RESET).addClass("affix");var a=this.$target.scrollTop(),b=this.$element.offset();return this.pinnedOffset=b.top-a},c.prototype.checkPositionWithEventLoop=function(){setTimeout(a.proxy(this.checkPosition,this),1)},c.prototype.checkPosition=function(){if(this.$element.is(":visible")){var b=this.$element.height(),d=this.options.offset,e=d.top,f=d.bottom,g=Math.max(a(document).height(),a(document.body).height());"object"!=typeof d&&(f=e=d),"function"==typeof e&&(e=d.top(this.$element)),"function"==typeof f&&(f=d.bottom(this.$element));var h=this.getState(g,b,e,f);if(this.affixed!=h){null!=this.unpin&&this.$element.css("top","");var i="affix"+(h?"-"+h:""),j=a.Event(i+".bs.affix");if(this.$element.trigger(j),j.isDefaultPrevented())return;this.affixed=h,this.unpin="bottom"==h?this.getPinnedOffset():null,this.$element.removeClass(c.RESET).addClass(i).trigger(i.replace("affix","affixed")+".bs.affix")}"bottom"==h&&this.$element.offset({top:g-b-f})}};var d=a.fn.affix;a.fn.affix=b,a.fn.affix.Constructor=c,a.fn.affix.noConflict=function(){return a.fn.affix=d,this},a(window).on("load",function(){a('[data-spy="affix"]').each(function(){var c=a(this),d=c.data();d.offset=d.offset||{},null!=d.offsetBottom&&(d.offset.bottom=d.offsetBottom),null!=d.offsetTop&&(d.offset.top=d.offsetTop),b.call(c,d)})})}(jQuery);
\ No newline at end of file
diff --git a/lib/cylc/cylc-review/static/js/cylc-review.js b/lib/cylc/cylc-review/static/js/cylc-review.js
deleted file mode 100644
index 9c743f90f21..00000000000
--- a/lib/cylc/cylc-review/static/js/cylc-review.js
+++ /dev/null
@@ -1,143 +0,0 @@
-/******************************************************************************
- * THIS FILE IS PART OF THE CYLC SUITE ENGINE.
- * Copyright (C) 2008-2019 NIWA & British Crown (Met Office) & Contributors.
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program.  If not, see .
- *
- ******************************************************************************/
-$(function() {
-    $(".collapse").collapse();
-    $(".livestamp").each(function() {
-        $(this).livestamp($(this).attr("title"));
-    });
-    $(".entry select.seq_log").change(function() {
-        this.form.submit();
-    });
-    var dt_as_m_and_s = function(dt) {
-        var s = dt.seconds();
-        if (s < 10) {
-            s = "0" + s.toString();
-        }
-        else {
-            s = s.toString();
-        }
-        var m = Math.floor(dt.asMinutes());
-        return m.toString() + ":" + s;
-    }
-
-    var toggle_fuzzy_time_init = true;
-    var toggle_fuzzy_time = function() {
-        var state = $("#toggle-fuzzy-time").attr("data-no-fuzzy-time");
-        if (toggle_fuzzy_time_init) {
-            state = (state == "0") ? "1" : "0";
-        }
-        toggle_fuzzy_time_init = false;
-        if (state == "1") {
-            $("form input[name='no_fuzzy_time']").attr("value", "0");
-            $("#toggle-fuzzy-time").attr("data-no-fuzzy-time", "0");
-            $(".livestamp").each(function() {
-                $(this).livestamp($(this).attr("title"));
-            });
-            $(".job-entry-head-init-time").html("queue \Δt");
-            $(".job-entry-head-exit-time").html("run \Δt");
-            $(".entry").each(function() {
-                var s_init = $(".t_init", this).attr("title");
-                if (s_init == null) {
-                    return;
-                }
-                var s_submit = $(".t_submit", this).attr("title");
-                var m_submit = moment(s_submit);
-                var m_init = moment(s_init);
-                if (m_init.isBefore(m_submit)) {
-                    m_init = m_submit;
-                }
-                var dt_q = moment.duration(m_init.diff(m_submit));
-                $(".t_init", this).text(dt_as_m_and_s(dt_q));
-                var s_exit = $(".t_exit", this).attr("title");
-                if (s_exit == null) {
-                    return;
-                }
-                var m_exit = moment(s_exit);
-                if (m_exit.isBefore(m_init)) {
-                    m_exit = m_init;
-                }
-                var dt_r = moment.duration(m_exit.diff(m_init));
-                $(".t_exit", this).text(dt_as_m_and_s(dt_r));
-            });
-            $("#toggle-fuzzy-time").addClass("active");
-        }
-        else {
-            $("form input[name='no_fuzzy_time']").attr("value", "1");
-            $("#toggle-fuzzy-time").attr("data-no-fuzzy-time", "1");
-            $(".livestamp").each(function() {
-                $(this).livestamp("destroy");
-            });
-            $(".job-entry-head-init-time").text("start time");
-            $(".job-entry-head-exit-time").text("exit time");
-            $(".entry").each(function() {
-                var s_init = $(".t_init", this).attr("title");
-                if (s_init == null) {
-                    return;
-                }
-                var m_submit = moment($(".t_submit", this).attr("title"));
-                var m_init = moment(s_init);
-                if (m_init.isSame(m_submit, "day")) {
-                    m_init.utc();
-                    s_init = m_init.format("HH:mm:ss");
-                }
-                $(".t_init", this).text(s_init);
-                var s_exit = $(".t_exit", this).attr("title");
-                if (s_exit == null) {
-                    return;
-                }
-                var m_exit = moment(s_exit);
-                if (m_exit.isSame(m_submit, "day")) {
-                    m_exit.utc();
-                    s_exit = m_exit.format("HH:mm:ss");
-                }
-                $(".t_exit", this).text(s_exit);
-            });
-            $("#toggle-fuzzy-time").removeClass("active");
-        }
-    }
-    toggle_fuzzy_time();
-    $("#toggle-fuzzy-time").click(toggle_fuzzy_time);
-    $("#page").change(function() {
-        this.form.submit();
-    });
-    $("#page-next").click(function() {
-        $("#page").prop("selectedIndex", $("#page").prop("selectedIndex") + 1);
-    });
-    $("#page-prev").click(function() {
-        $("#page").prop("selectedIndex", $("#page").prop("selectedIndex") - 1);
-    });
-    $("#per_page_max").click(function() {
-        $("#per_page").prop("disabled", $(this).prop("checked"));
-    });
-    $("#uncheck_task_statuses").click(function() {
-        $("input.task_status").prop("checked", false);
-    });
-    $("#all_task_statuses").change(function() {
-        if ($(this).prop("checked")) {
-            $("#uncheck_task_statuses").prop("disabled", true);
-            $("input.task_status").prop("checked", true);
-            $("input.task_status").prop("disabled", true);
-        }
-        else {
-            $("#uncheck_task_statuses").prop("disabled", false);
-            $("input.task_status").prop("disabled", false);
-        }
-    });
-    $("#all_task_statuses").change();
-});
diff --git a/lib/cylc/cylc-review/static/js/dataTables.bootstrap.js b/lib/cylc/cylc-review/static/js/dataTables.bootstrap.js
deleted file mode 100644
index 2c6959d1825..00000000000
--- a/lib/cylc/cylc-review/static/js/dataTables.bootstrap.js
+++ /dev/null
@@ -1,186 +0,0 @@
-/*! DataTables Bootstrap 3 integration
- * ©2011-2014 SpryMedia Ltd - datatables.net/license
- */
-
-/**
- * DataTables integration for Bootstrap 3. This requires Bootstrap 3 and
- * DataTables 1.10 or newer.
- *
- * This file sets the defaults and adds options to DataTables to style its
- * controls using Bootstrap. See http://datatables.net/manual/styling/bootstrap
- * for further information.
- */
-(function(window, document, undefined){
-
-var factory = function( $, DataTable ) {
-"use strict";
-
-
-/* Set the defaults for DataTables initialisation */
-$.extend( true, DataTable.defaults, {
-	dom:
-		"<'row'<'col-sm-6'l><'col-sm-6'f>>" +
-		"<'row'<'col-sm-12'tr>>" +
-		"<'row'<'col-sm-6'i><'col-sm-6'p>>",
-	renderer: 'bootstrap'
-} );
-
-
-/* Default class modification */
-$.extend( DataTable.ext.classes, {
-	sWrapper:      "dataTables_wrapper form-inline dt-bootstrap",
-	sFilterInput:  "form-control input-sm",
-	sLengthSelect: "form-control input-sm"
-} );
-
-
-/* Bootstrap paging button renderer */
-DataTable.ext.renderer.pageButton.bootstrap = function ( settings, host, idx, buttons, page, pages ) {
-	var api     = new DataTable.Api( settings );
-	var classes = settings.oClasses;
-	var lang    = settings.oLanguage.oPaginate;
-	var btnDisplay, btnClass;
-
-	var attach = function( container, buttons ) {
-		var i, ien, node, button;
-		var clickHandler = function ( e ) {
-			e.preventDefault();
-			if ( !$(e.currentTarget).hasClass('disabled') ) {
-				api.page( e.data.action ).draw( false );
-			}
-		};
-
-		for ( i=0, ien=buttons.length ; i 0 ?
-							'' : ' disabled');
-						break;
-
-					case 'previous':
-						btnDisplay = lang.sPrevious;
-						btnClass = button + (page > 0 ?
-							'' : ' disabled');
-						break;
-
-					case 'next':
-						btnDisplay = lang.sNext;
-						btnClass = button + (page < pages-1 ?
-							'' : ' disabled');
-						break;
-
-					case 'last':
-						btnDisplay = lang.sLast;
-						btnClass = button + (page < pages-1 ?
-							'' : ' disabled');
-						break;
-
-					default:
-						btnDisplay = button + 1;
-						btnClass = page === button ?
-							'active' : '';
-						break;
-				}
-
-				if ( btnDisplay ) {
-					node = $('
  • ', {
-							'class': classes.sPageButton+' '+btnClass,
-							'aria-controls': settings.sTableId,
-							'tabindex': settings.iTabIndex,
-							'id': idx === 0 && typeof button === 'string' ?
-								settings.sTableId +'_'+ button :
-								null
-						} )
-						.append( $('', {
-								'href': '#'
-							} )
-							.html( btnDisplay )
-						)
-						.appendTo( container );
-
-					settings.oApi._fnBindAction(
-						node, {action: button}, clickHandler
-					);
-				}
-			}
-		}
-	};
-
-	attach(
-		$(host).empty().html('
      ').children('ul'),
-		buttons
-	);
-};
-
-
-/*
- * TableTools Bootstrap compatibility
- * Required TableTools 2.1+
- */
-if ( DataTable.TableTools ) {
-	// Set the classes that TableTools uses to something suitable for Bootstrap
-	$.extend( true, DataTable.TableTools.classes, {
-		"container": "DTTT btn-group",
-		"buttons": {
-			"normal": "btn btn-default",
-			"disabled": "disabled"
-		},
-		"collection": {
-			"container": "DTTT_dropdown dropdown-menu",
-			"buttons": {
-				"normal": "",
-				"disabled": "disabled"
-			}
-		},
-		"print": {
-			"info": "DTTT_print_info"
-		},
-		"select": {
-			"row": "active"
-		}
-	} );
-
-	// Have the collection use a bootstrap compatible drop down
-	$.extend( true, DataTable.TableTools.DEFAULTS.oTags, {
-		"collection": {
-			"container": "ul",
-			"button": "li",
-			"liner": "a"
-		}
-	} );
-}
-
-}; // /factory
-
-
-// Define as an AMD module if possible
-if ( typeof define === 'function' && define.amd ) {
-	define( ['jquery', 'datatables'], factory );
-}
-else if ( typeof exports === 'object' ) {
-    // Node/CommonJS
-    factory( require('jquery'), require('datatables') );
-}
-else if ( jQuery ) {
-	// Otherwise simply initialise as normal, stopping multiple evaluation
-	factory( jQuery, jQuery.fn.dataTable );
-}
-
-
-})(window, document);
-
diff --git a/lib/cylc/cylc-review/static/js/jquery.dataTables.colVis.min.js b/lib/cylc/cylc-review/static/js/jquery.dataTables.colVis.min.js
deleted file mode 100644
index 01b31107d2d..00000000000
--- a/lib/cylc/cylc-review/static/js/jquery.dataTables.colVis.min.js
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- * File:        ColVis.min.js
- * Version:     1.0.6
- * Author:      Allan Jardine (www.sprymedia.co.uk)
- *
- * Copyright 2010-2011 Allan Jardine, all rights reserved.
- *
- * This source file is free software, under either the GPL v2 license or a
- * BSD (3 point) style license, as supplied with this software.
- *
- * This source file is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
- * or FITNESS FOR A PARTICULAR PURPOSE. See the license files for details.
- */
-(function(d){ColVis=function(a,b){if(!this.CLASS||this.CLASS!="ColVis")alert("Warning: ColVis must be initialised with the keyword 'new'");if(typeof b=="undefined")b={};this.s={dt:null,oInit:b,fnStateChange:null,activate:"click",sAlign:"left",buttonText:"Show / hide columns",hidden:true,aiExclude:[],abOriginal:[],bRestore:false,sRestore:"Restore original",iOverlayFade:500,fnLabel:null,sSize:"auto"};this.dom={wrapper:null,button:null,collection:null,background:null,catcher:null,buttons:[],restore:null};
-ColVis.aInstances.push(this);this.s.dt=a;this._fnConstruct();return this};ColVis.prototype={fnRebuild:function(){for(var a=this.dom.buttons.length-1;a>=0;a--)this.dom.buttons[a]!==null&&this.dom.collection.removeChild(this.dom.buttons[a]);this.dom.buttons.splice(0,this.dom.buttons.length);this.dom.restore&&this.dom.restore.parentNode(this.dom.restore);this._fnAddButtons();this._fnDrawCallback()},_fnConstruct:function(){this._fnApplyCustomisation();var a=this;this.dom.wrapper=document.createElement("div");
-this.dom.wrapper.className="ColVis TableTools";this.dom.button=this._fnDomBaseButton(this.s.buttonText);this.dom.button.className+=" ColVis_MasterButton";this.dom.wrapper.appendChild(this.dom.button);this.dom.catcher=this._fnDomCatcher();this.dom.collection=this._fnDomCollection();this.dom.background=this._fnDomBackground();this._fnAddButtons();for(var b=0,c=this.s.dt.aoColumns.length;b'+this.s.sRestore+"");d(b).click(function(){for(var e=0,f=a.s.abOriginal.length;e'+c+"");d(e).click(function(g){var h=!d("input",this).is(":checked");if(g.target.nodeName.toLowerCase()=="input")h=d("input",this).is(":checked");g=d.fn.dataTableExt.iApiIndex;d.fn.dataTableExt.iApiIndex=b._fnDataTablesApiIndex.call(b);b.s.dt.oInstance.fnSetColumnVis(a,h);d.fn.dataTableExt.iApiIndex=g;b.s.fnStateChange!==
-null&&b.s.fnStateChange.call(b,a,h)});return e},_fnDataTablesApiIndex:function(){for(var a=0,b=this.s.dt.oInstance.length;ai?c:i)+"px";f.style.width=(jh)e.style.left=h-b+"px";setTimeout(function(){d(e).animate({opacity:1},a.s.iOverlayFade);d(f).animate({opacity:0.1},a.s.iOverlayFade,"linear",function(){jQuery.browser.msie&&jQuery.browser.version=="6.0"&&a._fnDrawCallback()})},10);this.s.hidden=false},_fnCollectionHide:function(){var a=this;if(!this.s.hidden&&this.dom.collection!==null){this.s.hidden=
-true;d(this.dom.collection).animate({opacity:0},a.s.iOverlayFade,function(){this.style.display="none"});d(this.dom.background).animate({opacity:0},a.s.iOverlayFade,function(){document.body.removeChild(a.dom.background);document.body.removeChild(a.dom.catcher)})}}};ColVis.fnRebuild=function(a){var b=null;if(typeof a!="undefined")b=a.fnSettings().nTable;for(var c=0,e=ColVis.aInstances.length;c */function( window, document, undefined ) {
-
-(function( factory ) {
-	"use strict";
-
-	if ( typeof define === 'function' && define.amd ) {
-		// Define as an AMD module if possible
-		define( 'datatables', ['jquery'], factory );
-	}
-    else if ( typeof exports === 'object' ) {
-        // Node/CommonJS
-        factory( require( 'jquery' ) );
-    }
-	else if ( jQuery && !jQuery.fn.dataTable ) {
-		// Define using browser globals otherwise
-		// Prevent multiple instantiations if the script is loaded twice
-		factory( jQuery );
-	}
-}
-(/** @lends  */function( $ ) {
-	"use strict";
-
-	/**
-	 * DataTables is a plug-in for the jQuery Javascript library. It is a highly
-	 * flexible tool, based upon the foundations of progressive enhancement,
-	 * which will add advanced interaction controls to any HTML table. For a
-	 * full list of features please refer to
-	 * [DataTables.net](href="http://datatables.net).
-	 *
-	 * Note that the `DataTable` object is not a global variable but is aliased
-	 * to `jQuery.fn.DataTable` and `jQuery.fn.dataTable` through which it may
-	 * be  accessed.
-	 *
-	 *  @class
-	 *  @param {object} [init={}] Configuration object for DataTables. Options
-	 *    are defined by {@link DataTable.defaults}
-	 *  @requires jQuery 1.7+
-	 *
-	 *  @example
-	 *    // Basic initialisation
-	 *    $(document).ready( function {
-	 *      $('#example').dataTable();
-	 *    } );
-	 *
-	 *  @example
-	 *    // Initialisation with configuration options - in this case, disable
-	 *    // pagination and sorting.
-	 *    $(document).ready( function {
-	 *      $('#example').dataTable( {
-	 *        "paginate": false,
-	 *        "sort": false
-	 *      } );
-	 *    } );
-	 */
-	var DataTable;
-
-	
-	/*
-	 * It is useful to have variables which are scoped locally so only the
-	 * DataTables functions can access them and they don't leak into global space.
-	 * At the same time these functions are often useful over multiple files in the
-	 * core and API, so we list, or at least document, all variables which are used
-	 * by DataTables as private variables here. This also ensures that there is no
-	 * clashing of variable names and that they can easily referenced for reuse.
-	 */
-	
-	
-	// Defined else where
-	//  _selector_run
-	//  _selector_opts
-	//  _selector_first
-	//  _selector_row_indexes
-	
-	var _ext; // DataTable.ext
-	var _Api; // DataTable.Api
-	var _api_register; // DataTable.Api.register
-	var _api_registerPlural; // DataTable.Api.registerPlural
-	
-	var _re_dic = {};
-	var _re_new_lines = /[\r\n]/g;
-	var _re_html = /<.*?>/g;
-	var _re_date_start = /^[\w\+\-]/;
-	var _re_date_end = /[\w\+\-]$/;
-	
-	// Escape regular expression special characters
-	var _re_escape_regex = new RegExp( '(\\' + [ '/', '.', '*', '+', '?', '|', '(', ')', '[', ']', '{', '}', '\\', '$', '^', '-' ].join('|\\') + ')', 'g' );
-	
-	// U+2009 is thin space and U+202F is narrow no-break space, both used in many
-	// standards as thousands separators
-	var _re_formatted_numeric = /[',$£€¥%\u2009\u202F]/g;
-	
-	
-	var _empty = function ( d ) {
-		return !d || d === true || d === '-' ? true : false;
-	};
-	
-	
-	var _intVal = function ( s ) {
-		var integer = parseInt( s, 10 );
-		return !isNaN(integer) && isFinite(s) ? integer : null;
-	};
-	
-	// Convert from a formatted number with characters other than `.` as the
-	// decimal place, to a Javascript number
-	var _numToDecimal = function ( num, decimalPoint ) {
-		// Cache created regular expressions for speed as this function is called often
-		if ( ! _re_dic[ decimalPoint ] ) {
-			_re_dic[ decimalPoint ] = new RegExp( _fnEscapeRegex( decimalPoint ), 'g' );
-		}
-		return typeof num === 'string' && decimalPoint !== '.' ?
-			num.replace( /\./g, '' ).replace( _re_dic[ decimalPoint ], '.' ) :
-			num;
-	};
-	
-	
-	var _isNumber = function ( d, decimalPoint, formatted ) {
-		var strType = typeof d === 'string';
-	
-		if ( decimalPoint && strType ) {
-			d = _numToDecimal( d, decimalPoint );
-		}
-	
-		if ( formatted && strType ) {
-			d = d.replace( _re_formatted_numeric, '' );
-		}
-	
-		return _empty( d ) || (!isNaN( parseFloat(d) ) && isFinite( d ));
-	};
-	
-	
-	// A string without HTML in it can be considered to be HTML still
-	var _isHtml = function ( d ) {
-		return _empty( d ) || typeof d === 'string';
-	};
-	
-	
-	var _htmlNumeric = function ( d, decimalPoint, formatted ) {
-		if ( _empty( d ) ) {
-			return true;
-		}
-	
-		var html = _isHtml( d );
-		return ! html ?
-			null :
-			_isNumber( _stripHtml( d ), decimalPoint, formatted ) ?
-				true :
-				null;
-	};
-	
-	
-	var _pluck = function ( a, prop, prop2 ) {
-		var out = [];
-		var i=0, ien=a.length;
-	
-		// Could have the test in the loop for slightly smaller code, but speed
-		// is essential here
-		if ( prop2 !== undefined ) {
-			for ( ; i')
-			.css( {
-				position: 'absolute',
-				top: 0,
-				left: 0,
-				height: 1,
-				width: 1,
-				overflow: 'hidden'
-			} )
-			.append(
-				$('
      ')
-					.css( {
-						position: 'absolute',
-						top: 1,
-						left: 1,
-						width: 100,
-						overflow: 'scroll'
-					} )
-					.append(
-						$('
      ')
-							.css( {
-								width: '100%',
-								height: 10
-							} )
-					)
-			)
-			.appendTo( 'body' );
-	
-		var test = n.find('.test');
-	
-		// IE6/7 will oversize a width 100% element inside a scrolling element, to
-		// include the width of the scrollbar, while other browsers ensure the inner
-		// element is contained without forcing scrolling
-		browser.bScrollOversize = test[0].offsetWidth === 100;
-	
-		// In rtl text layout, some browsers (most, but not all) will place the
-		// scrollbar on the left, rather than the right.
-		browser.bScrollbarLeft = test.offset().left !== 1;
-	
-		n.remove();
-	}
-	
-	
-	/**
-	 * Array.prototype reduce[Right] method, used for browsers which don't support
-	 * JS 1.6. Done this way to reduce code size, since we iterate either way
-	 *  @param {object} settings dataTables settings object
-	 *  @memberof DataTable#oApi
-	 */
-	function _fnReduce ( that, fn, init, start, end, inc )
-	{
-		var
-			i = start,
-			value,
-			isSet = false;
-	
-		if ( init !== undefined ) {
-			value = init;
-			isSet = true;
-		}
-	
-		while ( i !== end ) {
-			if ( ! that.hasOwnProperty(i) ) {
-				continue;
-			}
-	
-			value = isSet ?
-				fn( value, that[i], i, that ) :
-				that[i];
-	
-			isSet = true;
-			i += inc;
-		}
-	
-		return value;
-	}
-	
-	/**
-	 * Add a column to the list used for the table with default values
-	 *  @param {object} oSettings dataTables settings object
-	 *  @param {node} nTh The th element for this column
-	 *  @memberof DataTable#oApi
-	 */
-	function _fnAddColumn( oSettings, nTh )
-	{
-		// Add column to aoColumns array
-		var oDefaults = DataTable.defaults.column;
-		var iCol = oSettings.aoColumns.length;
-		var oCol = $.extend( {}, DataTable.models.oColumn, oDefaults, {
-			"nTh": nTh ? nTh : document.createElement('th'),
-			"sTitle":    oDefaults.sTitle    ? oDefaults.sTitle    : nTh ? nTh.innerHTML : '',
-			"aDataSort": oDefaults.aDataSort ? oDefaults.aDataSort : [iCol],
-			"mData": oDefaults.mData ? oDefaults.mData : iCol,
-			idx: iCol
-		} );
-		oSettings.aoColumns.push( oCol );
-	
-		// Add search object for column specific search. Note that the `searchCols[ iCol ]`
-		// passed into extend can be undefined. This allows the user to give a default
-		// with only some of the parameters defined, and also not give a default
-		var searchCols = oSettings.aoPreSearchCols;
-		searchCols[ iCol ] = $.extend( {}, DataTable.models.oSearch, searchCols[ iCol ] );
-	
-		// Use the default column options function to initialise classes etc
-		_fnColumnOptions( oSettings, iCol, null );
-	}
-	
-	
-	/**
-	 * Apply options for a column
-	 *  @param {object} oSettings dataTables settings object
-	 *  @param {int} iCol column index to consider
-	 *  @param {object} oOptions object with sType, bVisible and bSearchable etc
-	 *  @memberof DataTable#oApi
-	 */
-	function _fnColumnOptions( oSettings, iCol, oOptions )
-	{
-		var oCol = oSettings.aoColumns[ iCol ];
-		var oClasses = oSettings.oClasses;
-		var th = $(oCol.nTh);
-	
-		// Try to get width information from the DOM. We can't get it from CSS
-		// as we'd need to parse the CSS stylesheet. `width` option can override
-		if ( ! oCol.sWidthOrig ) {
-			// Width attribute
-			oCol.sWidthOrig = th.attr('width') || null;
-	
-			// Style attribute
-			var t = (th.attr('style') || '').match(/width:\s*(\d+[pxem%]+)/);
-			if ( t ) {
-				oCol.sWidthOrig = t[1];
-			}
-		}
-	
-		/* User specified column options */
-		if ( oOptions !== undefined && oOptions !== null )
-		{
-			// Backwards compatibility
-			_fnCompatCols( oOptions );
-	
-			// Map camel case parameters to their Hungarian counterparts
-			_fnCamelToHungarian( DataTable.defaults.column, oOptions );
-	
-			/* Backwards compatibility for mDataProp */
-			if ( oOptions.mDataProp !== undefined && !oOptions.mData )
-			{
-				oOptions.mData = oOptions.mDataProp;
-			}
-	
-			if ( oOptions.sType )
-			{
-				oCol._sManualType = oOptions.sType;
-			}
-	
-			// `class` is a reserved word in Javascript, so we need to provide
-			// the ability to use a valid name for the camel case input
-			if ( oOptions.className && ! oOptions.sClass )
-			{
-				oOptions.sClass = oOptions.className;
-			}
-	
-			$.extend( oCol, oOptions );
-			_fnMap( oCol, oOptions, "sWidth", "sWidthOrig" );
-	
-			/* iDataSort to be applied (backwards compatibility), but aDataSort will take
-			 * priority if defined
-			 */
-			if ( typeof oOptions.iDataSort === 'number' )
-			{
-				oCol.aDataSort = [ oOptions.iDataSort ];
-			}
-			_fnMap( oCol, oOptions, "aDataSort" );
-		}
-	
-		/* Cache the data get and set functions for speed */
-		var mDataSrc = oCol.mData;
-		var mData = _fnGetObjectDataFn( mDataSrc );
-		var mRender = oCol.mRender ? _fnGetObjectDataFn( oCol.mRender ) : null;
-	
-		var attrTest = function( src ) {
-			return typeof src === 'string' && src.indexOf('@') !== -1;
-		};
-		oCol._bAttrSrc = $.isPlainObject( mDataSrc ) && (
-			attrTest(mDataSrc.sort) || attrTest(mDataSrc.type) || attrTest(mDataSrc.filter)
-		);
-	
-		oCol.fnGetData = function (rowData, type, meta) {
-			var innerData = mData( rowData, type, undefined, meta );
-	
-			return mRender && type ?
-				mRender( innerData, type, rowData, meta ) :
-				innerData;
-		};
-		oCol.fnSetData = function ( rowData, val, meta ) {
-			return _fnSetObjectDataFn( mDataSrc )( rowData, val, meta );
-		};
-	
-		// Indicate if DataTables should read DOM data as an object or array
-		// Used in _fnGetRowElements
-		if ( typeof mDataSrc !== 'number' ) {
-			oSettings._rowReadObject = true;
-		}
-	
-		/* Feature sorting overrides column specific when off */
-		if ( !oSettings.oFeatures.bSort )
-		{
-			oCol.bSortable = false;
-			th.addClass( oClasses.sSortableNone ); // Have to add class here as order event isn't called
-		}
-	
-		/* Check that the class assignment is correct for sorting */
-		var bAsc = $.inArray('asc', oCol.asSorting) !== -1;
-		var bDesc = $.inArray('desc', oCol.asSorting) !== -1;
-		if ( !oCol.bSortable || (!bAsc && !bDesc) )
-		{
-			oCol.sSortingClass = oClasses.sSortableNone;
-			oCol.sSortingClassJUI = "";
-		}
-		else if ( bAsc && !bDesc )
-		{
-			oCol.sSortingClass = oClasses.sSortableAsc;
-			oCol.sSortingClassJUI = oClasses.sSortJUIAscAllowed;
-		}
-		else if ( !bAsc && bDesc )
-		{
-			oCol.sSortingClass = oClasses.sSortableDesc;
-			oCol.sSortingClassJUI = oClasses.sSortJUIDescAllowed;
-		}
-		else
-		{
-			oCol.sSortingClass = oClasses.sSortable;
-			oCol.sSortingClassJUI = oClasses.sSortJUI;
-		}
-	}
-	
-	
-	/**
-	 * Adjust the table column widths for new data. Note: you would probably want to
-	 * do a redraw after calling this function!
-	 *  @param {object} settings dataTables settings object
-	 *  @memberof DataTable#oApi
-	 */
-	function _fnAdjustColumnSizing ( settings )
-	{
-		/* Not interested in doing column width calculation if auto-width is disabled */
-		if ( settings.oFeatures.bAutoWidth !== false )
-		{
-			var columns = settings.aoColumns;
-	
-			_fnCalculateColumnWidths( settings );
-			for ( var i=0 , iLen=columns.length ; i