Merge pull request #2966 from oliver-sanders/python3

Python3

.coveragerc

@@ -29,21 +29,14 @@ debug=
 #include
 note=
 omit=
-    */lib/cherrypy/*
-    */lib/cylc/gui/*
     */lib/cylc/tests/*
-    */lib/cylc/cylc_xdot.py
     */lib/cylc/profiler.py
-    */lib/cylc/cfgspec/gcylc.py
-    */lib/cylc/cfgspec/gscan.py
     */lib/cylc/profiling/*
-    */lib/cylc/mkdir_p.py
     */lib/parsec/OrderedDictCompat.py
     */lib/isodatetime/*
     */lib/jinja2/*
     */lib/markupsafe/*
     */lib/parsec/tests/*
-    */lib/xdot.py
 parallel = True
 plugins=
 source=
@@ -64,21 +57,14 @@ fail_under=0
 ignore_errors = False
 include=
 omit=
-    */lib/cherrypy/*
-    */lib/cylc/gui/*
     */lib/cylc/tests/*
-    */lib/cylc/cylc_xdot.py
     */lib/cylc/profiler.py
-    */lib/cylc/cfgspec/gcylc.py
-    */lib/cylc/cfgspec/gscan.py
     */lib/cylc/profiling/*
-    */lib/cylc/mkdir_p.py
     */lib/parsec/OrderedDictCompat.py
     */lib/isodatetime/*
     */lib/jinja2/*
     */lib/markupsafe/*
     */lib/parsec/tests/*
-    */lib/xdot.py
 partial_branches=
 precision=2
 show_missing=False

.gitignore

@@ -1,6 +1,7 @@
 
 # python bytecode
-*.pyc
+*.py[cod]
+__pycache__/
 
 # Generated documentation files (from 'cylc make-docs')
 # - Command reference from called script 'custom/make-commands.sh'.

.travis.yml

@@ -25,63 +25,93 @@
 # More information for Travis can be found at http://docs.travis-ci.com/user/getting-started/
 
 ---
-language: python
 cache: pip
-virtualenv:
-  # we need system packages in Travis-CI's virtualenv in order to access pygtk, installed via apt-get
-  system_site_packages: true
 dist: xenial
 addons:
   apt:
-    # instruct travis-ci to always run apt-get before each build
-    update: true
+    update: true  # instruct travis-ci to always run apt-get before each build
+
+language: python
 
 stages:
 - unit-test
-- test
+- functional-test
+- documentation
 
 env:
   global:
-  - PATH="${TRAVIS_BUILD_DIR}/bin:$PATH"
-  # Only run the generic tests on Travis CI.
-  - CYLC_TEST_RUN_PLATFORM=false
-  # Custom diff command to ignore Xlib errors (xvfb has not RANDR extension).
-  - CYLC_TEST_DIFF_CMD="diff -I Xlib -u"
-  # This coverage RC file is created under the script task
-  - COVERAGE_PROCESS_START="/tmp/.coveragerc"
-  matrix:
-  - CHUNK="1/4"
-  - CHUNK="2/4"
-  - CHUNK="3/4"
-  - CHUNK="4/4"
+    - PATH="${TRAVIS_BUILD_DIR}/bin:$PATH"
+    # Only run the generic tests on Travis CI.
+    - CYLC_TEST_RUN_PLATFORM=false
+    # Custom diff command to ignore Xlib errors (xvfb has not RANDR extension).
+    - CYLC_TEST_DIFF_CMD="diff -I Xlib -u"
+    # This coverage RC file is created under the script task
+    - COVERAGE_PROCESS_START="/tmp/.coveragerc"
+
+unittest: &unittest
+  stage: unit-test
+  install:
+    - .travis/install.sh unit-tests
+  script:
+    - export PYTHONPATH=$(pwd -P)/lib/
+    - pycodestyle
+    - pytest --cov-append --cov=lib/cylc --cov=lib/parsec
+  after_success:
+    # Report metrics, such as coverage
+    - coverage combine --append
+    - coverage xml --ignore-errors
+    - bash <(curl -s https://codecov.io/bash)
 
-# this is the default test stage, which is used for functional-tests
-install: .travis/install.sh functional-tests docs
-script:
-- export PYTHONPATH="${TRAVIS_BUILD_DIR}/.travis"
-# When we run cylc commands, there are processes being forked, that get a
-# new working directory. As .coveragerc contains relatives paths, it fails
-# to produce the correct coverage, unless we use absolute paths. The `sed`
-# call below tries to define the data_file, and sources locations for Travis.
-- sed -e "s|data_file=.coverage|data_file=${TRAVIS_BUILD_DIR}/.coverage|g; s|./bin|${TRAVIS_BUILD_DIR}/bin|g; s|./lib|${TRAVIS_BUILD_DIR}/lib|g" .coveragerc > /tmp/.coveragerc
-# And some tests fail if we touch files in the git working directory, due
-# to Cylc's version appearing with the "dirty" suffix. To avoid this, we
-# are using a new coveragerc created under the temporary directory.
-- coverage run .travis/cover.py
-- unset PYTHONPATH
-after_script: .travis/after_script.sh
-after_success:
-# Report metrics, such as coverage
-- coverage combine --append
-- coverage xml --ignore-errors
-- bash <(curl -s https://codecov.io/bash)
+regressiontest: &regressiontest
+  stage: functional-test
+  python:
+    - "3.7"
+  install:
+    - .travis/install.sh functional-tests docs
+  script:
+    - export PYTHONPATH="${TRAVIS_BUILD_DIR}/.travis"
+    - .travis/coverage-data.sh
+    - coverage run .travis/cover.py
+    - unset PYTHONPATH
+  after_failure:
+    - .travis/after_failure.sh
+  after_success:
+    # Report metrics, such as coverage
+    - coverage combine --append
+    - coverage xml --ignore-errors
+    - bash <(curl -s https://codecov.io/bash)
 
 jobs:
   include:
-  - stage: unit-test
+
+  # run unittests at each supported Python version
+  - <<: *unittest
+    python: "3.6"
+  - <<: *unittest
+    python: "3.7"
+
+  # run regression tests in chunks with the most contemporary
+  # supported Python version
+  - <<: *regressiontest
+    python: "3.7"
+    env: CHUNK="1/4"
+  - <<: *regressiontest
+    python: "3.7"
+    env: CHUNK="2/4"
+  - <<: *regressiontest
+    python: "3.7"
+    env: CHUNK="3/4"
+  - <<: *regressiontest
+    python: "3.7"
+    env: CHUNK="4/4"
+
+  # TODO: run shortlist of regression tests with the
+  #       least contemporary Python version
+
+  # build documentation
+  - stage: documentation
+    python: "3.7"
     install:
-    - .travis/install.sh unit-tests functional-tests
+      - .travis/install.sh docs
     script:
-    - pycodestyle --ignore=E402,W503,W504 lib/cylc lib/Jinja2Filters/*.py lib/parsec/*.py $(grep -l '#!.*\<python\>' bin/*)
-    - PYTHONPATH=$(pwd -P)/lib/ pytest --cov-append --cov=lib/cylc --cov=lib/parsec
-    after_script: true
+      - cylc make-docs

.travis/after_script.sh → .travis/after_failure.sh

@@ -20,5 +20,7 @@
 
 # Check output (more useful if you narrow down what tests get run)
 
+cylc scan -f --color=never
+
 find $HOME/cylc-run -name '*.err' -type f -exec echo '==== {} ====' \; -exec cat '{}' \;
 find /tmp/${USER}/cylctb-* -type f -exec echo '==== {} ====' \; -exec cat '{}' \;

.travis/cover.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
 
 # THIS FILE IS PART OF THE CYLC SUITE ENGINE.
 # Copyright (C) 2008-2019 NIWA & British Crown (Met Office) & Contributors.

bin/gcontrol → .travis/coverage-data.sh

@@ -16,6 +16,11 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-echo "WARNING: \"gcontrol\" is deprecated - it is now called \"gcylc\""
-echo "WARNING: This command alias will eventually be deleted."
-$(dirname $0)/cylc gui "$@"
+# When we run cylc commands, there are processes being forked, that get a
+# new working directory. As .coveragerc contains relatives paths, it fails
+# to produce the correct coverage, unless we use absolute paths. The `sed`
+# call below tries to define the data_file, and sources locations for Travis.
+sed -e "s|data_file=.coverage|data_file=${TRAVIS_BUILD_DIR}/.coverage|g; s|./bin|${TRAVIS_BUILD_DIR}/bin|g; s|./lib|${TRAVIS_BUILD_DIR}/lib|g" .coveragerc > /tmp/.coveragerc
+# And some tests fail if we touch files in the git working directory, due
+# to Cylc's version appearing with the "dirty" suffix. To avoid this, we
+# are using a new coveragerc created under the temporary directory.

.travis/install.sh

@@ -24,27 +24,23 @@ shopt -s extglob
 
 args=("$@")
 
-# pygtk via apt-get, necessary for both unit and functional tests
-sudo apt-get install graphviz libgraphviz-dev python-gtk2-dev heirloom-mailx
-# coverage dependencies
-pip install coverage pytest-cov mock
-
-# install dependencies required for running unit tests
-if grep 'unit-tests' <<< "${args[@]}"; then
-    pip install EmPy pyopenssl pycodestyle pytest mock
+if grep -E '(unit-tests|functional-tests)' <<< "${args[@]}"; then
+    sudo apt-get install heirloom-mailx
+    # coverage dependencies
+    pip install coverage pytest-cov
+    # common Cylc reqirements
+    pip install colorama python-jose zmq
 fi
 
-# install dependencies required for running functional tests
-if grep 'functional-tests' <<< "${args[@]}"; then
-    # pygraphviz needs special treatment to avoid an error from "from . import release"
-    pip install EmPy pyopenssl
-    pip install pygraphviz \
-      --install-option="--include-path=/usr/include/graphviz" \
-      --install-option="--library-path=/usr/lib/graphviz/"
+if grep 'unit-tests' <<< "${args[@]}"; then
+    pip install pycodestyle pytest
+    # TODO: EmPy removed from testing, see:  #2958
 fi
 
-# install dependencies required for building documentation, only when instructed to do so
+# install dependencies required for building documentation
 if grep 'docs' <<< "${args[@]}$"; then
+    pip install sphinx
+    # for PDF output via LaTeX builder
     sudo apt-get install texlive-latex-base
 fi
 

.travis/sitecustomize.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
 
 # THIS FILE IS PART OF THE CYLC SUITE ENGINE.
 # Copyright (C) 2008-2019 NIWA & British Crown (Met Office) & Contributors.
@@ -16,8 +16,9 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-# This file is used by Travis-CI to start the coverage process. In order to make
-# Cylc and Python aware of it, we export PYTHONPATH when running the tests
+# This file is used by Travis-CI to start the coverage process.
+# In order to make Cylc and Python aware of it, we export PYTHONPATH when
+# running the tests.
 
 import coverage
 coverage.process_startup()

ACKNOWLEDGEMENT.md

@@ -2,10 +2,6 @@
 
 Licences for non-cylc work included in this distribution can be found in the
 `licences/` directory.
- * `lib/cherrypy/`:
-    External software library released under a BSD license.
-    Minor modification to ignore an import warning.
-    See [cherrypy](http://www.cherrypy.org/).
  * `lib/isodatetime/`:
     Unmodified external software library released under the LGPL license.
     See [metomi/isodatetime](https://github.com/metomi/isodatetime/).
@@ -15,10 +11,6 @@ Licences for non-cylc work included in this distribution can be found in the
  * `lib/markupsafe/`:
     External software library released under a BSD license, used by Jinja2.
     See [MarkupSafe](http://www.pocoo.org/projects/markupsafe/).
- * `lib/xdot.py`:
-    External software released under the LGPL license.
-    Modifications based on version 0.6. See
-    [xdot](https://github.com/jrfonseca/xdot.py)
  * `lib/cylc/cylc-review/static/css/bootstrap-???`,
    `lib/cylc/cylc-review/static/images/glyphicons-halflings-???` &
    `lib/cylc/cylc-review/static/js/bootstrap.min.js`:

CHANGES.md

@@ -229,7 +229,7 @@ warnings emitted by `urllib3` under old Python versions (2.6). *We are
 aware of the security issues, but these warnings serve no purpose on affected
 platforms except to confuse and annoy users.*
 
-[#2676](https://github.com/cylc/cylc/pull/2676) - use `#!/usr/bin/env python2`
+[#2676](https://github.com/cylc/cylc/pull/2676) - use `#!/usr/bin/env python3`
 (i.e. Python-2 specific) in Cylc source files, to avoid issues with default
 Python 3 installations (note Cylc is going to Python 3 next year)
 

INSTALL.md

@@ -63,39 +63,27 @@ $ cylc check-software
 Checking your software...
 
 Individual results:
-===============================================================================
-Package (version requirements)                          Outcome (version found)
-===============================================================================
-                              *REQUIRED SOFTWARE*                              
-Python (2.6+, <3).....................FOUND & min. version MET (2.7.12.final.0)
-
-       *OPTIONAL SOFTWARE for the GUI & dependency graph visualisation*       
-Python:pygtk (2.0+)...........................FOUND & min. version MET (2.24.0)
-graphviz (any)...................................................FOUND (2.38.0)
-Python:pygraphviz (any)...........................................FOUND (1.3.1)
-
-                  *OPTIONAL SOFTWARE for the HTML User Guide*                  
-ImageMagick (any)...............................................FOUND (6.8.9-9)
-
-            *OPTIONAL SOFTWARE for the HTTPS communications layer*            
-Python:urllib3 (any).............................................FOUND (1.13.1)
-Python:OpenSSL (any).............................................FOUND (17.2.0)
-Python:requests (2.4.2+).......................FOUND & min. version MET (2.9.1)
-
-                 *OPTIONAL SOFTWARE for the LaTeX User Guide*                 
-TeX:framed (any)....................................................FOUND (n/a)
-TeX (3.0+)................................FOUND & min. version MET (3.14159265)
-TeX:preprint (any)..................................................FOUND (n/a)
-TeX:tex4ht (any)....................................................FOUND (n/a)
-TeX:tocloft (any)...................................................FOUND (n/a)
-TeX:texlive (any)...................................................FOUND (n/a)
-===============================================================================
+================================================================================
+Package (version requirements)                           Outcome (version found)
+================================================================================
+                              *REQUIRED SOFTWARE*
+Python (3+).............................FOUND & min. version MET (3.7.2.final.0)
+Python:zmq (any)..................................................FOUND (17.1.2)
+Python:jose (any)..................................................FOUND (2.0.2)
+Python:colorama (any)..............................................FOUND (0.4.1)
+
+              *OPTIONAL SOFTWARE for the configuration templating*
+Python:EmPy (any)..................................................FOUND (3.3.2)
+
+                 *OPTIONAL SOFTWARE for the HTML documentation*
+Python:sphinx (1.5.3+)..........................FOUND & min. version MET (1.8.4)
+================================================================================
 
 Summary:
-                         ****************************                         
-                             Core requirements: ok                             
-                            Full-functionality: ok                            
-                         ****************************  
+                          ****************************
+                             Core requirements: ok
+                             Full-functionality: ok
+                          ****************************
 ```
 
 ### Installing The Documentation

bin/cylc

@@ -140,7 +140,7 @@ __HELP__
 }
 
 category_help_command() {
-# Special case for printing categories and the gcylc gui
+# Special case for printing categories
     local COMMAND="${CYLC_HOME_BIN}/cylc-help"
     exec "${COMMAND}" "categories"
 }