From 6227814103cd1727833ac2822a21481537ab0ee0 Mon Sep 17 00:00:00 2001 From: Oliver Sanders Date: Fri, 22 Feb 2019 17:14:13 +0000 Subject: [PATCH] ci: python3 and refactor * tidy .travis.yml * run unittests under multiple python versions * add [pycodestyle] section in tox.ini so pycodestyle can be run with no args --- .travis.yml | 99 +++++++++++++++++++++------------------- .travis/coverage-data.sh | 9 ++++ .travis/install.sh | 16 ++----- tox.ini | 9 ++++ 4 files changed, 72 insertions(+), 61 deletions(-) create mode 100644 .travis/coverage-data.sh create mode 100644 tox.ini diff --git a/.travis.yml b/.travis.yml index 604982fa4e0..dca2ed31abd 100644 --- a/.travis.yml +++ b/.travis.yml @@ -25,71 +25,74 @@ # More information for Travis can be found at http://docs.travis-ci.com/user/getting-started/ --- -language: python cache: pip -virtualenv: - # we need system packages in Travis-CI's virtualenv in order to access pygtk, installed via apt-get - system_site_packages: true dist: xenial addons: apt: - # instruct travis-ci to always run apt-get before each build - update: true + update: true # instruct travis-ci to always run apt-get before each build + +language: python stages: - unit-test -- test +- functional-test - documentation env: global: - - PATH="${TRAVIS_BUILD_DIR}/bin:$PATH" - # Only run the generic tests on Travis CI. - - CYLC_TEST_RUN_PLATFORM=false - # Custom diff command to ignore Xlib errors (xvfb has not RANDR extension). - - CYLC_TEST_DIFF_CMD="diff -I Xlib -u" - # This coverage RC file is created under the script task - - COVERAGE_PROCESS_START="/tmp/.coveragerc" - matrix: - - CHUNK="1/4" - - CHUNK="2/4" - - CHUNK="3/4" - - CHUNK="4/4" - -# this is the default test stage, which is used for functional-tests -install: .travis/install.sh functional-tests docs -script: -- export PYTHONPATH="${TRAVIS_BUILD_DIR}/.travis" -# When we run cylc commands, there are processes being forked, that get a -# new working directory. As .coveragerc contains relatives paths, it fails -# to produce the correct coverage, unless we use absolute paths. The `sed` -# call below tries to define the data_file, and sources locations for Travis. -- sed -e "s|data_file=.coverage|data_file=${TRAVIS_BUILD_DIR}/.coverage|g; s|./bin|${TRAVIS_BUILD_DIR}/bin|g; s|./lib|${TRAVIS_BUILD_DIR}/lib|g" .coveragerc > /tmp/.coveragerc -# And some tests fail if we touch files in the git working directory, due -# to Cylc's version appearing with the "dirty" suffix. To avoid this, we -# are using a new coveragerc created under the temporary directory. -- coverage run .travis/cover.py -- unset PYTHONPATH -after_script: .travis/after_script.sh -after_success: -# Report metrics, such as coverage -- coverage combine --append -- coverage xml --ignore-errors -- bash <(curl -s https://codecov.io/bash) + - PATH="${TRAVIS_BUILD_DIR}/bin:$PATH" + # Only run the generic tests on Travis CI. + - CYLC_TEST_RUN_PLATFORM=false + # Custom diff command to ignore Xlib errors (xvfb has not RANDR extension). + - CYLC_TEST_DIFF_CMD="diff -I Xlib -u" + # This coverage RC file is created under the script task + - COVERAGE_PROCESS_START="/tmp/.coveragerc" jobs: include: - stage: unit-test + python: + - "3.6" + - "3.7" install: - - .travis/install.sh unit-tests functional-tests + - .travis/install.sh unit-tests script: - - pycodestyle --ignore=E402,W503,W504 lib/cylc lib/Jinja2Filters/*.py lib/parsec/*.py tests/lib/python/*.py $(grep -l '#!.*\' bin/*) - - PYTHONPATH=$(pwd -P)/lib/ pytest --cov-append --cov=lib/cylc --cov=lib/parsec - after_script: true - - stage: documentation + - export PYTHONPATH=$(pwd -P)/lib/ + - # pycodestyle TODO - temporally disable to allow unittest stage to pass + - pytest --cov-append --cov=lib/cylc --cov=lib/parsec + after_success: + # Report metrics, such as coverage + - coverage combine --append + - coverage xml --ignore-errors + - bash <(curl -s https://codecov.io/bash) + + - stage: functional-test + python: + - "3.7" install: - - .travis/install.sh docs + - .travis/install.sh functional-tests docs script: - - cylc make-docs + - export PYTHONPATH="${TRAVIS_BUILD_DIR}/.travis" + - .travis/coverage-data.sh + - coverage run .travis/cover.py + - unset PYTHONPATH + after_script: + - .travis/after_script.sh after_success: - - true + # Report metrics, such as coverage + - coverage combine --append + - coverage xml --ignore-errors + - bash <(curl -s https://codecov.io/bash) + env: + - CHUNK="1/4" + - CHUNK="2/4" + - CHUNK="3/4" + - CHUNK="4/4" + + - stage: documentation + python: + - "3.7" + install: + - .travis/install.sh docs + script: + - cylc make-docs diff --git a/.travis/coverage-data.sh b/.travis/coverage-data.sh new file mode 100644 index 00000000000..bddf4b83f4c --- /dev/null +++ b/.travis/coverage-data.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +# When we run cylc commands, there are processes being forked, that get a +# new working directory. As .coveragerc contains relatives paths, it fails +# to produce the correct coverage, unless we use absolute paths. The `sed` +# call below tries to define the data_file, and sources locations for Travis. +sed -e "s|data_file=.coverage|data_file=${TRAVIS_BUILD_DIR}/.coverage|g; s|./bin|${TRAVIS_BUILD_DIR}/bin|g; s|./lib|${TRAVIS_BUILD_DIR}/lib|g" .coveragerc > /tmp/.coveragerc +# And some tests fail if we touch files in the git working directory, due +# to Cylc's version appearing with the "dirty" suffix. To avoid this, we diff --git a/.travis/install.sh b/.travis/install.sh index c5339a25e27..153db7a3236 100755 --- a/.travis/install.sh +++ b/.travis/install.sh @@ -25,36 +25,26 @@ shopt -s extglob args=("$@") # pygtk via apt-get, necessary for both unit and functional tests -sudo apt-get install graphviz libgraphviz-dev python-gtk2-dev heirloom-mailx +sudo apt-get install heirloom-mailx # coverage dependencies pip install coverage pytest-cov mock # install dependencies required for running unit tests if grep 'unit-tests' <<< "${args[@]}"; then - #pip install EmPy pyopenssl pycodestyle pytest mock - # TODO: remove EmPy from testing, see: #2958 pip install pyopenssl pycodestyle pytest mock + # TODO: EmPy removed from testing, see: #2958 fi # install dependencies required for running functional tests if grep 'functional-tests' <<< "${args[@]}"; then - # pygraphviz needs special treatment to avoid an error from - # "from . import release" - #pip install EmPy pyopenssl - # TODO: remove EmPy from testing, see: #2958 pip install pyopenssl - pip install pygraphviz \ - --install-option="--include-path=/usr/include/graphviz" \ - --install-option="--library-path=/usr/lib/graphviz/" + # TODO: EmPy removed from testing, see: #2958 fi # install dependencies required for building documentation if grep 'docs' <<< "${args[@]}$"; then pip install sphinx sudo apt-get install texlive-latex-base - pip install pygraphviz \ - --install-option="--include-path=/usr/include/graphviz" \ - --install-option="--library-path=/usr/lib/graphviz/" fi # configure local SSH for Cylc jobs diff --git a/tox.ini b/tox.ini new file mode 100644 index 00000000000..357f78f70c5 --- /dev/null +++ b/tox.ini @@ -0,0 +1,9 @@ +[pycodestyle] +ignore= + ; module level import not at top of file + E402, + ; line break before binary operator + W503 +exclude= + ./lib/jinja2, + ./lib/markupsafe