Merge pull request #1 from cylc/master

sync latest changes into my fork

.codecov.yml

@@ -0,0 +1,46 @@
+# Codecov settings
+# After modifying this file, it might be worth to validate it with:
+# `curl --data-binary @.codecov.yml https://codecov.io/validate`
+
+# which branch to monitor and whether wait for ci success
+codecov:
+  branch: master
+  notify:
+    require_ci_to_pass: yes
+
+# define the colour bar limits here
+coverage:
+  precision: 2
+  round: down
+  range: "75...100"
+
+  # diff type
+  status:
+    project:
+      default:
+        # commits below this threshold will be marked as failed
+        target: '58%'
+        # how much we allow the coverage to drop
+        threshold: '5%'
+    patch:
+      default:
+        # basic
+        target: '58%'
+        threshold: '5%'
+        base: auto
+        # advanced
+        branches: null
+        if_no_uploads: error
+        if_not_found: success
+        if_ci_failed: error
+        only_pulls: false
+        flags: null
+        paths: null
+    changes: no
+
+# files to ignore
+ignore:
+  - "tests/**/*.py"
+
+# turn off comments to pull requests
+comment: off

.coveragerc

@@ -0,0 +1,99 @@
+# This is the Coverage.py configuration file. This is used by Travis-CI when running
+# the tests and collecting coverage
+
+[run]
+
+branch=True
+cover_pylib=False
+concurrency=thread
+data_file=.coverage
+disable_warnings=
+    trace-changed
+    module-not-python
+    module-not-imported
+    no-data-collected
+    module-not-measured
+    # include-ignored
+debug=
+#    callers
+#    dataio
+#    multiprocess
+#    dataio
+#    dataop
+#    pid
+#    plugin
+#    process
+#    sys
+#    trace
+# Include can be used only if source is not used!
+#include
+note=
+omit=
+    */lib/cherrypy/*
+    */lib/cylc/gui/*
+    */lib/cylc/tests/*
+    */lib/cylc/cylc_xdot.py
+    */lib/cylc/profiler.py
+    */lib/cylc/cfgspec/gcylc.py
+    */lib/cylc/cfgspec/gscan.py
+    */lib/cylc/profiling/*
+    */lib/cylc/mkdir_p.py
+    */lib/parsec/OrderedDictCompat.py
+    */lib/isodatetime/*
+    */lib/jinja2/*
+    */lib/markupsafe/*
+    */lib/parsec/tests/*
+    */lib/xdot.py
+parallel = True
+plugins=
+source=
+    ./bin
+    ./lib
+timid = False
+
+
+[report]
+
+exclude_lines =
+    pragma: no cover
+    def __repr__
+    raise NotImplementedError
+    if __name__ == .__main__.:
+    def parse_args
+fail_under=0
+ignore_errors = False
+include=
+omit=
+    */lib/cherrypy/*
+    */lib/cylc/gui/*
+    */lib/cylc/tests/*
+    */lib/cylc/cylc_xdot.py
+    */lib/cylc/profiler.py
+    */lib/cylc/cfgspec/gcylc.py
+    */lib/cylc/cfgspec/gscan.py
+    */lib/cylc/profiling/*
+    */lib/cylc/mkdir_p.py
+    */lib/parsec/OrderedDictCompat.py
+    */lib/isodatetime/*
+    */lib/jinja2/*
+    */lib/markupsafe/*
+    */lib/parsec/tests/*
+    */lib/xdot.py
+partial_branches=
+precision=2
+show_missing=False
+skip_covered=False
+sort=Name
+
+
+[html]
+
+directory=htmlcov
+extra_css=
+title=
+
+
+[xml]
+
+output=coverage.xml
+package_depth=99

.gitignore

@@ -42,3 +42,9 @@ passphrase
 
 # profiling
 .profiling
+
+# coverage
+.coverage
+.coverage.*
+coverage.xml
+htmlcov/

.travis.yml

@@ -31,6 +31,9 @@ matrix:
     include:
     - name: "1/4"
       env: CHUNK='1/4'
+      before_script:
+        # we must run unit tests only once
+        - PYTHONPATH=$(pwd -P)/lib/ pytest --cov-append --cov=lib/cylc --cov=lib/parsec
     - name: "2/4"
       env: CHUNK='2/4'
     - name: "3/4"
@@ -70,15 +73,25 @@ install:
     # Pygraphviz needs special treatment to avoid an error from "from . import release"
     - pip install pygraphviz --install-option="--include-path=/usr/include/graphviz" --install-option="--library-path=/usr/lib/graphviz/"
     - pip install EmPy pycodestyle pyopenssl
+    # Coverage dependencies
+    - pip install coverage pytest-cov mock
+    - export COVERAGE_PROCESS_START="${TRAVIS_BUILD_DIR}/.coveragerc"
 
 # Run tests
-script: 
+script:
     # Custom diff command to ignore Xlib errors (xvfb has not RANDR extension).
     - export CYLC_TEST_DIFF_CMD='diff -I Xlib -u'
     # Only run the generic tests on Travis CI.
     - export CYLC_TEST_RUN_PLATFORM=false
-    # Run tests with virtual frame buffer for X support.
-    - xvfb-run -a cylc test-battery --chunk $CHUNK --state=save -j 5 || (echo -e "\n\nRerunning Failed Tests...\n\n"; cylc test-battery --state=failed -j 5)
+    - export PYTHONPATH="${TRAVIS_BUILD_DIR}/.travis"
+    - coverage run .travis/cover.py
+    - unset PYTHONPATH
+
+after_success:
+    # Report metrics, such as coverage
+    - coverage combine --append
+    - coverage xml --ignore-errors
+    - bash <(curl -s https://codecov.io/bash)
 
 # Check output (more useful if you narrow down what tests get run)
 after_script:

lib/cylc/gui/color_rotator.py → .travis/cover.py

@@ -16,20 +16,20 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+import sys
 
-class ColorRotator(object):
+from subprocess import call
 
-    COLORS = ('#fcc', '#cfc', '#bbf', '#ffb')
 
-    def __init__(self, colors=COLORS):
-        self.colors = colors
-        self.current_color = 0
+def main():
+    # Run tests with virtual frame buffer for X support.
+    if call('xvfb-run -a cylc test-battery --chunk $CHUNK --state=save -j 5',
+            shell=True) != 0:
+        # Non-zero return code
+        sys.stderr.write('\n\nRerunning Failed Tests...\n\n')
+        # Exit with final return code
+        sys.exit(call('cylc test-battery --state=failed -j 5', shell=True))
 
-    def get_color(self):
-        index = self.current_color
-        if index == len(self.colors) - 1:
-            index = 0
-        else:
-            index += 1
-        self.current_color = index
-        return self.colors[index]
+
+if __name__ == '__main__':
+    main()

tests/triggering/18-conditional_simplifier-doc.t → .travis/sitecustomize.py

@@ -1,7 +1,8 @@
-#!/bin/bash
+#!/usr/bin/env python2
+
 # THIS FILE IS PART OF THE CYLC SUITE ENGINE.
 # Copyright (C) 2008-2018 NIWA & British Crown (Met Office) & Contributors.
-# 
+#
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
@@ -14,9 +15,9 @@
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
-#-------------------------------------------------------------------------------
-# cylc.conditional_simplifier doctest
-. "$(dirname "$0")/test_header"
-set_test_number 1
 
-run_ok "${TEST_NAME_BASE}" python -m 'cylc.conditional_simplifier'
+# This file is used by Travis-CI to start the coverage process. In order to make
+# Cylc and Python aware of it, we export PYTHONPATH when running the tests
+
+import coverage
+coverage.process_startup()

CHANGES.md

@@ -5,10 +5,134 @@ all changes see the [closed
 milestones](https://github.com/cylc/cylc/milestones?state=closed) for each
 release.
 
+-------------------------------------------------------------------------------
+## __cylc-7.8.0 (2018-11-27)__
+
+Minor release with over 120 issues closed. Significant issues include:
+
+### Enhancements
+
+[#2693](https://github.com/cylc/cylc/pull/2693) - __auto host selection__; and
+[#2809](https://github.com/cylc/cylc/pull/2809) - __auto migration__.  
+`cylc run` and `cylc restart` can now select the best host (based on several
+metrics) on which to launch suite server programs. And running suites
+can be told (via global config) to self-migrate to another available host, e.g.
+for server maintenance. (The pool of suite hosts should see a shared
+filesystem).
+
+[#2614](https://github.com/cylc/cylc/pull/2614) and 
+[#2821](https://github.com/cylc/cylc/pull/2821) - __web-based job log viewer__ -
+ `cylc review` (migration of "Rose Bush" from the Rose project). 
+
+
+[#2339](https://github.com/cylc/cylc/pull/2339) - __general external
+triggering__: tasks can trigger off of arbitrary user-defined Python functions
+called periodically by the suite server program, with built-in functions
+for suite-state (inter-suite) triggering and clock triggering (these
+deprecate the existing suite-state polling tasks and clock-triggered tasks).
+
+[#2734](https://github.com/cylc/cylc/pull/2734) - __EmPy templating__
+support, as an alternative to Jinja2. _"EmPy allows embedding plain Python code
+within templates and doesn't enforce any particular templating philosophy."_
+
+[#2733](https://github.com/cylc/cylc/pull/2733) - enhanced Jinja2 support:
+- __import pure Python modules__ in the same way as template modules
+- Jinja2Tests and Jinja2Globals, for custom "is" tests and global variables
+  (c.f. our existing Jinja2Filters for custom filters).
+
+[#2682](https://github.com/cylc/cylc/pull/2682) - new built-in
+Jinja2 filter to convert ISO8601 date-time durations to
+decimal seconds or hours.
+
+[#2842](https://github.com/cylc/cylc/pull/2842) - `cylc gui` and
+`cylc graph` - better integration with system desktop themes, including dark
+themes; and other minor graph visualization improvements. 
+
+[#2807](https://github.com/cylc/cylc/pull/2807) - task output events (event
+handlers can now be triggered when a task reports a registered output message).
+
+[#2868](https://github.com/cylc/cylc/pull/2868) - a new task
+runtime config item `exit-script`, for scripting to be executed at the
+last moment after successful job completion. (Companion of `err-script`).
+
+[#2781](https://github.com/cylc/cylc/pull/2781) and
+[#2854](https://github.com/cylc/cylc/pull/2854) - improved suite
+server program logging (including: `log/suite/err` is no longer used).
+
+[#2849](https://github.com/cylc/cylc/pull/2849) - record local
+background jobs by host name rather than "localhost".
+
+[#2877](https://github.com/cylc/cylc/pull/2877) - new batch system
+handler `pbs_multi_cluster`, supports PBS 13 clients fronting
+heterogeneous clusters with different home directories from the
+cylc remote. (Not needed for PBS 14+.) (For Rose suites this requires a
+corresponding change to `rose suite-run`:
+[metomi/rose#2252](https://github.com/metomi/rose/pull/2252).)
+
+[#2812](https://github.com/cylc/cylc/pull/2812) - `cylc gscan`:
+show application menu bar by default.
+
+[#2768](https://github.com/cylc/cylc/pull/2768) - `cylc gscan`: display Cylc
+version of running suites.
+
+[#2786](https://github.com/cylc/cylc/pull/2786) - make task try number
+available to event handlers (as for task job submit number).
+
+[#2771](https://github.com/cylc/cylc/pull/2771) - bash command completion:
+complete suite names for commands that take a suite name argument (see
+`etc/cylc-bash-completion`).
+
+[#2769](https://github.com/cylc/cylc/pull/2769) - `cylc check-software` now
+takes arguments to check for availability of specific modules.
+
+[#2763](https://github.com/cylc/cylc/pull/2763) - `cylc monitor` - clean exit
+on Ctrl-C.
+
+[#2704](https://github.com/cylc/cylc/pull/2704) - paginated `cylc help` output.
+
+[#2660](https://github.com/cylc/cylc/pull/2660) - new `gcylc.rc` config item to
+show grouped cyclepoint subgraphs by default.
+
+[#2766](https://github.com/cylc/cylc/pull/2766) - (development) formal test
+coverage reporting and integration with GitHub.
+
+[#2751](https://github.com/cylc/cylc/pull/2751) - (development) new contributor
+guidelines - see `CONTRIBUTING.md`.
+
+### Fixes
+
+[#2876](https://github.com/cylc/cylc/pull/2876) - avoid subprocess hang when
+executing commands that generate a lot of stdout (such as when submitting
+hundreds of jobs at once).
+
+[#2828](https://github.com/cylc/cylc/pull/2828) - `suite.rc` - fail validation
+on detecting trailing whitespace after a line continuation character.
+
+[#2807](https://github.com/cylc/cylc/pull/2807) - handle multiple events of
+the same type with the same message (e.g. warnings) from the same task job.
+
+[#2803](https://github.com/cylc/cylc/pull/2803) - reset job submit number
+correctly after aborting (backing out of) a trigger edit-run.
+
+[#2727](https://github.com/cylc/cylc/pull/2727) - `cylc gui`: fix dropdown list
+of log numbers for re-inserted tasks or after suite
+restart.
+
+[#2759](https://github.com/cylc/cylc/pull/2759) and
+[#2816](https://github.com/cylc/cylc/pull/2816) -
+suite registration tweaks and fixes.
+
+[#2861](https://github.com/cylc/cylc/pull/2861) - improved emacs
+syntax highlighting.
+
+[#2892](https://github.com/cylc/cylc/pull/2892) - print the bad host name along
+with "Name or service not known" exceptions.
+
+
 -------------------------------------------------------------------------------
 ## __cylc-7.7.2 (2018-07-26)__
 
-Minor maintenance release.
+Maintenance release.
 
 (Some minor changes not relevant to normal users may be omitted.)
 

README.md

@@ -2,6 +2,7 @@
 
 [![Build Status](https://travis-ci.org/cylc/cylc.svg?branch=master)](https://travis-ci.org/cylc/cylc)
 [![Codacy Badge](https://api.codacy.com/project/badge/Grade/1d6a97bf05114066ae30b63dcb0cdcf9)](https://www.codacy.com/app/Cylc/cylc?utm_source=github.com&utm_medium=referral&utm_content=cylc/cylc&utm_campaign=Badge_Grade)
+[![codecov](https://codecov.io/gh/cylc/cylc/branch/master/graph/badge.svg)](https://codecov.io/gh/cylc/cylc)
 [![DOI](https://zenodo.org/badge/1836229.svg)](https://zenodo.org/badge/latestdoi/1836229)
 [![DOI](http://joss.theoj.org/papers/10.21105/joss.00737/status.svg)](https://doi.org/10.21105/joss.00737)