Add coverage to the project reports.

.codecov.yml

@@ -0,0 +1,46 @@
+# Codecov settings
+# After modifying this file, it might be worth to validate it with:
+# `curl --data-binary @.codecov.yml https://codecov.io/validate`
+
+# which branch to monitor and whether wait for ci success
+codecov:
+  branch: master
+  notify:
+    require_ci_to_pass: yes
+
+# define the colour bar limits here
+coverage:
+  precision: 2
+  round: down
+  range: "75...100"
+
+  # diff type
+  status:
+    project:
+      default:
+        # commits below this threshold will be marked as failed
+        target: '58%'
+        # how much we allow the coverage to drop
+        threshold: '5%'
+    patch:
+      default:
+        # basic
+        target: '58%'
+        threshold: '5%'
+        base: auto
+        # advanced
+        branches: null
+        if_no_uploads: error
+        if_not_found: success
+        if_ci_failed: error
+        only_pulls: false
+        flags: null
+        paths: null
+    changes: no
+
+# files to ignore
+ignore:
+  - "tests/**/*.py"
+
+# turn off comments to pull requests
+comment: off

.coveragerc

@@ -0,0 +1,99 @@
+# This is the Coverage.py configuration file. This is used by Travis-CI when running
+# the tests and collecting coverage
+
+[run]
+
+branch=True
+cover_pylib=False
+concurrency=thread
+data_file=.coverage
+disable_warnings=
+    trace-changed
+    module-not-python
+    module-not-imported
+    no-data-collected
+    module-not-measured
+    # include-ignored
+debug=
+#    callers
+#    dataio
+#    multiprocess
+#    dataio
+#    dataop
+#    pid
+#    plugin
+#    process
+#    sys
+#    trace
+# Include can be used only if source is not used!
+#include
+note=
+omit=
+    */lib/cherrypy/*
+    */lib/cylc/gui/*
+    */lib/cylc/tests/*
+    */lib/cylc/cylc_xdot.py
+    */lib/cylc/profiler.py
+    */lib/cylc/cfgspec/gcylc.py
+    */lib/cylc/cfgspec/gscan.py
+    */lib/cylc/profiling/*
+    */lib/cylc/mkdir_p.py
+    */lib/parsec/OrderedDictCompat.py
+    */lib/isodatetime/*
+    */lib/jinja2/*
+    */lib/markupsafe/*
+    */lib/parsec/tests/*
+    */lib/xdot.py
+parallel = True
+plugins=
+source=
+    ./bin
+    ./lib
+timid = False
+
+
+[report]
+
+exclude_lines =
+    pragma: no cover
+    def __repr__
+    raise NotImplementedError
+    if __name__ == .__main__.:
+    def parse_args
+fail_under=0
+ignore_errors = False
+include=
+omit=
+    */lib/cherrypy/*
+    */lib/cylc/gui/*
+    */lib/cylc/tests/*
+    */lib/cylc/cylc_xdot.py
+    */lib/cylc/profiler.py
+    */lib/cylc/cfgspec/gcylc.py
+    */lib/cylc/cfgspec/gscan.py
+    */lib/cylc/profiling/*
+    */lib/cylc/mkdir_p.py
+    */lib/parsec/OrderedDictCompat.py
+    */lib/isodatetime/*
+    */lib/jinja2/*
+    */lib/markupsafe/*
+    */lib/parsec/tests/*
+    */lib/xdot.py
+partial_branches=
+precision=2
+show_missing=False
+skip_covered=False
+sort=Name
+
+
+[html]
+
+directory=htmlcov
+extra_css=
+title=
+
+
+[xml]
+
+output=coverage.xml
+package_depth=99

.gitignore

@@ -42,3 +42,9 @@ passphrase
 
 # profiling
 .profiling
+
+# coverage
+.coverage
+.coverage.*
+coverage.xml
+htmlcov/

.travis.yml

@@ -31,6 +31,9 @@ matrix:
     include:
     - name: "1/4"
       env: CHUNK='1/4'
+      before_script:
+        # we must run unit tests only once
+        - PYTHONPATH=$(pwd -P)/lib/ pytest --cov-append --cov=lib/cylc --cov=lib/parsec
     - name: "2/4"
       env: CHUNK='2/4'
     - name: "3/4"
@@ -70,15 +73,25 @@ install:
     # Pygraphviz needs special treatment to avoid an error from "from . import release"
     - pip install pygraphviz --install-option="--include-path=/usr/include/graphviz" --install-option="--library-path=/usr/lib/graphviz/"
     - pip install EmPy pycodestyle pyopenssl
+    # Coverage dependencies
+    - pip install coverage pytest-cov mock
+    - export COVERAGE_PROCESS_START="${TRAVIS_BUILD_DIR}/.coveragerc"
 
 # Run tests
-script: 
+script:
     # Custom diff command to ignore Xlib errors (xvfb has not RANDR extension).
     - export CYLC_TEST_DIFF_CMD='diff -I Xlib -u'
     # Only run the generic tests on Travis CI.
     - export CYLC_TEST_RUN_PLATFORM=false
-    # Run tests with virtual frame buffer for X support.
-    - xvfb-run -a cylc test-battery --chunk $CHUNK --state=save -j 5 || (echo -e "\n\nRerunning Failed Tests...\n\n"; cylc test-battery --state=failed -j 5)
+    - export PYTHONPATH="${TRAVIS_BUILD_DIR}/.travis"
+    - coverage run .travis/cover.py
+    - unset PYTHONPATH
+
+after_success:
+    # Report metrics, such as coverage
+    - coverage combine --append
+    - coverage xml --ignore-errors
+    - bash <(curl -s https://codecov.io/bash)
 
 # Check output (more useful if you narrow down what tests get run)
 after_script:

tests/graph_parser/00-unittests.t → .travis/cover.py

@@ -1,4 +1,5 @@
-#!/bin/bash
+#!/usr/bin/env python2
+
 # THIS FILE IS PART OF THE CYLC SUITE ENGINE.
 # Copyright (C) 2008-2018 NIWA & British Crown (Met Office) & Contributors.
 #
@@ -15,11 +16,20 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-# Run graph parser unit tests.
-. "$(dirname "$0")/test_header"
+import sys
+
+from subprocess import call
+
+
+def main():
+    # Run tests with virtual frame buffer for X support.
+    if call('xvfb-run -a cylc test-battery --chunk $CHUNK --state=save -j 5',
+            shell=True) != 0:
+        # Non-zero return code
+        sys.stderr.write('\n\nRerunning Failed Tests...\n\n')
+        # Exit with final return code
+        sys.exit(call('cylc test-battery --state=failed -j 5', shell=True))
 
-set_test_number 2
 
-TEST_NAME="$TEST_NAME_BASE-unit-tests"
-run_ok "${TEST_NAME_BASE}-cylc.graph-parser" python -m 'cylc.graph_parser'
-run_ok "${TEST_NAME_BASE}-cylc.cycling" python -m 'cylc.cycling.__init__'
+if __name__ == '__main__':
+    main()

tests/triggering/18-conditional_simplifier-doc.t → .travis/sitecustomize.py

@@ -1,7 +1,8 @@
-#!/bin/bash
+#!/usr/bin/env python2
+
 # THIS FILE IS PART OF THE CYLC SUITE ENGINE.
 # Copyright (C) 2008-2018 NIWA & British Crown (Met Office) & Contributors.
-# 
+#
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
@@ -14,9 +15,9 @@
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
-#-------------------------------------------------------------------------------
-# cylc.conditional_simplifier doctest
-. "$(dirname "$0")/test_header"
-set_test_number 1
 
-run_ok "${TEST_NAME_BASE}" python -m 'cylc.conditional_simplifier'
+# This file is used by Travis-CI to start the coverage process. In order to make
+# Cylc and Python aware of it, we export PYTHONPATH when running the tests
+
+import coverage
+coverage.process_startup()

README.md

@@ -2,6 +2,7 @@
 
 [![Build Status](https://travis-ci.org/cylc/cylc.svg?branch=master)](https://travis-ci.org/cylc/cylc)
 [![Codacy Badge](https://api.codacy.com/project/badge/Grade/1d6a97bf05114066ae30b63dcb0cdcf9)](https://www.codacy.com/app/Cylc/cylc?utm_source=github.com&utm_medium=referral&utm_content=cylc/cylc&utm_campaign=Badge_Grade)
+[![codecov](https://codecov.io/gh/cylc/cylc/branch/master/graph/badge.svg)](https://codecov.io/gh/cylc/cylc)
 [![DOI](https://zenodo.org/badge/1836229.svg)](https://zenodo.org/badge/latestdoi/1836229)
 [![DOI](http://joss.theoj.org/papers/10.21105/joss.00737/status.svg)](https://doi.org/10.21105/joss.00737)
 

lib/cylc/conditional_simplifier.py

@@ -189,8 +189,3 @@ def flatten_nested_expr(cls, expr):
         if isinstance(flattened, list):
             flattened = " ".join(flattened)
         return "({0})".format(flattened)
-
-
-if __name__ == "__main__":
-    import doctest
-    doctest.testmod()

lib/cylc/cycling/__init__.py

@@ -459,58 +459,3 @@ def __str__(self):
         if ',' in ret:
             ret = '(' + ret + ')'
         return ret
-
-
-if __name__ == "__main__":
-    import unittest
-
-    class TestBaseClasses(unittest.TestCase):
-        """Test the abstract base classes cannot be instantiated on their own
-        """
-        def test_simple_abstract_class_test(self):
-            """Cannot instantiate abstract classes, they must be defined in
-            the subclasses"""
-            self.assertRaises(TypeError, SequenceBase, "sequence-string",
-                              "context_string")
-            self.assertRaises(TypeError, IntervalBase, "value")
-            self.assertRaises(TypeError, PointBase, "value")
-
-    class TestParseExclusion(unittest.TestCase):
-        """Test cases for the parser function"""
-        def test_parse_exclusion_simple(self):
-            """Tests the simple case of exclusion parsing"""
-            expression = "PT1H!20000101T02Z"
-            sequence, exclusion = parse_exclusion(expression)
-
-            self.assertEqual(sequence, "PT1H")
-            self.assertEqual(exclusion, ['20000101T02Z'])
-
-        def test_parse_exclusions_list(self):
-            """Tests the simple case of exclusion parsing"""
-            expression = "PT1H!(T03, T06, T09)"
-            sequence, exclusion = parse_exclusion(expression)
-
-            self.assertEqual(sequence, "PT1H")
-            self.assertEqual(exclusion, ['T03', 'T06', 'T09'])
-
-        def test_parse_exclusions_list_spaces(self):
-            """Tests the simple case of exclusion parsing"""
-            expression = "PT1H!    (T03, T06,   T09)   "
-            sequence, exclusion = parse_exclusion(expression)
-
-            self.assertEqual(sequence, "PT1H")
-            self.assertEqual(exclusion, ['T03', 'T06', 'T09'])
-
-        def test_parse_bad_exclusion(self):
-            """Tests incorrectly formatted exclusions"""
-            expression1 = "T01/PT1H!(T06, T09), PT5M"
-            expression2 = "T01/PT1H!T03, PT17H, (T06, T09), PT5M"
-            expression3 = "T01/PT1H! PT8H, (T06, T09)"
-            expression4 = "T01/PT1H! T03, T06, T09"
-
-            self.assertRaises(Exception, parse_exclusion, expression1)
-            self.assertRaises(Exception, parse_exclusion, expression2)
-            self.assertRaises(Exception, parse_exclusion, expression3)
-            self.assertRaises(Exception, parse_exclusion, expression4)
-
-    unittest.main()