actions: coverage

.dockerignore

@@ -1,6 +1,7 @@
 *
 *.py[cod]
 __pycache__
+!.coveragerc
 !.docker*
 !README.md
 !bin

.github/workflows/test_fast.yml

@@ -35,12 +35,17 @@ jobs:
 
       - name: Doctests
         run: |
-          pytest -n 5 cylc/flow
+          pytest --cov --cov-append -n 5 cylc/flow
 
       - name: Unit Tests
         run: |
-          pytest -n 5 tests/unit
+          pytest --cov --cov-append -n 5 tests/unit
 
       - name: Integration Tests
         run: |
-          pytest -n 5 tests/integration
+          pytest --cov --cov-append -n 5 tests/integration
+
+      - name: Coverage
+        run: |
+          coverage report
+          bash <(curl -s https://codecov.io/bash)

.github/workflows/test_functional.yml

@@ -27,6 +27,7 @@ jobs:
       BASE: ${{ matrix.tests[0] }}
       CHUNK: ${{ matrix.tests[1] }}
       CYLC_TEST_PLATFORMS: ${{ matrix.tests[2] }}
+      CYLC_COVERAGE: 1
     steps:
       - name: Checkout
         uses: actions/checkout@v2
@@ -48,7 +49,7 @@ jobs:
       - name: Install
         run: |
           pip install git+https://github.com/metomi/rose@master
-          pip install ."[all]"
+          pip install -e ."[all]"
           pip install --no-deps git+https://github.com/cylc/cylc-rose.git@master
           mkdir "$HOME/cylc-run"
 
@@ -100,11 +101,6 @@ jobs:
                   --state=save,failed
           )
 
-      - name: Shutdown
-        if: always()
-        run: |
-          etc/bin/swarm kill
-
       - name: Debug
         if: failure()
         run: |
@@ -127,3 +123,35 @@ jobs:
         with:
           name: Upload cylc-run artifact
           path: cylc-run
+
+      - name: Fetch Remote Coverage
+        run: |
+          if [[ "${{ matrix.tests[2] }}" = _remote* ]]; then
+            host="$(cut -d ' ' -f 1 <<< "${{ matrix.tests[2] }}")"
+            # copy back the remote coverage files
+            rsync -av \
+              "${host}:/cylc/" \
+              '.' \
+              --include='.coverage*' \
+              --exclude='*' \
+              >rsyncout
+            cat rsyncout
+            # fiddle the python source location to match the local system
+            for db in $(grep --color=never '.coverage\.' rsyncout); do
+              sqlite3 "$db" "
+                UPDATE file
+                SET path = REPLACE(path, '/cylc/cylc/', '$PWD/cylc/')
+              "
+            done
+          fi
+
+      - name: Shutdown
+        if: always()
+        run: |
+          etc/bin/swarm kill
+
+      - name: Coverage
+        run: |
+          coverage combine -a
+          coverage report
+          bash <(curl -s https://codecov.io/bash)

.gitignore

@@ -23,7 +23,7 @@ __pycache__/
 
 # coverage
 .coverage
-.coverage.*
+.coverage*
 coverage.xml
 htmlcov/
 

cylc/flow/job_file.py

@@ -182,7 +182,12 @@ def _write_prelude(self, handle, job_conf):
         if cylc.flow.flags.debug:
             handle.write("\nexport CYLC_DEBUG=true")
         handle.write("\nexport CYLC_VERSION='%s'" % CYLC_VERSION)
-        for key in job_conf['platform']['copyable environment variables']:
+        env_vars = (
+            (job_conf['platform']['copyable environment variables'] or [])
+            # pass CYLC_COVERAGE into the job execution environment
+            + ['CYLC_COVERAGE']
+        )
+        for key in env_vars:
             if key in os.environ:
                 handle.write("\nexport %s='%s'" % (key, os.environ[key]))
 

cylc/flow/remote.py

@@ -279,11 +279,16 @@ def _construct_ssh_cmd(
 
     # Pass CYLC_VERSION and optionally, CYLC_CONF_PATH & CYLC_UTC through.
     command += ['env', quote(r'CYLC_VERSION=%s' % CYLC_VERSION)]
-    try:
-        command.append(
-            quote(r'CYLC_CONF_PATH=%s' % os.environ['CYLC_CONF_PATH']))
-    except KeyError:
-        pass
+
+    for envvar in [
+        'CYLC_CONF_PATH',
+        'CYLC_COVERAGE'
+    ]:
+        if envvar in os.environ:
+            command.append(
+                quote(f'{envvar}={os.environ[envvar]}')
+            )
+
     if set_UTC and os.getenv('CYLC_UTC') in ["True", "true"]:
         command.append(quote(r'CYLC_UTC=True'))
         command.append(quote(r'TZ=UTC'))

cylc/flow/scripts/cylc.py

@@ -16,6 +16,7 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """cylc main entry point"""
 
+from contextlib import contextmanager
 import os
 import sys
 from pathlib import Path
@@ -362,74 +363,170 @@ def cli_version(long=False):
     sys.exit(0)
 
 
+@contextmanager
+def pycoverage(cmd_args):
+    """Capture code coverage if configured to do so.
+
+    This requires Cylc to be installed in editible mode
+    (i.e. `pip install -e`) in order to access the coverage configuration
+    file, etc.
+
+    $ pip install -e /cylc/working/directory
+
+    Set the CYLC_COVERAGE env var as appropriate before running tests
+
+    $ export CYLC_COVERAGE=1
+
+    Coverage files will be written out to the working copy irrespective
+    of where in the filesystem the `cylc` command was run.
+
+    $ cd /cylc/working/directory
+    $ coverage combine
+    $ coverage report
+
+    For remote tasks the coverage files will be written to the cylc
+    working directory on the remote so you will have to scp them back
+    to your local working directory before running coverage combine:
+
+    $ cd /cylc/working/directory
+    $ ssh remote-host cd /cylc/remote/working/directory && coverage combine
+    $ scp \
+    >     remote-host/cylc/remote/working/directory/.coverage \
+    >    .coverage.remote-host.12345.12345
+    $ coverage combine
+    $ coverage report
+
+    Environment Variables:
+        CYLC_COVERAGE:
+            '0'
+                Do nothing / run as normal.
+            '1'
+                Collect coverage data.
+            '2'
+                Collect coverage data and log every command for which
+                coverage data was successfully recorded to
+                a .coverage_commands_captured file in the Cylc
+                working directory.
+
+    """
+    cylc_coverage = os.environ.get('CYLC_COVERAGE')
+    if cylc_coverage not in ('1', '2'):
+        yield
+        return
+
+    # import here to avoid unnecessary imports when not running coverage
+    import cylc.flow
+    import coverage
+    from pathlib import Path
+
+    # the cylc working directory
+    cylc_wc = Path(cylc.flow.__file__).parents[2]
+
+    # intiate coverage
+    try:
+        cov = coverage.Coverage(
+            # NOTE: coverage paths are all relative so we must hack them here
+            # to absolute values, otherwise when `cylc` scripts are run
+            # elsewhere on the filesystem they will fail to capture coverage
+            # data and will dump empty coverage files where they run.
+            config_file=str(cylc_wc / '.coveragerc'),
+            data_file=str(cylc_wc / '.coverage'),
+            source=[str(cylc_wc / 'cylc')]
+        )
+    except coverage.misc.CoverageException:
+        raise Exception(
+            # make sure this exception is visible in the traceback
+            '\n\n*****************************\n\n'
+            'Could not initiate coverage, likely because Cylc was not '
+            'installed in editible mode.'
+            '\n\n*****************************\n\n'
+        )
+
+    # start the coverage running
+    cov.start()
+    try:
+        # yield control back to cylc, return once the command exits
+        yield
+    finally:
+        # stop the coverage and save the data
+        cov.stop()
+        cov.save()
+        if cylc_coverage == '2':
+            with open(cylc_wc / '.coverage_commands_captured', 'a+') as ccc:
+                ccc.write(
+                    '$ cylc ' + (' '.join(cmd_args) + '\n'),
+                )
+
+
 @click.command(context_settings={'ignore_unknown_options': True})
 @click.option("--help", "-h", "help_", is_flag=True, is_eager=True)
 @click.option("--version", "-V", is_flag=True, is_eager=True)
 @click.argument("cmd-args", nargs=-1)
 def main(cmd_args, version, help_):
-    if not cmd_args:
-        if version:
-            cli_version()
-        else:
-            cli_help()
-    else:
-        cmd_args = list(cmd_args)
-        command = cmd_args.pop(0)
-
-        if command == "version":
-            cli_version("--long" in cmd_args)
-
-        if command == "help":
-            help_ = True
-            if not len(cmd_args):
-                cli_help()
-            elif cmd_args == ['all']:
-                print_command_list()
-                sys.exit(0)
+    with pycoverage(cmd_args):
+        if not cmd_args:
+            if version:
+                cli_version()
             else:
-                command = cmd_args.pop(0)
-
-        if command in ALIASES:
-            # this is an alias to a command
-            command = ALIASES[command]
-
-        if command in DEAD_ENDS:
-            # this command has been removed but not aliased
-            # display a helpful message and move on#
-            print(
-                cparse(
-                    f'<red>{DEAD_ENDS[command]}</red>'
-                )
-            )
-            sys.exit(42)
-
-        if command not in COMMANDS:
-            # check if this is a command abbreviation or exit
-            command = match_command(command)
-
-        if command == "graph-diff":
-            if len(cmd_args) > 2:
-                for arg in cmd_args[2:]:
-                    if arg.startswith("-"):
-                        cmd_args.insert(cmd_args.index(arg), "--")
-                        break
-        elif command == "jobs-submit":
-            if len(cmd_args) > 1:
-                for arg in cmd_args:
-                    if not arg.startswith("-"):
-                        cmd_args.insert(cmd_args.index(arg) + 1, "--")
-                        break
-        elif command == "message":
-            if cmd_args:
-                if cmd_args[0] in ['-s', '--severity', '-p', '--priority']:
-                    dd_index = 2
-                else:
-                    dd_index = 0
-                cmd_args.insert(dd_index, "--")
-
-        if help_:
-            execute_cmd(command, "--help")
+                cli_help()
         else:
-            if version:
-                cmd_args.append("--version")
-            execute_cmd(command, *cmd_args)
+            cmd_args = list(cmd_args)
+            command = cmd_args.pop(0)
+
+            if command == "version":
+                cli_version("--long" in cmd_args)
+
+            if command == "help":
+                help_ = True
+                if not len(cmd_args):
+                    cli_help()
+                elif cmd_args == ['all']:
+                    print_command_list()
+                    sys.exit(0)
+                else:
+                    command = cmd_args.pop(0)
+
+            if command in ALIASES:
+                # this is an alias to a command
+                command = ALIASES[command]
+
+            if command in DEAD_ENDS:
+                # this command has been removed but not aliased
+                # display a helpful message and move on#
+                print(
+                    cparse(
+                        f'<red>{DEAD_ENDS[command]}</red>'
+                    )
+                )
+                sys.exit(42)
+
+            if command not in COMMANDS:
+                # check if this is a command abbreviation or exit
+                command = match_command(command)
+
+            if command == "graph-diff":
+                if len(cmd_args) > 2:
+                    for arg in cmd_args[2:]:
+                        if arg.startswith("-"):
+                            cmd_args.insert(cmd_args.index(arg), "--")
+                            break
+            elif command == "jobs-submit":
+                if len(cmd_args) > 1:
+                    for arg in cmd_args:
+                        if not arg.startswith("-"):
+                            cmd_args.insert(cmd_args.index(arg) + 1, "--")
+                            break
+            elif command == "message":
+                if cmd_args:
+                    if cmd_args[0] in ['-s', '--severity', '-p', '--priority']:
+                        dd_index = 2
+                    else:
+                        dd_index = 0
+                    cmd_args.insert(dd_index, "--")
+
+            if help_:
+                execute_cmd(command, "--help")
+            else:
+                if version:
+                    cmd_args.append("--version")
+                execute_cmd(command, *cmd_args)