Diagnose timeouts in pytest CI jobs (#1158)

.github/workflows/checks.yml

@@ -52,46 +52,24 @@ jobs:
         run: |
           black --check .
 
-  pylint:
-    name: Code linting (pylint)
-    runs-on: ubuntu-latest
-    needs: [code-formatting]
-    steps:
-      - uses: actions/checkout@v3
-      - uses: actions/setup-python@v4
-        with:
-          python-version: '3.8'
-      - name: Install dev dependencies
-        run: |
-          pip install -r requirements-dev.txt
-          pip list
-      - name: Run pylint
-        run: |
-          pylint watertap
-
   tests:
-    name: Tests (py${{ matrix.python-version }}/${{ matrix.os }})
+    name: Tests (py${{ matrix.python-version }}/${{ matrix.os }}/rep=${{ matrix.rep }})
     runs-on: ${{ matrix.os-version }}
     needs: [code-formatting]
     strategy:
       fail-fast: false
       matrix:
+        rep: ["0", "1", "2", "3", "4", "5", "6", "7", "8", "9"]
         python-version:
           - '3.8'
           - '3.9'
           - '3.10'
           - '3.11'
         os:
           - linux
-          - win64
-          # - macos
         include:
           - os: linux
             os-version: ubuntu-20.04
-          - os: win64
-            os-version: windows-2019
-          # - os: macos
-          #   os-version: macos-10.15
           - python-version: '3.8'
             # limit uploading coverage report for a single Python version in the matrix
             cov_report: true
@@ -110,9 +88,6 @@ jobs:
         echo '::group::Output of "pip install" commands'
         pip install -r requirements-dev.txt
         echo '::endgroup::'
-        echo '::group::Output of "pip install -U ray" command'
-        pip install -U ray
-        echo '::endgroup::'
         echo '::group::Display installed packages'
         conda list
         pip list
@@ -125,159 +100,8 @@ jobs:
       if: matrix.cov_report
       run:
         |
-        echo PYTEST_ADDOPTS="$PYTEST_ADDOPTS --cov-report=xml" >> $GITHUB_ENV
-    - name: Test with pytest
-      run: |
-        pytest --pyargs watertap
-    - name: Upload coverage report to Codecov
-      if: matrix.cov_report
-      uses: codecov/codecov-action@v2
-    - name: Test documentation code
-      run: |
-        make -C docs doctest -d
-    # TODO: this should be moved to a dedicated job/workflow
-    # until then, we can leave this here as a reminder
-    - name: Test documentation links
-      if: 'false'
-      run: |
-        make -C docs linkcheck -d
-
-  user-mode-pytest:
-    name: pytest (user mode) (py${{ matrix.python-version }}/${{ matrix.os }})
-    runs-on: ${{ matrix.os-version }}
-    needs: [code-formatting]
-    strategy:
-      fail-fast: false
-      matrix:
-        python-version:
-          - '3.8'
-          - '3.11'
-        os:
-          - linux
-          - win64
-        include:
-          - os: linux
-            os-version: ubuntu-20.04
-          - os: win64
-            os-version: windows-2019
-    steps:
-    - name: Set up Python ${{ matrix.python-version }}
-      uses: conda-incubator/setup-miniconda@v2
-      with:
-        activate-environment: watertap
-        python-version: ${{ matrix.python-version }}
-    - name: Define install URL (default)
-      env:
-        _repo_full_name: watertap-org/watertap
-        _ref_to_install: main
-      run: |
-        echo "_install_url=https://github.com/${_repo_full_name}/archive/${_ref_to_install}.zip" >> $GITHUB_ENV
-    - name: Define install URL (for PRs)
-      if: github.event.pull_request
-      env:
-        _repo_full_name: ${{ github.event.pull_request.head.repo.full_name }}
-        _ref_to_install: ${{ github.event.pull_request.head.sha }}
-      run:
-        echo "_install_url=https://github.com/${_repo_full_name}/archive/${_ref_to_install}.zip" >> $GITHUB_ENV
-    - name: Install watertap and testing dependencies
-      run: |
-        echo '::group::Output of "pip install" commands'
-        pip install "watertap @ ${_install_url}" pytest
-        echo '::endgroup::'
-        echo '::group::Display installed packages'
-        conda list
-        pip list
-        pip show idaes-pse
-        echo '::endgroup::'
-        echo '::group::Output of "idaes get-extensions" command'
-        idaes get-extensions --verbose
-        echo '::endgroup::'
-    - name: Run pytest
-      run: |
-        pytest --pyargs watertap
-
-  notebooks:
-    name: Test notebooks (py${{ matrix.python-version }}/${{ matrix.os }})
-    runs-on: ${{ matrix.os-version }}
-    needs: [code-formatting]
-    strategy:
-      fail-fast: false
-      matrix:
-        python-version:
-          - '3.8'
-          - '3.11'
-        os:
-          - linux
-          - win64
-        include:
-          - os: linux
-            os-version: ubuntu-20.04
-          - os: win64
-            os-version: windows-2019
-    steps:
-    - uses: actions/checkout@v3
-    - name: Set up Python ${{ matrix.python-version }}
-      uses: conda-incubator/setup-miniconda@v2
-      with:
-        activate-environment: watertap-dev
-        python-version: ${{ matrix.python-version }}
-    - name: Install dependencies
-      run: |
-        echo '::group::Output of "conda install" commands'
-        conda install --quiet --yes pip setuptools wheel pandoc
-        echo '::endgroup::'
-        echo '::group::Output of "pip install" commands'
-        pip install -r requirements-dev.txt
-        echo '::endgroup::'
-        echo '::group::Display installed packages'
-        conda list
-        pip list
-        pip show idaes-pse
-        echo '::endgroup::'
-        echo '::group::Output of "idaes get-extensions" command'
-        idaes get-extensions --verbose
-        echo '::endgroup::'
-    - name: Run pytest with nbmake
-      run:
-        pytest --nbmake **/*.ipynb
-
-  macos:
-    name: macOS setup (EXPERIMENTAL)
-    runs-on: macos-11
-    needs: [code-formatting]
-    strategy:
-      fail-fast: false
-      matrix:
-        python-version:
-          - '3.8'
-    steps:
-    - uses: actions/checkout@v3
-    - name: Set up Python ${{ matrix.python-version }}
-      uses: conda-incubator/setup-miniconda@v2
-      with:
-        activate-environment: watertap
-        python-version: ${{ matrix.python-version }}
-    - name: Install WaterTAP (dev) without idaes get-extensions
-      run: |
-        echo '::group::Output of "conda install" commands'
-        conda install --quiet --yes pip=21.1 wheel setuptools pandoc
-        echo '::endgroup::'
-        echo '::group::Output of "pip install" commands'
-        pip install -r requirements-dev.txt
-        echo '::endgroup::'
-        echo '::group::Display installed packages'
-        conda list
-        pip list
-        pip show pyomo idaes-pse
-        echo '::endgroup::'
-    - name: Install Ipopt from conda-forge
-      run:
-        conda install --quiet --yes -c conda-forge ipopt=3.14.11
-    - name: Build Pyomo extensions
-      run: |
-        conda install --quiet --yes cmake
-        # some failures are expected, but this should succeed as long as pynumero is built correctly
-        pyomo build-extensions || python -c "from pyomo.contrib.pynumero.asl import AmplInterface; exit(0) if AmplInterface.available() else exit(1)"
-    - name: Run pytest
+        echo PYTEST_ADDOPTS="$PYTEST_ADDOPTS --cov-report=xml --cov-append" >> $GITHUB_ENV
+    - name: Test with pytest (tools)
       run: |
-        pytest --pyargs watertap -k 'not (nf_dspmde.nf_ui or nf_dspmde.nf_with_bypass_ui)'
+        timeout -s SIGINT 3m pytest --pyargs watertap -m 'tools' --verbose --capture=no --full-trace
+      timeout-minutes: 5

watertap/conftest.py

@@ -10,16 +10,26 @@
 # "https://github.com/watertap-org/watertap/"
 #################################################################################
 import contextlib
+import datetime
 import enum
 from pathlib import Path
-from typing import Container, Optional, Callable
+from typing import Container, Optional, Callable, List
 
 import pytest
 from _pytest.nodes import Item
 from _pytest.config import Config
 from _pytest.config.argparsing import Parser
 
 
+START = datetime.datetime.now()
+
+
+def _get_elapsed_seconds(end=None) -> float:
+    end = end or datetime.datetime.now()
+    delta = end - START
+    return delta.total_seconds()
+
+
 class MarkerSpec(enum.Enum):
     unit = "Quick tests that do not require a solver, must run in < 2 s"
     component = "Quick tests that may require a solver"
@@ -29,6 +39,7 @@ class MarkerSpec(enum.Enum):
     requires_idaes_solver = (
         "Tests that require a solver from the IDEAS extensions to pass"
     )
+    tools = "Tests pertaining to WaterTAP tools"
 
     @property
     def description(self) -> str:
@@ -83,3 +94,28 @@ def pytest_addoption(parser: Parser):
         default=False,
         dest="edb_no_mock",
     )
+
+
+def pytest_collection_modifyitems(items: List[Item]):
+    for item in items:
+        if "watertap/tools" in str(item.path):
+            item.add_marker("tools")
+
+
+@pytest.hookimpl(hookwrapper=True)
+def pytest_runtest_protocol(item):
+    import psutil
+
+    this_process = psutil.Process()
+
+    print(f"START {item}")
+    print(f"{_get_elapsed_seconds()=}")
+    print("processes before:")
+    for proc in this_process.children(recursive=True):
+        print(f"\t{proc}")
+    yield
+    print(f"\nprocesses after:")
+    for proc in this_process.children(recursive=True):
+        print(f"\t{proc}")
+    print(f"{_get_elapsed_seconds()=}")
+    print(f"END {item}")