diff --git a/.flake8 b/.flake8 deleted file mode 100644 index 612b95fc8..000000000 --- a/.flake8 +++ /dev/null @@ -1,10 +0,0 @@ -[flake8] -# Exceptions: -# - message_ix/core.py:716:5: C901 'Scenario.rename' is too complex (18) -# - message_ix/tools/add_year/__init__.py:86:1: C901 'add_year' is too complex (17) -# - message_ix/tools/add_year/__init__.py:533:1: C901 'interpolate_1d' is too complex (19) -# - message_ix/tools/add_year/__init__.py:687:1: C901 'interpolate_2d' is too complex (38) -# - message_ix/testing/__init__.py:91:1: C901 'make_austria' is too complex (18) -max-complexity = 14 -# For black -max-line-length = 88 diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml deleted file mode 100644 index b83451ad5..000000000 --- a/.github/workflows/lint.yaml +++ /dev/null @@ -1,27 +0,0 @@ -name: Lint - -on: - push: - branches: [ main ] - pull_request: - branches: [ main ] - schedule: - # 05:00 UTC = 06:00 CET = 07:00 CEST - - cron: "0 5 * * *" - -jobs: - lint: - uses: iiasa/actions/.github/workflows/lint.yaml@main - with: - # If the "Latest version testable on GitHub Actions" in pytest.yaml - # is not the latest 3.x stable version, adjust here to match: - python-version: "3.10" - # For PRs that depend on an ixmp PR branch, adjust the URL below - type-hint-packages: >- - asyncssh - genno - pytest - sphinx - types-requests - types-PyYAML - git+https://github.com/iiasa/ixmp.git@main diff --git a/.github/workflows/pytest.yaml b/.github/workflows/pytest.yaml index 6cd92c94b..80945a0b4 100644 --- a/.github/workflows/pytest.yaml +++ b/.github/workflows/pytest.yaml @@ -132,3 +132,19 @@ jobs: - name: Upload test coverage to Codecov.io uses: codecov/codecov-action@v3 + + pre-commit: + name: Code quality + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + + - name: Force recreation of pre-commit virtual environment for mypy + if: github.event_name == 'schedule' # Comment this line to run on a PR + run: gh cache list -L 999 | cut -f2 | grep pre-commit | xargs -I{} gh cache delete "{}" || true + env: { GH_TOKEN: "${{ github.token }}" } + + - uses: pre-commit/action@v3.0.0 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 000000000..ae0b01c5c --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,28 @@ +repos: +- repo: local + hooks: + - id: mypy + name: mypy + always_run: true + require_serial: true + pass_filenames: false + + language: python + entry: bash -c ". ${PRE_COMMIT_MYPY_VENV:-/dev/null}/bin/activate 2>/dev/null; mypy $0 $@" + additional_dependencies: + - mypy + - asyncssh + - git+https://github.com/iiasa/ixmp.git@main + - pytest + - Sphinx + - types-PyYAML + - types-requests + args: ["."] +- repo: https://github.com/psf/black + rev: 23.7.0 + hooks: + - id: black +- repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.0.287 + hooks: + - id: ruff diff --git a/doc/contributing.rst b/doc/contributing.rst index 7666b7391..d05375b52 100644 --- a/doc/contributing.rst +++ b/doc/contributing.rst @@ -97,6 +97,8 @@ Optionally: __ https://help.github.com/en/github/collaborating-with-issues-and-pull-requests/about-pull-requests#draft-pull-requests +.. _ci-workflows: + 3. Ensure checks pass --------------------- @@ -111,12 +113,15 @@ __ https://help.github.com/en/github/collaborating-with-issues-and-pull-requests pytest This workflow runs all Python and R tests; on Linux, macOS, and Windows; and for multiple versions of Python. - lint - This workflow checks for code style and other details: + It also: + + - Checks that the documentation can be built without fatal errors. + - Checks that the `code style`_ is applied. - - “Lint with flake8”: checks that `Code style`_ is met. - - “Test package build”: checks that the Python package for upload to PyPI, can be built cleanly and without errors. - - “Test documentation build”: checks that the documentation can be built without fatal errors. + publish + This workflow checks that the Python package (for upload to PyPI) can be built cleanly and without errors. + + The package is not actually uploaded, unless this workflow is started from a release candidate tag or on the creation of a new release on GitHub. nightly These tests run daily at 05:00 UTC. @@ -182,23 +187,37 @@ Code style - **Python** code: - Follow the `PEP 8 naming conventions `_. + - Apply `black `_ code formatting. + - Use `ruff `_ to check code quality. + In particular, through :file:`pyproject.toml`, :mod:`message_ix` uses the following rule sets to ensure: - - Apply the following to all code:: + - `"F" `_: code is free of basic errors, equivalent to Pyflakes or `flake8 `_. + - `"E", "W" `_: code conforms to `PEP 8 `_, equivalent to using pycodestyle. + - `"I" `_: :py:`import` statements are sorted in a consistent way, equivalent to `isort `_. + - `"C90" `_: the McCabe complexity of code is below a fixed threshold, equivalent to using `mccabe `_ via flake8. - isort -rc . && black . && mypy . && flake8 + - Add type hints to new or changed functions, methods, and global variables, and check these using the `mypy `_ static type checker. - Links to the documentation for these tools: + To simplify the use of these tools: - - `isort `_: sorts import lines at the top of code files in a consistent way. - - `black `_: applies consistent code style & formatting. - Plugins are available for popular code editors. - - `mypy `_: checks typing for inconsistencies. - - `flake8 `_: check code style against `PEP 8 `_. + - Black, ruff, and mypy can each be configured to run automatically within your code editor with an extension, plugin, or script (see their respective documentation for links and details). + These tools help apply the code style every time a file is saved, or even as you type. + - The source repository contains configuration for `pre-commit `_, a tool that invokes multiple actions via `git hooks `_. + This runs all of the above checks every time you do a :program:`git commit`. + To use this tool, install :program:`pre-commit` and install it in your local checkout of the Git repository: - The ``lint`` continuous integration workflow runs these on every pull request. - PRs that fail the checks must be corrected before they can be merged. + .. code-block:: - - Add type hints to new or changed functions, methods, and global variables. + pip install pre-commit + pre-commit install -f + + # Run the tools on all files to confirm they are working + pre-commit run --all-files + + To force mypy type checking to use packages from an existing `Python virtual environment `_ on your system (for instance, with development code), set the ``PRE_COMMIT_MYPY_VENV`` environment variable to the path to that environment. + + - The "Code quality" job in the "pytest" workflow :ref:`described above ` applies exactly the same checks for PR branches. + PRs that fail the checks must be corrected before they can be merged. - **GAMS** code: @@ -220,7 +239,7 @@ Documentation 2. Documentation pages, :file:`doc/*.rst`. 3. Inline documentation in :file:`message_ix/model/*.gms` files. - For (2) and (3), start each sentence on a new line, and do not hard-wrap sentences. + For (2) and (3), start each sentence on a new line, and do not hard-wrap within sentences. For (1), wrap at the same 88 characters as :command:`black` enforces for code. - Ensure Sphinx does not give warnings about ReST syntax for new or modified documentation. diff --git a/message_ix/reporting/__init__.py b/message_ix/reporting/__init__.py index 660bf8e66..60f475601 100644 --- a/message_ix/reporting/__init__.py +++ b/message_ix/reporting/__init__.py @@ -10,9 +10,9 @@ KeyExistsError, MissingKeyError, Quantity, + configure, ) from ixmp.reporting import Reporter as IXMPReporter -from ixmp.reporting import configure from .pyam import collapse_message_cols diff --git a/message_ix/testing/__init__.py b/message_ix/testing/__init__.py index 5f975e23d..cc7f4ffe4 100644 --- a/message_ix/testing/__init__.py +++ b/message_ix/testing/__init__.py @@ -513,7 +513,7 @@ def make_westeros( grid_efficiency = 0.9 common.update(unit="-") - for name, tec, c, l, value in [ + for name, tec, c, L, value in [ ("input", "bulb", "electricity", "final", 1.0), ("output", "bulb", "light", "useful", 1.0), ("input", "grid", "electricity", "secondary", 1.0), @@ -523,7 +523,7 @@ def make_westeros( ]: scen.add_par( name, - make_df(name, **common, technology=tec, commodity=c, level=l, value=value), + make_df(name, **common, technology=tec, commodity=c, level=L, value=value), ) name = "capacity_factor" diff --git a/pyproject.toml b/pyproject.toml index dbd4ce760..9c1dc805d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -77,9 +77,6 @@ omit = [ "message_ix/testing/nightly.py", ] -[tool.isort] -profile = "black" - [tool.mypy] exclude = ["doc/"] @@ -108,6 +105,18 @@ markers = [ "rmessageix: test of the message_ix R interface.", ] +[tool.ruff] +select = ["C9", "E", "F", "I", "W"] + +[tool.ruff.mccabe] +# Exceptions: +# - message_ix/core.py:716:5: C901 'Scenario.rename' is too complex (18) +# - message_ix/tools/add_year/__init__.py:86:1: C901 'add_year' is too complex (17) +# - message_ix/tools/add_year/__init__.py:533:1: C901 'interpolate_1d' is too complex (19) +# - message_ix/tools/add_year/__init__.py:687:1: C901 'interpolate_2d' is too complex (38) +# - message_ix/testing/__init__.py:91:1: C901 'make_austria' is too complex (18) +max-complexity = 14 + [tool.setuptools.packages] find = {}