From 849331009db6fd25b76f2149ad9e15608d0a778a Mon Sep 17 00:00:00 2001 From: Bill Little Date: Thu, 8 Jul 2021 23:04:29 +0100 Subject: [PATCH 1/7] pre-commit isort and black --check only for cirrus-ci --- .cirrus.yml | 2 +- .pre-commit-config.yaml | 4 ++-- noxfile.py | 24 ++++++++++++++++++++++-- 3 files changed, 25 insertions(+), 5 deletions(-) diff --git a/.cirrus.yml b/.cirrus.yml index 52abddcadd..326cfd74d9 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -34,7 +34,7 @@ env: # Increment the build number to force new pip cache upload. PIP_CACHE_BUILD: "0" # Pip packages to be upgraded/installed. - PIP_CACHE_PACKAGES: "nox pip setuptools wheel" + PIP_CACHE_PACKAGES: "nox pip pyyaml setuptools wheel" # Conda packages to be installed. CONDA_CACHE_PACKAGES: "nox pip" # Git commit hash for iris test data. diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index abcca5ecbe..4574d09b77 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -32,7 +32,7 @@ repos: hooks: - id: black pass_filenames: false - args: [--config=./pyproject.toml, --check, .] + args: [--config=./pyproject.toml, .] - repo: https://github.com/PyCQA/flake8 rev: 3.9.2 @@ -46,7 +46,7 @@ repos: hooks: - id: isort types: [file, python] - args: [--filter-files, --check] + args: [--filter-files] - repo: https://github.com/asottile/blacken-docs rev: v1.10.0 diff --git a/noxfile.py b/noxfile.py index 1570f62532..14b35e1e41 100755 --- a/noxfile.py +++ b/noxfile.py @@ -8,6 +8,8 @@ import hashlib import os from pathlib import Path +from tempfile import NamedTemporaryFile +from urllib.parse import unquote, urlsplit import nox from nox.logger import logger @@ -173,10 +175,28 @@ def lint(session: nox.sessions.Session): A `nox.sessions.Session` object. """ + import yaml + # Pip install the session requirements. session.install("pre-commit") - # Execute the pre-commit linting tasks. - cmd = ["pre-commit", "run", "--all-files"] + + # Load the pre-commit configuration YAML file. + with open(".pre-commit-config.yaml", "r") as fi: + config = yaml.load(fi, Loader=yaml.FullLoader) + + # Ensure black and isort only perform a status check + # with a custom pre-commit configuration. + for entry in config["repos"]: + package = Path(urlsplit(unquote(entry["repo"])).path).name + if package in ["black", "isort"]: + entry["hooks"][0]["args"].insert(0, "--check") + + with NamedTemporaryFile(mode="w+t", delete=False) as temp: + yaml.dump(config, temp) + + # Execute the pre-commit linting tasks with the custom + # pre-commit configuration. + cmd = ["pre-commit", "run", "--all-files", f"--config={temp.name}"] hooks = ["black", "blacken-docs", "flake8", "isort"] for hook in hooks: session.run(*cmd, hook) From 143130bff6093c4489ec9e057821a7a01628946b Mon Sep 17 00:00:00 2001 From: Bill Little Date: Thu, 8 Jul 2021 23:52:44 +0100 Subject: [PATCH 2/7] run discovered hooks, with exclusion --- noxfile.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/noxfile.py b/noxfile.py index 14b35e1e41..0660d0faf4 100755 --- a/noxfile.py +++ b/noxfile.py @@ -186,9 +186,12 @@ def lint(session: nox.sessions.Session): # Ensure black and isort only perform a status check # with a custom pre-commit configuration. + hooks = [] for entry in config["repos"]: - package = Path(urlsplit(unquote(entry["repo"])).path).name - if package in ["black", "isort"]: + hook = Path(urlsplit(unquote(entry["repo"])).path).name + hooks.append(hook) + if hook in ["black", "isort"]: + # Enforce a "--check" for the hook. entry["hooks"][0]["args"].insert(0, "--check") with NamedTemporaryFile(mode="w+t", delete=False) as temp: @@ -197,9 +200,9 @@ def lint(session: nox.sessions.Session): # Execute the pre-commit linting tasks with the custom # pre-commit configuration. cmd = ["pre-commit", "run", "--all-files", f"--config={temp.name}"] - hooks = ["black", "blacken-docs", "flake8", "isort"] for hook in hooks: - session.run(*cmd, hook) + if hook != "pre-commit-hooks": + session.run(*cmd, hook) @nox.session(python=PY_VER, venv_backend="conda") From 604a22248a17fb87878af51c4abf50c7d3665ed2 Mon Sep 17 00:00:00 2001 From: Bill Little Date: Fri, 9 Jul 2021 11:28:39 +0100 Subject: [PATCH 3/7] review actions --- noxfile.py | 26 ++++++++------------------ 1 file changed, 8 insertions(+), 18 deletions(-) diff --git a/noxfile.py b/noxfile.py index 0660d0faf4..e46a4ac2ce 100755 --- a/noxfile.py +++ b/noxfile.py @@ -8,7 +8,6 @@ import hashlib import os from pathlib import Path -from tempfile import NamedTemporaryFile from urllib.parse import unquote, urlsplit import nox @@ -184,25 +183,16 @@ def lint(session: nox.sessions.Session): with open(".pre-commit-config.yaml", "r") as fi: config = yaml.load(fi, Loader=yaml.FullLoader) - # Ensure black and isort only perform a status check - # with a custom pre-commit configuration. - hooks = [] - for entry in config["repos"]: - hook = Path(urlsplit(unquote(entry["repo"])).path).name - hooks.append(hook) - if hook in ["black", "isort"]: - # Enforce a "--check" for the hook. - entry["hooks"][0]["args"].insert(0, "--check") - - with NamedTemporaryFile(mode="w+t", delete=False) as temp: - yaml.dump(config, temp) - - # Execute the pre-commit linting tasks with the custom - # pre-commit configuration. - cmd = ["pre-commit", "run", "--all-files", f"--config={temp.name}"] + # Enumerate the pre-commit hooks. + hooks = [ + Path(urlsplit(unquote(entry["repo"])).path).name + for entry in config["repos"] + ] + + # Execute the pre-commit linting hooks. for hook in hooks: if hook != "pre-commit-hooks": - session.run(*cmd, hook) + session.run("pre-commit", "run", "--all-files", hook) @nox.session(python=PY_VER, venv_backend="conda") From f789b362c2825426406ad96eea8f0520b0b2b402 Mon Sep 17 00:00:00 2001 From: Bill Little Date: Fri, 9 Jul 2021 13:55:34 +0100 Subject: [PATCH 4/7] match on pre-commit hook id --- noxfile.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/noxfile.py b/noxfile.py index e46a4ac2ce..e9cf272d29 100755 --- a/noxfile.py +++ b/noxfile.py @@ -8,7 +8,6 @@ import hashlib import os from pathlib import Path -from urllib.parse import unquote, urlsplit import nox from nox.logger import logger @@ -183,16 +182,15 @@ def lint(session: nox.sessions.Session): with open(".pre-commit-config.yaml", "r") as fi: config = yaml.load(fi, Loader=yaml.FullLoader) - # Enumerate the pre-commit hooks. + # Enumerate singleton pre-commit hooks. hooks = [ - Path(urlsplit(unquote(entry["repo"])).path).name + entry["hooks"][0]["id"] for entry in config["repos"] + if len(entry["hooks"]) == 1 ] # Execute the pre-commit linting hooks. - for hook in hooks: - if hook != "pre-commit-hooks": - session.run("pre-commit", "run", "--all-files", hook) + [session.run("pre-commit", "run", "--all-files", hook) for hook in hooks] @nox.session(python=PY_VER, venv_backend="conda") From 1d9edbf3123511dc77ed79ba24283937e1f5e5e8 Mon Sep 17 00:00:00 2001 From: Bill Little Date: Fri, 9 Jul 2021 14:34:43 +0100 Subject: [PATCH 5/7] add clarifying comment to nox --- noxfile.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/noxfile.py b/noxfile.py index e9cf272d29..4038f28d61 100755 --- a/noxfile.py +++ b/noxfile.py @@ -182,15 +182,19 @@ def lint(session: nox.sessions.Session): with open(".pre-commit-config.yaml", "r") as fi: config = yaml.load(fi, Loader=yaml.FullLoader) - # Enumerate singleton pre-commit hooks. - hooks = [ + # Enumerate the ids of pre-commit hooks we want to run. + # We're only capturing the hook id of pre-commit repos with *one* + # registered hook. This is a simple approach to filtering out + # the "https://github.com/pre-commit/pre-commit-hooks" hooks, + # all of which we don't want to run within nox. + ids = [ entry["hooks"][0]["id"] for entry in config["repos"] if len(entry["hooks"]) == 1 ] # Execute the pre-commit linting hooks. - [session.run("pre-commit", "run", "--all-files", hook) for hook in hooks] + [session.run("pre-commit", "run", "--all-files", id) for id in ids] @nox.session(python=PY_VER, venv_backend="conda") From ecacdf3e62c922a84be696ee4fd3fddd82363b2c Mon Sep 17 00:00:00 2001 From: Bill Little Date: Fri, 9 Jul 2021 15:00:34 +0100 Subject: [PATCH 6/7] support pre-commit excluded hooks --- noxfile.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/noxfile.py b/noxfile.py index 4038f28d61..0f4152b683 100755 --- a/noxfile.py +++ b/noxfile.py @@ -182,18 +182,18 @@ def lint(session: nox.sessions.Session): with open(".pre-commit-config.yaml", "r") as fi: config = yaml.load(fi, Loader=yaml.FullLoader) - # Enumerate the ids of pre-commit hooks we want to run. - # We're only capturing the hook id of pre-commit repos with *one* - # registered hook. This is a simple approach to filtering out - # the "https://github.com/pre-commit/pre-commit-hooks" hooks, - # all of which we don't want to run within nox. + # List of pre-commit hook ids that we don't want to run. + excluded = ["no-commit-to-branch"] + + # Enumerate the ids of pre-commit hooks we do want to run. ids = [ - entry["hooks"][0]["id"] + hook["id"] for entry in config["repos"] - if len(entry["hooks"]) == 1 + for hook in entry["hooks"] + if hook["id"] not in excluded ] - # Execute the pre-commit linting hooks. + # Execute the pre-commit hooks. [session.run("pre-commit", "run", "--all-files", id) for id in ids] From 18017e6c3c8e0d22bee15d8ea5dd14bdc2953f4a Mon Sep 17 00:00:00 2001 From: Bill Little Date: Fri, 9 Jul 2021 15:12:31 +0100 Subject: [PATCH 7/7] rebrand lint to precommit --- .cirrus.yml | 8 ++++---- noxfile.py | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.cirrus.yml b/.cirrus.yml index 326cfd74d9..b728e31867 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -105,7 +105,7 @@ iris_test_data_template: &IRIS_TEST_DATA_TEMPLATE # # Linting # -lint_task: +precommit_task: only_if: ${SKIP_LINT_TASK} == "" << : *CREDITS_TEMPLATE auto_cancellation: true @@ -113,17 +113,17 @@ lint_task: image: python:3.8 cpu: 2 memory: 4G - name: "${CIRRUS_OS}: linting" + name: "${CIRRUS_OS}: pre-commit hooks" pip_cache: folder: ~/.cache/pip fingerprint_script: - echo "${CIRRUS_TASK_NAME} py${PYTHON_VERSION}" - echo "$(date +%Y).$(expr $(date +%U) / ${CACHE_PERIOD}):${PIP_CACHE_BUILD} ${PIP_CACHE_PACKAGES}" - lint_script: + precommit_script: - pip list - python -m pip install --retries 3 --upgrade ${PIP_CACHE_PACKAGES} - pip list - - nox --session lint + - nox --session precommit # diff --git a/noxfile.py b/noxfile.py index 0f4152b683..f7cd86e09f 100755 --- a/noxfile.py +++ b/noxfile.py @@ -163,9 +163,9 @@ def prepare_venv(session: nox.sessions.Session) -> None: @nox.session -def lint(session: nox.sessions.Session): +def precommit(session: nox.sessions.Session): """ - Perform pre-commit linting of iris codebase. + Perform pre-commit hooks of iris codebase. Parameters ----------