Add pre-commit python hooks & tidy helper scripts

.github/workflows/codeql.yml

@@ -10,6 +10,9 @@ on:
   schedule:
     - cron: "0 5 * * *"
 
+env:
+  python-version: "3.10"
+
 jobs:
   analyze:
     name: Analyze
@@ -23,6 +26,9 @@ jobs:
       matrix:
         language: ["csharp", "java", "python"]
     steps:
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ env.python-version }}
       - name: Checkout repository
         uses: actions/checkout@v4
       - name: Initialize CodeQL
@@ -38,11 +44,11 @@ jobs:
         uses: actions/setup-dotnet@v4.1.0
       - name: Build .Net
         if: ${{ matrix.language == 'csharp' }}
-        run: ./bin/smi/build.py -c Release
+        run: ./bin/smi/smi_build.py -c Release
       - name: Build java
         if: ${{ matrix.language == 'java' }}
         # NOTE(rkm 2023-03-21) Ensure test code is detected but don't actually run anything
-        run: ./bin/ctp/test.py --install-libs -DskipTests
+        run: ./bin/ctp/ctp_test.py --install-libs -DskipTests
       - name: SecurityCodescan
         if: ${{ matrix.language == 'csharp' }}
         run: |

.github/workflows/main.yml

@@ -22,15 +22,18 @@ env:
   java-version: 11
   java-distribution: temurin
   # python
-  python-version: 3.6
+  python-version: "3.10"
 
 jobs:
   init:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-24.04
     outputs:
       matrix: ${{ steps.matrix.outputs.matrix }}
       build_ref: ${{ steps.build_ref.outputs.build_ref }}
     steps:
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ env.python-version }}
       - name: checkout
         uses: actions/checkout@v4
       - name: set matrix
@@ -59,16 +62,19 @@ jobs:
       - name: "[linux] enable disk caching"
         if: ${{ matrix.os == 'linux' }}
         run: sudo apt-get install -y libeatmydata1
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ env.python-version }}
       - name: checkout
         uses: actions/checkout@v4
       - name: setup .NET
         # NOTE(rkm 2022-02-20) Uses global.json
         uses: actions/setup-dotnet@v4.1.0
       - name: download tessdata
-        run: ./bin/smi/downloadTessdata.py
+        run: ./bin/smi/download_tessdata.py
       - name: "[linux] start services"
         if: ${{ matrix.os == 'linux' }}
-        run: ./bin/smi/startDockerLinux.py ${{ env.db-password }}
+        run: ./bin/smi/smi_start_docker_linux.py ${{ env.db-password }}
       - name: "[linux] re-write database strings"
         if: ${{ matrix.os == 'linux' }}
         run: |
@@ -96,7 +102,7 @@ jobs:
           set -euxo pipefail
           cov=""
           [ "${{ matrix.os }}" == "windows" ] && cov="--no-coverage"
-          ./bin/smi/buildTestPackage.py \
+          ./bin/smi/smi_build_test_package.py \
               ${{ needs.init.outputs.build_ref }} \
               "$cov"
       - name: upload coverage to codecov
@@ -126,6 +132,9 @@ jobs:
       matrix: ${{ fromJson(needs.init.outputs.matrix) }}
     runs-on: ${{ matrix.image }}
     steps:
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ env.python-version }}
       - name: "[linux] enable disk caching"
         if: ${{ matrix.os == 'linux' }}
         run: sudo apt-get install -y libeatmydata1
@@ -139,12 +148,12 @@ jobs:
           cache: maven
       - name: "[linux] start services"
         if: ${{ matrix.os == 'linux' }}
-        run: ./bin/ctp/startDockerLinux.py
+        run: ./bin/ctp/ctp_start_docker_linux.py
       - name: "[windows] skip integration tests"
         if: ${{ matrix.os == 'windows' }}
         run: echo "JAVA_TESTS=--skip-integration-tests" >> $GITHUB_ENV
       - name: build, test, and package ctp
-        run: ./bin/ctp/buildTestPackage.py --install-libs ${{ needs.init.outputs.build_ref }} ${{ env.JAVA_TESTS }}
+        run: ./bin/ctp/ctp_build_test_package.py --install-libs ${{ needs.init.outputs.build_ref }} ${{ env.JAVA_TESTS }}
       - name: "[linux] upload packages"
         if: ${{ matrix.os == 'linux' }}
         uses: actions/upload-artifact@v4
@@ -155,7 +164,7 @@ jobs:
   upload-to-release:
     if: contains(github.ref, 'refs/tags/v')
     needs: [init, smi, ctp]
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-24.04
     steps:
       - name: "[linux] enable disk caching"
         run: sudo apt-get install -y libeatmydata1

.meta/flake8.conf

@@ -0,0 +1,8 @@
+[flake8]
+
+# https://github.com/psf/black
+max-line-length = 120
+
+# Ignoring:
+# E203 - Whitespace before ':'
+extend-ignore = E203

.meta/isort.cfg

@@ -0,0 +1,3 @@
+[isort]
+profile = black
+force_single_line = True

.meta/mypy.ini

@@ -0,0 +1,9 @@
+[mypy]
+check_untyped_defs = true
+disallow_any_generics = true
+disallow_incomplete_defs = true
+disallow_untyped_defs = true
+no_implicit_optional = true
+warn_redundant_casts = true
+warn_unused_ignores = true
+mypy_path = ./bin

.pre-commit-config.yaml

@@ -1,4 +1,6 @@
 ---
+default_language_version:
+  python: python3.10
 exclude: |
     (?x)^(
         .*/packages.lock.json|
@@ -74,3 +76,50 @@ repos:
     hooks:
       - id: markdown-link-check
         args: [--quiet, --config, .meta/markdown-link-check.json]
+  # Python
+  - repo: https://github.com/pre-commit/pygrep-hooks
+    rev: "v1.10.0"
+    hooks:
+      - id: python-check-blanket-noqa
+      - id: python-check-mock-methods
+      - id: python-use-type-annotations
+      - id: python-no-log-warn
+  - repo: https://github.com/asottile/add-trailing-comma
+    rev: "v3.1.0"
+    hooks:
+      - id: add-trailing-comma
+  - repo: https://github.com/asottile/pyupgrade
+    rev: "v3.19.0"
+    hooks:
+      - id: pyupgrade
+        args: [--py310-plus]
+  - repo: https://github.com/asottile/yesqa
+    rev: "v1.5.0"
+    hooks:
+      - id: yesqa
+        additional_dependencies:
+        - pep8-naming==0.14.1
+  - repo: https://github.com/pre-commit/mirrors-mypy
+    rev: "v1.13.0"
+    hooks:
+      - id: mypy
+        args: [--config-file, .meta/mypy.ini]
+  - repo: https://github.com/psf/black-pre-commit-mirror
+    rev: "24.10.0"
+    hooks:
+      - id: black
+  - repo: https://github.com/pycqa/isort
+    rev: 5.13.2
+    hooks:
+      - id: isort
+        name: isort (python)
+        args: [--settings=.meta/isort.cfg]
+  - repo: https://github.com/pycqa/flake8
+    rev: "7.1.1"
+    hooks:
+      - id: flake8
+        additional_dependencies:
+        - flake8-bugbear==24.10.31
+        - flake8-assert-msg==1.1.1
+        - pep8-naming==0.14.1
+        args: [--config, .meta/flake8.conf]

bin/checkDocs.py

@@ -1,15 +1,11 @@
 #!/usr/bin/env python3
-
 import collections
-import glob
 import os
 import re
-import sys
-
-import common as C
 
+import common as c
 
-_VERB_CS_PATH = f"{C.PROJ_ROOT}/src/SmiServices/ServiceVerbs.cs"
+_VERB_CS_PATH = f"{c.PROJ_ROOT}/src/SmiServices/ServiceVerbs.cs"
 _VERB_RE = re.compile(r'\[Verb\(("\S*?")')
 
 
@@ -63,7 +59,7 @@ def main() -> int:
 
     for tool_type in ("application", "service"):
         for tool in tools[tool_type]:
-            doc_path = f"{C.PROJ_ROOT}/docs/{tool_type}s/{tool}.md"
+            doc_path = f"{c.PROJ_ROOT}/docs/{tool_type}s/{tool}.md"
             if not os.path.isfile(doc_path):
                 print(f"Missing {doc_path}")
                 rc |= 1

bin/common.py

@@ -1,44 +1,38 @@
 """Common build variables and functions"""
 
 import argparse
-import hashlib
 import functools
+import hashlib
 import os
 import subprocess
-from pathlib import Path
-from typing import Dict
-from typing import Optional
-from typing import Sequence
-from typing import Union
+from collections.abc import Sequence
 
-PROJ_ROOT = (Path(__file__).parent / "..").resolve()
-DIST_DIR = PROJ_ROOT / "dist"
-STR_LIKE = Union[str, Path]
+PROJ_ROOT = os.path.realpath(os.path.join(os.path.dirname(__file__), ".."))
+DIST_DIR = f"{PROJ_ROOT}/dist"
 
 
-def add_clean_arg(parser: argparse.ArgumentParser) -> None:    
+def add_clean_arg(parser: argparse.ArgumentParser) -> None:
     parser.add_argument(
         "--clean",
         action="store_true",
         help="Cleanup any existing files",
     )
 
 
-def add_tag_arg(parser: argparse.ArgumentParser) -> None:    
+def add_tag_arg(parser: argparse.ArgumentParser) -> None:
     parser.add_argument(
         "tag",
         help="The git tag for the release",
     )
 
 
 def run(
-    cmd: Sequence[STR_LIKE],
+    cmd: Sequence[str],
     *,
-    cwd: Optional[str] = None,
-    env: Dict[str, str] = None,
+    cwd: str | None = None,
+    env: dict[str, str] | None = None,
 ) -> None:
 
-    cmd = [str(x) for x in cmd]
     cwd = cwd or PROJ_ROOT
     if env is None:
         env = {}
@@ -47,22 +41,26 @@ def run(
     subprocess.check_call(cmd, cwd=cwd, env={**os.environ, **env})
 
 
-def create_checksums(dist_tag_dir: Path, product: str) -> None:
-    file_checksums = {x.name: _md5sum(x) for x in dist_tag_dir.iterdir()}
+def create_checksums(dist_tag_dir: str, product: str) -> None:
+    file_checksums = {
+        x: _md5sum(f"{dist_tag_dir}/{x}") for x in os.listdir(dist_tag_dir)
+    }
     print("\n=== Checksums ===")
-    with open(dist_tag_dir / f"MD5SUM-{product}.txt", "w") as md5_file:
+    with open(f"{dist_tag_dir}/MD5SUM-{product}.txt", "w") as md5_file:
         for file_name, md5sum in file_checksums.items():
             line = f"{md5sum} {file_name}\n"
             print(line, end="")
             md5_file.write(line)
 
 
-def verify_md5(file_path: Path, expected_md5: str):
+def verify_md5(file_path: str, expected_md5: str) -> None:
     actual_md5 = _md5sum(file_path)
-    assert expected_md5 == actual_md5
+    assert (
+        expected_md5 == actual_md5
+    ), f"Checksum mismatch: expected={expected_md5}, actual={actual_md5}"
 
 
-def _md5sum(file_path: Path) -> str:
+def _md5sum(file_path: str) -> str:
     with open(file_path, mode="rb") as f:
         d = hashlib.md5()
         for buf in iter(functools.partial(f.read, 128), b""):
@@ -81,28 +79,31 @@ def get_docker_parser() -> argparse.ArgumentParser:
 
 
 def start_containers(
-    compose_file: Path,
+    compose_file: str,
     *,
-    env: Dict[str, str] = None,
+    env: dict[str, str] | None = None,
     docker: str,
-    checks: Sequence[str]
+    checks: Sequence[str],
 ) -> None:
 
     user = ("--user", f"{os.geteuid()}:{os.getegid()}") if docker == "docker" else ()
-    volume = f"-v{compose_file.parent}:/run"
+    volume = f"-v{os.path.dirname(compose_file)}:/run"
     if docker == "podman":
         volume += ":z"
 
     cmd = (
-        docker, "run",
+        docker,
+        "run",
         "--rm",
         volume,
         *user,
         "safewaters/docker-lock",
         *(
-            "lock", "rewrite",
-            "--lockfile-name", f"{compose_file.name}.lock"
-        )
+            "lock",
+            "rewrite",
+            "--lockfile-name",
+            f"{os.path.basename(compose_file)}.lock",
+        ),
     )
     run(cmd)
 
@@ -112,7 +113,8 @@ def start_containers(
     cmd = (
         f"{docker}",
         "compose",
-        "-f", compose_file,
+        "-f",
+        compose_file,
         "up",
         "--quiet-pull",
         "--detach",
@@ -126,7 +128,8 @@ def start_containers(
     for c in checks:
         cmd = (
             "./bin/wait-for.bash",
-            "--timeout", "60s",
+            "--timeout",
+            "60s",
             f"{docker} exec {c}",
         )
         try:
@@ -136,12 +139,14 @@ def start_containers(
             cmd = (
                 f"{docker}",
                 "compose",
-                "-f", compose_file,
-                "logs"
+                "-f",
+                compose_file,
+                "logs",
             )
             run(cmd)
             raise
 
+
 def is_ci() -> bool:
     ci = os.environ.get("CI", None)
     if ci is None: