diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index c63dd2a72..13ee9fa7e 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -10,6 +10,9 @@ on: schedule: - cron: "0 5 * * *" +env: + python-version: "3.10" + jobs: analyze: name: Analyze @@ -23,6 +26,9 @@ jobs: matrix: language: ["csharp", "java", "python"] steps: + - uses: actions/setup-python@v5 + with: + python-version: ${{ env.python-version }} - name: Checkout repository uses: actions/checkout@v4 - name: Initialize CodeQL @@ -38,11 +44,11 @@ jobs: uses: actions/setup-dotnet@v4.1.0 - name: Build .Net if: ${{ matrix.language == 'csharp' }} - run: ./bin/smi/build.py -c Release + run: ./bin/smi/smi_build.py -c Release - name: Build java if: ${{ matrix.language == 'java' }} # NOTE(rkm 2023-03-21) Ensure test code is detected but don't actually run anything - run: ./bin/ctp/test.py --install-libs -DskipTests + run: ./bin/ctp/ctp_test.py --install-libs -DskipTests - name: SecurityCodescan if: ${{ matrix.language == 'csharp' }} run: | diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 3468fa3dd..b69cb0a36 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -22,15 +22,18 @@ env: java-version: 11 java-distribution: temurin # python - python-version: 3.6 + python-version: "3.10" jobs: init: - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04 outputs: matrix: ${{ steps.matrix.outputs.matrix }} build_ref: ${{ steps.build_ref.outputs.build_ref }} steps: + - uses: actions/setup-python@v5 + with: + python-version: ${{ env.python-version }} - name: checkout uses: actions/checkout@v4 - name: set matrix @@ -59,16 +62,19 @@ jobs: - name: "[linux] enable disk caching" if: ${{ matrix.os == 'linux' }} run: sudo apt-get install -y libeatmydata1 + - uses: actions/setup-python@v5 + with: + python-version: ${{ env.python-version }} - name: checkout uses: actions/checkout@v4 - name: setup .NET # NOTE(rkm 2022-02-20) Uses global.json uses: actions/setup-dotnet@v4.1.0 - name: download tessdata - run: ./bin/smi/downloadTessdata.py + run: ./bin/smi/download_tessdata.py - name: "[linux] start services" if: ${{ matrix.os == 'linux' }} - run: ./bin/smi/startDockerLinux.py ${{ env.db-password }} + run: ./bin/smi/smi_start_docker_linux.py ${{ env.db-password }} - name: "[linux] re-write database strings" if: ${{ matrix.os == 'linux' }} run: | @@ -96,7 +102,7 @@ jobs: set -euxo pipefail cov="" [ "${{ matrix.os }}" == "windows" ] && cov="--no-coverage" - ./bin/smi/buildTestPackage.py \ + ./bin/smi/smi_build_test_package.py \ ${{ needs.init.outputs.build_ref }} \ "$cov" - name: upload coverage to codecov @@ -126,6 +132,9 @@ jobs: matrix: ${{ fromJson(needs.init.outputs.matrix) }} runs-on: ${{ matrix.image }} steps: + - uses: actions/setup-python@v5 + with: + python-version: ${{ env.python-version }} - name: "[linux] enable disk caching" if: ${{ matrix.os == 'linux' }} run: sudo apt-get install -y libeatmydata1 @@ -139,12 +148,12 @@ jobs: cache: maven - name: "[linux] start services" if: ${{ matrix.os == 'linux' }} - run: ./bin/ctp/startDockerLinux.py + run: ./bin/ctp/ctp_start_docker_linux.py - name: "[windows] skip integration tests" if: ${{ matrix.os == 'windows' }} run: echo "JAVA_TESTS=--skip-integration-tests" >> $GITHUB_ENV - name: build, test, and package ctp - run: ./bin/ctp/buildTestPackage.py --install-libs ${{ needs.init.outputs.build_ref }} ${{ env.JAVA_TESTS }} + run: ./bin/ctp/ctp_build_test_package.py --install-libs ${{ needs.init.outputs.build_ref }} ${{ env.JAVA_TESTS }} - name: "[linux] upload packages" if: ${{ matrix.os == 'linux' }} uses: actions/upload-artifact@v4 @@ -155,7 +164,7 @@ jobs: upload-to-release: if: contains(github.ref, 'refs/tags/v') needs: [init, smi, ctp] - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04 steps: - name: "[linux] enable disk caching" run: sudo apt-get install -y libeatmydata1 diff --git a/.meta/flake8.conf b/.meta/flake8.conf new file mode 100644 index 000000000..3f5824b38 --- /dev/null +++ b/.meta/flake8.conf @@ -0,0 +1,8 @@ +[flake8] + +# https://github.com/psf/black +max-line-length = 120 + +# Ignoring: +# E203 - Whitespace before ':' +extend-ignore = E203 diff --git a/.meta/isort.cfg b/.meta/isort.cfg new file mode 100644 index 000000000..77300c8d5 --- /dev/null +++ b/.meta/isort.cfg @@ -0,0 +1,3 @@ +[isort] +profile = black +force_single_line = True diff --git a/.meta/mypy.ini b/.meta/mypy.ini new file mode 100644 index 000000000..154a1a472 --- /dev/null +++ b/.meta/mypy.ini @@ -0,0 +1,9 @@ +[mypy] +check_untyped_defs = true +disallow_any_generics = true +disallow_incomplete_defs = true +disallow_untyped_defs = true +no_implicit_optional = true +warn_redundant_casts = true +warn_unused_ignores = true +mypy_path = ./bin diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 4a4577e4b..1a90ee8d0 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,4 +1,6 @@ --- +default_language_version: + python: python3.10 exclude: | (?x)^( .*/packages.lock.json| @@ -74,3 +76,50 @@ repos: hooks: - id: markdown-link-check args: [--quiet, --config, .meta/markdown-link-check.json] + # Python + - repo: https://github.com/pre-commit/pygrep-hooks + rev: "v1.10.0" + hooks: + - id: python-check-blanket-noqa + - id: python-check-mock-methods + - id: python-use-type-annotations + - id: python-no-log-warn + - repo: https://github.com/asottile/add-trailing-comma + rev: "v3.1.0" + hooks: + - id: add-trailing-comma + - repo: https://github.com/asottile/pyupgrade + rev: "v3.19.0" + hooks: + - id: pyupgrade + args: [--py310-plus] + - repo: https://github.com/asottile/yesqa + rev: "v1.5.0" + hooks: + - id: yesqa + additional_dependencies: + - pep8-naming==0.14.1 + - repo: https://github.com/pre-commit/mirrors-mypy + rev: "v1.13.0" + hooks: + - id: mypy + args: [--config-file, .meta/mypy.ini] + - repo: https://github.com/psf/black-pre-commit-mirror + rev: "24.10.0" + hooks: + - id: black + - repo: https://github.com/pycqa/isort + rev: 5.13.2 + hooks: + - id: isort + name: isort (python) + args: [--settings=.meta/isort.cfg] + - repo: https://github.com/pycqa/flake8 + rev: "7.1.1" + hooks: + - id: flake8 + additional_dependencies: + - flake8-bugbear==24.10.31 + - flake8-assert-msg==1.1.1 + - pep8-naming==0.14.1 + args: [--config, .meta/flake8.conf] diff --git a/bin/checkDocs.py b/bin/checkDocs.py index d6045b9b6..205760d7e 100755 --- a/bin/checkDocs.py +++ b/bin/checkDocs.py @@ -1,15 +1,11 @@ #!/usr/bin/env python3 - import collections -import glob import os import re -import sys - -import common as C +import common as c -_VERB_CS_PATH = f"{C.PROJ_ROOT}/src/SmiServices/ServiceVerbs.cs" +_VERB_CS_PATH = f"{c.PROJ_ROOT}/src/SmiServices/ServiceVerbs.cs" _VERB_RE = re.compile(r'\[Verb\(("\S*?")') @@ -63,7 +59,7 @@ def main() -> int: for tool_type in ("application", "service"): for tool in tools[tool_type]: - doc_path = f"{C.PROJ_ROOT}/docs/{tool_type}s/{tool}.md" + doc_path = f"{c.PROJ_ROOT}/docs/{tool_type}s/{tool}.md" if not os.path.isfile(doc_path): print(f"Missing {doc_path}") rc |= 1 diff --git a/bin/common.py b/bin/common.py index b729cee3d..a71aac724 100644 --- a/bin/common.py +++ b/bin/common.py @@ -1,22 +1,17 @@ """Common build variables and functions""" import argparse -import hashlib import functools +import hashlib import os import subprocess -from pathlib import Path -from typing import Dict -from typing import Optional -from typing import Sequence -from typing import Union +from collections.abc import Sequence -PROJ_ROOT = (Path(__file__).parent / "..").resolve() -DIST_DIR = PROJ_ROOT / "dist" -STR_LIKE = Union[str, Path] +PROJ_ROOT = os.path.realpath(os.path.join(os.path.dirname(__file__), "..")) +DIST_DIR = f"{PROJ_ROOT}/dist" -def add_clean_arg(parser: argparse.ArgumentParser) -> None: +def add_clean_arg(parser: argparse.ArgumentParser) -> None: parser.add_argument( "--clean", action="store_true", @@ -24,7 +19,7 @@ def add_clean_arg(parser: argparse.ArgumentParser) -> None: ) -def add_tag_arg(parser: argparse.ArgumentParser) -> None: +def add_tag_arg(parser: argparse.ArgumentParser) -> None: parser.add_argument( "tag", help="The git tag for the release", @@ -32,13 +27,12 @@ def add_tag_arg(parser: argparse.ArgumentParser) -> None: def run( - cmd: Sequence[STR_LIKE], + cmd: Sequence[str], *, - cwd: Optional[str] = None, - env: Dict[str, str] = None, + cwd: str | None = None, + env: dict[str, str] | None = None, ) -> None: - cmd = [str(x) for x in cmd] cwd = cwd or PROJ_ROOT if env is None: env = {} @@ -47,22 +41,26 @@ def run( subprocess.check_call(cmd, cwd=cwd, env={**os.environ, **env}) -def create_checksums(dist_tag_dir: Path, product: str) -> None: - file_checksums = {x.name: _md5sum(x) for x in dist_tag_dir.iterdir()} +def create_checksums(dist_tag_dir: str, product: str) -> None: + file_checksums = { + x: _md5sum(f"{dist_tag_dir}/{x}") for x in os.listdir(dist_tag_dir) + } print("\n=== Checksums ===") - with open(dist_tag_dir / f"MD5SUM-{product}.txt", "w") as md5_file: + with open(f"{dist_tag_dir}/MD5SUM-{product}.txt", "w") as md5_file: for file_name, md5sum in file_checksums.items(): line = f"{md5sum} {file_name}\n" print(line, end="") md5_file.write(line) -def verify_md5(file_path: Path, expected_md5: str): +def verify_md5(file_path: str, expected_md5: str) -> None: actual_md5 = _md5sum(file_path) - assert expected_md5 == actual_md5 + assert ( + expected_md5 == actual_md5 + ), f"Checksum mismatch: expected={expected_md5}, actual={actual_md5}" -def _md5sum(file_path: Path) -> str: +def _md5sum(file_path: str) -> str: with open(file_path, mode="rb") as f: d = hashlib.md5() for buf in iter(functools.partial(f.read, 128), b""): @@ -81,28 +79,31 @@ def get_docker_parser() -> argparse.ArgumentParser: def start_containers( - compose_file: Path, + compose_file: str, *, - env: Dict[str, str] = None, + env: dict[str, str] | None = None, docker: str, - checks: Sequence[str] + checks: Sequence[str], ) -> None: user = ("--user", f"{os.geteuid()}:{os.getegid()}") if docker == "docker" else () - volume = f"-v{compose_file.parent}:/run" + volume = f"-v{os.path.dirname(compose_file)}:/run" if docker == "podman": volume += ":z" cmd = ( - docker, "run", + docker, + "run", "--rm", volume, *user, "safewaters/docker-lock", *( - "lock", "rewrite", - "--lockfile-name", f"{compose_file.name}.lock" - ) + "lock", + "rewrite", + "--lockfile-name", + f"{os.path.basename(compose_file)}.lock", + ), ) run(cmd) @@ -112,7 +113,8 @@ def start_containers( cmd = ( f"{docker}", "compose", - "-f", compose_file, + "-f", + compose_file, "up", "--quiet-pull", "--detach", @@ -126,7 +128,8 @@ def start_containers( for c in checks: cmd = ( "./bin/wait-for.bash", - "--timeout", "60s", + "--timeout", + "60s", f"{docker} exec {c}", ) try: @@ -136,12 +139,14 @@ def start_containers( cmd = ( f"{docker}", "compose", - "-f", compose_file, - "logs" + "-f", + compose_file, + "logs", ) run(cmd) raise + def is_ci() -> bool: ci = os.environ.get("CI", None) if ci is None: diff --git a/bin/ctp/build.py b/bin/ctp/build.py deleted file mode 100755 index 54392c718..000000000 --- a/bin/ctp/build.py +++ /dev/null @@ -1,48 +0,0 @@ -#!/usr/bin/env python3 - -import argparse -import glob -import os -import shutil -import sys -from pathlib import Path -from typing import Optional -from typing import Sequence - -sys.path.append(os.path.join(os.path.dirname(__file__), '..')) -import common as C - -import javaCommon as JC -import installLibs as L - - -def main(argv: Optional[Sequence[str]] = None) -> int: - - parser = argparse.ArgumentParser() - C.add_clean_arg(parser) - JC.add_common_args(parser) - args, rest = parser.parse_known_args(argv) - - if args.install_libs: - rc = L.main() - if rc: - return rc - - stages = ["compile"] - if args.clean: - stages.insert(0, "clean") - - cmd = ( - JC.mvn_exe(), - "-ntp", - *stages, - "-f", (C.PROJ_ROOT / "src/common/com.smi.microservices.parent").resolve(), - *rest, - ) - C.run(cmd) - - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/bin/ctp/ctp_build.py b/bin/ctp/ctp_build.py new file mode 100755 index 000000000..c6f106ba5 --- /dev/null +++ b/bin/ctp/ctp_build.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python3 +import argparse +import os +import sys +from collections.abc import Sequence + +import install_libs +import java_common + +sys.path.append(os.path.join(os.path.dirname(__file__), "..")) +import common # noqa: E402 + + +def main(argv: Sequence[str] | None = None) -> int: + + parser = argparse.ArgumentParser() + common.add_clean_arg(parser) + java_common.add_common_args(parser) + args, rest = parser.parse_known_args(argv) + + if args.install_libs: + rc = install_libs.main() + if rc: + return rc + + stages = ["compile"] + if args.clean: + stages.insert(0, "clean") + + cmd = ( + java_common.mvn_exe(), + "-ntp", + *stages, + "-f", + f"{common.PROJ_ROOT}/src/common/com.smi.microservices.parent", + *rest, + ) + common.run(cmd) + + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/bin/ctp/buildTestPackage.py b/bin/ctp/ctp_build_test_package.py similarity index 63% rename from bin/ctp/buildTestPackage.py rename to bin/ctp/ctp_build_test_package.py index af307eaf7..e100727ad 100755 --- a/bin/ctp/buildTestPackage.py +++ b/bin/ctp/ctp_build_test_package.py @@ -1,28 +1,23 @@ #!/usr/bin/env python3 - import argparse -import glob import os -import shutil import sys -from pathlib import Path -sys.path.append(os.path.join(os.path.dirname(__file__), '..')) -import common as C +import ctp_build +import ctp_package +import ctp_test +import java_common -import build as JB -import package as JP -import test as JT -import javaCommon as JC -import installLibs as L +sys.path.append(os.path.join(os.path.dirname(__file__), "..")) +import common # noqa: E402 def main() -> int: parser = argparse.ArgumentParser() - C.add_clean_arg(parser) - C.add_tag_arg(parser) - JC.add_common_args(parser) + common.add_clean_arg(parser) + common.add_tag_arg(parser) + java_common.add_common_args(parser) parser.add_argument( "--skip-tests", action="store_true", @@ -41,17 +36,17 @@ def main() -> int: # Build and test if args.skip_tests: - rc = JB.main(build_args) + rc = ctp_build.main(build_args) else: if args.skip_integration_tests: build_args.append("-PunitTests") - rc = JT.main(build_args) + rc = ctp_test.main(build_args) if rc: return rc # Package - rc = JP.main((args.tag,)) + rc = ctp_package.main((args.tag,)) return 0 diff --git a/bin/ctp/ctp_package.py b/bin/ctp/ctp_package.py new file mode 100755 index 000000000..b1fb9bf37 --- /dev/null +++ b/bin/ctp/ctp_package.py @@ -0,0 +1,64 @@ +#!/usr/bin/env python3 +import argparse +import glob +import os +import shutil +import sys +from collections.abc import Sequence + +import java_common + +sys.path.append(os.path.join(os.path.dirname(__file__), "..")) +import common # noqa: E402 + + +def main(argv: Sequence[str] | None = None) -> int: + + parser = argparse.ArgumentParser() + common.add_clean_arg(parser) + common.add_tag_arg(parser) + java_common.add_common_args(parser) + args, rest = parser.parse_known_args(argv) + + dist_tag_dir = f"{common.DIST_DIR}/{args.tag}" + + stages = ["package"] + + if args.clean: + stages.insert(0, "clean") + if os.path.isdir(dist_tag_dir): + shutil.rmtree(dist_tag_dir) + os.makedirs(dist_tag_dir, exist_ok=True) + + cmd = ( + java_common.mvn_exe(), + "-ntp", + *stages, + "-DskipTests", + "assembly:single@create-deployable", + "-f", + f"{common.PROJ_ROOT}/src/common/com.smi.microservices.parent", + *rest, + ) + common.run(cmd) + + zips = tuple( + x + for x in glob.glob( + f"{common.PROJ_ROOT}/src/**/*deploy-distribution.zip", + recursive=True, + ) + ) + assert 1 == len(zips), "Expected 1 zip file (CTP)" + shutil.copyfile( + zips[0], + f"{dist_tag_dir}/{os.path.basename(zips[0]).split('-')[0]}-{args.tag}.zip", + ) + + common.create_checksums(dist_tag_dir, "ctp") + + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/bin/ctp/ctp_start_docker_linux.py b/bin/ctp/ctp_start_docker_linux.py new file mode 100755 index 000000000..8572fd1cc --- /dev/null +++ b/bin/ctp/ctp_start_docker_linux.py @@ -0,0 +1,32 @@ +#!/usr/bin/env python3 +import os +import sys + +sys.path.append(os.path.join(os.path.dirname(__file__), "..")) +import common # noqa: E402 + +_COMPOSE_FILE_NAME = "linux-java.yml" +_COMPOSE_FILE_PATH = f"{common.PROJ_ROOT}/utils/docker-compose/{_COMPOSE_FILE_NAME}" +assert os.path.isfile( + _COMPOSE_FILE_PATH, +), f"Compose file does not exist: {_COMPOSE_FILE_PATH}" + + +def main() -> int: + + parser = common.get_docker_parser() + args = parser.parse_args() + + docker = "podman" if args.podman else "docker" + + common.start_containers( + _COMPOSE_FILE_PATH, + docker=docker, + checks=("rabbitmq rabbitmq-diagnostics -q ping",), + ) + + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/bin/ctp/ctp_stop_docker_linux.py b/bin/ctp/ctp_stop_docker_linux.py new file mode 100755 index 000000000..a3c1db247 --- /dev/null +++ b/bin/ctp/ctp_stop_docker_linux.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python3 +import os +import sys + +sys.path.append(os.path.join(os.path.dirname(__file__), "..")) +import common # noqa: E402 + +_COMPOSE_FILE_NAME = "linux-java.yml" +_COMPOSE_FILE_PATH = f"{common.PROJ_ROOT}/utils/docker-compose/{_COMPOSE_FILE_NAME}" +assert os.path.isfile( + _COMPOSE_FILE_PATH, +), f"Compose file does not exist: {_COMPOSE_FILE_PATH}" + + +def main() -> int: + + parser = common.get_docker_parser() + args = parser.parse_args() + + docker = "docker" if not args.podman else "podman" + + cmd = ( + f"{docker}-compose", + "-f", + _COMPOSE_FILE_PATH, + "down", + "--timeout", + "0", + ) + common.run(cmd) + + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/bin/ctp/ctp_test.py b/bin/ctp/ctp_test.py new file mode 100755 index 000000000..c5d4b5a58 --- /dev/null +++ b/bin/ctp/ctp_test.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python3 +import argparse +import os +import sys +from collections.abc import Sequence + +import install_libs +import java_common + +sys.path.append(os.path.join(os.path.dirname(__file__), "..")) +import common # noqa: E402 + + +def main(argv: Sequence[str] | None = None) -> int: + + parser = argparse.ArgumentParser() + common.add_clean_arg(parser) + java_common.add_common_args(parser) + args, rest = parser.parse_known_args(argv) + + if args.install_libs: + rc = install_libs.main() + if rc: + return rc + + stages = ["test"] + if args.clean: + stages.insert(0, "clean") + + cmd = ( + java_common.mvn_exe(), + "-ntp", + "-DtrimStackTrace=false", + *stages, + "-f", + f"{common.PROJ_ROOT}/src/common/com.smi.microservices.parent", + *rest, + ) + common.run(cmd) + + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/bin/ctp/installLibs.py b/bin/ctp/install_libs.py similarity index 73% rename from bin/ctp/installLibs.py rename to bin/ctp/install_libs.py index 02ee33653..76275b9f5 100755 --- a/bin/ctp/installLibs.py +++ b/bin/ctp/install_libs.py @@ -1,17 +1,14 @@ #!/usr/bin/env python3 - import os import sys -from pathlib import Path - -sys.path.append(os.path.join(os.path.dirname(__file__), '..')) -import common as C -import javaCommon as JC +import java_common +sys.path.append(os.path.join(os.path.dirname(__file__), "..")) +import common as c # noqa: E402 -_CTP_LIB_DIR = Path(f"{C.PROJ_ROOT}/lib/ctp") -assert _CTP_LIB_DIR.is_dir() +_CTP_LIB_DIR = f"{c.PROJ_ROOT}/lib/ctp" +assert os.path.isdir(_CTP_LIB_DIR), "Could not find CTP libs dir" _CTP_JARS = [ "CTP", @@ -31,9 +28,10 @@ def main() -> int: base_cmd = ( - JC.mvn_exe(), - "--quiet", "--no-transfer-progress", - "install:install-file" + java_common.mvn_exe(), + "--quiet", + "--no-transfer-progress", + "install:install-file", ) base_jar_cmd = ( @@ -43,7 +41,7 @@ def main() -> int: "-DgeneratePom=true", ) for j in _CTP_JARS: - C.run( + c.run( ( *base_jar_cmd, f"-DartifactId={j}", @@ -53,7 +51,7 @@ def main() -> int: cwd=_CTP_LIB_DIR, ) for j in _DAT_JARS: - C.run( + c.run( ( *base_jar_cmd, f"-DartifactId={j}", @@ -63,7 +61,7 @@ def main() -> int: cwd=_CTP_LIB_DIR, ) - C.run( + c.run( ( *base_cmd, "-Dfile=DAT.jar", diff --git a/bin/ctp/javaCommon.py b/bin/ctp/java_common.py similarity index 69% rename from bin/ctp/javaCommon.py rename to bin/ctp/java_common.py index 059a746e2..d0fab6dc9 100644 --- a/bin/ctp/javaCommon.py +++ b/bin/ctp/java_common.py @@ -10,8 +10,9 @@ def add_common_args( parser.add_argument( "--install-libs", action="store_true", - help="Install the CTP libraries if specified" + help="Install the CTP libraries if specified", ) + def mvn_exe() -> str: - return "mvn" if os.name == "posix" else "mvn.cmd" + return "mvn" if os.name == "posix" else "mvn.cmd" diff --git a/bin/ctp/package.py b/bin/ctp/package.py deleted file mode 100755 index 8fcc81452..000000000 --- a/bin/ctp/package.py +++ /dev/null @@ -1,69 +0,0 @@ -#!/usr/bin/env python3 - -import argparse -import glob -import os -import shutil -import sys -from pathlib import Path -from typing import Optional -from typing import Sequence - -sys.path.append(os.path.join(os.path.dirname(__file__), '..')) -import common as C - -import build as JB -import installLibs as L -import javaCommon as JC - - -def main(argv: Optional[Sequence[str]] = None) -> int: - - parser = argparse.ArgumentParser() - C.add_clean_arg(parser) - C.add_tag_arg(parser) - JC.add_common_args(parser) - args, rest = parser.parse_known_args(argv) - - dist_tag_dir = C.DIST_DIR / args.tag - - stages = ["package"] - - if args.clean: - stages.insert(0, "clean") - if dist_tag_dir.is_dir(): - shutil.rmtree(dist_tag_dir) - dist_tag_dir.mkdir(parents=True, exist_ok=True) - - cmd = ( - JC.mvn_exe(), - "-ntp", - *stages, - "-DskipTests", - "assembly:single@create-deployable", - "-f", (C.PROJ_ROOT / "src/common/com.smi.microservices.parent").resolve(), - *rest, - ) - C.run(cmd) - - zips = { - Path(x) for x in - glob.glob( - f"{C.PROJ_ROOT}/src/**/*deploy-distribution.zip", - recursive=True, - ) - } - assert 1 == len(zips), "Expected 1 zip file (CTP)" - for zip_path in zips: - shutil.copyfile( - zip_path, - dist_tag_dir / f"{zip_path.name.split('-')[0]}-{args.tag}.zip", - ) - - C.create_checksums(dist_tag_dir, "ctp") - - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/bin/ctp/startDockerLinux.py b/bin/ctp/startDockerLinux.py deleted file mode 100755 index 1c2a43fa8..000000000 --- a/bin/ctp/startDockerLinux.py +++ /dev/null @@ -1,33 +0,0 @@ -#!/usr/bin/env python3 - -import os -import sys - -sys.path.append(os.path.join(os.path.dirname(__file__), '..')) -import common as C - -_COMPOSE_FILE_NAME = "linux-java.yml" -_COMPOSE_FILE_PATH = (C.PROJ_ROOT / "utils/docker-compose" / _COMPOSE_FILE_NAME).resolve() -assert _COMPOSE_FILE_PATH.is_file() - - -def main() -> int: - - parser = C.get_docker_parser() - args = parser.parse_args() - - docker = "podman" if args.podman else "docker" - - C.start_containers( - _COMPOSE_FILE_PATH, - docker=docker, - checks=( - "rabbitmq rabbitmq-diagnostics -q ping", - ), - ) - - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/bin/ctp/stopDockerLinux.py b/bin/ctp/stopDockerLinux.py deleted file mode 100755 index 9454d40ce..000000000 --- a/bin/ctp/stopDockerLinux.py +++ /dev/null @@ -1,33 +0,0 @@ -#!/usr/bin/env python3 - -import os -import sys - -sys.path.append(os.path.join(os.path.dirname(__file__), '..')) -import common as C - -_COMPOSE_FILE_NAME = "linux-java.yml" -_COMPOSE_FILE_PATH = (C.PROJ_ROOT / "utils/docker-compose" / _COMPOSE_FILE_NAME).resolve() -assert _COMPOSE_FILE_PATH.is_file() - - -def main() -> int: - - parser = C.get_docker_parser() - args = parser.parse_args() - - docker = "docker" if not args.podman else "podman" - - cmd = ( - f"{docker}-compose", - "-f", _COMPOSE_FILE_PATH, - "down", - "--timeout", 0, - ) - C.run(cmd) - - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/bin/ctp/test.py b/bin/ctp/test.py deleted file mode 100755 index b18cf9d74..000000000 --- a/bin/ctp/test.py +++ /dev/null @@ -1,50 +0,0 @@ -#!/usr/bin/env python3 - -import argparse -import glob -import os -import shutil -import sys -from pathlib import Path -from typing import Optional -from typing import Sequence - -sys.path.append(os.path.join(os.path.dirname(__file__), '..')) -import common as C - -import build as JB -import installLibs as L -import javaCommon as JC - - -def main(argv: Optional[Sequence[str]] = None) -> int: - - parser = argparse.ArgumentParser() - C.add_clean_arg(parser) - JC.add_common_args(parser) - args, rest = parser.parse_known_args(argv) - - if args.install_libs: - rc = L.main() - if rc: - return rc - - stages = ["test"] - if args.clean: - stages.insert(0, "clean") - - cmd = ( - JC.mvn_exe(), - "-ntp", - "-DtrimStackTrace=false", - *stages, - "-f", (C.PROJ_ROOT / "src/common/com.smi.microservices.parent").resolve(), - *rest, - ) - C.run(cmd) - - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/bin/release/createReleaseChangelog.py b/bin/release/createReleaseChangelog.py index 08238e9fd..f5fdeaf76 100755 --- a/bin/release/createReleaseChangelog.py +++ b/bin/release/createReleaseChangelog.py @@ -1,12 +1,10 @@ #!/usr/bin/env python3 - import argparse import re -from typing import Optional -from typing import Sequence +from collections.abc import Sequence -def main(argv: Optional[Sequence[str]] = None) -> int: +def main(argv: Sequence[str] | None = None) -> int: parser = argparse.ArgumentParser() parser.add_argument("tag", help="The git tag for the release") @@ -36,7 +34,8 @@ def main(argv: Optional[Sequence[str]] = None) -> int: f.write(content) print(f"Wrote {output_file}") + return 0 if __name__ == "__main__": - exit(main()) + raise SystemExit(main()) diff --git a/bin/release/updateChangelog.py b/bin/release/updateChangelog.py index 2e4d70dc7..a66366070 100755 --- a/bin/release/updateChangelog.py +++ b/bin/release/updateChangelog.py @@ -6,39 +6,29 @@ import collections import datetime import fileinput +import glob import json +import os import subprocess -import sys import urllib.request -from pathlib import Path -from typing import Dict -from typing import Optional -from typing import Sequence -from typing import Union +from collections.abc import Sequence - -_NEWS_DIR = Path("./news/") +_NEWS_DIR = "./news/" _MARKER = "" _UNRELEASED_LINK = "[unreleased]:" _ORG = "SMI" _REPO = "SmiServices" -_STR_LIKE = Union[str, Path] - -def _run(cmd: Sequence[_STR_LIKE]): +def _run(cmd: Sequence[str]) -> None: subprocess.check_call(("echo", *cmd)) subprocess.check_call(cmd) def _get_pr_author(pr_ref: int) -> str: - url = ( - f"https://api.github.com/repos/" - f"{_ORG}/{_REPO}/" - f"pulls/{pr_ref}" - ) + url = f"https://api.github.com/repos/" f"{_ORG}/{_REPO}/" f"pulls/{pr_ref}" try: resp = urllib.request.urlopen(url) except urllib.error.HTTPError as e: @@ -47,12 +37,15 @@ def _get_pr_author(pr_ref: int) -> str: return data["user"]["login"] -def _print_fragments(version: str, fragments: Dict[str, Dict[int, str]]) -> None: +def _print_fragments(version: str, fragments: dict[str, dict[int, str]]) -> None: today = datetime.datetime.today().strftime("%Y-%m-%d") print(f"## [{version[1:]}] {today}") - def _print_type_fragment(frag_type: str, fragments: Dict[str, Dict[int, str]]): + def _print_type_fragment( + frag_type: str, + fragments: dict[str, dict[int, str]], + ) -> None: print(f"\n### {frag_type.capitalize()}\n") for pr_ref in sorted(fragments[frag_type]): first, *rest = fragments[frag_type][pr_ref].splitlines() @@ -97,7 +90,7 @@ def _print_links(last_tag: str, next_tag: str) -> None: print(diff_link_str) -def main(argv: Optional[Sequence[str]] = None) -> int: +def main(argv: Sequence[str] | None = None) -> int: parser = argparse.ArgumentParser() parser.add_argument( @@ -111,13 +104,13 @@ def main(argv: Optional[Sequence[str]] = None) -> int: args = parser.parse_args(argv) # Gather the news files for the next release - fragments = collections.defaultdict(dict) - fragment_files = list(_NEWS_DIR.glob("*-*.md")) + fragments: dict[str, dict[int, str]] = collections.defaultdict(dict) + fragment_files = list(glob.glob(f"{_NEWS_DIR}/*-*.md")) for fragment_file in fragment_files: with open(fragment_file) as f: contents = f.read() - pr_ref, _, frag_type = fragment_file.stem.partition("-") - fragments[frag_type][pr_ref] = contents + pr_ref, _, frag_type = os.path.splitext(fragment_file)[0].partition("-") + fragments[frag_type][int(pr_ref)] = contents # Write-out the new CHANGELOG with fileinput.FileInput("CHANGELOG.md", inplace=True) as f: @@ -134,8 +127,9 @@ def main(argv: Optional[Sequence[str]] = None) -> int: # Now delete all the news files for news_file in fragment_files: cmd = ( - "git", "rm", - news_file + "git", + "rm", + news_file, ) _run(cmd) diff --git a/bin/smi/dotnetCommon.py b/bin/smi/dotnet_common.py similarity index 89% rename from bin/smi/dotnetCommon.py rename to bin/smi/dotnet_common.py index a3a7a0894..949ee54db 100644 --- a/bin/smi/dotnetCommon.py +++ b/bin/smi/dotnet_common.py @@ -5,7 +5,7 @@ def add_args( parser: argparse.ArgumentParser, - configuration: str = "debug" + configuration: str = "debug", ) -> None: parser.add_argument( "-c", diff --git a/bin/smi/downloadRdmpCli.py b/bin/smi/downloadRdmpCli.py index ddb4f3587..ec68601cb 100755 --- a/bin/smi/downloadRdmpCli.py +++ b/bin/smi/downloadRdmpCli.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 - import argparse import os import shutil @@ -8,17 +7,15 @@ import tempfile import urllib.request import zipfile -from pathlib import Path -from typing import Sequence -from typing import Optional +from collections.abc import Sequence -sys.path.append(os.path.join(os.path.dirname(__file__), '..')) -import common as C +sys.path.append(os.path.join(os.path.dirname(__file__), "..")) +import common # noqa: E402 -_RDMP_CLI_DIR = (C.PROJ_ROOT / "rdmp-cli").resolve() +_RDMP_CLI_DIR = f"{common.PROJ_ROOT}/rdmp-cli" -def main(argv: Optional[Sequence[str]] = None) -> int: +def main(argv: Sequence[str] | None = None) -> int: parser = argparse.ArgumentParser() parser.add_argument( @@ -27,16 +24,16 @@ def main(argv: Optional[Sequence[str]] = None) -> int: parser.add_argument( "--clean", action="store_true", - help="Delete the file and re-download if it already exists" + help="Delete the file and re-download if it already exists", ) args = parser.parse_args(argv) if args.clean: shutil.rmtree(_RDMP_CLI_DIR) - elif _RDMP_CLI_DIR.is_dir(): + elif os.path.isdir(_RDMP_CLI_DIR): print(f"Error: {_RDMP_CLI_DIR} exists") return 1 - _RDMP_CLI_DIR.mkdir() + os.mkdir(_RDMP_CLI_DIR) platform = "linux" if os.name == "posix" else "win" url = ( @@ -45,13 +42,13 @@ def main(argv: Optional[Sequence[str]] = None) -> int: ) print(f"Downloading {url}") with tempfile.TemporaryDirectory() as tmpdir: - _file = Path(tmpdir) / f"rdmp-cli-{args.version}-x64.zip" + _file = f"{tmpdir}/rdmp-cli-{args.version}-x64.zip" urllib.request.urlretrieve(url, _file) with zipfile.ZipFile(_file) as zf: zf.extractall(path=_RDMP_CLI_DIR) - rdmp = _RDMP_CLI_DIR / ("rdmp.exe" if os.name == "nt" else "rdmp") + rdmp = f"{_RDMP_CLI_DIR}/{'rdmp.exe' if os.name == 'nt' else 'rdmp'}" st = os.stat(rdmp) os.chmod(rdmp, st.st_mode | stat.S_IXUSR | stat.S_IXGRP) diff --git a/bin/smi/downloadTessdata.py b/bin/smi/download_tessdata.py similarity index 56% rename from bin/smi/downloadTessdata.py rename to bin/smi/download_tessdata.py index cc7a6af20..109d60a11 100755 --- a/bin/smi/downloadTessdata.py +++ b/bin/smi/download_tessdata.py @@ -1,49 +1,43 @@ #!/usr/bin/env python3 - import argparse import os import sys import urllib.request -from pathlib import Path -from typing import Sequence -from typing import Optional +from collections.abc import Sequence -sys.path.append(os.path.join(os.path.dirname(__file__), '..')) -import common as C +sys.path.append(os.path.join(os.path.dirname(__file__), "..")) +import common # noqa: E402 -_PATH = C.PROJ_ROOT / "data/tessdata" +_PATH = f"{common.PROJ_ROOT}/data/tessdata" _FILE = "eng.traineddata" _VERSION = "4.1.0" _MD5SUM = "57e0df3d84fed9fbf8c7a8e589f8f012" -def main(argv: Optional[Sequence[str]] = None) -> int: +def main(argv: Sequence[str] | None = None) -> int: parser = argparse.ArgumentParser() parser.add_argument( "--clean", action="store_true", - help="Delete the file and re-download if it already exists" + help="Delete the file and re-download if it already exists", ) args = parser.parse_args(argv) - output_file = Path(_PATH, _FILE) - if output_file.is_file(): + output_file = f"{_PATH}/{_FILE}" + if os.path.isfile(output_file): if args.clean: os.unlink(output_file) else: print(f"{output_file} exists") - C.verify_md5(output_file, _MD5SUM) + common.verify_md5(output_file, _MD5SUM) return 0 - url = ( - "https://github.com/tesseract-ocr/tessdata/raw/" - f"{_VERSION}/{_FILE}" - ) + url = "https://github.com/tesseract-ocr/tessdata/raw/" f"{_VERSION}/{_FILE}" print(f"Downloading {url}") urllib.request.urlretrieve(url, output_file) - C.verify_md5(output_file, _MD5SUM) + common.verify_md5(output_file, _MD5SUM) return 0 diff --git a/bin/smi/build.py b/bin/smi/smi_build.py similarity index 65% rename from bin/smi/build.py rename to bin/smi/smi_build.py index a67d634b1..c18f13893 100755 --- a/bin/smi/build.py +++ b/bin/smi/smi_build.py @@ -1,22 +1,19 @@ #!/usr/bin/env python3 - import argparse import os -import platform import sys -from typing import Optional -from typing import Sequence +from collections.abc import Sequence -sys.path.append(os.path.join(os.path.dirname(__file__), '..')) -import common as C +import dotnet_common as dc -import dotnetCommon as DC +sys.path.append(os.path.join(os.path.dirname(__file__), "..")) +import common as c # noqa: E402 -def main(argv: Optional[Sequence[str]] = None) -> int: +def main(argv: Sequence[str] | None = None) -> int: parser = argparse.ArgumentParser() - DC.add_args(parser) + dc.add_args(parser) parser.add_argument( "--clean", action="store_true", @@ -24,19 +21,22 @@ def main(argv: Optional[Sequence[str]] = None) -> int: ) args = parser.parse_args(argv) + cmd: tuple[str, ...] + if args.clean: cmd = ( "dotnet", "clean", "-p:UseCurrentRuntimeIdentifier=True", - "--verbosity", "quiet", + "--verbosity", + "quiet", "--nologo", ) - C.run(cmd) + c.run(cmd) # NOTE This isn't necessarily needed, but the lockfile processing isn't # otherwise transparent in the build output - if C.is_ci(): + if c.is_ci(): cmd = ( "dotnet", "restore", @@ -46,19 +46,21 @@ def main(argv: Optional[Sequence[str]] = None) -> int: "--locked-mode", "--force", ) - C.run(cmd) + c.run(cmd) cmd = ( "dotnet", "build", "-warnaserror", "--use-current-runtime", - "--configuration", args.configuration, - "--no-restore" if C.is_ci() else "", - "--verbosity", "quiet", + "--configuration", + args.configuration, + "--no-restore" if c.is_ci() else "", + "--verbosity", + "quiet", "--nologo", ) - C.run(cmd) + c.run(cmd) return 0 diff --git a/bin/smi/buildTestPackage.py b/bin/smi/smi_build_test_package.py similarity index 53% rename from bin/smi/buildTestPackage.py rename to bin/smi/smi_build_test_package.py index 665ef71d7..3534c9626 100755 --- a/bin/smi/buildTestPackage.py +++ b/bin/smi/smi_build_test_package.py @@ -1,24 +1,23 @@ #!/usr/bin/env python3 - import argparse import os import sys -sys.path.append(os.path.join(os.path.dirname(__file__), '..')) -import common as C +import dotnet_common +import smi_build +import smi_package +import smi_test -import dotnetCommon as DC -import build as DB -import package as DP -import test as DT +sys.path.append(os.path.join(os.path.dirname(__file__), "..")) +import common # noqa: E402 def main() -> int: parser = argparse.ArgumentParser() - C.add_clean_arg(parser) - C.add_tag_arg(parser) - DC.add_args(parser, "release") + common.add_clean_arg(parser) + common.add_tag_arg(parser) + dotnet_common.add_args(parser, "release") parser.add_argument( "--skip-tests", action="store_true", @@ -32,29 +31,33 @@ def main() -> int: cfg_args = ("-c", args.configuration) # Build - build_args = [*cfg_args,] + build_args = [*cfg_args] if args.clean: build_args.append("--clean") - rc = DB.main(build_args) + rc = smi_build.main(build_args) if rc: return rc # Test if not args.skip_tests: - rc = DT.main(( - *cfg_args, - "--no-coverage" if args.no_coverage else "", - "--no-build", - )) + rc = smi_test.main( + ( + *cfg_args, + "--no-coverage" if args.no_coverage else "", + "--no-build", + ), + ) if rc: return rc # Package - rc = DP.main(( - *cfg_args, - "--no-build", - args.tag, - )) + rc = smi_package.main( + ( + *cfg_args, + "--no-build", + args.tag, + ), + ) return rc diff --git a/bin/smi/package.py b/bin/smi/smi_package.py similarity index 60% rename from bin/smi/package.py rename to bin/smi/smi_package.py index d4d91c328..1b2c60fe6 100755 --- a/bin/smi/package.py +++ b/bin/smi/smi_package.py @@ -1,17 +1,14 @@ #!/usr/bin/env python3 - import argparse import os -import platform import shutil import sys -from typing import Optional -from typing import Sequence +from collections.abc import Sequence -sys.path.append(os.path.join(os.path.dirname(__file__), '..')) -import common as C +import dotnet_common -import dotnetCommon as DC +sys.path.append(os.path.join(os.path.dirname(__file__), "..")) +import common # noqa: E402 _LINUX = "linux" _WINDOWS = "win" @@ -25,50 +22,54 @@ def runtime_platform() -> str: raise ValueError(os.name) -def main(argv: Optional[Sequence[str]] = None) -> int: +def main(argv: Sequence[str] | None = None) -> int: parser = argparse.ArgumentParser() - C.add_clean_arg(parser) - C.add_tag_arg(parser) - DC.add_args(parser) + common.add_clean_arg(parser) + common.add_tag_arg(parser) + dotnet_common.add_args(parser) parser.add_argument( "--no-build", action="store_true", ) args = parser.parse_args(argv) - dist_tag_dir = C.DIST_DIR / args.tag + dist_tag_dir = f"{common.DIST_DIR}/{args.tag}" - if args.clean and dist_tag_dir.is_dir(): + if args.clean and os.path.isdir(dist_tag_dir): shutil.rmtree(dist_tag_dir) - dist_tag_dir.mkdir(parents=True, exist_ok=True) + os.makedirs(dist_tag_dir, exist_ok=True) platform = runtime_platform() rid = f"{platform}-x64" smi_services_output_dir = f"smi-services-{args.tag}-{rid}" - cmd = ( + cmd: tuple[str, ...] = ( "dotnet", "publish", "-warnaserror", "--use-current-runtime", - "--configuration", args.configuration, + "--configuration", + args.configuration, "--no-build" if args.no_build else "", "-p:PublishTrimmed=false", "--self-contained", - "--output", dist_tag_dir / smi_services_output_dir, - "--verbosity", "quiet", + "--output", + f"{dist_tag_dir}/{smi_services_output_dir}", + "--verbosity", + "quiet", "--nologo", "src/SmiServices", ) - C.run(cmd) + common.run(cmd) if platform == _LINUX: cmd = ( "tar", - "-C", dist_tag_dir, + "-C", + dist_tag_dir, "-czf", - dist_tag_dir / f"{smi_services_output_dir}.tgz", + f"{dist_tag_dir}/{smi_services_output_dir}.tgz", smi_services_output_dir, ) elif platform == _WINDOWS: @@ -80,17 +81,17 @@ def main(argv: Optional[Sequence[str]] = None) -> int: "-tzip", "-mx9", "-r", - dist_tag_dir / f"{smi_services_output_dir}.zip", + f"{dist_tag_dir}/{smi_services_output_dir}.zip", smi_services_output_dir, ) else: print(f"Error: No case for platform {platform}", file=sys.stderr) return 1 - C.run(cmd) - shutil.rmtree(dist_tag_dir / smi_services_output_dir) + common.run(cmd) + shutil.rmtree(f"{dist_tag_dir}/{smi_services_output_dir}") - C.create_checksums(dist_tag_dir, "smiservices") + common.create_checksums(dist_tag_dir, "smiservices") return 0 diff --git a/bin/smi/startDockerLinux.py b/bin/smi/smi_start_docker_linux.py similarity index 65% rename from bin/smi/startDockerLinux.py rename to bin/smi/smi_start_docker_linux.py index 748a11961..7f8783edb 100755 --- a/bin/smi/startDockerLinux.py +++ b/bin/smi/smi_start_docker_linux.py @@ -1,21 +1,22 @@ #!/usr/bin/env python3 - import os import platform import sys -sys.path.append(os.path.join(os.path.dirname(__file__), '..')) -import common as C +sys.path.append(os.path.join(os.path.dirname(__file__), "..")) +import common # noqa: E402 _PLAT = "-arm" if platform.processor() == "arm" else "" _COMPOSE_FILE_NAME = f"linux-dotnet{_PLAT}.yml" -_COMPOSE_FILE_PATH = (C.PROJ_ROOT / "utils/docker-compose" / _COMPOSE_FILE_NAME).resolve() -assert _COMPOSE_FILE_PATH.is_file() +_COMPOSE_FILE_PATH = f"{common.PROJ_ROOT}/utils/docker-compose/{_COMPOSE_FILE_NAME}" +assert os.path.isfile( + _COMPOSE_FILE_PATH, +), f"Compose file does not exist: {_COMPOSE_FILE_PATH}" def main() -> int: - parser = C.get_docker_parser() + parser = common.get_docker_parser() parser.add_argument( "db_password", ) @@ -23,7 +24,7 @@ def main() -> int: docker = "podman" if args.podman else "docker" - C.start_containers( + common.start_containers( _COMPOSE_FILE_PATH, docker=docker, env={ @@ -40,11 +41,14 @@ def main() -> int: # Start MongoDB replication cmd = ( - docker, "exec", + docker, + "exec", "mongodb", - "/usr/bin/mongo", "--eval", 'rs.initiate({_id:"rs0",members:[{_id:0,host:"localhost:27017"}]})', + "/usr/bin/mongo", + "--eval", + 'rs.initiate({_id:"rs0",members:[{_id:0,host:"localhost:27017"}]})', ) - C.run(cmd) + common.run(cmd) return 0 diff --git a/bin/smi/smi_stop_docker_linux.py b/bin/smi/smi_stop_docker_linux.py new file mode 100755 index 000000000..1f876cac1 --- /dev/null +++ b/bin/smi/smi_stop_docker_linux.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python3 +import os +import platform +import sys + +sys.path.append(os.path.join(os.path.dirname(__file__), "..")) +import common # noqa: E402 + +_PLATFORM = "-arm" if platform.processor() == "arm" else "" +_COMPOSE_FILE_NAME = f"linux-dotnet{_PLATFORM}.yml" +_COMPOSE_FILE_PATH = f"{common.PROJ_ROOT}/utils/docker-compose/{_COMPOSE_FILE_NAME}" +assert os.path.isfile( + _COMPOSE_FILE_PATH, +), f"Compose file does not exist: {_COMPOSE_FILE_PATH}" + + +def main() -> int: + + parser = common.get_docker_parser() + args = parser.parse_args() + + docker = "docker" if not args.podman else "podman" + + cmd = ( + f"{docker}-compose", + "-f", + _COMPOSE_FILE_PATH, + "down", + "--timeout", + "0", + ) + common.run(cmd) + + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/bin/smi/smi_test.py b/bin/smi/smi_test.py new file mode 100755 index 000000000..c7d4b809c --- /dev/null +++ b/bin/smi/smi_test.py @@ -0,0 +1,109 @@ +#!/usr/bin/env python3 +import argparse +import glob +import json +import os +import shutil +import sys +from collections.abc import Sequence + +import dotnet_common +import download_tessdata +import smi_build + +sys.path.append(os.path.join(os.path.dirname(__file__), "..")) +import common # noqa: E402 + + +def main(argv: Sequence[str] | None = None) -> int: + + parser = argparse.ArgumentParser() + dotnet_common.add_args(parser) + parser.add_argument( + "--no-build", + action="store_true", + ) + parser.add_argument( + "--no-coverage", + action="store_true", + ) + group = parser.add_mutually_exclusive_group() + group.add_argument( + "--test", + nargs=1, + help="Run a specific test class", + ) + group.add_argument( + "--unit-only", + action="store_true", + help="Run unit tests only", + ) + args, _ = parser.parse_known_args(argv) + + rc = download_tessdata.main([]) + if rc: + return rc + + if not args.no_build: + rc = smi_build.main(("--configuration", args.configuration)) + if rc: + return rc + + with open(f"{common.PROJ_ROOT}/global.json") as f: + global_json = json.load(f) + sdk_version_major = global_json["sdk"]["version"].split(".")[0] + + net_glob = { + x + for x in glob.glob( + f"{common.PROJ_ROOT}/**/net{sdk_version_major}", + recursive=True, + ) + } + for build_dir in net_glob: + try: + os.symlink( + f"{common.PROJ_ROOT}/data/microserviceConfigs/default.yaml", + f"{build_dir}/default.yaml", + ) + except FileExistsError: + pass + + cov_dir = f"{common.PROJ_ROOT}/coverage" + if not args.no_coverage: + if os.path.isdir(cov_dir): + shutil.rmtree(cov_dir) + + cmd = ("dotnet", "tool", "restore") + common.run(cmd) + + filter: tuple[str, ...] = ("--filter", args.test[0]) if args.test else () + unit_only = ( + ("--filter", "FullyQualifiedName!~SmiServices.IntegrationTests") + if args.unit_only + else () + ) + cmd = ( + "dotnet", + "dotnet-coverage", + "collect", + "--settings", + "coverage.settings", + "--", + "dotnet", + "test", + "-p:UseCurrentRuntimeIdentifier=True", + "-warnaserror", + "--configuration", + args.configuration, + "--no-build" if args.no_build else "", + *unit_only, + *filter, + ) + common.run(cmd) + + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/bin/smi/stopDockerLinux.py b/bin/smi/stopDockerLinux.py deleted file mode 100755 index de7ed1ca6..000000000 --- a/bin/smi/stopDockerLinux.py +++ /dev/null @@ -1,35 +0,0 @@ -#!/usr/bin/env python3 - -import os -import platform -import sys - -sys.path.append(os.path.join(os.path.dirname(__file__), '..')) -import common as C - -_PLATFORM = "-arm" if platform.processor() == "arm" else "" -_COMPOSE_FILE_NAME = f"linux-dotnet{_PLAT}.yml" -_COMPOSE_FILE_PATH = (C.PROJ_ROOT / "utils/docker-compose" / _COMPOSE_FILE_NAME).resolve() -assert _COMPOSE_FILE_PATH.is_file() - - -def main() -> int: - - parser = C.get_docker_parser() - args = parser.parse_args() - - docker = "docker" if not args.podman else "podman" - - cmd = ( - f"{docker}-compose", - "-f", _COMPOSE_FILE_PATH, - "down", - "--timeout", 0, - ) - C.run(cmd) - - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/bin/smi/test.py b/bin/smi/test.py deleted file mode 100755 index 732889e57..000000000 --- a/bin/smi/test.py +++ /dev/null @@ -1,102 +0,0 @@ -#!/usr/bin/env python3 - -import argparse -import glob -import json -import os -import platform -import shutil -import subprocess -import sys -from pathlib import Path -from typing import Optional -from typing import Sequence - -sys.path.append(os.path.join(os.path.dirname(__file__), '..')) -import common as C - -import build as DB -import dotnetCommon as DC -import downloadTessdata - - -def main(argv: Optional[Sequence[str]] = None) -> int: - - parser = argparse.ArgumentParser() - DC.add_args(parser) - parser.add_argument( - "--no-build", - action="store_true", - ) - parser.add_argument( - "--no-coverage", - action="store_true", - ) - group = parser.add_mutually_exclusive_group() - group.add_argument( - "--test", - nargs=1, - help="Run a specific test class", - ) - group.add_argument( - "--unit-only", - action="store_true", - help="Run unit tests only" - ) - args, _ = parser.parse_known_args(argv) - - rc = downloadTessdata.main([]) - if rc: - return rc - - if not args.no_build: - rc = DB.main(("--configuration", args.configuration)) - if rc: - return rc - - with open(C.PROJ_ROOT / "global.json") as f: - global_json = json.load(f) - sdk_version_major = global_json["sdk"]["version"].split(".")[0] - - netX_glob = { - Path(x) for x in - glob.glob(f"{C.PROJ_ROOT}/**/net{sdk_version_major}", recursive=True) - } - for build_dir in netX_glob: - try: - os.symlink( - Path(f"{C.PROJ_ROOT}/data/microserviceConfigs/default.yaml").resolve(), - build_dir / "default.yaml", - ) - except FileExistsError: - pass - - cov_dir = C.PROJ_ROOT / "coverage" - if not args.no_coverage: - if cov_dir.is_dir(): - shutil.rmtree(cov_dir) - - cmd = ("dotnet", "tool", "restore") - C.run(cmd) - - f = ("--filter", args.test[0]) if args.test else () - unit_only = ("--filter", "FullyQualifiedName!~SmiServices.IntegrationTests") if args.unit_only else () - cmd = ( - "dotnet", "dotnet-coverage", "collect", - "--settings", "coverage.settings", - "--", - "dotnet", "test", - "-p:UseCurrentRuntimeIdentifier=True", - "-warnaserror", - "--configuration", args.configuration, - "--no-build" if args.no_build else "", - *unit_only, - *f, - ) - C.run(cmd) - - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/bin/smi/writeDatabaseStrings.py b/bin/smi/writeDatabaseStrings.py index 6bcbce80a..e36047478 100755 --- a/bin/smi/writeDatabaseStrings.py +++ b/bin/smi/writeDatabaseStrings.py @@ -1,17 +1,20 @@ #!/usr/bin/env python3 - import argparse import os import sys -sys.path.append(os.path.join(os.path.dirname(__file__), '..')) -import common as C +sys.path.append(os.path.join(os.path.dirname(__file__), "..")) +import common # noqa: E402 -_RELATIONAL_YAML = (C.PROJ_ROOT / "tests/SmiServices.UnitTests/RelationalDatabases.yaml").resolve() -assert _RELATIONAL_YAML.is_file() +_RELATIONAL_YAML = ( + f"{common.PROJ_ROOT}/tests/SmiServices.UnitTests/RelationalDatabases.yaml" +) +assert os.path.isfile(_RELATIONAL_YAML), f"YAML file does not exist: {_RELATIONAL_YAML}" -_TEST_DBS_TXT = (C.PROJ_ROOT / "tests/SmiServices.UnitTests/TestDatabases.txt").resolve() -assert _TEST_DBS_TXT.is_file() +_TEST_DBS_TXT = f"{common.PROJ_ROOT}/tests/SmiServices.UnitTests/TestDatabases.txt" +assert os.path.isfile( + _TEST_DBS_TXT, +), f"Test databases file does not exist: {_TEST_DBS_TXT}" def main() -> int: @@ -31,11 +34,16 @@ def main() -> int: if "localdb" in args.mssql_server: f.write(f"SqlServer: 'Server={args.mssql_server};'\n") else: - f.write(f"SqlServer: 'Server={args.mssql_server};User Id=sa;Password={args.db_password};TrustServerCertificate=true;'\n") + f.write( + f"SqlServer: 'Server={args.mssql_server};User Id=sa;" + f"Password={args.db_password};TrustServerCertificate=true;'\n", + ) # NOTE(rkm 2022-02-27) We don't run MySQL in Windows in GitHub actions - if not (os.name == "nt" and C.is_ci()): - f.write(f"MySql: 'server=127.0.0.1;Uid=root;Pwd={args.db_password};sslmode=None'\n") + if not (os.name == "nt" and common.is_ci()): + f.write( + f"MySql: 'server=127.0.0.1;Uid=root;Pwd={args.db_password};sslmode=None'\n", + ) with open(_RELATIONAL_YAML) as f: print(f"{_RELATIONAL_YAML}:") @@ -46,8 +54,10 @@ def main() -> int: f.write("Prefix: TEST_\n") f.write("Username: sa\n") f.write(f"Password: {args.db_password}\n") - if not (os.name == "nt" and C.is_ci()): - f.write(f"MySql: server=127.0.0.1;Uid=root;Pwd={args.db_password};sslmode=None\n") + if not (os.name == "nt" and common.is_ci()): + f.write( + f"MySql: server=127.0.0.1;Uid=root;Pwd={args.db_password};sslmode=None\n", + ) with open(_TEST_DBS_TXT) as f: print(f"{_TEST_DBS_TXT}:") diff --git a/news/2022-meta.md b/news/2022-meta.md new file mode 100644 index 000000000..7ebc5d57a --- /dev/null +++ b/news/2022-meta.md @@ -0,0 +1 @@ +Add pre-commit python hooks & tidy helper scripts