From 8bd0513664734aeb99a0fe10e0adffa11c8682f1 Mon Sep 17 00:00:00 2001 From: Leon Wright Date: Wed, 18 Sep 2024 13:23:59 +0800 Subject: [PATCH 1/5] chore: Update Ubuntu / Python This updates the container to Ubuntu 24.04, which comes with Python 3.12 --- netkan/Dockerfile | 42 ++++++++++++++++++------------------------ 1 file changed, 18 insertions(+), 24 deletions(-) diff --git a/netkan/Dockerfile b/netkan/Dockerfile index fc7a387..e1dadc4 100644 --- a/netkan/Dockerfile +++ b/netkan/Dockerfile @@ -1,4 +1,4 @@ -FROM ubuntu:22.04 as ubuntu_with_python +FROM ubuntu:24.04 AS ubuntu_with_python # Don't prompt for time zone ENV DEBIAN_FRONTEND=noninteractive @@ -6,53 +6,47 @@ ENV DEBIAN_FRONTEND=noninteractive # Install Git and Python RUN apt-get update \ && apt-get install -y --no-install-recommends \ - git libffi-dev openssh-client \ - python3 python-is-python3 \ - && apt-get clean + git libffi-dev openssh-client \ + python3 python-is-python3 ca-certificates curl && \ + apt-get clean && \ + curl -sSL https://bootstrap.pypa.io/get-pip.py -o get-pip.py && \ + rm /usr/lib/python3.12/EXTERNALLY-MANAGED && \ + python3 get-pip.py --no-input && rm get-pip.py +RUN groupmod -n 'netkan' 'ubuntu' && \ + usermod -l 'netkan' -d /home/netkan -m ubuntu -FROM ubuntu_with_python as base -RUN apt-get install -y --no-install-recommends \ - python3-pip python3-setuptools python3-dev -RUN useradd -ms /bin/bash netkan -ADD . /netkan +FROM ubuntu_with_python AS base +COPY . /netkan WORKDIR /netkan -RUN pip install pip --upgrade RUN chown -R netkan:netkan /netkan USER netkan RUN pip install --user . --no-warn-script-location -FROM ubuntu_with_python as production +FROM ubuntu_with_python AS production COPY --from=base /home/netkan/.local /home/netkan/.local -RUN useradd -Ms /bin/bash netkan RUN chown -R netkan:netkan /home/netkan WORKDIR /home/netkan USER netkan -ADD .gitconfig . -ENV PATH "$PATH:/home/netkan/.local/bin" +COPY .gitconfig . +ENV PATH="$PATH:/home/netkan/.local/bin" RUN /home/netkan/.local/bin/netkan --help ENTRYPOINT [".local/bin/netkan"] CMD ["--help"] -FROM production as test +FROM production AS test USER root -RUN apt-get install -y --no-install-recommends \ - python3-pip python3-setuptools python3-dev -RUN pip install pip --upgrade -ADD . /netkan +COPY . /netkan RUN chown -R netkan:netkan /netkan USER netkan WORKDIR /netkan RUN pip install --user .[test] RUN /home/netkan/.local/bin/pytest -v -FROM production as dev +FROM production AS dev USER root -RUN apt-get install -y --no-install-recommends \ - python3-pip python3-setuptools python3-dev -RUN pip install pip --upgrade ADD . /netkan RUN chown -R netkan:netkan /netkan -ADD run_dev.sh /usr/local/bin/ +COPY run_dev.sh /usr/local/bin/ USER netkan RUN pip install --user /netkan/.[development] ENTRYPOINT ["/usr/local/bin/run_dev.sh"] From 61c232e8c03058cc94609b27ca95b20953dc2d01 Mon Sep 17 00:00:00 2001 From: Leon Wright Date: Wed, 18 Sep 2024 13:24:57 +0800 Subject: [PATCH 2/5] chore: Switch to PyProject.toml This switches away from setup.py to the now standardised pyproject.toml. --- netkan/.coveragerc | 7 --- netkan/.pylintrc | 14 ------ netkan/MANIFEST.in | 2 + netkan/mypy.ini | 15 ------ netkan/pyproject.toml | 110 ++++++++++++++++++++++++++++++++++++++++++ netkan/pytest.ini | 7 --- netkan/setup.py | 68 -------------------------- 7 files changed, 112 insertions(+), 111 deletions(-) delete mode 100644 netkan/.coveragerc delete mode 100644 netkan/.pylintrc create mode 100644 netkan/MANIFEST.in delete mode 100644 netkan/mypy.ini create mode 100644 netkan/pyproject.toml delete mode 100644 netkan/pytest.ini delete mode 100644 netkan/setup.py diff --git a/netkan/.coveragerc b/netkan/.coveragerc deleted file mode 100644 index c7109ee..0000000 --- a/netkan/.coveragerc +++ /dev/null @@ -1,7 +0,0 @@ -[report] -include = - netkan/* - -exclude_lines = - pragma: no cover - if TYPE_CHECKING: diff --git a/netkan/.pylintrc b/netkan/.pylintrc deleted file mode 100644 index 9afe8c1..0000000 --- a/netkan/.pylintrc +++ /dev/null @@ -1,14 +0,0 @@ -[MESSAGES CONTROL] -disable = - line-too-long, - duplicate-code, - missing-module-docstring, - missing-class-docstring, - missing-function-docstring, - too-few-public-methods, - too-many-instance-attributes, - too-many-statements, - too-many-return-statements, - too-many-branches, - too-many-arguments, - consider-using-with, diff --git a/netkan/MANIFEST.in b/netkan/MANIFEST.in new file mode 100644 index 0000000..09c8303 --- /dev/null +++ b/netkan/MANIFEST.in @@ -0,0 +1,2 @@ +global-include *.md *.jinja2 *.graphql +recursive-exclude tests * diff --git a/netkan/mypy.ini b/netkan/mypy.ini deleted file mode 100644 index dee230d..0000000 --- a/netkan/mypy.ini +++ /dev/null @@ -1,15 +0,0 @@ -[mypy] -ignore_missing_imports = true -warn_redundant_casts = true -show_error_context = true -show_column_numbers = true -show_error_codes = true - -# Don't increase strictness for tests -[mypy-netkan.*] -check_untyped_defs = true -disallow_incomplete_defs = true -disallow_untyped_defs = true -disallow_any_generics = true -warn_unreachable = true -strict_equality = true diff --git a/netkan/pyproject.toml b/netkan/pyproject.toml new file mode 100644 index 0000000..2ad5024 --- /dev/null +++ b/netkan/pyproject.toml @@ -0,0 +1,110 @@ +[build-system] +requires = ["setuptools"] +build-backend = "setuptools.build_meta" + +[project] +name = "netkan" +version = "1.0" +dependencies = [ + "boto3", + "click", + "gitpython", + "pynamodb", + # 2019-11-01 capping to 2.8.0 - https://github.com/boto/botocore/commit/e87e7a745fd972815b235a9ee685232745aa94f9 + "python-dateutil>=2.1,<2.8.1", + "requests", + "flask", + "jinja2", + "internetarchive!=3.0.1", + "gunicorn>=19.9,!=20.0.0", + "discord.py>=1.6.0,<=1.7.3", + "PyGithub", + "ruamel.yaml", +] +requires-python = ">=3.8" +authors = [{ name = "Leon Wright", email = "techman83@gmail.com" }] +description = "NetKAN Infra" + +[project.urls] +Repository = "https://github.com/KSP-CKAN/NetKAN-infra/" +Issues = "https://github.com/KSP-CKAN/NetKAN-infra/issues" + +[project.scripts] +netkan = "netkan.cli:netkan" + +[project.optional-dependencies] +development = [ + "ptvsd", + "autopep8", + "boto3-stubs[essential,cloudwatch]", + "coverage", + "troposphere", + "pytest", + "mypy", + "pytest-mypy", + "pylint", + "pytest-pylint", + "types-python-dateutil", + "types-click", + "types-requests", + "types-Flask", + "types-Jinja2", +] +test = [ + "boto3-stubs[essential,cloudwatch]", + "coverage", + "pytest", + "mypy", + "pytest-mypy", + "pylint", + "pytest-pylint", + "types-python-dateutil", + "types-click", + "types-requests", + "types-Flask", + "types-Jinja2", +] + +[tool.coverage.run] +branch = true +omit = ["tests/*"] + +[tool.coverage.report] +exclude_lines = ["if TYPE_CHECKING:", " pass"] + +[tool.pytest.ini_options] +python_files = "tests/__init__.py" +addopts = "-p no:cacheprovider --mypy --pylint" +filterwarnings = ["ignore", "default:::netkan.*", "default:::tests.*"] + +[tool.mypy] +ignore_missing_imports = true +warn_redundant_casts = true +show_error_context = true +show_column_numbers = true +show_error_codes = true + +[[tool.mypy.overrides]] +module = "netkan.*" +check_untyped_defs = true +disallow_incomplete_defs = true +disallow_untyped_defs = true +disallow_any_generics = true +warn_unreachable = true +strict_equality = true + +[tool.pylint."MESSAGES CONTROL"] +disable = """ + line-too-long, + duplicate-code, + missing-module-docstring, + missing-class-docstring, + missing-function-docstring, + too-few-public-methods, + too-many-instance-attributes, + too-many-statements, + too-many-return-statements, + too-many-branches, + too-many-arguments, + consider-using-with, +""" diff --git a/netkan/pytest.ini b/netkan/pytest.ini deleted file mode 100644 index b1fbcc3..0000000 --- a/netkan/pytest.ini +++ /dev/null @@ -1,7 +0,0 @@ -[pytest] -python_files = tests/__init__.py -addopts = -p no:cacheprovider --mypy --pylint -filterwarnings = - ignore - default:::netkan.* - default:::tests.* diff --git a/netkan/setup.py b/netkan/setup.py deleted file mode 100644 index 9653780..0000000 --- a/netkan/setup.py +++ /dev/null @@ -1,68 +0,0 @@ -#!/usr/bin/env python - -from setuptools import setup, find_packages - -setup( - name='netkan_indexer', - version='1.0', - description='NetKAN Indexer', - author='Leon Wright', - author_email='techman83@gmail.com', - packages=find_packages(), - package_data={ - "": ["*.md", "*.jinja2", "*.graphql"], - }, - install_requires=[ - 'boto3', - 'click', - 'gitpython', - 'pynamodb', - # 2019-11-01 capping to 2.8.0 - https://github.com/boto/botocore/commit/e87e7a745fd972815b235a9ee685232745aa94f9 - 'python-dateutil>=2.1,<2.8.1', - 'requests', - 'flask', - 'jinja2', - 'internetarchive!=3.0.1', - 'gunicorn>=19.9,!=20.0.0', - 'discord.py>=1.6.0,<=1.7.3', - 'PyGithub', - 'ruamel.yaml', - ], - entry_points={ - 'console_scripts': [ - 'netkan=netkan.cli:netkan', - ], - }, - extras_require={ - 'development': [ - 'ptvsd', - 'autopep8', - 'boto3-stubs[essential,cloudwatch]', - 'coverage', - 'troposphere', - 'pytest', - 'pytest-mypy', - 'mypy', - 'pytest-pylint', - 'pylint', - 'types-python-dateutil', - 'types-click', - 'types-requests', - 'types-Flask', - 'types-Jinja2', - ], - 'test': [ - 'boto3-stubs[essential,cloudwatch]', - 'coverage', - 'pytest', - 'pytest-mypy', - 'mypy', - 'pytest-pylint', - 'types-python-dateutil', - 'types-click', - 'types-requests', - 'types-Flask', - 'types-Jinja2', - ] - }, -) From 6207434e6ccf1c54443b871e3937a3f377f3bd68 Mon Sep 17 00:00:00 2001 From: Leon Wright Date: Wed, 18 Sep 2024 13:40:27 +0800 Subject: [PATCH 3/5] ci: Update builds to Python 3.12 --- .github/workflows/coverage-build.yml | 3 +-- .github/workflows/deploy.yml | 3 +-- .github/workflows/test.yml | 3 +-- 3 files changed, 3 insertions(+), 6 deletions(-) diff --git a/.github/workflows/coverage-build.yml b/.github/workflows/coverage-build.yml index 8073b2f..d17e154 100644 --- a/.github/workflows/coverage-build.yml +++ b/.github/workflows/coverage-build.yml @@ -16,9 +16,8 @@ jobs: - name: Setup Python uses: actions/setup-python@v5 with: - python-version: "3.11" + python-version: "3.12" cache: pip - cache-dependency-path: netkan/setup.py - name: Install test dependencies run: pip install .[test] - name: force our git config diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index a035d39..5eb7f0b 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -67,9 +67,8 @@ jobs: - name: Set up Python uses: actions/setup-python@v5 with: - python-version: 3.11 + python-version: 3.12 cache: pip - cache-dependency-path: netkan/setup.py - name: Install Dependencies run: pip install netkan/. - name: Re-deploy Containers diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index e5c9f45..b671fd4 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -12,9 +12,8 @@ jobs: - name: Setup Python uses: actions/setup-python@v5 with: - python-version: 3.11 + python-version: 3.12 cache: pip - cache-dependency-path: netkan/setup.py - name: Install test dependencies working-directory: netkan run: pip install .[test] From 4dbb492186093fea47e8f372dcc4c630786727b0 Mon Sep 17 00:00:00 2001 From: Leon Wright Date: Wed, 18 Sep 2024 13:41:12 +0800 Subject: [PATCH 4/5] build: Add toml extension as recommends --- .vscode/extensions.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.vscode/extensions.json b/.vscode/extensions.json index 7c69955..d200844 100644 --- a/.vscode/extensions.json +++ b/.vscode/extensions.json @@ -1,6 +1,7 @@ { "recommendations": [ "ms-vscode-remote.vscode-remote-extensionpack", - "ms-python.python" + "ms-python.python", + "tamasfe.even-better-toml" ] } From 341b1ef443bb741f0eb4b0bbaf1f7fbc37233ede Mon Sep 17 00:00:00 2001 From: Leon Wright Date: Wed, 18 Sep 2024 17:09:11 +0800 Subject: [PATCH 5/5] chore: Tidy up docker compose --- docker-compose.yml | 35 +++++++---------------------------- 1 file changed, 7 insertions(+), 28 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 27f7f95..6104340 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,5 +1,3 @@ -version: '3.4' - volumes: certbot: @@ -11,15 +9,15 @@ services: environment: SSH_KEY: ${CKAN_NETKAN_SSHKEY} CKANMETA_REMOTES: ${CKAN_METADATA_PATHS} - CKANMETA_USER: ${CKAN_METADATA_USER} - CKANMETA_REPOS: ${CKAN_METADATA_REPOS} + CKAN_USER: ${CKAN_METADATA_USER} + CKAN_REPOS: ${CKAN_METADATA_REPOS} AWS_DEFAULT_REGION: ${CKAN_AWS_DEFAULT_REGION} AWS_SECRET_ACCESS_KEY: ${CKAN_AWS_SECRET_ACCESS_KEY} AWS_ACCESS_KEY_ID: ${CKAN_AWS_ACCESS_KEY_ID} GH_Token: ${CKAN_GH_Token} SQS_QUEUE: OutboundDev.fifo SQS_TIMEOUT: 30 - STATUS_DB: DevNetKANStatus + STATUS_DB: DevMultiKANStatus DISCORD_WEBHOOK_ID: ${DISCORD_WEBHOOK_ID} DISCORD_WEBHOOK_TOKEN: ${DISCORD_WEBHOOK_TOKEN} volumes: @@ -30,6 +28,7 @@ services: context: netkan/. target: dev environment: + GH_Token: ${CKAN_GH_Token} NETKAN_REMOTES: ${NETKAN_METADATA_PATHS} SSH_KEY: ${CKAN_NETKAN_SSHKEY} CKANMETA_REMOTES: ${CKAN_METADATA_PATHS} @@ -37,26 +36,6 @@ services: AWS_SECRET_ACCESS_KEY: ${CKAN_AWS_SECRET_ACCESS_KEY} AWS_ACCESS_KEY_ID: ${CKAN_AWS_ACCESS_KEY_ID} INFLATION_QUEUES: ksp=InboundDevKsp.fifo ksp2=InboundDevKsp2.fifo - GAME_ID: ksp2 - MAX_QUEUED: 1 - DISCORD_WEBHOOK_ID: ${DISCORD_WEBHOOK_ID} - DISCORD_WEBHOOK_TOKEN: ${DISCORD_WEBHOOK_TOKEN} - volumes: - - ./netkan:/home/netkan/netkan - command: scheduler --dev - scheduler: - build: - context: netkan/. - target: dev - environment: - NETKAN_REMOTES: ${NETKAN_METADATA_PATHS} - SSH_KEY: ${CKAN_NETKAN_SSHKEY} - CKANMETA_REMOTES: ${CKAN_METADATA_PATHS} - AWS_DEFAULT_REGION: ${CKAN_AWS_DEFAULT_REGION} - AWS_SECRET_ACCESS_KEY: ${CKAN_AWS_SECRET_ACCESS_KEY} - AWS_ACCESS_KEY_ID: ${CKAN_AWS_ACCESS_KEY_ID} - INFLATION_QUEUES: ksp=InboundDevKsp.fifo ksp2=InboundDevKsp2.fifo - GAME_ID: ksp2 MAX_QUEUED: 1 DISCORD_WEBHOOK_ID: ${DISCORD_WEBHOOK_ID} DISCORD_WEBHOOK_TOKEN: ${DISCORD_WEBHOOK_TOKEN} @@ -109,8 +88,8 @@ services: DISCORD_WEBHOOK_ID: ${DISCORD_WEBHOOK_ID} DISCORD_WEBHOOK_TOKEN: ${DISCORD_WEBHOOK_TOKEN} CKANMETA_REMOTES: ${CKAN_METADATA_PATHS} - CKANMETA_USER: ${CKAN_METADATA_USER} - CKANMETA_REPOS: ${CKAN_METADATA_REPOS} + CKAN_USER: ${CKAN_METADATA_USER} + CKAN_REPOS: ${CKAN_METADATA_REPOS} entrypoint: .local/bin/gunicorn command: -b 0.0.0.0:5000 --access-logfile - "netkan.webhooks:create_app()" adder: @@ -145,7 +124,7 @@ services: context: netkan/. target: dev environment: - STATUS_DB: DevNetKANStatus + STATUS_DB: DevMultiKANStatus STATUS_BUCKET: ckan-test-status AWS_DEFAULT_REGION: ${CKAN_AWS_DEFAULT_REGION} AWS_SECRET_ACCESS_KEY: ${CKAN_AWS_SECRET_ACCESS_KEY}