From fbfff0aec40e65a21db28e64c6435f2df7ecd7dc Mon Sep 17 00:00:00 2001 From: Daniel Thorn Date: Fri, 1 Nov 2024 09:52:34 -0700 Subject: [PATCH] Add docker dev environment --- .devcontainer/Dockerfile | 26 ++++++++++++++++ .devcontainer/devcontainer.json | 28 +++++++++++++++++ .dockerignore | 11 +++++++ .github/workflows/build.yaml | 15 ++++----- .gitignore | 2 ++ Makefile | 55 +++++++++++++++++++++++++++++++++ bin/lint.sh | 39 +++++++++++++++++++++++ bin/test.sh | 27 ++++++++++++++++ docker-compose.yml | 36 ++++++++++++++++++++- docker/Dockerfile | 35 +++++++++++++++++++++ docker/config/local_dev.env | 8 +++++ pyproject.toml | 1 + requirements.in | 1 + requirements.txt | 4 +++ 14 files changed, 278 insertions(+), 10 deletions(-) create mode 100644 .devcontainer/Dockerfile create mode 100644 .devcontainer/devcontainer.json create mode 100644 .dockerignore create mode 100644 Makefile create mode 100755 bin/lint.sh create mode 100755 bin/test.sh create mode 100644 docker/Dockerfile create mode 100644 docker/config/local_dev.env diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile new file mode 100644 index 0000000..b12c06e --- /dev/null +++ b/.devcontainer/Dockerfile @@ -0,0 +1,26 @@ +# This should be kept in sync with the python version used in docker/Dockerfile and +# docker/images/fakesentry/Dockerfile +FROM --platform=linux/amd64 mcr.microsoft.com/devcontainers/python:3.11-bullseye@sha256:105bf6a63ab025206f019a371a735fec6553db0be520030c7a2fd0e002947232 + +ARG userid=10001 +ARG groupid=10001 + +WORKDIR /app + +# add a non-privileged user for installing and running the application +# We use --non-unique in case $groupid/$userid collide with the existing "vscode" user. +# useradd -g app --uid $userid --non-unique --shell /usr/sbin/nologin --create-home app && \ +RUN groupadd --gid $groupid --non-unique app && \ + useradd -g app --uid $userid --non-unique --shell /bin/bash --create-home app && \ + chown app:app /app/ + +# Install Debian packages +RUN apt-get update && \ + apt-get install -y ripgrep tig + +# Install Python dependencies +COPY requirements.txt /app/ +RUN pip install -U 'pip>=20' && \ + pip install --no-cache-dir --no-deps --only-binary :all: -r requirements.txt && \ + pip install --no-cache-dir ipython && \ + pip check --disable-pip-version-check diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 0000000..4e423d0 --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,28 @@ +{ + "name": "Obs Common", + "dockerComposeFile": [ + "../docker-compose.yml" + ], + "service": "devcontainer", + "runServices": [ + "devcontainer" + ], + "shutdownAction": "none", + "workspaceFolder": "/app", + "customizations": { + "vscode": { + "settings": { + "python.testing.unittestEnabled": false, + "python.testing.pytestEnabled": true, + "[python]": { + "editor.defaultFormatter": "charliermarsh.ruff" + } + }, + "extensions": [ + "charliermarsh.ruff" + ] + } + }, + "remoteUser": "app", + "updateRemoteUserUID": false +} diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..c09f17d --- /dev/null +++ b/.dockerignore @@ -0,0 +1,11 @@ +*.py[co] +*.sw[po] +.docker-build +.DS_Store +.env +.pytest_cache +.python-version +build +dist/ +obs_common.egg-info/ +venv/ diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index eaf4143..8622f7c 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -26,6 +26,7 @@ jobs: run: | python -m venv venv venv/bin/pip install -r requirements.txt + venv/bin/pip install -e . --no-deps - name: Verify requirements.txt if: ${{ matrix.python-version == '3.11' }} run: | @@ -34,22 +35,18 @@ jobs: - name: Run lint check if: ${{ matrix.python-version == '3.11' }} run: | - venv/bin/ruff format --check obs_common tests - venv/bin/ruff check obs_common tests + PATH="venv/bin:$PATH" bin/lint.sh - name: Run tests env: SENTRY_DSN: http://public@localhost:8090/1 STORAGE_EMULATOR_HOST: http://localhost:8001 + PUBSUB_EMULATOR_HOST: http://localhost:5010 run: | - docker compose up -d fakesentry gcs-emulator - venv/bin/pytest tests/ + docker compose up -d fakesentry gcs-emulator pubsub + # Run outside docker because we are testing the matrix python version + PATH="venv/bin:$PATH" bin/test.sh # stop services immediate and ignore errors docker compose down -t0 || true - - name: License Check - if: ${{ matrix.python-version == '3.11' }} - run: | - venv/bin/pip install -e . --no-deps - venv/bin/license-check build-and-release: permissions: diff --git a/.gitignore b/.gitignore index b4554ff..c09f17d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,8 @@ *.py[co] *.sw[po] +.docker-build .DS_Store +.env .pytest_cache .python-version build diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..b4485a3 --- /dev/null +++ b/Makefile @@ -0,0 +1,55 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at https://mozilla.org/MPL/2.0/. + +# Include .env and export it so variables set in there are available +# in Makefile. +include .env +export + +.DEFAULT_GOAL := help +.PHONY: help +help: + @echo "Usage: make RULE" + @echo "" + @grep -E '^[a-zA-Z0-9_-]+:.*?## .*$$' Makefile \ + | grep -v grep \ + | sed -n 's/^\(.*\): \(.*\)##\(.*\)/\1\3/p' \ + | column -t -s '|' + @echo "" + @echo "Adjust your .env file to set configuration." + +.docker-build: + make build + +.env: + touch .env + +.PHONY: build +build: .env ## | Build docker images. + docker compose --progress plain build + touch .docker-build + +.PHONY: shell +shell: .env .docker-build ## | Open a shell in docker. + docker compose run --rm shell + +.PHONY: devcontainer +devcontainer: .env .docker-build ## | Run VS Code development container. + docker compose up --detach devcontainer + +.PHONY: rebuildreqs +rebuildreqs: .env .docker-build ## | Rebuild requirements.txt file after requirements.in changes. + docker compose run --rm --no-deps shell pip-compile --allow-unsafe --generate-hashes --strip-extras --quiet + +.PHONY: lint +lint: .env .docker-build ## | Lint code. + docker compose run --rm --no-deps shell bin/lint.sh + +.PHONY: lintfix +lintfix: .env .docker-build ## | Reformat code. + docker compose run --rm --no-deps shell bin/lint.sh --fix + +.PHONY: test +test: .env .docker-build ## | Run tests. + docker compose run --rm shell bin/test.sh diff --git a/bin/lint.sh b/bin/lint.sh new file mode 100755 index 0000000..63ac369 --- /dev/null +++ b/bin/lint.sh @@ -0,0 +1,39 @@ +#!/bin/bash + +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at https://mozilla.org/MPL/2.0/. + +# Usage: bin/run_lint.sh [--fix] +# +# Runs linting and code fixing. +# +# This should be called from inside a container or venv. + +set -euo pipefail + +FILES="bin obs_common tests" +PYTHON_VERSION=$(python --version) + + +if [[ "${1:-}" == "--fix" ]]; then + echo ">>> ruff fix (${PYTHON_VERSION})" + ruff format $FILES + ruff check --fix $FILES + +else + echo ">>> ruff (${PYTHON_VERSION})" + ruff check $FILES + ruff format --check $FILES + + echo ">>> license check (${PYTHON_VERSION})" + if [[ -d ".git" ]]; then + # If the .git directory exists, we can let license-check do + # git ls-files. + license-check + else + # The .git directory doesn't exist, so run it on all the Python + # files in the tree. + license-check . + fi +fi diff --git a/bin/test.sh b/bin/test.sh new file mode 100755 index 0000000..edfe544 --- /dev/null +++ b/bin/test.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at https://mozilla.org/MPL/2.0/. + +# Usage: bin/run_tests.sh +# +# Runs tests. +# +# This should be called from inside a container and after the dependent +# services have been launched. It depends on: +# +# * elasticsearch +# * postgresql + +set -euo pipefail + +echo ">>> pytest" + +# Wait for services to be ready (both have the same endpoint url) +urlwait "http://${PUBSUB_EMULATOR_HOST}" 10 +urlwait "${STORAGE_EMULATOR_HOST}/storage/v1/b" 10 +waitfor --verbose --codes=200,404 "${SENTRY_DSN}" + +# Run tests +pytest $@ diff --git a/docker-compose.yml b/docker-compose.yml index d45c9dc..7a7bcaa 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,9 +1,43 @@ services: + shell: + build: + context: . + dockerfile: docker/Dockerfile + args: + userid: ${USE_UID:-10001} + groupid: ${USE_GID:-10001} + image: local/obs-common-shell + env_file: + - docker/config/local_dev.env + links: + - fakesentry + - gcs-emulator + - pubsub + volumes: + - .:/app + + devcontainer: + build: + dockerfile: .devcontainer/Dockerfile + args: + userid: ${USE_UID:-10001} + groupid: ${USE_GID:-10001} + image: local/obs-common-devcontainer + entrypoint: ["sleep", "inf"] + env_file: + - docker/config/local_dev.env + links: + - fakesentry + - gcs-emulator + - pubsub + volumes: + - .:/app + # https://github.com/willkg/kent fakesentry: build: context: docker/images/fakesentry - image: local/tecken_fakesentry + image: local/obs-common-fakesentry ports: - "${EXPOSE_SENTRY_PORT:-8090}:8090" command: run --host 0.0.0.0 --port 8090 diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 0000000..598e009 --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,35 @@ +# This should be kept in sync with the python version used in .devcontainer/Dockerfile and +# docker/images/fakesentry/Dockerfile +FROM --platform=linux/amd64 python:3.11.10-slim-bullseye@sha256:f6a64ef0a5cc14855b15548056a8fc77f4c3526b79883fa6709a8e23f676ac34 + +# Set up user and group +ARG groupid=10001 +ARG userid=10001 + +WORKDIR /app/ +RUN groupadd --gid $groupid app && \ + useradd -g app --uid $userid --shell /usr/sbin/nologin --create-home app && \ + chown app:app /app/ + +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + git \ + ripgrep \ + tig && \ + apt-get autoremove -y && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* + +# Install Python dependencies +COPY requirements.txt /app/ + +RUN pip install -U 'pip>=20' && \ + pip install --no-cache-dir --no-deps --only-binary :all: -r requirements.txt && \ + pip install --no-cache-dir ipython && \ + pip check --disable-pip-version-check + +COPY . /app + +RUN pip install -e . --no-deps + +CMD ["/bin/bash"] diff --git a/docker/config/local_dev.env b/docker/config/local_dev.env new file mode 100644 index 0000000..4907f85 --- /dev/null +++ b/docker/config/local_dev.env @@ -0,0 +1,8 @@ +# Set Pub/Sub library to use emulator +PUBSUB_EMULATOR_HOST=pubsub:5010 + +# Set GCS library to use emulator +STORAGE_EMULATOR_HOST=http://gcs-emulator:8001 + +# Set up fakesentry +SENTRY_DSN=http://public@fakesentry:8090/1 diff --git a/pyproject.toml b/pyproject.toml index ecc890b..1c2469d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,6 +23,7 @@ service-status = "obs_common.service_status:main" gcs-cli = "obs_common.gcs_cli:main" pubsub-cli = "obs_common.pubsub_cli:main" sentry-wrap = "obs_common.sentry_wrap:cli_main" +waitfor = "obs_common.waitfor:main" [build-system] requires = ["setuptools", "setuptools_scm[toml]>=6.2"] diff --git a/requirements.in b/requirements.in index 1455054..9ea7e95 100644 --- a/requirements.in +++ b/requirements.in @@ -8,3 +8,4 @@ requests==2.32.3 ruff==0.7.1 sentry-sdk==2.17.0 twine==5.1.1 +urlwait==1.0 diff --git a/requirements.txt b/requirements.txt index bbb77a5..b7ceee0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -640,6 +640,10 @@ urllib3==2.2.2 \ # requests # sentry-sdk # twine +urlwait==1.0 \ + --hash=sha256:a9bf2da792fa6983fa93f6360108e16615066ab0f9cfb7f53e5faee5f5dffaac \ + --hash=sha256:eae2c20001efc915166cac79c04bac0088ad5787ec64b36f27afd2f359953b2b + # via -r requirements.in wheel==0.44.0 \ --hash=sha256:2376a90c98cc337d18623527a97c31797bd02bad0033d41547043a1cbfbe448f \ --hash=sha256:a29c3f2817e95ab89aa4660681ad547c0e9547f20e75b0562fe7723c9a2a9d49