Add Docker configuration and containerization guidelines (#3)

.github/workflows/monorepo_build_docker.yaml

@@ -0,0 +1,36 @@
+name: Build Docker Images For Mono-Repo Structure
+
+on:
+  pull_request:
+    branches:
+      - main
+
+jobs:
+  mono-repo-build-docker:
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: ${{ matrix.directory }}
+    strategy:
+      matrix:
+        directory: [
+          monorepo/services/service_a,
+          monorepo/services/service_b
+        ]
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Extract service name
+        id: extract-service-name
+        run: echo "service_name=$(basename ${{ matrix.directory }})" >> $GITHUB_OUTPUT
+
+      - name: Build Docker
+        uses: docker/build-push-action@v6
+        with:
+          context: ${{ matrix.directory }}
+          tags: ${{ steps.extract-service-name.outputs.service_name }}:latest

.github/workflows/standard_build_docker.yaml

@@ -0,0 +1,26 @@
+name: Build Docker Image For Standard Structure
+
+on:
+  pull_request:
+    branches:
+      - main
+
+jobs:
+  standard-repo-build-docker:
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: standard
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Build Docker
+        uses: docker/build-push-action@v6
+        with:
+          context: ./standard
+          tags: service:latest

README.md

@@ -1,18 +1,24 @@
 # py-manage
 
-A personal, opinionated guide to managing Python projects. A comprehensive writing and argumentation can be found
-in ["Python Project Management Primer"]().
+A personal, opinionated guide to managing Python projects. Literature that extends the argumentation and explains it on 
+a more detailed level:
+- ["Python Project Management Primer"](https://martynassubonis.substack.com/p/python-project-management-primer).
+- ["Optimizing Docker Images for Python Services and GPU-Accelerated Workbenches"]().
 
 ## Table of Contents
 
 1. [Tooling](#tooling)
-2. [Standard Project Structure](#standard-project-structure)
-3. [Mono-Repository Structure](#mono-repository-structure)
-4. [Workflows](#workflows)
+2. [Dockerfile Optimizations](#dockerfile-optimizations)
+3. [CUDA Development Workbenches](#cuda-development-workbenches)
+4. [Standard Project Structure](#standard-project-structure)
+5. [Mono-Repository Structure](#mono-repository-structure)
+6. [Workflows](#workflows)
     1. [Starting a New Project](#starting-a-new-project)
     2. [Installing an Existing Project](#installing-an-existing-project)
     3. [Developing Locally](#developing-locally)
-    4. [CI/CD](#cicd)
+    4. [Building Docker Images](#building-docker-images)
+    5. [Running Docker Containers Locally](#running-docker-containers-locally)
+    6. [CI/CD](#cicd)
 
 ## Tooling
 
@@ -22,6 +28,35 @@ in ["Python Project Management Primer"]().
 - Use [poetry](https://github.com/python-poetry/poetry) to manage `Python` dependencies and packaging.
 - [minor] use [ruff](https://github.com/astral-sh/ruff) as a linter/formatter.
 
+## Dockerfile Optimizations
+
+This guide recommends the following techniques:
+
+- [Multi-stage builds](https://docs.docker.com/build/guide/multi-stage/):
+-
+    - To parallelize builds to increase speed.
+-
+    - To separate build and runtime stages to reduce final image size.
+- [Effective cache utilization](https://docs.docker.com/build/cache/#optimizing-how-you-use-the-build-cache) to speed-up
+  build times by:
+-
+    - Positioning expensive layers early.
+-
+    - Placing frequently changing layers last.
+-
+    - Keeping layers small (including only necessary files and dependencies).
+-
+    - Minimizing layer count.
+
+Examples could be found in both, [standard](standard/Dockerfile) and [monorepo](monorepo/services/service_a/Dockerfile)
+structures.
+
+## CUDA Development Workbenches
+
+This guide also covers how to build comprehensive CUDA environment workbenches, that can reliably build deep learning
+frameworks, such as [PyTorch](https://github.com/pytorch/pytorch), from source. The same Dockerfile file can be found
+in [standard](standard/workbench/Dockerfile) and [monorepo](monorepo/workbench/Dockerfile) structures.
+
 ## Standard Project Structure
 
 For details on the standard project structure, refer to the [standard structure documentation](standard/README.md).
@@ -66,6 +101,26 @@ poetry run python -m unittest discover
 poetry run mypy .
 ```
 
+### Building Docker Images
+
+Follow these steps to build Docker images for services:
+
+```bash
+# Inside the service directory
+export IMAGE_TAG=python-service-x
+docker build -f Dockerfile -t $IMAGE_TAG .
+```
+
+### Running Docker Containers Locally
+
+Follow these steps to run Docker containers locally:
+
+```bash
+# Be sure to have built the Docker image before
+export IMAGE_TAG=python-service-x
+docker run -p 8080:8080 --name local $IMAGE_TAG
+```
+
 ### CI/CD
 
 For details on CI/CD workflows, refer to the respective documentation:

monorepo/README.md

@@ -12,6 +12,7 @@ monorepo/
 ├── poetry.toml
 ├── README.md
 ├── LICENSE
+├── workbench/  # Optional
 ├── packages/
 │   ├── package_a/
 │   │   ├── .python-version
@@ -22,9 +23,9 @@ monorepo/
 │   │   ├── README.md
 │   │   ├── LICENSE
 │   │   ├── package_a/
-|   │   │   ├── __init__.py
-|   │   │   ├── module_x.py
-|   │   │   └── ...
+│   │   │   ├── __init__.py
+│   │   │   ├── module_x.py
+│   │   │   └── ...
 │   │   └── tests/
 │   │       ├── __init__.py
 │   │       ├── test_module_x.py
@@ -38,9 +39,9 @@ monorepo/
 │       ├── README.md
 │       ├── LICENSE
 │       ├── package_b/
-|       │   ├── __init__.py
-|       │   ├── module_y.py
-|       │   └── ...
+│       │   ├── __init__.py
+│       │   ├── module_y.py
+│       │   └── ...
 │       └── tests/
 │           ├── __init__.py
 │           ├── test_module_y.py
@@ -112,3 +113,4 @@ monorepo/
 - [Linting and formatting workflow](../.github/workflows/monorepo_lint_format_check.yaml).
 - [Parallelized testing and static typing workflow](../.github/workflows/monorepo_tests_static_typing.yaml).
 - [Package building and publishing workflow](../.github/workflows/monorepo_build_dry_publish.yaml).
+- [Docker images building workflow](../.github/workflows/monorepo_build_docker.yaml).

monorepo/services/service_a/Dockerfile

@@ -0,0 +1,25 @@
+FROM python:3.12.4-slim AS builder
+
+RUN pip install --upgrade pip==24.1.1 && \
+    pip install poetry==1.8.3
+
+WORKDIR /app
+
+COPY pyproject.toml poetry.toml poetry.lock ./
+
+RUN poetry install --only main
+
+FROM python:3.12.4-slim AS runtime
+
+WORKDIR /app
+
+ENV PATH="/app/.venv/bin:$PATH"
+
+COPY src src
+COPY main.py .
+
+EXPOSE 8080
+
+COPY --from=builder /app/.venv .venv
+
+CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8080"]

monorepo/services/service_b/Dockerfile

@@ -0,0 +1,25 @@
+FROM python:3.12.4-slim AS builder
+
+RUN pip install --upgrade pip==24.1.1 && \
+    pip install poetry==1.8.3
+
+WORKDIR /app
+
+COPY pyproject.toml poetry.toml poetry.lock ./
+
+RUN poetry install --only main
+
+FROM python:3.12.4-slim AS runtime
+
+WORKDIR /app
+
+ENV PATH="/app/.venv/bin:$PATH"
+
+COPY src src
+COPY main.py .
+
+EXPOSE 8080
+
+COPY --from=builder /app/.venv .venv
+
+CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8080"]

monorepo/workbench/Dockerfile

@@ -0,0 +1,79 @@
+FROM alpine/git AS pytorch-source
+
+# Carefully select commit hash, as they are not guaranteed to uniformly build successfully.
+
+RUN git clone https://github.com/pytorch/pytorch.git pytorch && \
+    cd pytorch && \
+    git checkout 7919f0b && \
+    git submodule update --init --recursive && \
+    cd ..
+
+FROM nvidia/cuda:12.5.1-cudnn-devel-ubuntu22.04
+
+ENV DEBIAN_FRONTEND=noninteractive
+
+# For this Dockerefile we will use pyenv. Installing Python via pyenv creates an isolated environment, separate from
+# Ubuntu's system Python. This approach prevents accidental modifications to system-critical components that rely on
+# the default Python, ensuring overall system stability and avoiding potential breakage of Ubuntu's built-in tools and
+# packages. It also provides us with flexibility to choose any desired Python version.
+
+# Install essential build packages for Python (built by pyenv) and PyTorch, as they are built from source.
+# Additional utility packages (curl, git, wget) are included for common development tasks.
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends \
+        build-essential \
+        ca-certificates \
+        cmake \
+        curl \
+        git \
+        libbz2-dev \
+        libffi-dev \
+        liblzma-dev \
+        libncurses5-dev \
+        libncursesw5-dev \
+        libreadline-dev \
+        libsqlite3-dev \
+        libssl-dev \
+        libxml2-dev \
+        libxmlsec1-dev \
+        llvm \
+        ninja-build \
+        openssh-client \
+        tk-dev \
+        wget \
+        xz-utils \
+        zlib1g-dev && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*
+
+# pyenv configuration
+ENV PYENV_GIT_TAG="v2.4.7"
+ENV PYENV_ROOT=/root/.pyenv
+ENV PATH="$PYENV_ROOT/shims:$PYENV_ROOT/bin:$PATH"
+
+ENV PYTHON_VERSION=3.12.3
+
+RUN curl https://pyenv.run | bash && pyenv install $PYTHON_VERSION
+
+WORKDIR /app
+
+RUN pyenv local $PYTHON_VERSION
+
+ENV PIP_INSTALL_VERSION=24.1.2
+ENV POETRY_VERSION=1.8.3
+
+RUN pip install --upgrade pip==$PIP_INSTALL_VERSION && \
+    pip install --no-cache-dir poetry==$POETRY_VERSION && \
+    pip cache purge
+
+# Set up environment for PyTorch build
+ENV USE_CUDA=1
+ENV USE_CUDNN=1
+ENV CMAKE_PREFIX_PATH="$(dirname $(which python))/../"
+
+COPY --from=pytorch-source git/pytorch pytorch
+COPY pyproject.toml poetry.toml poetry.lock ./
+RUN poetry install --no-root && poetry add -e -vvv ./pytorch && rm -rf pytorch
+
+EXPOSE 8888
+CMD ["poetry", "run", "jupyter", "lab", "--ip=0.0.0.0", "--port=8888", "--no-browser", "--allow-root"]