From 19f374ca6bc4fc24078bab664cb947e541a9a48c Mon Sep 17 00:00:00 2001 From: Viktor Bozhinov Date: Fri, 20 May 2022 10:51:49 +0000 Subject: [PATCH 01/34] Set python image #354 --- Dockerfile | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 Dockerfile diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..61717b5d --- /dev/null +++ b/Dockerfile @@ -0,0 +1,3 @@ +# Dockerfile to build and serve datagateway-api + +FROM python:3.6-slim-bullseye From 70a9e6d8355c3b501ce075ea36d436fe9cca5fb4 Mon Sep 17 00:00:00 2001 From: Viktor Bozhinov Date: Fri, 20 May 2022 10:52:25 +0000 Subject: [PATCH 02/34] Upgrade pip #354 --- Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Dockerfile b/Dockerfile index 61717b5d..876038ec 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,3 +1,5 @@ # Dockerfile to build and serve datagateway-api FROM python:3.6-slim-bullseye + +RUN python -m pip install --upgrade pip From 046a22b48bad21bf614e488fd3877e8fa24d458f Mon Sep 17 00:00:00 2001 From: Viktor Bozhinov Date: Fri, 20 May 2022 10:52:50 +0000 Subject: [PATCH 03/34] Install Poetry #354 --- Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 876038ec..22a27aac 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,4 +2,5 @@ FROM python:3.6-slim-bullseye -RUN python -m pip install --upgrade pip +RUN python -m pip install --upgrade pip \ + && pip install poetry From 634d8b12cad2cc315f6a453a063898cbc018bb1c Mon Sep 17 00:00:00 2001 From: Viktor Bozhinov Date: Fri, 20 May 2022 10:53:46 +0000 Subject: [PATCH 04/34] Install a different version of setuptools #354 --- Dockerfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 22a27aac..f0e31271 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,4 +3,6 @@ FROM python:3.6-slim-bullseye RUN python -m pip install --upgrade pip \ - && pip install poetry + && pip install poetry \ + && poetry run pip uninstall -y setuptools \ + && poetry run pip install 'setuptools<58.0.0' From 86a7dce60258cb5f877fbe716d42d144cd36d9d9 Mon Sep 17 00:00:00 2001 From: Viktor Bozhinov Date: Fri, 20 May 2022 10:54:21 +0000 Subject: [PATCH 05/34] Install Gunicorn #354 --- Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index f0e31271..f22f68b4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,4 +5,5 @@ FROM python:3.6-slim-bullseye RUN python -m pip install --upgrade pip \ && pip install poetry \ && poetry run pip uninstall -y setuptools \ - && poetry run pip install 'setuptools<58.0.0' + && poetry run pip install 'setuptools<58.0.0' \ + && poetry run pip install 'gunicorn==20.1.0' From e10a88c1352fcf7d25db0295af53f2f9b131846a Mon Sep 17 00:00:00 2001 From: Viktor Bozhinov Date: Fri, 20 May 2022 10:54:51 +0000 Subject: [PATCH 06/34] Install the app dependencies #354 --- Dockerfile | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index f22f68b4..7b73e61c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,8 +2,13 @@ FROM python:3.6-slim-bullseye +WORKDIR /datagateway-api + +COPY poetry.lock pyproject.toml ./ + RUN python -m pip install --upgrade pip \ && pip install poetry \ && poetry run pip uninstall -y setuptools \ && poetry run pip install 'setuptools<58.0.0' \ - && poetry run pip install 'gunicorn==20.1.0' + && poetry run pip install 'gunicorn==20.1.0' \ + && poetry install --no-dev From 525e037675d7a8dc5f97993a25781eefc63cb606 Mon Sep 17 00:00:00 2001 From: Viktor Bozhinov Date: Fri, 20 May 2022 10:55:18 +0000 Subject: [PATCH 07/34] Serve the app on a Gunicorn server #354 --- Dockerfile | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Dockerfile b/Dockerfile index 7b73e61c..ea9a821d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,3 +12,8 @@ RUN python -m pip install --upgrade pip \ && poetry run pip install 'setuptools<58.0.0' \ && poetry run pip install 'gunicorn==20.1.0' \ && poetry install --no-dev + +COPY datagateway_api ./datagateway_api + +# Serve the application using gunicorn - production ready WSGI server +ENTRYPOINT ["poetry", "run", "gunicorn", "-c", "gunicorn.conf.py", "datagateway_api.wsgi:application"] From 9d911773b3eb42154b941342c74eeb12a1cb2c8f Mon Sep 17 00:00:00 2001 From: Viktor Bozhinov Date: Mon, 23 May 2022 12:32:30 +0000 Subject: [PATCH 08/34] Pin Poetry version #354 --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index ea9a821d..422b59ce 100644 --- a/Dockerfile +++ b/Dockerfile @@ -7,7 +7,7 @@ WORKDIR /datagateway-api COPY poetry.lock pyproject.toml ./ RUN python -m pip install --upgrade pip \ - && pip install poetry \ + && pip install 'poetry==1.1.13' \ && poetry run pip uninstall -y setuptools \ && poetry run pip install 'setuptools<58.0.0' \ && poetry run pip install 'gunicorn==20.1.0' \ From 3274d20ee8d1dc4095162035af24271e9ca9f753 Mon Sep 17 00:00:00 2001 From: Viktor Bozhinov Date: Mon, 23 May 2022 13:46:55 +0000 Subject: [PATCH 09/34] Define Actions job for building and pushing Docker image to Harbor #355 --- .github/workflows/ci-build.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci-build.yml b/.github/workflows/ci-build.yml index b3518471..29965e2c 100644 --- a/.github/workflows/ci-build.yml +++ b/.github/workflows/ci-build.yml @@ -4,7 +4,7 @@ on: pull_request: push: branches: - - main + - k8s-deployment jobs: tests: runs-on: ubuntu-20.04 @@ -329,3 +329,10 @@ jobs: # different to SciGateway preprod - name: Diff SQL dumps run: diff -s ~/generator_script_dump_main.sql ~/generator_script_dump_1.sql + + docker: + name: Docker + runs-on: ubuntu-20.04 + steps: + - name: Check out repo + uses: actions/checkout@v2 From bce3b3dd942397305701a8d7e856eb820a917cb9 Mon Sep 17 00:00:00 2001 From: Viktor Bozhinov Date: Mon, 23 May 2022 13:47:59 +0000 Subject: [PATCH 10/34] Configure job to only run after other jobs succeed #355 --- .github/workflows/ci-build.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci-build.yml b/.github/workflows/ci-build.yml index 29965e2c..7dc118fa 100644 --- a/.github/workflows/ci-build.yml +++ b/.github/workflows/ci-build.yml @@ -331,6 +331,7 @@ jobs: run: diff -s ~/generator_script_dump_main.sql ~/generator_script_dump_1.sql docker: + needs: [tests, linting, formatting, safety, generator-script-testing] name: Docker runs-on: ubuntu-20.04 steps: From 49c57b70a031d7423d23af0344976bbb55e480c7 Mon Sep 17 00:00:00 2001 From: Viktor Bozhinov Date: Mon, 23 May 2022 13:49:17 +0000 Subject: [PATCH 11/34] Configure job to login to Harbor #355 --- .github/workflows/ci-build.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/ci-build.yml b/.github/workflows/ci-build.yml index 7dc118fa..669fc9d0 100644 --- a/.github/workflows/ci-build.yml +++ b/.github/workflows/ci-build.yml @@ -337,3 +337,10 @@ jobs: steps: - name: Check out repo uses: actions/checkout@v2 + + - name: Login to Harbor + uses: docker/login-action@v1 + with: + registry: harbor.stfc.ac.uk/datagateway + username: ${{ secrets.HARBOR_USERNAME }} + password: ${{ secrets.HARBOR_TOKEN }} From 14feac18ae803aee04a0d81f6a2cccbc5ff8355a Mon Sep 17 00:00:00 2001 From: Viktor Bozhinov Date: Mon, 23 May 2022 13:50:33 +0000 Subject: [PATCH 12/34] Configure job to extract Docker metadata #355 --- .github/workflows/ci-build.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/ci-build.yml b/.github/workflows/ci-build.yml index 669fc9d0..570f73db 100644 --- a/.github/workflows/ci-build.yml +++ b/.github/workflows/ci-build.yml @@ -344,3 +344,9 @@ jobs: registry: harbor.stfc.ac.uk/datagateway username: ${{ secrets.HARBOR_USERNAME }} password: ${{ secrets.HARBOR_TOKEN }} + + - name: Extract metadata (tags, labels) for Docker + id: meta + uses: docker/metadata-action@v3 + with: + images: harbor.stfc.ac.uk/datagateway/datagateway-api From d7965416e201556e5bb41bc9ed5227b89e5c9042 Mon Sep 17 00:00:00 2001 From: Viktor Bozhinov Date: Mon, 23 May 2022 13:52:04 +0000 Subject: [PATCH 13/34] Configure job to build image #355 --- .github/workflows/ci-build.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/ci-build.yml b/.github/workflows/ci-build.yml index 570f73db..3631fc78 100644 --- a/.github/workflows/ci-build.yml +++ b/.github/workflows/ci-build.yml @@ -350,3 +350,11 @@ jobs: uses: docker/metadata-action@v3 with: images: harbor.stfc.ac.uk/datagateway/datagateway-api + + - name: Build Docker image + uses: docker/build-push-action@v2 + with: + context: . + push: false + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} From b51d647e96915306e3fcc86debbb9bccddae96f6 Mon Sep 17 00:00:00 2001 From: Viktor Bozhinov Date: Mon, 23 May 2022 13:53:23 +0000 Subject: [PATCH 14/34] Configure job to push image to Harbor on pushes to k8s-deployment branch #355 --- .github/workflows/ci-build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-build.yml b/.github/workflows/ci-build.yml index 3631fc78..36b018a7 100644 --- a/.github/workflows/ci-build.yml +++ b/.github/workflows/ci-build.yml @@ -351,10 +351,10 @@ jobs: with: images: harbor.stfc.ac.uk/datagateway/datagateway-api - - name: Build Docker image + - name: ${{ github.event_name == 'push' && github.ref == 'refs/heads/k8s-deployment' && 'Build and push Docker image to Harbor' || 'Build Docker image' }} uses: docker/build-push-action@v2 with: context: . - push: false + push: ${{ github.event_name == 'push' && github.ref == 'refs/heads/k8s-deployment' }} tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} From 92241ac4f92300d2ee9986b6162b060da6b46906 Mon Sep 17 00:00:00 2001 From: Viktor Bozhinov Date: Mon, 23 May 2022 13:54:43 +0000 Subject: [PATCH 15/34] Add job documentation and TODOs #355 --- .github/workflows/ci-build.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/ci-build.yml b/.github/workflows/ci-build.yml index 36b018a7..50d02083 100644 --- a/.github/workflows/ci-build.yml +++ b/.github/workflows/ci-build.yml @@ -331,6 +331,9 @@ jobs: run: diff -s ~/generator_script_dump_main.sql ~/generator_script_dump_1.sql docker: + # This job triggers only if all the other jobs succeed and does different things depending on the context. + # The job builds the Docker image in all cases and also pushes the image to Harbor only if something is + # pushed to the k8s-deployment branch. needs: [tests, linting, formatting, safety, generator-script-testing] name: Docker runs-on: ubuntu-20.04 @@ -351,10 +354,12 @@ jobs: with: images: harbor.stfc.ac.uk/datagateway/datagateway-api + # TODO - Change github.ref to refs/heads/main before merging k8s-deployment into main - name: ${{ github.event_name == 'push' && github.ref == 'refs/heads/k8s-deployment' && 'Build and push Docker image to Harbor' || 'Build Docker image' }} uses: docker/build-push-action@v2 with: context: . + # TODO - Change github.ref to refs/heads/main before merging k8s-deployment into main push: ${{ github.event_name == 'push' && github.ref == 'refs/heads/k8s-deployment' }} tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} From 17b64fd4bf5a85edef3fd353ed52e6e92420064d Mon Sep 17 00:00:00 2001 From: Viktor Bozhinov Date: Mon, 23 May 2022 14:30:33 +0000 Subject: [PATCH 16/34] Configure dependabot to maintain GH Actions dependencies #355 --- .github/dependabot.yml | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 .github/dependabot.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 00000000..438cb6fa --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,7 @@ +version: 2 +updates: + - package-ecosystem: "github-actions" + directory: "/" + schedule: + # Check for updates to GitHub Actions every weekday + interval: "daily" From 02c9432c628f9a61ef289df63fc55033e7e996cf Mon Sep 17 00:00:00 2001 From: Viktor Bozhinov Date: Mon, 30 May 2022 10:41:49 +0000 Subject: [PATCH 17/34] Add TODO for branch name of push events #355 --- .github/workflows/ci-build.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci-build.yml b/.github/workflows/ci-build.yml index 50d02083..bb2626b8 100644 --- a/.github/workflows/ci-build.yml +++ b/.github/workflows/ci-build.yml @@ -3,6 +3,7 @@ on: workflow_dispatch: pull_request: push: + # TODO - Change this to main before merging k8s-deployment into main branches: - k8s-deployment jobs: From 92cffd9f72983512261191ae7d02a78ad3e729bf Mon Sep 17 00:00:00 2001 From: Alan Kyffin Date: Mon, 20 Feb 2023 11:25:39 +0000 Subject: [PATCH 18/34] Update Dockerfile and add entrypoint script --- Dockerfile | 47 ++++++++++++++++++++++++++++--------- docker/docker-entrypoint.sh | 9 +++++++ 2 files changed, 45 insertions(+), 11 deletions(-) create mode 100755 docker/docker-entrypoint.sh diff --git a/Dockerfile b/Dockerfile index 422b59ce..b0e3438a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,19 +1,44 @@ # Dockerfile to build and serve datagateway-api -FROM python:3.6-slim-bullseye +# Build stage +FROM python:3-alpine as builder -WORKDIR /datagateway-api +WORKDIR /datagateway-api-build -COPY poetry.lock pyproject.toml ./ +COPY . . -RUN python -m pip install --upgrade pip \ - && pip install 'poetry==1.1.13' \ - && poetry run pip uninstall -y setuptools \ - && poetry run pip install 'setuptools<58.0.0' \ - && poetry run pip install 'gunicorn==20.1.0' \ - && poetry install --no-dev +RUN python3 -m pip install poetry \ + && poetry build -COPY datagateway_api ./datagateway_api + +# Install & run stage +FROM python:3-alpine + +WORKDIR /datagateway-api-run + +COPY --from=builder /datagateway-api-build/dist/datagateway_api-*.whl . + +# Workaround for https://github.com/icatproject/python-icat/issues/99 +RUN python3 -m pip install 'setuptools<58.0.0' + +RUN python3 -m pip install datagateway_api-*.whl gunicorn \ + && DATAGATEWAY_API_LOCATION="$(python3 -m pip show datagateway_api | awk '/^Location:/ { print $2 }')" \ + && cp "$DATAGATEWAY_API_LOCATION/datagateway_api/search_api_mapping.json.example" "$DATAGATEWAY_API_LOCATION/datagateway_api/search_api_mapping.json" \ + && cp "$DATAGATEWAY_API_LOCATION/datagateway_api/config.yaml.example" "/datagateway-api-run/config.yaml" \ + && ln -s "/datagateway-api-run/config.yaml" "$DATAGATEWAY_API_LOCATION/datagateway_api/config.yaml" \ + && addgroup -S datagateway-api \ + && adduser -S -D -G datagateway-api -H -h /datagateway-api-run datagateway-api \ + && chown datagateway-api:datagateway-api /datagateway-api-run /datagateway-api-run/config.yaml + +USER datagateway-api + +ENV ICAT_URL="http://localhost" +ENV ICAT_CHECK_CERT="true" +ENV LOG_LOCATION="/dev/stdout" + +COPY docker/docker-entrypoint.sh . +ENTRYPOINT ["./docker-entrypoint.sh"] # Serve the application using gunicorn - production ready WSGI server -ENTRYPOINT ["poetry", "run", "gunicorn", "-c", "gunicorn.conf.py", "datagateway_api.wsgi:application"] +CMD ["gunicorn", "-b", "0.0.0.0:8000", "datagateway_api.wsgi"] +EXPOSE 8000 diff --git a/docker/docker-entrypoint.sh b/docker/docker-entrypoint.sh new file mode 100755 index 00000000..eb4205e2 --- /dev/null +++ b/docker/docker-entrypoint.sh @@ -0,0 +1,9 @@ +#!/bin/sh + +# Set values in config.yaml from environment variables +sed -i "s|icat_url: \".*\"|icat_url: \"$ICAT_URL\"|" config.yaml +sed -i "s|icat_check_cert: \".*\"|icat_check_cert: \"$ICAT_CHECK_CERT\"|" config.yaml +sed -i "s|log_location: \".*\"|log_location: \"$LOG_LOCATION\"|" config.yaml + +# Run the CMD instruction +exec "$@" From dfb54bac1808117c93dd7bcab4b3c4f8a0919786 Mon Sep 17 00:00:00 2001 From: Alan Kyffin Date: Tue, 21 Feb 2023 11:56:07 +0000 Subject: [PATCH 19/34] Use specific python and alpine versions in the base image --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index b0e3438a..7c12c142 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ # Dockerfile to build and serve datagateway-api # Build stage -FROM python:3-alpine as builder +FROM python:3.11-alpine3.17 as builder WORKDIR /datagateway-api-build @@ -12,7 +12,7 @@ RUN python3 -m pip install poetry \ # Install & run stage -FROM python:3-alpine +FROM python:3.11-alpine3.17 WORKDIR /datagateway-api-run From f03d85365907c48aa46d807747c52b32addb085d Mon Sep 17 00:00:00 2001 From: Alan Kyffin Date: Tue, 21 Feb 2023 13:03:41 +0000 Subject: [PATCH 20/34] Pin python package versions in Dockerfile --- Dockerfile | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index 7c12c142..58a5dfad 100644 --- a/Dockerfile +++ b/Dockerfile @@ -7,7 +7,7 @@ WORKDIR /datagateway-api-build COPY . . -RUN python3 -m pip install poetry \ +RUN python3 -m pip install 'poetry~=1.3.2' \ && poetry build @@ -18,10 +18,11 @@ WORKDIR /datagateway-api-run COPY --from=builder /datagateway-api-build/dist/datagateway_api-*.whl . +RUN python3 -m pip install \ + 'gunicorn~=20.1.0' \ # Workaround for https://github.com/icatproject/python-icat/issues/99 -RUN python3 -m pip install 'setuptools<58.0.0' - -RUN python3 -m pip install datagateway_api-*.whl gunicorn \ + 'setuptools<58.0.0' \ + datagateway_api-*.whl \ && DATAGATEWAY_API_LOCATION="$(python3 -m pip show datagateway_api | awk '/^Location:/ { print $2 }')" \ && cp "$DATAGATEWAY_API_LOCATION/datagateway_api/search_api_mapping.json.example" "$DATAGATEWAY_API_LOCATION/datagateway_api/search_api_mapping.json" \ && cp "$DATAGATEWAY_API_LOCATION/datagateway_api/config.yaml.example" "/datagateway-api-run/config.yaml" \ From 14073b37a46c695325305ccf697a3533d1d18613 Mon Sep 17 00:00:00 2001 From: Alan Kyffin Date: Tue, 21 Feb 2023 13:14:00 +0000 Subject: [PATCH 21/34] Use a cache mount to speed up pip and poetry --- Dockerfile | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 58a5dfad..54783988 100644 --- a/Dockerfile +++ b/Dockerfile @@ -7,7 +7,8 @@ WORKDIR /datagateway-api-build COPY . . -RUN python3 -m pip install 'poetry~=1.3.2' \ +RUN --mount=type=cache,target=/root/.cache \ + python3 -m pip install 'poetry~=1.3.2' \ && poetry build @@ -18,7 +19,8 @@ WORKDIR /datagateway-api-run COPY --from=builder /datagateway-api-build/dist/datagateway_api-*.whl . -RUN python3 -m pip install \ +RUN --mount=type=cache,target=/root/.cache \ + python3 -m pip install \ 'gunicorn~=20.1.0' \ # Workaround for https://github.com/icatproject/python-icat/issues/99 'setuptools<58.0.0' \ From f701c9e29337a45b8eb9602b5166e2f09949b63b Mon Sep 17 00:00:00 2001 From: Alan Kyffin Date: Tue, 21 Feb 2023 13:27:45 +0000 Subject: [PATCH 22/34] Comment the RUN step --- Dockerfile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Dockerfile b/Dockerfile index 54783988..f2952463 100644 --- a/Dockerfile +++ b/Dockerfile @@ -26,9 +26,12 @@ RUN --mount=type=cache,target=/root/.cache \ 'setuptools<58.0.0' \ datagateway_api-*.whl \ && DATAGATEWAY_API_LOCATION="$(python3 -m pip show datagateway_api | awk '/^Location:/ { print $2 }')" \ +# Create search_api_mapping.json from its .example file && cp "$DATAGATEWAY_API_LOCATION/datagateway_api/search_api_mapping.json.example" "$DATAGATEWAY_API_LOCATION/datagateway_api/search_api_mapping.json" \ +# Create config.yaml from its .example file. It will need to be editted by the entrypoint script so create it in our non-root user's home directory and create a symlink && cp "$DATAGATEWAY_API_LOCATION/datagateway_api/config.yaml.example" "/datagateway-api-run/config.yaml" \ && ln -s "/datagateway-api-run/config.yaml" "$DATAGATEWAY_API_LOCATION/datagateway_api/config.yaml" \ +# Create a non-root user to run as && addgroup -S datagateway-api \ && adduser -S -D -G datagateway-api -H -h /datagateway-api-run datagateway-api \ && chown datagateway-api:datagateway-api /datagateway-api-run /datagateway-api-run/config.yaml From 6ad492c1253b2e3c260271b17e150b92b90ad422 Mon Sep 17 00:00:00 2001 From: Alan Kyffin Date: Tue, 21 Feb 2023 13:33:51 +0000 Subject: [PATCH 23/34] Move things out of the datagateway-api-run directory --- Dockerfile | 10 +++++----- docker/docker-entrypoint.sh | 6 +++--- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/Dockerfile b/Dockerfile index f2952463..43258617 100644 --- a/Dockerfile +++ b/Dockerfile @@ -17,14 +17,14 @@ FROM python:3.11-alpine3.17 WORKDIR /datagateway-api-run -COPY --from=builder /datagateway-api-build/dist/datagateway_api-*.whl . +COPY --from=builder /datagateway-api-build/dist/datagateway_api-*.whl /tmp RUN --mount=type=cache,target=/root/.cache \ python3 -m pip install \ 'gunicorn~=20.1.0' \ # Workaround for https://github.com/icatproject/python-icat/issues/99 'setuptools<58.0.0' \ - datagateway_api-*.whl \ + /tmp/datagateway_api-*.whl \ && DATAGATEWAY_API_LOCATION="$(python3 -m pip show datagateway_api | awk '/^Location:/ { print $2 }')" \ # Create search_api_mapping.json from its .example file && cp "$DATAGATEWAY_API_LOCATION/datagateway_api/search_api_mapping.json.example" "$DATAGATEWAY_API_LOCATION/datagateway_api/search_api_mapping.json" \ @@ -34,7 +34,7 @@ RUN --mount=type=cache,target=/root/.cache \ # Create a non-root user to run as && addgroup -S datagateway-api \ && adduser -S -D -G datagateway-api -H -h /datagateway-api-run datagateway-api \ - && chown datagateway-api:datagateway-api /datagateway-api-run /datagateway-api-run/config.yaml + && chown -R datagateway-api:datagateway-api /datagateway-api-run USER datagateway-api @@ -42,8 +42,8 @@ ENV ICAT_URL="http://localhost" ENV ICAT_CHECK_CERT="true" ENV LOG_LOCATION="/dev/stdout" -COPY docker/docker-entrypoint.sh . -ENTRYPOINT ["./docker-entrypoint.sh"] +COPY docker/docker-entrypoint.sh /usr/local/bin/ +ENTRYPOINT ["docker-entrypoint.sh"] # Serve the application using gunicorn - production ready WSGI server CMD ["gunicorn", "-b", "0.0.0.0:8000", "datagateway_api.wsgi"] diff --git a/docker/docker-entrypoint.sh b/docker/docker-entrypoint.sh index eb4205e2..365bc4cb 100755 --- a/docker/docker-entrypoint.sh +++ b/docker/docker-entrypoint.sh @@ -1,9 +1,9 @@ #!/bin/sh # Set values in config.yaml from environment variables -sed -i "s|icat_url: \".*\"|icat_url: \"$ICAT_URL\"|" config.yaml -sed -i "s|icat_check_cert: \".*\"|icat_check_cert: \"$ICAT_CHECK_CERT\"|" config.yaml -sed -i "s|log_location: \".*\"|log_location: \"$LOG_LOCATION\"|" config.yaml +sed -i "s|icat_url: \".*\"|icat_url: \"$ICAT_URL\"|" /datagateway-api-run/config.yaml +sed -i "s|icat_check_cert: \".*\"|icat_check_cert: \"$ICAT_CHECK_CERT\"|" /datagateway-api-run/config.yaml +sed -i "s|log_location: \".*\"|log_location: \"$LOG_LOCATION\"|" /datagateway-api-run/config.yaml # Run the CMD instruction exec "$@" From e7630dc43f798d3620baa8c2035f80f4cb4b26c3 Mon Sep 17 00:00:00 2001 From: Alan Kyffin Date: Tue, 21 Feb 2023 14:29:19 +0000 Subject: [PATCH 24/34] Remove workaround that is no longer needed --- Dockerfile | 2 -- 1 file changed, 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 43258617..e48da77f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -22,8 +22,6 @@ COPY --from=builder /datagateway-api-build/dist/datagateway_api-*.whl /tmp RUN --mount=type=cache,target=/root/.cache \ python3 -m pip install \ 'gunicorn~=20.1.0' \ -# Workaround for https://github.com/icatproject/python-icat/issues/99 - 'setuptools<58.0.0' \ /tmp/datagateway_api-*.whl \ && DATAGATEWAY_API_LOCATION="$(python3 -m pip show datagateway_api | awk '/^Location:/ { print $2 }')" \ # Create search_api_mapping.json from its .example file From 0e948f7c2291efc937c117c49f4e7cc6649b9d2a Mon Sep 17 00:00:00 2001 From: Alan Kyffin Date: Wed, 22 Feb 2023 15:35:56 +0000 Subject: [PATCH 25/34] log_location value should not be quoted --- docker/docker-entrypoint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/docker-entrypoint.sh b/docker/docker-entrypoint.sh index 365bc4cb..cfd89161 100755 --- a/docker/docker-entrypoint.sh +++ b/docker/docker-entrypoint.sh @@ -3,7 +3,7 @@ # Set values in config.yaml from environment variables sed -i "s|icat_url: \".*\"|icat_url: \"$ICAT_URL\"|" /datagateway-api-run/config.yaml sed -i "s|icat_check_cert: \".*\"|icat_check_cert: \"$ICAT_CHECK_CERT\"|" /datagateway-api-run/config.yaml -sed -i "s|log_location: \".*\"|log_location: \"$LOG_LOCATION\"|" /datagateway-api-run/config.yaml +sed -i "s|log_location: .*|log_location: $LOG_LOCATION|" /datagateway-api-run/config.yaml # Run the CMD instruction exec "$@" From 7eb6ae4103b8033afb060b33d90cf528e9c098e0 Mon Sep 17 00:00:00 2001 From: Alan Kyffin Date: Wed, 22 Feb 2023 16:05:08 +0000 Subject: [PATCH 26/34] Only copy necessary files to build container --- Dockerfile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index e48da77f..cbcd3b0f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,7 +5,8 @@ FROM python:3.11-alpine3.17 as builder WORKDIR /datagateway-api-build -COPY . . +COPY README.md poetry.lock pyproject.toml ./ +COPY datagateway_api/ datagateway_api/ RUN --mount=type=cache,target=/root/.cache \ python3 -m pip install 'poetry~=1.3.2' \ @@ -17,7 +18,7 @@ FROM python:3.11-alpine3.17 WORKDIR /datagateway-api-run -COPY --from=builder /datagateway-api-build/dist/datagateway_api-*.whl /tmp +COPY --from=builder /datagateway-api-build/dist/datagateway_api-*.whl /tmp/ RUN --mount=type=cache,target=/root/.cache \ python3 -m pip install \ From 530630725c7f94bdfee3ae5b8e1e1cafa4b7b817 Mon Sep 17 00:00:00 2001 From: Alan Kyffin Date: Wed, 22 Feb 2023 16:05:51 +0000 Subject: [PATCH 27/34] Improve readability of RUN instructions --- Dockerfile | 34 +++++++++++++++++++++------------- 1 file changed, 21 insertions(+), 13 deletions(-) diff --git a/Dockerfile b/Dockerfile index cbcd3b0f..22579a7e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,8 +9,10 @@ COPY README.md poetry.lock pyproject.toml ./ COPY datagateway_api/ datagateway_api/ RUN --mount=type=cache,target=/root/.cache \ - python3 -m pip install 'poetry~=1.3.2' \ - && poetry build + set -eux; \ + \ + python3 -m pip install 'poetry~=1.3.2'; \ + poetry build; # Install & run stage @@ -21,19 +23,25 @@ WORKDIR /datagateway-api-run COPY --from=builder /datagateway-api-build/dist/datagateway_api-*.whl /tmp/ RUN --mount=type=cache,target=/root/.cache \ + set -eux; \ + \ python3 -m pip install \ 'gunicorn~=20.1.0' \ - /tmp/datagateway_api-*.whl \ - && DATAGATEWAY_API_LOCATION="$(python3 -m pip show datagateway_api | awk '/^Location:/ { print $2 }')" \ -# Create search_api_mapping.json from its .example file - && cp "$DATAGATEWAY_API_LOCATION/datagateway_api/search_api_mapping.json.example" "$DATAGATEWAY_API_LOCATION/datagateway_api/search_api_mapping.json" \ -# Create config.yaml from its .example file. It will need to be editted by the entrypoint script so create it in our non-root user's home directory and create a symlink - && cp "$DATAGATEWAY_API_LOCATION/datagateway_api/config.yaml.example" "/datagateway-api-run/config.yaml" \ - && ln -s "/datagateway-api-run/config.yaml" "$DATAGATEWAY_API_LOCATION/datagateway_api/config.yaml" \ -# Create a non-root user to run as - && addgroup -S datagateway-api \ - && adduser -S -D -G datagateway-api -H -h /datagateway-api-run datagateway-api \ - && chown -R datagateway-api:datagateway-api /datagateway-api-run + /tmp/datagateway_api-*.whl; \ + \ + DATAGATEWAY_API_LOCATION="$(python3 -m pip show datagateway_api | awk '/^Location:/ { print $2 }')"; \ + \ + # Create search_api_mapping.json from its .example file \ + cp "$DATAGATEWAY_API_LOCATION/datagateway_api/search_api_mapping.json.example" "$DATAGATEWAY_API_LOCATION/datagateway_api/search_api_mapping.json"; \ + \ + # Create config.yaml from its .example file. It will need to be editted by the entrypoint script so create it in our non-root user's home directory and create a symlink \ + cp "$DATAGATEWAY_API_LOCATION/datagateway_api/config.yaml.example" /datagateway-api-run/config.yaml; \ + ln -s /datagateway-api-run/config.yaml "$DATAGATEWAY_API_LOCATION/datagateway_api/config.yaml"; \ + \ + # Create a non-root user to run as \ + addgroup -S datagateway-api; \ + adduser -S -D -G datagateway-api -H -h /datagateway-api-run datagateway-api; \ + chown -R datagateway-api:datagateway-api /datagateway-api-run; USER datagateway-api From 14434c9c0cd28c5c14864693a4ba7eaa7db78149 Mon Sep 17 00:00:00 2001 From: Alan Kyffin Date: Mon, 27 Feb 2023 10:04:32 +0000 Subject: [PATCH 28/34] Use a temp file instead of sed -i in entrypoint script --- docker/docker-entrypoint.sh | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/docker/docker-entrypoint.sh b/docker/docker-entrypoint.sh index cfd89161..19b9ec97 100755 --- a/docker/docker-entrypoint.sh +++ b/docker/docker-entrypoint.sh @@ -1,9 +1,17 @@ -#!/bin/sh +#!/bin/sh -eu + +# Use a tempfile instead of sed -i so that only the file, not the directory needs to be writable +TEMPFILE="$(mktemp)" # Set values in config.yaml from environment variables -sed -i "s|icat_url: \".*\"|icat_url: \"$ICAT_URL\"|" /datagateway-api-run/config.yaml -sed -i "s|icat_check_cert: \".*\"|icat_check_cert: \"$ICAT_CHECK_CERT\"|" /datagateway-api-run/config.yaml -sed -i "s|log_location: .*|log_location: $LOG_LOCATION|" /datagateway-api-run/config.yaml +# No quotes for icat_check_cert because it's boolean +sed -e "s|icat_url: \".*\"|icat_url: \"$ICAT_URL\"|" \ + -e "s|icat_check_cert: .*|icat_check_cert: $ICAT_CHECK_CERT|" \ + -e "s|log_location: \".*\"|log_location: \"$LOG_LOCATION\"|" \ + datagateway_api/config.yaml > "$TEMPFILE" + +cat "$TEMPFILE" > datagateway_api/config.yaml +rm "$TEMPFILE" # Run the CMD instruction exec "$@" From f3dfc20cf39ab848b690b0e33efaf16e7ce6ee3f Mon Sep 17 00:00:00 2001 From: Alan Kyffin Date: Mon, 27 Feb 2023 11:14:25 +0000 Subject: [PATCH 29/34] Create a symlink to the installed python module --- Dockerfile | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/Dockerfile b/Dockerfile index 22579a7e..707690b4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -29,19 +29,20 @@ RUN --mount=type=cache,target=/root/.cache \ 'gunicorn~=20.1.0' \ /tmp/datagateway_api-*.whl; \ \ + # Create a symlink to the installed python module \ DATAGATEWAY_API_LOCATION="$(python3 -m pip show datagateway_api | awk '/^Location:/ { print $2 }')"; \ + ln -s "$DATAGATEWAY_API_LOCATION/datagateway_api/" datagateway_api; \ \ - # Create search_api_mapping.json from its .example file \ - cp "$DATAGATEWAY_API_LOCATION/datagateway_api/search_api_mapping.json.example" "$DATAGATEWAY_API_LOCATION/datagateway_api/search_api_mapping.json"; \ - \ - # Create config.yaml from its .example file. It will need to be editted by the entrypoint script so create it in our non-root user's home directory and create a symlink \ - cp "$DATAGATEWAY_API_LOCATION/datagateway_api/config.yaml.example" /datagateway-api-run/config.yaml; \ - ln -s /datagateway-api-run/config.yaml "$DATAGATEWAY_API_LOCATION/datagateway_api/config.yaml"; \ + # Create config.yaml and search_api_mapping.json from their .example files \ + cp datagateway_api/config.yaml.example datagateway_api/config.yaml; \ + cp datagateway_api/search_api_mapping.json.example datagateway_api/search_api_mapping.json; \ \ # Create a non-root user to run as \ addgroup -S datagateway-api; \ adduser -S -D -G datagateway-api -H -h /datagateway-api-run datagateway-api; \ - chown -R datagateway-api:datagateway-api /datagateway-api-run; + \ + # Change ownership of config.yaml - the entrypoint script will need to edit it \ + chown datagateway-api:datagateway-api datagateway_api/config.yaml; USER datagateway-api From 47efb67bfffc006092ffac87afb13450583a8d0e Mon Sep 17 00:00:00 2001 From: Viktor Bozhinov <45173816+VKTB@users.noreply.github.com> Date: Wed, 31 May 2023 16:19:58 +0000 Subject: [PATCH 30/34] Address TODOs --- .github/workflows/ci-build.yml | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci-build.yml b/.github/workflows/ci-build.yml index 18022dd1..ea1db3d4 100644 --- a/.github/workflows/ci-build.yml +++ b/.github/workflows/ci-build.yml @@ -3,9 +3,8 @@ on: workflow_dispatch: pull_request: push: - # TODO - Change this to main before merging k8s-deployment into main branches: - - k8s-deployment + - main jobs: tests: runs-on: ubuntu-20.04 @@ -357,7 +356,7 @@ jobs: docker: # This job triggers only if all the other jobs succeed and does different things depending on the context. # The job builds the Docker image in all cases and also pushes the image to Harbor only if something is - # pushed to the k8s-deployment branch. + # pushed to the main branch. needs: [tests, linting, formatting, safety, generator-script-testing] name: Docker runs-on: ubuntu-20.04 @@ -378,12 +377,10 @@ jobs: with: images: harbor.stfc.ac.uk/datagateway/datagateway-api - # TODO - Change github.ref to refs/heads/main before merging k8s-deployment into main - - name: ${{ github.event_name == 'push' && github.ref == 'refs/heads/k8s-deployment' && 'Build and push Docker image to Harbor' || 'Build Docker image' }} + - name: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' && 'Build and push Docker image to Harbor' || 'Build Docker image' }} uses: docker/build-push-action@v2 with: context: . - # TODO - Change github.ref to refs/heads/main before merging k8s-deployment into main - push: ${{ github.event_name == 'push' && github.ref == 'refs/heads/k8s-deployment' }} + push: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }} tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} From 41fe074f718c6ae87816cd413569523c9fe097ae Mon Sep 17 00:00:00 2001 From: Viktor Bozhinov <45173816+VKTB@users.noreply.github.com> Date: Wed, 31 May 2023 16:43:37 +0000 Subject: [PATCH 31/34] Change default value of ICAT_CHECK_CERT ENV --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 707690b4..93bcc5e4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -47,7 +47,7 @@ RUN --mount=type=cache,target=/root/.cache \ USER datagateway-api ENV ICAT_URL="http://localhost" -ENV ICAT_CHECK_CERT="true" +ENV ICAT_CHECK_CERT="false" ENV LOG_LOCATION="/dev/stdout" COPY docker/docker-entrypoint.sh /usr/local/bin/ From c0019ae05697707abc1c8f16d63d444e7a395ff9 Mon Sep 17 00:00:00 2001 From: Viktor Bozhinov <45173816+VKTB@users.noreply.github.com> Date: Thu, 1 Jun 2023 15:55:42 +0000 Subject: [PATCH 32/34] Upgrade and pin actions to commit SHAs --- .github/workflows/ci-build.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci-build.yml b/.github/workflows/ci-build.yml index ea1db3d4..32e6c9ec 100644 --- a/.github/workflows/ci-build.yml +++ b/.github/workflows/ci-build.yml @@ -362,10 +362,10 @@ jobs: runs-on: ubuntu-20.04 steps: - name: Check out repo - uses: actions/checkout@v2 + uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab # v3.5.2 - name: Login to Harbor - uses: docker/login-action@v1 + uses: docker/login-action@f4ef78c080cd8ba55a85445d5b36e214a81df20a # v2.1.0 with: registry: harbor.stfc.ac.uk/datagateway username: ${{ secrets.HARBOR_USERNAME }} @@ -373,12 +373,12 @@ jobs: - name: Extract metadata (tags, labels) for Docker id: meta - uses: docker/metadata-action@v3 + uses: docker/metadata-action@c4ee3adeed93b1fa6a762f209fb01608c1a22f1e # v4.4.0 with: images: harbor.stfc.ac.uk/datagateway/datagateway-api - name: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' && 'Build and push Docker image to Harbor' || 'Build Docker image' }} - uses: docker/build-push-action@v2 + uses: docker/build-push-action@3b5e8027fcad23fda98b2e3ac259d8d67585f671 # v4.0.0 with: context: . push: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }} From 6fb038a3240ee25ef5d7610e8282c13c1fd79408 Mon Sep 17 00:00:00 2001 From: Viktor Bozhinov <45173816+VKTB@users.noreply.github.com> Date: Fri, 16 Jun 2023 10:59:59 +0100 Subject: [PATCH 33/34] ci(docker): bump actions/checkout to 3.5.3 in docker job --- .github/workflows/ci-build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-build.yml b/.github/workflows/ci-build.yml index 0635d13f..30d302d0 100644 --- a/.github/workflows/ci-build.yml +++ b/.github/workflows/ci-build.yml @@ -370,7 +370,7 @@ jobs: runs-on: ubuntu-20.04 steps: - name: Check out repo - uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab # v3.5.2 + uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3 - name: Login to Harbor uses: docker/login-action@f4ef78c080cd8ba55a85445d5b36e214a81df20a # v2.1.0 From 378ab9b9ecdbf4ffc1571ec9d44f0ad4b1485901 Mon Sep 17 00:00:00 2001 From: Viktor Bozhinov <45173816+VKTB@users.noreply.github.com> Date: Thu, 31 Aug 2023 08:42:10 +0000 Subject: [PATCH 34/34] Update README --- README.md | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/README.md b/README.md index 51b1f713..7e5d91a9 100644 --- a/README.md +++ b/README.md @@ -375,6 +375,32 @@ If using Python 3.10, please use Payara 5 on the ICAT stack which the API is bei pointed at. There is a known issue when making HTTPS connections to Payara (via Python ICAT). +It is also possible to run the API inside Docker. The `Dockerfile` can be used to build +a Docker image which in turn can be used to create a container. The `Dockerfile` is +configured to create a production image and runs a Gunicorn serve on port `8000` when a +container is started. Environment variables have also been defined in the `Dockerfile` +to allow for values to be passed at runtime to future running containers. These values +are used by the `docker/docker-entrypoint.sh` script to update the config values in the +`config.yaml` file. The environment varialbes are: +- `ICAT_URL` (Default value: `http://localhost`) +- `ICAT_CHECK_CERT` (Default value: `false`) +- `LOG_LOCATION` (Default value: `/dev/stdout`) + +To build an image, run: +```bash +docker build -t datagateway_api_image . +``` + +To start a container on port `8000` from the image that you just built, run: +```bash +docker run -p 8000:8000 --name datagateway_api_container datagateway_api_image +``` + +If you want to pass values for the environment variables then instead run: +```bash +docker run -p 8000:8000 --name datagateway_api_container --env ICAT_URL=https://127.0.0.1:8181 --env ICAT_CHECK_CERT=true --env LOG_LOCATION=/datagateway-api-run/logs.log datagateway_api_image +``` + ## DataGateway API Authentication Each request requires a valid session ID to be provided in the Authorization header.