Merge branch 'main' into remove-isis-endpoints-#432

ral-facilities · Sep 1, 2023 · e667c78 · e667c78
2 parents 12e0304 + 60c539c
commit e667c78
Show file tree

Hide file tree

Showing 8 changed files with 757 additions and 128 deletions.
diff --git a/.github/workflows/ci-build.yml b/.github/workflows/ci-build.yml
@@ -19,18 +19,18 @@ jobs:
 
       # Setup Java & Python
       - name: Setup Java
-        uses: actions/setup-java@5ffc13f4174014e2d4d4572b3d74c3fa61aeb2c2 # v3.11.0
+        uses: actions/setup-java@cd89f46ac9d01407894225f350157564c9c7cee2 # v3.12.0
         with:
           distribution: 'temurin'
           java-version: 8
       - name: Setup Python
-        uses: actions/setup-python@57ded4d7d5e986d7296eab16560982c6dd7c923b # v4.6.0
+        uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4.7.0
         with:
           python-version: ${{ matrix.python-version }}
           architecture: x64
 
       - name: Checkout DataGateway API
-        uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab # v3.5.2
+        uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0
 
       # Install Nox, Poetry and API's dependencies
       - name: Install Nox
@@ -70,12 +70,12 @@ jobs:
         run: nox -p ${{ matrix.python-version }} -s unit_tests -- --cov=datagateway_api --cov-report=xml
       - name: Upload unit test code coverage report
         if: matrix.python-version == '3.6'
-        uses: codecov/codecov-action@894ff025c7b54547a9a2a1e9f228beae737ad3c2 # v3.1.3
+        uses: codecov/codecov-action@eaaf4bedf32dbdc6b720b63067d99c4d77d6047d # v3.1.4
 
       # ICAT Ansible clone and install dependencies
       - name: Checkout icat-ansible
         if: success()
-        uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab # v3.5.2
+        uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0
         with:
           repository: icatproject-contrib/icat-ansible
           ref: master
@@ -122,7 +122,7 @@ jobs:
           cd /home/runner/install/icat.server/ && ./setup -vv install
 
       - name: Checkout DataGateway API
-        uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab # v3.5.2
+        uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0
 
       # Prep for using the API for tests
       - name: Create log file
@@ -151,19 +151,19 @@ jobs:
         run: nox -p ${{ matrix.python-version }} -s integration_tests -- --cov=datagateway_api --cov-report=xml
       - name: Upload integration test code coverage report
         if: matrix.python-version == '3.6'
-        uses: codecov/codecov-action@894ff025c7b54547a9a2a1e9f228beae737ad3c2 # v3.1.3
+        uses: codecov/codecov-action@eaaf4bedf32dbdc6b720b63067d99c4d77d6047d # v3.1.4
 
   linting:
     runs-on: ubuntu-20.04
     name: Linting
     steps:
       - name: Setup Python
-        uses: actions/setup-python@57ded4d7d5e986d7296eab16560982c6dd7c923b # v4.6.0
+        uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4.7.0
         with:
           python-version: "3.9.7"
           architecture: x64
       - name: Checkout DataGateway API
-        uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab # v3.5.2
+        uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0
 
       - name: Install Nox
         run: pip install nox==2020.8.22
@@ -178,12 +178,12 @@ jobs:
     name: Code Formatting
     steps:
       - name: Setup Python
-        uses: actions/setup-python@57ded4d7d5e986d7296eab16560982c6dd7c923b # v4.6.0
+        uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4.7.0
         with:
           python-version: "3.9.7"
           architecture: x64
       - name: Checkout DataGateway API
-        uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab # v3.5.2
+        uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0
 
       - name: Install Nox
         run: pip install nox==2020.8.22
@@ -198,12 +198,12 @@ jobs:
     name: Dependency Safety
     steps:
       - name: Setup Python
-        uses: actions/setup-python@57ded4d7d5e986d7296eab16560982c6dd7c923b # v4.6.0
+        uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4.7.0
         with:
           python-version: "3.9.7"
           architecture: x64
       - name: Checkout DataGateway API
-        uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab # v3.5.2
+        uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0
 
       - name: Install Nox
         run: pip install nox==2020.8.22
@@ -223,19 +223,19 @@ jobs:
 
       # Setup Java & Python
       - name: Setup Java
-        uses: actions/setup-java@5ffc13f4174014e2d4d4572b3d74c3fa61aeb2c2 # v3.11.0
+        uses: actions/setup-java@cd89f46ac9d01407894225f350157564c9c7cee2 # v3.12.0
         with:
           distribution: 'temurin'
           java-version: 8
       - name: Setup Python
-        uses: actions/setup-python@57ded4d7d5e986d7296eab16560982c6dd7c923b # v4.6.0
+        uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4.7.0
         with:
           python-version: "3.9.7"
           architecture: x64
 
       # ICAT Ansible clone and install dependencies
       - name: Checkout icat-ansible
-        uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab # v3.5.2
+        uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0
         with:
           repository: icatproject-contrib/icat-ansible
           ref: master
@@ -271,7 +271,7 @@ jobs:
           ansible-playbook icat-ansible/icatdb_minimal_hosts.yml -i icat-ansible/hosts --vault-password-file icat-ansible/vault_pass.txt -vv
 
       - name: Checkout DataGateway API
-        uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab # v3.5.2
+        uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0
 
       - name: Create config.yaml
         run: cd /home/runner/work/datagateway-api/datagateway-api; cp datagateway_api/config.yaml.example datagateway_api/config.yaml
@@ -330,7 +330,7 @@ jobs:
         run: cd /home/runner/install/icat.server; sudo mysql -uroot -D icatdb < create_triggers_mysql_5_0.sql
 
       - name: Checkout DataGateway API (default branch)
-        uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab # v3.5.2
+        uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0
         with:
           ref: ${{ github.event.repository.default_branch }}
 
@@ -360,3 +360,35 @@ jobs:
       # different to SciGateway preprod
       - name: Diff SQL dumps
         run: diff -s ~/generator_script_dump_main.sql ~/generator_script_dump_1.sql
+
+  docker:
+    # This job triggers only if all the other jobs succeed and does different things depending on the context.
+    # The job builds the Docker image in all cases and also pushes the image to Harbor only if something is
+    # pushed to the main branch.
+    needs: [tests, linting, formatting, safety, generator-script-testing]
+    name: Docker
+    runs-on: ubuntu-20.04
+    steps:
+    - name: Check out repo
+      uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.5.3
+
+    - name: Login to Harbor
+      uses: docker/login-action@f4ef78c080cd8ba55a85445d5b36e214a81df20a # v2.1.0
+      with:
+        registry: harbor.stfc.ac.uk/datagateway
+        username: ${{ secrets.HARBOR_USERNAME }}
+        password: ${{ secrets.HARBOR_TOKEN }}
+
+    - name: Extract metadata (tags, labels) for Docker
+      id: meta
+      uses: docker/metadata-action@c4ee3adeed93b1fa6a762f209fb01608c1a22f1e # v4.4.0
+      with:
+        images: harbor.stfc.ac.uk/datagateway/datagateway-api
+
+    - name: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' && 'Build and push Docker image to Harbor' || 'Build Docker image' }}
+      uses: docker/build-push-action@3b5e8027fcad23fda98b2e3ac259d8d67585f671 # v4.0.0
+      with:
+        context: .
+        push: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }}
+        tags: ${{ steps.meta.outputs.tags }}
+        labels: ${{ steps.meta.outputs.labels }}
diff --git a/.github/workflows/release-build.yml b/.github/workflows/release-build.yml
@@ -11,12 +11,12 @@ jobs:
 
     steps:
       - name: Checkout repo
-        uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab # v3.5.2
+        uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0
         with:
           fetch-depth: 0
           token: ${{ secrets.ADMIN_PAT }}
 
       - name: Python Semantic Release
-        uses: python-semantic-release/python-semantic-release@1aed73030dfc573d11ac590f7b7243a0f390b0fc # v7.34.4
+        uses: python-semantic-release/python-semantic-release@3abfb7ac216b9ad439de24fda60eca84038e850e # v8.0.8
         with:
           github_token: ${{ secrets.ADMIN_PAT }}
diff --git a/Dockerfile b/Dockerfile
@@ -0,0 +1,58 @@
+# Dockerfile to build and serve datagateway-api
+
+# Build stage
+FROM python:3.11-alpine3.17 as builder
+
+WORKDIR /datagateway-api-build
+
+COPY README.md poetry.lock pyproject.toml ./
+COPY datagateway_api/ datagateway_api/
+
+RUN --mount=type=cache,target=/root/.cache \
+    set -eux; \
+    \
+    python3 -m pip install 'poetry~=1.3.2'; \
+    poetry build;
+
+
+# Install & run stage
+FROM python:3.11-alpine3.17
+
+WORKDIR /datagateway-api-run
+
+COPY --from=builder /datagateway-api-build/dist/datagateway_api-*.whl /tmp/
+
+RUN --mount=type=cache,target=/root/.cache \
+    set -eux; \
+    \
+    python3 -m pip install \
+        'gunicorn~=20.1.0' \
+        /tmp/datagateway_api-*.whl; \
+    \
+    # Create a symlink to the installed python module \
+    DATAGATEWAY_API_LOCATION="$(python3 -m pip show datagateway_api | awk '/^Location:/ { print $2 }')"; \
+    ln -s "$DATAGATEWAY_API_LOCATION/datagateway_api/" datagateway_api; \
+    \
+    # Create config.yaml and search_api_mapping.json from their .example files \
+    cp datagateway_api/config.yaml.example datagateway_api/config.yaml; \
+    cp datagateway_api/search_api_mapping.json.example datagateway_api/search_api_mapping.json; \
+    \
+    # Create a non-root user to run as \
+    addgroup -S datagateway-api; \
+    adduser -S -D -G datagateway-api -H -h /datagateway-api-run datagateway-api; \
+    \
+    # Change ownership of config.yaml - the entrypoint script will need to edit it \
+    chown datagateway-api:datagateway-api datagateway_api/config.yaml;
+
+USER datagateway-api
+
+ENV ICAT_URL="http://localhost"
+ENV ICAT_CHECK_CERT="false"
+ENV LOG_LOCATION="/dev/stdout"
+
+COPY docker/docker-entrypoint.sh /usr/local/bin/
+ENTRYPOINT ["docker-entrypoint.sh"]
+
+# Serve the application using gunicorn - production ready WSGI server
+CMD ["gunicorn", "-b", "0.0.0.0:8000", "datagateway_api.wsgi"]
+EXPOSE 8000
diff --git a/README.md b/README.md
@@ -375,6 +375,32 @@ If using Python 3.10, please use Payara 5 on the ICAT stack which the API is bei
 pointed at. There is a known issue when making HTTPS connections to Payara (via Python
 ICAT).
 
+It is also possible to run the API inside Docker. The `Dockerfile` can be used to build
+a Docker image which in turn can be used to create a container. The `Dockerfile` is
+configured to create a production image and runs a Gunicorn serve on port `8000` when a
+container is started. Environment variables have also been defined in the `Dockerfile`
+to allow for values to be passed at runtime to future running containers. These values
+are used by the `docker/docker-entrypoint.sh` script to update the config values in the
+`config.yaml` file. The environment varialbes are:
+- `ICAT_URL` (Default value: `http://localhost`)
+- `ICAT_CHECK_CERT` (Default value: `false`)
+- `LOG_LOCATION` (Default value: `/dev/stdout`)
+
+To build an image, run:
+```bash
+docker build -t datagateway_api_image .
+```
+
+To start a container on port `8000` from the image that you just built, run:
+```bash
+docker run -p 8000:8000 --name datagateway_api_container datagateway_api_image 
+```
+
+If you want to pass values for the environment variables then instead run:
+```bash
+docker run -p 8000:8000 --name datagateway_api_container --env ICAT_URL=https://127.0.0.1:8181 --env ICAT_CHECK_CERT=true --env LOG_LOCATION=/datagateway-api-run/logs.log datagateway_api_image
+```
+
 ## DataGateway API Authentication
 
 Each request requires a valid session ID to be provided in the Authorization header.

diff --git a/docker/docker-entrypoint.sh b/docker/docker-entrypoint.sh
@@ -0,0 +1,17 @@
+#!/bin/sh -eu
+
+# Use a tempfile instead of sed -i so that only the file, not the directory needs to be writable
+TEMPFILE="$(mktemp)"
+
+# Set values in config.yaml from environment variables
+# No quotes for icat_check_cert because it's boolean
+sed -e "s|icat_url: \".*\"|icat_url: \"$ICAT_URL\"|" \
+    -e "s|icat_check_cert: .*|icat_check_cert: $ICAT_CHECK_CERT|" \
+    -e "s|log_location: \".*\"|log_location: \"$LOG_LOCATION\"|" \
+    datagateway_api/config.yaml > "$TEMPFILE"
+
+cat "$TEMPFILE" > datagateway_api/config.yaml
+rm "$TEMPFILE"
+
+# Run the CMD instruction
+exec "$@"
diff --git a/noxfile.py b/noxfile.py
@@ -72,13 +72,8 @@ def safety(session):
             f"--output={requirements.name}",
             external=True,
         )
-        # Ignore 50916 as the latest version of pydantic and
-        # Ignore 51457 as the latest version of pytest and
-        # Ignore 51668 as the latest version of SQLAchemy and
-        # Ignore 52322 and 52518 as the latest version of Gitpython does not
-        # support python 3.6 which is still used in production
-        # Ignore 53325, 53326, 54456, and 55261 as the fixed versions do not support
-        # python 3.6
+        # Ignore vulnerabilities as the patched versions of dependencies that they
+        # relate to don't support Python 3.6 which is still required for production
         session.run(
             "safety",
             "check",
@@ -102,6 +97,14 @@ def safety(session):
             "54456",
             "--ignore",
             "55261",
+            "--ignore",
+            "58910",
+            "--ignore",
+            "58755",
+            "--ignore",
+            "59062",
+            "--ignore",
+            "59473",
         )
 
         try: