Merge pull request #1461 from dodona-edu/feature/enhanced-docker

Enhance self-hosting support with prebuilt docker images

.github/workflows/ci-cd.yml

@@ -34,7 +34,7 @@ jobs:
       - name: Run tests 🧪
         run: |
           cd api/
-          docker pull ghcr.io/dodona-edu/dolos:latest
+          docker pull ghcr.io/dodona-edu/dolos-cli:latest
           bundle exec rails db:prepare
           bundle exec rails test
 
@@ -722,7 +722,7 @@ jobs:
              dodona@dolos.ugent.be:demo
 
   docker:
-    name: "Publish docker container 🚢"
+    name: "Publish docker containers 🚢"
     needs: publish
     runs-on: ubuntu-latest
     steps:
@@ -740,14 +740,23 @@ jobs:
         id: parse_tag
         run: "echo ${{ github.ref }} | sed 's#^refs/tags/v#version=#' >> $GITHUB_OUTPUT"
 
-      - name: Build the new image
-        run: docker build docker/ -t ghcr.io/dodona-edu/dolos:${{ steps.parse_tag.outputs.version }}
+      - name: Build the Dolos images
+        run: |
+          docker build docker/ -t ghcr.io/dodona-edu/dolos-cli:${{ steps.parse_tag.outputs.version }}
+          docker build web/ -t ghcr.io/dodona-edu/dolos-web:${{ steps.parse_tag.outputs.version }}
+          docker build api/ -t ghcr.io/dodona-edu/dolos-api:${{ steps.parse_tag.outputs.version }}
 
       - name: Login to the container registry
         run: echo ${{ secrets.GITHUB_TOKEN }} | docker login ghcr.io -u $GITHUB_ACTOR --password-stdin
 
-      - name: Tag the new image with latest
-        run: docker tag ghcr.io/dodona-edu/dolos:${{ steps.parse_tag.outputs.version }} ghcr.io/dodona-edu/dolos:latest
+      - name: Tag the new images with latest
+        run: |
+          docker tag ghcr.io/dodona-edu/dolos-cli:${{ steps.parse_tag.outputs.version }} ghcr.io/dodona-edu/dolos-cli:latest
+          docker tag ghcr.io/dodona-edu/dolos-web:${{ steps.parse_tag.outputs.version }} ghcr.io/dodona-edu/dolos-web:latest
+          docker tag ghcr.io/dodona-edu/dolos-api:${{ steps.parse_tag.outputs.version }} ghcr.io/dodona-edu/dolos-api:latest
 
-      - name: Push the new image
-        run: docker push --all-tags ghcr.io/dodona-edu/dolos
+      - name: Push the images
+        run: |
+          docker push --all-tags ghcr.io/dodona-edu/dolos-cli
+          docker push --all-tags ghcr.io/dodona-edu/dolos-web
+          docker push --all-tags ghcr.io/dodona-edu/dolos-api

.github/workflows/validation.yml

@@ -0,0 +1,58 @@
+name: Validation
+on:
+  push:
+    branches: [main]
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  install-cli:
+    name: "Install and run CLI on ${{ matrix.os }}"
+    strategy:
+      matrix:
+        os:
+          - macos-14
+          - macos-13
+          - ubuntu-latest
+      fail-fast: false
+    runs-on: ${{ matrix.os }}
+    steps:
+      - name: Setup Node 📚
+        uses: actions/setup-node@v4
+        with:
+          node-version: 21
+
+      - name: Install dolos
+        shell: bash
+        run: |
+          npm install -g @dodona/dolos
+          dolos --version
+
+      - name: Prepare analysis
+        run: |
+          wget https://dolos.ugent.be/simple-dataset.zip
+
+      - name: Run analysis on samples
+        run: |
+          dolos simple-dataset.zip
+
+  self-hosting:
+    name: "Test docker-compose for self-hosting Dolos web app"
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout 🛎️
+        uses: actions/checkout@v4
+
+      - name: Run test script
+        run: ./test_compose.sh
+
+  packaging:
+    name: "Test packing and installing Dolos"
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout 🛎️
+        uses: actions/checkout@v4
+
+      - name: Run test script
+        run: ./test_package.sh

api/Dockerfile

@@ -2,6 +2,8 @@
 FROM ruby:3.2.3
 RUN apt-get update -qq && apt-get -y install mariadb-client
 
+LABEL org.opencontainers.image.source https://github.com/dodona-edu/dolos
+
 ADD config.ru Gemfile Gemfile.lock Rakefile /dolos/
 WORKDIR /dolos
 RUN bundle install && mkdir log

api/app/jobs/analyze_dataset_job.rb

@@ -8,7 +8,7 @@ class AnalyzeDatasetJob < ApplicationJob
 
   TMPDIR_PATH = '/tmp'.freeze
   OUTPUT_DIRNAME = 'result'.freeze
-  DOLOS_IMAGE = 'ghcr.io/dodona-edu/dolos:latest'.freeze
+  DOLOS_IMAGE = 'ghcr.io/dodona-edu/dolos-cli:latest'.freeze
   TIMEOUT = 60.seconds
   MEMORY_LIMIT = 2_000_000_000
   OUTPUT_LIMIT = 65_000

api/config/routes.rb

@@ -1,11 +1,14 @@
 Rails.application.routes.draw do
+  # Define your application routes per the DSL in https://guides.rubyonrails.org/routing.html
   resources :reports, except: %i[index update] do
     member do
       get 'data/:file', to: 'reports#data', as: 'data'
     end
   end
+
   resources :datasets, only: %i[show]
-  # Define your application routes per the DSL in https://guides.rubyonrails.org/routing.html
+
+  get "up" => "rails/health#show"
 
   # Defines the root path route ("/")
   # root "articles#index"

docker-compose.yml

@@ -1,3 +1,4 @@
+
 name: dolos
 version: "3.9"
 services:
@@ -9,9 +10,13 @@ services:
       MARIADB_ROOT_PASSWORD: dolos
     networks:
       - dolos
+    healthcheck:
+      start_period: 2s
+      interval: 5s
+      test: healthcheck.sh --su-mysql --connect --innodb_initialized
   web:
-    build:
-      context: ./web/
+    image: ghcr.io/dodona-edu/dolos-web
+    # build: ./web/
     ports:
       - "8080:8080"
     environment:
@@ -23,8 +28,13 @@ services:
       - api
     networks:
       - dolos
+    healthcheck:
+      start_period: 2s
+      interval: 5s
+      test: curl --fail http://localhost:8080 || exit 1
   api:
-    build: ./api/
+    image: ghcr.io/dodona-edu/dolos-api
+    # build: ./api/
     command: bash -c "rm -f tmp/pids/server.pid && bundle exec rails db:prepare && bundle exec rails s -p 3000 -b '0.0.0.0'"
     volumes:
       - dolos-api-storage:/dolos/storage
@@ -42,11 +52,17 @@ services:
       RAILS_LOG_TO_STDOUT: true
       SECRET_KEY_BASE_DUMMY: true
     depends_on:
-      - worker
+      db:
+        condition: service_healthy
     networks:
       - dolos
+    healthcheck:
+      start_period: 2s
+      interval: 10s
+      test: curl --fail http://localhost:3000/up || exit 1
   worker:
-    build: ./api/
+    image: ghcr.io/dodona-edu/dolos-api
+    # build: ./api/
     command: bash -c "bundle exec rails jobs:work"
     volumes:
       - dolos-api-storage:/dolos/storage
@@ -60,9 +76,13 @@ services:
       SECRET_KEY_BASE_DUMMY: true
       RAILS_LOG_TO_STDOUT: true
     depends_on:
-      - db
+      - api
     networks:
       - dolos
+    healthcheck:
+      start_period: 2s
+      interval: 5s
+      test: ruby -e true || exit 1
 volumes:
   dolos-db-data:
   dolos-api-storage:

docker/Dockerfile

@@ -1,6 +1,8 @@
 FROM node:20-alpine3.17
 ARG dolos_version=2.5.1
 
+LABEL org.opencontainers.image.source https://github.com/dodona-edu/dolos
+
 # node-gyp needs python3 and a compiler to build tree-sitter
 RUN apk --no-cache add python3 build-base unzip &&\
     npm -g install @dodona/dolos@$dolos_version &&\

docs/docs/docker.md

@@ -1,11 +1,11 @@
-# Install Dolos with Docker
+# Install Dolos CLI with Docker
 
-The lastest Dolos version comes pre-installed in a [Docker container image](https://github.com/dodona-edu/dolos/pkgs/container/dolos)
+The latest Dolos version comes pre-installed in a [Docker container image](https://github.com/dodona-edu/dolos/pkgs/container/dolos-cli)
 that is available from GitHub's container registry.
 Use the following commando to pull the image:
 
 ```shell
-docker pull ghcr.io/dodona-edu/dolos:latest
+docker pull ghcr.io/dodona-edu/dolos-cli:latest
 ```
 
 ## Run Dolos CLI in Docker
@@ -23,7 +23,7 @@ Here is, for example, a containerized version of the command from the [Running D
 to run a plagiarism detection analysis and open an interatie web app where the analysis results can be explored:
 
 ```shell
-docker run --init --network host -v "$PWD:/dolos" ghcr.io/dodona-edu/dolos -l javascript -f web *.js
+docker run --init --network host -v "$PWD:/dolos" ghcr.io/dodona-edu/dolos-cli -l javascript -f web *.js
 ```
 
 :::: tip

docs/docs/hosting-dolos.md

@@ -9,23 +9,17 @@ These instructions are for **local** hosting only. The section [external hosting
 
 [Docker](https://www.docker.com/) is a containerization technology that allows you to run our service without the hassle of installing the different dependencies and services yourself.
 
-::: info
-
-The instructions below are for **Linux** and **MacOS** systems only.
-[Contact us](/about/contact) if you want to run the Dolos web app on windows.
-
-:::
 
 Run Dolos on your own system using these instructions:
 
 1. Ensure [Git](https://git-scm.com/downloads), [Docker Engine](https://docs.docker.com/engine/install/) and [Docker Compose](https://docs.docker.com/compose/install/) are installed on the system where you will be running Dolos on.
-2. Clone the Docker repository with Git and enter the new directory
+2. In a terminal, clone the Docker repository with Git and enter the new directory
   ```
     git clone https://github.com/dodona-edu/dolos.git
     cd dolos/
   ```
-3. Run `docker-compose build` in this directory to pull and fetch all needed container images.
-4. Run `docker pull ghcr.io/dodona-edu/dolos:latest` to ensure the container running the Dolos CLI is up-to-date.
+3. Run `docker-compose pull` in this directory to pull and fetch all needed container images.
+4. Run `docker pull ghcr.io/dodona-edu/dolos-cli:latest` to ensure the container running the Dolos CLI is present and up-to-date.
 5. Run `docker-compose up` to start the services.
 
 You can now visit the web app running locally on <http://localhost:8080>.

test_compose.sh

@@ -0,0 +1,46 @@
+#!/usr/bin/env sh
+set -e
+
+# This script tests whether self-hosting the Dolos web-app works using the docker-compose.yml
+
+docker compose --progress quiet down
+docker compose --progress quiet pull
+docker pull ghcr.io/dodona-edu/dolos-cli:latest
+docker compose up --wait --detach
+
+echo "Upload zipfile"
+
+report_url="$(
+  curl -s --fail \
+    --form "dataset[name]=Example" \
+    --form "dataset[zipfile]=@./samples/javascript/simple-dataset.zip" \
+    http://localhost:3000/reports \
+    | jq -r '.url'
+    )"
+
+
+echo "Polling $report_url until finished or failed"
+
+while sleep 1; do
+  report_json="$(curl -s --fail "$report_url")"
+  report_status="$(printf "%s" "$report_json" | jq -r '.status')"
+  echo "Status is '$report_status'"
+  case "$report_status" in
+    "queued" | "running")
+      ;;
+    "finished")
+      break
+      ;;
+    *)
+      echo "Something went wrong:"
+      echo "$report_json"
+      exit 1
+      ;;
+  esac
+done
+
+curl -s --location --fail "$report_url/data/pairs.csv"
+
+echo "Everything is working as expected"
+
+docker compose down

test_package.sh

@@ -1,7 +1,10 @@
 #!/usr/bin/env sh
-# Run the current script in a docker container
+
+# This script tests whether packaging Dolos with its different components works
+
+# Run the current script in a docker container with a clean repository
 exec time docker run -v "$PWD:/repo:ro" --rm --entrypoint="" node:20 \
-  sh -c 'git clone --recursive --no-remote-submodules --shallow-submodules /repo /dolos && cd /dolos && tail -n+5 test_package.sh | sh -'
+  sh -c 'git clone --recursive --no-remote-submodules --shallow-submodules /repo /dolos && cd /dolos && tail -n+8 test_package.sh | sh -'
 
 ### Docker script starts here
 set -e

web/Dockerfile

@@ -6,7 +6,7 @@ ADD public/ /web/public
 ADD src/ /web/src
 ADD index.html index.d.ts index.js package.json tsconfig.json tsconfig.node.json vite.config.ts /web/
 
-RUN npm install
+RUN npm install && apk add --no-cache curl
 
 ENV VITE_HOST=0.0.0.0
 ENV VITE_PORT=8080