diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..46aa2a0 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,77 @@ +# Multi-stage unique docker build script, provides both dev & prod environments + +# ---------------------------------------------------- +# Base-image +# ---------------------------------------------------- +FROM python:3.9-slim-buster as common-base +# Django directions: https://blog.ploetzli.ch/2020/efficient-multi-stage-build-django-docker/ +# Pip on docker : https://pythonspeed.com/articles/multi-stage-docker-python/ +# https://blog.mikesir87.io/2018/07/leveraging-multi-stage-builds-single-dockerfile-dev-prod/ +# https://pythonspeed.com/articles/base-image-python-docker-images/ + +# Default environment: Dev +ARG ENV=dev + +ENV PYTHONFAULTHANDLER=1 \ + PYTHONUNBUFFERED=1 \ + PYTHONHASHSEED=random \ + PIP_NO_CACHE_DIR=off \ + PIP_DISABLE_PIP_VERSION_CHECK=on \ + PIP_DEFAULT_TIMEOUT=100 + +ENV HOST=0.0.0.0 \ + PORT=8000 + +WORKDIR /app + +COPY ./docker/install-packages.sh . +RUN ./install-packages.sh + +# ---------------------------------------------------- +# Install dependencies +# ---------------------------------------------------- +FROM common-base AS dependencies +ENV PATH="/opt/venv/bin:$PATH" + +# apt-get install build-essential -y +COPY requirements.txt /app/ + +RUN pip install --target /opt/packages -r requirements.txt + +# ---------------------------------------------------- +# Copy project +# ---------------------------------------------------- +FROM common-base AS app-run +COPY --from=dependencies /opt/packages /opt/packages +ENV PYTHONPATH "${PYTHONPATH}:/opt/packages" +# ENV PYTHONPATH="$PYTHONPATH:/app/lemarche:/app/config" +COPY ./ark ./ark +COPY ./ark_import ./ark_import +COPY ./arklet ./arklet +COPY ./manage.py ./manage.py +COPY ./docker/entrypoint.sh ./entrypoint.sh + +# ---------------------------------------------------- +# Run Dev +# ---------------------------------------------------- +FROM app-run AS dev +ENV ENV="dev" \ + ARKLET_DEBUG="True" + +CMD ["bash"] + +# ---------------------------------------------------- +# Run Prod +# ---------------------------------------------------- +FROM app-run AS prod +ENV ENV="prod" \ + ARKLET_DEBUG="False" + +CMD ["./entrypoint.sh"] + +# # For some _real_ performance, at cost of ease of use: +# FROM python:3.9-alpine as prod +# ENV PATH="/opt/venv/bin:$PATH" +# COPY . . +# RUN apk add python3-dev build-base linux-headers pcre-dev +# RUN pip install uwsgi diff --git a/README.md b/README.md index c892b59..81d4430 100644 --- a/README.md +++ b/README.md @@ -6,12 +6,19 @@ See https://arks.org/ ## What is Arklet? Arklet is a Python Django application for minting, binding, and resolving ARKs. It is intended to follow best practices set out by https://arks.org/. -## Running Locally with Postgres +## Running -Create the default config YAML file: **/etc/arklet.yml** +### Locally with Postgres -- Set a value for **ARKLET_DJANGO_SECRET_KEY**. -- For local development, set **ARKLET_DEBUG** to True. +Create the default `.env` file in the project's root directory + +``` +# /!\ Set your own secret key /!\ +ARKLET_DJANGO_SECRET_KEY=[YOUR_SECRET] + +# For local development, set to True +ARKLET_DEBUG=True +``` Run Postgres, install into virtual environment, and start the app: ``` @@ -29,11 +36,73 @@ python manage.py createsuperuser python manage.py runserver ``` +### Separate dockers +Using docker, we can use a [this provided](./docker/env.docker.local) config file. + +See above for running PostgreSQL, and run the **Arklet** docker as follows (in *bash*): +``` +docker build \ + --target dev \ + -t "arklet" -f ./Dockerfile . \ + --build-arg ENV=DEV \ +&& docker run --rm -it \ + -p 8000:8000 \ + --env-file=./docker/env.docker.local \ + -e ARKLETDEBUG="true" \ + --name arklet \ + -v `pwd`/ark:/app/ark \ + -v `pwd`/ark_import:/app/ark_import \ + -v `pwd`/arklet:/app/arklet \ + arklet +``` + +### With docker-compose +Using the provided `docker-compose.yml` with default settings in the [docker +configuration directory](./docker) : + +``` +docker-compose up +``` + +By default, the folders `ark`, `ark_import` and `arklet` are mounted in the +container. Should you wish to attach a console to the `arklet` container (needed +to create the django superuser) : +``` +# In another shell +docker exec -it arklet_django /bin/bash +# You're now in the docker container +./manage.py createsuperuser +``` + +### First steps Create your first NAAN, Key, and Shoulder in the admin: 127.0.0.1:8000/admin +And by the way, you now host a working ARK resolver! You can already +try the following ones : +- [http://127.0.0.1:8000/ark:/13960/t5n960f7n](http://127.0.0.1:8000/ark:/13960/t5n960f7n) +- [http://127.0.0.1:8000/ark:/67375/C0X-SPWFRSGR-N](http://127.0.0.1:8000/ark:/67375/C0X-SPWFRSGR-N) +- [http://127.0.0.1:8000/ark:/12148/bpt6k65358454](http://127.0.0.1:8000/ark:/12148/bpt6k65358454) +- ... + Happy minting, binding, and resolving! ## Configuration Options -See arklet/settings.py for the full list of options to put in your config file. \ No newline at end of file +See arklet/settings.py for the full list of options to put in your config file. + +## Deploying +### With docker +Using the provided Dockerfile (is you wish to set a build target, use `prod`, +but being the default target you can skip this), provide the following values +in your environment: + +- ARKLET_DJANGO_SECRET_KEY=[YOUR_SECRET] +- ARKLET_DEBUG=False +- ARKLET_HOST=0.0.0.0 +- ARKLET_PORT=[Port of choice] +- ARKLET_POSTGRES_NAME=[DB NAME] +- ARKLET_POSTGRES_USER=[DB USER] +- ARKLET_POSTGRES_PASSWORD=[DB PASS] +- ARKLET_POSTGRES_HOST=[DB HOST] +- ARKLET_POSTGRES_PORT=[DB PORT] diff --git a/ark/views.py b/ark/views.py index 23ac653..507e9bb 100644 --- a/ark/views.py +++ b/ark/views.py @@ -163,10 +163,9 @@ def resolve_ark(request, ark: str): return HttpResponseRedirect(ark.url) except Ark.DoesNotExist: try: - naan = Naan.objects.get(naan=naan) - resolver = naan.url or "https://n2t.net" - # TODO: more robust resolver URL creation - return HttpResponseRedirect(f"{resolver}/ark:/{naan.naan}/{assigned_name}") + naan_obj = Naan.objects.get(naan=naan) + return HttpResponseRedirect(f"{naan_obj.url}/ark:/{naan_obj.naan}/{assigned_name}") except Naan.DoesNotExist: - # TODO: make nice page saying we don't know about the ARK or NAAN - raise Http404 + resolver = "https://n2t.net" + # TODO: more robust resolver URL creation + return HttpResponseRedirect(f"{resolver}/ark:/{naan}/{assigned_name}") diff --git a/arklet/settings.py b/arklet/settings.py index cc09a9d..a8d98e7 100644 --- a/arklet/settings.py +++ b/arklet/settings.py @@ -16,11 +16,22 @@ import sentry_sdk from sentry_sdk.integrations.django import DjangoIntegration import yaml - - -with open(os.environ.get("ARKLET_CONF", "/etc/arklet.yml")) as f: - conf = yaml.safe_load(f) - +import environ + +env = environ.Env( + # set default values + ARKLET_HOST=(str, "127.0.0.1"), + ARKLET_DEBUG=(bool, False), + ARKLET_POSTGRES_NAME=(str, "arklet"), + ARKLET_POSTGRES_HOST=(str, "127.0.0.1"), + ARKLET_POSTGRES_PORT=(str, "5432"), + ARKLET_POSTGRES_USER=(str, "arklet"), + ARKLET_POSTGRES_PASSWORD=(str, "arklet"), + ARKLET_SENTRY_DSN=(str, ""), + ARKLET_SENTRY_TRANSACTIONS_PER_TRACE=(int, 1), + ARKLET_STATIC_ROOT=(str, "static"), + ARKLET_MEDIA_ROOT=(str, "media"), +) # Build paths inside the project like this: BASE_DIR / 'subdir'. BASE_DIR = Path(__file__).resolve().parent.parent @@ -29,13 +40,13 @@ # See https://docs.djangoproject.com/en/3.2/howto/deployment/checklist/ # SECURITY WARNING: keep the secret key used in production secret! -SECRET_KEY = conf["ARKLET_DJANGO_SECRET_KEY"] # Intentionally no default value +SECRET_KEY = env.str("ARKLET_DJANGO_SECRET_KEY") # Intentionally no default value # SECURITY WARNING: don't run with debug turned on in production! -DEBUG = conf.get("ARKLET_DEBUG", False) +DEBUG = env("ARKLET_DEBUG") ALLOWED_HOSTS = [ - conf.get("ARKLET_HOST", "127.0.0.1"), + env("ARKLET_HOST"), "wbgrp-svc302.us.archive.org", "qa-ark.archive.org", "ark.archive.org", @@ -107,11 +118,11 @@ # "ENGINE": "django.db.backends.sqlite3", # "NAME": BASE_DIR / "db.sqlite3", "ENGINE": "django.db.backends.postgresql", - "NAME": conf.get("ARKLET_POSTGRES_NAME", "arklet"), - "HOST": conf.get("ARKLET_POSTGRES_HOST", "127.0.0.1"), - "PORT": conf.get("ARKLET_POSTGRES_PORT", "5432"), - "USER": conf.get("ARKLET_POSTGRES_USER", "arklet"), - "PASSWORD": conf.get("ARKLET_POSTGRES_PASSWORD", "arklet"), + "NAME": env("ARKLET_POSTGRES_NAME"), + "HOST": env("ARKLET_POSTGRES_HOST"), + "PORT": env("ARKLET_POSTGRES_PORT"), + "USER": env("ARKLET_POSTGRES_USER"), + "PASSWORD": env("ARKLET_POSTGRES_PASSWORD"), "DISABLE_SERVER_SIDE_CURSORS": True, } } @@ -158,17 +169,17 @@ STATIC_URL = "/static/" -STATIC_ROOT = conf.get("ARKLET_STATIC_ROOT") +STATIC_ROOT = env.str("ARKLET_STATIC_ROOT") -MEDIA_ROOT = conf.get("ARKLET_MEDIA_ROOT") +MEDIA_ROOT = env.str("ARKLET_MEDIA_ROOT") # Default primary key field type # https://docs.djangoproject.com/en/3.2/ref/settings/#default-auto-field DEFAULT_AUTO_FIELD = "django.db.models.BigAutoField" -SENTRY_DSN = conf.get("ARKLET_SENTRY_DSN", "") -SENTRY_SAMPLE_RATE = 1 / int(conf.get("ARKLET_SENTRY_TRANSACTIONS_PER_TRACE", 1)) +SENTRY_DSN = env("ARKLET_SENTRY_DSN") +SENTRY_SAMPLE_RATE = 1 / int(env("ARKLET_SENTRY_TRANSACTIONS_PER_TRACE")) sentry_sdk.init( dsn=SENTRY_DSN, integrations=[DjangoIntegration()], diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..6717443 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,35 @@ +version: "3.7" + +services: + postgres: + container_name: arkled_db + image: postgres:14-alpine + env_file: + - ./docker/env.docker.local + volumes: + - postgres:/var/lib/postgresql/data + restart: always + ports: + - "5432:5432" + + arklet: + container_name: arklet_django + restart: always + build: + context: . + target: dev + dockerfile: ./Dockerfile + command: /app/entrypoint.sh + volumes: + - ./ark:/app/ark + - ./ark_import:/app/ark_import + - ./arklet:/app/arklet + env_file: + - ./docker/env.docker.local + ports: + - "8000:8000" + depends_on: + - postgres + +volumes: + postgres: diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 0000000..177d748 --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,104 @@ +# ---------------------------------------------------- +# Base-image +# ---------------------------------------------------- +FROM python:3.9-slim-buster as common-base +# Django directions: https://blog.ploetzli.ch/2020/efficient-multi-stage-build-django-docker/ +# Pip on docker : https://pythonspeed.com/articles/multi-stage-docker-python/ +# https://blog.mikesir87.io/2018/07/leveraging-multi-stage-builds-single-dockerfile-dev-prod/ +# https://pythonspeed.com/articles/base-image-python-docker-images/ + +# Default environment: Dev +ARG ENV=dev + +ENV PYTHONFAULTHANDLER=1 \ + PYTHONUNBUFFERED=1 \ + PYTHONHASHSEED=random \ + PIP_NO_CACHE_DIR=off \ + PIP_DISABLE_PIP_VERSION_CHECK=on \ + PIP_DEFAULT_TIMEOUT=100 \ + POETRY_VERSION=1.1.1 \ + NODE_VERSION=15 + +ENV HOST=0.0.0.0 \ + PORT=8000 + +WORKDIR /app + +COPY install-packages.sh . +RUN ./install-packages.sh + +# ---------------------------------------------------- +# Install dependencies +# ---------------------------------------------------- +FROM common-base AS dependencies +RUN python -m venv /opt/venv +ENV PATH="/opt/venv/bin:$PATH" + +RUN pip install "poetry==$POETRY_VERSION" +RUN pip install -I uwsgi + +# apt-get install build-essential -y +COPY poetry.lock pyproject.toml /app/ + +RUN poetry config virtualenvs.create false && \ + poetry config virtualenvs.path /opt/venv && \ + poetry install $(test $ENV == "prod" && echo "--no-dev") --no-interaction --no-ansi + +# ---------------------------------------------------- +# Build project +# ---------------------------------------------------- +FROM common-base AS app-run +COPY --from=dependencies /opt/venv /opt/venv +ENV PATH="/opt/venv/bin:$PATH" \ + VIRTUALENV="/opt/venv" \ + PYTHONPATH="$PYTHONPATH:/app/lemarche:/app/config" +COPY ./lemarche ./lemarche +COPY ./config ./config +COPY ./manage.py ./manage.py +COPY ./pyproject.toml ./pyproject.toml +COPY ./docker ./docker + +# ---------------------------------------------------- +# Run Dev +# ---------------------------------------------------- +FROM app-run AS dev +ENV DJANGO_SETTINGS_MODULE="config.settings.dev" \ + ENV="dev" \ + DEBUG="True" + +RUN echo '[ ! -z "$TERM" -a -r /etc/motd ] && cat /etc/issue && cat /etc/motd' \ + >> /etc/bash.bashrc \ + ; echo "\ +===================================================================\n\ += Bitoubi API Dev Docker container =\n\ +===================================================================\n\ +\n\ +(c) plateforme de l'Inclusion\n\ +\n\ +Source directory is /app \n\ + +Run API with :\n\ +> python ./manage.py runserver \$HOST:\$PORT\n\ +\n\ +"\ + > /etc/motd + +CMD ["bash"] + +# ---------------------------------------------------- +# Run Dev +# ---------------------------------------------------- +FROM app-run AS prod +ENV DJANGO_SETTINGS_MODULE="config.settings.prod" \ + ENV="prod" \ + DEBUG="False" + +CMD [".docker/dev/entrypoint.sh"] + +# # For some _real_ performance, at cost of ease of use: +# FROM python:3.9-alpine as prod +# COPY --from=dependencies /opt/venv /opt/venv +# ENV PATH="/opt/venv/bin:$PATH" +# COPY . . +# RUN apk add python3-dev build-base linux-headers pcre-dev +# RUN pip install uwsgi diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh new file mode 100755 index 0000000..83a06a7 --- /dev/null +++ b/docker/entrypoint.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +# Getting static files for Admin panel hosting! +set -e + +# White while DB is spinning up +echo "pg_isready -h $ARKLET_POSTGRES_HOST -p $ARKLET_POSTGRES_PORT" +while ! pg_isready -h $ARKLET_POSTGRES_HOST -p $ARKLET_POSTGRES_PORT; do + >&2 echo "Postgres is unavailable - sleeping" + sleep 1 +done + +# Following rules depend on what you expect from django dev docker +# ./manage.py collectstatic --noinput +# ./manage.py compress --force + +./manage.py migrate +./manage.py createsuperuser + +./manage.py runserver 0.0.0.0:$ARKLET_PORT +# gunicorn config.wsgi:application -w 2 -b :8880 --reload diff --git a/docker/env.docker.local b/docker/env.docker.local new file mode 100644 index 0000000..68ae02b --- /dev/null +++ b/docker/env.docker.local @@ -0,0 +1,30 @@ +# Config for POSTGRES +# ########################### +# Avoid a log error when starting the itou_postgres container: +# > Role "root" does not exist. +# Without this variable, the default Unix account ('root') +# is used automatically when starting postgres. +# https://www.postgresql.org/docs/current/libpq-envars.html +PGUSER=postgres + +# PostgreSQL +POSTGRES_DB=arklet +POSTGRES_USER=postgres +POSTGRES_PASSWORD=postgres # ggignore +POSTGRES_HOST=postgres +POSTGRESQL_PORT=5432 + +# Config for ARKLET +# ######################## +# Django + +ARKLET_DJANGO_SECRET_KEY=[YOUR_SECRET] ##ggignore +ARKLET_DEBUG=True +ARKLET_HOST=127.0.0.1 +ARKLET_PORT=8000 + +ARKLET_POSTGRES_NAME=arklet +ARKLET_POSTGRES_USER=postgres +ARKLET_POSTGRES_PASSWORD=postgres # ggignore +ARKLET_POSTGRES_HOST=postgres +ARKLET_POSTGRES_PORT=5432 diff --git a/docker/install-packages.sh b/docker/install-packages.sh new file mode 100755 index 0000000..b91e1ce --- /dev/null +++ b/docker/install-packages.sh @@ -0,0 +1,39 @@ +#!/bin/bash +# Why this script ? See : +# https://pythonspeed.com/articles/system-packages-docker/ + +# Bash "strict mode", to help catch problems and bugs in the shell +# script. Every bash script you write should include this. See +# http://redsymbol.net/articles/unofficial-bash-strict-mode/ for +# details. +set -euo pipefail + +# Tell apt-get we're never going to be able to give manual +# feedback: +export DEBIAN_FRONTEND=noninteractive + + +# Update the package listing, so we know what package exist: +apt-get update -qq + +# Install security updates: +apt-get -y upgrade + +# Install a new package, without unnecessary recommended packages: +apt-get -y install --no-install-recommends \ + vim-tiny \ + postgresql-client \ + binutils \ + libproj-dev \ + gdal-bin \ + iputils-ping \ + libpcre3 \ + libpcre3-dev \ + libsass1 \ + gettext \ + build-essential \ + gcc + +# Delete cached files we don't need anymore: +apt-get clean +rm -rf /var/lib/apt/lists/* diff --git a/requirements.txt b/requirements.txt index 5adb622..6319233 100644 --- a/requirements.txt +++ b/requirements.txt @@ -20,3 +20,4 @@ sqlparse==0.4.2 tomli==1.2.2 typing-extensions==3.10.0.2 urllib3==1.26.7 +django-environ==0.8.0