diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..0a407ba --- /dev/null +++ b/.dockerignore @@ -0,0 +1,12 @@ +.cache +.dockerignore +.gitignore +.git +.github +.env +.pylintrc +__pycache__ +*.pyc +*.egg-info +.idea/ +.vscode diff --git a/.env.sample b/.env.sample new file mode 100644 index 0000000..86879c9 --- /dev/null +++ b/.env.sample @@ -0,0 +1,7 @@ +PYTHONPATH=/app/ +DEBUG=0 +DJANGO_SETTINGS_MODULE=config.settings.test +DJANGO_SECRET_KEY=t3st-s3cr3t#-!k3y +ETH_HASH_BACKEND=pysha3 + +DATABASE_URL=psql://postgres:postgres@db:5432/postgres diff --git a/.env.test b/.env.test new file mode 100644 index 0000000..fabd26c --- /dev/null +++ b/.env.test @@ -0,0 +1,9 @@ +PYTHONPATH=/app/ +DEBUG=0 +DJANGO_SETTINGS_MODULE=config.settings.test +DJANGO_SECRET_KEY=t3st-s3cr3t#-!k3y +ETH_HASH_BACKEND=pysha3 + +DATABASE_URL=psql://postgres:postgres@db:5432/postgres +# Only required for testing +ETHEREUM_MAINNET_NODE=https://ethereum.publicnode.com diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..176a458 --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +* text=auto diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 0000000..26d7482 --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,3 @@ +# These owners will be the default owners for everything in +# the repo. Unless a later match takes precedence. +* @safe-global/core-api diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 0000000..5504f1c --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,30 @@ +--- +name: Bug report +about: Create a report to help us improve +title: '' +labels: bug + +--- + +**Describe the bug** +A clear and concise description of what the bug is. + +**To Reproduce** +Steps to reproduce the behavior: +1. Do POST on '...' + - Provide `json` you are submitting to the service (if it applies) +2. Then GET on '....' +3. Links to issues in other repos (if possible) + +**Expected behavior** +A clear and concise description of what you expected to happen. + +**Environment (please complete the following information):** + - Staging or production? + - Which chain? + - OS: [e.g. iOS] + - Browser [e.g. chrome, safari] + - Version [e.g. 22] + +**Additional context** +Add any other context about the problem here. diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 0000000..67a2881 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,34 @@ +--- +name: Feature request +about: Suggest an idea for this project +title: '' +labels: enhancement + +--- + +# What is needed? +A clear and concise description of what you want to happen. + +# Background +More information about the feature needed + +# Related issues +Paste here the related links for the issues on the clients/safe project if applicable. Please provide at least one of the following: +- Links to epics in your repository +- Images taken from mocks +- Gitbook or any form of written documentation links, etc. Any of these alternatives will help us contextualise your request. + +# Endpoint +If applicable, description on the endpoint and the result you expect: + +## URL +`GET /api/v1/safes/
/creation/` + +## Response +``` +{ + created: "", + transactionHash: "", + creator: "" +} +``` diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 0000000..32d5f19 --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,10 @@ +### Make sure these boxes are checked! 📦✅ + +- [ ] You ran `./run_tests.sh` +- [ ] You ran `pre-commit run -a` + +### What was wrong? 👾 + +Closes # + +### How was it fixed? 🎯 diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..2a5f193 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,25 @@ +version: 2 +updates: + - package-ecosystem: pip + directory: "/" + schedule: + interval: weekly + day: monday + reviewers: + - "safe-global/safe-services" + + - package-ecosystem: docker + directory: "/docker/web" + schedule: + interval: weekly + day: monday + reviewers: + - "safe-global/safe-services" + + - package-ecosystem: github-actions + directory: "/" + schedule: + interval: weekly + day: monday + reviewers: + - "safe-global/safe-services" diff --git a/.github/release.yml b/.github/release.yml new file mode 100644 index 0000000..1c0307d --- /dev/null +++ b/.github/release.yml @@ -0,0 +1,15 @@ +changelog: + categories: + - title: 🏕 Features + labels: + - '*' + exclude: + labels: + - dependencies + - breaking_change + - title: 🛠 Breaking Changes + labels: + - breaking_change + - title: 👒 Dependencies + labels: + - dependencies diff --git a/.github/workflows/cla.yml b/.github/workflows/cla.yml new file mode 100644 index 0000000..c12a7e7 --- /dev/null +++ b/.github/workflows/cla.yml @@ -0,0 +1,36 @@ +name: "CLA Assistant" +on: + issue_comment: + types: [ created ] + pull_request_target: + types: [ opened,closed,synchronize ] + +jobs: + CLAssistant: + runs-on: ubuntu-latest + steps: + - name: "CLA Assistant" + if: (github.event.comment.body == 'recheck' || github.event.comment.body == 'I have read the CLA Document and I hereby sign the CLA') || github.event_name == 'pull_request_target' + # Beta Release + uses: cla-assistant/github-action@v2.3.1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + # the below token should have repo scope and must be manually added by you in the repository's secret + PERSONAL_ACCESS_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }} + with: + path-to-signatures: 'signatures/version1/cla.json' + path-to-document: 'https://safe.global/cla/' + # branch should not be protected + branch: 'cla-signatures' + allowlist: hectorgomezv,moisses89,luarx,fmrsabino,luarx,rmeissner,Uxio0,*bot # may need to update this expression if we add new bots + + #below are the optional inputs - If the optional inputs are not given, then default values will be taken + #remote-organization-name: enter the remote organization name where the signatures should be stored (Default is storing the signatures in the same repository) + #remote-repository-name: enter the remote repository name where the signatures should be stored (Default is storing the signatures in the same repository) + #create-file-commit-message: 'For example: Creating file for storing CLA Signatures' + #signed-commit-message: 'For example: $contributorName has signed the CLA in #$pullRequestNo' + #custom-notsigned-prcomment: 'pull request comment with Introductory message to ask new contributors to sign' + #custom-pr-sign-comment: 'The signature to be committed in order to sign the CLA' + #custom-allsigned-prcomment: 'pull request comment when all contributors has signed, defaults to **CLA Assistant Lite bot** All Contributors have signed the CLA.' + #lock-pullrequest-aftermerge: false - if you don't want this bot to automatically lock the pull request after merging (default - true) + #use-dco-flag: true - If you are using DCO instead of CLA diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml new file mode 100644 index 0000000..68e2fc1 --- /dev/null +++ b/.github/workflows/python.yml @@ -0,0 +1,121 @@ +name: Python CI +on: + push: + branches: + - main + - develop + pull_request: + release: + types: [ released ] + +jobs: + linting: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ["3.11"] + + steps: + - uses: actions/checkout@v4 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: pip install pre-commit + - name: Run pre-commit + run: pre-commit run --all-files + + test-app: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ["3.11"] + steps: + - name: Setup and run ganache + run: | + docker run --detach --publish 8545:8545 --network-alias ganache -e DOCKER=true trufflesuite/ganache:latest --defaultBalanceEther 10000 --gasLimit 10000000 -a 30 --chain.chainId 1337 --chain.networkId 1337 -d + - uses: actions/checkout@v4 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + cache: 'pip' + cache-dependency-path: 'requirements*.txt' + - name: Install dependencies + run: | + pip install wheel + pip install -r requirements-test.txt + env: + PIP_USE_MIRRORS: true + - name: Run tests and coverage + run: | + python manage.py check + # python manage.py makemigrations --check --dry-run + coverage run --source=$SOURCE_FOLDER -m pytest -rxXs --reruns 3 + env: + SOURCE_FOLDER: safe_locking_service + DJANGO_SETTINGS_MODULE: config.settings.test + ETHEREUM_MAINNET_NODE: ${{ secrets.ETHEREUM_MAINNET_NODE }} + ETHEREUM_NODES_URLS: http://localhost:8545 + ETH_HASH_BACKEND: pysha3 + - name: Coveralls + uses: coverallsapp/github-action@v2 + docker-deploy: + runs-on: ubuntu-latest + needs: + - linting + - test-app + if: github.ref == 'refs/heads/main' || github.ref == 'refs/heads/develop' || (github.event_name == 'release' && github.event.action == 'released') + steps: + - uses: actions/checkout@v4 + - uses: docker/setup-qemu-action@v3 + with: + platforms: arm64 + - uses: docker/setup-buildx-action@v3 + - name: Dockerhub login + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKER_USER }} + password: ${{ secrets.DOCKER_PASSWORD }} + - name: Deploy main + if: github.ref == 'refs/heads/main' + uses: docker/build-push-action@v5 + with: + context: . + file: docker/web/Dockerfile + push: true + tags: safeglobal/safe-locking-service:staging + platforms: | + linux/amd64 + linux/arm64 + cache-from: type=gha + cache-to: type=gha,mode=max + - name: Deploy Develop + if: github.ref == 'refs/heads/develop' + uses: docker/build-push-action@v5 + with: + context: . + file: docker/web/Dockerfile + push: true + tags: safeglobal/safe-locking-service:develop + platforms: | + linux/amd64 + linux/arm64 + cache-from: type=gha + cache-to: type=gha,mode=max + - name: Deploy Tag + if: (github.event_name == 'release' && github.event.action == 'released') + uses: docker/build-push-action@v5 + with: + context: . + file: docker/web/Dockerfile + push: true + tags: | + safeglobal/safe-locking-service:${{ github.event.release.tag_name }} + safeglobal/safe-locking-service:latest + platforms: | + linux/amd64 + linux/arm64 + cache-from: type=gha + cache-to: type=gha,mode=max diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..3297b71 --- /dev/null +++ b/.gitignore @@ -0,0 +1,127 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# pyenv +.python-version + +# celery beat schedule file +celerybeat-schedule + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ + +# Django +staticfiles/ +safe_locking_service/media + +### VisualStudioCode template +.vscode/* +!.vscode/settings.json +!.vscode/tasks.json +!.vscode/launch.json +!.vscode/extensions.json + +# Provided default Pycharm Run/Debug Configurations should be tracked by git +# In case of local modifications made by Pycharm, use update-index command +# for each changed file, like this: +# git update-index --assume-unchanged .idea/safe_price_service.iml +### JetBrains template +# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and Webstorm +# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 + +# User-specific stuff: +.idea/ +.vscode/ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..4a96652 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,28 @@ +# See https://pre-commit.com for more information +# See https://pre-commit.com/hooks.html for more hooks +repos: + - repo: https://github.com/PyCQA/isort + rev: 5.12.0 + hooks: + - id: isort + - repo: https://github.com/psf/black + rev: 23.11.0 + hooks: + - id: black + - repo: https://github.com/PyCQA/flake8 + rev: 6.1.0 + hooks: + - id: flake8 + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.5.0 + hooks: + - id: check-docstring-first + - id: check-merge-conflict + - id: debug-statements + - id: detect-private-key + - id: requirements-txt-fixer + - id: trailing-whitespace + - id: end-of-file-fixer + types: [python] + - id: check-yaml + - id: check-added-large-files diff --git a/README.md b/README.md new file mode 100644 index 0000000..99f54b4 --- /dev/null +++ b/README.md @@ -0,0 +1,35 @@ +[![Python CI](https://github.com/safe-global/safe-locking-service/actions/workflows/python.yml/badge.svg?branch=main)](https://github.com/safe-global/safe-locking-service/actions/workflows/python.yml) +[![Coverage Status](https://coveralls.io/repos/github/safe-global/safe-locking-service/badge.svg?branch=main)](https://coveralls.io/github/safe-global/safe-locking-service?branch=main) +[![pre-commit](https://img.shields.io/badge/pre--commit-enabled-brightgreen?logo=pre-commit&logoColor=white)](https://github.com/pre-commit/pre-commit) +![Python 3.11](hhttps://img.shields.io/badge/Python-3.11-blue.svg) +![Django 5](https://img.shields.io/badge/Django-5-blue.svg) +[![Docker Image Version (latest semver)](https://img.shields.io/docker/v/safeglobal/safe-locking-service?label=Docker&sort=semver)](https://hub.docker.com/r/safeglobal/safe-locking-service) + +# Safe Locking Service + +Keeps track emitted events by Safe token locking contract. https://github.com/safe-global/safe-locking + +## Configuration +```bash +cp .env.sample .env +``` + +Configure environment variables on `.env`: + +- `DJANGO_SECRET_KEY`: **IMPORTANT: Update it with a secure generated string**. +- `ETHEREUM_NODES_URLS`: RPC node url. + +## Execution + +```bash +docker compose build +docker compose up +``` + +Then go to http://localhost:8000 to see the service documentation. + +## Endpoints +To be defined + +## Contributors +[See contributors](https://github.com/safe-global/safe-locking-service/graphs/contributors) diff --git a/config/__init__.py b/config/__init__.py new file mode 100644 index 0000000..6d300d0 --- /dev/null +++ b/config/__init__.py @@ -0,0 +1,3 @@ +from .celery_app import app as celery_app + +__all__ = ("celery_app",) diff --git a/config/celery_app.py b/config/celery_app.py new file mode 100644 index 0000000..9da2788 --- /dev/null +++ b/config/celery_app.py @@ -0,0 +1,41 @@ +import os + +from celery import Celery +from celery.signals import setup_logging + + +@setup_logging.connect +def on_celery_setup_logging(**kwargs): + """ + Use Django logging instead of celery logger + :param kwargs: + :return: + """ + from logging.config import dictConfig + + from django.conf import settings + + # Patch all the code to use Celery logger (if not just logs inside tasks.py are displayed with the + # task_id and task_name). This way every log will have the context information + if not settings.CELERY_ALWAYS_EAGER: + for _, logger in settings.LOGGING["loggers"].items(): + key = "handlers" + if key in logger: + logger[key] = ["celery_console"] + dictConfig(settings.LOGGING) + + +# set the default Django settings module for the 'celery' program. +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "config.settings.local") + +app = Celery("safe_locking_service") + +# Using a string here means the worker doesn't have to serialize +# the configuration object to child processes. +# - namespace='CELERY' means all celery-related configuration keys +# should have a `CELERY_` prefix. +app.config_from_object("django.conf:settings") +# app.config_from_object("django.conf:settings", namespace="CELERY") + +# Load task modules from all registered Django app configs. +app.autodiscover_tasks() diff --git a/config/gunicorn.py b/config/gunicorn.py new file mode 100644 index 0000000..247ed43 --- /dev/null +++ b/config/gunicorn.py @@ -0,0 +1,8 @@ +""" +Store gunicorn variables in this file, so they can be read by Django +""" +import os + +gunicorn_request_timeout = os.environ.get("WEB_WORKER_TIMEOUT", 60) +gunicorn_worker_connections = os.environ.get("WEB_WORKER_CONNECTIONS", 1000) +gunicorn_workers = os.environ.get("WEB_CONCURRENCY", 2) diff --git a/config/settings/__init__.py b/config/settings/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/config/settings/base.py b/config/settings/base.py new file mode 100644 index 0000000..97eb158 --- /dev/null +++ b/config/settings/base.py @@ -0,0 +1,352 @@ +""" +Base settings to build other settings files upon. +""" + +from pathlib import Path + +import environ +from corsheaders.defaults import default_headers as default_cors_headers + +from ..gunicorn import ( + gunicorn_request_timeout, + gunicorn_worker_connections, + gunicorn_workers, +) + +ROOT_DIR = Path(__file__).resolve(strict=True).parent.parent.parent +APPS_DIR = ROOT_DIR / "safe_locking_service" + +env = environ.Env() + +READ_DOT_ENV_FILE = env.bool("DJANGO_READ_DOT_ENV_FILE", default=False) +DOT_ENV_FILE = env("DJANGO_DOT_ENV_FILE", default=None) +if READ_DOT_ENV_FILE or DOT_ENV_FILE: + DOT_ENV_FILE = DOT_ENV_FILE or ".env" + # OS environment variables take precedence over variables from .env + env.read_env(str(ROOT_DIR / DOT_ENV_FILE)) + +# GENERAL +# ------------------------------------------------------------------------------ +# https://docs.djangoproject.com/en/dev/ref/settings/#debug +DEBUG = env.bool("DEBUG", False) +# Local time zone. Choices are +# http://en.wikipedia.org/wiki/List_of_tz_zones_by_name +# though not all of them may be available with every OS. +# In Windows, this must be set to your system time zone. +TIME_ZONE = "UTC" +# https://docs.djangoproject.com/en/dev/ref/settings/#language-code +LANGUAGE_CODE = "en-us" +# https://docs.djangoproject.com/en/dev/ref/settings/#site-id +SITE_ID = 1 +# https://docs.djangoproject.com/en/dev/ref/settings/#use-i18n +USE_I18N = True +# https://docs.djangoproject.com/en/dev/ref/settings/#use-tz +USE_TZ = True +# https://docs.djangoproject.com/en/3.2/ref/settings/#force-script-name +FORCE_SCRIPT_NAME = env("FORCE_SCRIPT_NAME", default=None) + +# GUNICORN +GUNICORN_REQUEST_TIMEOUT = gunicorn_request_timeout +GUNICORN_WORKER_CONNECTIONS = gunicorn_worker_connections +GUNICORN_WORKERS = gunicorn_workers + +# DATABASES +# ------------------------------------------------------------------------------ +# https://docs.djangoproject.com/en/dev/ref/settings/#databases +DB_STATEMENT_TIMEOUT = env.int("DB_STATEMENT_TIMEOUT", 60_000) +DATABASES = { + "default": env.db("DATABASE_URL"), +} +DATABASES["default"]["ATOMIC_REQUESTS"] = False +DATABASES["default"]["ENGINE"] = "django_db_geventpool.backends.postgresql_psycopg2" +DATABASES["default"]["CONN_MAX_AGE"] = 0 +DB_MAX_CONNS = env.int("DB_MAX_CONNS", default=50) +DATABASES["default"]["OPTIONS"] = { + # https://github.com/jneight/django-db-geventpool#settings + "MAX_CONNS": DB_MAX_CONNS, + "REUSE_CONNS": env.int("DB_REUSE_CONNS", default=DB_MAX_CONNS), + "options": f"-c statement_timeout={DB_STATEMENT_TIMEOUT}", +} + +# URLS +# ------------------------------------------------------------------------------ +# https://docs.djangoproject.com/en/dev/ref/settings/#root-urlconf +ROOT_URLCONF = "config.urls" +# https://docs.djangoproject.com/en/dev/ref/settings/#wsgi-application +WSGI_APPLICATION = "config.wsgi.application" + +# APPS +# ------------------------------------------------------------------------------ +DJANGO_APPS = [ + "django.contrib.admin", + "django.contrib.auth", + "django.contrib.contenttypes", + "django.contrib.sessions", + "django.contrib.sites", + "django.contrib.messages", + "django.contrib.staticfiles", + # 'django.contrib.humanize', # Handy template tags +] +THIRD_PARTY_APPS = [ + "django_extensions", + "corsheaders", + "rest_framework", + "drf_yasg", +] +LOCAL_APPS = [ + "safe_locking_service.locking_events.apps.LockingEventsConfig", +] +# https://docs.djangoproject.com/en/dev/ref/settings/#installed-apps +INSTALLED_APPS = DJANGO_APPS + THIRD_PARTY_APPS + LOCAL_APPS + +# MIDDLEWARE +# ------------------------------------------------------------------------------ +# https://docs.djangoproject.com/en/dev/ref/settings/#middleware +MIDDLEWARE = [ + "safe_locking_service.utils.loggers.LoggingMiddleware", + "corsheaders.middleware.CorsMiddleware", + "django.middleware.security.SecurityMiddleware", + "django.contrib.sessions.middleware.SessionMiddleware", + "django.middleware.http.ConditionalGetMiddleware", + "django.middleware.common.CommonMiddleware", + "django.middleware.csrf.CsrfViewMiddleware", + "django.contrib.auth.middleware.AuthenticationMiddleware", + "django.contrib.messages.middleware.MessageMiddleware", + "django.middleware.clickjacking.XFrameOptionsMiddleware", +] + +# STATIC +# ------------------------------------------------------------------------------ +# https://docs.djangoproject.com/en/dev/ref/settings/#static-root +STATIC_ROOT = str(ROOT_DIR / "staticfiles") + +# https://docs.djangoproject.com/en/dev/ref/settings/#static-url +STATIC_URL = "static/" +# https://docs.djangoproject.com/en/dev/ref/contrib/staticfiles/#std:setting-STATICFILES_DIRS +STATICFILES_DIRS = [ + str(APPS_DIR / "static"), +] +# https://docs.djangoproject.com/en/dev/ref/contrib/staticfiles/#staticfiles-finders +STATICFILES_FINDERS = [ + "django.contrib.staticfiles.finders.FileSystemFinder", + "django.contrib.staticfiles.finders.AppDirectoriesFinder", +] + +# MEDIA +# ------------------------------------------------------------------------------ +# https://docs.djangoproject.com/en/dev/ref/settings/#media-root +MEDIA_ROOT = str(APPS_DIR / "media") +# https://docs.djangoproject.com/en/dev/ref/settings/#media-url +MEDIA_URL = "/media/" + +# TEMPLATES +# ------------------------------------------------------------------------------ +# https://docs.djangoproject.com/en/dev/ref/settings/#templates +TEMPLATES = [ + { + # https://docs.djangoproject.com/en/dev/ref/settings/#std:setting-TEMPLATES-BACKEND + "BACKEND": "django.template.backends.django.DjangoTemplates", + # https://docs.djangoproject.com/en/dev/ref/settings/#template-dirs + "DIRS": [ + str(APPS_DIR / "templates"), + ], + "OPTIONS": { + # https://docs.djangoproject.com/en/dev/ref/settings/#template-debug + "debug": DEBUG, + # https://docs.djangoproject.com/en/dev/ref/settings/#template-loaders + # https://docs.djangoproject.com/en/dev/ref/templates/api/#loader-types + "loaders": [ + "django.template.loaders.filesystem.Loader", + "django.template.loaders.app_directories.Loader", + ], + # https://docs.djangoproject.com/en/dev/ref/settings/#template-context-processors + "context_processors": [ + "django.template.context_processors.debug", + "django.template.context_processors.request", + "django.contrib.auth.context_processors.auth", + "django.template.context_processors.i18n", + "django.template.context_processors.media", + "django.template.context_processors.static", + "django.template.context_processors.tz", + "django.contrib.messages.context_processors.messages", + ], + }, + }, +] + + +# CORS +# ------------------------------------------------------------------------------ +CORS_ALLOW_ALL_ORIGINS = True +CORS_ALLOW_HEADERS = list(default_cors_headers) + [ + "if-match", + "if-modified-since", + "if-none-match", +] +CORS_EXPOSE_HEADERS = ["etag"] + +# FIXTURES +# ------------------------------------------------------------------------------ +# https://docs.djangoproject.com/en/dev/ref/settings/#fixture-dirs +FIXTURE_DIRS = (str(APPS_DIR / "fixtures"),) + +# ADMIN +# ------------------------------------------------------------------------------ +# Django Admin URL regex. +ADMIN_URL = "admin/" + +# Celery +# ------------------------------------------------------------------------------ +INSTALLED_APPS += [ + "django_celery_beat", +] + +# http://docs.celeryproject.org/en/latest/userguide/configuration.html#std:setting-broker_url +CELERY_BROKER_URL = env("CELERY_BROKER_URL", default="django://") +# https://docs.celeryproject.org/en/stable/userguide/optimizing.html#broker-connection-pools +# https://docs.celeryq.dev/en/latest/userguide/optimizing.html#broker-connection-pools +# Configured to 0 due to connection issues https://github.com/celery/celery/issues/4355 +CELERY_BROKER_POOL_LIMIT = env.int("CELERY_BROKER_POOL_LIMIT", default=0) +# https://docs.celeryq.dev/en/stable/userguide/configuration.html#broker-heartbeat +CELERY_BROKER_HEARTBEAT = env.int("CELERY_BROKER_HEARTBEAT", default=0) + +# https://docs.celeryq.dev/en/stable/userguide/configuration.html#std-setting-broker_connection_max_retries +CELERY_BROKER_CONNECTION_MAX_RETRIES = env.int( + "CELERY_BROKER_CONNECTION_MAX_RETRIES", default=0 +) +# https://docs.celeryq.dev/en/stable/userguide/configuration.html#broker-channel-error-retry +CELERY_BROKER_CHANNEL_ERROR_RETRY = env.bool( + "CELERY_BROKER_CHANNEL_ERROR_RETRY", default=True +) +# http://docs.celeryproject.org/en/latest/userguide/configuration.html#std:setting-result_backend +CELERY_RESULT_BACKEND = env("CELERY_RESULT_BACKEND", default="redis://") +# http://docs.celeryproject.org/en/latest/userguide/configuration.html#std:setting-accept_content +CELERY_ACCEPT_CONTENT = ["json"] +# http://docs.celeryproject.org/en/latest/userguide/configuration.html#std:setting-task_serializer +CELERY_TASK_SERIALIZER = "json" +# http://docs.celeryproject.org/en/latest/userguide/configuration.html#std:setting-result_serializer +CELERY_RESULT_SERIALIZER = "json" +# We are not interested in keeping results of tasks +CELERY_IGNORE_RESULT = True +# http://docs.celeryproject.org/en/latest/userguide/configuration.html#std:setting-task_always_eager +CELERY_ALWAYS_EAGER = False +# https://docs.celeryproject.org/en/latest/userguide/configuration.html#task-default-priority +# Higher = more priority on RabbitMQ, opposite on Redis ¯\_(ツ)_/¯ +CELERY_TASK_DEFAULT_PRIORITY = 5 +# https://docs.celeryproject.org/en/stable/userguide/configuration.html#task-queue-max-priority +CELERY_TASK_QUEUE_MAX_PRIORITY = 10 +# https://docs.celeryproject.org/en/latest/userguide/configuration.html#broker-transport-options +CELERY_BROKER_TRANSPORT_OPTIONS = {} + +# https://docs.celeryq.dev/en/stable/userguide/configuration.html#std-setting-task_routes +# CELERY_ROUTES = ( +# [ +# ( +# "safe_locking_service.locking_events.tasks.*", +# {"queue": "events", "delivery_mode": "transient"}, +# ), +# ]) + +# Django REST Framework +# ------------------------------------------------------------------------------ +REST_FRAMEWORK = { + "PAGE_SIZE": 10, + "DEFAULT_PAGINATION_CLASS": "rest_framework.pagination.LimitOffsetPagination", + "DEFAULT_PERMISSION_CLASSES": ("rest_framework.permissions.AllowAny",), + "DEFAULT_RENDERER_CLASSES": ( + "djangorestframework_camel_case.render.CamelCaseJSONRenderer", + ), + "DEFAULT_PARSER_CLASSES": ( + "djangorestframework_camel_case.parser.CamelCaseJSONParser", + ), + "DEFAULT_AUTHENTICATION_CLASSES": ( + # 'rest_framework.authentication.BasicAuthentication', + "rest_framework.authentication.SessionAuthentication", + "rest_framework.authentication.TokenAuthentication", + ), + "DEFAULT_VERSIONING_CLASS": "rest_framework.versioning.NamespaceVersioning", + # TODO Check this + "EXCEPTION_HANDLER": "safe_locking_service.utils.exceptions.custom_exception_handler", +} + +# LOGGING +# ------------------------------------------------------------------------------ +# See: https://docs.djangoproject.com/en/dev/ref/settings/#logging +# A sample logging configuration. The only tangible logging +# performed by this configuration is to send an email to +# the site admins bon every HTTP 500 error when DEBUG=False. +# See https://docs.djangoproject.com/en/dev/topics/logging for +# more details on how to customize your logging configuration. +LOGGING = { + "version": 1, + "disable_existing_loggers": False, + "filters": { + "require_debug_false": {"()": "django.utils.log.RequireDebugFalse"}, + "ignore_succeeded_none": { + "()": "safe_locking_service.utils.loggers.IgnoreSucceededNone" + }, + }, + "formatters": { + "short": {"format": "%(asctime)s %(message)s"}, + "verbose": { + "format": "%(asctime)s [%(levelname)s] [%(processName)s] %(message)s" + }, + }, + "handlers": { + "mail_admins": { + "level": "ERROR", + "filters": ["require_debug_false"], + "class": "django.utils.log.AdminEmailHandler", + }, + "console": { + "level": "DEBUG", + "class": "logging.StreamHandler", + "formatter": "verbose", + }, + "console_short": { + "class": "logging.StreamHandler", + "formatter": "short", + }, + }, + "loggers": { + "": { + "handlers": ["console"], + "level": "INFO", + }, + "web3.providers": { + "level": "DEBUG" if DEBUG else "WARNING", + }, + "django.geventpool": { + "level": "DEBUG" if DEBUG else "WARNING", + }, + "LoggingMiddleware": { + "handlers": ["console_short"], + "level": "INFO", + "propagate": False, + }, + "safe_locking_service": { + "level": "DEBUG" if DEBUG else "INFO", + "handlers": ["console"], + "propagate": False, + }, + "django.request": { + "handlers": ["mail_admins"], + "level": "ERROR", + "propagate": True, + }, + "django.security.DisallowedHost": { + "level": "ERROR", + "handlers": ["console", "mail_admins"], + "propagate": True, + }, + }, +} + +SWAGGER_SETTINGS = { + "SECURITY_DEFINITIONS": { + "api_key": {"type": "apiKey", "in": "header", "name": "Authorization"} + }, +} + +# Ethereum +ETHEREUM_NODE_URL = env("ETHEREUM_NODE_URL", default=None) diff --git a/config/settings/local.py b/config/settings/local.py new file mode 100644 index 0000000..28245f3 --- /dev/null +++ b/config/settings/local.py @@ -0,0 +1,49 @@ +from .base import * # noqa +from .base import env + +# GENERAL +# ------------------------------------------------------------------------------ +# https://docs.djangoproject.com/en/dev/ref/settings/#debug +DEBUG = True +# https://docs.djangoproject.com/en/dev/ref/settings/#secret-key +SECRET_KEY = env( + "DJANGO_SECRET_KEY", + default="aHdCBMHXuxIxEhfRGFRp7Cp3N9CqEZEEAvwZVlBCazKExkEnzvVs4bYWC8Qqh9lg", +) +# https://docs.djangoproject.com/en/dev/ref/settings/#allowed-hosts +ALLOWED_HOSTS = env.list("DJANGO_ALLOWED_HOSTS", default=["*"]) + +REDIS_URL = env.str("REDIS_URL", default=None) + +# CACHES +# ------------------------------------------------------------------------------ +# https://docs.djangoproject.com/en/dev/ref/settings/#caches +CACHES = { + "default": { + "BACKEND": "django.core.cache.backends.locmem.LocMemCache", + } +} + +# django-debug-toolbar +# ------------------------------------------------------------------------------ +# https://django-debug-toolbar.readthedocs.io/en/latest/installation.html#prerequisites +INSTALLED_APPS += ["debug_toolbar"] # noqa F405 + +# https://django-debug-toolbar.readthedocs.io/en/latest/installation.html#middleware +MIDDLEWARE += [ # noqa F405 + "debug_toolbar.middleware.DebugToolbarMiddleware", + "debug_toolbar_force.middleware.ForceDebugToolbarMiddleware", +] +# https://django-debug-toolbar.readthedocs.io/en/latest/configuration.html#debug-toolbar-config +DEBUG_TOOLBAR_CONFIG = { + "DISABLE_PANELS": [ + "debug_toolbar.panels.redirects.RedirectsPanel", + ], + "SHOW_TEMPLATE_CONTEXT": True, +} +# https://django-debug-toolbar.readthedocs.io/en/latest/installation.html#internal-ips +INTERNAL_IPS = ["127.0.0.1", "10.0.2.2"] + +# CELERY +# ------------------------------------------------------------------------------ +CELERY_RESULT_BACKEND = env("CELERY_RESULT_BACKEND", default=REDIS_URL) diff --git a/config/settings/production.py b/config/settings/production.py new file mode 100644 index 0000000..699f42c --- /dev/null +++ b/config/settings/production.py @@ -0,0 +1,32 @@ +from .base import * # noqa +from .base import env + +# GENERAL +# ------------------------------------------------------------------------------ +# https://docs.djangoproject.com/en/dev/ref/settings/#secret-key +SECRET_KEY = env("DJANGO_SECRET_KEY") +# https://docs.djangoproject.com/en/dev/ref/settings/#allowed-hosts +ALLOWED_HOSTS = env.list("DJANGO_ALLOWED_HOSTS", default=["*"]) + +# DATABASES +# ------------------------------------------------------------------------------ +# DATABASES['default'] = env.db('DATABASE_URL') # noqa F405 +DATABASES["default"]["ATOMIC_REQUESTS"] = False # noqa F405 + +# SECURITY +# ------------------------------------------------------------------------------ +# https://docs.djangoproject.com/en/dev/ref/middleware/#x-content-type-options-nosniff +SECURE_CONTENT_TYPE_NOSNIFF = env.bool( + "DJANGO_SECURE_CONTENT_TYPE_NOSNIFF", default=True +) +# https://docs.djangoproject.com/en/3.2/ref/settings/#csrf-trusted-origins +CSRF_TRUSTED_ORIGINS = env.list("CSRF_TRUSTED_ORIGINS", default=[]) + +# ADMIN +# ------------------------------------------------------------------------------ +# Django Admin URL regex. +ADMIN_URL = env("DJANGO_ADMIN_URL", default="admin/") + +# Gunicorn +# ------------------------------------------------------------------------------ +INSTALLED_APPS += ["gunicorn"] # noqa F405 diff --git a/config/settings/test.py b/config/settings/test.py new file mode 100644 index 0000000..51b02b8 --- /dev/null +++ b/config/settings/test.py @@ -0,0 +1,42 @@ +""" +With these settings, tests run faster. +""" + +from .base import * # noqa +from .base import env + +# GENERAL +# ------------------------------------------------------------------------------ +# https://docs.djangoproject.com/en/dev/ref/settings/#debug +DEBUG = True +# https://docs.djangoproject.com/en/dev/ref/settings/#secret-key +SECRET_KEY = env( + "DJANGO_SECRET_KEY", + default="q8lVkJGsIiHcTSQKaWIBsMVPOGnCnF6f7NDGup8KdDNmviSaZVhP0Nq3q3MolmFU", +) +# https://docs.djangoproject.com/en/dev/ref/settings/#test-runner +TEST_RUNNER = "django.test.runner.DiscoverRunner" + +# CACHES +# ------------------------------------------------------------------------------ +# https://docs.djangoproject.com/en/dev/ref/settings/#caches +CACHES = { + "default": { + "BACKEND": "django.core.cache.backends.dummy.DummyCache", + }, +} + +# PASSWORDS +# ------------------------------------------------------------------------------ +# https://docs.djangoproject.com/en/dev/ref/settings/#password-hashers +PASSWORD_HASHERS = ["django.contrib.auth.hashers.MD5PasswordHasher"] + +LOGGING["loggers"] = { # noqa F405 + "safe_locking_service": { + "level": "DEBUG", + } +} + +ETHEREUM_TEST_PRIVATE_KEY = ( + "6370fd033278c143179d81c5526140625662b8daa446c22ee2d73db3707e620c" +) diff --git a/config/urls.py b/config/urls.py new file mode 100644 index 0000000..16d3a19 --- /dev/null +++ b/config/urls.py @@ -0,0 +1,86 @@ +from django.conf import settings +from django.conf.urls import include +from django.contrib import admin +from django.http import HttpResponse +from django.urls import path, re_path +from django.views import defaults as default_views + +from drf_yasg import openapi +from drf_yasg.views import get_schema_view +from rest_framework import permissions + +schema_view = get_schema_view( + openapi.Info( + title="Safe Locking Service API", + default_version="v1", + description="Get events emitted by Safe token locking contract.", + license=openapi.License(name="MIT License"), + ), + validators=["flex", "ssv"], + public=True, + permission_classes=[permissions.AllowAny], +) + +schema_cache_timeout = 60 * 5 # 5 minutes + +swagger_urlpatterns = [ + path( + "", + schema_view.with_ui("swagger", cache_timeout=schema_cache_timeout), + name="schema-swagger-ui", + ), + re_path( + r"^swagger(?P\.json|\.yaml)$", + schema_view.without_ui(cache_timeout=schema_cache_timeout), + name="schema-json", + ), + path( + "redoc/", + schema_view.with_ui("redoc", cache_timeout=schema_cache_timeout), + name="schema-redoc", + ), +] + +urlpatterns_v1 = [ + path( + "", + include("safe_locking_service.locking_events.urls", namespace="locking_events"), + ), +] + +urlpatterns = swagger_urlpatterns + [ + path(settings.ADMIN_URL, admin.site.urls), + path("api/v1/", include((urlpatterns_v1, "v1"))), + path("check/", lambda request: HttpResponse("Ok"), name="check"), +] + + +if settings.DEBUG: + # This allows the error pages to be debugged during development, just visit + # these url in browser to see how these error pages look like. + urlpatterns += [ + path( + "400/", + default_views.bad_request, + kwargs={"exception": Exception("Bad Request!")}, + ), + path( + "403/", + default_views.permission_denied, + kwargs={"exception": Exception("Permission Denied")}, + ), + path( + "404/", + default_views.page_not_found, + kwargs={"exception": Exception("Page not Found")}, + ), + path("500/", default_views.server_error), + ] + if "debug_toolbar" in settings.INSTALLED_APPS: + import debug_toolbar + + urlpatterns = [ + path("__debug__/", include(debug_toolbar.urls)), + ] + urlpatterns + +admin.autodiscover() diff --git a/config/wsgi.py b/config/wsgi.py new file mode 100644 index 0000000..7d5efff --- /dev/null +++ b/config/wsgi.py @@ -0,0 +1,42 @@ +""" +WSGI config for Safe Locking Service project. + +This module contains the WSGI application used by Django's development server +and any production WSGI deployments. It should expose a module-level variable +named ``application``. Django's ``runserver`` and ``runfcgi`` commands discover +this application via the ``WSGI_APPLICATION`` setting. + +Usually you will have the standard Django WSGI application here, but it also +might make sense to replace the whole Django WSGI application with a custom one +that later delegates to the Django one. For example, you could introduce WSGI +middleware here, or combine a Django application with an application of another +framework. + +""" +import os +import sys + +from django.core.wsgi import get_wsgi_application + +# This allows easy placement of apps within the interior +# safe_locking_service directory. +app_path = os.path.abspath( + os.path.join(os.path.dirname(os.path.abspath(__file__)), os.pardir) +) +sys.path.append(os.path.join(app_path, "safe_locking_service")) + + +# We defer to a DJANGO_SETTINGS_MODULE already in the environment. This breaks +# if running multiple sites in the same mod_wsgi process. To fix this, use +# mod_wsgi daemon mode with each site in its own daemon process, or use +# os.environ["DJANGO_SETTINGS_MODULE"] = "config.settings.production" +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "config.settings.production") + +# This application object is used by any WSGI server configured to use this +# file. This includes Django's development server, if the WSGI_APPLICATION +# setting points here. +application = get_wsgi_application() + +# Apply WSGI middleware here. +# from helloworld.wsgi import HelloWorldApplication +# application = HelloWorldApplication(application) diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..de667af --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,60 @@ +version: '3.5' + +volumes: + nginx-shared: + +services: + nginx: + image: nginx:alpine + hostname: nginx + ports: + - "8000:8000" + volumes: + - ./docker/nginx/nginx.conf:/etc/nginx/nginx.conf:ro + - nginx-shared:/nginx + depends_on: + - web + + web: + build: + context: . + dockerfile: docker/web/Dockerfile + env_file: + - .env + working_dir: /app + ports: + - "8888:8888" + volumes: + - nginx-shared:/nginx + command: docker/web/run_web.sh + + rabbitmq: + image: rabbitmq:alpine + ports: + - "5672:5672" + + db: + image: postgres:14-alpine + ports: + - "5432:5432" + environment: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + + indexer-worker: &worker + build: + context: . + dockerfile: docker/web/Dockerfile + env_file: + - .env + environment: + RUN_MIGRATIONS: 1 + WORKER_QUEUES: "default,events" + depends_on: + - db + - rabbitmq + command: docker/web/celery/worker/run.sh + + scheduler: + <<: *worker + command: docker/web/celery/scheduler/run.sh diff --git a/docker/nginx/nginx.conf b/docker/nginx/nginx.conf new file mode 100644 index 0000000..6b93ea7 --- /dev/null +++ b/docker/nginx/nginx.conf @@ -0,0 +1,68 @@ +# https://github.com/KyleAMathews/docker-nginx/blob/master/nginx.conf +# https://linode.com/docs/web-servers/nginx/configure-nginx-for-optimized-performance/ +# https://docs.gunicorn.org/en/stable/deploy.html + +worker_processes 1; + +events { + worker_connections 2000; # increase if you have lots of clients + accept_mutex off; # set to 'on' if nginx worker_processes > 1 + use epoll; # Enable epoll for Linux 2.6+ + # 'use kqueue;' to enable for FreeBSD, OSX +} + +http { + include mime.types; + # fallback in case we can't determine a type + default_type application/octet-stream; + sendfile on; + + upstream app_server { + # ip_hash; # For load-balancing + # + # fail_timeout=0 means we always retry an upstream even if it failed + # to return a good HTTP response + server unix:/nginx/gunicorn.socket fail_timeout=0; + + # for a TCP configuration + # server web:8000 fail_timeout=0; + keepalive 32; + } + + server { + access_log off; + listen 8000 deferred; + charset utf-8; + keepalive_timeout 75s; + + # https://thoughts.t37.net/nginx-optimization-understanding-sendfile-tcp-nodelay-and-tcp-nopush-c55cdd276765 + # tcp_nopush on; + # tcp_nodelay on; + + gzip on; + gzip_min_length 1000; + gzip_comp_level 2; + # text/html is always included by default + gzip_types text/plain text/css application/json application/javascript application/x-javascript text/javascript text/xml application/xml application/rss+xml application/atom+xml application/rdf+xml; + gzip_disable "MSIE [1-6]\."; + + location /static { + alias /nginx/staticfiles; + expires 365d; + } + + location / { + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Host $host; + # we don't want nginx trying to do something clever with + # redirects, we set the Host: header above already. + proxy_redirect off; + proxy_pass http://app_server/; + + proxy_set_header X-Forwarded-Host $server_name; + proxy_set_header X-Real-IP $remote_addr; + add_header Front-End-Https on; + } + } +} diff --git a/docker/web/Dockerfile b/docker/web/Dockerfile new file mode 100644 index 0000000..bb10c49 --- /dev/null +++ b/docker/web/Dockerfile @@ -0,0 +1,38 @@ +FROM python:3.11-slim + +ARG APP_HOME=/app +WORKDIR ${APP_HOME} +ENV PYTHONUNBUFFERED=1 + +# https://eth-hash.readthedocs.io/en/latest/quickstart.html#specify-backend-by-environment-variable +# `pysha3` is way faster than `pycryptodome` for CPython +ENV ETH_HASH_BACKEND=pysha3 + +COPY requirements.txt ./ +RUN set -ex \ + && buildDeps=" \ + build-essential \ + git \ + libssl-dev \ + libpq-dev \ + " \ + && apt-get update \ + && apt-get upgrade -y \ + && apt-get install -y --no-install-recommends $buildDeps tmux postgresql-client \ + && pip install -U --no-cache-dir wheel setuptools pip \ + && pip install --no-cache-dir -r requirements.txt \ + && apt-get purge -y --auto-remove $buildDeps \ + && rm -rf /var/lib/apt/lists/* + +# /nginx mount point must be created before so it doesn't have root permissions +# ${APP_HOME} root folder will not be updated by COPY --chown, so permissions need to be adjusted +RUN groupadd -g 999 python && \ + useradd -u 999 -r -g python python && \ + mkdir -p /nginx && \ + chown -R python:python /nginx ${APP_HOME} +COPY --chown=python:python . . + +# Use numeric ids so kubernetes identifies the user correctly +USER 999:999 + +RUN DJANGO_SETTINGS_MODULE=config.settings.production DJANGO_DOT_ENV_FILE=.env.sample python manage.py collectstatic --noinput diff --git a/docker/web/celery/scheduler/run.sh b/docker/web/celery/scheduler/run.sh new file mode 100755 index 0000000..cbb5a47 --- /dev/null +++ b/docker/web/celery/scheduler/run.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +set -euo pipefail + +# DEBUG set in .env +if [ ${DEBUG:-0} = 1 ]; then + log_level="debug" +else + log_level="info" +fi + +# Wait for migrations +sleep 10 + +echo "==> $(date +%H:%M:%S) ==> Running Celery beat <==" +exec celery -C -A config.celery_app beat \ + -S django_celery_beat.schedulers:DatabaseScheduler \ + --loglevel $log_level \ No newline at end of file diff --git a/docker/web/celery/worker/run.sh b/docker/web/celery/worker/run.sh new file mode 100755 index 0000000..3011fb4 --- /dev/null +++ b/docker/web/celery/worker/run.sh @@ -0,0 +1,37 @@ +#!/bin/bash + +set -euo pipefail + +MAX_MEMORY_PER_CHILD="${WORKER_MAX_MEMORY_PER_CHILD:-2097152}" +MAX_TASKS_PER_CHILD="${MAX_TASKS_PER_CHILD:-1000000}" +TASK_CONCURRENCY="${CELERYD_CONCURRENCY:-1000}" + +# DEBUG set in .env_docker_compose +if [ ${DEBUG:-0} = 1 ]; then + log_level="debug" +else + log_level="info" +fi + +if [ ${RUN_MIGRATIONS:-0} = 1 ]; then + echo "==> $(date +%H:%M:%S) ==> Migrating Django models... " + python manage.py migrate --noinput + + echo "==> $(date +%H:%M:%S) ==> Setting up service... " + python manage.py setup_service + + echo "==> $(date +%H:%M:%S) ==> Setting contracts... " + python manage.py update_safe_contracts_logo +fi + +echo "==> $(date +%H:%M:%S) ==> Check RPC connected matches previously used RPC... " +python manage.py check_chainid_matches + +echo "==> $(date +%H:%M:%S) ==> Running Celery worker with a max_memory_per_child of ${MAX_MEMORY_PER_CHILD} <==" +exec celery -C -A config.celery_app worker \ + --loglevel $log_level --pool=gevent \ + --concurrency=${TASK_CONCURRENCY} \ + --max-memory-per-child=${MAX_MEMORY_PER_CHILD} \ + --max-tasks-per-child=${MAX_TASKS_PER_CHILD} \ + --without-heartbeat --without-gossip \ + --without-mingle -Q "$WORKER_QUEUES" \ No newline at end of file diff --git a/docker/web/run_web.sh b/docker/web/run_web.sh new file mode 100755 index 0000000..9224f94 --- /dev/null +++ b/docker/web/run_web.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +set -euo pipefail + +echo "==> $(date +%H:%M:%S) ==> Collecting statics... " +DOCKER_SHARED_DIR=/nginx +rm -rf $DOCKER_SHARED_DIR/* +# STATIC_ROOT=$DOCKER_SHARED_DIR/staticfiles python manage.py collectstatic --noinput & +cp -r staticfiles/ $DOCKER_SHARED_DIR/ + +echo "==> $(date +%H:%M:%S) ==> Running Gunicorn... " +exec gunicorn --config gunicorn.conf.py --pythonpath "$PWD" -b unix:$DOCKER_SHARED_DIR/gunicorn.socket -b 0.0.0.0:8888 config.wsgi:application diff --git a/gunicorn.conf.py b/gunicorn.conf.py new file mode 100644 index 0000000..28d6c3c --- /dev/null +++ b/gunicorn.conf.py @@ -0,0 +1,24 @@ +from config.gunicorn import ( + gunicorn_request_timeout, + gunicorn_worker_connections, + gunicorn_workers, +) + +access_logfile = "-" +error_logfile = "-" +max_requests = 20_000 # Restart a worker after it has processed a given number of requests (for memory leaks) +max_requests_jitter = ( + 10_000 # Randomize max_requests to prevent all workers restarting at the same time +) +# graceful_timeout = 90 # https://stackoverflow.com/a/24305939 +keep_alive = 2 +log_file = "-" +log_level = "info" +logger_class = "safe_locking_service.utils.loggers.CustomGunicornLogger" +preload_app = False # Load application code before the worker processes are forked (problems with gevent patching) +# For timeout to work with gevent, a custom GeventWorker needs to be used +timeout = gunicorn_request_timeout + +worker_class = "gunicorn_custom_workers.MyGeventWorker" # "gevent" +worker_connections = gunicorn_worker_connections +workers = gunicorn_workers diff --git a/gunicorn_custom_workers.py b/gunicorn_custom_workers.py new file mode 100644 index 0000000..17844c0 --- /dev/null +++ b/gunicorn_custom_workers.py @@ -0,0 +1,27 @@ +import gevent +from gunicorn.workers.ggevent import GeventWorker + + +class MyGeventWorker(GeventWorker): + def patch_psycopg2(self) -> bool: + try: + from psycogreen.gevent import patch_psycopg + + patch_psycopg() + self.log.info("Patched Psycopg2 for gevent") + return True + except ImportError: + self.log.info("Cannot patch psycopg2 for gevent, install psycogreen") + return False + + def patch(self): + super().patch() + self.log.info("Patched all for gevent") + self.patch_psycopg2() + + def handle_request(self, listener_name, req, sock, addr): + try: + with gevent.Timeout(self.cfg.timeout): + super().handle_request(listener_name, req, sock, addr) + except gevent.Timeout: + self.log.error("TimeoutError on %s", req.path) diff --git a/manage.py b/manage.py new file mode 100644 index 0000000..4702e46 --- /dev/null +++ b/manage.py @@ -0,0 +1,30 @@ +#!/usr/bin/env python +import os +import sys +from pathlib import Path + +if __name__ == "__main__": + os.environ.setdefault("DJANGO_SETTINGS_MODULE", "config.settings.local") + + try: + from django.core.management import execute_from_command_line + except ImportError: + # The above import may fail for some other reason. Ensure that the + # issue is really that Django is missing to avoid masking other + # exceptions on Python 2. + try: + import django # noqa + except ImportError: + raise ImportError( + "Couldn't import Django. Are you sure it's installed and " + "available on your PYTHONPATH environment variable? Did you " + "forget to activate a virtual environment?" + ) + raise + + # This allows easy placement of apps within the interior + # safe_locking_service directory. + current_path = Path(__file__).parent.resolve() + sys.path.append(str(current_path / "safe_locking_service")) + + execute_from_command_line(sys.argv) diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 0000000..a2d8fcd --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,9 @@ +-r requirements.txt +-r requirements-test.txt +flake8 +ipdb +ipython +isort +pre-commit +pylint +pylint-django diff --git a/requirements-test.txt b/requirements-test.txt new file mode 100644 index 0000000..a410fd1 --- /dev/null +++ b/requirements-test.txt @@ -0,0 +1,12 @@ +-r requirements.txt +coverage==7.4.1 +django-stubs==4.2.7 +django-test-migrations==1.3.0 +factory-boy==3.3.0 +faker==23.2.1 +mypy==1.8.0 +pytest==8.0.1 +pytest-django==4.8.0 +pytest-env==1.1.3 +pytest-rerunfailures==13.0 +pytest-sugar==1.0.0 diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..a8c6274 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,22 @@ +cachetools==5.3.2 +celery==5.3.6 +django==4.2.10 +django-cache-memoize==0.2.0 +django-celery-beat==2.5.0 +django-cors-headers==4.3.1 +django-db-geventpool==4.0.2 +django-debug-toolbar +django-debug-toolbar-force +django-environ==0.11.2 +django-extensions==3.2.3 +djangorestframework==3.14.0 +djangorestframework-camel-case==1.4.2 +docutils==0.20.1 +drf-yasg[validation]==1.21.7 +gunicorn[gevent]==21.2.0 +hexbytes==0.3.1 +packaging>=21 +psycopg2==2.9.9 +requests==2.31.0 +safe-eth-py[django]==6.0.0b16 +web3==6.15.1 diff --git a/run_tests.sh b/run_tests.sh new file mode 100755 index 0000000..f2cda5e --- /dev/null +++ b/run_tests.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +set -euo pipefail + +export DJANGO_SETTINGS_MODULE=config.settings.test +export DJANGO_DOT_ENV_FILE=.env.test +docker compose -f docker-compose.yml -f docker-compose.dev.yml build --force-rm db redis ganache rabbitmq +docker compose -f docker-compose.yml -f docker-compose.dev.yml up --no-start db redis ganache rabbitmq +docker compose -f docker-compose.yml -f docker-compose.dev.yml start db redis ganache rabbitmq + +sleep 10 + +python manage.py check +pytest -rxXs diff --git a/safe_locking_service/__init__.py b/safe_locking_service/__init__.py new file mode 100644 index 0000000..3181f24 --- /dev/null +++ b/safe_locking_service/__init__.py @@ -0,0 +1,5 @@ +__version__ = "0.0.1" +__version_info__ = tuple( + int(num) if num.isdigit() else num + for num in __version__.replace("-", ".", 1).split(".") +) diff --git a/safe_locking_service/locking_events/__init__.py b/safe_locking_service/locking_events/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/safe_locking_service/locking_events/admin.py b/safe_locking_service/locking_events/admin.py new file mode 100644 index 0000000..4185d36 --- /dev/null +++ b/safe_locking_service/locking_events/admin.py @@ -0,0 +1,3 @@ +# from django.contrib import admin + +# Register your models here. diff --git a/safe_locking_service/locking_events/apps.py b/safe_locking_service/locking_events/apps.py new file mode 100644 index 0000000..f3c839b --- /dev/null +++ b/safe_locking_service/locking_events/apps.py @@ -0,0 +1,7 @@ +from django.apps import AppConfig + + +class LockingEventsConfig(AppConfig): + name = "safe_locking_service.locking_events" + verbose_name = "Safe Locking Service" + default_auto_field = "django.db.models.BigAutoField" diff --git a/safe_locking_service/locking_events/management/__init__.py b/safe_locking_service/locking_events/management/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/safe_locking_service/locking_events/management/commands/__init__.py b/safe_locking_service/locking_events/management/commands/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/safe_locking_service/locking_events/migrations/__init__.py b/safe_locking_service/locking_events/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/safe_locking_service/locking_events/models.py b/safe_locking_service/locking_events/models.py new file mode 100644 index 0000000..0b4331b --- /dev/null +++ b/safe_locking_service/locking_events/models.py @@ -0,0 +1,3 @@ +# from django.db import models + +# Create your models here. diff --git a/safe_locking_service/locking_events/tasks.py b/safe_locking_service/locking_events/tasks.py new file mode 100644 index 0000000..3017f69 --- /dev/null +++ b/safe_locking_service/locking_events/tasks.py @@ -0,0 +1 @@ +# Expected to add indexing tasks diff --git a/safe_locking_service/locking_events/tests.py b/safe_locking_service/locking_events/tests.py new file mode 100644 index 0000000..a79ca8b --- /dev/null +++ b/safe_locking_service/locking_events/tests.py @@ -0,0 +1,3 @@ +# from django.test import TestCase + +# Create your tests here. diff --git a/safe_locking_service/locking_events/urls.py b/safe_locking_service/locking_events/urls.py new file mode 100644 index 0000000..c54b99b --- /dev/null +++ b/safe_locking_service/locking_events/urls.py @@ -0,0 +1,9 @@ +from django.urls import path + +from . import views + +app_name = "tokens" + +urlpatterns = [ + path("about/", views.AboutView.as_view(), name="about"), +] diff --git a/safe_locking_service/locking_events/views.py b/safe_locking_service/locking_events/views.py new file mode 100644 index 0000000..6d664b9 --- /dev/null +++ b/safe_locking_service/locking_events/views.py @@ -0,0 +1,25 @@ +from django.utils.decorators import method_decorator +from django.views.decorators.cache import cache_page + +from rest_framework.generics import GenericAPIView +from rest_framework.response import Response + +from safe_locking_service import __version__ + + +class AboutView(GenericAPIView): + """ + Returns information and configuration of the service + """ + + @method_decorator(cache_page(5 * 60)) # 5 minutes + def get(self, request, format=None): + content = { + "name": "Safe Locking Service", + "version": __version__, + "api_version": request.version, + "secure": request.is_secure(), + "host": request.get_host(), + "headers": [x for x in request.META.keys() if "FORWARD" in x], + } + return Response(content) diff --git a/safe_locking_service/static/.gitignore b/safe_locking_service/static/.gitignore new file mode 100644 index 0000000..e69de29 diff --git a/safe_locking_service/static/safe/favicon.png b/safe_locking_service/static/safe/favicon.png new file mode 100644 index 0000000..2ba2897 Binary files /dev/null and b/safe_locking_service/static/safe/favicon.png differ diff --git a/safe_locking_service/static/safe/logo.png b/safe_locking_service/static/safe/logo.png new file mode 100755 index 0000000..5cbcc55 Binary files /dev/null and b/safe_locking_service/static/safe/logo.png differ diff --git a/safe_locking_service/static/safe/logo.svg b/safe_locking_service/static/safe/logo.svg new file mode 100644 index 0000000..df57691 --- /dev/null +++ b/safe_locking_service/static/safe/logo.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/safe_locking_service/static/safe/safe_contract_logo.png b/safe_locking_service/static/safe/safe_contract_logo.png new file mode 100644 index 0000000..14eb5bb Binary files /dev/null and b/safe_locking_service/static/safe/safe_contract_logo.png differ diff --git a/safe_locking_service/templates/drf-yasg/swagger-ui.html b/safe_locking_service/templates/drf-yasg/swagger-ui.html new file mode 100644 index 0000000..1a4f792 --- /dev/null +++ b/safe_locking_service/templates/drf-yasg/swagger-ui.html @@ -0,0 +1,25 @@ +{% extends "drf-yasg/swagger-ui.html" %} +{% load static %} + +{% block favicon %} + +{% endblock %} + +{% block main_styles %} + + + +{% endblock %} diff --git a/safe_locking_service/utils/__init__.py b/safe_locking_service/utils/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/safe_locking_service/utils/exceptions.py b/safe_locking_service/utils/exceptions.py new file mode 100644 index 0000000..06b8875 --- /dev/null +++ b/safe_locking_service/utils/exceptions.py @@ -0,0 +1,39 @@ +import logging + +from rest_framework import status +from rest_framework.response import Response +from rest_framework.views import exception_handler + +logger = logging.getLogger(__name__) + + +def custom_exception_handler(exc, context): + if isinstance(exc, NodeConnectionException): + response = Response(status=status.HTTP_503_SERVICE_UNAVAILABLE) + + if str(exc): + exception_str = "{}: {}".format(exc.__class__.__name__, exc) + else: + exception_str = exc.__class__.__name__ + response.data = { + "exception": "Problem connecting to Ethereum network", + "trace": exception_str, + } + + logger.warning( + "%s - Exception: %s - Data received %s", + context["request"].build_absolute_uri(), + exception_str, + context["request"].data, + exc_info=exc, + ) + else: + # Call REST framework's default exception handler, + # to get the standard error response. + response = exception_handler(exc, context) + + return response + + +class NodeConnectionException(IOError): + pass diff --git a/safe_locking_service/utils/loggers.py b/safe_locking_service/utils/loggers.py new file mode 100644 index 0000000..11b9e11 --- /dev/null +++ b/safe_locking_service/utils/loggers.py @@ -0,0 +1,60 @@ +import logging +import time + +from django.http import HttpRequest + +from gunicorn import glogging + + +def get_milliseconds_now(): + return int(time.time() * 1000) + + +class IgnoreCheckUrl(logging.Filter): + def filter(self, record: logging.LogRecord) -> bool: + message = record.getMessage() + return not ("GET /check/" in message and "200" in message) + + +class IgnoreSucceededNone(logging.Filter): + """ + Ignore Celery messages + They are usually emitted when a redis lock is active + """ + + def filter(self, rec: logging.LogRecord): + message = rec.getMessage() + return not ("Task" in message and "succeeded" in message and "None" in message) + + +class CustomGunicornLogger(glogging.Logger): + def setup(self, cfg): + super().setup(cfg) + + # Add filters to Gunicorn logger + logger = logging.getLogger("gunicorn.access") + logger.addFilter(IgnoreCheckUrl()) + + +class LoggingMiddleware: + def __init__(self, get_response): + self.get_response = get_response + self.logger = logging.getLogger("LoggingMiddleware") + + def __call__(self, request: HttpRequest): + milliseconds = get_milliseconds_now() + response = self.get_response(request) + if request.resolver_match: + route = ( + request.resolver_match.route if request.resolver_match else request.path + ) + delta = get_milliseconds_now() - milliseconds + self.logger.info( + "MT::%s::%s::%s::%d::%s", + request.method, + route, + delta, + response.status_code, + request.path, + ) + return response diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..2292422 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,60 @@ +[flake8] +max-line-length = 88 +select = C,E,F,W,B,B950 +extend-ignore = E203,E501,F841,W503 +exclude = .tox,.git,*/static/CACHE/*,docs,node_modules,venv + +[pycodestyle] +max-line-length = 120 +exclude = .tox,.git,*/static/CACHE/*,docs,node_modules,venv + +[isort] +profile = black +default_section = THIRDPARTY +known_first_party = safe_locking_service +known_gnosis = py_eth_sig_utils,gnosis +known_django = django +sections = FUTURE,STDLIB,DJANGO,THIRDPARTY,GNOSIS,FIRSTPARTY,LOCALFOLDER + +[tool:pytest] +env = + DJANGO_SETTINGS_MODULE=config.settings.test + DJANGO_DOT_ENV_FILE=.env.test + +[mypy] +python_version = 3.11 +check_untyped_defs = True +ignore_missing_imports = True +warn_unused_ignores = True +warn_redundant_casts = True +warn_unused_configs = True +plugins = mypy_django_plugin.main + +[mypy.plugins.django-stubs] +django_settings_module = config.settings.test + +[mypy-*.migrations.*] +# Django migrations should not produce any errors: +ignore_errors = True + +[coverage:report] +exclude_lines = +# Have to re-enable the standard pragma + pragma: no cover + + # Don't complain if tests don't hit defensive assertion code: + raise NotImplementedError + + # Don't complain if non-runnable code isn't run: + if __name__ == .__main__.: + if settings.DEBUG + + # Ignore pass lines + pass + +[coverage:run] +include = safe_locking_service/* +omit = + *__init__.py* + *tests* + */migrations/*