V3

.circleci/config.yml

@@ -0,0 +1,199 @@
+version: 2.1
+
+orbs:
+  python: circleci/python@0.2.1
+
+jobs:
+  backend-build:
+    machine:
+      image: ubuntu-1604:201903-01
+    steps:
+      - checkout
+      # restore cache and docker layer cache
+      - restore_cache:
+          keys:
+            - jarrback-{{ .Environment.CACHE_VERSION }}.{{ .Branch }}
+          paths:
+            - ~/caches/jarr-server.tar
+            - ~/caches/jarr-worker.tar
+      - run:
+          name: Loading docker image layer caches
+          command: |
+            set +o pipefail
+            docker load -i ~/caches/jarr-server.tar | true
+            docker load -i ~/caches/jarr-worker.tar | true
+      - run:
+          name: Building images for server and worker
+          command: |
+            mkdir -p ~/caches
+            make build-server
+            make build-worker
+      - run:
+          name: Saving docker image layer caches
+          command: |
+            docker save -o ~/caches/jarr-server.tar jarr-server
+            docker save -o ~/caches/jarr-worker.tar jarr-worker
+      - save_cache:
+          key: jarrback-{{ .Environment.CACHE_VERSION }}.{{ .Branch }}-{{ epoch }}
+          paths:
+            - ~/caches/jarr-server.tar
+            - ~/caches/jarr-worker.tar
+
+  front-build:
+    machine:
+      image: ubuntu-1604:201903-01
+    steps:
+      - checkout
+      # restore cache and docker layer cache
+      - restore_cache:
+          keys:
+            - jarrfront-{{ .Environment.CACHE_VERSION }}.{{ .Branch }}
+          paths:
+            - ~/caches/jarr-front.tar
+      - run:
+          name: Loading docker image layer caches
+          command: |
+            set +o pipefail
+            docker load -i ~/caches/jarr-front.tar | true
+      - run:
+          name: Building images for front
+          command: |
+            mkdir -p ~/caches
+            make build-front PUBLIC_URL=https://app.jarr.info REACT_APP_API_URL=https://api.jarr.info
+      - run:
+          name: Saving docker image layer caches
+          command: |
+            docker save -o ~/caches/jarr-front.tar jarr-front
+      - save_cache:
+          key: jarrfront-{{ .Environment.CACHE_VERSION }}.{{ .Branch }}-{{ epoch }}
+          paths:
+            - ~/caches/jarr-front.tar
+
+  backend-test:
+    machine:
+      image: ubuntu-1604:201903-01
+    steps:
+      - checkout
+      # restore cache and docker layer cache
+      - restore_cache:
+          keys:
+            - jarrback-{{ .Environment.CACHE_VERSION }}.{{ .Branch }}
+          paths:
+            - ~/caches/jarr-server.tar
+            - ~/caches/jarr-worker.tar
+      - run:
+          name: Loading docker image layer caches
+          command: |
+            docker load -i ~/caches/jarr-server.tar
+            docker load -i ~/caches/jarr-worker.tar
+      - run:
+          name: Moving test config to default config
+          command: |
+            mkdir -p ~/coverage_results
+            rm example_conf/jarr.test.json example_conf/jarr.json
+            mv example_conf/jarr.circleci.json example_conf/jarr.json
+      - run:
+          name: Starting test env (postgres)
+          command: |
+            docker-compose --file Dockerfiles/circleci-env.yml up -d
+            sleep 10s
+      - run:
+          name: Creating postgres database
+          command: >-
+            docker-compose --file Dockerfiles/circleci-env.yml
+            exec postgresql su postgres -c
+            "createdb jarr --no-password"
+      - run:
+          name: Populating database with tables
+          command: >-
+            docker-compose --file Dockerfiles/circleci-env.yml
+            exec server bash -c "pipenv run ./manager.py db_create"
+      - run:
+          name: Installing testing dependencies
+          command: >-
+            docker-compose --file Dockerfiles/circleci-env.yml
+            exec server bash -c "pipenv sync --dev --bare"
+      - run:
+          name: Running tests
+          command: >-
+            docker-compose --file Dockerfiles/circleci-env.yml
+            exec server bash -c
+            "pipenv run nosetests tests/ -vv --with-coverage --cover-package=jarr"
+      - run:
+          name: Uplaoding coverage to codacy
+          command: >-
+            docker-compose --file Dockerfiles/circleci-env.yml
+            exec -u root server bash -c "chown -R jarr:jarr /jarr/coverage_results";
+            docker-compose --file Dockerfiles/circleci-env.yml
+            exec server bash -c
+            "pipenv run coverage xml -o coverage_results/coverage.xml";
+            pip install --upgrade codacy-coverage;
+            python-codacy-coverage -r ~/coverage_results/coverage.xml
+      - run:
+          name: Running PEP8 linter
+          command: >-
+            docker-compose --file Dockerfiles/circleci-env.yml
+            exec server bash -c
+            "pipenv run pycodestyle --ignore=E126,E127,E128,W503 jarr/ --exclude=jarr/migrations"
+      - run:
+          name: Running MyPy linter
+          command: >-
+            docker-compose --file Dockerfiles/circleci-env.yml
+            exec server bash -c
+            "pipenv run mypy jarr --ignore-missing-imports"
+
+  push-docker-images:
+    machine:
+      image: ubuntu-1604:201903-01
+    steps:
+      - restore_cache:
+          keys:
+            - jarrback-{{ .Environment.CACHE_VERSION }}.{{ .Branch }}
+          paths:
+            - ~/caches/jarr-server.tar
+            - ~/caches/jarr-worker.tar
+      - restore_cache:
+          keys:
+            - jarrfront-{{ .Environment.CACHE_VERSION }}.{{ .Branch }}
+          paths:
+            - ~/caches/jarr-front.tar
+      - run:
+          name: Loading ALL docker image layer caches
+          command: |
+            set +o pipefail
+            docker load -i ~/caches/jarr-server.tar
+            docker load -i ~/caches/jarr-worker.tar
+            docker load -i ~/caches/jarr-front.tar
+      - run:
+          name: Pushing tested images to registry
+          command: |
+            echo "$DOCKER_PASS" | docker login --username $DOCKER_USER --password-stdin
+            if [ "$CIRCLE_BRANCH" = "master" ]; then
+              export DOCKER_TAG=latest
+            else
+              export DOCKER_TAG=$CIRCLE_BRANCH
+            fi
+            docker image tag jarr-server $DOCKER_USER/jarr-server:$DOCKER_TAG
+            docker image tag jarr-worker $DOCKER_USER/jarr-worker:$DOCKER_TAG
+            docker image tag jarr-front $DOCKER_USER/jarr-front:$DOCKER_TAG
+            docker push $DOCKER_USER/jarr-server:$DOCKER_TAG
+            docker push $DOCKER_USER/jarr-worker:$DOCKER_TAG
+            docker push $DOCKER_USER/jarr-front:$DOCKER_TAG
+
+workflows:
+  jarr-testing:
+    jobs:
+      - backend-build
+      - front-build
+      - backend-test:
+          requires:
+            - backend-build
+      - push-docker-images:
+          requires:
+            - backend-test
+            - front-build
+          filters:
+            branches:
+              only:
+                - master
+                - develop  # TODO: remove when merging to master

.gitignore

@@ -22,9 +22,10 @@ eproject.cfg
 venv
 build
 
-src/conf.py
 .coverage
 
 # node files
 node_modules
-src/web/static/js/bundle.min.js
+
+# docker compose
+.jarr-data/

.gitmodules

@@ -1,3 +0,0 @@
-[submodule "submodules/bootstrap"]
-	path = submodules/bootstrap
-	url = https://github.com/twbs/bootstrap.git

Dockerfiles/circleci-env.yml

@@ -0,0 +1,18 @@
+version: '3.2'
+services:
+  postgresql:
+    image: postgres:11
+    ports:
+      - 0.0.0.0:5432:5432
+    environment:
+      - POSTGRES_PASSWORD=password
+  redis:
+    image: redis
+    ports:
+      - 0.0.0.0:6379:6379
+  server:
+    image: jarr-server:latest
+    volumes:
+      - "/home/circleci/project/tests:/jarr/tests"
+      - "/home/circleci/project/example_conf:/etc/jarr"
+      - "/home/circleci/coverage_results:/jarr/coverage_results:rw"

Dockerfiles/dev-env.yml

@@ -0,0 +1,29 @@
+version: '3.2'
+services:
+  redis:
+    image: redis
+    ports:
+      - 0.0.0.0:6379:6379
+  rabbitmq:
+    image: rabbitmq
+    ports:
+      - 0.0.0.0:25672:25672
+      - 0.0.0.0:4369:4369
+      - 0.0.0.0:5671:5671
+      - 0.0.0.0:5672:5672
+  postgresql:
+    image: postgres:11
+    environment:
+      - POSTGRES_PASSWORD=password
+      - POSTGRES_HOST_AUTH_METHOD=trust
+    ports:
+      - 0.0.0.0:5432:5432
+    volumes:
+      - ../.jarr-data:/var/lib/postgresql/data
+    command:
+      - "postgres"
+      - "-c"
+      - "log_min_duration_statement=100"
+
+volumes:
+  db-data:

Dockerfiles/front

@@ -0,0 +1,13 @@
+FROM node:12.16.1 as build-deps
+ARG PUBLIC_URL=
+ARG REACT_APP_API_URL=0.0.0.0:8000
+WORKDIR /jarr
+RUN yarn global add serve
+COPY jsclient/package.json jsclient/yarn.lock ./
+COPY jsclient/src/ /jarr/src
+COPY jsclient/public/ /jarr/public
+RUN yarn \
+    && yarn build \
+    && rm -rf src public node_modules package.json yarn.lock
+EXPOSE 80
+CMD ["serve", "-p", "80", "-s", "/jarr/build"]

Dockerfiles/migration

@@ -0,0 +1,6 @@
+FROM jarr_python-base:latest
+
+COPY example_conf/logging.ini /etc/jarr/logging.ini
+COPY example_conf/gunicorn.py /etc/jarr/gunicorn.py
+EXPOSE 8000
+CMD ["pipenv", "run", "/jarr/manager.py", "db", "upgrade"]

Dockerfiles/pythonbase

@@ -0,0 +1,24 @@
+FROM python:3.7-slim
+
+RUN apt-get update \
+    && apt-get install --yes --no-install-recommends \
+        gcc \
+        python3-dev \
+        libxml2-dev \
+        postgresql-server-dev-all \
+    && pip install --no-cache-dir --upgrade pip pipenv \
+    && apt clean --yes \
+    && apt autoremove --yes \
+    && rm -rf /var/log/apt /var/cache/apt /var/log/apt
+RUN useradd jarr --create-home --shell /bin/bash --home-dir /jarr --user-group
+COPY example_conf/jarr.json /etc/jarr/jarr.json
+WORKDIR /jarr
+USER jarr
+COPY ./Pipfil* /jarr/
+RUN pipenv sync
+COPY ./manager.py /jarr/manager.py
+COPY ./migrations/ /jarr/migrations
+COPY ./*.py /jarr/
+COPY ./jarr/ /jarr/jarr
+
+VOLUME /etc/jarr/

Dockerfiles/server

@@ -0,0 +1,6 @@
+FROM jarr-base:latest
+
+COPY example_conf/logging.ini /etc/jarr/logging.ini
+COPY example_conf/gunicorn.py /etc/jarr/gunicorn.py
+EXPOSE 8000
+CMD ["pipenv", "run", "gunicorn", "-c", "/etc/jarr/gunicorn.py", "--log-config", "/etc/jarr/logging.ini", "-b", "0.0.0.0:8000", "wsgi:application"]

Dockerfiles/worker

@@ -0,0 +1,3 @@
+FROM jarr-base:latest
+EXPOSE 8001
+CMD ["pipenv", "run", "celery", "worker", "--app", "ep_celery.celery_app"]