Improve deployment

.env.template

@@ -1,4 +1,4 @@
-# get the complete env from other volunteers, please
+AIRFLOW_HOME=/opt/airflow
 BIGQUERY_PROJECT=pycontw-225217
-DATATEAM_DISCORD_WEBHOOK=<>
-GOOGLE_APPLICATION_CREDENTIALS=/usr/local/airflow/service-account.json
+GOOGLE_APPLICATION_CREDENTIALS=/opt/airflow/service-account.json
+AIRFLOW__CORE__FERNET_KEY=paste-your-fernet-key-here

.github/workflows/python.yml

@@ -21,8 +21,8 @@ jobs:
 
       - name: Install dependencies
         run: |
-          pip install -U poetry==1.6.1
-          poetry install
+          pip install -r requirements.txt -r requirements-dev.txt -c constraints-3.8.txt
+          pip install black==19.10b0 click==7.1.2
 
       - name: Run linters
         run: make lint

.gitignore

@@ -1,4 +1,5 @@
 # project stuff
+.env.production
 .env.staging
 env
 env.sh

Dockerfile

@@ -1,7 +1,9 @@
-FROM apache/airflow:1.10.13-python3.8
+ARG AIRFLOW_VERSION=1.10.15
+ARG PYTHON_VERSION=3.8
+
+FROM apache/airflow:${AIRFLOW_VERSION}-python${PYTHON_VERSION}
+
 USER root
-ENV POETRY_CACHE_DIR='/var/cache/pypoetry' \
-    GOOGLE_APPLICATION_CREDENTIALS='/usr/local/airflow/service-account.json'
 
 RUN apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 \
     && apt-key adv --keyserver keyserver.ubuntu.com --recv-keys B7B3B788A8D3785C \
@@ -10,19 +12,22 @@ RUN apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 \
     # 1. if you don't need postgres, remember to remove postgresql-dev and sqlalchemy
     # 2. libglib2.0-0 libsm6 libxext6 libxrender-dev libgl1-mesa-dev are required by opencv
     # 3. git is required by pip install git+https
-    && pip install --no-cache-dir -U poetry==1.6.1 \
     && apt-get clean \
     && rm -rf /var/lib/apt/lists/*
 
-COPY pyproject.toml pyproject.toml
-COPY poetry.toml poetry.toml
-COPY poetry.lock poetry.lock
+COPY entrypoint.sh /entrypoint.sh
 
-RUN python -m poetry install --no-interaction --no-ansi --no-dev \
-    # Cleaning poetry installation's cache for production:
-    && rm -rf "$POETRY_CACHE_DIR" \
-    && pip uninstall -yq poetry
+RUN chmod +x /entrypoint.sh
 
 USER airflow
-COPY dags /usr/local/airflow/dags
-COPY airflow.cfg airflow.cfg
+
+COPY ./requirements.txt ${AIRFLOW_HOME}/requirements.txt
+COPY ./constraints-3.8.txt ${AIRFLOW_HOME}/constraints-3.8.txt
+
+RUN pip install --no-cache-dir -r ${AIRFLOW_HOME}/requirements.txt --constraint constraints-3.8.txt
+
+COPY airflow.cfg ${AIRFLOW_HOME}/airflow.cfg
+
+COPY --chown=airflow:root dags ${AIRFLOW_HOME}/dags
+
+ENTRYPOINT ["/entrypoint.sh"]

Dockerfile.test

@@ -1,5 +1,36 @@
-FROM davidtnfsh/pycon_etl:prod
-ENV AIRFLOW_TEST_MODE=True
-ENV FERNET_KEY="uMnRC6ingT/WjPzPiXLvbWTJYzaA3sJdJRVkinceVp4="
+ARG AIRFLOW_VERSION=1.10.15
+ARG PYTHON_VERSION=3.8
+
+FROM apache/airflow:${AIRFLOW_VERSION}-python${PYTHON_VERSION}
+
 USER root
-COPY airflow.test.cfg airflow.cfg
+
+RUN apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 \
+    && apt-key adv --keyserver keyserver.ubuntu.com --recv-keys B7B3B788A8D3785C \
+    && apt-get update \
+    && apt-get install -y --no-install-recommends git \
+    # 1. if you don't need postgres, remember to remove postgresql-dev and sqlalchemy
+    # 2. libglib2.0-0 libsm6 libxext6 libxrender-dev libgl1-mesa-dev are required by opencv
+    # 3. git is required by pip install git+https
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
+
+COPY entrypoint.sh /entrypoint.sh
+
+RUN chmod +x /entrypoint.sh
+
+USER airflow
+
+COPY ./requirements.txt ${AIRFLOW_HOME}/requirements.txt
+COPY ./requirements-dev.txt ${AIRFLOW_HOME}/requirements-dev.txt
+COPY ./constraints-3.8.txt ${AIRFLOW_HOME}/constraints-3.8.txt
+
+RUN pip install --no-cache-dir -r ${AIRFLOW_HOME}/requirements.txt -r ${AIRFLOW_HOME}/requirements-dev.txt --constraint constraints-3.8.txt
+
+COPY airflow.cfg ${AIRFLOW_HOME}/airflow.cfg
+
+COPY --chown=airflow:root dags ${AIRFLOW_HOME}/dags
+
+ENV AIRFLOW_TEST_MODE=True
+
+ENTRYPOINT ["/entrypoint.sh"]

Makefile

@@ -1,17 +1,30 @@
-VENV_PREFIX=poetry run
-
 lint:
-	$(VENV_PREFIX) black . --check
-	$(VENV_PREFIX) isort --check-only .
-	$(VENV_PREFIX) flake8 .
-	$(VENV_PREFIX) mypy dags/ tests/
+	black . --check
+	isort --check-only .
+	flake8 .
+	mypy dags/ tests/
 
 format:
-	$(VENV_PREFIX) black .
-	$(VENV_PREFIX) isort .
+	black .
+	isort .
 
 test:
-	PYTHONPATH=./dags $(VENV_PREFIX) pytest
+	PYTHONPATH=./dags pytest
 
 coverage:
-	PYTHONPATH=./dags $(VENV_PREFIX) pytest --cov=dags tests
+	PYTHONPATH=./dags pytest --cov=dags tests
+
+build-dev:
+	docker-compose -f ./docker-compose-dev.yml build
+
+deploy-dev:
+	docker-compose -f ./docker-compose-dev.yml up -d
+
+down-dev:
+	docker-compose -f ./docker-compose-dev.yml down
+
+deploy-prod:
+	docker-compose -f ./docker-compose.yml up -d
+
+down-prod:
+	docker-compose -f ./docker-compose.yml down