switch dependency management and package installation to poetry (#1304)

* Replace setup.py and requirements files with pyproject.toml and poetry

* Switch Dockerfiles, tox, and CI over to using Poetry instead of pip for dependency and package installation

* Fix formatting issues after upgrading black

api/Dockerfile

@@ -7,14 +7,14 @@ RUN groupadd --gid 10001 app && \
 
 WORKDIR /app
 
-COPY requirements/ /app/requirements/
-RUN pip install -r requirements/base.txt
+COPY requirements/poetry.txt /tmp/poetry.txt
+RUN pip install -r /tmp/poetry.txt
 
 COPY . /app
-RUN APP_TYPE="admin" pip install -e .
 
-ENV FLASK_APP shipit_api.admin.flask:app
 ENV WEB_CONCURRENCY=3
 USER app
 
-CMD ["/app/docker.d/init.sh"]
+RUN poetry install --only main
+
+CMD ["/app/docker.d/init.sh", "admin"]

api/Dockerfile.public

@@ -7,15 +7,14 @@ RUN groupadd --gid 10001 app && \
 
 WORKDIR /app
 
-COPY requirements/ /app/requirements/
-RUN pip install -r requirements/public.txt
+COPY requirements/poetry.txt /tmp/poetry.txt
+RUN pip install -r /tmp/poetry.txt
 
-COPY docker.d/init.sh /app/docker.d/
 COPY . /app
-RUN pip install -e .
 
-ENV FLASK_APP shipit_api.public.flask:app
 ENV WEB_CONCURRENCY=3
 USER app
 
-CMD ["/app/docker.d/init.sh"]
+RUN poetry install --only main
+
+CMD ["/app/docker.d/init.sh", "public"]

api/Dockerfile.test

@@ -2,13 +2,17 @@ ARG PYTHON_VERSION
 
 FROM python:${PYTHON_VERSION}
 
+COPY requirements/poetry.txt /tmp/poetry.txt
+RUN pip install -r /tmp/poetry.txt
+
 WORKDIR /app
 
-COPY MANIFEST.in setup.py tox.ini /app/
-COPY requirements/ /app/requirements/
+COPY pyproject.toml /app/
+
+RUN poetry install --only tox --no-root
 
-RUN pip install -r requirements/local.txt
+COPY tox.ini /app/
 
 COPY src/ /app/src/
 
-ENTRYPOINT ["/usr/local/bin/tox", "-e"]
+ENTRYPOINT ["/usr/local/bin/poetry", "run", "tox", "-e"]

api/docker.d/init.sh

@@ -4,6 +4,15 @@ pushd `dirname $0` &>/dev/null
 MY_DIR=$(pwd)
 popd &>/dev/null
 
+if [ $1 == "public" ]; then
+    export FLASK_APP="shipit_api.public.flask:app"
+elif [ $1 == "admin" ]; then
+    export FLASK_APP="shipit_api.admin.flask:app"
+else
+    echo "first arg must be 'public' or 'admin'"
+    exit 1
+fi
+
 EXTRA_ARGS=""
 if [ "$APP_CHANNEL" == "development" ]
 then
@@ -14,7 +23,9 @@ then
     key="${MY_DIR}/key.pem"
 
     # Local development only - we don't want these in deployed environments
-    EXTRA_ARGS="--bind $HOST:$PORT --workers 3 --timeout 3600 --reload --reload-engine=poll --certfile=$cert --keyfile=$key"
+    # More than 1 worker causes race conditions when migrating the database.
+    # They resolve themselves after a restart...but it's preferable not to have them at all.
+    EXTRA_ARGS="--bind $HOST:$PORT --workers 1 --timeout 3600 --reload --reload-engine=poll --certfile=$cert --keyfile=$key"
 fi
 
-exec /usr/local/bin/gunicorn $FLASK_APP --log-file - $EXTRA_ARGS
+exec poetry run gunicorn $FLASK_APP --log-file - $EXTRA_ARGS

api/pyproject.toml

@@ -29,3 +29,85 @@ use_parentheses=true
 
 [tool.coverage.run]
 omit = ["api/tests/*"]
+
+[tool.poetry]
+name = "shipit-api"
+version = "1.0.0"
+description = ""
+authors = ["Release Engineering <releng@mozilla.com>"]
+license = "MPL2"
+packages = [
+    {include = "backend_common", from = "src"},
+    {include = "cli_common", from = "src"},
+    {include = "shipit_api", from = "src"},
+]
+
+[tool.poetry.scripts]
+shipit_upload_product_details = "shipit_api.admin.cli:download_product_details"
+shipit_rebuild_product_details = "shipit_api.admin.cli:rebuild_product_details"
+shipit_import = "shipit_api.admin.cli:shipit_import"
+shipit_trigger_product_details = "shipit_api.admin.cli:trigger_product_details"
+
+[tool.poetry.dependencies]
+# Required by both public and admin apps
+python = "^3.9"
+connexion = {version = "*", extras = ["swagger-ui"]}
+dockerflow = "*"
+flask = "*"
+flask_cors = "*"
+flask_migrate = "*"
+flask_talisman = "*"
+# flask_oidc imports JSONWebSignatureSerializer which was removed in 2.1+
+# https://github.com/pallets/itsdangerous/pull/273
+itsdangerous = "<2.1"
+mozilla-version = "*"
+python-decouple = "*"
+sentry-sdk = {version = "*", extras = ["flask"]}
+slugid = "*"
+sqlalchemy = "<=1.4.44"
+# deployment
+gunicorn = "*"
+psycopg2 = "*"
+
+# Required only by the admin app
+aioamqp = "*"
+aiohttp = {version = "*", extras = ["speedup"]}
+arrow = "*"
+backoff = "*"
+# to support signals in Flask
+blinker = "*"
+flask_login = "*"
+flask_oidc = "*"
+json-e = "*"
+kombu = "*"
+mohawk = "*"
+mypy = "*"
+mypy_extensions = "*"
+pyyaml = "*"
+taskcluster = "*"
+
+[tool.poetry.group.test.dependencies]
+python = "^3.9"
+aioresponses = "*"
+responses = "*"
+pytest = "*"
+pytest-asyncio = "*"
+pytest-cov = "*"
+black = "*"
+check-manifest = "*"
+isort = "*"
+coverage = "*"
+dpath = "*"
+flake8 = "*"
+openapi-spec-validator = "*"
+oyaml = "*"
+
+[tool.poetry.group.ccov-upload.dependencies]
+requests = "*"
+
+[tool.poetry.group.tox.dependencies]
+tox = "*"
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"

api/requirements/README

@@ -0,0 +1,3 @@
+Requirements files are only used to get Poetry installed. From there, Poetry handles all of the application-specific packages, dependencies, et. al.
+
+To pick up a new version of Poetry, run `pip-compile -r poetry.in --generate-hashes`.