From 7f3eab33e6019af67f147ce301235875bf9363a2 Mon Sep 17 00:00:00 2001 From: Aly Sivji Date: Fri, 14 Sep 2018 15:25:30 -0500 Subject: [PATCH 1/3] Get pyspy running in a container --- Makefile | 4 ++-- docker-compose.yml | 10 ++++++++++ scripts/py-spy/Dockerfile | 5 +++++ 3 files changed, 17 insertions(+), 2 deletions(-) create mode 100644 scripts/py-spy/Dockerfile diff --git a/Makefile b/Makefile index a3334d9..ad9370e 100644 --- a/Makefile +++ b/Makefile @@ -28,11 +28,11 @@ build: docker-compose build up: - docker-compose up -d api db + docker-compose up -d make migrate-up start: - docker-compose start api db + docker-compose start stop: docker-compose stop diff --git a/docker-compose.yml b/docker-compose.yml index eadb327..af451ed 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -112,3 +112,13 @@ services: - elasticsearch ports: - "5601:5601" + + # dev tools + py-spy: + build: + context: ./scripts/py-spy/ + pid: "service:api" + cap_add: + - SYS_PTRACE + stdin_open: true + tty: true diff --git a/scripts/py-spy/Dockerfile b/scripts/py-spy/Dockerfile new file mode 100644 index 0000000..4a6fbdc --- /dev/null +++ b/scripts/py-spy/Dockerfile @@ -0,0 +1,5 @@ +FROM python:3.7.0-stretch + +RUN pip install py-spy + +CMD ["python"] From 5faa2deecd598040914d68b8a43dfa1327b0f8e2 Mon Sep 17 00:00:00 2001 From: Aly Sivji Date: Fri, 14 Sep 2018 16:02:34 -0500 Subject: [PATCH 2/3] update readme and makefile --- Makefile | 11 +++++++---- README.md | 11 +++++++++++ docker-compose.yml | 4 +++- 3 files changed, 21 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index ad9370e..d96014a 100644 --- a/Makefile +++ b/Makefile @@ -52,14 +52,17 @@ logs: shell: ## Shell into web container docker-compose exec api bash -shell-root: # Shell into web container as root - docker-compose exec -u root api bash +shell-db: ## Shell into postgres process inside db container + docker-compose exec db psql -w --username "sivdev_user" --dbname "sivdev" shell-ipython: ## Shell into ipython with falcon context docker-compose exec api python /app/scripts/ipython_shell.py -shell-db: ## Shell into postgres process inside db container - docker-compose exec db psql -w --username "sivdev_user" --dbname "sivdev" +shell-profiler: + docker-compose exec profiler bash + +shell-root: # Shell into web container as root + docker-compose exec -u root api bash migration: ## Create migrations using alembic docker-compose exec api alembic revision --autogenerate -m "$(m)" diff --git a/README.md b/README.md index e167bfb..3ce8567 100644 --- a/README.md +++ b/README.md @@ -77,6 +77,17 @@ Movie.search("top gun", page=1, per_page=5) * Remove database from request before sending response * Migrations with [Alembic](http://alembic.zzzcomputing.com/en/latest/) +### Profiler + +* Attach [py-spy](https://github.com/benfred/py-spy) to Python process to profile +* Connect to Gunicorn process to profile web application + * [Gunicorn architecture](http://docs.gunicorn.org/en/stable/design.html) + +```console +USAGE: + py-spy --duration --pid --rate +``` + ### Testing * [pytest](https://docs.pytest.org/en/latest/) diff --git a/docker-compose.yml b/docker-compose.yml index af451ed..211ca16 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -114,11 +114,13 @@ services: - "5601:5601" # dev tools - py-spy: + profiler: build: context: ./scripts/py-spy/ pid: "service:api" cap_add: - SYS_PTRACE + volumes: + - ./scripts/py-spy/flame-graphs:/flame-graphs stdin_open: true tty: true From 49a78e25b10efb9afd1ea1fe65be630405d19695 Mon Sep 17 00:00:00 2001 From: Aly Sivji Date: Fri, 14 Sep 2018 16:38:30 -0500 Subject: [PATCH 3/3] Move dockerfiles into own directory --- docker-compose.yml | 10 ++++++---- Dockerfile => docker/Dockerfile | 0 docker/README.md | 3 +++ {scripts => docker}/py-spy/Dockerfile | 0 {scripts => docker}/rq-dashboard/Dockerfile | 0 {scripts => docker}/rq-scheduler/Dockerfile | 0 6 files changed, 9 insertions(+), 4 deletions(-) rename Dockerfile => docker/Dockerfile (100%) create mode 100644 docker/README.md rename {scripts => docker}/py-spy/Dockerfile (100%) rename {scripts => docker}/rq-dashboard/Dockerfile (100%) rename {scripts => docker}/rq-scheduler/Dockerfile (100%) diff --git a/docker-compose.yml b/docker-compose.yml index 211ca16..d140374 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -47,6 +47,7 @@ services: api: build: context: . + dockerfile: ./docker/Dockerfile command: ["gunicorn", "app:api", "-b", "0.0.0.0:7000", "--reload", "--timeout", "100000"] environment: API_URL: http://api:7000 @@ -75,7 +76,7 @@ services: # task queue rq-dash: build: - context: ./scripts/rq-dashboard/ + context: ./docker/rq-dashboard/ environment: RQ_DASHBOARD_REDIS_URL: redis://redis:6379 depends_on: @@ -85,7 +86,7 @@ services: command: ["rq-dashboard"] rq-scheduler: build: - context: ./scripts/rq-scheduler/ + context: ./docker/rq-scheduler/ environment: RQ_REDIS_HOST: redis RQ_REDIS_PORT: 6379 @@ -95,6 +96,7 @@ services: worker: build: context: . + dockerfile: ./docker/Dockerfile environment: GMAIL_ADDRESS: $GMAIL_ADDRESS GMAIL_APPLICATION_PASSWORD: $GMAIL_APPLICATION_PASSWORD @@ -116,11 +118,11 @@ services: # dev tools profiler: build: - context: ./scripts/py-spy/ + context: ./docker/py-spy/ pid: "service:api" cap_add: - SYS_PTRACE volumes: - - ./scripts/py-spy/flame-graphs:/flame-graphs + - ./docker/py-spy/flame-graphs:/flame-graphs stdin_open: true tty: true diff --git a/Dockerfile b/docker/Dockerfile similarity index 100% rename from Dockerfile rename to docker/Dockerfile diff --git a/docker/README.md b/docker/README.md new file mode 100644 index 0000000..21b6834 --- /dev/null +++ b/docker/README.md @@ -0,0 +1,3 @@ +# Docker Images + +Supporting materials required to make Docker images for webapp. diff --git a/scripts/py-spy/Dockerfile b/docker/py-spy/Dockerfile similarity index 100% rename from scripts/py-spy/Dockerfile rename to docker/py-spy/Dockerfile diff --git a/scripts/rq-dashboard/Dockerfile b/docker/rq-dashboard/Dockerfile similarity index 100% rename from scripts/rq-dashboard/Dockerfile rename to docker/rq-dashboard/Dockerfile diff --git a/scripts/rq-scheduler/Dockerfile b/docker/rq-scheduler/Dockerfile similarity index 100% rename from scripts/rq-scheduler/Dockerfile rename to docker/rq-scheduler/Dockerfile