Akeneo - split job-queue-consumer from php fpm container (#206)

* initial changes to support division of akeneo php-fpm container

* initial changes to support division of akeneo php-fpm container

* changes to support division of php-fpm container and configuration of N job queue consumer daemons

* changes to support division of php-fpm container and configuration of N job queue consumer daemons

* changes to support division of php-fpm container and configuration of N job queue consumer daemons

* changes to support division of php-fpm container and configuration of N job queue consumer daemons

Co-authored-by: g-foster <gavin.foster@inviqa.com>

src/akeneo/_twig/docker-compose.yml/application.yml.twig

@@ -0,0 +1,71 @@
+{% set blocks  = '_twig/docker-compose.yml/' %}
+{% set dockersync = false %}
+{% if @('host.os') == 'darwin' and @('docker-sync') == 'yes' %}
+{% set dockersync = true %}
+{% endif %}
+
+  console:
+{% if @('app.build') == 'dynamic' %}
+    build:
+      context: ./
+      dockerfile: .my127ws/docker/image/console/Dockerfile
+    entrypoint: [/entrypoint.dynamic.sh]
+    command: [sleep, infinity]
+    volumes:
+      - {{ (dockersync) ? @('workspace.name') ~ '-sync:/app:nocopy' : './:/app:delegated' }}
+      - ./.my127ws/application:/home/build/application
+      - ./.my127ws/docker/image/console/root/lib/task:/lib/task
+      - ./.my127ws:/.my127ws
+{% else %}
+    image: {{ @('docker.repository') ~ ':' ~ @('app.version') ~ '-console' }}
+{% endif %}
+    labels:
+      - traefik.enable=false
+    environment:
+{{ deep_merge_to_yaml([@('services.php-base.environment'), @('services.console.environment')], 2, 6) | raw }}
+{% include blocks ~ 'environment.yml.twig' %}
+    networks:
+      - private
+
+  nginx:
+{% if @('app.build') == 'dynamic' %}
+    build: .my127ws/docker/image/nginx
+    volumes:
+      - {{ (dockersync) ? @('workspace.name') ~ '-sync:/app:nocopy' : './:/app:delegated' }}
+{% else %}
+    image: {{ @('docker.repository') ~ ':' ~ @('app.version') ~ '-nginx' }}
+{% endif %}
+    labels:
+      - traefik.backend={{ @('workspace.name') }}
+      - traefik.frontend.rule=Host:{{ @('hostname') }}
+      - traefik.docker.network=my127ws
+      - traefik.port=80
+{% if @('services.nginx.environment') %}
+    environment:
+{{ to_nice_yaml(@('services.nginx.environment'), 2, 6) | raw }}
+{% endif %}
+    links:
+      - php-fpm:php-fpm
+    networks:
+      private:
+        aliases:
+          - {{ @('hostname') }}
+      shared: {}
+
+  php-fpm:
+{% if @('app.build') == 'dynamic' %}
+    build: .my127ws/docker/image/php-fpm
+    image: {{ @('workspace.name') ~ '-php-fpm:dev' }}
+    volumes:
+      - {{ (dockersync) ? @('workspace.name') ~ '-sync:/app:nocopy' : './:/app:delegated' }}
+      - ./.my127ws:/.my127ws
+{% else %}
+    image: {{ @('docker.repository') ~ ':' ~ @('app.version') ~ '-php-fpm' }}
+{% endif %}
+    labels:
+      - traefik.enable=false
+    networks:
+      - private
+    environment:
+{{ deep_merge_to_yaml([@('services.php-base.environment'), @('services.php-fpm.environment')], 2, 6) | raw }}
+{% include blocks ~ 'environment.yml.twig' %}

src/akeneo/_twig/docker-compose.yml/service/job-queue-consumer.yml.twig

@@ -0,0 +1,18 @@
+  job-queue-consumer:
+{% if @('app.build') == 'dynamic' %}
+    build:
+      context: ./
+      dockerfile: .my127ws/docker/image/job-queue-consumer/Dockerfile
+    volumes:
+      - {{ (dockersync) ? @('workspace.name') ~ '-sync:/app:nocopy' : './:/app:delegated' }}
+      - ./.my127ws/application:/home/build/application
+{% else %}
+    image: {{ @('docker.repository') ~ ':' ~ @('app.version') ~ '-job-queue-consumer' }}
+{% endif %}
+    environment:
+{{ deep_merge_to_yaml([@('services.php-base.environment'),@('services.job-queue-consumer.environment')], 2, 6) | raw }}
+{% include blocks ~ 'environment.yml.twig' %}
+    networks:
+      - private
+    labels:
+      - traefik.enable=false

src/akeneo/docker/image/job-queue-consumer/Dockerfile.twig

@@ -0,0 +1,24 @@
+{% if @('app.build') == 'static' %}
+FROM {{ @('docker.repository') ~ ':' ~ @('app.version') }}-php-fpm
+{% else %}
+FROM {{ @('workspace.name') ~ '-php-fpm:dev' }}
+{% endif %}
+
+WORKDIR /app
+COPY .my127ws/docker/image/job-queue-consumer/root /
+
+{% if @('app.build') == 'static' %}
+RUN bash /fix_app_permissions.sh
+{% else %}
+VOLUME /app
+{% endif %}
+ENV APP_MODE {{ @('app.mode') }}
+
+{% if @('app.build') == 'static' %}
+RUN chmod +x /entrypoint.sh
+ENTRYPOINT ["/entrypoint.sh"]
+CMD ["sleep", "infinity"]
+{% else %}
+ENTRYPOINT ["/entrypoint.dynamic.sh"]
+CMD ["sleep", "infinity"]
+{% endif %}

src/akeneo/docker/image/php-fpm/root/etc/supervisor/conf.d/akeneo-job-queue-consumer.conf → src/akeneo/docker/image/job-queue-consumer/root/etc/supervisor/conf.d/container-cmd.conf

@@ -1,5 +1,8 @@
 [program:akeneo-job-queue-consumer]
-command=php /app/bin/console --env=%(ENV_AKENEO_APP_MODE) akeneo:batch:job-queue-consumer-daemon
+process_name=job-queue-consumer_%(process_num)02d
+numprocs=%(ENV_AKENEO_JOB_QUEUE_CONSUMER_PROCESSES)s
+numprocs_start=01
+command=php /app/bin/console --env=%(ENV_AKENEO_APP_MODE)s akeneo:batch:job-queue-consumer-daemon
 stdout_logfile=/dev/stdout
 stdout_logfile_maxbytes=0
 stderr_logfile=/dev/stderr

src/akeneo/harness.yml

@@ -17,6 +17,7 @@ attributes:
       - chrome
       - elasticsearch
       - mysql
+      - job-queue-consumer
     file_storage_directory: /app/app/file_storage
     uploads_directory: /app/app/uploads
     web_directory: /app/web

src/akeneo/harness/attributes/docker.yml

@@ -11,8 +11,11 @@ attributes:
     php-base:
       environment:
         AKENEO_SECRET: = @('akeneo.secret')
-        AUTOSTART_AKENEO_JOB_QUEUE_CONSUMER: "true"
         AKENEO_APP_MODE:  = @('akeneo.app.mode')
+    job-queue-consumer:
+      environment:
+        AUTOSTART_AKENEO_JOB_QUEUE_CONSUMER: "true"
+        AKENEO_JOB_QUEUE_CONSUMER_PROCESSES: 5
   pipeline:
     base:
       services:

src/akeneo/harness/config/confd.yml

@@ -34,3 +34,5 @@ confd('harness:/'):
   - { src: helm/qa/requirements.yaml }
   # harness-akeneo templates
   - { src: application/overlay/app/config/parameters.yml }
+  - { src: docker/image/nginx/root/docker-entrypoint.d/config_render.sh }
+  - { src: docker/image/job-queue-consumer/Dockerfile }

src/akeneo/harness/config/q-akeneo-pipeline.yml

@@ -0,0 +1,21 @@
+
+command('app build'): |
+  #!bash(workspace:/)|@
+  ws app build console
+  ws app build php-fpm
+  ws app build job-queue-consumer
+  ws app build nginx
+
+command('app build job-queue-consumer'): |
+  #!bash(workspace:/)|@
+  passthru docker build -t @('docker.repository'):@('app.version')-job-queue-consumer -f .my127ws/docker/image/job-queue-consumer/Dockerfile .
+
+command('app publish'): |
+  #!bash(workspace:/)|@
+  run docker login -u="@('docker.username')" -p="@('docker.password')" @('docker.repository')
+  run docker push @('docker.repository'):@('app.version')-console
+  run docker push @('docker.repository'):@('app.version')-php-fpm
+  run docker push @('docker.repository'):@('app.version')-job-queue-consumer
+  run docker push @('docker.repository'):@('app.version')-nginx
+  run docker logout @('docker.repository')
+

src/akeneo/harness/scripts/enable.sh

@@ -0,0 +1,59 @@
+#!/usr/bin/env bash
+
+main()
+{
+    passthru ws networks external
+    if [ ! -f .my127ws/.flag-built ]; then
+
+        passthru docker-compose down
+
+        if [[ "$HAS_ASSETS" = "yes" ]]; then
+            ws assets download
+        fi
+
+        $APP_BUILD
+        touch .my127ws/.flag-built
+
+    else
+        passthru docker-compose up -d
+        passthru docker-compose exec -T -u build console app welcome
+    fi
+
+    if [[ "$APP_BUILD" = "dynamic" && "$USE_DOCKER_SYNC" = "yes" ]]; then
+        passthru docker-sync start
+    fi
+}
+
+dynamic()
+{
+    # we synchronise then stop docker-sync as leaving it running during the build
+    # will often cause it to crash.
+
+    if [[ "$USE_DOCKER_SYNC" = "yes" ]]; then
+        passthru gem install docker-sync --no-document
+        passthru docker-sync start
+        passthru docker-sync stop
+    fi
+
+    passthru "docker-compose config --services | grep -v php-fpm | xargs docker-compose pull"
+    passthru "docker-compose config --services | grep -v job-queue-consumer | xargs docker-compose build --pull"
+    passthru docker-compose build job-queue-consumer
+
+    passthru "docker-compose config --services | grep -v job-queue-consumer | xargs docker-compose up -d"
+
+    passthru docker-compose exec -T -u build console app build
+    passthru docker-compose exec -T -u build console app init
+
+    passthru docker-compose up -d job-queue-consumer
+}
+
+static()
+{
+    ws app build
+
+    passthru "docker-compose config --services | grep -v job-queue-consumer | xargs docker-compose up -d"
+    passthru docker-compose exec -T -u build console app init
+    passthru docker-compose up -d job-queue-consumer
+}
+
+main

src/akeneo/helm/app/templates/application/job-queue-consumer/deployment.yaml

@@ -0,0 +1,88 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ .Values.resourcePrefix }}job-queue-consumer
+  labels:
+    app.service: {{ .Values.resourcePrefix }}job-queue-consumer
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app.service: {{ .Values.resourcePrefix }}job-queue-consumer
+  template:
+    metadata:
+      labels:
+        app.service: {{ .Values.resourcePrefix }}job-queue-consumer
+    spec:
+      {{- if not (eq "" (include "application.volumes.wwwDataPaths" .)) }}
+      initContainers:
+      - name: job-queue-consumer-volume-permissions
+        image: busybox
+        command: 
+        - "/bin/chown"
+        - "-R"
+        - "www-data"
+        {{- include "application.volumes.wwwDataPaths" . | indent 8 }}
+        volumeMounts:
+        {{- if not (and (eq "" (include "application.volumeMounts.backend" .)) (eq "" (include "application.volumeMounts.all" .)) ) }}
+        {{- include "application.volumeMounts.backend" . | indent 8 }}
+        {{- include "application.volumeMounts.all" . | indent 8 }}
+        {{- else }} []
+        {{- end }}
+      {{- end }}
+      containers:
+      - name: job-queue-consumer
+        env:
+        {{- range $key, $value := index .Values.docker.services "php-base" "environment" }}
+        {{- $tp := typeOf $value }}
+        - name: {{ $key | quote }}
+        {{- if eq $tp "string" }}
+          value: {{ tpl $value $ | quote }}
+        {{- else}}
+          value: {{ $value | quote }}
+        {{- end }}
+        {{- end }}
+        {{- range $key, $value := index .Values.docker.services "job-queue-consumer" "environment" }}
+        {{- $tp := typeOf $value }}
+        - name: {{ $key | quote }}
+        {{- if eq $tp "string" }}
+          value: {{ tpl $value $ | quote }}
+        {{- else}}
+          value: {{ $value | quote }}
+        {{- end }}
+        {{- end }}
+        {{- range $key, $value := .Values.environment }}
+        - name: {{ $key | quote }}
+          value: {{ $value | quote }}
+        {{- end }}
+        image: {{ .Values.docker.image.job_queue_consumer }}
+        imagePullPolicy: Always
+        ports:
+        - containerPort: 9000
+        resources:
+          limits:
+            memory: {{ .Values.resources.limits.memory.job_queue_consumer }}
+          requests:
+            memory: {{ .Values.resources.requests.memory.job_queue_consumer }}
+        readinessProbe:
+          tcpSocket:
+            port: 9000
+          initialDelaySeconds: 5
+          periodSeconds: 10
+        volumeMounts:
+        {{- if not (and (eq "" (include "application.volumeMounts.backend" .)) (eq "" (include "application.volumeMounts.all" .)) ) }}
+        {{- include "application.volumeMounts.backend" . | indent 8 }}
+        {{- include "application.volumeMounts.all" . | indent 8 }}
+        {{- else }} []
+        {{- end }}
+
+      imagePullSecrets:
+      - name: {{ .Values.resourcePrefix }}docker-config
+      restartPolicy: Always
+      volumes:
+      {{- if not (and (eq "" (include "application.volumes.backend" .)) (eq "" (include "application.volumes.all" .)) ) }}
+      {{- include "application.volumes.backend" . | indent 6 }}
+      {{- include "application.volumes.all" . | indent 6 }}
+      {{- else }} []
+      {{- end }}
+status: {}

src/akeneo/helm/app/templates/values.yaml.twig

@@ -0,0 +1,36 @@
+{% set blocks  = 'helm/app/_twig/values.yaml/' %}
+
+environment:
+{% include blocks ~ 'environment.yml.twig' %}
+
+docker:
+  config: {{ @('docker.config') }}
+  image:
+    console: {{ @('docker.repository') ~ ':' ~ @('app.version') ~ '-console' }}
+    fpm: {{ @('docker.repository') ~ ':' ~ @('app.version') ~ '-php-fpm' }}
+    nginx: {{ @('docker.repository') ~ ':' ~ @('app.version') ~ '-nginx' }}
+    job_queue_consumer: {{ @('docker.repository') ~ ':' ~ @('app.version') ~ '-job-queue-consumer' }}
+    elasticsearch: {{ @('elasticsearch.image') ~ ':' ~ @('elasticsearch.tag') }}
+  services:
+{{ deep_merge_to_yaml([@('services'), @('pipeline.base.services')], 2, 4) | raw }}
+
+service:
+  mysql: {{ ("mysql" in @('app.services')) ? 'true' : 'false' }}
+  postgres: {{ ("postgres" in @('app.services')) ? 'true' : 'false' }}
+  redis: {{ ("redis" in @('app.services')) ? 'true' : 'false' }}
+  redis_session: {{ ("redis-session" in @('app.services')) ? 'true' : 'false' }}
+  memcached: {{ ("memcached" in @('app.services')) ? 'true' : 'false' }}
+  elasticsearch: {{ ("elasticsearch" in @('app.services')) ? 'true' : 'false' }}
+  job-queue-consumer: {{ ("job-queue-consumer" in @('app.services')) ? 'true' : 'false' }}
+
+resources:
+{% include blocks ~ 'resources.yml.twig' %}
+
+persistence:
+{{ to_nice_yaml(@('pipeline.base.persistence'), 2, 2) | raw }}
+
+resourcePrefix: {{ @('pipeline.base.resourcePrefix') | json_encode | raw }}
+
+ingress: "standard" # standard or istio
+istio:
+{{ to_nice_yaml(@('pipeline.base.istio'), 2, 2) | raw }}