Add cron container, with it enabled for Akeneo (#210)

* Add cron container for Akeneo

* Add cron container for Akeneo

* Add cron container for Akeneo

* Add cron container for Akeneo

* Move cron container to base harness

* Move cron container to base harness

* Move cron container to base harness

* Move cron container to base harness

* Move cron container to base harness

* Move cron container to base harness

* Move cron container to base harness

* Move cron container to base harness

* Move cron container to base harness

* Move cron container to base harness

* Move cron container to base harness - fix test build issues

* Revert "Move cron container to base harness - fix test build issues"

This reverts commit 9f3da30.

* Move cron container to base harness - fix test build issues

* trigger build

* Improve the quoting of env vars

* Improve the quoting of env vars

* Improve env vars and logging

* remove unneeded sleep

Co-authored-by: g-foster <gavin.foster@inviqa.com>

src/_base/_twig/docker-compose.yml/application.yml.twig

@@ -55,6 +55,9 @@
   php-fpm:
 {% if @('app.build') == 'dynamic' %}
     build: .my127ws/docker/image/php-fpm
+{% if ("cron" in @('app.services')) %}
+    image: {{ @('workspace.name') ~ '-php-fpm:dev' }}
+{% endif %}
     volumes:
       - {{ (dockersync) ? @('workspace.name') ~ '-sync:/app:nocopy' : './:/app:delegated' }}
       - ./.my127ws:/.my127ws

src/_base/_twig/docker-compose.yml/service/cron.yml.twig

@@ -0,0 +1,18 @@
+  cron:
+{% if @('app.build') == 'dynamic' %}
+    build:
+      context: ./
+      dockerfile: .my127ws/docker/image/cron/Dockerfile
+    volumes:
+      - {{ (dockersync) ? @('workspace.name') ~ '-sync:/app:nocopy' : './:/app:delegated' }}
+      - ./.my127ws/application:/home/build/application
+{% else %}
+    image: {{ @('docker.repository') ~ ':' ~ @('app.version') ~ '-cron' }}
+{% endif %}
+    environment:
+{{ deep_merge_to_yaml([@('services.php-base.environment'),@('services.cron.environment')], 2, 6) | raw }}
+{% include blocks ~ 'environment.yml.twig' %}
+    networks:
+      - private
+    labels:
+      - traefik.enable=false

src/_base/docker/image/console/root/lib/task/database/available.sh

@@ -20,7 +20,7 @@ function task_database_available()
 
     while ! $command &> /dev/null; do
 
-        if (( counter > 8 )); then
+        if (( counter > 120 )); then
             (>&2 echo "timeout while waiting on ${DB_PLATFORM} to become available")
             exit 1
         fi

src/_base/docker/image/cron/Dockerfile.twig

@@ -0,0 +1,29 @@
+{% if @('app.build') == 'static' %}
+FROM {{ @('docker.repository') ~ ':' ~ @('app.version') }}-php-fpm
+{% else %}
+FROM {{ @('workspace.name') ~ '-php-fpm:dev' }}
+{% endif %}
+
+# install cron
+RUN apt update
+RUN apt install -y cron sudo
+
+WORKDIR /app
+COPY .my127ws/docker/image/cron/root /
+RUN chmod +x /cron-run-with-env.sh
+
+{% if @('app.build') == 'static' %}
+RUN bash /fix_app_permissions.sh
+{% else %}
+VOLUME /app
+{% endif %}
+ENV APP_MODE {{ @('app.mode') }}
+
+{% if @('app.build') == 'static' %}
+RUN chmod +x /entrypoint.sh
+ENTRYPOINT ["/entrypoint.sh"]
+CMD ["sleep", "infinity"]
+{% else %}
+ENTRYPOINT ["/entrypoint.dynamic.sh"]
+CMD ["sleep", "infinity"]
+{% endif %}

src/_base/docker/image/cron/root/cron-run-with-env.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+IFS=$'\n'
+env_vars="$(cat /app/env.sh)"
+/usr/bin/env - "$env_vars" sudo --preserve-env -u www-data "$@" > /proc/1/fd/1 2> /proc/1/fd/2

src/_base/docker/image/cron/root/crontab.twig

@@ -0,0 +1,3 @@
+{% for cronjob in @('backend.cron.jobs') -%}
+{{ cronjob|raw }}
+{% endfor %}

src/_base/docker/image/cron/root/entrypoint.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+# this is used to load env vars in crontab commands
+env > /app/env.sh
+
+# run
+crontab /crontab
+exec cron -f -L 15

src/_base/harness/attributes/common.yml

@@ -77,6 +77,8 @@ attributes.default:
       steps: []
     migrate:
       steps: []
+    cron:
+      jobs: []
 
   frontend:
     path: /app

src/_base/harness/attributes/docker-base.yml

@@ -33,6 +33,8 @@ attributes:
         POSTGRES_PASSWORD: = @('database.pass')
         POSTGRES_USER: = @('database.user')
         PGDATA: /var/lib/postgresql/data/pgdata
+    cron:
+      environment: []
   pipeline:
     base:
       services:

src/_base/harness/config/confd.yml

@@ -22,12 +22,15 @@ confd('harness:/'):
   - { src: docker/image/php-fpm/root/fix_app_permissions.sh }
   - { src: docker/image/php-fpm/root/usr/local/etc/php/php.ini }
   - { src: docker/image/php-fpm/root/usr/local/etc/php/conf.d/docker-php-ext-xdebug.ini }
+  - { src: docker/image/cron/Dockerfile }
+  - { src: docker/image/cron/root/crontab }
   - { src: application/overlay/Jenkinsfile }
   - { src: application/overlay/auth.json }
   - { src: application/overlay/.dockerignore, dst: workspace:/.dockerignore }
   - { src: application/skeleton/README.md }
   - { src: docker-compose.yml, dst: workspace:/docker-compose.yml }
   - { src: docker-sync.yml, dst: workspace:/docker-sync.yml }
+  - { src: harness/scripts/enable.sh }
   - { src: helm/app/values.yaml }
   - { src: helm/app/Chart.yaml }
   - { src: helm/qa/requirements.yaml }

src/_base/harness/config/pipeline.yml

@@ -9,6 +9,10 @@ command('app build console'): |
   #!bash(workspace:/)|@
   passthru docker build --pull -t @('docker.repository'):@('app.version')-console -f .my127ws/docker/image/console/Dockerfile .
 
+command('app build cron'): |
+  #!bash(workspace:/)|@
+  passthru docker build -t @('docker.repository'):@('app.version')-cron -f .my127ws/docker/image/cron/Dockerfile .
+
 command('app build php-fpm'): |
   #!bash(harness:/docker/image/php-fpm)|@
   passthru docker build -t @('docker.repository'):@('app.version')-php-fpm .

src/_base/harness/scripts/enable.sh → src/_base/harness/scripts/enable.sh.twig

@@ -35,9 +35,15 @@ dynamic()
         passthru docker-sync stop
     fi
 
-    passthru docker-compose pull
-    passthru docker-compose build --pull
-    passthru docker-compose up -d
+    {% if ("cron" in @('app.services')) %}
+        passthru "docker-compose config --services | grep -v php-fpm | xargs docker-compose pull"
+        passthru "docker-compose config --services | grep -v cron | xargs docker-compose build --pull"
+        passthru docker-compose build cron
+    {% else %}
+        passthru docker-compose pull
+        passthru docker-compose build --pull
+        passthru docker-compose up -d
+    {% endif %}
 
     passthru docker-compose exec -T -u build console app build
     passthru docker-compose exec -T -u build console app init

src/_base/helm/app/_twig/values.yaml/resources.yml.twig

@@ -10,6 +10,7 @@
       postgres: {{ @('resources.requests.memory.postgres') }}
       redis: {{ @('resources.requests.memory.redis') }}
       redis_session: {{ @('resources.requests.memory.redis_session') }}
+      cron: {{ @('resources.requests.memory.cron') }}
   limits:
     memory:
       console: {{ @('resources.limits.memory.console') }}
@@ -21,4 +22,5 @@
       mysql: {{ @('resources.limits.memory.mysql') }}
       postgres: {{ @('resources.limits.memory.postgres') }}
       redis: {{ @('resources.limits.memory.redis') }}
-      redis_session: {{ @('resources.limits.memory.redis_session') }}
+      redis_session: {{ @('resources.limits.memory.redis_session') }}
+      cron: {{ @('resources.limits.memory.cron') }}

src/_base/helm/app/templates/application/cron/deployment.yaml

@@ -0,0 +1,88 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ .Values.resourcePrefix }}cron
+  labels:
+    app.service: {{ .Values.resourcePrefix }}cron
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app.service: {{ .Values.resourcePrefix }}cron
+  template:
+    metadata:
+      labels:
+        app.service: {{ .Values.resourcePrefix }}cron
+    spec:
+      {{- if not (eq "" (include "application.volumes.wwwDataPaths" .)) }}
+      initContainers:
+      - name: cron-volume-permissions
+        image: busybox
+        command: 
+        - "/bin/chown"
+        - "-R"
+        - "www-data"
+        {{- include "application.volumes.wwwDataPaths" . | indent 8 }}
+        volumeMounts:
+        {{- if not (and (eq "" (include "application.volumeMounts.backend" .)) (eq "" (include "application.volumeMounts.all" .)) ) }}
+        {{- include "application.volumeMounts.backend" . | indent 8 }}
+        {{- include "application.volumeMounts.all" . | indent 8 }}
+        {{- else }} []
+        {{- end }}
+      {{- end }}
+      containers:
+      - name: cron
+        env:
+        {{- range $key, $value := index .Values.docker.services "php-base" "environment" }}
+        {{- $tp := typeOf $value }}
+        - name: {{ $key | quote }}
+        {{- if eq $tp "string" }}
+          value: {{ tpl $value $ | quote }}
+        {{- else}}
+          value: {{ $value | quote }}
+        {{- end }}
+        {{- end }}
+        {{- range $key, $value := index .Values.docker.services "cron" "environment" }}
+        {{- $tp := typeOf $value }}
+        - name: {{ $key | quote }}
+        {{- if eq $tp "string" }}
+          value: {{ tpl $value $ | quote }}
+        {{- else}}
+          value: {{ $value | quote }}
+        {{- end }}
+        {{- end }}
+        {{- range $key, $value := .Values.environment }}
+        - name: {{ $key | quote }}
+          value: {{ $value | quote }}
+        {{- end }}
+        image: {{ .Values.docker.image.cron }}
+        imagePullPolicy: Always
+        ports:
+        - containerPort: 9000
+        resources:
+          limits:
+            memory: {{ .Values.resources.limits.memory.cron }}
+          requests:
+            memory: {{ .Values.resources.requests.memory.cron }}
+        readinessProbe:
+          tcpSocket:
+            port: 9000
+          initialDelaySeconds: 5
+          periodSeconds: 10
+        volumeMounts:
+        {{- if not (and (eq "" (include "application.volumeMounts.backend" .)) (eq "" (include "application.volumeMounts.all" .)) ) }}
+        {{- include "application.volumeMounts.backend" . | indent 8 }}
+        {{- include "application.volumeMounts.all" . | indent 8 }}
+        {{- else }} []
+        {{- end }}
+
+      imagePullSecrets:
+      - name: {{ .Values.resourcePrefix }}docker-config
+      restartPolicy: Always
+      volumes:
+      {{- if not (and (eq "" (include "application.volumes.backend" .)) (eq "" (include "application.volumes.all" .)) ) }}
+      {{- include "application.volumes.backend" . | indent 6 }}
+      {{- include "application.volumes.all" . | indent 6 }}
+      {{- else }} []
+      {{- end }}
+status: {}

src/_base/helm/app/values.yaml.twig

@@ -10,6 +10,7 @@ docker:
     fpm: {{ @('docker.repository') ~ ':' ~ @('app.version') ~ '-php-fpm' }}
     nginx: {{ @('docker.repository') ~ ':' ~ @('app.version') ~ '-nginx' }}
     elasticsearch: {{ @('elasticsearch.image') ~ ':' ~ @('elasticsearch.tag') }}
+    cron: {{ @('docker.repository') ~ ':' ~ @('app.version') ~ '-cron' }}
   services:
 {{ deep_merge_to_yaml([@('services'), @('pipeline.base.services')], 2, 4) | raw }}
 
@@ -20,6 +21,7 @@ service:
   postgres: {{ ("postgres" in @('app.services')) ? 'true' : 'false' }}
   redis: {{ ("redis" in @('app.services')) ? 'true' : 'false' }}
   redis_session: {{ ("redis-session" in @('app.services')) ? 'true' : 'false' }}
+  cron: {{ ("cron" in @('app.services')) ? 'true' : 'false' }}
 
 resources:
 {% include blocks ~ 'resources.yml.twig' %}

src/akeneo/harness.yml

@@ -18,6 +18,7 @@ attributes:
       - elasticsearch
       - mysql
       - job-queue-consumer
+      - cron
     file_storage_directory: /app/app/file_storage
     uploads_directory: /app/app/uploads
     web_directory: /app/web
@@ -59,6 +60,19 @@ attributes:
         - run "bin/console pim:product:index --all"
     migrate:
       steps: []
+    cron:
+      jobs:
+        - 30 1 * * * /cron-run-with-env.sh /app/bin/console pim:versioning:refresh --env=prod
+        - 0 2 * * * /cron-run-with-env.sh /app/bin/console pim:completeness:calculate --env=prod
+        - 20 0 1 * * /cron-run-with-env.sh /app/bin/console akeneo:batch:purge-job-execution --env=prod
+        - 0 1 * * * /cron-run-with-env.sh /app/bin/console pim:asset:send-expiration-notification --env=prod
+        - 30 4 * * * /cron-run-with-env.sh /app/bin/console pim:volume:aggregate --env=prod
+        - 0 4 * * * /cron-run-with-env.sh /app/bin/console pim:versioning:purge --more-than-days 90 --no-interaction --force --env=prod
+        - 0 5 * * * /cron-run-with-env.sh /app/bin/console akeneo:rule:run --env=prod
+        - 20 0 * * * /cron-run-with-env.sh /app/bin/console pimee:project:notify-before-due-date --env=prod
+        - 0 2 * * * /cron-run-with-env.sh /app/bin/console pimee:project:recalculate --env=prod
+        - 30 */1 * * * /cron-run-with-env.sh /app/bin/console pimee:franklin-insights:fetch-products --env=prod
+        - 0 23 * * * /cron-run-with-env.sh /app/bin/console akeneo:reference-entity:refresh-records --all --env=prod
   frontend:
     path: /app
     build:

src/akeneo/harness/attributes/docker.yml

@@ -2,6 +2,15 @@ attributes:
   akeneo:
     app:
       mode: dev # dev or prod
+  resources:
+    requests:
+      memory:
+        job_queue_consumer: "1024Mi"
+        cron: "1024Mi"
+    limits:
+      memory:
+        job_queue_consumer: "1024Mi"
+        cron: "1024Mi"
   nginx:
     copy_directories:
      - = @('app.web_directory')