Pull external images in parallel before building service images

src/_base/harness/config/external-images.yml

@@ -0,0 +1,47 @@
+function('external_images', [services]): |
+  #!php
+
+  $upstreamImages = call_user_func_array(
+    'array_merge',
+    array_map(
+      function ($service) {
+        return $service['upstream'];
+      },
+      $services
+    )
+  );
+
+  $externalImages = array_filter(
+    $upstreamImages,
+    function ($image) use ($services) {
+      $serviceUpstreamImages = array_filter(
+        $services,
+        function ($service) use ($image) {
+          return $service['image'] == $image;
+        }
+      );
+      return count($serviceUpstreamImages) == 0;
+    }
+  );
+
+  # workspace commands don't allow non-string types
+  = join(' ', $externalImages);
+
+
+command('external-images config'):
+  env:
+    IMAGES: = external_images(docker_service_images())
+  exec: |
+    #!php
+    $compose = ['version' => '3', 'services' => []];
+    foreach (explode(' ', $env['IMAGES']) as $image) {
+      $compose['services'][str_replace(['/', ':'], '_', $image)] = ['image' => $image];
+    }
+    echo \Symfony\Component\Yaml\Yaml::dump($compose, 100, 2);
+
+command('external-images pull'):
+  env:
+    BASE_IMAGES: = external_images(docker_service_images())
+  exec: |
+    #!bash(workspace:/)|@
+    passthru 'ws external-images config | docker-compose -f - pull'

src/_base/harness/config/functions.yml

@@ -114,6 +114,30 @@ function('get_docker_external_networks'): |
   }
   = join(" ", $externalNetworks);
 
+function('docker_service_images'): |
+  #!php
+  $configRaw = shell_exec('docker-compose config');
+  $config = \Symfony\Component\Yaml\Yaml::parse($configRaw);
+  $images = [];
+
+  foreach ($config['services'] as $serviceName => $service) {
+    if (!isset($service['build'])) {
+      continue;
+    }
+    $context = rtrim($service['build']['context'], '/');
+    $dockerfile = $service['build']['dockerfile'] ?? 'Dockerfile';
+
+    if (preg_match_all('/^FROM\s+([^\s]*)/m', file_get_contents($context.'/'.$dockerfile), $matches) === false) {
+      continue;
+    }
+    $images[$serviceName] = [
+      'image' => $service['image'] ?? null,
+      'upstream' => $matches[1],
+    ];
+  }
+
+  = $images;
+
 function('get_docker_registry', [dockerRepository]): |
   #!php
   $dockerRepoParts = explode('/', $dockerRepository);

src/_base/harness/config/pipeline.yml

@@ -4,6 +4,8 @@ command('app build'):
     HAS_CRON: "= ('cron' in @('app.services') ? 'true' : 'false')"
   exec: |
     #!bash(workspace:/)|@
+    ws external-images pull
+
     # dependency ordered build
     passthru docker-compose build console
     passthru docker-compose build php-fpm nginx

src/_base/harness/scripts/enable.sh.twig

@@ -39,9 +39,11 @@ dynamic()
         passthru ws mutagen pause
     fi
 
+    ws external-images pull
+
     {% if ("cron" in @('app.services')) %}
         passthru "docker-compose config --services | grep -v php-fpm | xargs docker-compose pull"
-        passthru "docker-compose config --services | grep -v cron | xargs docker-compose build --pull"
+        passthru "docker-compose config --services | grep -v cron | xargs docker-compose build"
         passthru docker-compose build cron
         # Bring up all but cron
         passthru "docker-compose config --services | grep -v cron | xargs docker-compose up -d"

src/akeneo/harness/config/q-akeneo-pipeline.yml

@@ -1,6 +1,8 @@
 
 command('app build'): |
   #!bash(workspace:/)|@
+  ws external-images pull
+
   passthru docker-compose build console
   passthru docker-compose build php-fpm nginx
   passthru docker-compose build job-queue-consumer cron

src/spryker/harness/config/spryker-pipeline.yml

@@ -1,5 +1,7 @@
 
 command('app build'): |
   #!bash(workspace:/)|@
+  ws external-images pull
+
   passthru docker-compose build console
   passthru docker-compose build php-fpm nginx jenkins-runner