pipeline(precompile): allow configuration to force local compilation …

…on bosh director

Add a parameter to allow some bosh releases to be recompiled on some bosh directors
```
<root-deployment-name>:
  precompile:
    disabled-cached-releases: [] # Default: [] - List of bosh releases to compile on local bosh director, even when compiled
```

concourse/pipelines/template/bosh-precompile-pipeline.yml.erb

@@ -40,19 +40,22 @@
         end
       end
     end
-# Psych.dump
   raise "Inconsistency detected on #{root_deployment_name}: #{releases_inconsistencies[root_deployment_name].to_yaml}" if releases_inconsistencies[root_deployment_name].size >0
 
   uniq_releases_per_root_deployments = Hash.new { |h, k| h[k] = {} }
   releases_per_root_deployments_reference = releases_per_root_deployments.dup
   releases_per_root_deployments.each do |root_depl, releases_info|
     depends_on = config&.dig(root_depl,'precompile', 'depends-on') || []
+    disabled_cached_releases = config&.dig(root_depl,'precompile', 'disabled-cached-releases') || []
     releases_info.each do |release_name, details|
       already_defined = false
-      depends_on.each do |depend_root_depl|
-        details_from_parent = releases_per_root_deployments_reference&.dig(depend_root_depl, release_name)
-        standardized_parent_details = extract_repo_and_version(details_from_parent);
-        already_defined = already_defined || standardized_parent_details == extract_repo_and_version(details)
+      use_cached_release = disabled_cached_releases.include?(release_name)
+      unless use_cached_release
+        depends_on.each do |depend_root_depl|
+          details_from_parent = releases_per_root_deployments_reference&.dig(depend_root_depl, release_name)
+          standardized_parent_details = extract_repo_and_version(details_from_parent)
+          already_defined = already_defined || standardized_parent_details == extract_repo_and_version(details)
+        end
       end
       uniq_releases_per_root_deployments[root_depl][release_name] = details unless already_defined
     end
@@ -244,6 +247,7 @@ jobs:
 <%else %>
   - name: push-boshreleases
     <% jobs['Utils'] << 'push-boshreleases' %>
+    serial: true
     on_failure: *on_failure
     plan:
       - in_parallel:
@@ -349,6 +353,7 @@ jobs:
   <% if offline_stemcells_enabled %>
   - name: upload-stemcell-to-s3
     <% jobs['Utils'] << 'upload-stemcell-to-s3' %>
+    serial: true
     on_failure: *on_failure
     plan:
       - in_parallel:
@@ -380,6 +385,7 @@ jobs:
 
   - name: upload-stemcell-to-director
     <% jobs['Utils'] << 'upload-stemcell-to-director' %>
+    serial: true
     on_failure: *on_failure
     plan:
       - in_parallel:

docs/reference_dataset/config_repository/private-config.yml

@@ -61,3 +61,4 @@ log:
 #    excluded_deployments: [] # Default: [] # List deployments to exclude from precompile. As precompile does not use secrets repository to determine enabled deployments, it might be convenient to exclude a deployment not enabled in secrets.
 #    disabled_compiled_download_url_deployments: [] # Default: [] - List of deployments to exclude from bosh manifest precompile patching
 #    disabled_compiled_download_url_deployments_prefix: [] # Default: [] - List of deployments prefix to exclude from bosh manifest precompile patching
+#    disabled-cached-releases: [] # Default: [] - List of bosh releases to compile on local bosh director, even when compiled

docs/reference_dataset/template_repository/shared-config.yml

@@ -61,4 +61,5 @@ default:
 #  precompile:
 #    excluded_deployments: [] # Default: [] # List of deployments to exclude from precompile. Since precompile does not use secrets repository to determine enabled deployments, it may be useful to exclude a deployment that is not enabled in secrets.
 #    disabled_compiled_download_url_deployments: [] # Default: [] - List of deployments to exclude from bosh manifest precompile patching
-#    disabled_compiled_download_url_deployments_prefix: [] # Default: [] - List of deployments prefix to exclude from bosh manifest precompile patching
+#    disabled_compiled_download_url_deployments_prefix: [] # Default: [] - List of deployments prefix to exclude from bosh manifest precompile patching
+#    disabled-cached-releases: [] # Default: [] - List of bosh releases to compile on local bosh director, even when compiled

spec/scripts/generate-depls/fixtures/references/empty-bosh-precompile.yml

@@ -90,6 +90,7 @@ resources:
       skip_ssl_verification: true
 jobs:
   - name: push-boshreleases
+    serial: true
     on_failure: *on_failure
     plan:
       - in_parallel:
@@ -160,6 +161,7 @@ jobs:
                     echo "No errors detected"
                   fi
   - name: upload-stemcell-to-s3
+    serial: true
     on_failure: *on_failure
     plan:
       - in_parallel:
@@ -187,6 +189,7 @@ jobs:
           STEMCELL_BASE_LOCATION: https://bosh.io/d/stemcells
           VERSIONS_FILE: templates-resource/empty-depls/root-deployment.yml
   - name: upload-stemcell-to-director
+    serial: true
     on_failure: *on_failure
     plan:
       - in_parallel:

spec/scripts/generate-depls/fixtures/references/simple-depls-bosh-precompile-ref.yml

@@ -126,6 +126,7 @@ resources:
     version: { ref: v((releases.zookeeper_boshrelease.version)) }
 jobs:
   - name: push-boshreleases
+    serial: true
     on_failure: *on_failure
     plan:
       - in_parallel:
@@ -196,6 +197,7 @@ jobs:
                     echo "No errors detected"
                   fi
   - name: upload-stemcell-to-s3
+    serial: true
     on_failure: *on_failure
     plan:
       - in_parallel:
@@ -223,6 +225,7 @@ jobs:
           STEMCELL_BASE_LOCATION: https://bosh.io/d/stemcells
           VERSIONS_FILE: templates-resource/simple-depls/root-deployment.yml
   - name: upload-stemcell-to-director
+    serial: true
     on_failure: *on_failure
     plan:
       - in_parallel: