concourse(bosh-precompile): ensure stemcell upload

As we detect bosh releases already compiled by other root deployments, we may encounter precompile pipeline without any bosh releases. To avoid empty pipeline generation, we need to tune bosh-precompile template to always upload stemcell when precompile is enabled.

concourse/pipelines/template/bosh-precompile-pipeline.yml.erb

@@ -5,7 +5,7 @@
   disabled_deployments = multi_root_dependencies.select do |root_depl_name, deployment_info|
     deployment_info['status'] == 'disabled'
   end
-
+  precompile_pipeline_enabled = config&.fetch('precompile-mode', true) && true
   precompile_excluded_deployments = {}
   root_deployments.each do |name|
     precompile_excluded_deployments[name] = config&.dig(name,'precompile','excluded-deployments') || []
@@ -94,7 +94,7 @@ resource_types:
     tag: v2.12.0
 
 resources:
-<% unless uniq_releases.empty? %>
+<% if precompile_pipeline_enabled || ! uniq_releases.empty? %>
   - name: failure-alert
     icon: slack
     type: slack-notification
@@ -229,10 +229,10 @@ resources:
 <% end %>
 
 jobs:
-<% if uniq_releases.empty? %>
+<% if uniq_releases.empty? && !precompile_pipeline_enabled %>
   - name: this-is-an-empty-pipeline
 <%else %>
-
+  <% if ! uniq_releases.empty? %>
   - name: push-boshreleases
     <% jobs['Utils'] << 'push-boshreleases' %>
     on_failure: *on_failure
@@ -337,6 +337,7 @@ jobs:
                   else
                     echo "No errors detected"
                   fi
+  <% end %>
   <% if offline_stemcells_enabled %>
   - name: upload-stemcell-to-s3
     <% jobs['Utils'] << 'upload-stemcell-to-s3' %>
@@ -621,7 +622,6 @@ jobs:
     <% end %>
     <% end %>
 
-  <% if filtered_deployments.any? %>
   - name: init-concourse-boshrelease-and-stemcell-for-<%= root_deployment_name %>
     <% jobs["Utils"] << "init-concourse-boshrelease-and-stemcell-for-#{root_deployment_name}" %>
     on_failure: *on_failure
@@ -679,8 +679,6 @@ jobs:
         FLY_USERNAME: ((concourse-<%= root_deployment_name %>-username))
         FLY_PASSWORD: "((concourse-<%= root_deployment_name %>-password))"
         FLY_TEAM: <%= current_team %>
-  <% end %>
-
 
 groups:
   - name: <%= root_deployment_name.downcase %>

spec/lib/template_processor/template_processor_for_bosh_precompile_pipeline_spec.rb

@@ -174,6 +174,25 @@
       @processed_template = subject.process(@pipelines_dir + '/*')
     end
 
+    context 'when precompile is enabled without bosh deployment' do
+      let(:multi_root_dependencies) { {root_deployment_name => {} } }
+      let(:loaded_config) do
+        my_config_yaml = <<~YAML
+          offline-mode:
+            stemcells: true
+          precompile-mode: true
+        YAML
+        YAML.safe_load(my_config_yaml)
+      end
+      let(:expected_jobs) { %W[init-concourse-boshrelease-and-stemcell-for-#{root_deployment_name} upload-stemcell-to-director upload-stemcell-to-s3] }
+
+      it 'generates compile and export job for dependencies describe in disabled deployments' do
+        filtered_generated_jobs = generated_pipeline['jobs']&.flat_map { |job| job['name'] }
+        expect(filtered_generated_jobs).to match_array(expected_jobs)
+      end
+
+    end
+
     context 'when disabled deployments are presents' do
       let(:expected_compiled_exported_deployments) { %W[compile-and-export-bosh compile-and-export-route-registrar-boshrelease compile-and-export-haproxy-boshrelease compile-and-export-cf-routing-release] }
       let(:expected_uploaded_deployments) { %W[upload-compiled-bosh upload-compiled-route-registrar-boshrelease upload-compiled-haproxy-boshrelease upload-compiled-cf-routing-release] }

spec/lib/template_processor/template_processor_spec.rb

@@ -227,6 +227,7 @@
               boshreleases: true
               stemcells: true
               docker-images: false
+            precompile-mode: false
           YAML
           YAML.safe_load(my_config_yaml)
         end

spec/scripts/generate-depls/fixtures/references/empty-bosh-precompile.yml

@@ -23,5 +23,169 @@ resource_types:
     repository: cloudfoundry/bosh-deployment-resource
     tag: v2.12.0
 resources:
+  - name: failure-alert
+    icon: slack
+    type: slack-notification
+    source:
+      url: ((slack-webhook))
+      proxy: ((slack-proxy))
+      proxy_https_tunnel: ((slack-proxy-https-tunnel))
+      disable: ((slack-disable))
+  - name: secrets-full-writer
+    icon: source-pull
+    type: git
+    source:
+      uri: ((secrets-uri))
+      branch: ((secrets-branch))
+      skip_ssl_verification: true
+  - name: paas-templates-empty-depls-limited
+    icon: home-analytics
+    type: git
+    source:
+      uri: ((paas-templates-uri))
+      paths:
+        - ".gitmodules"
+        - "shared-config.yml"
+        - "meta-inf.yml"
+        - "empty-depls/template"
+        - "empty-depls/**/deployment-dependencies*.yml"
+        - "empty-depls/root-deployment.yml"
+      branch: ((paas-templates-precompile-branch))
+      skip_ssl_verification: true
+  - name: cf-ops-automation
+    type: git
+    icon: rocket
+    source:
+      uri: ((cf-ops-automation-uri))
+      branch: ((cf-ops-automation-branch))
+      tag_filter: ((cf-ops-automation-tag-filter))
+      skip_ssl_verification: true
+  - name: ((stemcell-main-name))
+    type: s3
+    icon: home-floor-l
+    source:
+      bucket: ((s3-stemcell-bucket))
+      region_name: ((s3-stemcell-region-name))
+      regexp: ((stemcell-name-prefix))((stemcell-main-name))/bosh-stemcell-(.*)-((stemcell-main-name)).tgz
+      access_key_id: ((s3-stemcell-access-key-id))
+      secret_access_key: ((s3-stemcell-secret-key))
+      endpoint: ((s3-stemcell-endpoint))
+      skip_ssl_verification: ((s3-stemcell-skip-ssl-verification))
+    version: { path: ((stemcell-name-prefix))((stemcell-main-name))/bosh-stemcell-((stemcell.version))-((stemcell-main-name)).tgz }
+  - name: secrets-empty-depls-trigger
+    icon: source-commit
+    type: git
+    source:
+      uri: ((secrets-uri))
+      paths: ["private-config.yml", "empty-depls", "shared"]
+      branch: ((secrets-branch))
+      skip_ssl_verification: true
+  - name: paas-templates-empty-depls-versions
+    icon: home-thermometer
+    type: git
+    source:
+      uri: ((paas-templates-uri))
+      paths: ["empty-depls/root-deployment.yml"]
+      branch: ((paas-templates-precompile-branch))
+      skip_ssl_verification: true
 jobs:
-  - name: this-is-an-empty-pipeline
+  - name: upload-stemcell-to-s3
+    on_failure: *on_failure
+    plan:
+      - in_parallel:
+          - get: cf-ops-automation
+            params: { submodules: none }
+            trigger: true
+          - get : secrets-full-writer
+            params: { submodules: none }
+          - get: paas-templates-empty-depls-limited
+            params: { submodules: none }
+            trigger: true
+      - task: upload-stemcells
+        attempts: 2
+        input_mapping: { templates-resource: paas-templates-empty-depls-limited }
+        file:  cf-ops-automation/concourse/tasks/s3_stemcells_upload/task.yml
+        params:
+          BUCKET: ((s3-stemcell-bucket))
+          REGION_NAME: ((s3-stemcell-region-name))
+          ACCESS_KEY_ID: ((s3-stemcell-access-key-id))
+          SECRET_ACCESS_KEY: ((s3-stemcell-secret-key))
+          ENDPOINT: ((s3-stemcell-endpoint))
+          SKIP_SSL_VERIFICATION: ((s3-stemcell-skip-ssl-verification))
+          STEMCELL_PREFIX: ((stemcell-name-prefix))
+          STEMCELL_MAIN_NAME: ((stemcell-main-name))
+          STEMCELL_BASE_LOCATION: https://bosh.io/d/stemcells
+          VERSIONS_FILE: templates-resource/empty-depls/root-deployment.yml
+  - name: upload-stemcell-to-director
+    on_failure: *on_failure
+    plan:
+      - in_parallel:
+          - get: cf-ops-automation
+            params: { submodules: none }
+            trigger: true
+          - get : secrets-full-writer
+            params: { submodules: none }
+          - get: ((stemcell-main-name))
+            trigger: true
+            attempts: 2
+      - task: upload-to-director
+        attempts: 2
+        input_mapping: { stemcell: ((stemcell-main-name)), config-resource: secrets-full-writer }
+        file:  cf-ops-automation/concourse/tasks/bosh_upload_stemcell/task.yml
+        params:
+          BOSH_CLIENT: ((bosh-username))
+          BOSH_CLIENT_SECRET: ((bosh-password))
+          BOSH_ENVIRONMENT: ((bosh-target))
+          BOSH_CA_CERT: config-resource/shared/certs/internal_paas-ca/server-ca.crt
+  - name: init-concourse-boshrelease-and-stemcell-for-empty-depls
+    on_failure: *on_failure
+    plan:
+    - in_parallel:
+        - get: cf-ops-automation
+          params: { submodules: none}
+          attempts: 2
+          #trigger: true
+        - get: secrets-empty-depls-trigger
+          params: { submodules: none}
+          attempts: 2
+        - get: paas-templates-empty-depls-versions
+          params: { submodules: none}
+          attempts: 2
+          trigger: true
+    - task: generate-empty-depls-flight-plan
+      output_mapping: {result-dir: init-empty-depls-plan}
+      config:
+        platform: linux
+        image_resource:
+          type: registry-image
+          source:
+            repository: orangecloudfoundry/curl-ssl
+            tag: daaa098ccf63a8460a8e88897faba9f8c3aeb79b
+        outputs:
+          - name: result-dir
+        run:
+          path: sh
+          args:
+          - -ec
+          - |
+            echo "check-resource -r $BUILD_PIPELINE_NAME/((stemcell-main-name)) --from path:((stemcell-name-prefix))((stemcell-main-name))/bosh-stemcell-((stemcell.version))-((stemcell-main-name)).tgz" | tee -a result-dir/flight-plan
+        params:
+          BUILD_PIPELINE_NAME: empty-depls-bosh-precompile-generated
+    - task: fly-into-concourse
+      input_mapping: {fly-cmd: init-empty-depls-plan}
+      output_mapping: {flight-report: concourse-empty-depls-init-report}
+      file: cf-ops-automation/concourse/tasks/fly_execute_commands.yml
+      params:
+        ATC_EXTERNAL_URL: ((concourse-empty-depls-target))
+        FLY_USERNAME: ((concourse-empty-depls-username))
+        FLY_PASSWORD: "((concourse-empty-depls-password))"
+        FLY_TEAM: empty-depls
+groups:
+  - name: empty-depls
+    jobs:
+      - "*"
+  - name: "utils"
+    jobs:
+      - init-concourse-boshrelease-and-stemcell-for-empty-depls
+      - upload-stemcell-to-director
+      - upload-stemcell-to-s3