From 885b136ad40b3b6c729a38df0d2cccbc9913eaf5 Mon Sep 17 00:00:00 2001 From: Olivier ORAND Date: Thu, 27 Apr 2023 08:35:56 +0000 Subject: [PATCH] pipeline(bosh): fine grain management of compiled download url With jammy stemcell we encounter incompatible bosh releases, so we want to keep the old stemcell and still have minimal management by COA, without adding multi stemcell support. --- .../pipelines/template/bosh-pipeline.yml.erb | 22 +++++++++++-------- .../config_repository/private-config.yml | 4 +++- .../template_repository/shared-config.yml | 10 +++++---- 3 files changed, 22 insertions(+), 14 deletions(-) diff --git a/concourse/pipelines/template/bosh-pipeline.yml.erb b/concourse/pipelines/template/bosh-pipeline.yml.erb index 422e7207..c80bc531 100755 --- a/concourse/pipelines/template/bosh-pipeline.yml.erb +++ b/concourse/pipelines/template/bosh-pipeline.yml.erb @@ -10,7 +10,6 @@ terraform_config_path = all_ci_deployments.dig(root_deployment_name, PipelineHelpers::TERRAFORM_CONFIG_DIRNAME_KEY, 'state_file_path') terraform_config_path_value = ", \"#{terraform_config_path}\"" end - disabled_deployments = all_dependencies.select do |_, deployment_info| deployment_info['status'] == 'disabled' end @@ -32,14 +31,9 @@ offline_boshreleases_enabled = config&.dig('offline-mode', 'boshreleases') && true lock_releases_enabled = config&.fetch('lock-releases', false) && true precompile_pipeline_enabled = config&.fetch('precompile-mode', true) && true - precompile_excluded_deployments = config&.dig(root_deployment_name,'precompile','excluded-deployments') || [] - download_server_url = if offline_boshreleases_enabled - # we expect s3-endpoint to end with '/' - "((s3-#{precompile_pipeline_enabled ? "compiled-release" : "br" }-endpoint))((s3-#{ precompile_pipeline_enabled ? "compiled-release" : "br" }-bucket))" - else - "https://bosh.io/d/github.com/" - end + disabled_compiled_download_url_deployments = config&.dig(root_deployment_name,'precompile','disabled_compiled_download_url_deployments') || [] + disabled_compiled_download_url_deployments_prefix = config&.dig(root_deployment_name,'precompile','disabled_compiled_download_url_deployments_prefix') || [] jobs = Hash.new {|h,k| h[k]=[]} @@ -899,9 +893,19 @@ jobs: VERSIONS_FILE: templates-resource/<%= root_deployment_name %>/root-deployment.yml MANIFEST_YAML_FILE: manifest-dir/interpolated-<%= name %>.yml DEPLOYMENT_NAME: <%= name %> + <% + deployment_precompile_enabled = precompile_pipeline_enabled && !disabled_compiled_download_url_deployments.include?(name) + disabled_compiled_download_url_deployments_prefix.each { |prefix | deployment_precompile_enabled = deployment_precompile_enabled && !name.start_with?(prefix) } + download_server_url = if offline_boshreleases_enabled + # we expect s3-endpoint to end with '/' + "((s3-#{deployment_precompile_enabled ? "compiled-release" : "br" }-endpoint))((s3-#{ deployment_precompile_enabled ? "compiled-release" : "br" }-bucket))" + else + "https://bosh.io/d/github.com/" + end + %> DOWNLOAD_SERVER_URL: <%= download_server_url %> OFFLINE_MODE_ENABLED: <%= offline_boshreleases_enabled %> - PRECOMPILE_MODE_ENABLED: <%= precompile_pipeline_enabled %> + PRECOMPILE_MODE_ENABLED: <%= deployment_precompile_enabled %> LOCK_RELEASES: <%= lock_releases_enabled %> - put: <%= name %>-deployment attempts: <%= concourse_retry[:bosh_push] %> diff --git a/docs/reference_dataset/config_repository/private-config.yml b/docs/reference_dataset/config_repository/private-config.yml index bb2d26c4..a575cc40 100644 --- a/docs/reference_dataset/config_repository/private-config.yml +++ b/docs/reference_dataset/config_repository/private-config.yml @@ -53,4 +53,6 @@ # git: # shallow-clone-depth: 1 # Default: 0, ie disabled # precompile: -# excluded_deployments: [] # Default: [] # List deployments to exclude from precompile. As precompile does not use secrets repository to determine enabled deployments, it might be convenient to exclude a deployment not enabled in secrets. \ No newline at end of file +# excluded_deployments: [] # Default: [] # List deployments to exclude from precompile. As precompile does not use secrets repository to determine enabled deployments, it might be convenient to exclude a deployment not enabled in secrets. +# disabled_compiled_download_url_deployments: [] # Default: [] - List of deployments to exclude from bosh manifest precompile patching +# disabled_compiled_download_url_deployments_prefix: [] # Default: [] - List of deployments prefix to exclude from bosh manifest precompile patching diff --git a/docs/reference_dataset/template_repository/shared-config.yml b/docs/reference_dataset/template_repository/shared-config.yml index f01d04fa..8290aad9 100644 --- a/docs/reference_dataset/template_repository/shared-config.yml +++ b/docs/reference_dataset/template_repository/shared-config.yml @@ -1,8 +1,8 @@ --- -# When offline-mode is enabled, bytes are retrieved from an S3 compliant location, otherwise download from bosh-io +# If offline mode is enabled, bytes are fetched from an S3-compatible location, otherwise they are downloaded from bosh-io. offline-mode: - boshreleases: false # Choose where boshreleases are downloaded from. Default: false - stemcells: false # Choose where stemcells are downloaded from. Default: true + boshreleases: false # Enable or disable boshreleases offline mode, thus its download location. Default: false (disabled) + stemcells: false # Enable or disable stemcells offline mode, thus its download location. Default: true (enabled) lock-releases: false # Default: false. See https://bosh.io/docs/locking-compiled-releases/ # Shared pipelines configuration @@ -59,4 +59,6 @@ default: # parallel_execution_limit: 10 # Default: -1, ie unlimited # serial_group_naming_strategy: SerialGroupMd5NamingStrategy #Default: SerialGroupRoundRobinNamingStrategy # precompile: -# excluded_deployments: [] # Default: [] # List deployments to exclude from precompile. As precompile does not use secrets repository to determine enabled deployments, it might be convenient to exclude a deployment not enabled in secrets. \ No newline at end of file +# excluded_deployments: [] # Default: [] # List of deployments to exclude from precompile. Since precompile does not use secrets repository to determine enabled deployments, it may be useful to exclude a deployment that is not enabled in secrets. +# disabled_compiled_download_url_deployments: [] # Default: [] - List of deployments to exclude from bosh manifest precompile patching +# disabled_compiled_download_url_deployments_prefix: [] # Default: [] - List of deployments prefix to exclude from bosh manifest precompile patching \ No newline at end of file