From 139cb5a961d8de68b8e02c45cc47f5289a3623af Mon Sep 17 00:00:00 2001 From: Rene Groeschke Date: Tue, 18 Feb 2025 14:51:58 +0100 Subject: [PATCH] [7.17] [Gradle] Fix build finished hooks on ci when using configuration cache (#116888) (#122744) * [Gradle] Fix build finished hooks on ci when using configuration cache (#116888) Fixes two incompatibilities with Gradle configuration cache in our build scan build finished hook: referencing static methods from build script referencing gradle object from closure (cherry picked from commit 929d39820a657b423911ccfe9f974cffac5a9e34) # Conflicts: # build-tools-internal/src/main/groovy/elasticsearch.build-scan.gradle * Update elasticsearch.build-scan.gradle fix license * Fix license header --- .../groovy/elasticsearch.build-scan.gradle | 249 +++++++----------- .../gradle/internal/util/CiUtils.java | 17 ++ .../gradle/internal/util/CiUtils.java.new | 38 +++ 3 files changed, 157 insertions(+), 147 deletions(-) create mode 100644 build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/util/CiUtils.java create mode 100644 build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/util/CiUtils.java.new diff --git a/build-tools-internal/src/main/groovy/elasticsearch.build-scan.gradle b/build-tools-internal/src/main/groovy/elasticsearch.build-scan.gradle index 9dfa5f5570324..527dae5ad1f83 100644 --- a/build-tools-internal/src/main/groovy/elasticsearch.build-scan.gradle +++ b/build-tools-internal/src/main/groovy/elasticsearch.build-scan.gradle @@ -10,173 +10,128 @@ import java.lang.management.ManagementFactory; import java.time.LocalDateTime; import org.elasticsearch.gradle.Architecture -import org.elasticsearch.gradle.OS import org.elasticsearch.gradle.internal.info.BuildParams -import org.gradle.initialization.BuildRequestMetaData +import org.elasticsearch.gradle.OS +import static org.elasticsearch.gradle.internal.util.CiUtils.safeName -buildScan { - URL jenkinsUrl = System.getenv('JENKINS_URL') ? new URL(System.getenv('JENKINS_URL')) : null - String buildKiteUrl = System.getenv('BUILDKITE_BUILD_URL') ? System.getenv('BUILDKITE_BUILD_URL') : null +import java.lang.management.ManagementFactory +import java.time.LocalDateTime - // Automatically publish scans from Elasticsearch CI - if (jenkinsUrl?.host?.endsWith('elastic.co') || jenkinsUrl?.host?.endsWith('elastic.dev') || System.getenv('BUILDKITE') == 'true') { - publishAlways() - buildScan.server = 'https://gradle-enterprise.elastic.co' - } +// Resolving this early to avoid issues with the build scan plugin in combination with the configuration cache usage +def taskNames = gradle.startParameter.taskNames.join(' ') + +develocity { - background { - tag OS.current().name() - tag Architecture.current().name() + buildScan { - // Tag if this build is run in FIPS mode - if (BuildParams.inFipsJvm) { - tag 'FIPS' + def onCI = System.getenv('CI') ? Boolean.parseBoolean(System.getenv('CI')) : false + + // Disable async upload in CI to ensure scan upload completes before CI agent is terminated + uploadInBackground = onCI == false + + // Automatically publish scans from Elasticsearch CI + if (onCI) { + publishing.onlyIf { true } + server = 'https://gradle-enterprise.elastic.co' + } else if( server.isPresent() == false) { + publishing.onlyIf { false } } - // Jenkins-specific build scan metadata - if (jenkinsUrl) { - // Disable async upload in CI to ensure scan upload completes before CI agent is terminated - uploadInBackground = false - - String buildNumber = System.getenv('BUILD_NUMBER') - String buildUrl = System.getenv('BUILD_URL') - String jobName = System.getenv('JOB_NAME') - String nodeName = System.getenv('NODE_NAME') - String jobBranch = System.getenv('ghprbTargetBranch') ?: System.getenv('JOB_BRANCH') - - // Link to Jenkins worker logs and system metrics - if (nodeName) { - link 'System logs', "https://ci-stats.elastic.co/app/infra#/logs?&logFilter=(expression:'host.name:${nodeName}',kind:kuery)" - buildFinished { - link 'System metrics', "https://ci-stats.elastic.co/app/metrics/detail/host/${nodeName}" - } - } + def fips = BuildParams.inFipsJvm + def gitRevision = BuildParams.gitRevision - // Parse job name in the case of matrix builds - // Matrix job names come in the form of "base-job-name/matrix_param1=value1,matrix_param2=value2" - def splitJobName = jobName.split('/') - if (splitJobName.length > 1 && splitJobName.last() ==~ /^([a-zA-Z0-9_\-]+=[a-zA-Z0-9_\-&\.]+,?)+$/) { - def baseJobName = splitJobName.dropRight(1).join('/') - tag baseJobName - tag splitJobName.last() - value 'Job Name', baseJobName - def matrixParams = splitJobName.last().split(',') - matrixParams.collect { it.split('=') }.each { param -> - value "MATRIX_${param[0].toUpperCase()}", param[1] - } - } else { - tag jobName - value 'Job Name', jobName - } + background { + tag OS.current().name() + tag Architecture.current().name() - tag 'CI' - link 'CI Build', buildUrl - link 'GCP Upload', "https://console.cloud.google.com/storage/browser/_details/elasticsearch-ci-artifacts/jobs/${URLEncoder.encode(jobName, "UTF-8")}/build/${buildNumber}.tar.bz2" - value 'Job Number', buildNumber - if (jobBranch) { - tag jobBranch - value 'Git Branch', jobBranch + // Tag if this build is run in FIPS mode + if (fips) { + tag 'FIPS' } - System.getenv().getOrDefault('NODE_LABELS', '').split(' ').each { - value 'Jenkins Worker Label', it - } + if (onCI) { //Buildkite-specific build scan metadata + String buildKiteUrl = System.getenv('BUILDKITE_BUILD_URL') + def branch = System.getenv('BUILDKITE_PULL_REQUEST_BASE_BRANCH') ?: System.getenv('BUILDKITE_BRANCH') + def repoMatcher = System.getenv('BUILDKITE_REPO') =~ /(https:\/\/github\.com\/|git@github\.com:)(\S+)\.git/ + def repository = repoMatcher.matches() ? repoMatcher.group(2) : "" + def jobLabel = System.getenv('BUILDKITE_LABEL') ?: '' + def jobName = safeName(jobLabel) - // Add SCM information - def isPrBuild = System.getenv('ROOT_BUILD_CAUSE_GHPRBCAUSE') != null - if (isPrBuild) { - value 'Git Commit ID', System.getenv('ghprbActualCommit') - tag "pr/${System.getenv('ghprbPullId')}" - tag 'pull-request' - link 'Source', "https://github.com/elastic/elasticsearch/tree/${System.getenv('ghprbActualCommit')}" - link 'Pull Request', System.getenv('ghprbPullLink') - } else { - value 'Git Commit ID', BuildParams.gitRevision - link 'Source', "https://github.com/elastic/elasticsearch/tree/${BuildParams.gitRevision}" - } - } else if (buildKiteUrl) { //Buildkite-specific build scan metadata - // Disable async upload in CI to ensure scan upload completes before CI agent is terminated - uploadInBackground = false - - def branch = System.getenv('BUILDKITE_PULL_REQUEST_BASE_BRANCH') ?: System.getenv('BUILDKITE_BRANCH') - def repoMatcher = System.getenv('BUILDKITE_REPO') =~ /(https:\/\/github\.com\/|git@github\.com:)(\S+)\.git/ - def repository = repoMatcher.matches() ? repoMatcher.group(2) : "" - def jobLabel = System.getenv('BUILDKITE_LABEL') ?: '' - def jobName = safeName(jobLabel) - - tag 'CI' - link 'CI Build', "${buildKiteUrl}#${System.getenv('BUILDKITE_JOB_ID')}" - value 'Job Number', System.getenv('BUILDKITE_BUILD_NUMBER') - value 'Build ID', System.getenv('BUILDKITE_BUILD_ID') - value 'Job ID', System.getenv('BUILDKITE_JOB_ID') - - value 'Pipeline', System.getenv('BUILDKITE_PIPELINE_SLUG') - tag System.getenv('BUILDKITE_PIPELINE_SLUG') - - value 'Job Name', jobName - tag jobName - if (jobLabel.contains("/")) { - jobLabel.split("/").collect {safeName(it) }.each {matrix -> - tag matrix + tag 'CI' + link 'CI Build', "${buildKiteUrl}#${System.getenv('BUILDKITE_JOB_ID')}" + value 'Job Number', System.getenv('BUILDKITE_BUILD_NUMBER') + value 'Build ID', System.getenv('BUILDKITE_BUILD_ID') + value 'Job ID', System.getenv('BUILDKITE_JOB_ID') + + value 'Pipeline', System.getenv('BUILDKITE_PIPELINE_SLUG') + tag System.getenv('BUILDKITE_PIPELINE_SLUG') + + value 'Job Name', jobName + tag jobName + if (jobLabel.contains("/")) { + jobLabel.split("/").collect { safeName(it) }.each { matrix -> + tag matrix + } } - } - def uptime = ManagementFactory.getRuntimeMXBean().getUptime() / 1000; - def metricsStartTime = LocalDateTime.now().minusSeconds(uptime.longValue()).minusMinutes(15).toString() - def metricsEndTime = LocalDateTime.now().plusMinutes(15).toString() + def uptime = ManagementFactory.getRuntimeMXBean().getUptime() / 1000; + def metricsStartTime = LocalDateTime.now().minusSeconds(uptime.longValue()).minusMinutes(15).toString() + def metricsEndTime = LocalDateTime.now().plusMinutes(15).toString() - link 'Agent Metrics', "https://es-buildkite-agents.elastic.dev/app/metrics/detail/host/${System.getenv('BUILDKITE_AGENT_NAME')}?_a=(time:(from:%27${metricsStartTime}Z%27,interval:%3E%3D1m,to:%27${metricsEndTime}Z%27))" - link 'Agent Logs', "https://es-buildkite-agents.elastic.dev/app/logs/stream?logFilter=(filters:!(),query:(language:kuery,query:%27host.name:%20${System.getenv('BUILDKITE_AGENT_NAME')}%27),timeRange:(from:%27${metricsStartTime}Z%27,to:%27${metricsEndTime}Z%27))" + link 'Agent Metrics', + "https://es-buildkite-agents.elastic.dev/app/metrics/detail/host/${System.getenv('BUILDKITE_AGENT_NAME')}?_a=(time:(from:%27${metricsStartTime}Z%27,interval:%3E%3D1m,to:%27${metricsEndTime}Z%27))" + link 'Agent Logs', + "https://es-buildkite-agents.elastic.dev/app/logs/stream?logFilter=(filters:!(),query:(language:kuery,query:%27host.name:%20${System.getenv('BUILDKITE_AGENT_NAME')}%27),timeRange:(from:%27${metricsStartTime}Z%27,to:%27${metricsEndTime}Z%27))" - if (branch) { - tag branch - value 'Git Branch', branch - } + if (branch) { + tag branch + value 'Git Branch', branch + } - // Add SCM information - def prId = System.getenv('BUILDKITE_PULL_REQUEST') - if (prId != 'false') { - def prBaseUrl = (System.getenv('BUILDKITE_PULL_REQUEST_REPO') - ".git").replaceFirst("git://", "https://") - value 'Git Commit ID', System.getenv('BUILDKITE_COMMIT') - tag "pr/${prId}" - tag 'pull-request' - link 'Source', "${prBaseUrl}/tree/${System.getenv('BUILDKITE_COMMIT')}" - link 'Pull Request', "https://github.com/${repository}/pull/${prId}" - } else { - value 'Git Commit ID', BuildParams.gitRevision - link 'Source', "https://github.com/${repository}/tree/${BuildParams.gitRevision}" - } + // Add SCM information + def prId = System.getenv('BUILDKITE_PULL_REQUEST') + if (prId != 'false') { + def prBaseUrl = (System.getenv('BUILDKITE_PULL_REQUEST_REPO') - ".git").replaceFirst("git://", "https://") + value 'Git Commit ID', System.getenv('BUILDKITE_COMMIT') + tag "pr/${prId}" + tag 'pull-request' + link 'Source', "${prBaseUrl}/tree/${System.getenv('BUILDKITE_COMMIT')}" + link 'Pull Request', "https://github.com/${repository}/pull/${prId}" + } else { + value 'Git Commit ID', gitRevision + link 'Source', "https://github.com/${repository}/tree/${gitRevision}" + } - buildFinished { result -> - buildScanPublished { scan -> - // Attach build scan link as build metadata - // See: https://buildkite.com/docs/pipelines/build-meta-data - new ProcessBuilder('buildkite-agent', 'meta-data', 'set', "build-scan-${System.getenv('BUILDKITE_JOB_ID')}", "${scan.buildScanUri}") - .start() - .waitFor() - - // Add a build annotation - // See: https://buildkite.com/docs/agent/v3/cli-annotate - def body = """
${System.getenv('BUILDKITE_LABEL')} :gradle: ${result.failure ? 'failed' : 'successful'} build: gradle ${gradle.startParameter.taskNames.join(' ')}
""" - def process = [ - 'buildkite-agent', - 'annotate', - '--context', - result.failure ? 'gradle-build-scans-failed' : 'gradle-build-scans', - '--append', - '--style', - result.failure ? 'error' : 'info' - ].execute() - process.withWriter { it.write(body) } // passing the body in as an argument has issues on Windows, so let's use stdin of the process instead - process.waitFor() + buildFinished { result -> + + buildScanPublished { scan + -> + // Attach build scan link as build metadata + // See: https://buildkite.com/docs/pipelines/build-meta-data + new ProcessBuilder('buildkite-agent', 'meta-data', 'set', "build-scan-${System.getenv('BUILDKITE_JOB_ID')}", "${scan.buildScanUri}") + .start() + .waitFor() + + // Add a build annotation + // See: https://buildkite.com/docs/agent/v3/cli-annotate + def body = """
${System.getenv('BUILDKITE_LABEL')} :gradle: ${result.failures ? 'failed' : 'successful'} build: gradle ${taskNames}
""" + def process = [ + 'buildkite-agent', + 'annotate', + '--context', + result.failures ? 'gradle-build-scans-failed' : 'gradle-build-scans', + '--append', + '--style', + result.failures ? 'error' : 'info' + ].execute() + process.withWriter { it.write(body) } + // passing the body in as an argument has issues on Windows, so let's use stdin of the process instead + process.waitFor() + } } + } else { + tag 'LOCAL' } - } else { - tag 'LOCAL' } } } - -static def safeName(String string) { - return string.replaceAll(/[^a-zA-Z0-9_\-\.]+/, ' ').trim().replaceAll(' ', '_').toLowerCase() -} diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/util/CiUtils.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/util/CiUtils.java new file mode 100644 index 0000000000000..9c56b6c1d9854 --- /dev/null +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/util/CiUtils.java @@ -0,0 +1,17 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.gradle.internal.util; + +public class CiUtils { + + static String safeName(String input) { + return input.replaceAll("[^a-zA-Z0-9_\\-\\.]+", " ").trim().replaceAll(" ", "_").toLowerCase(); + } + +} diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/util/CiUtils.java.new b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/util/CiUtils.java.new new file mode 100644 index 0000000000000..6956ff3ac6abc --- /dev/null +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/util/CiUtils.java.new @@ -0,0 +1,38 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.gradle.internal.util; + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + + +public class CiUtils { + + static String safeName(String input) { + return input.replaceAll("[^a-zA-Z0-9_\\-\\.]+", " ").trim().replaceAll(" ", "_").toLowerCase(); + } + +}