From ece31ee2d90ca327ce9f233a41f362ea9e4ada68 Mon Sep 17 00:00:00 2001 From: Sruti Parthiban Date: Wed, 18 Aug 2021 18:38:08 -0700 Subject: [PATCH 1/2] Updates to gradle build file Signed-off-by: Sruti Parthiban --- build.gradle | 86 ++++++++++++++++++++++++++++++---------------------- 1 file changed, 50 insertions(+), 36 deletions(-) diff --git a/build.gradle b/build.gradle index 64b58417..2564b188 100644 --- a/build.gradle +++ b/build.gradle @@ -9,7 +9,7 @@ * GitHub history for details. */ - /* +/* * Copyright <2019> Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). @@ -29,7 +29,8 @@ buildscript { ext { opensearch_version = System.getProperty("opensearch.version", "1.0.0") } - // This isn't applying from repositories.gradle so repeating it here + + // Used to resolve build file dependencies repositories { mavenCentral() mavenLocal() @@ -69,6 +70,15 @@ spotbugsTest { ext { opensearchVersion = '1.0.0' isSnapshot = "true" == System.getProperty("build.snapshot", "true") + + // Name of the default git branch. This is used for checking out the RCA repository + gitDefaultBranch = 'main' + + // If true, then the build will clone the RCA Project into $rcaProjectDir + cloneRcaProject = "true" == System.getProperty("cloneRcaProject", "true") + + // By default we will look for RCA in a peer directory + rcaProjectDir = findProperty("rcaProjectDir") ?: "../performance-analyzer-rca" } group = "org.opensearch" @@ -79,12 +89,8 @@ if (isSnapshot) { apply plugin: 'opensearch.opensearchplugin' -ext { - projectSubstitutions = [:] - licenseFile = rootProject.file('LICENSE.txt') - noticeFile = rootProject.file('NOTICE.txt') - gitDefaultBranch = 'main' -} +licenseFile = rootProject.file('LICENSE.txt') +noticeFile = rootProject.file('NOTICE.txt') spotless { java { @@ -107,8 +113,6 @@ test { licenseHeaders.enabled = true validateNebulaPom.enabled = false - - loggerUsageCheck.enabled = false opensearchplugin { @@ -129,7 +133,7 @@ compileJava { } javadoc { - options.addStringOption("-add-exports", "jdk.attach/sun.tools.attach=ALL-UNNAMED") + options.addStringOption("-add-exports", "jdk.attach/sun.tools.attach=ALL-UNNAMED") } project.afterEvaluate { @@ -152,7 +156,6 @@ repositories { jcenter() } - configurations { includeJars } @@ -207,7 +210,6 @@ jacocoTestCoverageVerification { } } - // to run coverage verification during the build (and fail when appropriate) check.dependsOn jacocoTestCoverageVerification @@ -230,13 +232,13 @@ dependencies { } configurations.all { - resolutionStrategy { - force 'junit:junit:4.13.1' - force "com.fasterxml.jackson.core:jackson-annotations:${jacksonVersion}" - force "com.fasterxml.jackson.core:jackson-core:${jacksonVersion}" - force "com.fasterxml.jackson.core:jackson-databind:${jacksonVersion}" - force "com.fasterxml.jackson.module:jackson-module-paranamer:${jacksonVersion}" - } + resolutionStrategy { + force 'junit:junit:4.13.1' + force "com.fasterxml.jackson.core:jackson-annotations:${jacksonVersion}" + force "com.fasterxml.jackson.core:jackson-core:${jacksonVersion}" + force "com.fasterxml.jackson.core:jackson-databind:${jacksonVersion}" + force "com.fasterxml.jackson.module:jackson-module-paranamer:${jacksonVersion}" + } } compile ('com.google.guava:guava:28.2-jre') { @@ -297,48 +299,54 @@ gradle.startParameter.excludedTaskNames += [ "forbiddenApisMain", "forbiddenApisTest", "thirdPartyAudit", "testingConventions"] -import java.nio.file.Paths + import org.ajoberstar.gradle.git.tasks.GitClone import org.opensearch.gradle.test.RestIntegTestTask -String rcaDir -String rcaArtifactsDir - static def propEnabled(property) { return System.getProperty(property) != null && System.getProperty(property).toLowerCase().equals("true") } // The following Gradle tasks are used to create a PA/RCA enabled OpenSearch cluster // Pass the -Dtests.enableIT property to Gradle to run ITs + /** * cloneGitRepo clones the performance-analyzer-rca repo if the -Dtests.enableIT=true flag is passed * to the Gradle JVM */ task cloneGitRepo(type: GitClone) { - rcaDir = Paths.get(getProject().getBuildDir().toString(), "performance-analyzer-rca").toString() - def destination = file(rcaDir) + def destination = file(rcaProjectDir) uri = "https://github.com/opensearch-project/performance-analyzer-rca.git" branch = gitDefaultBranch destinationPath = destination bare = false - enabled = !destination.exists() // to clone only once + enabled = cloneRcaProject && !destination.exists() // to clone only once + + doFirst { + logger.info("Cloning performance-analyzer-rca into ${rcaProjectDir}") + } } task buildRca() { dependsOn(cloneGitRepo) + + doFirst { + logger.info("Building performance-analyzer-rca project in ${rcaProjectDir}") + } + doLast { exec { - workingDir("$rcaDir") + workingDir("$rcaProjectDir") commandLine './gradlew', 'build', '-x', 'test' } exec { - workingDir("$rcaDir") + workingDir("$rcaProjectDir") commandLine './gradlew', 'publishToMavenLocal' } exec { def licenseDir = "$projectDir/licenses" workingDir("$licenseDir") - commandLine 'rm', "performanceanalyzer-rca-1.0.0.0.jar.sha1" + commandLine 'rm', "-f", "performanceanalyzer-rca-1.0.0.0.jar.sha1" } exec { workingDir("$projectDir") @@ -347,16 +355,23 @@ task buildRca() { } } +// This value is set by the unpackRca task +def rcaArtifactsDir + task unpackRca(type: Copy) { dependsOn(buildRca) - from(zipTree("$rcaDir/build/distributions/performance-analyzer-rca.zip")) { + from(zipTree("$rcaProjectDir/build/distributions/performance-analyzer-rca.zip")) { + } + into "$rcaArtifactsDir/build/distributions" + rcaArtifactsDir = "$rcaProjectDir/build/distributions/performance-analyzer-rca/" + + doLast { + logger.info("Unpacked performance-analyzer-rca artifacts into ${rcaProjectDir}") } - into "$rcaDir/build/distributions" - rcaArtifactsDir = "$rcaDir/build/distributions/performance-analyzer-rca/" } bundlePlugin { - dependsOn 'cloneGitRepo' + dependsOn 'unpackRca' from("$rcaArtifactsDir/pa_config") { into "pa_config" } @@ -399,7 +414,7 @@ task setupOpenSearchCluster() { } doLast { exec { - workingDir(rcaDir) + workingDir(rcaProjectDir) commandLine './gradlew', 'enableRca' } sleep(30000) @@ -527,4 +542,3 @@ afterEvaluate { } bundlePlugin.mustRunAfter unpackRca -build.dependsOn unpackRca From 4bd15eb081a1d1b37ffe8730dda95a0caf17db5e Mon Sep 17 00:00:00 2001 From: Sruti Parthiban Date: Thu, 19 Aug 2021 15:14:08 -0700 Subject: [PATCH 2/2] Add ability to specify RCA branch Signed-off-by: Sruti Parthiban --- .github/workflows/cd.yml | 19 +------------------ .github/workflows/gradle.yml | 23 +++-------------------- build.gradle | 12 ++++++------ 3 files changed, 10 insertions(+), 44 deletions(-) diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml index 157acb5a..a4aea7f2 100644 --- a/.github/workflows/cd.yml +++ b/.github/workflows/cd.yml @@ -10,11 +10,6 @@ jobs: runs-on: ubuntu-latest steps: - - name: Checkout RCA package - uses: actions/checkout@v2 - with: - repository: opensearch-project/performance-analyzer-rca - path: ./tmp/rca - name: Checkout Performance Analyzer package uses: actions/checkout@v2 with: @@ -23,22 +18,10 @@ jobs: uses: actions/setup-java@v1 with: java-version: 1.12 - - name: Build RCA with Gradle - working-directory: ./tmp/rca - run: ./gradlew build -x test - - name: Publish RCA jar to maven local - working-directory: ./tmp/rca - run: ./gradlew publishToMavenLocal - - name: Build PA gradle using the new RCA jar - working-directory: ./tmp/pa - run: rm -f licenses/performanceanalyzer-rca-1.0.0.0.jar.sha1 - - name: Update SHA - working-directory: ./tmp/pa - run: ./gradlew updateShas - name: Build Artifacts working-directory: ./tmp/pa run: | - ./gradlew build buildDeb buildRpm --refresh-dependencies -Dbuild.snapshot=false -x javadoc + ./gradlew build buildDeb buildRpm --refresh-dependencies -Dbuild.snapshot=false -Dperformance-analyzer-rca.build=true -Dperformance-analyzer-rca.branch=main -x javadoc mkdir artifacts artifact=`ls build/distributions/*.zip` rpm_artifact=`ls build/distributions/*.rpm` diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml index f9f5581f..59b9e6e7 100644 --- a/.github/workflows/gradle.yml +++ b/.github/workflows/gradle.yml @@ -9,15 +9,10 @@ on: - "*" jobs: - build_rca_pkg: + build_pa_pkg: runs-on: [ubuntu-latest] - name: Building RCA package + name: Building PA package steps: - - name: Checkout RCA package - uses: actions/checkout@v2 - with: - repository: opensearch-project/performance-analyzer-rca - path: ./tmp/rca - name: Checkout Performance Analyzer package uses: actions/checkout@v2 with: @@ -30,12 +25,6 @@ jobs: uses: actions/setup-java@v1 with: java-version: 1.12 - - name: Build RCA with Gradle - working-directory: ./tmp/rca - run: ./gradlew build -x test - - name: Publish RCA jar to maven local - working-directory: ./tmp/rca - run: ./gradlew publishToMavenLocal # dependencies: OpenSearch - name: Checkout OpenSearch uses: actions/checkout@v2 @@ -46,12 +35,6 @@ jobs: - name: Build OpenSearch working-directory: ./OpenSearch run: ./gradlew publishToMavenLocal -Dbuild.snapshot=false - - name: Build PA gradle using the new RCA jar - working-directory: ./tmp/pa - run: rm -f licenses/performanceanalyzer-rca-1.0.0.0.jar.sha1 - - name: Update SHA - working-directory: ./tmp/pa - run: ./gradlew updateShas # Explicitly set the docker-compose program path so that our build scripts in RCA can run the program # This is necessary because of the Github Actions environment and the workingDir of the Gradle environment - name: Set docker-compose path @@ -61,7 +44,7 @@ jobs: run: sudo sysctl -w vm.max_map_count=262144 - name: Build PA and run Unit Tests working-directory: ./tmp/pa - run: ./gradlew build + run: ./gradlew build -Dperformance-analyzer-rca.build=true -Dperformance-analyzer-rca.branch=main - name: Generate Jacoco coverage report working-directory: ./tmp/pa run: ./gradlew jacocoTestReport diff --git a/build.gradle b/build.gradle index 2564b188..307af6f9 100644 --- a/build.gradle +++ b/build.gradle @@ -71,14 +71,14 @@ ext { opensearchVersion = '1.0.0' isSnapshot = "true" == System.getProperty("build.snapshot", "true") - // Name of the default git branch. This is used for checking out the RCA repository - gitDefaultBranch = 'main' + // The RCA branch that will be built. Default value is main. + rcaProjectBranch = findProperty("performance-analyzer-rca.branch") ?: "main" // If true, then the build will clone the RCA Project into $rcaProjectDir - cloneRcaProject = "true" == System.getProperty("cloneRcaProject", "true") + cloneRcaProject = "true" == System.getProperty("performance-analyzer-rca.build", "true") // By default we will look for RCA in a peer directory - rcaProjectDir = findProperty("rcaProjectDir") ?: "../performance-analyzer-rca" + rcaProjectDir = findProperty("performance-analyzer-rca.path") ?: "../performance-analyzer-rca" } group = "org.opensearch" @@ -317,7 +317,7 @@ static def propEnabled(property) { task cloneGitRepo(type: GitClone) { def destination = file(rcaProjectDir) uri = "https://github.com/opensearch-project/performance-analyzer-rca.git" - branch = gitDefaultBranch + branch = rcaProjectBranch destinationPath = destination bare = false enabled = cloneRcaProject && !destination.exists() // to clone only once @@ -362,7 +362,7 @@ task unpackRca(type: Copy) { dependsOn(buildRca) from(zipTree("$rcaProjectDir/build/distributions/performance-analyzer-rca.zip")) { } - into "$rcaArtifactsDir/build/distributions" + into "$rcaProjectDir/build/distributions" rcaArtifactsDir = "$rcaProjectDir/build/distributions/performance-analyzer-rca/" doLast {