From af89e4d9f01509af59cf0986eb48629dd3d67d82 Mon Sep 17 00:00:00 2001 From: dblock Date: Fri, 20 Aug 2021 18:46:41 +0000 Subject: [PATCH 1/3] Fix snapshot build using OpenSearch 1.1. Signed-off-by: dblock --- .github/workflows/gradle.yml | 8 ++++---- build.gradle | 18 +++++++++--------- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml index 59b9e6e7..2c3c8abf 100644 --- a/.github/workflows/gradle.yml +++ b/.github/workflows/gradle.yml @@ -31,10 +31,10 @@ jobs: with: repository: 'opensearch-project/OpenSearch' path: OpenSearch - ref: '1.0' + ref: '1.x' - name: Build OpenSearch working-directory: ./OpenSearch - run: ./gradlew publishToMavenLocal -Dbuild.snapshot=false + run: ./gradlew publishToMavenLocal -Dopensearch.version=1.1.0-SNAPSHOT # Explicitly set the docker-compose program path so that our build scripts in RCA can run the program # This is necessary because of the Github Actions environment and the workingDir of the Gradle environment - name: Set docker-compose path @@ -44,7 +44,7 @@ jobs: run: sudo sysctl -w vm.max_map_count=262144 - name: Build PA and run Unit Tests working-directory: ./tmp/pa - run: ./gradlew build -Dperformance-analyzer-rca.build=true -Dperformance-analyzer-rca.branch=main + run: ./gradlew build -Dperformance-analyzer-rca.build=true -Dperformance-analyzer-rca.branch=main -Dopensearch.version=1.1.0-SNAPSHOT - name: Generate Jacoco coverage report working-directory: ./tmp/pa run: ./gradlew jacocoTestReport @@ -55,4 +55,4 @@ jobs: run: bash <(curl -s https://codecov.io/bash) -f ./build/reports/jacoco/test/jacocoTestReport.xml - name: Run Integration Tests working-directory: ./tmp/pa - run: ./gradlew integTest -Dtests.enableIT -Dtests.useDockerCluster + run: ./gradlew integTest -Dtests.enableIT -Dtests.useDockerCluster -Dopensearch.version=1.1.0-SNAPSHOT diff --git a/build.gradle b/build.gradle index 1188d859..86a79c18 100644 --- a/build.gradle +++ b/build.gradle @@ -27,7 +27,7 @@ buildscript { ext { - opensearch_version = System.getProperty("opensearch.version", "1.0.0") + opensearch_version = System.getProperty("opensearch.version", "1.1.0-SNAPSHOT") } // Used to resolve build file dependencies @@ -68,7 +68,6 @@ spotbugsTest { } ext { - opensearchVersion = '1.0.0' isSnapshot = "true" == System.getProperty("build.snapshot", "true") // The RCA branch that will be built. Default value is main. @@ -82,7 +81,7 @@ ext { } group = "org.opensearch" -version = "${opensearchVersion}.0" +version = opensearch_version - '-SNAPSHOT' + '.0' if (isSnapshot) { version += "-SNAPSHOT" } @@ -248,7 +247,7 @@ dependencies { compile 'org.apache.commons:commons-lang3:3.9' compile 'org.bouncycastle:bcprov-jdk15on:1.68' compile 'org.bouncycastle:bcpkix-jdk15on:1.68' - compile 'org.opensearch:performanceanalyzer-rca:1.0.0.0' + compile "org.opensearch:performanceanalyzer-rca:${version}" compile "com.fasterxml.jackson.core:jackson-annotations:${jacksonVersion}" compile "com.fasterxml.jackson.core:jackson-databind:${jacksonVersion}" compile "com.fasterxml.jackson.module:jackson-module-paranamer:${jacksonVersion}" @@ -331,25 +330,26 @@ task buildRca() { dependsOn(cloneGitRepo) doFirst { - logger.info("Building performance-analyzer-rca project in ${rcaProjectDir}") + logger.info("Building performance-analyzer-rca project in ${rcaProjectDir} with -Dopensearch.version=${opensearch_version}") } doLast { exec { workingDir("$rcaProjectDir") - commandLine './gradlew', 'build', '-x', 'test' + commandLine './gradlew', 'build', '-x', 'test', "-Dopensearch.version=${opensearch_version}" } exec { workingDir("$rcaProjectDir") - commandLine './gradlew', 'publishToMavenLocal' + commandLine './gradlew', 'publishToMavenLocal', "-Dopensearch.version=${opensearch_version}" } exec { def licenseDir = "$projectDir/licenses" workingDir("$licenseDir") - commandLine 'rm', "-f", "performanceanalyzer-rca-1.0.0.0.jar.sha1" + commandLine 'rm', "-f", "performanceanalyzer-rca-${project.version}.jar.sha1" } } } + buildRca.finalizedBy updateShas // This value is set by the unpackRca task @@ -357,7 +357,7 @@ def rcaArtifactsDir task unpackRca(type: Copy) { dependsOn(buildRca) - from(zipTree("$rcaProjectDir/build/distributions/performance-analyzer-rca.zip")) { + from(zipTree("$rcaProjectDir/build/distributions/performance-analyzer-rca-${version}.zip")) { } into "$rcaProjectDir/build/distributions" rcaArtifactsDir = "$rcaProjectDir/build/distributions/performance-analyzer-rca/" From 099c81b0bcc9895af3819e9a49876d632db761f9 Mon Sep 17 00:00:00 2001 From: dblock Date: Fri, 20 Aug 2021 20:11:15 +0000 Subject: [PATCH 2/3] Allow for a different repo for RCA. Signed-off-by: dblock --- .github/workflows/gradle.yml | 36 +++++++++++++++++++----------------- build.gradle | 21 +++++++++++---------- 2 files changed, 30 insertions(+), 27 deletions(-) diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml index 2c3c8abf..6532b219 100644 --- a/.github/workflows/gradle.yml +++ b/.github/workflows/gradle.yml @@ -13,18 +13,10 @@ jobs: runs-on: [ubuntu-latest] name: Building PA package steps: - - name: Checkout Performance Analyzer package - uses: actions/checkout@v2 - with: - path: ./tmp/pa - # fetch the main branch to make it available for spotless ratcheting - - name: Fetch 'main' branch - working-directory: ./tmp/pa - run: git fetch --depth=1 origin main - - name: Set up JDK 1.12 + - name: Set up JDK uses: actions/setup-java@v1 with: - java-version: 1.12 + java-version: 14 # dependencies: OpenSearch - name: Checkout OpenSearch uses: actions/checkout@v2 @@ -34,7 +26,13 @@ jobs: ref: '1.x' - name: Build OpenSearch working-directory: ./OpenSearch - run: ./gradlew publishToMavenLocal -Dopensearch.version=1.1.0-SNAPSHOT + run: ./gradlew publishToMavenLocal + + # Performance Analyzer in ./tmp/performance-analyzer + - name: Checkout Performance Analyzer package + uses: actions/checkout@v2 + with: + path: ./tmp/performance-analyzer # Explicitly set the docker-compose program path so that our build scripts in RCA can run the program # This is necessary because of the Github Actions environment and the workingDir of the Gradle environment - name: Set docker-compose path @@ -42,17 +40,21 @@ jobs: # Set the vm.max_map_count system property to the minimum required to run OpenSearch - name: Set vm.max_map_count run: sudo sysctl -w vm.max_map_count=262144 - - name: Build PA and run Unit Tests - working-directory: ./tmp/pa - run: ./gradlew build -Dperformance-analyzer-rca.build=true -Dperformance-analyzer-rca.branch=main -Dopensearch.version=1.1.0-SNAPSHOT + - name: Build Performance Analyzer and run Unit Tests + working-directory: ./tmp/performance-analyzer + run: | + ./gradlew build -Dperformance-analyzer-rca.build=true \ + -Dperformance-analyzer-rca.repo="https://github.com/dblock/performance-analyzer-rca.git" \ + -Dperformance-analyzer-rca.branch=fix-snapshot-build \ + -Dopensearch.version=1.1.0-SNAPSHOT - name: Generate Jacoco coverage report - working-directory: ./tmp/pa + working-directory: ./tmp/performance-analyzer run: ./gradlew jacocoTestReport - name: Upload coverage report - working-directory: ./tmp/pa + working-directory: ./tmp/performance-analyzer env: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} run: bash <(curl -s https://codecov.io/bash) -f ./build/reports/jacoco/test/jacocoTestReport.xml - name: Run Integration Tests - working-directory: ./tmp/pa + working-directory: ./tmp/performance-analyzer run: ./gradlew integTest -Dtests.enableIT -Dtests.useDockerCluster -Dopensearch.version=1.1.0-SNAPSHOT diff --git a/build.gradle b/build.gradle index 86a79c18..e35de81c 100644 --- a/build.gradle +++ b/build.gradle @@ -70,14 +70,15 @@ spotbugsTest { ext { isSnapshot = "true" == System.getProperty("build.snapshot", "true") - // The RCA branch that will be built. Default value is main. - rcaProjectBranch = findProperty("performance-analyzer-rca.branch") ?: "main" + // The RCA branch that will be built. Default branch is main. + rcaProjectRepo = System.getProperty("performance-analyzer-rca.repo", "https://github.com/opensearch-project/performance-analyzer-rca.git") + rcaProjectBranch = System.getProperty("performance-analyzer-rca.branch", "main") // If true, then the build will clone the RCA Project into $rcaProjectDir cloneRcaProject = "true" == System.getProperty("performance-analyzer-rca.build", "true") // By default we will look for RCA in a peer directory - rcaProjectDir = findProperty("performance-analyzer-rca.path") ?: "../performance-analyzer-rca" + rcaProjectDir = System.getProperty("performance-analyzer-rca.path", "../performance-analyzer-rca") } group = "org.opensearch" @@ -310,24 +311,24 @@ static def propEnabled(property) { // Pass the -Dtests.enableIT property to Gradle to run ITs /** - * cloneGitRepo clones the performance-analyzer-rca repo if the -Dtests.enableIT=true flag is passed + * cloneRcaGitRepo clones the performance-analyzer-rca repo if the -Dtests.enableIT=true flag is passed * to the Gradle JVM */ -task cloneGitRepo(type: GitClone) { +task cloneRcaGitRepo(type: GitClone) { def destination = file(rcaProjectDir) - uri = "https://github.com/opensearch-project/performance-analyzer-rca.git" + uri = rcaProjectRepo branch = rcaProjectBranch destinationPath = destination bare = false enabled = cloneRcaProject && !destination.exists() // to clone only once doFirst { - logger.info("Cloning performance-analyzer-rca into ${rcaProjectDir}") + logger.info("Cloning performance-analyzer-rca into ${rcaProjectDir} from ${rcaProjectRepo}#${rcaProjectBranch}") } } task buildRca() { - dependsOn(cloneGitRepo) + dependsOn(cloneRcaGitRepo) doFirst { logger.info("Building performance-analyzer-rca project in ${rcaProjectDir} with -Dopensearch.version=${opensearch_version}") @@ -402,10 +403,10 @@ bundlePlugin { /** * setupOpenSearchCluster spins up a local 2 node OpenSearch cluster using the enableRca task in the performance-analyzer-rca - * repo. The performance-analyzer-rca repo is cloned as part of the cloneGitRepo task. + * repo. The performance-analyzer-rca repo is cloned as part of the cloneRcaGitRepo task. */ task setupOpenSearchCluster() { - dependsOn(cloneGitRepo) + dependsOn(cloneRcaGitRepo) onlyIf = { propEnabled("tests.enableIT") } From 7550cc989a492633f7222ec44e2cfe1acce4eb69 Mon Sep 17 00:00:00 2001 From: dblock Date: Fri, 20 Aug 2021 22:05:25 +0000 Subject: [PATCH 3/3] Project.version -> version. Signed-off-by: dblock --- build.gradle | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build.gradle b/build.gradle index e35de81c..3804ec86 100644 --- a/build.gradle +++ b/build.gradle @@ -346,7 +346,7 @@ task buildRca() { exec { def licenseDir = "$projectDir/licenses" workingDir("$licenseDir") - commandLine 'rm', "-f", "performanceanalyzer-rca-${project.version}.jar.sha1" + commandLine 'rm', "-f", "performanceanalyzer-rca-${version}.jar.sha1" } } } @@ -461,7 +461,7 @@ afterEvaluate { ospackage { packageName = "opensearch-performance-analyzer" release = isSnapshot ? "0.1" : '1' - version = "${project.version}" - "-SNAPSHOT" + version = "${version}" - "-SNAPSHOT" into '/usr/share/opensearch/plugins' from(zipTree(bundlePlugin.archivePath)) {