Add bug fixes from main and upgrade to version 1.0.1 (#56)

* Create writer file if metrics are available (#31) Signed-off-by: Sruti Parthiban <partsrut@amazon.com> * Add tests to check for writer file only if metrics are present (#35) Signed-off-by: Sruti Parthiban <partsrut@amazon.com> * Merge pull request #36 from opensearch-project/khushbr-writer-purge-fix Fixing Event Log file cleanup issue * Moving deleteAllFiles() to inside scheduleExecutor() * Fixing the Link Checker errors, updating the official documentation * nit: Fixing spotlessJava indentation issue * Merge pull request #37 from khushbr/feature/purge-fix Handling purging of lingering files before scheduleExecutor start. * Fix failing file handler test (#38) Signed-off-by: Sruti Parthiban <partsrut@amazon.com> * Remove dependency on main branch when running spotless. (#47) Signed-off-by: Marc Handalian <handalm@amazon.com> * Updates to gradle build file (#48) * Updates to gradle build file Signed-off-by: Sruti Parthiban <partsrut@amazon.com> * Add ability to specify RCA branch Signed-off-by: Sruti Parthiban <partsrut@amazon.com> * Fix build when opensearch_version flag is provided. (#52) Signed-off-by: Marc Handalian <handalm@amazon.com> * Update the version to 1.0.1 Signed-off-by: Sruti Parthiban <partsrut@amazon.com> Co-authored-by: Khushboo Rajput <59671881+khushbr@users.noreply.github.com> Co-authored-by: Khushboo Rajput <khushbr@amazon.com> Co-authored-by: Marc Handalian <handalm@amazon.com>
opensearch-project · Aug 25, 2021 · 498539e · 498539e
1 parent 8bad5d2
commit 498539e
Show file tree

Hide file tree

Showing 11 changed files with 232 additions and 139 deletions.
diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml
@@ -10,11 +10,6 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - name: Checkout RCA package
-        uses: actions/checkout@v2
-        with:
-          repository: opensearch-project/performance-analyzer-rca
-          path: ./tmp/rca
       - name: Checkout Performance Analyzer package
         uses: actions/checkout@v2
         with:
@@ -23,22 +18,10 @@ jobs:
         uses: actions/setup-java@v1
         with:
           java-version: 1.12
-      - name: Build RCA with Gradle
-        working-directory:  ./tmp/rca
-        run: ./gradlew build -x test
-      - name: Publish RCA jar to maven local
-        working-directory: ./tmp/rca
-        run: ./gradlew publishToMavenLocal
-      - name: Build PA gradle using the new RCA jar
-        working-directory: ./tmp/pa
-        run: rm -f licenses/performanceanalyzer-rca-1.0.0.0.jar.sha1
-      - name: Update SHA
-        working-directory: ./tmp/pa
-        run: ./gradlew updateShas
       - name: Build Artifacts
         working-directory: ./tmp/pa
         run: |
-          ./gradlew build buildDeb buildRpm --refresh-dependencies -Dbuild.snapshot=false -x javadoc
+          ./gradlew build buildDeb buildRpm --refresh-dependencies -Dbuild.snapshot=false -Dperformance-analyzer-rca.build=true -Dperformance-analyzer-rca.branch=main -x javadoc
           mkdir artifacts
           artifact=`ls build/distributions/*.zip`
           rpm_artifact=`ls build/distributions/*.rpm`

diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml
@@ -9,33 +9,21 @@ on:
       - "*"
 
 jobs:
-  build_rca_pkg:
+  build_pa_pkg:
     runs-on: [ubuntu-latest]
-    name: Building RCA package
+    name: Building PA package
     steps:
-    - name: Checkout RCA package
-      uses: actions/checkout@v2
-      with:
-        repository: opensearch-project/performance-analyzer-rca
-        path: ./tmp/rca
     - name: Checkout Performance Analyzer package
       uses: actions/checkout@v2
       with:
         path: ./tmp/pa
-      # fetch the main branch to make it available for spotless ratcheting
-    - name: Fetch 'main' branch
+    - name: Fetch '1.0.1' branch
       working-directory:  ./tmp/pa
-      run: git fetch --depth=1 origin main
+      run: git fetch --depth=1 origin 1.0.1
     - name: Set up JDK 1.12
       uses: actions/setup-java@v1
       with:
         java-version: 1.12
-    - name: Build RCA with Gradle
-      working-directory:  ./tmp/rca
-      run: ./gradlew build -x test
-    - name: Publish RCA jar to maven local
-      working-directory: ./tmp/rca
-      run: ./gradlew publishToMavenLocal
     # dependencies: OpenSearch
     - name: Checkout OpenSearch
       uses: actions/checkout@v2
@@ -46,12 +34,6 @@ jobs:
     - name: Build OpenSearch
       working-directory: ./OpenSearch
       run: ./gradlew publishToMavenLocal -Dbuild.snapshot=false
-    - name: Build PA gradle using the new RCA jar
-      working-directory: ./tmp/pa
-      run: rm -f licenses/performanceanalyzer-rca-1.0.0.0.jar.sha1
-    - name: Update SHA
-      working-directory: ./tmp/pa
-      run: ./gradlew updateShas
       # Explicitly set the docker-compose program path so that our build scripts in RCA can run the program
       # This is necessary because of the Github Actions environment and the workingDir of the Gradle environment
     - name: Set docker-compose path
@@ -61,7 +43,7 @@ jobs:
       run: sudo sysctl -w vm.max_map_count=262144
     - name: Build PA and run Unit Tests
       working-directory: ./tmp/pa
-      run: ./gradlew build
+      run: ./gradlew build -Dperformance-analyzer-rca.build=true -Dperformance-analyzer-rca.branch=1.0.1
     - name: Generate Jacoco coverage report
       working-directory: ./tmp/pa
       run: ./gradlew jacocoTestReport

diff --git a/README.md b/README.md
@@ -1,7 +1,7 @@
 [![Java CI](https://github.com/opensearch-project/performance-analyzer/workflows/Java%20CI/badge.svg)](https://github.com/opensearch-project/performance-analyzer/actions?query=workflow%3A%22Java+CI%22)
 [![CD](https://github.com/opensearch-project/performance-analyzer/workflows/CD/badge.svg)](https://github.com/opensearch-project/performance-analyzer/actions?query=workflow%3ACD)
 [![codecov](https://codecov.io/gh/opensearch-project/performance-analyzer/branch/main/graph/badge.svg)](https://codecov.io/gh/opensearch-project/performance-analyzer)
-[![Documentation](https://img.shields.io/badge/api-reference-blue.svg)](https://docs-beta.opensearch.org/monitoring-plugins/pa/api/)
+[![Documentation](https://img.shields.io/badge/api-reference-blue.svg)](https://opensearch.org/docs/monitoring-plugins/pa/api/)
 [![Chat](https://img.shields.io/badge/chat-on%20forums-blue)](https://discuss.opendistrocommunity.dev/c/performance-analyzer/)
 ![PRs welcome!](https://img.shields.io/badge/PRs-welcome!-success)
 
@@ -96,7 +96,7 @@ See the [design doc](https://github.com/opensearch-project/performance-analyzer-
 
 ## Documentation
 
-Please refer to the [technical documentation](https://docs-beta.opensearch.org/monitoring-plugins/pa/index/) for detailed information on installing and configuring Performance Analyzer.
+Please refer to the [technical documentation](https://opensearch.org/docs/monitoring-plugins/pa/index/) for detailed information on installing and configuring Performance Analyzer.
 
 ## Contributing
 

diff --git a/build.gradle b/build.gradle
@@ -9,7 +9,7 @@
  * GitHub history for details.
  */
 
- /*
+/*
  * Copyright <2019> Amazon.com, Inc. or its affiliates. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License").
@@ -29,7 +29,8 @@ buildscript {
     ext {
         opensearch_version = System.getProperty("opensearch.version", "1.0.0")
     }
-    // This isn't applying from repositories.gradle so repeating it here
+
+    // Used to resolve build file dependencies
     repositories {
         mavenCentral()
         mavenLocal()
@@ -67,8 +68,17 @@ spotbugsTest {
 }
 
 ext {
-    opensearchVersion = '1.0.0'
+    opensearchVersion = '1.0.1'
     isSnapshot = "true" == System.getProperty("build.snapshot", "true")
+
+    // The RCA branch that will be built. Default value is main.
+    rcaProjectBranch = findProperty("performance-analyzer-rca.branch") ?: "main"
+
+    // If true, then the build will clone the RCA Project into $rcaProjectDir
+    cloneRcaProject = "true" == System.getProperty("performance-analyzer-rca.build", "true")
+
+    // By default we will look for RCA in a peer directory
+    rcaProjectDir = findProperty("performance-analyzer-rca.path") ?: "../performance-analyzer-rca"
 }
 
 group = "org.opensearch"
@@ -79,18 +89,11 @@ if (isSnapshot) {
 
 apply plugin: 'opensearch.opensearchplugin'
 
-ext {
-    projectSubstitutions = [:]
-    licenseFile = rootProject.file('LICENSE.txt')
-    noticeFile = rootProject.file('NOTICE.txt')
-    gitDefaultBranch = 'main'
-}
+licenseFile = rootProject.file('LICENSE.txt')
+noticeFile = rootProject.file('NOTICE.txt')
 
 spotless {
     java {
-        // only apply formatting rules to updated files
-        ratchetFrom "origin/${gitDefaultBranch}"
-
         licenseHeaderFile(file('license-header'))
         googleJavaFormat().aosp()
         importOrder()
@@ -110,8 +113,6 @@ test {
 
 licenseHeaders.enabled = true
 validateNebulaPom.enabled = false
-
-
 loggerUsageCheck.enabled = false
 
 opensearchplugin {
@@ -132,7 +133,7 @@ compileJava {
 }
 
 javadoc {
-     options.addStringOption("-add-exports", "jdk.attach/sun.tools.attach=ALL-UNNAMED")
+    options.addStringOption("-add-exports", "jdk.attach/sun.tools.attach=ALL-UNNAMED")
 }
 
 project.afterEvaluate {
@@ -155,7 +156,6 @@ repositories {
     jcenter()
 }
 
-
 configurations {
     includeJars
 }
@@ -210,7 +210,6 @@ jacocoTestCoverageVerification {
     }
 }
 
-
 // to run coverage verification during the build (and fail when appropriate)
 check.dependsOn jacocoTestCoverageVerification
 
@@ -233,13 +232,13 @@ dependencies {
     }
 
     configurations.all {
-       resolutionStrategy {
-          force 'junit:junit:4.13.1'
-          force "com.fasterxml.jackson.core:jackson-annotations:${jacksonVersion}"
-          force "com.fasterxml.jackson.core:jackson-core:${jacksonVersion}"
-          force "com.fasterxml.jackson.core:jackson-databind:${jacksonVersion}"
-          force "com.fasterxml.jackson.module:jackson-module-paranamer:${jacksonVersion}"
-       }
+        resolutionStrategy {
+            force 'junit:junit:4.13.1'
+            force "com.fasterxml.jackson.core:jackson-annotations:${jacksonVersion}"
+            force "com.fasterxml.jackson.core:jackson-core:${jacksonVersion}"
+            force "com.fasterxml.jackson.core:jackson-databind:${jacksonVersion}"
+            force "com.fasterxml.jackson.module:jackson-module-paranamer:${jacksonVersion}"
+        }
     }
 
     compile ('com.google.guava:guava:28.2-jre') {
@@ -249,7 +248,7 @@ dependencies {
     compile 'org.apache.commons:commons-lang3:3.9'
     compile 'org.bouncycastle:bcprov-jdk15on:1.68'
     compile 'org.bouncycastle:bcpkix-jdk15on:1.68'
-    compile 'org.opensearch:performanceanalyzer-rca:1.0.0.0'
+    compile 'org.opensearch:performanceanalyzer-rca:1.0.1.0'
     compile "com.fasterxml.jackson.core:jackson-annotations:${jacksonVersion}"
     compile "com.fasterxml.jackson.core:jackson-databind:${jacksonVersion}"
     compile "com.fasterxml.jackson.module:jackson-module-paranamer:${jacksonVersion}"
@@ -300,66 +299,76 @@ gradle.startParameter.excludedTaskNames += [ "forbiddenApisMain",
                                              "forbiddenApisTest",
                                              "thirdPartyAudit",
                                              "testingConventions"]
-import java.nio.file.Paths
+
 import org.ajoberstar.gradle.git.tasks.GitClone
 import org.opensearch.gradle.test.RestIntegTestTask
 
-String rcaDir
-String rcaArtifactsDir
-
 static def propEnabled(property) {
     return System.getProperty(property) != null && System.getProperty(property).toLowerCase().equals("true")
 }
 
 // The following Gradle tasks are used to create a PA/RCA enabled OpenSearch cluster
 // Pass the -Dtests.enableIT property to Gradle to run ITs
+
 /**
  * cloneGitRepo clones the performance-analyzer-rca repo if the -Dtests.enableIT=true flag is passed
  * to the Gradle JVM
  */
 task cloneGitRepo(type: GitClone) {
-    rcaDir = Paths.get(getProject().getBuildDir().toString(), "performance-analyzer-rca").toString()
-    def destination = file(rcaDir)
+    def destination = file(rcaProjectDir)
     uri = "https://github.com/opensearch-project/performance-analyzer-rca.git"
-    branch = gitDefaultBranch
+    branch = rcaProjectBranch
     destinationPath = destination
     bare = false
-    enabled = !destination.exists() // to clone only once
+    enabled = cloneRcaProject && !destination.exists() // to clone only once
+
+    doFirst {
+        logger.info("Cloning performance-analyzer-rca into ${rcaProjectDir}")
+    }
 }
 
 task buildRca() {
     dependsOn(cloneGitRepo)
+
+    doFirst {
+        logger.info("Building performance-analyzer-rca project in ${rcaProjectDir}")
+    }
+
     doLast {
         exec {
-            workingDir("$rcaDir")
+            workingDir("$rcaProjectDir")
             commandLine './gradlew', 'build', '-x', 'test'
         }
         exec {
-            workingDir("$rcaDir")
+            workingDir("$rcaProjectDir")
             commandLine './gradlew', 'publishToMavenLocal'
         }
         exec {
             def licenseDir = "$projectDir/licenses"
             workingDir("$licenseDir")
-            commandLine 'rm', "performanceanalyzer-rca-1.0.0.0.jar.sha1"
-        }
-        exec {
-            workingDir("$projectDir")
-            commandLine './gradlew', 'updateShas'
+            commandLine 'rm', "-f", "performanceanalyzer-rca-1.0.1.0.jar.sha1"
         }
     }
 }
+buildRca.finalizedBy updateShas
+
+// This value is set by the unpackRca task
+def rcaArtifactsDir
 
 task unpackRca(type: Copy) {
     dependsOn(buildRca)
-    from(zipTree("$rcaDir/build/distributions/performance-analyzer-rca.zip")) {
+    from(zipTree("$rcaProjectDir/build/distributions/performance-analyzer-rca.zip")) {
+    }
+    into "$rcaProjectDir/build/distributions"
+    rcaArtifactsDir = "$rcaProjectDir/build/distributions/performance-analyzer-rca/"
+
+    doLast {
+        logger.info("Unpacked performance-analyzer-rca artifacts into ${rcaProjectDir}")
     }
-    into "$rcaDir/build/distributions"
-    rcaArtifactsDir = "$rcaDir/build/distributions/performance-analyzer-rca/"
 }
 
 bundlePlugin {
-    dependsOn 'cloneGitRepo'
+    dependsOn 'unpackRca'
     from("$rcaArtifactsDir/pa_config") {
         into "pa_config"
     }
@@ -402,7 +411,7 @@ task setupOpenSearchCluster() {
     }
     doLast {
         exec {
-            workingDir(rcaDir)
+            workingDir(rcaProjectDir)
             commandLine './gradlew', 'enableRca'
         }
         sleep(30000)
@@ -486,7 +495,7 @@ afterEvaluate {
         url 'https://opensearch.org/downloads.html'
         summary '''
          Performance Analyzer plugin for OpenSearch.
-         Reference documentation can be found at https://docs-beta.opensearch.org/.
+         Reference documentation can be found at https://opensearch.org/docs/.
     '''.stripIndent().replace('\n', ' ').trim()
     }
 
@@ -530,4 +539,3 @@ afterEvaluate {
 }
 
 bundlePlugin.mustRunAfter unpackRca
-build.dependsOn unpackRca
diff --git a/licenses/performanceanalyzer-rca-1.0.0.0.jar.sha1 b/licenses/performanceanalyzer-rca-1.0.0.0.jar.sha1
diff --git a/licenses/performanceanalyzer-rca-1.0.1.0.jar.sha1 b/licenses/performanceanalyzer-rca-1.0.1.0.jar.sha1
@@ -0,0 +1 @@
+b2fd334b34e89480c910969a677a9284095d2e69
diff --git a/src/main/java/org/opensearch/performanceanalyzer/PerformanceAnalyzerPlugin.java b/src/main/java/org/opensearch/performanceanalyzer/PerformanceAnalyzerPlugin.java
@@ -79,7 +79,6 @@
 import org.opensearch.performanceanalyzer.collectors.MasterServiceEventMetrics;
 import org.opensearch.performanceanalyzer.collectors.MasterServiceMetrics;
 import org.opensearch.performanceanalyzer.collectors.MasterThrottlingMetricsCollector;
-import org.opensearch.performanceanalyzer.collectors.MetricsPurgeActivity;
 import org.opensearch.performanceanalyzer.collectors.NetworkInterfaceCollector;
 import org.opensearch.performanceanalyzer.collectors.NodeDetailsCollector;
 import org.opensearch.performanceanalyzer.collectors.NodeStatsAllShardsMetricsCollector;
@@ -223,8 +222,6 @@ public PerformanceAnalyzerPlugin(final Settings settings, final java.nio.file.Pa
         scheduledMetricCollectorsExecutor.addScheduledMetricCollector(new OSMetricsCollector());
         scheduledMetricCollectorsExecutor.addScheduledMetricCollector(new HeapMetricsCollector());
 
-        scheduledMetricCollectorsExecutor.addScheduledMetricCollector(new MetricsPurgeActivity());
-
         scheduledMetricCollectorsExecutor.addScheduledMetricCollector(
                 new NodeDetailsCollector(configOverridesWrapper));
         scheduledMetricCollectorsExecutor.addScheduledMetricCollector(

diff --git a/src/main/java/org/opensearch/performanceanalyzer/util/Utils.java b/src/main/java/org/opensearch/performanceanalyzer/util/Utils.java
@@ -69,7 +69,7 @@ public static void configureMetrics() {
         MetricsConfiguration.CONFIG_MAP.put(NodeStatsAllShardsMetricsCollector.class, cdefault);
         MetricsConfiguration.CONFIG_MAP.put(NodeStatsFixedShardsMetricsCollector.class, cdefault);
         MetricsConfiguration.CONFIG_MAP.put(
-                MasterServiceEventMetrics.class, new MetricsConfiguration.MetricConfig(1000, 0, 0));
+                MasterServiceEventMetrics.class, new MetricsConfiguration.MetricConfig(1000, 0));
         MetricsConfiguration.CONFIG_MAP.put(MasterServiceMetrics.class, cdefault);
         MetricsConfiguration.CONFIG_MAP.put(FaultDetectionMetricsCollector.class, cdefault);
         MetricsConfiguration.CONFIG_MAP.put(ShardStateCollector.class, cdefault);