Additional opensearch changes and formatting changes

opensearch-project · Apr 16, 2021 · 0ecfafc · 0ecfafc
1 parent 072ce05
commit 0ecfafc
Show file tree

Hide file tree

Showing 53 changed files with 192 additions and 280 deletions.
diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml
@@ -31,7 +31,7 @@ jobs:
         run: ./gradlew publishToMavenLocal
       - name: Build PA gradle using the new RCA jar
         working-directory: ./tmp/pa
-        run: rm licenses/performanceanalyzer-rca-1.13.jar.sha1
+        run: rm -f licenses/performanceanalyzer-rca-1.13.jar.sha1
       - name: Update SHA
         working-directory: ./tmp/pa
         run: ./gradlew updateShas

diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml
@@ -38,15 +38,15 @@ jobs:
       run: ./gradlew publishToMavenLocal
     - name: Build PA gradle using the new RCA jar
       working-directory: ./tmp/pa
-      run: rm licenses/performanceanalyzer-rca-1.13.jar.sha1
+      run: rm -f licenses/performanceanalyzer-rca-1.13.jar.sha1
     - name: Update SHA
       working-directory: ./tmp/pa
       run: ./gradlew updateShas
       # Explicitly set the docker-compose program path so that our build scripts in RCA can run the program
       # This is necessary because of the Github Actions environment and the workingDir of the Gradle environment
     - name: Set docker-compose path
       run: DOCKER_COMPOSE_LOCATION=$(which docker-compose)
-      # Set the vm.max_map_count system property to the minimum required to run Elasticsearch
+      # Set the vm.max_map_count system property to the minimum required to run OpenSearch
     - name: Set vm.max_map_count
       run: sudo sysctl -w vm.max_map_count=262144
     - name: Build PA and run Unit Tests

diff --git a/MAINTAINERS.md b/MAINTAINERS.md
@@ -3,8 +3,6 @@
 ## Maintainers
 | Maintainer | GitHub ID | Affiliation |
 | --------------- | --------- | ----------- |
-| Charlotte Henkle  | [CEHENKLE](https://github.com/CEHENKLE) | Amazon |
-| Nick Knize  | [nknize](https://github.com/nknize) | Amazon |
 | Joshua Tokle  | [jotok](https://github.com/jotok) | Amazon |
 | Sruti Parthiban  | [sruti1312](https://github.com/sruti1312) | Amazon |
 | Haoru Xiao  | [hxiao608](https://github.com/hxiao608) | Amazon |

diff --git a/build.gradle b/build.gradle
@@ -36,7 +36,7 @@ buildscript {
     }
 
     dependencies {
-        classpath "org.opensearch.gradle:build-tools:${opensearch_version}"
+        classpath "org.opensearch.gradle:build-tools:${opensearch_version}-SNAPSHOT"
         classpath 'org.ajoberstar:gradle-git:0.2.3'
     }
 }

diff --git a/packaging/deb/postinst b/packaging/deb/postinst
@@ -23,10 +23,10 @@ chown opensearch /var/lib/opensearch/rca_enabled.conf
 chown -R opensearch "$OPENSEARCH_HOME/performance-analyzer-rca"
 chmod a+rw /tmp
 
-if ! grep -q '## OpenDistro Performance Analyzer' /etc/opensearch/jvm.options; then
+if ! grep -q '## OpenSearch Performance Analyzer' /etc/opensearch/jvm.options; then
    CLK_TCK=`/usr/bin/getconf CLK_TCK`
    echo >> /etc/opensearch/jvm.options
-   echo '## OpenDistro Performance Analyzer' >> /etc/opensearch/jvm.options
+   echo '## OpenSearch Performance Analyzer' >> /etc/opensearch/jvm.options
    echo "-Dclk.tck=$CLK_TCK" >> /etc/opensearch/jvm.options
    echo "-Djdk.attach.allowAttachSelf=true" >> /etc/opensearch/jvm.options
    echo "-Djava.security.policy=file:///usr/share/opensearch/plugins/opendistro-performance-analyzer/pa_config/opensearch_security.policy" >> /etc/opensearch/jvm.options
@@ -47,7 +47,7 @@ fi
 
 if [ "x$IS_UPGRADE" != "xtrue" ]; then
     if command -v systemctl > /dev/null; then
-        echo '# Enabling opendistro performance analyzer to start and stop along with opensearch.service'
+        echo '# Enabling OpenSearch performance analyzer to start and stop along with opensearch.service'
         systemctl daemon-reload
         systemctl enable opendistro-performance-analyzer.service || true
 

diff --git a/packaging/opendistro-performance-analyzer.service b/packaging/opendistro-performance-analyzer.service
@@ -1,5 +1,5 @@
 [Unit]
-Description=Opendistro for Elasticsearch Performance Analyzer
+Description=OpenSearch Performance Analyzer
 PartOf=opensearch.service
 
 [Service]

diff --git a/packaging/rpm/postinst b/packaging/rpm/postinst
@@ -29,10 +29,10 @@ chown opensearch /var/lib/opensearch/rca_enabled.conf
 chown -R opensearch "$OPENSEARCH_HOME/performance-analyzer-rca"
 chmod a+rw /tmp
 
-if ! grep -q '## OpenDistro Performance Analyzer' /etc/opensearch/jvm.options; then
+if ! grep -q '## OpenSearch Performance Analyzer' /etc/opensearch/jvm.options; then
    CLK_TCK=`/usr/bin/getconf CLK_TCK`
    echo >> /etc/opensearch/jvm.options
-   echo '## OpenDistro Performance Analyzer' >> /etc/opensearch/jvm.options
+   echo '## OpenSearch Performance Analyzer' >> /etc/opensearch/jvm.options
    echo "-Dclk.tck=$CLK_TCK" >> /etc/opensearch/jvm.options
    echo "-Djdk.attach.allowAttachSelf=true" >> /etc/opensearch/jvm.options
    echo "-Djava.security.policy=file:///usr/share/opensearch/plugins/opendistro-performance-analyzer/pa_config/opensearch_security.policy" >> /etc/opensearch/jvm.options
@@ -57,7 +57,7 @@ esac
 
 if [ "x$IS_UPGRADE" != "xtrue" ]; then
     if command -v systemctl > /dev/null; then
-        echo '# Enabling opendistro performance analyzer to start and stop along with opensearch.service'
+        echo '# Enabling OpenSearch performance analyzer to start and stop along with opensearch.service'
         systemctl daemon-reload
         systemctl enable opendistro-performance-analyzer.service || true
 

diff --git a/.../java/com/amazon/opendistro/opensearch/performanceanalyzer/PerformanceAnalyzerPlugin.java b/.../java/com/amazon/opendistro/opensearch/performanceanalyzer/PerformanceAnalyzerPlugin.java
@@ -242,8 +242,7 @@ public PerformanceAnalyzerPlugin(final Settings settings, final java.nio.file.Pa
         scheduledMetricCollectorsExecutor.addScheduledMetricCollector(
                 new AdmissionControlMetricsCollector());
         scheduledMetricCollectorsExecutor.addScheduledMetricCollector(
-                new ElectionTermCollector(
-                        performanceAnalyzerController,configOverridesWrapper));
+                new ElectionTermCollector(performanceAnalyzerController, configOverridesWrapper));
         try {
             Class.forName(ShardIndexingPressureMetricsCollector.SHARD_INDEXING_PRESSURE_CLASS_NAME);
             scheduledMetricCollectorsExecutor.addScheduledMetricCollector(

diff --git a/...distro/opensearch/performanceanalyzer/collectors/ClusterApplierServiceStatsCollector.java b/...distro/opensearch/performanceanalyzer/collectors/ClusterApplierServiceStatsCollector.java
@@ -135,14 +135,15 @@ public Object getClusterApplierServiceStats()
         Method method =
                 ClusterApplierService.class.getMethod(
                         GET_CLUSTER_APPLIER_SERVICE_STATS_METHOD_NAME);
-        return method.invoke(OpenSearchResources.INSTANCE.getClusterService().getClusterApplierService());
+        return method.invoke(
+                OpenSearchResources.INSTANCE.getClusterService().getClusterApplierService());
     }
 
     /**
-     * ClusterApplierServiceStats is OpenSearch is a tracker for total time taken to apply cluster state and
-     * the number of times it has failed. To calculate point in time metric, we will have to store
-     * its previous state and calculate the diff to get the point in time latency. This might return
-     * as 0 if there is no cluster update since last retrieval.
+     * ClusterApplierServiceStats is OpenSearch is a tracker for total time taken to apply cluster
+     * state and the number of times it has failed. To calculate point in time metric, we will have
+     * to store its previous state and calculate the diff to get the point in time latency. This
+     * might return as 0 if there is no cluster update since last retrieval.
      *
      * @param currentMetrics Current Cluster update stats in OpenSearch
      * @return point in time latency.
@@ -161,10 +162,10 @@ private double computeRate(final double currentTotalCount) {
     }
 
     /**
-     * ClusterApplierServiceStats is OpenSearch is a tracker for total time taken to apply cluster state and
-     * the number of times it has failed. To calculate point in time metric, we will have to store
-     * its previous state and calculate the diff to get the point in time failure. This might return
-     * as 0 if there is no cluster update since last retrieval.
+     * ClusterApplierServiceStats is OpenSearch is a tracker for total time taken to apply cluster
+     * state and the number of times it has failed. To calculate point in time metric, we will have
+     * to store its previous state and calculate the diff to get the point in time failure. This
+     * might return as 0 if there is no cluster update since last retrieval.
      *
      * @param currentMetrics Current Cluster update stats in OpenSearch
      * @return point in time failure.

diff --git a/...om/amazon/opendistro/opensearch/performanceanalyzer/collectors/ElectionTermCollector.java b/...om/amazon/opendistro/opensearch/performanceanalyzer/collectors/ElectionTermCollector.java
@@ -15,6 +15,7 @@
 
 package com.amazon.opendistro.opensearch.performanceanalyzer.collectors;
 
+
 import com.amazon.opendistro.opensearch.performanceanalyzer.OpenSearchResources;
 import com.amazon.opendistro.opensearch.performanceanalyzer.PerformanceAnalyzerApp;
 import com.amazon.opendistro.opensearch.performanceanalyzer.config.PerformanceAnalyzerController;
@@ -32,16 +33,19 @@
 /**
  * This class starts publishing election term metric. These metric is emitted from cluster state.
  */
-public class ElectionTermCollector extends PerformanceAnalyzerMetricsCollector implements MetricsProcessor {
-    public static final int SAMPLING_TIME_INTERVAL = MetricsConfiguration.CONFIG_MAP.get(ElectionTermCollector.class).samplingInterval;
+public class ElectionTermCollector extends PerformanceAnalyzerMetricsCollector
+        implements MetricsProcessor {
+    public static final int SAMPLING_TIME_INTERVAL =
+            MetricsConfiguration.CONFIG_MAP.get(ElectionTermCollector.class).samplingInterval;
     private static final Logger LOG = LogManager.getLogger(ElectionTermCollector.class);
     private static final int KEYS_PATH_LENGTH = 0;
     private final ConfigOverridesWrapper configOverridesWrapper;
     private final PerformanceAnalyzerController controller;
     private StringBuilder value;
 
-    public ElectionTermCollector(PerformanceAnalyzerController controller,
-                                 ConfigOverridesWrapper configOverridesWrapper) {
+    public ElectionTermCollector(
+            PerformanceAnalyzerController controller,
+            ConfigOverridesWrapper configOverridesWrapper) {
         super(SAMPLING_TIME_INTERVAL, "ElectionTermCollector");
         value = new StringBuilder();
         this.controller = controller;
@@ -55,7 +59,8 @@ public String getMetricsPath(long startTime, String... keysPath) {
             throw new RuntimeException("keys length should be " + KEYS_PATH_LENGTH);
         }
 
-        return PerformanceAnalyzerMetrics.generatePath(startTime, PerformanceAnalyzerMetrics.sElectionTermPath);
+        return PerformanceAnalyzerMetrics.generatePath(
+                startTime, PerformanceAnalyzerMetrics.sElectionTermPath);
     }
 
     @Override
@@ -73,21 +78,26 @@ public void collectMetrics(long startTime) {
             value.setLength(0);
             value.append(PerformanceAnalyzerMetrics.getJsonCurrentMilliSeconds())
                     .append(PerformanceAnalyzerMetrics.sMetricNewLineDelimitor);
-            value.append(new ElectionTermMetrics(
-                    OpenSearchResources.INSTANCE.getClusterService().state()
-                            .term()).serialize());
+            value.append(
+                    new ElectionTermMetrics(
+                                    OpenSearchResources.INSTANCE.getClusterService().state().term())
+                            .serialize());
             saveMetricValues(value.toString(), startTime);
 
             PerformanceAnalyzerApp.WRITER_METRICS_AGGREGATOR.updateStat(
-                    WriterMetrics.ELECTION_TERM_COLLECTOR_EXECUTION_TIME, "",
+                    WriterMetrics.ELECTION_TERM_COLLECTOR_EXECUTION_TIME,
+                    "",
                     System.currentTimeMillis() - mCurrT);
 
         } catch (Exception ex) {
             PerformanceAnalyzerApp.ERRORS_AND_EXCEPTIONS_AGGREGATOR.updateStat(
-                    ExceptionsAndErrors.ELECTION_TERM_COLLECTOR_ERROR, "",
+                    ExceptionsAndErrors.ELECTION_TERM_COLLECTOR_ERROR,
+                    "",
                     System.currentTimeMillis() - mCurrT);
-            LOG.debug("Exception in Collecting Election term Metrics: {} for startTime {}",
-                    () -> ex.toString(), () -> startTime);
+            LOG.debug(
+                    "Exception in Collecting Election term Metrics: {} for startTime {}",
+                    () -> ex.toString(),
+                    () -> startTime);
         }
     }
 

diff --git a/...mazon/opendistro/opensearch/performanceanalyzer/collectors/MasterServiceEventMetrics.java b/...mazon/opendistro/opensearch/performanceanalyzer/collectors/MasterServiceEventMetrics.java
@@ -86,7 +86,8 @@ public String getMetricsPath(long startTime, String... keysPath) {
     public void collectMetrics(long startTime) {
         try {
             if (OpenSearchResources.INSTANCE.getClusterService() == null
-                    || OpenSearchResources.INSTANCE.getClusterService().getMasterService() == null) {
+                    || OpenSearchResources.INSTANCE.getClusterService().getMasterService()
+                            == null) {
                 return;
             }
 
@@ -194,7 +195,8 @@ Field getMasterServiceTPExecutorField() throws Exception {
 
     // - Separated to have a unit test; and catch any code changes around this field
     Field getPrioritizedTPExecutorCurrentField() throws Exception {
-        Field currentField = PrioritizedOpenSearchThreadPoolExecutor.class.getDeclaredField("current");
+        Field currentField =
+                PrioritizedOpenSearchThreadPoolExecutor.class.getDeclaredField("current");
         currentField.setAccessible(true);
         return currentField;
     }
@@ -213,7 +215,8 @@ Method getPrioritizedTPExecutorAddPendingMethod() throws Exception {
         classArray[1] = List.class;
         classArray[2] = boolean.class;
         Method addPendingMethod =
-                PrioritizedOpenSearchThreadPoolExecutor.class.getDeclaredMethod("addPending", classArray);
+                PrioritizedOpenSearchThreadPoolExecutor.class.getDeclaredMethod(
+                        "addPending", classArray);
         addPendingMethod.setAccessible(true);
         return addPendingMethod;
     }

diff --git a/...com/amazon/opendistro/opensearch/performanceanalyzer/collectors/MasterServiceMetrics.java b/...com/amazon/opendistro/opensearch/performanceanalyzer/collectors/MasterServiceMetrics.java
diff --git a/...pendistro/opensearch/performanceanalyzer/collectors/MasterThrottlingMetricsCollector.java b/...pendistro/opensearch/performanceanalyzer/collectors/MasterThrottlingMetricsCollector.java
@@ -66,7 +66,8 @@ public void collectMetrics(long startTime) {
         try {
             long mCurrT = System.currentTimeMillis();
             if (OpenSearchResources.INSTANCE.getClusterService() == null
-                    || OpenSearchResources.INSTANCE.getClusterService().getMasterService() == null) {
+                    || OpenSearchResources.INSTANCE.getClusterService().getMasterService()
+                            == null) {
                 return;
             }
             if (!isMasterThrottlingFeatureAvailable()) {
@@ -114,7 +115,8 @@ private boolean isMasterThrottlingFeatureAvailable() {
 
     private long getTotalMasterThrottledTaskCount() throws Exception {
         Method method = MasterService.class.getMethod(THROTTLED_PENDING_TASK_COUNT_METHOD_NAME);
-        return (long) method.invoke(OpenSearchResources.INSTANCE.getClusterService().getMasterService());
+        return (long)
+                method.invoke(OpenSearchResources.INSTANCE.getClusterService().getMasterService());
     }
 
     private long getRetryingPendingTaskCount() throws Exception {