Merge branch 'seg-rep/checkpoint-publish' of https://github.com/Rishi…

…kesh1159/OpenSearch into seg-rep/checkpoint-publish Signed-off-by: Rishikesh1159 <rishireddy1159@gmail.com>
opensearch-project · May 6, 2022 · 5907f0b · 5907f0b
2 parents fa39d47 + 7777d1e
commit 5907f0b
Show file tree

Hide file tree

Showing 654 changed files with 2,890 additions and 622 deletions.
diff --git a/.ci/bwcVersions b/.ci/bwcVersions
@@ -39,5 +39,6 @@ BWC_VERSION:
   - "1.3.0"
   - "1.3.1"
   - "1.3.2"
+  - "1.3.3"
   - "2.0.0"
   - "2.1.0"
diff --git a/buildSrc/src/main/java/org/opensearch/gradle/PublishPlugin.java b/buildSrc/src/main/java/org/opensearch/gradle/PublishPlugin.java
@@ -135,11 +135,11 @@ public String call() throws Exception {
                     publication.artifact(project.getTasks().getByName("sourcesJar"));
                     publication.artifact(project.getTasks().getByName("javadocJar"));
                 }
-
-                generatePomTask.configure(
-                    t -> t.dependsOn(String.format("generatePomFileFor%sPublication", Util.capitalize(publication.getName())))
-                );
             }
+
+            generatePomTask.configure(
+                t -> t.dependsOn(String.format("generatePomFileFor%sPublication", Util.capitalize(publication.getName())))
+            );
         });
 
     }

diff --git a/buildSrc/src/main/java/org/opensearch/gradle/pluginzip/Publish.java b/buildSrc/src/main/java/org/opensearch/gradle/pluginzip/Publish.java
@@ -14,6 +14,7 @@
 import org.gradle.api.publish.maven.MavenPublication;
 import org.gradle.api.publish.maven.plugins.MavenPublishPlugin;
 import java.nio.file.Path;
+import org.gradle.api.Task;
 
 public class Publish implements Plugin<Project> {
     private Project project;
@@ -63,12 +64,19 @@ static String getProperty(String name, Project project) {
     @Override
     public void apply(Project project) {
         this.project = project;
-        project.afterEvaluate(evaluatedProject -> { configMaven(project); });
-        project.getGradle().getTaskGraph().whenReady(graph -> {
-            if (graph.hasTask(LOCALMAVEN)) {
-                project.getTasks().getByName(PLUGIN_ZIP_PUBLISH_POM_TASK).setEnabled(false);
+        project.afterEvaluate(evaluatedProject -> {
+            configMaven(project);
+            Task validatePluginZipPom = project.getTasks().findByName("validatePluginZipPom");
+            if (validatePluginZipPom != null) {
+                project.getTasks().getByName("validatePluginZipPom").dependsOn("generatePomFileForNebulaPublication");
+            }
+            Task publishPluginZipPublicationToZipStagingRepository = project.getTasks()
+                .findByName("publishPluginZipPublicationToZipStagingRepository");
+            if (validatePluginZipPom != null) {
+                project.getTasks()
+                    .getByName("publishPluginZipPublicationToZipStagingRepository")
+                    .dependsOn("generatePomFileForNebulaPublication");
             }
-
         });
     }
 }
diff --git a/buildSrc/src/test/java/org/opensearch/gradle/pluginzip/PublishTests.java b/buildSrc/src/test/java/org/opensearch/gradle/pluginzip/PublishTests.java
@@ -28,6 +28,8 @@
 import org.codehaus.plexus.util.xml.pull.XmlPullParserException;
 import java.io.FileReader;
 import org.gradle.api.tasks.bundling.Zip;
+import java.util.List;
+import java.util.ArrayList;
 
 public class PublishTests extends GradleUnitTestCase {
 
@@ -56,6 +58,7 @@ public void testZipPublish() throws IOException, XmlPullParserException {
         writeString(new File(projectDir, "settings.gradle"), "");
         // Generate the build.gradle file
         String buildFileContent = "apply plugin: 'maven-publish' \n"
+            + "apply plugin: 'java' \n"
             + "publishing {\n"
             + "  repositories {\n"
             + "       maven {\n"
@@ -74,10 +77,13 @@ public void testZipPublish() throws IOException, XmlPullParserException {
             + "}";
         writeString(new File(projectDir, "build.gradle"), buildFileContent);
         // Execute the task publishPluginZipPublicationToZipStagingRepository
+        List<String> allArguments = new ArrayList<String>();
+        allArguments.add("build");
+        allArguments.add(zipPublishTask);
         GradleRunner runner = GradleRunner.create();
         runner.forwardOutput();
         runner.withPluginClasspath();
-        runner.withArguments(zipPublishTask);
+        runner.withArguments(allArguments);
         runner.withProjectDir(projectDir);
         BuildResult result = runner.build();
         // Check if task publishMavenzipPublicationToZipstagingRepository has ran well
@@ -87,6 +93,7 @@ public void testZipPublish() throws IOException, XmlPullParserException {
             new File("build/functionalTest/local-staging-repo/org/opensearch/plugin/sample-plugin/2.0.0.0/sample-plugin-2.0.0.0.zip")
                 .exists()
         );
+        assertEquals(SUCCESS, result.task(":" + "build").getOutcome());
         // Parse the maven file and validate the groupID to org.opensearch.plugin
         MavenXpp3Reader reader = new MavenXpp3Reader();
         Model model = reader.read(

diff --git a/distribution/tools/plugin-cli/src/main/java/org/opensearch/plugins/InstallPluginCommand.java b/distribution/tools/plugin-cli/src/main/java/org/opensearch/plugins/InstallPluginCommand.java
@@ -374,7 +374,12 @@ private String getOpenSearchUrl(
                 stagingHash
             );
         } else {
-            baseUrl = String.format(Locale.ROOT, "https://artifacts.opensearch.org/releases/plugins/%s/%s", pluginId, version);
+            baseUrl = String.format(
+                Locale.ROOT,
+                "https://artifacts.opensearch.org/releases/plugins/%s/%s",
+                pluginId,
+                Build.CURRENT.getQualifiedVersion()
+            );
         }
         final String platformUrl = String.format(
             Locale.ROOT,

diff --git a/...tion/tools/plugin-cli/src/test/java/org/opensearch/plugins/InstallPluginCommandTests.java b/...tion/tools/plugin-cli/src/test/java/org/opensearch/plugins/InstallPluginCommandTests.java
@@ -1049,7 +1049,7 @@ public void assertInstallPluginFromUrl(
 
     public void testOfficialPlugin() throws Exception {
         String url = "https://artifacts.opensearch.org/releases/plugins/analysis-icu/"
-            + Version.CURRENT
+            + Build.CURRENT.getQualifiedVersion()
             + "/analysis-icu-"
             + Build.CURRENT.getQualifiedVersion()
             + ".zip";
@@ -1093,7 +1093,7 @@ public void testOfficialPluginStaging() throws Exception {
 
     public void testOfficialPlatformPlugin() throws Exception {
         String url = "https://artifacts.opensearch.org/releases/plugins/analysis-icu/"
-            + Version.CURRENT
+            + Build.CURRENT.getQualifiedVersion()
             + "/analysis-icu-"
             + Platforms.PLATFORM_NAME
             + "-"
@@ -1159,7 +1159,7 @@ public void testMavenChecksumWithoutFilename() throws Exception {
 
     public void testOfficialChecksumWithoutFilename() throws Exception {
         String url = "https://artifacts.opensearch.org/releases/plugins/analysis-icu/"
-            + Version.CURRENT
+            + Build.CURRENT.getQualifiedVersion()
             + "/analysis-icu-"
             + Build.CURRENT.getQualifiedVersion()
             + ".zip";
@@ -1184,7 +1184,7 @@ public void testOfficialChecksumWithoutFilename() throws Exception {
 
     public void testOfficialShaMissing() throws Exception {
         String url = "https://artifacts.opensearch.org/releases/plugins/analysis-icu/"
-            + Version.CURRENT
+            + Build.CURRENT.getQualifiedVersion()
             + "/analysis-icu-"
             + Build.CURRENT.getQualifiedVersion()
             + ".zip";
@@ -1229,7 +1229,7 @@ public void testMavenShaMissing() throws Exception {
 
     public void testInvalidShaFileMissingFilename() throws Exception {
         String url = "https://artifacts.opensearch.org/releases/plugins/analysis-icu/"
-            + Version.CURRENT
+            + Build.CURRENT.getQualifiedVersion()
             + "/analysis-icu-"
             + Build.CURRENT.getQualifiedVersion()
             + ".zip";
@@ -1254,7 +1254,7 @@ public void testInvalidShaFileMissingFilename() throws Exception {
 
     public void testInvalidShaFileMismatchFilename() throws Exception {
         String url = "https://artifacts.opensearch.org/releases/plugins/analysis-icu/"
-            + Version.CURRENT
+            + Build.CURRENT.getQualifiedVersion()
             + "/analysis-icu-"
             + Build.CURRENT.getQualifiedVersion()
             + ".zip";
@@ -1279,7 +1279,7 @@ public void testInvalidShaFileMismatchFilename() throws Exception {
 
     public void testInvalidShaFileContainingExtraLine() throws Exception {
         String url = "https://artifacts.opensearch.org/releases/plugins/analysis-icu/"
-            + Version.CURRENT
+            + Build.CURRENT.getQualifiedVersion()
             + "/analysis-icu-"
             + Build.CURRENT.getQualifiedVersion()
             + ".zip";
@@ -1304,7 +1304,7 @@ public void testInvalidShaFileContainingExtraLine() throws Exception {
 
     public void testSha512Mismatch() throws Exception {
         String url = "https://artifacts.opensearch.org/releases/plugins/analysis-icu/"
-            + Version.CURRENT
+            + Build.CURRENT.getQualifiedVersion()
             + "/analysis-icu-"
             + Build.CURRENT.getQualifiedVersion()
             + ".zip";
@@ -1349,7 +1349,7 @@ public void testSha1Mismatch() throws Exception {
     public void testPublicKeyIdMismatchToExpectedPublicKeyId() throws Exception {
         final String icu = "analysis-icu";
         final String url = "https://artifacts.opensearch.org/releases/plugins/analysis-icu/"
-            + Version.CURRENT
+            + Build.CURRENT.getQualifiedVersion()
             + "/"
             + icu
             + "-"
@@ -1386,7 +1386,7 @@ public void testPublicKeyIdMismatchToExpectedPublicKeyId() throws Exception {
     public void testFailedSignatureVerification() throws Exception {
         final String icu = "analysis-icu";
         final String url = "https://artifacts.opensearch.org/releases/plugins/analysis-icu/"
-            + Version.CURRENT
+            + Build.CURRENT.getQualifiedVersion()
             + "/"
             + icu
             + "-"

diff --git a/release-notes/opensearch.release-notes-1.3.2.md b/release-notes/opensearch.release-notes-1.3.2.md
@@ -0,0 +1,16 @@
+## Version 1.3.2 Release Notes
+
+### Upgrades
+
+* Update bundled JDK to 11.0.15+10 ([#3134](https://github.com/opensearch-project/OpenSearch/pull/3134))
+* [CVE-2020-36518] Update jackson-databind to 2.12.6.1 ([#2706](https://github.com/opensearch-project/OpenSearch/pull/2706))
+
+### Enhancements
+
+* Use G1GC on JDK11+ ([#2971](https://github.com/opensearch-project/OpenSearch/pull/2971))
+
+### Infrastructure
+
+* Fix opensearch-env always sources the environment from hardcoded file ([#2909](https://github.com/opensearch-project/OpenSearch/pull/2909))
+* Backporting changes for version workflow ([#2676](https://github.com/opensearch-project/OpenSearch/pull/2676))
+* Bump the version to 1.3.2 ([#2675](https://github.com/opensearch-project/OpenSearch/pull/2675))
diff --git a/...c/src/main/resources/rest-api-spec/test/search.aggregation/190_percentiles_hdr_metric.yml b/...c/src/main/resources/rest-api-spec/test/search.aggregation/190_percentiles_hdr_metric.yml
@@ -160,6 +160,16 @@ setup:
 ---
 "Filtered test":
 
+  - do:
+      bulk:
+        refresh: true
+        body:
+          - index:
+              _index: test_1
+              _id:    5
+          - int_field: 126
+            double_field: 126.0
+            string_field: foo
   - do:
       search:
         rest_total_hits_as_int: true
@@ -180,22 +190,22 @@ setup:
                 field: double_field
                 hdr: {}
 
-  - match: { hits.total: 3 }
-  - length: { hits.hits: 3 }
+  - match: { hits.total: 4 }
+  - length: { hits.hits: 4 }
 
   - match: { aggregations.percentiles_int.values.1\.0: 51.0 }
   - match: { aggregations.percentiles_int.values.5\.0: 51.0 }
   - match: { aggregations.percentiles_int.values.25\.0: 51.0 }
   - match: { aggregations.percentiles_int.values.50\.0: 101.03125 }
-  - match: { aggregations.percentiles_int.values.75\.0: 101.03125 }
+  - match: { aggregations.percentiles_int.values.75\.0: 126.03125 }
   - match: { aggregations.percentiles_int.values.95\.0: 151.09375 }
   - match: { aggregations.percentiles_int.values.99\.0: 151.09375 }
 
   - match: { aggregations.percentiles_double.values.1\.0: 51.0 }
   - match: { aggregations.percentiles_double.values.5\.0: 51.0 }
   - match: { aggregations.percentiles_double.values.25\.0: 51.0 }
   - match: { aggregations.percentiles_double.values.50\.0: 101.03125 }
-  - match: { aggregations.percentiles_double.values.75\.0: 101.03125 }
+  - match: { aggregations.percentiles_double.values.75\.0: 126.03125 }
   - match: { aggregations.percentiles_double.values.95\.0: 151.09375 }
   - match: { aggregations.percentiles_double.values.99\.0: 151.09375 }
 

diff --git a/server/build.gradle b/server/build.gradle
@@ -121,7 +121,7 @@ dependencies {
   // percentiles aggregation
   api 'com.tdunning:t-digest:3.2'
   // precentil ranks aggregation
-  api 'org.hdrhistogram:HdrHistogram:2.1.9'
+  api 'org.hdrhistogram:HdrHistogram:2.1.12'
 
   // lucene spatial
   api "org.locationtech.spatial4j:spatial4j:${versions.spatial4j}", optional
@@ -317,10 +317,6 @@ tasks.named("thirdPartyAudit").configure {
             'com.google.common.geometry.S2$Metric',
             'com.google.common.geometry.S2LatLng'
     )
-
-    if (BuildParams.runtimeJavaVersion > JavaVersion.VERSION_1_8) {
-        ignoreMissingClasses 'javax.xml.bind.DatatypeConverter'
-    }
 }
 
 tasks.named("dependencyLicenses").configure {

diff --git a/server/licenses/HdrHistogram-2.1.12.jar.sha1 b/server/licenses/HdrHistogram-2.1.12.jar.sha1
@@ -0,0 +1 @@
+6eb7552156e0d517ae80cc2247be1427c8d90452
diff --git a/server/licenses/HdrHistogram-2.1.9.jar.sha1 b/server/licenses/HdrHistogram-2.1.9.jar.sha1
diff --git a/server/src/internalClusterTest/java/org/opensearch/gateway/RecoveryFromGatewayIT.java b/server/src/internalClusterTest/java/org/opensearch/gateway/RecoveryFromGatewayIT.java
@@ -62,6 +62,7 @@
 import org.opensearch.index.shard.ShardPath;
 import org.opensearch.indices.IndicesService;
 import org.opensearch.indices.recovery.RecoveryState;
+import org.opensearch.indices.replication.common.ReplicationLuceneIndex;
 import org.opensearch.plugins.Plugin;
 import org.opensearch.test.OpenSearchIntegTestCase;
 import org.opensearch.test.OpenSearchIntegTestCase.ClusterScope;
@@ -547,7 +548,7 @@ public void testReuseInFileBasedPeerRecovery() throws Exception {
         final Set<String> files = new HashSet<>();
         for (final RecoveryState recoveryState : initialRecoveryReponse.shardRecoveryStates().get("test")) {
             if (recoveryState.getTargetNode().getName().equals(replicaNode)) {
-                for (final RecoveryState.FileDetail file : recoveryState.getIndex().fileDetails()) {
+                for (final ReplicationLuceneIndex.FileMetadata file : recoveryState.getIndex().fileDetails()) {
                     files.add(file.name());
                 }
                 break;
@@ -607,7 +608,7 @@ public Settings onNodeStopped(String nodeName) throws Exception {
             long reused = 0;
             int filesRecovered = 0;
             int filesReused = 0;
-            for (final RecoveryState.FileDetail file : recoveryState.getIndex().fileDetails()) {
+            for (final ReplicationLuceneIndex.FileMetadata file : recoveryState.getIndex().fileDetails()) {
                 if (files.contains(file.name()) == false) {
                     recovered += file.length();
                     filesRecovered++;

diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/recovery/IndexRecoveryIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/recovery/IndexRecoveryIT.java
@@ -102,6 +102,7 @@
 import org.opensearch.indices.NodeIndicesStats;
 import org.opensearch.indices.analysis.AnalysisModule;
 import org.opensearch.indices.recovery.RecoveryState.Stage;
+import org.opensearch.indices.replication.common.ReplicationLuceneIndex;
 import org.opensearch.node.NodeClosedException;
 import org.opensearch.node.RecoverySettingsChunkSizePlugin;
 import org.opensearch.plugins.AnalysisPlugin;
@@ -836,7 +837,7 @@ private IndicesStatsResponse createAndPopulateIndex(String name, int nodeCount,
         return client().admin().indices().prepareStats(name).execute().actionGet();
     }
 
-    private void validateIndexRecoveryState(RecoveryState.Index indexState) {
+    private void validateIndexRecoveryState(ReplicationLuceneIndex indexState) {
         assertThat(indexState.time(), greaterThanOrEqualTo(0L));
         assertThat(indexState.recoveredFilesPercent(), greaterThanOrEqualTo(0.0f));
         assertThat(indexState.recoveredFilesPercent(), lessThanOrEqualTo(100.0f));

diff --git a/server/src/main/java/org/opensearch/Version.java b/server/src/main/java/org/opensearch/Version.java
@@ -86,6 +86,7 @@ public class Version implements Comparable<Version>, ToXContentFragment {
     public static final Version V_1_3_0 = new Version(1030099, org.apache.lucene.util.Version.LUCENE_8_10_1);
     public static final Version V_1_3_1 = new Version(1030199, org.apache.lucene.util.Version.LUCENE_8_10_1);
     public static final Version V_1_3_2 = new Version(1030299, org.apache.lucene.util.Version.LUCENE_8_10_1);
+    public static final Version V_1_3_3 = new Version(1030399, org.apache.lucene.util.Version.LUCENE_8_10_1);
     public static final Version V_2_0_0 = new Version(2000099, org.apache.lucene.util.Version.LUCENE_9_1_0);
     public static final Version V_2_1_0 = new Version(2010099, org.apache.lucene.util.Version.LUCENE_9_1_0);
     public static final Version V_3_0_0 = new Version(3000099, org.apache.lucene.util.Version.LUCENE_9_2_0);

diff --git a/server/src/main/java/org/opensearch/index/AbstractIndexComponent.java b/server/src/main/java/org/opensearch/index/AbstractIndexComponent.java
@@ -36,6 +36,11 @@
 import org.opensearch.common.logging.DeprecationLogger;
 import org.opensearch.common.logging.Loggers;
 
+/**
+ * A base class for new index components
+ *
+ * @opensearch.internal
+ */
 public abstract class AbstractIndexComponent implements IndexComponent {
 
     protected final Logger logger;

diff --git a/server/src/main/java/org/opensearch/index/CompositeIndexEventListener.java b/server/src/main/java/org/opensearch/index/CompositeIndexEventListener.java
@@ -51,6 +51,8 @@
 
 /**
  * A composite {@link IndexEventListener} that forwards all callbacks to an immutable list of IndexEventListener
+ *
+ * @opensearch.internal
  */
 final class CompositeIndexEventListener implements IndexEventListener {
 

diff --git a/server/src/main/java/org/opensearch/index/Index.java b/server/src/main/java/org/opensearch/index/Index.java
@@ -47,6 +47,8 @@
 
 /**
  * A value class representing the basic required properties of an OpenSearch index.
+ *
+ * @opensearch.internal
  */
 public class Index implements Writeable, ToXContentObject {
 

diff --git a/server/src/main/java/org/opensearch/index/IndexComponent.java b/server/src/main/java/org/opensearch/index/IndexComponent.java
@@ -32,6 +32,11 @@
 
 package org.opensearch.index;
 
+/**
+ * Actions that can be executed on an Index Component
+ *
+ * @opensearch.internal
+ */
 public interface IndexComponent {
 
     Index index();

diff --git a/server/src/main/java/org/opensearch/index/IndexModule.java b/server/src/main/java/org/opensearch/index/IndexModule.java
@@ -109,6 +109,8 @@
  *      <li>Settings update listener - Custom settings update listener can be registered via
  *      {@link #addSettingsUpdateConsumer(Setting, Consumer)}</li>
  * </ul>
+ *
+ * @opensearch.internal
  */
 public final class IndexModule {
-Original file line number
+Diff line change
@@ Expand Up / @@ -39,5 +39,6 @@ BWC_VERSION: @@
       - "1.3.0"
       - "1.3.1"
       - "1.3.2"
+      - "1.3.3"
       - "2.0.0"
       - "2.1.0"