Merge branch 'main' into graceful-node-decommission-wrr-2

.github/pull_request_template.md

@@ -1,3 +1,10 @@
+<!--  Thanks for sending a pull request, here are some tips:
+
+1. If this is a fix for an undisclosed security vulnerability, please STOP. All security vulnerability reporting and fixes should be done as per our security policy https://github.com/opensearch-project/OpenSearch/security/policy
+2. If this is your first time, please read our contributor guidelines: https://github.com/opensearch-project/OpenSearch/blob/main/CONTRIBUTING.md and developer guide https://github.com/opensearch-project/OpenSearch/blob/main/DEVELOPER_GUIDE.md
+3. Ensure you have added or ran the appropriate tests for your PR: https://github.com/opensearch-project/OpenSearch/blob/main/TESTING.md
+-->
+
 ### Description
 [Describe what this change achieves]
 

.gitignore

@@ -11,6 +11,7 @@ out/
 !.idea/inspectionProfiles/Project_Default.xml
 !.idea/runConfigurations/Debug_OpenSearch.xml
 !.idea/vcs.xml
+!.idea/icon.svg
 
 # These files are generated in the main tree by IntelliJ
 benchmarks/src/main/generated/*

DEVELOPER_GUIDE.md

@@ -38,21 +38,22 @@
     - [Gradle Plugins](#gradle-plugins)
       - [Distribution Download Plugin](#distribution-download-plugin)
     - [Creating fat-JAR of a Module](#creating-fat-jar-of-a-module)
-  - [Misc](#misc)
-    - [git-secrets](#git-secrets)
-      - [Installation](#installation)
-      - [Configuration](#configuration)
   - [Components](#components)
     - [Build libraries & interfaces](#build-libraries--interfaces)
     - [Clients & Libraries](#clients--libraries)
     - [Plugins](#plugins-1)
-    - [Indexing & search](#indexing--search)
+    - [Indexing & Search](#indexing--search)
     - [Aggregations](#aggregations)
     - [Distributed Framework](#distributed-framework)
-  - [Submitting Changes](#submitting-changes)
-  - [Backports](#backports)
-  - [LineLint](#linelint)
-- [Lucene Snapshots](#lucene-snapshots)
+  - [Misc](#misc)
+    - [Git Secrets](#git-secrets)
+      - [Installation](#installation)
+      - [Configuration](#configuration)
+    - [Submitting Changes](#submitting-changes)
+    - [Backports](#backports)
+    - [LineLint](#linelint)
+    - [Lucene Snapshots](#lucene-snapshots)
+    - [Flakey Tests](#flakey-tests)
 
 # Developer Guide
 
@@ -414,37 +415,12 @@ Refer the installed JAR as any other maven artifact, e.g.
 </dependency>
 ```
 
-## Misc
-
-### git-secrets
-
-Security is our top priority. Avoid checking in credentials.
-
-#### Installation
-Install [awslabs/git-secrets](https://github.com/awslabs/git-secrets) by running the following commands.
-```
-git clone https://github.com/awslabs/git-secrets.git
-cd git-secrets
-make install
-```
-
-#### Configuration
-You can configure git secrets per repository, you need to change the directory to the root of the repository and run the following command.
-```
-git secrets --install
-✓ Installed commit-msg hook to .git/hooks/commit-msg
-✓ Installed pre-commit hook to .git/hooks/pre-commit
-✓ Installed prepare-commit-msg hook to .git/hooks/prepare-commit-msg
-```
-Then, you need to apply patterns for git-secrets, you can install the AWS standard patterns by running the following command.
-```
-git secrets --register-aws
-```
-
 ## Components
+
 As you work in the OpenSearch repo you may notice issues getting labeled with component labels.  It's a housekeeping task to help group together similar pieces of work.  You can pretty much ignore it, but if you're curious, here's what the different labels mean:
 
 ### Build libraries & interfaces
+
 Tasks to make sure the build tasks are useful and packaging and distribution are easy.
 
 Includes:
@@ -458,6 +434,7 @@ Includes:
 
 
 ### Clients & Libraries
+
 APIs and communication mechanisms for external connections to OpenSearch.  This includes the “library” directory in OpenSearch (a set of common functions).
 
 Includes:
@@ -467,6 +444,7 @@ Includes:
 - CLI
 
 ### Plugins
+
 Anything touching the plugin infrastructure within core OpenSearch.
 
 Includes:
@@ -476,7 +454,8 @@ Includes:
 - Plugin interfaces
 
 
-### Indexing & search
+### Indexing & Search
+
 The critical path of indexing and search, including:  Measure index and search, performance, Improving the performance of indexing and search, ensure synchronization OpenSearch APIs with upstream Lucene change (e.g. new field types, changing doc values and codex).
 
 Includes:
@@ -487,6 +466,7 @@ Includes:
 - DocValues
 
 ### Aggregations
+
 Making sure OpenSearch can be used as a compute engine.
 
 Includes:
@@ -495,6 +475,7 @@ Includes:
 - Framework
 
 ### Distributed Framework
+
 Work to make sure that OpenSearch can scale in a distributed manner.
 
 Includes:
@@ -506,15 +487,43 @@ Includes:
 - Shard Strategies
 - Circuit Breakers
 
-## Submitting Changes
+## Misc
+
+### Git Secrets
+
+Security is our top priority. Avoid checking in credentials.
+
+#### Installation
+Install [awslabs/git-secrets](https://github.com/awslabs/git-secrets) by running the following commands.
+```
+git clone https://github.com/awslabs/git-secrets.git
+cd git-secrets
+make install
+```
+
+#### Configuration
+You can configure git secrets per repository, you need to change the directory to the root of the repository and run the following command.
+```
+git secrets --install
+✓ Installed commit-msg hook to .git/hooks/commit-msg
+✓ Installed pre-commit hook to .git/hooks/pre-commit
+✓ Installed prepare-commit-msg hook to .git/hooks/prepare-commit-msg
+```
+Then, you need to apply patterns for git-secrets, you can install the AWS standard patterns by running the following command.
+```
+git secrets --register-aws
+```
+
+### Submitting Changes
 
 See [CONTRIBUTING](CONTRIBUTING.md).
 
-## Backports
+### Backports
 
 The Github workflow in [`backport.yml`](.github/workflows/backport.yml) creates backport PRs automatically when the original PR with an appropriate label `backport <backport-branch-name>` is merged to main with the backport workflow run successfully on the PR. For example, if a PR on main needs to be backported to `1.x` branch, add a label `backport 1.x` to the PR and make sure the backport workflow runs on the PR along with other checks. Once this PR is merged to main, the workflow will create a backport PR to the `1.x` branch.
 
-## LineLint
+### LineLint
+
 A linter in [`code-hygiene.yml`](.github/workflows/code-hygiene.yml) that validates simple newline and whitespace rules in all sorts of files. It can:
 - Recursively check a directory tree for files that do not end in a newline
 - Automatically fix these files by adding a newline or trimming extra newlines.
@@ -529,7 +538,20 @@ Pass a list of files or directories to limit your search.
 
     linelint README.md LICENSE
 
-# Lucene Snapshots
+### Lucene Snapshots
+
 The Github workflow in [lucene-snapshots.yml](.github/workflows/lucene-snapshots.yml) is a Github worfklow executable by maintainers to build a top-down snapshot build of lucene.
-These snapshots are available to test compatibility with upcoming changes to Lucene by updating the version at [version.properties](buildsrc/version.properties) with the `version-snapshot-sha` version.
-Example: `lucene = 10.0.0-snapshot-2e941fc`.
+These snapshots are available to test compatibility with upcoming changes to Lucene by updating the version at [version.properties](buildsrc/version.properties) with the `version-snapshot-sha` version. Example: `lucene = 10.0.0-snapshot-2e941fc`.
+
+### Flakey Tests
+
+OpenSearch has a very large test suite with long running, often failing (flakey), integration tests. Such individual tests are labelled as [Flakey Random Test Failure](https://github.com/opensearch-project/OpenSearch/issues?q=is%3Aopen+is%3Aissue+label%3A%22Flakey+Random+Test+Failure%22). Your help is wanted fixing these!
+
+If you encounter a build/test failure in CI that is unrelated to the change in your pull request, it may be a known flakey test, or a new test failure.
+
+1. Follow failed CI links, and locate the failing test(s).
+2. Copy-paste the failure into a comment of your PR.
+3. Search through [issues](https://github.com/opensearch-project/OpenSearch/issues?q=is%3Aopen+is%3Aissue+label%3A%22Flakey+Random+Test+Failure%22) using the name of the failed test for whether this is a known flakey test. 
+5. If an existing issue is found, paste a link to the known issue in a comment to your PR.
+6. If no existing issue is found, open one.
+7. Retry CI via the GitHub UX or by pushing an update to your PR.

qa/mixed-cluster/build.gradle

@@ -38,6 +38,10 @@ apply plugin: 'opensearch.standalone-test'
 apply from : "$rootDir/gradle/bwc-test.gradle"
 apply plugin: 'opensearch.rest-resources'
 
+dependencies {
+  testImplementation project(":client:rest-high-level")
+}
+
 restResources {
   restTests {
     includeCore '*'

qa/mixed-cluster/src/test/java/org/opensearch/backwards/SearchingIT.java

@@ -0,0 +1,78 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.backwards;
+
+import org.apache.hc.core5.http.HttpHost;
+import org.opensearch.LegacyESVersion;
+import org.opensearch.Version;
+import org.opensearch.action.get.MultiGetRequest;
+import org.opensearch.action.get.MultiGetResponse;
+import org.opensearch.backwards.IndexingIT.Nodes;
+import org.opensearch.client.Request;
+import org.opensearch.client.RequestOptions;
+import org.opensearch.client.Response;
+import org.opensearch.client.ResponseException;
+import org.opensearch.client.RestClient;
+import org.opensearch.client.RestHighLevelClient;
+import org.opensearch.cluster.metadata.IndexMetadata;
+import org.opensearch.common.Strings;
+import org.opensearch.common.settings.Settings;
+import org.opensearch.common.xcontent.json.JsonXContent;
+import org.opensearch.common.xcontent.support.XContentMapValues;
+import org.opensearch.index.seqno.SeqNoStats;
+import org.opensearch.rest.RestStatus;
+import org.opensearch.test.rest.OpenSearchRestTestCase;
+import org.opensearch.test.rest.yaml.ObjectPath;
+
+import java.io.IOException;
+import java.net.URISyntaxException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
+
+public class SearchingIT extends OpenSearchRestTestCase {
+    public void testMultiGet() throws Exception {
+        final Set<HttpHost> nodes = buildNodes();
+
+        final MultiGetRequest multiGetRequest = new MultiGetRequest();
+        multiGetRequest.add("index", "id1");
+
+        try (RestHighLevelClient client = new RestHighLevelClient(RestClient.builder(nodes.toArray(HttpHost[]::new)))) {
+            MultiGetResponse response = client.mget(multiGetRequest, RequestOptions.DEFAULT);
+            assertEquals(1, response.getResponses().length);
+
+            assertTrue(response.getResponses()[0].isFailed());
+            assertNotNull(response.getResponses()[0].getFailure());
+            assertEquals(response.getResponses()[0].getFailure().getId(), "id1");
+            assertEquals(response.getResponses()[0].getFailure().getIndex(), "index");
+            assertThat(response.getResponses()[0].getFailure().getMessage(), containsString("no such index [index]"));
+       }
+    }
+
+    private Set<HttpHost> buildNodes() throws IOException, URISyntaxException {
+        Response response = client().performRequest(new Request("GET", "_nodes"));
+        ObjectPath objectPath = ObjectPath.createFromResponse(response);
+        Map<String, Object> nodesAsMap = objectPath.evaluate("nodes");
+        final Set<HttpHost> nodes = new HashSet<>();
+        for (String id : nodesAsMap.keySet()) {
+            nodes.add(HttpHost.create((String) objectPath.evaluate("nodes." + id + ".http.publish_address")));
+        }
+
+        return nodes;
+    }
+}

rest-api-spec/src/main/resources/rest-api-spec/test/index/80_geo_point.yml

@@ -12,6 +12,9 @@ setup:
 
 ---
 "Single point test":
+  - skip:
+      version: " - 2.3.99"
+      reason: "geojson format is supported in 2.4 and above"
   - do:
       bulk:
         refresh: true
@@ -75,6 +78,9 @@ setup:
 
 ---
 "Multi points test":
+  - skip:
+      version: " - 2.3.99"
+      reason: "geojson format is supported in 2.4 and above"
   - do:
       bulk:
         refresh: true

server/src/main/java/org/opensearch/action/get/MultiGetResponse.java

@@ -63,6 +63,10 @@ public class MultiGetResponse extends ActionResponse implements Iterable<MultiGe
     private static final ParseField ID = new ParseField("_id");
     private static final ParseField ERROR = new ParseField("error");
     private static final ParseField DOCS = new ParseField("docs");
+    // In mixed clusters, the 1.x cluster could still return the '_type' in the response payload, it has to
+    // be handled gracefully
+    @Deprecated(forRemoval = true)
+    private static final ParseField TYPE = new ParseField("_type");
 
     /**
      * Represents a failure.
@@ -212,6 +216,7 @@ private static MultiGetItemResponse parseItem(XContentParser parser) throws IOEx
                     currentFieldName = parser.currentName();
                     if (INDEX.match(currentFieldName, parser.getDeprecationHandler()) == false
                         && ID.match(currentFieldName, parser.getDeprecationHandler()) == false
+                        && TYPE.match(currentFieldName, parser.getDeprecationHandler()) == false
                         && ERROR.match(currentFieldName, parser.getDeprecationHandler()) == false) {
                         getResult = GetResult.fromXContentEmbedded(parser, index, id);
                     }

server/src/main/java/org/opensearch/index/snapshots/blobstore/SlicedInputStream.java

@@ -35,6 +35,7 @@
 
 import java.io.IOException;
 import java.io.InputStream;
+import java.util.Objects;
 
 /**
  *  A {@link SlicedInputStream} is a logical
@@ -100,6 +101,11 @@ public final int read() throws IOException {
 
     @Override
     public final int read(byte[] buffer, int offset, int length) throws IOException {
+        Objects.checkFromIndexSize(offset, length, buffer.length);
+        if (length == 0) {
+            return 0;
+        }
+
         final InputStream stream = currentStream();
         if (stream == null) {
             return -1;