Merge main into replace_latches

Signed-off-by: Ryan Bogan <rbogan@amazon.com>
opensearch-project · Jan 18, 2023 · 269b146 · 269b146
2 parents b3610fd + 60db7b5
commit 269b146
Show file tree

Hide file tree

Showing 78 changed files with 1,389 additions and 224 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -3,6 +3,7 @@
 ## [Unreleased]
 ### Dependencies
 - Bumps `azure-core-http-netty` from 1.12.7 to 1.12.8
+- Bumps `reactor-netty` from 1.1.1 to 1.1.2
 
 
 All notable changes to this project are documented in this file.
@@ -32,12 +33,13 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Added support for feature flags in opensearch.yml ([#4959](https://github.com/opensearch-project/OpenSearch/pull/4959))
 - Add query for initialized extensions ([#5658](https://github.com/opensearch-project/OpenSearch/pull/5658))
 - Revert 'Added jackson dependency to server' and change extension reading ([#5768](https://github.com/opensearch-project/OpenSearch/pull/5768))
+- Add support to disallow search request with preference parameter with strict weighted shard routing([#5874](https://github.com/opensearch-project/OpenSearch/pull/5874))
 - Replace latches with CompletableFutures for extensions ([#5646](https://github.com/opensearch-project/OpenSearch/pull/5646))
 
 ### Dependencies
 - Bumps `log4j-core` from 2.18.0 to 2.19.0
 - Bumps `reactor-netty-http` from 1.0.18 to 1.0.23
-- Bumps `jettison` from 1.5.0 to 1.5.1
+- Bumps `jettison` from 1.5.0 to 1.5.3
 - Bumps `forbiddenapis` from 3.3 to 3.4
 - Bumps `gson` from 2.9.0 to 2.10
 - Bumps `avro` from 1.11.0 to 1.11.1
@@ -107,6 +109,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 
 ## [Unreleased 2.x]
 ### Added
+- Added cluster manager throttling stats in nodes/stats API ([#5790](https://github.com/opensearch-project/OpenSearch/pull/5790))
 
 ### Dependencies
 
@@ -121,4 +124,4 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 ### Security
 
 [Unreleased 3.0]: https://github.com/opensearch-project/OpenSearch/compare/2.4...HEAD
-[Unreleased 2.x]: https://github.com/opensearch-project/OpenSearch/compare/2.4...2.x
+[Unreleased 2.x]: https://github.com/opensearch-project/OpenSearch/compare/2.4...2.x
diff --git a/buildSrc/version.properties b/buildSrc/version.properties
@@ -17,7 +17,7 @@ supercsv          = 2.4.0
 log4j             = 2.17.1
 slf4j             = 1.7.36
 asm               = 9.4
-jettison          = 1.5.1
+jettison          = 1.5.3
 woodstox          = 6.4.0
 kotlin            = 1.7.10
 antlr4            = 4.11.1

diff --git a/plugins/discovery-azure-classic/licenses/jettison-1.5.1.jar.sha1 b/plugins/discovery-azure-classic/licenses/jettison-1.5.1.jar.sha1
diff --git a/plugins/discovery-azure-classic/licenses/jettison-1.5.3.jar.sha1 b/plugins/discovery-azure-classic/licenses/jettison-1.5.3.jar.sha1
@@ -0,0 +1 @@
+964d35bbdecbbc33cf2f2044e8a1648d9f6f1474
diff --git a/plugins/repository-azure/build.gradle b/plugins/repository-azure/build.gradle
@@ -57,7 +57,7 @@ dependencies {
   api 'com.azure:azure-storage-blob:12.20.0'
   api 'org.reactivestreams:reactive-streams:1.0.4'
   api 'io.projectreactor:reactor-core:3.5.1'
-  api 'io.projectreactor.netty:reactor-netty:1.1.1'
+  api 'io.projectreactor.netty:reactor-netty:1.1.2'
   api 'io.projectreactor.netty:reactor-netty-core:1.0.24'
   api 'io.projectreactor.netty:reactor-netty-http:1.0.24'
   api "org.slf4j:slf4j-api:${versions.slf4j}"

diff --git a/plugins/repository-azure/licenses/reactor-netty-1.1.1.jar.sha1 b/plugins/repository-azure/licenses/reactor-netty-1.1.1.jar.sha1
diff --git a/plugins/repository-azure/licenses/reactor-netty-1.1.2.jar.sha1 b/plugins/repository-azure/licenses/reactor-netty-1.1.2.jar.sha1
@@ -0,0 +1 @@
+e3bc060179a0776016f629e3b1a1d1c8e51b7112
diff --git a/release-notes/opensearch.release-notes-2.5.0.md b/release-notes/opensearch.release-notes-2.5.0.md
@@ -2,37 +2,56 @@
 
 ## [2.5]
 ### Added
-- Added jackson dependency to server ([#5366](https://github.com/opensearch-project/OpenSearch/pull/5366))
-- Added experimental support for extensions ([#5347](https://github.com/opensearch-project/OpenSearch/pull/5347)), ([#5518](https://github.com/opensearch-project/OpenSearch/pull/5518)), ([#5597](https://github.com/opensearch-project/OpenSearch/pull/5597)), ([#5615](https://github.com/opensearch-project/OpenSearch/pull/5615)))
-- Adding support to register settings dynamically ([#5495](https://github.com/opensearch-project/OpenSearch/pull/5495))
-- Adding auto release workflow ([#5582](https://github.com/opensearch-project/OpenSearch/pull/5582))
+- Prevent deletion of snapshots that are backing searchable snapshot indexes ([#5365](https://github.com/opensearch-project/OpenSearch/pull/5365))
+- Reject bulk requests with invalid actions ([#5384](https://github.com/opensearch-project/OpenSearch/issues/5384))
+- Add max_shard_size parameter for shrink API ([#5229](https://github.com/opensearch-project/OpenSearch/pull/5229))
+- Add experimental support for extensions ([#5347](https://github.com/opensearch-project/OpenSearch/pull/5347)), ([#5518](https://github.com/opensearch-project/OpenSearch/pull/5518)), ([#5597](https://github.com/opensearch-project/OpenSearch/pull/5597)), ([#5615](https://github.com/opensearch-project/OpenSearch/pull/5615)))
+- Add support to register settings dynamically ([#5495](https://github.com/opensearch-project/OpenSearch/pull/5495))
+- Add auto release workflow ([#5582](https://github.com/opensearch-project/OpenSearch/pull/5582))
 - Add CI bundle pattern to distribution download ([#5348](https://github.com/opensearch-project/OpenSearch/pull/5348))
 - Experimental support for extended backward compatiblity in searchable snapshots ([#5429](https://github.com/opensearch-project/OpenSearch/pull/5429))
 - Add support of default replica count cluster setting ([#5610](https://github.com/opensearch-project/OpenSearch/pull/5610))
 - Add support for refresh level durability ([#5253](https://github.com/opensearch-project/OpenSearch/pull/5253))
-- Added Request level Durability using Remote Translog functionality ([#5757](https://github.com/opensearch-project/OpenSearch/pull/5757))
+- Add Request level Durability using Remote Translog functionality ([#5757](https://github.com/opensearch-project/OpenSearch/pull/5757))
 - Support to fail open requests on search shard failures with weighted traffic routing ([#5072](https://github.com/opensearch-project/OpenSearch/pull/5072))
 - Support versioning for Weighted routing apis([#5255](https://github.com/opensearch-project/OpenSearch/pull/5255))
 - Add support for discovered cluster manager and remove local weights ([#5680](https://github.com/opensearch-project/OpenSearch/pull/5680))
-- Added new level to get health per awareness attribute in _cluster/health ([#5694](https://github.com/opensearch-project/OpenSearch/pull/5694))
+- Add new level to get health per awareness attribute in _cluster/health ([#5694](https://github.com/opensearch-project/OpenSearch/pull/5694))
 
 ### Changed
+- Change http code for DecommissioningFailedException from 500 to 400 ([#5283](https://github.com/opensearch-project/OpenSearch/pull/5283))
 - Pre conditions check before updating weighted routing metadata ([#4955](https://github.com/opensearch-project/OpenSearch/pull/4955))
-- Support remote translog transfer for request level durability([#4480](https://github.com/opensearch-project/OpenSearch/pull/4480))
+- Support remote translog transfer for request level durability ([#4480](https://github.com/opensearch-project/OpenSearch/pull/4480))
 - Gracefully handle concurrent zone decommission action ([#5542](https://github.com/opensearch-project/OpenSearch/pull/5542))
 
 ### Deprecated
 - Refactor fuzziness interface on query builders ([#5433](https://github.com/opensearch-project/OpenSearch/pull/5433))
 
 ### Fixed
+- Fix case sensitivity for wildcard queries ([#5462](https://github.com/opensearch-project/OpenSearch/pull/5462))
 - Apply cluster manager throttling settings during bootstrap ([#5524](https://github.com/opensearch-project/OpenSearch/pull/5524))
 - Update thresholds map when cluster manager throttling setting is removed ([#5524](https://github.com/opensearch-project/OpenSearch/pull/5524))
 - Fix backward compatibility for static cluster manager throttling threshold setting ([#5633](https://github.com/opensearch-project/OpenSearch/pull/5633))
 - Fix index exclusion behavior in snapshot restore and clone APIs ([#5626](https://github.com/opensearch-project/OpenSearch/pull/5626))
 - Fix graph filter error in search ([#5665](https://github.com/opensearch-project/OpenSearch/pull/5665))
-- [Test] Renaming PIT tests to IT to fix intermittent test failures ([#5750](https://github.com/opensearch-project/OpenSearch/pull/5750))
 
 ### Dependencies
+- Bumps `bcpg-fips` from 1.0.5.1 to 1.0.7.1 ([#5148](https://github.com/opensearch-project/OpenSearch/pull/5148))
+- Bumps `commons-compress` from 1.21 to 1.22 ([#5104](https://github.com/opensearch-project/OpenSearch/pull/5104))
+- Bumps `geoip2` from 3.0.1 to 3.0.2 in /modules/ingest-geoip ([#5201](https://github.com/opensearch-project/OpenSearch/pull/5201))
+- Bumps `gson` from 2.9.0 to 2.10 in /plugins/repository-hdfs ([#5184](https://github.com/opensearch-project/OpenSearch/pull/5184))
+- Bumps `protobuf-java` from 3.21.8 to 3.21.9 in /test/fixtures/hdfs-fixture ([#5185](https://github.com/opensearch-project/OpenSearch/pull/5185))
+- Bumps `gradle-extra-configurations-plugin` from 7.0.0 to 8.0.0 ([#5330](https://github.com/opensearch-project/OpenSearch/pull/5330))
+- Bumps `jcodings` from 1.0.57 to 1.0.58 ([#5330](https://github.com/opensearch-project/OpenSearch/pull/5330))
+- Bumps `google-http-client-jackson2` from 1.35.0 to 1.42.3 ([#5330](https://github.com/opensearch-project/OpenSearch/pull/5330))
+- Bumps `azure-core` from 1.33.0 to 1.34.0 ([#5330](https://github.com/opensearch-project/OpenSearch/pull/5330))
+- Bumps `azure-core-http-netty` from 1.12.4 to 1.12.7 ([#5330](https://github.com/opensearch-project/OpenSearch/pull/5330))
+- Bumps `maxmind-db` from 2.0.0 to 2.1.0 ([#5330](https://github.com/opensearch-project/OpenSearch/pull/5330))
+- Bumps `json-schema-validator` from 1.0.69 to 1.0.73 ([#5330](https://github.com/opensearch-project/OpenSearch/pull/5330))
+- Bumps `proto-google-common-protos` from 2.8.0 to 2.10.0 ([#5330](https://github.com/opensearch-project/OpenSearch/pull/5330))
+- Bumps `protobuf-java` from 3.21.7 to 3.21.9 ([#5330](https://github.com/opensearch-project/OpenSearch/pull/5330))
+- Bumps `gradle` from 7.5 to 7.6 ([#5382](https://github.com/opensearch-project/OpenSearch/pull/5382))
+- Bumps `jackson` from 2.14.0 to 2.14.1 ([#5355](https://github.com/opensearch-project/OpenSearch/pull/5355))
 - Bumps `apache-rat` from 0.13 to 0.15 ([#5686](https://github.com/opensearch-project/OpenSearch/pull/5686))
 - Bumps `reactor-netty` from 1.0.18 to 1.1.1 ([#5685](https://github.com/opensearch-project/OpenSearch/pull/5685))
 - Bumps `gradle-info-plugin` from 7.1.3 to 12.0.0 ([#5684](https://github.com/opensearch-project/OpenSearch/pull/5684))
diff --git a/server/src/internalClusterTest/java/org/opensearch/search/SearchWeightedRoutingIT.java b/server/src/internalClusterTest/java/org/opensearch/search/SearchWeightedRoutingIT.java
@@ -20,13 +20,15 @@
 import org.opensearch.action.search.SearchResponse;
 import org.opensearch.cluster.node.DiscoveryNode;
 import org.opensearch.cluster.node.DiscoveryNodes;
+import org.opensearch.cluster.routing.PreferenceBasedSearchNotAllowedException;
 import org.opensearch.cluster.routing.WeightedRouting;
 import org.opensearch.cluster.routing.allocation.decider.AwarenessAllocationDecider;
 import org.opensearch.common.collect.ImmutableOpenMap;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.index.query.QueryBuilders;
 import org.opensearch.index.search.stats.SearchStats;
 import org.opensearch.plugins.Plugin;
+import org.opensearch.rest.RestStatus;
 import org.opensearch.search.aggregations.Aggregations;
 import org.opensearch.search.aggregations.bucket.terms.Terms;
 import org.opensearch.snapshots.mockstore.MockRepository;
@@ -796,4 +798,75 @@ public void testMultiGetWithNetworkDisruption_FailOpenDisabled() throws Exceptio
         );
     }
 
+    /**
+     * Assert that preference based search is not allowed with strict weighted shard routing
+     * @throws Exception throws exception
+     */
+    public void testStrictWeightedRouting() throws Exception {
+
+        Settings commonSettings = Settings.builder()
+            .put("cluster.routing.allocation.awareness.attributes", "zone")
+            .put("cluster.routing.allocation.awareness.force.zone.values", "a,b,c")
+            .put("cluster.routing.weighted.fail_open", true)
+            .put("cluster.routing.weighted.strict", true)
+            .build();
+
+        int nodeCountPerAZ = 1;
+        Map<String, List<String>> nodeMap = setupCluster(nodeCountPerAZ, commonSettings);
+
+        int numShards = 10;
+        int numReplicas = 1;
+        setUpIndexing(numShards, numReplicas);
+
+        logger.info("--> setting shard routing weights for weighted round robin");
+        Map<String, Double> weights = Map.of("a", 1.0, "b", 1.0, "c", 0.0);
+        setShardRoutingWeights(weights);
+        String nodeInZoneA = nodeMap.get("a").get(0);
+        String customPreference = randomAlphaOfLength(10);
+
+        assertThrows(
+            PreferenceBasedSearchNotAllowedException.class,
+            () -> internalCluster().client(nodeMap.get("b").get(0))
+                .prepareSearch()
+                .setSize(0)
+                .setPreference(randomFrom("_local", "_only_nodes:" + nodeInZoneA, "_prefer_nodes:" + nodeInZoneA, customPreference))
+                .get()
+        );
+
+    }
+
+    /**
+     *  Assert that preference based search works with non-strict weighted shard routing
+     * @throws Exception
+     */
+    public void testPreferenceSearchWithWeightedRouting() throws Exception {
+        Settings commonSettings = Settings.builder()
+            .put("cluster.routing.allocation.awareness.attributes", "zone")
+            .put("cluster.routing.allocation.awareness.force.zone.values", "a,b,c")
+            .put("cluster.routing.weighted.fail_open", true)
+            .put("cluster.routing.weighted.strict", false)
+            .build();
+
+        int nodeCountPerAZ = 1;
+        Map<String, List<String>> nodeMap = setupCluster(nodeCountPerAZ, commonSettings);
+
+        int numShards = 10;
+        int numReplicas = 1;
+        setUpIndexing(numShards, numReplicas);
+
+        logger.info("--> setting shard routing weights for weighted round robin");
+        Map<String, Double> weights = Map.of("a", 1.0, "b", 1.0, "c", 0.0);
+        setShardRoutingWeights(weights);
+
+        String customPreference = randomAlphaOfLength(10);
+        String nodeInZoneA = nodeMap.get("a").get(0);
+
+        SearchResponse searchResponse = internalCluster().client(nodeMap.get("b").get(0))
+            .prepareSearch()
+            .setSize(0)
+            .setPreference(randomFrom("_local", "_only_nodes:" + nodeInZoneA, "_prefer_nodes:" + nodeInZoneA, customPreference))
+            .get();
+        assertEquals(RestStatus.OK.getStatus(), searchResponse.status().getStatus());
+    }
+
 }
diff --git a/server/src/main/java/org/opensearch/LegacyESVersion.java b/server/src/main/java/org/opensearch/LegacyESVersion.java
@@ -48,6 +48,10 @@
  */
 public class LegacyESVersion extends Version {
 
+    public static final LegacyESVersion V_6_0_0 = new LegacyESVersion(6000099, org.apache.lucene.util.Version.fromBits(7, 0, 0));
+    public static final LegacyESVersion V_6_5_0 = new LegacyESVersion(6050099, org.apache.lucene.util.Version.fromBits(7, 0, 0));
+    public static final LegacyESVersion V_7_2_0 = new LegacyESVersion(7020099, org.apache.lucene.util.Version.LUCENE_8_0_0);
+
     // todo move back to Version.java if retiring legacy version support
     protected static final ImmutableOpenIntMap<Version> idToVersion;
     protected static final ImmutableOpenMap<String, Version> stringToVersion;
@@ -237,4 +241,22 @@ public String toString() {
         }
         return sb.toString();
     }
+
+    @Override
+    protected Version computeMinIndexCompatVersion() {
+        final int prevLuceneVersionMajor = this.luceneVersion.major - 1;
+        final int bwcMajor;
+        if (major == 5) {
+            bwcMajor = 2; // we jumped from 2 to 5
+        } else if (major == 7) {
+            return LegacyESVersion.fromId(6000026);
+        } else {
+            bwcMajor = major - 1;
+        }
+        final int bwcMinor = 0;
+        return new LegacyESVersion(
+            bwcMajor * 1000000 + bwcMinor * 10000 + 99,
+            org.apache.lucene.util.Version.fromBits(prevLuceneVersionMajor, 0, 0)
+        );
+    }
 }
diff --git a/server/src/main/java/org/opensearch/OpenSearchException.java b/server/src/main/java/org/opensearch/OpenSearchException.java
@@ -34,6 +34,7 @@
 
 import org.opensearch.action.support.replication.ReplicationOperation;
 import org.opensearch.cluster.action.shard.ShardStateAction;
+import org.opensearch.cluster.routing.PreferenceBasedSearchNotAllowedException;
 import org.opensearch.cluster.routing.UnsupportedWeightedRoutingStateException;
 import org.opensearch.cluster.service.ClusterManagerThrottlingException;
 import org.opensearch.common.CheckedFunction;
@@ -72,6 +73,7 @@
 import static org.opensearch.Version.V_2_1_0;
 import static org.opensearch.Version.V_2_4_0;
 import static org.opensearch.Version.V_2_5_0;
+import static org.opensearch.Version.V_2_6_0;
 import static org.opensearch.Version.V_3_0_0;
 import static org.opensearch.cluster.metadata.IndexMetadata.INDEX_UUID_NA_VALUE;
 import static org.opensearch.common.xcontent.XContentParserUtils.ensureExpectedToken;
@@ -1632,6 +1634,12 @@ private enum OpenSearchExceptionHandle {
             167,
             V_2_5_0
         ),
+        PREFERENCE_BASED_SEARCH_NOT_ALLOWED_EXCEPTION(
+            PreferenceBasedSearchNotAllowedException.class,
+            PreferenceBasedSearchNotAllowedException::new,
+            168,
+            V_2_6_0
+        ),
         INDEX_CREATE_BLOCK_EXCEPTION(
             org.opensearch.cluster.block.IndexCreateBlockException.class,
             org.opensearch.cluster.block.IndexCreateBlockException::new,

diff --git a/server/src/main/java/org/opensearch/Version.java b/server/src/main/java/org/opensearch/Version.java
@@ -316,8 +316,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
      * lazily once.
      */
     static class DeclaredVersionsHolder {
-        // use LegacyESVersion.class since it inherits Version fields
-        protected static final List<Version> DECLARED_VERSIONS = Collections.unmodifiableList(getDeclaredVersions(LegacyESVersion.class));
+        protected static final List<Version> DECLARED_VERSIONS = Collections.unmodifiableList(getDeclaredVersions(Version.class));
     }
 
     // lazy initialized because we don't yet have the declared versions ready when instantiating the cached Version
@@ -394,7 +393,7 @@ public Version minimumIndexCompatibilityVersion() {
         return res;
     }
 
-    private Version computeMinIndexCompatVersion() {
+    protected Version computeMinIndexCompatVersion() {
         final int bwcMajor;
         if (major == 5) {
             bwcMajor = 2; // we jumped from 2 to 5