Merge branch 'opensearch-project:main' into seg-rep/force-replication

.ci/bwcVersions

@@ -1,47 +1,4 @@
 BWC_VERSION:
-  - "7.0.0"
-  - "7.0.1"
-  - "7.1.0"
-  - "7.1.1"
-  - "7.2.0"
-  - "7.2.1"
-  - "7.3.0"
-  - "7.3.1"
-  - "7.3.2"
-  - "7.4.0"
-  - "7.4.1"
-  - "7.4.2"
-  - "7.5.0"
-  - "7.5.1"
-  - "7.5.2"
-  - "7.6.0"
-  - "7.6.1"
-  - "7.6.2"
-  - "7.7.0"
-  - "7.7.1"
-  - "7.8.0"
-  - "7.8.1"
-  - "7.9.0"
-  - "7.9.1"
-  - "7.9.2"
-  - "7.9.3"
-  - "7.10.0"
-  - "7.10.1"
-  - "7.10.2"
-  - "1.0.0"
-  - "1.1.0"
-  - "1.2.0"
-  - "1.2.1"
-  - "1.2.2"
-  - "1.2.3"
-  - "1.2.4"
-  - "1.2.5"
-  - "1.3.0"
-  - "1.3.1"
-  - "1.3.2"
-  - "1.3.3"
-  - "1.3.4"
-  - "1.3.5"
   - "2.0.0"
   - "2.0.1"
   - "2.0.2"

CHANGELOG.md

@@ -1,12 +1,5 @@
 # CHANGELOG
 
-## [Unreleased]
-### Dependencies
-- Bumps `commons-compress` from 1.21 to 1.22
-- Bumps `jcodings` from 1.0.57 to 1.0.58
-- Bumps `google-http-client-jackson2` from 1.35.0 to 1.42.3
-- Bumps `maxmind-db` from 2.0.0 to 2.1.0
-
 All notable changes to this project are documented in this file.
 
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). See the [CONTRIBUTING guide](./CONTRIBUTING.md#Changelog) for instructions on how to add changelog entries.
@@ -42,6 +35,16 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Bumps `com.diffplug.spotless` from 6.10.0 to 6.11.0 ([#4547](https://github.com/opensearch-project/OpenSearch/pull/4547))
 - Bumps `reactor-core` from 3.4.18 to 3.4.23 ([#4548](https://github.com/opensearch-project/OpenSearch/pull/4548))
 - Bumps `jempbox` from 1.8.16 to 1.8.17 ([#4550](https://github.com/opensearch-project/OpenSearch/pull/4550))
+- Bumps `commons-compress` from 1.21 to 1.22
+- Bumps `jcodings` from 1.0.57 to 1.0.58 ([#5233](https://github.com/opensearch-project/OpenSearch/pull/5233))
+- Bumps `google-http-client-jackson2` from 1.35.0 to 1.42.3 ([#5234](https://github.com/opensearch-project/OpenSearch/pull/5234))
+- Bumps `maxmind-db` from 2.0.0 to 2.1.0 ([#5236](https://github.com/opensearch-project/OpenSearch/pull/5236))
+- Bumps `azure-core` from 1.33.0 to 1.34.0 ([#5235](https://github.com/opensearch-project/OpenSearch/pull/5235))
+- Bumps `azure-core-http-netty` from 1.12.4 to 1.12.7 ([#5235](https://github.com/opensearch-project/OpenSearch/pull/5235))
+- Bumps `spock-core` from from 2.1-groovy-3.0 to 2.3-groovy-3.0 ([#5315](https://github.com/opensearch-project/OpenSearch/pull/5315))
+- Bumps `json-schema-validator` from 1.0.69 to 1.0.73 ([#5316](https://github.com/opensearch-project/OpenSearch/pull/5316))
+- Bumps `proto-google-common-protos` from 2.8.0 to 2.10.0 ([#5318](https://github.com/opensearch-project/OpenSearch/pull/5318))
+- Bumps `protobuf-java` from 3.21.7 to 3.21.9 ([#5319](https://github.com/opensearch-project/OpenSearch/pull/5319))
 - Update Apache Lucene to 9.5.0-snapshot-a4ef70f ([#4979](https://github.com/opensearch-project/OpenSearch/pull/4979))
 - Update to Gradle 7.6 and JDK-19 ([#4973](https://github.com/opensearch-project/OpenSearch/pull/4973))
 
@@ -71,7 +74,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 ### Fixed
 - Fix 'org.apache.hc.core5.http.ParseException: Invalid protocol version' under JDK 16+ ([#4827](https://github.com/opensearch-project/OpenSearch/pull/4827))
 - Fixed compression support for h2c protocol ([#4944](https://github.com/opensearch-project/OpenSearch/pull/4944))
-- Add jvm option to allow security manager ([#5194](https://github.com/opensearch-project/OpenSearch/pull/5194))
+- Reject bulk requests with invalid actions ([#5299](https://github.com/opensearch-project/OpenSearch/issues/5299))
+
 ### Security
 
 ## [Unreleased 2.x]
@@ -90,4 +94,4 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 ### Security
 
 [Unreleased 3.0]: https://github.com/opensearch-project/OpenSearch/compare/2.4...HEAD
-[Unreleased 2.x]: https://github.com/opensearch-project/OpenSearch/compare/2.4...2.x
+[Unreleased 2.x]: https://github.com/opensearch-project/OpenSearch/compare/2.4...2.x

buildSrc/build.gradle

@@ -118,7 +118,7 @@ dependencies {
   api 'com.avast.gradle:gradle-docker-compose-plugin:0.15.2'
   api "org.yaml:snakeyaml:${props.getProperty('snakeyaml')}"
   api 'org.apache.maven:maven-model:3.6.2'
-  api 'com.networknt:json-schema-validator:1.0.69'
+  api 'com.networknt:json-schema-validator:1.0.73'
   api "com.fasterxml.jackson.core:jackson-databind:${props.getProperty('jackson_databind')}"
 
   testFixturesApi "junit:junit:${props.getProperty('junit')}"
@@ -127,7 +127,7 @@ dependencies {
   testFixturesApi gradleTestKit()
   testImplementation 'com.github.tomakehurst:wiremock-jre8-standalone:2.33.2'
   testImplementation "org.mockito:mockito-core:${props.getProperty('mockito')}"
-  integTestImplementation('org.spockframework:spock-core:2.1-groovy-3.0') {
+  integTestImplementation('org.spockframework:spock-core:2.3-groovy-3.0') {
     exclude module: "groovy"
   }
 }

buildSrc/src/main/java/org/opensearch/gradle/BwcVersions.java

@@ -105,7 +105,7 @@ public class BwcVersions {
     private final Map<Integer, List<Version>> groupByMajor;
     private final Map<Version, UnreleasedVersionInfo> unreleased;
 
-    public class UnreleasedVersionInfo {
+    public static class UnreleasedVersionInfo {
         public final Version version;
         public final String branch;
         public final String gradleProjectPath;
@@ -149,13 +149,7 @@ public BwcVersions(SortedSet<Version> allVersions, Version currentVersionPropert
 
         groupByMajor = allVersions.stream()
             // We only care about the last 2 majors when it comes to BWC.
-            // It might take us time to remove the older ones from versionLines, so we allow them to exist.
-            // Adjust the major number since OpenSearch 1.x is released after predecessor version 7.x
-            .filter(
-                version -> (version.getMajor() == 1 ? 7 : version.getMajor()) > (currentVersion.getMajor() == 1
-                    ? 7
-                    : currentVersion.getMajor()) - 2
-            )
+            .filter(version -> version.getMajor() > currentVersion.getMajor() - 2)
             .collect(Collectors.groupingBy(Version::getMajor, Collectors.toList()));
 
         assertCurrentVersionMatchesParsed(currentVersionProperty);
@@ -174,9 +168,7 @@ public BwcVersions(SortedSet<Version> allVersions, Version currentVersionPropert
 
     private void assertNoOlderThanTwoMajors() {
         Set<Integer> majors = groupByMajor.keySet();
-        // until OpenSearch 3.0 we will need to carry three major support
-        // (1, 7, 6) && (2, 1, 7) since OpenSearch 1.0 === Legacy 7.x
-        int numSupportedMajors = (currentVersion.getMajor() < 3) ? 3 : 2;
+        int numSupportedMajors = 2;
         if (majors.size() != numSupportedMajors && currentVersion.getMinor() != 0 && currentVersion.getRevision() != 0) {
             throw new IllegalStateException("Expected exactly 2 majors in parsed versions but found: " + majors);
         }
@@ -207,7 +199,7 @@ public void forPreviousUnreleased(Consumer<UnreleasedVersionInfo> consumer) {
             .map(version -> new UnreleasedVersionInfo(version, getBranchFor(version), getGradleProjectPathFor(version)))
             .collect(Collectors.toList());
 
-        collect.forEach(uvi -> consumer.accept(uvi));
+        collect.forEach(consumer);
     }
 
     private String getGradleProjectPathFor(Version version) {
@@ -271,18 +263,9 @@ public List<Version> getUnreleased() {
         // The current version is being worked, is always unreleased
         unreleased.add(currentVersion);
 
-        // No unreleased versions for 1.0.0
-        // todo remove this hack
-        if (currentVersion.equals(Version.fromString("1.0.0"))) {
-            return unmodifiableList(unreleased);
-        }
-
         // the tip of the previous major is unreleased for sure, be it a minor or a bugfix
         if (currentVersion.getMajor() != 1) {
-            final Version latestOfPreviousMajor = getLatestVersionByKey(
-                this.groupByMajor,
-                currentVersion.getMajor() == 1 ? 7 : currentVersion.getMajor() - 1
-            );
+            final Version latestOfPreviousMajor = getLatestVersionByKey(this.groupByMajor, currentVersion.getMajor() - 1);
             unreleased.add(latestOfPreviousMajor);
             if (latestOfPreviousMajor.getRevision() == 0) {
                 // if the previous major is a x.y.0 release, then the tip of the minor before that (y-1) is also unreleased
@@ -311,7 +294,7 @@ public List<Version> getUnreleased() {
             }
         }
 
-        return unmodifiableList(unreleased.stream().sorted().distinct().collect(Collectors.toList()));
+        return unreleased.stream().sorted().distinct().collect(Collectors.toUnmodifiableList());
     }
 
     private Version getLatestInMinor(int major, int minor) {
@@ -342,7 +325,7 @@ private Map<Integer, List<Version>> getReleasedMajorGroupedByMinor() {
 
     public void compareToAuthoritative(List<Version> authoritativeReleasedVersions) {
         Set<Version> notReallyReleased = new HashSet<>(getReleased());
-        notReallyReleased.removeAll(authoritativeReleasedVersions);
+        authoritativeReleasedVersions.forEach(notReallyReleased::remove);
         if (notReallyReleased.isEmpty() == false) {
             throw new IllegalStateException(
                 "out-of-date released versions"
@@ -370,32 +353,21 @@ private List<Version> getReleased() {
             .stream()
             .flatMap(Collection::stream)
             .filter(each -> unreleased.contains(each) == false)
-            // this is to make sure we only consider OpenSearch versions
-            // TODO remove this filter once legacy ES versions are no longer supported
-            .filter(v -> v.onOrAfter("1.0.0"))
             .collect(Collectors.toList());
     }
 
     public List<Version> getIndexCompatible() {
         int currentMajor = currentVersion.getMajor();
         int prevMajor = getPreviousMajor(currentMajor);
-        List<Version> result = Stream.concat(groupByMajor.get(prevMajor).stream(), groupByMajor.get(currentMajor).stream())
+        return Stream.concat(groupByMajor.get(prevMajor).stream(), groupByMajor.get(currentMajor).stream())
             .filter(version -> version.equals(currentVersion) == false)
-            .collect(Collectors.toList());
-        if (currentMajor == 1) {
-            // add 6.x compatible for OpenSearch 1.0.0
-            return unmodifiableList(Stream.concat(groupByMajor.get(prevMajor - 1).stream(), result.stream()).collect(Collectors.toList()));
-        } else if (currentMajor == 2) {
-            // add 7.x compatible for OpenSearch 2.0.0
-            return unmodifiableList(Stream.concat(groupByMajor.get(7).stream(), result.stream()).collect(Collectors.toList()));
-        }
-        return unmodifiableList(result);
+            .collect(Collectors.toUnmodifiableList());
     }
 
     public List<Version> getWireCompatible() {
         List<Version> wireCompat = new ArrayList<>();
         int currentMajor = currentVersion.getMajor();
-        int lastMajor = currentMajor == 1 ? 6 : currentMajor == 2 ? 7 : currentMajor - 1;
+        int lastMajor = currentMajor - 1;
         List<Version> lastMajorList = groupByMajor.get(lastMajor);
         if (lastMajorList == null) {
             throw new IllegalStateException("Expected to find a list of versions for version: " + lastMajor);
@@ -405,20 +377,6 @@ public List<Version> getWireCompatible() {
             wireCompat.add(lastMajorList.get(i));
         }
 
-        // if current is OpenSearch 1.0.0 add all of the 7.x line:
-        if (currentMajor == 1) {
-            List<Version> previousMajor = groupByMajor.get(7);
-            for (Version v : previousMajor) {
-                wireCompat.add(v);
-            }
-        } else if (currentMajor == 2) {
-            // add all of the 1.x line:
-            List<Version> previousMajor = groupByMajor.get(1);
-            for (Version v : previousMajor) {
-                wireCompat.add(v);
-            }
-        }
-
         wireCompat.addAll(groupByMajor.get(currentMajor));
         wireCompat.remove(currentVersion);
         wireCompat.sort(Version::compareTo);
@@ -438,7 +396,7 @@ public List<Version> getUnreleasedWireCompatible() {
     }
 
     private int getPreviousMajor(int currentMajor) {
-        return currentMajor == 1 ? 7 : currentMajor - 1;
+        return currentMajor - 1;
     }
 
 }

libs/grok/build.gradle

@@ -29,7 +29,7 @@
  */
 
 dependencies {
-  api 'org.jruby.joni:joni:2.1.43'
+  api 'org.jruby.joni:joni:2.1.44'
   // joni dependencies:
   api 'org.jruby.jcodings:jcodings:1.0.58'
 

libs/grok/licenses/joni-2.1.44.jar.sha1

@@ -0,0 +1 @@
+35746c2aee04ce459a2aa8dc2d626946c5dfb051

plugins/repository-azure/build.gradle

@@ -44,9 +44,9 @@ opensearchplugin {
 }
 
 dependencies {
-  api 'com.azure:azure-core:1.33.0'
+  api 'com.azure:azure-core:1.34.0'
   api 'com.azure:azure-storage-common:12.18.1'
-  api 'com.azure:azure-core-http-netty:1.12.4'
+  api 'com.azure:azure-core-http-netty:1.12.7'
   api "io.netty:netty-codec-dns:${versions.netty}"
   api "io.netty:netty-codec-socks:${versions.netty}"
   api "io.netty:netty-codec-http2:${versions.netty}"

plugins/repository-azure/licenses/azure-core-1.34.0.jar.sha1

@@ -0,0 +1 @@
+59827c9aeab1c67053fc598207781e56fb8709f6

plugins/repository-azure/licenses/azure-core-http-netty-1.12.7.jar.sha1

@@ -0,0 +1 @@
+e7739b5c0d9c968afcb6100f15f3491143d47814