Merge branch 'main' into dependabot/gradle/buildSrc/com.netflix.nebul…

…a-gradle-extra-configurations-plugin-10.0.0

Signed-off-by: Tianli Feng <ftianli@amazon.com>

.ci/bwcVersions

@@ -17,3 +17,4 @@ BWC_VERSION:
   - "2.6.0"
   - "2.6.1"
   - "2.7.0"
+  - "2.8.0"

.github/dependabot.yml

@@ -207,6 +207,14 @@ updates:
     labels:
       - "dependabot"
       - "dependencies"
+  - directory: /distribution/archives/darwin-arm64-tar/
+    open-pull-requests-limit: 1
+    package-ecosystem: gradle
+    schedule:
+      interval: weekly
+    labels:
+      - "dependabot"
+      - "dependencies"
   - directory: /distribution/archives/integ-test-zip/
     open-pull-requests-limit: 1
     package-ecosystem: gradle

.github/pull_request_template.md

@@ -8,8 +8,9 @@
 ### Description
 [Describe what this change achieves]
 
-### Issues Resolved
-[List any issues this PR will resolve]
+### Related Issues
+Resolves #[Issue number to be closed when this PR is merged]
+<!-- List any other related issues here -->
 
 ### Check List
 - [ ] New functionality includes testing.

.github/workflows/version.yml

@@ -105,7 +105,7 @@ jobs:
           echo Adding bwc version $NEXT_VERSION after $CURRENT_VERSION
           sed -i "s/- \"$CURRENT_VERSION\"/\0\n  - \"$NEXT_VERSION\"/g" .ci/bwcVersions
           echo Adding $NEXT_VERSION_UNDERSCORE after $CURRENT_VERSION_UNDERSCORE
-          sed -i "s/public static final Version $CURRENT_VERSION_UNDERSCORE = new Version(\([[:digit:]]\+\)\(.*\));/\0\n    public static final Version $NEXT_VERSION_UNDERSCORE = new Version($NEXT_VERSION_ID\2);/g" server/src/main/java/org/opensearch/Version.java
+          sed -i "s/public static final Version $CURRENT_VERSION_UNDERSCORE = new Version(\([[:digit:]]\+\)\(.*\));/\0\n    public static final Version $NEXT_VERSION_UNDERSCORE = new Version($NEXT_VERSION_ID\2);/g" libs/core/src/main/java/org/opensearch/Version.java
 
       - name: Create Pull Request
         uses: peter-evans/create-pull-request@v3

CHANGELOG.md

@@ -8,6 +8,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Support for HTTP/2 (server-side) ([#3847](https://github.com/opensearch-project/OpenSearch/pull/3847))
 - Add getter for path field in NestedQueryBuilder ([#4636](https://github.com/opensearch-project/OpenSearch/pull/4636))
 - Allow mmap to use new JDK-19 preview APIs in Apache Lucene 9.4+ ([#5151](https://github.com/opensearch-project/OpenSearch/pull/5151))
+- Add events correlation engine plugin ([#6854](https://github.com/opensearch-project/OpenSearch/issues/6854))
+- Add connectToNodeAsExtension in TransportService ([#6866](https://github.com/opensearch-project/OpenSearch/pull/6866))
 
 ### Dependencies
 - Bump `log4j-core` from 2.18.0 to 2.19.0
@@ -16,7 +18,6 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Bump `woodstox-core` from 6.3.0 to 6.3.1
 - Bump `xmlbeans` from 5.1.0 to 5.1.1 ([#4354](https://github.com/opensearch-project/OpenSearch/pull/4354))
 - Bump `azure-storage-common` from 12.18.0 to 12.18.1 ([#4164](https://github.com/opensearch-project/OpenSearch/pull/4664))
-- Bump `org.gradle.test-retry` from 1.4.0 to 1.5.2
 - Bump `reactor-netty-core` from 1.0.19 to 1.0.22 ([#4447](https://github.com/opensearch-project/OpenSearch/pull/4447))
 - Bump `reactive-streams` from 1.0.3 to 1.0.4 ([#4488](https://github.com/opensearch-project/OpenSearch/pull/4488))
 - Bump `reactor-core` from 3.4.23 to 3.5.1 ([#5604](https://github.com/opensearch-project/OpenSearch/pull/5604))
@@ -36,16 +37,17 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Bump `com.google.code.gson:gson` from 2.10 to 2.10.1
 - Bump `com.maxmind.geoip2:geoip2` from 4.0.0 to 4.0.1
 - Bump `com.networknt:json-schema-validator` from 1.0.76 to 1.0.78
-- Bump `com.netflix.nebula:gradle-info-plugin` from 12.0.0 to 12.0.1
+- Bump `com.netflix.nebula:gradle-info-plugin` from 12.0.0 to 12.1.0
 - Bump `com.avast.gradle:gradle-docker-compose-plugin` from 0.16.11 to 0.16.12
 - Bump `org.apache.commons:commons-compress` from 1.22 to 1.23.0
 - Bump `org.apache.commons:commons-configuration2` from 2.8.0 to 2.9.0
-- Bump `com.netflix.nebula:nebula-publishing-plugin` from 19.2.0 to 20.2.0
-- Bump `net.minidev:json-smart` from 2.4.9 to 2.4.10
-- Bump `com.netflix.nebula.ospackage-base` from 11.0.0 to 11.2.0
-- Bump `org.apache.hadoop:hadoop-minicluster` from 3.3.4 to 3.3.5
+- Bump `com.netflix.nebula:nebula-publishing-plugin` from 19.2.0 to 20.3.0
+- Bump `com.netflix.nebula.ospackage-base` from 11.0.0 to 11.3.0
 - Bump `org.apache.commons:commons-compress` from 1.22 to 1.23.0
 - Bump `com.netflix.nebula:gradle-extra-configurations-plugin` from 9.0.0 to 10.0.0 in /buildSrc ([#7068](https://github.com/opensearch-project/OpenSearch/pull/7068))
+- Bump `com.google.protobuf:protobuf-java` from 3.22.2 to 3.22.3
+- Bump `com.diffplug.spotless` from 6.17.0 to 6.18.0
+- Bump `io.opencensus:opencensus-api` from 0.18.0 to 0.31.1 ([#7291](https://github.com/opensearch-project/OpenSearch/pull/7291))
 
 ### Changed
 - [CCR] Add getHistoryOperationsFromTranslog method to fetch the history snapshot from translogs ([#3948](https://github.com/opensearch-project/OpenSearch/pull/3948))
@@ -77,66 +79,28 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Fix 'org.apache.hc.core5.http.ParseException: Invalid protocol version' under JDK 16+ ([#4827](https://github.com/opensearch-project/OpenSearch/pull/4827))
 - Fix compression support for h2c protocol ([#4944](https://github.com/opensearch-project/OpenSearch/pull/4944))
 - Support OpenSSL Provider with default Netty allocator ([#5460](https://github.com/opensearch-project/OpenSearch/pull/5460))
-- Avoid negative memory result in IndicesQueryCache stats calculation ([#6917](https://github.com/opensearch-project/OpenSearch/pull/6917))
+- Replaces ZipInputStream with ZipFile to fix Zip Slip vulnerability ([#7230](https://github.com/opensearch-project/OpenSearch/pull/7230))
 
 ### Security
 
 ## [Unreleased 2.x]
 ### Added
-- Add GeoTile and GeoHash Grid aggregations on GeoShapes. ([#5589](https://github.com/opensearch-project/OpenSearch/pull/5589))
-- Disallow multiple data paths for search nodes ([#6427](https://github.com/opensearch-project/OpenSearch/pull/6427))
-- [Segment Replication] Allocation and rebalancing based on average primary shard count per index ([#6422](https://github.com/opensearch-project/OpenSearch/pull/6422))
-- The truncation limit of the OpenSearchJsonLayout logger is now configurable ([#6569](https://github.com/opensearch-project/OpenSearch/pull/6569))
-- Add 'base_path' setting to File System Repository ([#6558](https://github.com/opensearch-project/OpenSearch/pull/6558))
-- Return success on DeletePits when no PITs exist. ([#6544](https://github.com/opensearch-project/OpenSearch/pull/6544))
-- Add initial search pipelines ([#6587](https://github.com/opensearch-project/OpenSearch/pull/6587))
-- Add node repurpose command for search nodes ([#6517](https://github.com/opensearch-project/OpenSearch/pull/6517))
-- Add wait_for_completion parameter to resize, open, and forcemerge APIs ([#6434](https://github.com/opensearch-project/OpenSearch/pull/6434))
-- [Segment Replication] Apply backpressure when replicas fall behind ([#6563](https://github.com/opensearch-project/OpenSearch/pull/6563))
-- [Remote Store] Integrate remote segment store in peer recovery flow ([#6664](https://github.com/opensearch-project/OpenSearch/pull/6664))
-- Enable sort optimization for all NumericTypes ([#6464](https://github.com/opensearch-project/OpenSearch/pull/6464)
-- Remove 'cluster_manager' role attachment when using 'node.master' deprecated setting ([#6331](https://github.com/opensearch-project/OpenSearch/pull/6331))
-- Add new cluster settings to ignore weighted round-robin routing and fallback to default behaviour. ([#6834](https://github.com/opensearch-project/OpenSearch/pull/6834))
-- Add experimental support for ZSTD compression. ([#3577](https://github.com/opensearch-project/OpenSearch/pull/3577))
-- [Segment Replication] Add point in time and scroll query compatibility. ([#6644](https://github.com/opensearch-project/OpenSearch/pull/6644))
+- [Extensions] Moving Extensions APIs to protobuf serialization. ([#6960](https://github.com/opensearch-project/OpenSearch/pull/6960))
+- Add descending order search optimization through reverse segment read. ([#7244](https://github.com/opensearch-project/OpenSearch/pull/7244))
 
 ### Dependencies
-- Bump `org.apache.logging.log4j:log4j-core` from 2.18.0 to 2.20.0 ([#6490](https://github.com/opensearch-project/OpenSearch/pull/6490))
-- Bump `com.azure:azure-storage-common` from 12.19.3 to 12.20.0 ([#6492](https://github.com/opensearch-project/OpenSearch/pull/6492)
-- Bump `snakeyaml` from 1.33 to 2.0 ([#6511](https://github.com/opensearch-project/OpenSearch/pull/6511))
-- Bump `io.projectreactor.netty:reactor-netty` from 1.1.3 to 1.1.4
-- Bump `com.avast.gradle:gradle-docker-compose-plugin` from 0.15.2 to 0.16.11
-- Bump `net.minidev:json-smart` from 2.4.8 to 2.4.9
-- Bump `com.google.protobuf:protobuf-java` to 3.22.2 ([#6994](https://github.com/opensearch-project/OpenSearch/pull/6994))
-- Bump Netty to 4.1.90.Final ([#6677](https://github.com/opensearch-project/OpenSearch/pull/6677)
-- Bump `com.diffplug.spotless` from 6.15.0 to 6.17.0
-- Bump `org.apache.zookeeper:zookeeper` from 3.8.0 to 3.8.1
-- Bump `net.minidev:json-smart` from 2.4.7 to 2.4.10
-- Bump `org.apache.maven:maven-model` from 3.6.2 to 3.9.1
-- Bump `org.codehaus.jettison:jettison` from 1.5.3 to 1.5.4 ([#6878](https://github.com/opensearch-project/OpenSearch/pull/6878))
-- Add `com.github.luben:zstd-jni:1.5.4-1` ([#3577](https://github.com/opensearch-project/OpenSearch/pull/3577))
-- Bump: Netty from 4.1.90.Final to 4.1.91.Final , ASM 9.4 to ASM 9.5, ByteBuddy 1.14.2 to 1.14.3 ([#6981](https://github.com/opensearch-project/OpenSearch/pull/6981))
-- Bump `com.azure:azure-storage-blob` from 12.15.0 to 12.21.1
+- Bump `jackson` from 2.14.2 to 2.15.0 ([#7286](https://github.com/opensearch-project/OpenSearch/pull/7286)
 
 ### Changed
-- Require MediaType in Strings.toString API ([#6009](https://github.com/opensearch-project/OpenSearch/pull/6009))
-- [Refactor] XContent base classes from xcontent to core library ([#5902](https://github.com/opensearch-project/OpenSearch/pull/5902))
-- Changed `opensearch-env` to respect already set `OPENSEARCH_HOME` environment variable ([#6956](https://github.com/opensearch-project/OpenSearch/pull/6956/))
-- Added a new field type: flat_object ([#6507](https://github.com/opensearch-project/OpenSearch/pull/6507))
+- Enable `./gradlew build` on MacOS by disabling bcw tests ([#7303](https://github.com/opensearch-project/OpenSearch/pull/7303))
 
 ### Deprecated
-- Map, List, and Set in org.opensearch.common.collect ([#6609](https://github.com/opensearch-project/OpenSearch/pull/6609))
 
 ### Removed
 
 ### Fixed
-- Added depth check in doc parser for deep nested document ([#5199](https://github.com/opensearch-project/OpenSearch/pull/5199))
-- Added equals/hashcode for named DocValueFormat.DateTime inner class ([#6357](https://github.com/opensearch-project/OpenSearch/pull/6357))
-- Fixed bug for searchable snapshot to take 'base_path' of blob into account ([#6558](https://github.com/opensearch-project/OpenSearch/pull/6558))
-- Fix fuzziness validation ([#5805](https://github.com/opensearch-project/OpenSearch/pull/5805))
-- Fix GetSnapshots to not return non-existent snapshots with ignore_unavailable=true ([#6839](https://github.com/opensearch-project/OpenSearch/pull/6839))
 
 ### Security
 
 [Unreleased 3.0]: https://github.com/opensearch-project/OpenSearch/compare/2.x...HEAD
-[Unreleased 2.x]: https://github.com/opensearch-project/OpenSearch/compare/2.5...2.x
+[Unreleased 2.x]: https://github.com/opensearch-project/OpenSearch/compare/2.5...2.x

benchmarks/src/main/java/org/opensearch/benchmark/store/remote/filecache/FileCacheBenchmark.java

@@ -107,27 +107,10 @@ public void setup() {
     /**
      * Stubbed out IndexInput that does nothing but report a fixed size
      */
-    private static class FixedSizeStubIndexInput extends CachedIndexInput {
-        private FixedSizeStubIndexInput() {
-            super(FixedSizeStubIndexInput.class.getSimpleName());
-        }
-
-        @Override
-        public boolean isClosed() {
-            return false;
-        }
-
-        @Override
-        public void close() {}
-
-        @Override
-        public long getFilePointer() {
-            throw new UnsupportedOperationException();
-        }
-
+    private static class FixedSizeStubIndexInput implements CachedIndexInput {
         @Override
-        public void seek(long pos) {
-            throw new UnsupportedOperationException();
+        public IndexInput getIndexInput() {
+            return null;
         }
 
         @Override
@@ -136,18 +119,13 @@ public long length() {
         }
 
         @Override
-        public IndexInput slice(String sliceDescription, long offset, long length) {
-            throw new UnsupportedOperationException();
+        public boolean isClosed() {
+            return false;
         }
 
         @Override
-        public byte readByte() {
-            throw new UnsupportedOperationException();
-        }
+        public void close() throws Exception {
 
-        @Override
-        public void readBytes(byte[] b, int offset, int len) {
-            throw new UnsupportedOperationException();
         }
     }
 }

build.gradle

@@ -53,7 +53,7 @@ plugins {
   id 'lifecycle-base'
   id 'opensearch.docker-support'
   id 'opensearch.global-build-info'
-  id "com.diffplug.spotless" version "6.17.0" apply false
+  id "com.diffplug.spotless" version "6.18.0" apply false
   id "org.gradle.test-retry" version "1.5.2" apply false
   id "test-report-aggregation"
   id 'jacoco-report-aggregation'
@@ -225,8 +225,16 @@ tasks.register("verifyVersions") {
  */
 
 boolean bwc_tests_enabled = true
+
 /* place an issue link here when committing bwc changes */
-final String bwc_tests_disabled_issue = ""
+String bwc_tests_disabled_issue = ""
+
+/* there's no existing MacOS release, therefore disable bcw tests */
+if (Os.isFamily(Os.FAMILY_MAC)) {
+  bwc_tests_enabled = false
+  bwc_tests_disabled_issue = "https://github.com/opensearch-project/OpenSearch/issues/4173"
+}
+
 if (bwc_tests_enabled == false) {
   if (bwc_tests_disabled_issue.isEmpty()) {
     throw new GradleException("bwc_tests_disabled_issue must be set when bwc_tests_enabled == false")

buildSrc/build.gradle

@@ -61,7 +61,7 @@ Properties props = VersionPropertiesLoader.loadBuildSrcVersion(project.file('ver
 version = props.getProperty("opensearch")
 
 def generateVersionProperties = tasks.register("generateVersionProperties", WriteProperties) {
-  outputFile = "${buildDir}/version.properties"
+  destinationFile = file("${buildDir}/version.properties")
   comment = 'Generated version properties'
   properties(props)
 }
@@ -106,8 +106,8 @@ dependencies {
   api 'org.apache.commons:commons-compress:1.22'
   api 'org.apache.ant:ant:1.10.13'
   api 'com.netflix.nebula:gradle-extra-configurations-plugin:10.0.0'
-  api 'com.netflix.nebula:nebula-publishing-plugin:20.2.0'
-  api 'com.netflix.nebula:gradle-info-plugin:12.0.1'
+  api 'com.netflix.nebula:nebula-publishing-plugin:20.3.0'
+  api 'com.netflix.nebula:gradle-info-plugin:12.1.0'
   api 'org.apache.rat:apache-rat:0.15'
   api 'commons-io:commons-io:2.11.0'
   api "net.java.dev.jna:jna:5.13.0"
@@ -187,6 +187,7 @@ if (project != rootProject) {
   dependencies {
     reaper project('reaper')
     distribution project(':distribution:archives:darwin-tar')
+    distribution project(':distribution:archives:darwin-arm64-tar')
     distribution project(':distribution:archives:linux-arm64-tar')
     distribution project(':distribution:archives:linux-tar')
     distribution project(':distribution:archives:windows-zip')

buildSrc/src/main/java/org/opensearch/gradle/OpenSearchTestBasePlugin.java

@@ -35,6 +35,7 @@
 import com.github.jengelman.gradle.plugins.shadow.ShadowBasePlugin;
 import org.opensearch.gradle.info.BuildParams;
 import org.opensearch.gradle.info.GlobalBuildInfoPlugin;
+import org.opensearch.gradle.jvm.JvmTestSuiteHelper;
 import org.opensearch.gradle.test.ErrorReportingTestListener;
 import org.opensearch.gradle.util.Util;
 import org.gradle.api.Action;
@@ -223,7 +224,15 @@ public void execute(Task t) {
                 // Add the shadow JAR artifact itself
                 FileCollection shadowJar = project.files(project.getTasks().named("shadowJar"));
 
-                test.setClasspath(test.getClasspath().minus(mainRuntime).plus(shadowConfig).plus(shadowJar));
+                // See please https://docs.gradle.org/8.1/userguide/upgrading_version_8.html#test_task_default_classpath
+                test.setClasspath(
+                    JvmTestSuiteHelper.getDefaultTestSuite(project)
+                        .map(suite -> suite.getSources().getRuntimeClasspath())
+                        .orElseGet(() -> test.getClasspath())
+                        .minus(mainRuntime)
+                        .plus(shadowConfig)
+                        .plus(shadowJar)
+                );
             });
         });
     }

buildSrc/src/main/java/org/opensearch/gradle/info/GlobalBuildInfoPlugin.java

@@ -74,8 +74,8 @@
 
 public class GlobalBuildInfoPlugin implements Plugin<Project> {
     private static final Logger LOGGER = Logging.getLogger(GlobalBuildInfoPlugin.class);
-    private static final String DEFAULT_LEGACY_VERSION_JAVA_FILE_PATH = "server/src/main/java/org/opensearch/LegacyESVersion.java";
-    private static final String DEFAULT_VERSION_JAVA_FILE_PATH = "server/src/main/java/org/opensearch/Version.java";
+    private static final String DEFAULT_LEGACY_VERSION_JAVA_FILE_PATH = "libs/core/src/main/java/org/opensearch/LegacyESVersion.java";
+    private static final String DEFAULT_VERSION_JAVA_FILE_PATH = "libs/core/src/main/java/org/opensearch/Version.java";
     private static Integer _defaultParallel = null;
 
     private final JvmMetadataDetector jvmMetadataDetector;
@@ -152,7 +152,7 @@ private static BwcVersions resolveBwcVersions(File root) {
             versionLines.addAll(IOUtils.readLines(fis2, "UTF-8"));
             return new BwcVersions(versionLines);
         } catch (IOException e) {
-            throw new IllegalStateException("Unable to resolve to resolve bwc versions from versionsFile.", e);
+            throw new IllegalStateException("Unable to resolve bwc versions from versionsFile.", e);
         }
     }
 

buildSrc/src/main/java/org/opensearch/gradle/jvm/JvmTestSuiteHelper.java

@@ -0,0 +1,40 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.gradle.jvm;
+
+import org.gradle.api.Project;
+import org.gradle.api.plugins.JvmTestSuitePlugin;
+import org.gradle.api.plugins.jvm.JvmTestSuite;
+import org.gradle.testing.base.TestSuite;
+import org.gradle.testing.base.TestingExtension;
+
+import java.util.Optional;
+
+public final class JvmTestSuiteHelper {
+    private JvmTestSuiteHelper() {}
+
+    /**
+     * Gets the default test suite. This method assumes the Java plugin is applied,
+     * adapted from {@link org.gradle.api.plugins.internal.JavaPluginHelper} since it is not
+     * available in Gradle releases predated 8.1.
+     */
+    public static Optional<JvmTestSuite> getDefaultTestSuite(Project project) {
+        TestingExtension testing = project.getExtensions().findByType(TestingExtension.class);
+        if (testing == null) {
+            return Optional.empty();
+        }
+
+        TestSuite defaultTestSuite = testing.getSuites().findByName(JvmTestSuitePlugin.DEFAULT_TEST_SUITE_NAME);
+        if (!(defaultTestSuite instanceof JvmTestSuite)) {
+            return Optional.empty();
+        }
+
+        return Optional.of((JvmTestSuite) defaultTestSuite);
+    }
+}