Merge remote-tracking branch 'upstream/main' into decommission/init-m…

…erge

.github/CODEOWNERS

@@ -1,3 +1 @@
-# This should match the owning team set up in https://github.com/orgs/opensearch-project/teams
-*   @opensearch-project/opensearch-core @reta
-
+*   @reta @anasalkouz @andrross @reta @Bukhtawar @CEHENKLE @dblock @setiah @kartg @kotwanikunal @mch2 @nknize @owaiskazi19 @adnapibar @Rishikesh1159 @ryanbogan @saratvemulapalli @shwetathareja @dreamer-89 @tlfeng @VachaShah @xuezhou25

.github/workflows/gradle-check.yml

@@ -8,8 +8,15 @@ on:
   pull_request_target:
     types: [opened, synchronize, reopened]
 
+permissions:
+  contents: read # to fetch code (actions/checkout)
+
 jobs:
   gradle-check:
+    permissions:
+      contents: read # to fetch code (actions/checkout)
+      pull-requests: write # to create or update comment (peter-evans/create-or-update-comment)
+
     runs-on: ubuntu-latest
     timeout-minutes: 130
     steps:

.github/workflows/links.yml

@@ -2,6 +2,8 @@ name: Link Checker
 on:
   schedule:
     - cron:  '0 0 * * *'
+permissions:
+  contents: read # to fetch code (actions/checkout)
 jobs:
   linkchecker:
     if: github.repository == 'opensearch-project/OpenSearch'

.github/workflows/version.yml

@@ -5,6 +5,7 @@ on:
     tags:
       - '*.*.*'
 
+permissions: {}
 jobs:
   build:
     runs-on: ubuntu-latest

CHANGELOG.md

@@ -6,13 +6,16 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 
 ## [Unreleased 3.0]
 ### Added
+- Hardened token permissions in GitHub workflows ([#4587](https://github.com/opensearch-project/OpenSearch/pull/4587))
 - Support for HTTP/2 (server-side) ([#3847](https://github.com/opensearch-project/OpenSearch/pull/3847))
 - Add getter for path field in NestedQueryBuilder ([#4636](https://github.com/opensearch-project/OpenSearch/pull/4636))
 - Apply reproducible builds configuration for OpenSearch plugins through gradle plugin ([#4746](https://github.com/opensearch-project/OpenSearch/pull/4746))
 - Add project health badges to the README.md ([#4843](https://github.com/opensearch-project/OpenSearch/pull/4843))
 - [Test] Add IAE test for deprecated edgeNGram analyzer name ([#5040](https://github.com/opensearch-project/OpenSearch/pull/5040))
 - Allow mmap to use new JDK-19 preview APIs in Apache Lucene 9.4+ ([#5151](https://github.com/opensearch-project/OpenSearch/pull/5151))
 - Add feature flag for extensions ([#5211](https://github.com/opensearch-project/OpenSearch/pull/5211))
+- Added jackson dependency to server ([#5366] (https://github.com/opensearch-project/OpenSearch/pull/5366))
+- Added experimental extensions to main ([#5347](https://github.com/opensearch-project/OpenSearch/pull/5347))
 
 ### Dependencies
 - Bumps `log4j-core` from 2.18.0 to 2.19.0
@@ -55,6 +58,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Migrate client transports to Apache HttpClient / Core 5.x ([#4459](https://github.com/opensearch-project/OpenSearch/pull/4459))
 - Support remote translog transfer for request level durability([#4480](https://github.com/opensearch-project/OpenSearch/pull/4480))
 - Changed http code on create index API with bad input raising NotXContentException from 500 to 400 ([#4773](https://github.com/opensearch-project/OpenSearch/pull/4773))
+- Change http code for DecommissioningFailedException from 500 to 400 ([#5283](https://github.com/opensearch-project/OpenSearch/pull/5283))
 
 ### Deprecated
 
@@ -77,12 +81,14 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Fix 'org.apache.hc.core5.http.ParseException: Invalid protocol version' under JDK 16+ ([#4827](https://github.com/opensearch-project/OpenSearch/pull/4827))
 - Fixed compression support for h2c protocol ([#4944](https://github.com/opensearch-project/OpenSearch/pull/4944))
 - Reject bulk requests with invalid actions ([#5299](https://github.com/opensearch-project/OpenSearch/issues/5299))
+- Support OpenSSL Provider with default Netty allocator ([#5460](https://github.com/opensearch-project/OpenSearch/pull/5460))
 
 ### Security
 
 ## [Unreleased 2.x]
 ### Added
 - Prevent deletion of snapshots that are backing searchable snapshot indexes ([#5069](https://github.com/opensearch-project/OpenSearch/pull/5069))
+- Add max_shard_size parameter for shrink API ([#5229](https://github.com/opensearch-project/OpenSearch/pull/5229))
 
 ### Dependencies
 - Bumps `bcpg-fips` from 1.0.5.1 to 1.0.7.1
@@ -95,6 +101,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 ### Deprecated
 ### Removed
 ### Fixed
+- Fix 1.x compatibility bug with stored Tasks ([#5412](https://github.com/opensearch-project/OpenSearch/pull/5412))
+- Fix case sensitivity for wildcard queries ([#5462](https://github.com/opensearch-project/OpenSearch/pull/5462))
 ### Security
 
 [Unreleased 3.0]: https://github.com/opensearch-project/OpenSearch/compare/2.4...HEAD

MAINTAINERS.md

@@ -5,7 +5,6 @@
 
 | Maintainer | GitHub ID | Affiliation |
 | --------------- | --------- | ----------- |
-| Abbas Hussain | [abbashus](https://github.com/abbashus) | Amazon |
 | Anas Alkouz | [anasalkouz](https://github.com/anasalkouz) | Amazon |
 | Andrew Ross   | [andrross](https://github.com/andrross)| Amazon |
 | Andriy Redko | [reta](https://github.com/reta) | Aiven |
@@ -22,8 +21,8 @@
 | Rishikesh Pasham | [Rishikesh1159](https://github.com/Rishikesh1159) | Amazon|
 | Ryan Bogan | [ryanbogan](https://github.com/ryanbogan) | Amazon |
 | Sarat Vemulapalli | [saratvemulapalli](https://github.com/saratvemulapalli) | Amazon |
-| Shweta Thareja |[shwetathareja](https://github.com/shwetathareja) | Amazon |
-| Suraj Singh |[dreamer-89](https://github.com/dreamer-89) | Amazon |
+| Shweta Thareja | [shwetathareja](https://github.com/shwetathareja) | Amazon |
+| Suraj Singh | [dreamer-89](https://github.com/dreamer-89) | Amazon |
 | Tianli Feng | [tlfeng](https://github.com/tlfeng) | Amazon |
 | Vacha Shah | [VachaShah](https://github.com/VachaShah) | Amazon |
 | Xue Zhou | [xuezhou25](https://github.com/xuezhou25) | Amazon |
@@ -32,6 +31,7 @@
 
 | Maintainer | GitHub ID | Affiliation |
 | --------------- | --------- | ----------- |
+| Abbas Hussain | [abbashus](https://github.com/abbashus) | Amazon |
 | Megha Sai Kavikondala | [meghasaik](https://github.com/meghasaik) | Amazon |
 
 [This document](https://github.com/opensearch-project/.github/blob/main/MAINTAINERS.md) explains what maintainers do in this repo, and how they should be doing it. If you're interested in contributing, see [CONTRIBUTING](CONTRIBUTING.md).

client/rest-high-level/src/main/java/org/opensearch/client/indices/ResizeRequest.java

@@ -39,6 +39,7 @@
 import org.opensearch.common.settings.Settings;
 import org.opensearch.common.xcontent.ToXContentObject;
 import org.opensearch.common.xcontent.XContentBuilder;
+import org.opensearch.common.unit.ByteSizeValue;
 
 import java.io.IOException;
 import java.util.Collections;
@@ -58,6 +59,7 @@ public class ResizeRequest extends TimedRequest implements Validatable, ToXConte
     private final String targetIndex;
     private Settings settings = Settings.EMPTY;
     private Set<Alias> aliases = new HashSet<>();
+    private ByteSizeValue maxShardSize;
 
     /**
      * Creates a new resize request
@@ -155,6 +157,24 @@ public ActiveShardCount getWaitForActiveShards() {
         return waitForActiveShards;
     }
 
+    /**
+     * Sets the maximum size of a primary shard in the new shrunken index.
+     * This parameter can be used to calculate the lowest factor of the source index's shards number
+     * which satisfies the maximum shard size requirement.
+     *
+     * @param maxShardSize the maximum size of a primary shard in the new shrunken index
+     */
+    public void setMaxShardSize(ByteSizeValue maxShardSize) {
+        this.maxShardSize = maxShardSize;
+    }
+
+    /**
+     * Returns the maximum size of a primary shard in the new shrunken index.
+     */
+    public ByteSizeValue getMaxShardSize() {
+        return maxShardSize;
+    }
+
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         builder.startObject();

client/rest-high-level/src/test/java/org/opensearch/client/IndicesRequestConvertersTests.java

@@ -79,6 +79,7 @@
 import org.opensearch.common.xcontent.XContentType;
 import org.opensearch.test.OpenSearchTestCase;
 import org.junit.Assert;
+import org.opensearch.common.unit.ByteSizeValue;
 
 import java.io.IOException;
 import java.util.Arrays;
@@ -701,6 +702,8 @@ private void resizeTest(ResizeType resizeType, CheckedFunction<ResizeRequest, Re
         RequestConvertersTests.setRandomWaitForActiveShards(resizeRequest::setWaitForActiveShards, expectedParams);
         if (resizeType == ResizeType.SPLIT) {
             resizeRequest.setSettings(Settings.builder().put("index.number_of_shards", 2).build());
+        } else if (resizeType == ResizeType.SHRINK) {
+            resizeRequest.setMaxShardSize(new ByteSizeValue(randomIntBetween(1, 1000)));
         }
 
         Request request = function.apply(resizeRequest);

modules/ingest-geoip/build.gradle

@@ -41,8 +41,6 @@ opensearchplugin {
 dependencies {
   api('com.maxmind.geoip2:geoip2:3.0.2')
   // geoip2 dependencies:
-  api("com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}")
-  api("com.fasterxml.jackson.core:jackson-databind:${versions.jackson_databind}")
   api('com.maxmind.db:maxmind-db:2.1.0')
 
   testImplementation 'org.elasticsearch:geolite2-databases:20191119'

modules/transport-netty4/src/main/java/org/opensearch/transport/Netty4NioServerSocketChannel.java

@@ -0,0 +1,62 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.transport;
+
+import io.netty.channel.socket.InternetProtocolFamily;
+import io.netty.channel.socket.nio.NioServerSocketChannel;
+import io.netty.util.internal.SocketUtils;
+import io.netty.util.internal.logging.InternalLogger;
+import io.netty.util.internal.logging.InternalLoggerFactory;
+
+import java.nio.channels.ServerSocketChannel;
+import java.nio.channels.SocketChannel;
+import java.nio.channels.spi.SelectorProvider;
+import java.util.List;
+
+public class Netty4NioServerSocketChannel extends NioServerSocketChannel {
+    private static final InternalLogger logger = InternalLoggerFactory.getInstance(Netty4NioServerSocketChannel.class);
+
+    public Netty4NioServerSocketChannel() {
+        super();
+    }
+
+    public Netty4NioServerSocketChannel(SelectorProvider provider) {
+        super(provider);
+    }
+
+    public Netty4NioServerSocketChannel(SelectorProvider provider, InternetProtocolFamily family) {
+        super(provider, family);
+    }
+
+    public Netty4NioServerSocketChannel(ServerSocketChannel channel) {
+        super(channel);
+    }
+
+    @Override
+    protected int doReadMessages(List<Object> buf) throws Exception {
+        SocketChannel ch = SocketUtils.accept(javaChannel());
+
+        try {
+            if (ch != null) {
+                buf.add(new Netty4NioSocketChannel(this, ch));
+                return 1;
+            }
+        } catch (Throwable t) {
+            logger.warn("Failed to create a new channel from an accepted socket.", t);
+
+            try {
+                ch.close();
+            } catch (Throwable t2) {
+                logger.warn("Failed to close a socket.", t2);
+            }
+        }
+
+        return 0;
+    }
+}

modules/transport-netty4/src/main/java/org/opensearch/transport/NettyAllocator.java

@@ -39,7 +39,6 @@
 import io.netty.buffer.UnpooledByteBufAllocator;
 import io.netty.channel.Channel;
 import io.netty.channel.ServerChannel;
-import io.netty.channel.socket.nio.NioServerSocketChannel;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.opensearch.common.Booleans;
@@ -181,7 +180,7 @@ public static Class<? extends ServerChannel> getServerChannelType() {
         if (ALLOCATOR instanceof NoDirectBuffers) {
             return CopyBytesServerSocketChannel.class;
         } else {
-            return NioServerSocketChannel.class;
+            return Netty4NioServerSocketChannel.class;
         }
     }
 

plugins/discovery-ec2/build.gradle

@@ -46,8 +46,6 @@ dependencies {
   api "commons-logging:commons-logging:${versions.commonslogging}"
   api "org.apache.logging.log4j:log4j-1.2-api:${versions.log4j}"
   api "commons-codec:commons-codec:${versions.commonscodec}"
-  api "com.fasterxml.jackson.core:jackson-databind:${versions.jackson_databind}"
-  api "com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}"
 }
 
 restResources {

plugins/ingest-attachment/build.gradle

@@ -38,10 +38,10 @@ opensearchplugin {
 }
 
 versions << [
-  'tika'  : '2.5.0',
-  'pdfbox': '2.0.25',
-  'poi'   : '5.2.2',
-  'mime4j': '0.8.3'
+  'tika'  : '2.6.0',
+  'pdfbox': '2.0.27',
+  'poi'   : '5.2.3',
+  'mime4j': '0.8.8'
 ]
 
 dependencies {

plugins/ingest-attachment/licenses/apache-mime4j-core-0.8.8.jar.sha1

@@ -0,0 +1 @@
+7330de23c52f71617cbec7f1d2760dae32e687cd

plugins/ingest-attachment/licenses/apache-mime4j-dom-0.8.8.jar.sha1

@@ -0,0 +1 @@
+e76715563a6bd150f84ccb0adb920aec8faf4779

plugins/ingest-attachment/licenses/fontbox-2.0.27.jar.sha1

@@ -0,0 +1 @@
+d08c064d18b2b149da937d15c0d1708cba03f29d

plugins/ingest-attachment/licenses/pdfbox-2.0.27.jar.sha1

@@ -0,0 +1 @@
+416a9dfce3714116bfdf793b15368df04266845f

plugins/ingest-attachment/licenses/poi-5.2.3.jar.sha1

@@ -0,0 +1 @@
+2fb22ae74ad5aea6af1a9c64b9542f2ccf348604

plugins/ingest-attachment/licenses/poi-ooxml-5.2.3.jar.sha1

@@ -0,0 +1 @@
+02efd11c940adb18c03eb9ce7ad88fc40ee6a196

plugins/ingest-attachment/licenses/poi-ooxml-lite-5.2.3.jar.sha1

@@ -0,0 +1 @@
+db113c8e9051b0ff967f4911fa20336c8325a7c5