Merge branch 'main' into fs

Signed-off-by: Bukhtawar Khan <bukhtawa@amazon.com>

.github/workflows/links.yml

@@ -1,22 +1,24 @@
 name: Link Checker
 on:
-  push:
-    branches: [ main ]
-  pull_request:
-    branches: [ main ]
-
+  schedule:
+    - cron:  '0 0 * * *'
 jobs:
   linkchecker:
 
     runs-on: ubuntu-latest
 
     steps:
       - uses: actions/checkout@v2
+      - name: Load Excludes
+        run: |
+          LYCHEE_EXCLUDE=$(sed -e :a  -e 'N;s/\n/ /;ta' .lycheeexclude)
+          echo "LYCHEE_EXCLUDE=$LYCHEE_EXCLUDE" >> $GITHUB_ENV
+
       - name: lychee Link Checker
         id: lychee
         uses: lycheeverse/lychee-action@master
         with:
-          args: --accept=200,403,429  --exclude=http://www.unicode.org/Public/PROGRAMS/CVTUTF --exclude=http://www.eclipse.org/jetty/downloads.php --exclude=http://project.carrot2.org/license.html --exclude=http://bitbucket.org/jpbarrette/moman/overview/ --exclude=http://opensource.adobe.com/wiki/display/cmap/Downloads --exclude=http://viewvc.jboss.org/cgi-bin/viewvc.cgi/jbosscache/experimental/jsr166/ --exclude=http://eid-applet.googlecode.com/ --exclude=http://www.ecma-international.org/publications/files/ECMA-ST/Ecma%20PATENT/Patent%20statements%20ok/ECMA-376%20Edition%202%20Microsoft%20Patent%20Declaration.pdf --exclude=http://www.ecma-international.org/publications/files/ECMA-ST/Ecma%20PATENT/Patent%20statements%20ok/ECMA-376%20Adobe%20Patent%20Declaration.pdf --exclude=http://snapshot/ --exclude=http://www.darwinsys.com/file/ --exclude=https://www.sqlite.org/copyright.html --exclude=http://www.bouncycastle.org/ --exclude=http://www.icu-project.org --exclude=http://www.sjp.pl/slownik/en/ --exclude=http://www.brics.dk/automaton/ --exclude=http://jaspell.sourceforge.net/ --exclude=http://www.opensource.org/licenses/bsd-license.php --exclude=http://www.jcraft.com/jzlib/ --exclude=http://www.slf4j.org/ --exclude=http://www.python.org/download/releases/2.4.2/license/ --exclude=http://www.python.org/download/releases/3.1.2/license/ --exclude=http://www.jcip.net --exclude=http://www.7-zip.org/sdk.html --exclude=http://www.mozilla.org/MPL/ --exclude=http://www.ecma-international.org/publications/standards/Ecma-376.htm --exclude=http://www.ecma-international.org/memento/Ecmabylaws.htm --exclude=http://www.microsoft.com/openspecifications/en/us/programs/osp/default.aspx --exclude=http://www.bea.com/ --exclude=http://www.unidata.ucar.edu/software/netcdf-java/ --exclude=http://www.clker.com/clipart-13653.html --exclude=http://www.joda.org/ --exclude=http://www.eclipse.org/licenses/edl-v10.html --exclude=http://www.eclipse.org/legal/cpl-v10.html --exclude=http://www.mozilla.org/MPL/MPL-1.1.html --exclude=http://source.icu-project.org/repos/icu/icu/trunk/license.html --exclude=http://unicode.org/copyright.html --exclude-mail "**/*.html" "**/*.md" "**/*.txt" "**/*.json"
+          args: --accept=200,403,429 --exclude ${{ env.LYCHEE_EXCLUDE }} --exclude-mail "**/*.html" "**/*.md" "**/*.txt" "**/*.json"
         env:
           GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}}
       - name: Fail if there were link errors

.lycheeexclude

@@ -0,0 +1,13 @@
+http://bitbucket.org/jpbarrette/moman/overview/
+http://eid-applet.googlecode.com/
+http://opensource.adobe.com/wiki/display/cmap/Downloads
+http://project.carrot2.org/license.html
+http://source.icu-project.org/
+http://site.icu-project.org/
+http://www.icu-project.org/
+http://snapshot/ 
+http://viewvc.jboss.org/cgi-bin/viewvc.cgi/jbosscache/experimental/jsr166/
+http://www.eclipse.org/jetty/downloads.php
+http://www.ecma-international.org/publications/files/ECMA-ST/Ecma%20PATENT/Patent%20statements%20ok/ECMA-376%20*
+http://www.unicode.org/Public/PROGRAMS/CVTUTF
+http://sgjp.pl/morfeusz/

DEVELOPER_GUIDE.md

@@ -49,9 +49,9 @@ Fork [opensearch-project/OpenSearch](https://github.com/opensearch-project/OpenS
 
 OpenSearch builds using Java 11 at a minimum. This means you must have a JDK 11 installed with the environment variable `JAVA_HOME` referencing the path to Java home for your JDK 11 installation, e.g. `JAVA_HOME=/usr/lib/jvm/jdk-11`.
 
-By default, tests use the same runtime as `JAVA_HOME`. However, since OpenSearch supports JDK 8, the build supports compiling with JDK 11 and testing on a different version of JDK runtime. To do this, set `RUNTIME_JAVA_HOME` pointing to the Java home of another JDK installation, e.g. `RUNTIME_JAVA_HOME=/usr/lib/jvm/jdk-8`.
+By default, tests use the same runtime as `JAVA_HOME`. However, since OpenSearch also supports JDK 8 as the runtime, the build supports compiling with JDK 11 and testing on a different version of JDK runtime. To do this, set `RUNTIME_JAVA_HOME` pointing to the Java home of another JDK installation, e.g. `RUNTIME_JAVA_HOME=/usr/lib/jvm/jdk-8`.
 
-To run the full suite of tests you will also need `JAVA8_HOME`, `JAVA9_HOME`, `JAVA10_HOME`, `JAVA11_HOME`, and `JAVA12_HOME`.
+To run the full suite of tests you will also need `JAVA8_HOME`, `JAVA11_HOME`, and `JAVA14_HOME`. They are required by the [backwards compatibility test](./TESTING.md#testing-backwards-compatibility).
 
 #### Docker
 
@@ -147,7 +147,18 @@ Follow links in the [Java Tutorial](https://code.visualstudio.com/docs/java/java
 
 ### Eclipse
 
-We would like to support Eclipse, but few of us use it and has fallen into disrepair. Please [contribute](CONTRIBUTING.md).
+When importing to Eclipse, you need to have [Eclipse Buildship](https://projects.eclipse.org/projects/tools.buildship) plugin installed and, preferrably, have JDK 11 set as default JRE in **Preferences -> Java -> Installed JREs**. Once this is done, generate Eclipse projects using Gradle wrapper:
+
+    ./gradlew eclipse
+
+You can now import the OpenSearch project into Eclipse as follows.
+
+1. Select **File > Import -> Existing Gradle Project**
+2. In the subsequent dialog navigate to the root of `build.gradle` file
+3. In the subsequent dialog, if JDK 11 is not set as default JRE, please make sure to check **[Override workpace settings]**, keep **[Gradle Wrapper]** and provide the correct path to JDK11 using **[Java Home]** property under **[Advanced Options]**. Otherwise, you may run into cryptic import failures and only top level project is going to be imported.
+4. In the subsequent dialog, you sould see **[Gradle project structure]** populated, please click **[Finish]** to complete the import
+
+**Note:** it may look non-intuitive why one needs to use Gradle wrapper and then import existing Gradle project (in general, **File > Import -> Existing Gradle Project** should be enough). Practially, as it stands now, Eclipse Buildship plugin does not import OpenSearch project dependencies correctly but does work in conjuction with Gradle wrapper.
 
 ## Project Layout
 
@@ -282,14 +293,30 @@ of this is `junit`.
 
 ### git-secrets
 
-Security is our top priority. Avoid checking in credentials, install [awslabs/git-secrets](https://github.com/awslabs/git-secrets).
+Security is our top priority. Avoid checking in credentials.
 
+#### Installation
+Install [awslabs/git-secrets](https://github.com/awslabs/git-secrets) by running the following commands.
 ```
 git clone https://github.com/awslabs/git-secrets.git
 cd git-secrets
 make install
 ```
 
+#### Configuration
+You can configure git secrets per repository, you need to change the directory to the root of the repository and run the following command.
+```
+git secrets --install
+✓ Installed commit-msg hook to .git/hooks/commit-msg
+✓ Installed pre-commit hook to .git/hooks/pre-commit
+✓ Installed prepare-commit-msg hook to .git/hooks/prepare-commit-msg
+```
+Then, you need to apply patterns for git-secrets, you can install the AWS standard patterns by running the following command.
+```
+git secrets --register-aws
+```
+
+
 ## Submitting Changes
 
 See [CONTRIBUTING](CONTRIBUTING.md).

NOTICE.txt

@@ -1,5 +1,5 @@
 OpenSearch (https://opensearch.org/)
-Copyright 2021 OpenSearch Contributors
+Copyright OpenSearch Contributors
 
 This product includes software developed by
 Elasticsearch (http://www.elastic.co).

README.md

@@ -34,4 +34,4 @@ This project is licensed under the [Apache v2.0 License](LICENSE.txt).
 
 ## Copyright
 
-Copyright 2020-2021 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+Copyright OpenSearch Contributors. See [NOTICE](NOTICE.txt) for details.

TESTING.md

@@ -363,7 +363,13 @@ These test tasks can use the `--tests`, `--info`, and `--debug` parameters just
 
 # Testing backwards compatibility
 
-Backwards compatibility tests exist to test upgrading from each supported version to the current version. To run them all use:
+Backwards compatibility tests exist to test upgrading from each supported version to the current version. 
+
+The test can be run for any versions which the current version will be compatible with. Tests are run for released versions download the distributions from the artifact repository, see [DistributionDownloadPlugin](./buildSrc/src/main/java/org/opensearch/gradle/DistributionDownloadPlugin.java) for the repository location. Tests are run for versions that are not yet released automatically check out the branch and build from source to get the distributions, see [BwcVersions](./buildSrc/src/main/java/org/opensearch/gradle/BwcVersions.java) and [distribution/bwc/build.gradle](./distribution/bwc/build.gradle) for more information.
+
+The minimum JDK versions for runtime and compiling need to be installed, and environment variables `JAVAx_HOME`, such as `JAVA8_HOME`, pointing to the JDK installations are required to run the tests against unreleased versions, since the distributions are created by building from source. The required JDK versions for each branch are located at [.ci/java-versions.properties](.ci/java-versions.properties), see [BwcSetupExtension](./buildSrc/src/main/java/org/opensearch/gradle/internal/BwcSetupExtension.java) for more information. 
+
+To run all the backwards compatibility tests use:
 
     ./gradlew bwcTest
 
@@ -377,8 +383,6 @@ Use -Dtest.class and -Dtests.method to run a specific bwcTest test. For example
      -Dtests.class=org.opensearch.upgrades.RecoveryIT \
      -Dtests.method=testHistoryUUIDIsGenerated
 
-Tests are run for versions that are not yet released but with which the current version will be compatible with. These are automatically checked out and built from source. See [BwcVersions](./buildSrc/src/main/java/org/opensearch/gradle/BwcVersions.java) and [distribution/bwc/build.gradle](./distribution/bwc/build.gradle) for more information.
-
 When running `./gradlew check`, minimal bwc checks are also run against compatible versions that are not yet released.
 
 ## BWC Testing against a specific remote/branch

buildSrc/build.gradle

@@ -103,7 +103,7 @@ dependencies {
   api localGroovy()
 
   api 'commons-codec:commons-codec:1.13'
-  api 'org.apache.commons:commons-compress:1.19'
+  api 'org.apache.commons:commons-compress:1.21'
   api 'org.apache.ant:ant:1.10.9'
   api 'com.netflix.nebula:gradle-extra-configurations-plugin:3.0.3'
   api 'com.netflix.nebula:nebula-publishing-plugin:4.4.4'

buildSrc/src/integTest/groovy/org/opensearch/gradle/internal/InternalDistributionArchiveCheckPluginFuncTest.groovy

@@ -108,7 +108,7 @@ Copyright 2009-2018 Acme Coorp"""
         result.task(":darwin-tar:checkNotice").outcome == TaskOutcome.FAILED
         normalizedOutput(result.output).contains("> expected line [2] in " +
                 "[./darwin-tar/build/tar-extracted/opensearch-${VersionProperties.getOpenSearch()}/NOTICE.txt] " +
-                "to be [Copyright 2021 OpenSearch Contributors] but was [Copyright 2009-2018 Acme Coorp]")
+                "to be [Copyright OpenSearch Contributors] but was [Copyright 2009-2018 Acme Coorp]")
     }
 
     void license(File file = file("licenses/APACHE-LICENSE-2.0.txt")) {

buildSrc/src/main/java/org/opensearch/gradle/BwcVersions.java

@@ -379,14 +379,17 @@ public List<Version> getIndexCompatible() {
         if (currentMajor == 1) {
             // add 6.x compatible for OpenSearch 1.0.0
             return unmodifiableList(Stream.concat(groupByMajor.get(prevMajor - 1).stream(), result.stream()).collect(Collectors.toList()));
+        } else if (currentMajor == 2) {
+            // add 7.x compatible for OpenSearch 2.0.0
+            return unmodifiableList(Stream.concat(groupByMajor.get(7).stream(), result.stream()).collect(Collectors.toList()));
         }
         return unmodifiableList(result);
     }
 
     public List<Version> getWireCompatible() {
         List<Version> wireCompat = new ArrayList<>();
         int currentMajor = currentVersion.getMajor();
-        int lastMajor = currentMajor == 1 ? 6 : currentMajor - 1;
+        int lastMajor = currentMajor == 1 ? 6 : currentMajor == 2 ? 7 : currentMajor - 1;
         List<Version> lastMajorList = groupByMajor.get(lastMajor);
         if (lastMajorList == null) {
             throw new IllegalStateException("Expected to find a list of versions for version: " + lastMajor);
@@ -402,7 +405,14 @@ public List<Version> getWireCompatible() {
             for (Version v : previousMajor) {
                 wireCompat.add(v);
             }
+        } else if (currentMajor == 2) {
+            // add all of the 1.x line:
+            List<Version> previousMajor = groupByMajor.get(1);
+            for (Version v : previousMajor) {
+                wireCompat.add(v);
+            }
         }
+
         wireCompat.addAll(groupByMajor.get(currentMajor));
         wireCompat.remove(currentVersion);
         wireCompat.sort(Version::compareTo);

buildSrc/src/main/java/org/opensearch/gradle/Version.java

@@ -77,7 +77,7 @@ public Version(int major, int minor, int revision) {
         // currently snapshot is not taken into account
         int id = major * 10000000 + minor * 100000 + revision * 1000;
         // identify if new OpenSearch version 1
-        this.id = major == 1 ? id ^ MASK : id;
+        this.id = major == 1 || major == 2 ? id ^ MASK : id;
     }
 
     private static int parseSuffixNumber(String substring) {

buildSrc/src/main/java/org/opensearch/gradle/internal/InternalDistributionArchiveCheckPlugin.java

@@ -105,7 +105,7 @@ private TaskProvider<Task> registerCheckNoticeTask(Project project, TaskProvider
                 public void execute(Task task) {
                     final List<String> noticeLines = Arrays.asList(
                         "OpenSearch (https://opensearch.org/)",
-                        "Copyright 2021 OpenSearch Contributors"
+                        "Copyright OpenSearch Contributors"
                     );
                     final Path noticePath = checkExtraction.get()
                         .getDestinationDir()

buildSrc/src/main/resources/plugin-descriptor.properties

@@ -36,7 +36,7 @@ version=${version}
 # 'name': the plugin name
 name=${name}
 #
-# 'classname': the name of the class to load, fully-qualified.
+# 'classname': the name of the class to load, fully-qualified
 classname=${classname}
 #
 # 'java.version': version of java the code is built against
@@ -45,14 +45,16 @@ classname=${classname}
 # separated by "."'s and may have leading zeros
 java.version=${javaVersion}
 #
-# 'opensearch.version': version of opensearch compiled against
+# 'opensearch.version': semantic version of opensearch the plugin is compatible with
+# does not include -SNAPSHOT if compiled against a snapshot build
 opensearch.version=${opensearchVersion}
+#
 ### optional elements for plugins:
 #
-# 'custom.foldername': the custom name of the folder in which the plugin is installed.
+# 'custom.foldername': the custom name of the folder in which the plugin is installed
 custom.foldername=${customFolderName}
 #
-#  'extended.plugins': other plugins this plugin extends through SPI
+# 'extended.plugins': other plugins this plugin extends through SPI
 extended.plugins=${extendedPlugins}
 #
 # 'has.native.controller': whether or not the plugin has a native controller

buildSrc/src/test/java/org/opensearch/gradle/BwcOpenSearchVersionsTests.java

@@ -37,6 +37,10 @@ public class BwcOpenSearchVersionsTests extends GradleUnitTestCase {
     static {
         sampleVersions.put("1.0.0", asList("5_6_13", "6_6_1", "6_8_15", "7_0_0", "7_9_1", "7_10_0", "7_10_1", "7_10_2", "1_0_0"));
         sampleVersions.put("1.1.0", asList("5_6_13", "6_6_1", "6_8_15", "7_0_0", "7_9_1", "7_10_0", "7_10_1", "7_10_2", "1_0_0", "1_1_0"));
+        sampleVersions.put(
+            "2.0.0",
+            asList("5_6_13", "6_6_1", "6_8_15", "7_0_0", "7_9_1", "7_10_0", "7_10_1", "7_10_2", "1_0_0", "1_1_0", "2_0_0")
+        );
     }
 
     public void testWireCompatible() {

buildSrc/src/test/java/org/opensearch/gradle/DistributionDownloadPluginTests.java

@@ -52,11 +52,11 @@ public class DistributionDownloadPluginTests extends GradleUnitTestCase {
     private static Project packagesProject;
     private static Project bwcProject;
 
-    private static final Version BWC_MAJOR_VERSION = Version.fromString("4.0.0");
-    private static final Version BWC_MINOR_VERSION = Version.fromString("3.1.0");
-    private static final Version BWC_STAGED_VERSION = Version.fromString("3.0.0");
-    private static final Version BWC_BUGFIX_VERSION = Version.fromString("3.0.1");
-    private static final Version BWC_MAINTENANCE_VERSION = Version.fromString("2.90.1");
+    private static final Version BWC_MAJOR_VERSION = Version.fromString("5.0.0");
+    private static final Version BWC_MINOR_VERSION = Version.fromString("4.1.0");
+    private static final Version BWC_STAGED_VERSION = Version.fromString("4.0.0");
+    private static final Version BWC_BUGFIX_VERSION = Version.fromString("4.0.1");
+    private static final Version BWC_MAINTENANCE_VERSION = Version.fromString("3.90.1");
     private static final BwcVersions BWC_MINOR = new BwcVersions(
         new TreeSet<>(Arrays.asList(BWC_BUGFIX_VERSION, BWC_MINOR_VERSION, BWC_MAJOR_VERSION)),
         BWC_MAJOR_VERSION

buildSrc/src/test/java/org/opensearch/gradle/VersionTests.java

@@ -65,7 +65,7 @@ public void testRelaxedVersionParsing() {
 
     public void testCompareWithStringVersions() {
         // 1.10.2 is now rebased to OpenSearch version; so this needs to report
-        assertTrue("OpenSearch 1.10.20 is not interpreted as after Legacy 2.0.0", Version.fromString("1.10.20").after("2.0.0"));
+        assertTrue("OpenSearch 1.10.20 is not interpreted as after Legacy 3.0.0", Version.fromString("1.10.20").after("3.0.0"));
         assertTrue(
             "7.0.0-alpha1 should be equal to 7.0.0-alpha1",
             Version.fromString("7.0.0-alpha1").equals(Version.fromString("7.0.0-alpha1"))

buildSrc/version.properties

@@ -1,4 +1,4 @@
-opensearch        = 1.1.0
+opensearch        = 2.0.0
 lucene            = 8.9.0
 
 bundled_jdk_vendor = adoptopenjdk
@@ -38,6 +38,8 @@ commonscodec      = 1.13
 hamcrest          = 2.1
 securemock        = 1.2
 mocksocket        = 1.2
+mockito           = 1.9.5
+objenesis         = 1.0
 
 # benchmark dependencies
 jmh               = 1.19

client/rest/build.gradle

@@ -51,8 +51,8 @@ dependencies {
   testImplementation "com.carrotsearch.randomizedtesting:randomizedtesting-runner:${versions.randomizedrunner}"
   testImplementation "junit:junit:${versions.junit}"
   testImplementation "org.hamcrest:hamcrest:${versions.hamcrest}"
-  testImplementation "org.elasticsearch:securemock:${versions.securemock}"
-  testImplementation "org.elasticsearch:mocksocket:${versions.mocksocket}"
+  testImplementation "org.mockito:mockito-core:${versions.mockito}"
+  testImplementation "org.objenesis:objenesis:${versions.objenesis}"
 }
 
 tasks.withType(CheckForbiddenApis).configureEach {

client/rest/src/test/java/org/opensearch/client/RestClientBuilderIntegTests.java

@@ -37,7 +37,6 @@
 import com.sun.net.httpserver.HttpsConfigurator;
 import com.sun.net.httpserver.HttpsServer;
 import org.apache.http.HttpHost;
-import org.elasticsearch.mocksocket.MockHttpServer;
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
 
@@ -73,7 +72,7 @@ public class RestClientBuilderIntegTests extends RestClientTestCase {
 
     @BeforeClass
     public static void startHttpServer() throws Exception {
-        httpsServer = MockHttpServer.createHttps(new InetSocketAddress(InetAddress.getLoopbackAddress(), 0), 0);
+        httpsServer = HttpsServer.create(new InetSocketAddress(InetAddress.getLoopbackAddress(), 0), 0);
         httpsServer.setHttpsConfigurator(new HttpsConfigurator(getSslContext()));
         httpsServer.createContext("/", new ResponseHandler());
         httpsServer.start();

client/rest/src/test/java/org/opensearch/client/RestClientGzipCompressionTests.java

@@ -39,7 +39,6 @@
 import org.apache.http.HttpHost;
 import org.apache.http.entity.ContentType;
 import org.apache.http.entity.StringEntity;
-import org.elasticsearch.mocksocket.MockHttpServer;
 import org.junit.AfterClass;
 import org.junit.Assert;
 import org.junit.BeforeClass;
@@ -61,7 +60,7 @@ public class RestClientGzipCompressionTests extends RestClientTestCase {
 
     @BeforeClass
     public static void startHttpServer() throws Exception {
-        httpServer = MockHttpServer.createHttp(new InetSocketAddress(InetAddress.getLoopbackAddress(), 0), 0);
+        httpServer = HttpServer.create(new InetSocketAddress(InetAddress.getLoopbackAddress(), 0), 0);
         httpServer.createContext("/", new GzipResponseHandler());
         httpServer.start();
     }