Merge branch 'master' into security_api_keys

buildSrc/build.gradle

@@ -245,7 +245,7 @@ class VersionPropertiesLoader {
                       elasticsearch
       )
     }
-    String qualifier = systemProperties.getProperty("build.version_qualifier", "alpha1");
+    String qualifier = systemProperties.getProperty("build.version_qualifier", "");
     if (qualifier.isEmpty() == false) {
       if (qualifier.matches("(alpha|beta|rc)\\d+") == false) {
         throw new IllegalStateException("Invalid qualifier: " + qualifier)

buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy

@@ -66,7 +66,7 @@ class BuildPlugin implements Plugin<Project> {
     void apply(Project project) {
         if (project.pluginManager.hasPlugin('elasticsearch.standalone-rest-test')) {
               throw new InvalidUserDataException('elasticsearch.standalone-test, '
-                + 'elasticearch.standalone-rest-test, and elasticsearch.build '
+                + 'elasticsearch.standalone-rest-test, and elasticsearch.build '
                 + 'are mutually exclusive')
         }
         final String minimumGradleVersion

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy

@@ -343,7 +343,7 @@ class ClusterFormationTasks {
         if (minimumMasterNodes > 0) {
             esConfig['discovery.zen.minimum_master_nodes'] = minimumMasterNodes
         }
-        if (node.config.numNodes > 1) {
+        if (minimumMasterNodes > 1) {
             // don't wait for state.. just start up quickly
             // this will also allow new and old nodes in the BWC case to become the master
             esConfig['discovery.initial_state_timeout'] = '0s'
@@ -719,7 +719,7 @@ class ClusterFormationTasks {
                 unicastHosts.addAll(node.config.otherUnicastHostAddresses.call())
                 String unicastHost = node.config.unicastTransportUri(node, null, project.ant)
                 if (unicastHost != null) {
-                    unicastHosts.addAll(Arrays.asList(unicastHost.split(",")))
+                    unicastHosts.add(unicastHost)
                 }
             }
             String unicastHostsTxt = String.join("\n", unicastHosts)

buildSrc/version.properties

@@ -1,5 +1,5 @@
 elasticsearch     = 7.0.0
-lucene            = 8.0.0-snapshot-31d7dfe6b1
+lucene            = 8.0.0-snapshot-6d9c714052
 
 # optional dependencies
 spatial4j         = 0.7
@@ -16,6 +16,7 @@ slf4j             = 1.6.2
 jna               = 4.5.1
 
 netty             = 4.1.30.Final
+joda              = 2.10.1
 
 # test dependencies
 randomizedrunner  = 2.7.0

client/rest-high-level/src/main/java/org/elasticsearch/client/CcrClient.java

@@ -21,6 +21,8 @@
 
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.client.ccr.PauseFollowRequest;
+import org.elasticsearch.client.ccr.PutFollowRequest;
+import org.elasticsearch.client.ccr.PutFollowResponse;
 import org.elasticsearch.client.core.AcknowledgedResponse;
 
 import java.io.IOException;
@@ -41,6 +43,51 @@ public final class CcrClient {
         this.restHighLevelClient = restHighLevelClient;
     }
 
+    /**
+     * Executes the put follow api, which creates a follower index and then the follower index starts following
+     * the leader index.
+     *
+     * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/ccr-put-follow.html">
+     * the docs</a> for more.
+     *
+     * @param request the request
+     * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     * @return the response
+     * @throws IOException in case there is a problem sending the request or parsing back the response
+     */
+    public PutFollowResponse putFollow(PutFollowRequest request, RequestOptions options) throws IOException {
+        return restHighLevelClient.performRequestAndParseEntity(
+            request,
+            CcrRequestConverters::putFollow,
+            options,
+            PutFollowResponse::fromXContent,
+            Collections.emptySet()
+        );
+    }
+
+    /**
+     * Asynchronously executes the put follow api, which creates a follower index and then the follower index starts
+     * following the leader index.
+     *
+     * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/ccr-put-follow.html">
+     * the docs</a> for more.
+     *
+     * @param request the request
+     * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     */
+    public void putFollowAsync(PutFollowRequest request,
+                               RequestOptions options,
+                               ActionListener<PutFollowResponse> listener) {
+        restHighLevelClient.performRequestAsyncAndParseEntity(
+            request,
+            CcrRequestConverters::putFollow,
+            options,
+            PutFollowResponse::fromXContent,
+            listener,
+            Collections.emptySet()
+        );
+    }
+
     /**
      * Instructs a follower index the pause the following of a leader index.
      *

client/rest-high-level/src/main/java/org/elasticsearch/client/CcrRequestConverters.java

@@ -20,10 +20,27 @@
 package org.elasticsearch.client;
 
 import org.apache.http.client.methods.HttpPost;
+import org.apache.http.client.methods.HttpPut;
 import org.elasticsearch.client.ccr.PauseFollowRequest;
+import org.elasticsearch.client.ccr.PutFollowRequest;
+
+import java.io.IOException;
+
+import static org.elasticsearch.client.RequestConverters.REQUEST_BODY_CONTENT_TYPE;
+import static org.elasticsearch.client.RequestConverters.createEntity;
 
 final class CcrRequestConverters {
 
+    static Request putFollow(PutFollowRequest putFollowRequest) throws IOException {
+        String endpoint = new RequestConverters.EndpointBuilder()
+            .addPathPart(putFollowRequest.getFollowerIndex())
+            .addPathPartAsIs("_ccr", "follow")
+            .build();
+        Request request = new Request(HttpPut.METHOD_NAME, endpoint);
+        request.setEntity(createEntity(putFollowRequest, REQUEST_BODY_CONTENT_TYPE));
+        return request;
+    }
+
     static Request pauseFollow(PauseFollowRequest pauseFollowRequest) {
         String endpoint = new RequestConverters.EndpointBuilder()
             .addPathPart(pauseFollowRequest.getFollowerIndex())

client/rest-high-level/src/main/java/org/elasticsearch/client/IndexLifecycleClient.java

@@ -20,7 +20,7 @@
 package org.elasticsearch.client;
 
 import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.action.support.master.AcknowledgedResponse;
+import org.elasticsearch.client.core.AcknowledgedResponse;
 import org.elasticsearch.client.indexlifecycle.DeleteLifecyclePolicyRequest;
 import org.elasticsearch.client.indexlifecycle.ExplainLifecycleRequest;
 import org.elasticsearch.client.indexlifecycle.ExplainLifecycleResponse;
@@ -29,9 +29,9 @@
 import org.elasticsearch.client.indexlifecycle.LifecycleManagementStatusRequest;
 import org.elasticsearch.client.indexlifecycle.LifecycleManagementStatusResponse;
 import org.elasticsearch.client.indexlifecycle.PutLifecyclePolicyRequest;
-import org.elasticsearch.client.indexlifecycle.RetryLifecyclePolicyRequest;
 import org.elasticsearch.client.indexlifecycle.RemoveIndexLifecyclePolicyRequest;
 import org.elasticsearch.client.indexlifecycle.RemoveIndexLifecyclePolicyResponse;
+import org.elasticsearch.client.indexlifecycle.RetryLifecyclePolicyRequest;
 import org.elasticsearch.client.indexlifecycle.StartILMRequest;
 import org.elasticsearch.client.indexlifecycle.StopILMRequest;
 

client/rest-high-level/src/main/java/org/elasticsearch/client/LicenseClient.java

@@ -26,6 +26,8 @@
 import org.elasticsearch.client.license.StartTrialResponse;
 import org.elasticsearch.client.license.StartBasicRequest;
 import org.elasticsearch.client.license.StartBasicResponse;
+import org.elasticsearch.client.license.GetBasicStatusResponse;
+import org.elasticsearch.client.license.GetTrialStatusResponse;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.Streams;
 import org.elasticsearch.common.xcontent.DeprecationHandler;
@@ -172,6 +174,28 @@ public void startBasicAsync(StartBasicRequest request, RequestOptions options,
             StartBasicResponse::fromXContent, listener, emptySet());
     }
 
+    /**
+     * Retrieve the license trial status
+     * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     * @return the response
+     * @throws IOException in case there is a problem sending the request or parsing back the response
+     */
+    public GetTrialStatusResponse getTrialStatus(RequestOptions options) throws IOException {
+        return restHighLevelClient.performRequestAndParseEntity(Validatable.EMPTY,
+            request -> LicenseRequestConverters.getLicenseTrialStatus(), options, GetTrialStatusResponse::fromXContent, emptySet());
+    }
+
+    /**
+     * Retrieve the license basic status
+     * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     * @return the response
+     * @throws IOException in case there is a problem sending the request or parsing back the response
+     */
+    public GetBasicStatusResponse getBasicStatus(RequestOptions options) throws IOException {
+        return restHighLevelClient.performRequestAndParseEntity(Validatable.EMPTY,
+            request -> LicenseRequestConverters.getLicenseBasicStatus(), options, GetBasicStatusResponse::fromXContent, emptySet());
+    }
+
     /**
      * Converts an entire response into a json string
      *

client/rest-high-level/src/main/java/org/elasticsearch/client/LicenseRequestConverters.java

@@ -88,4 +88,12 @@ static Request startBasic(StartBasicRequest startBasicRequest) {
         }
         return request;
     }
+
+    static Request getLicenseTrialStatus() {
+        return new Request(HttpGet.METHOD_NAME, "/_xpack/license/trial_status");
+    }
+
+    static Request getLicenseBasicStatus() {
+        return new Request(HttpGet.METHOD_NAME, "/_xpack/license/basic_status");
+    }
 }

client/rest-high-level/src/main/java/org/elasticsearch/client/MigrationClient.java

@@ -21,6 +21,11 @@
 
 import org.elasticsearch.client.migration.IndexUpgradeInfoRequest;
 import org.elasticsearch.client.migration.IndexUpgradeInfoResponse;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.client.tasks.TaskSubmissionResponse;
+import org.elasticsearch.index.reindex.BulkByScrollResponse;
+import org.elasticsearch.client.migration.IndexUpgradeRequest;
+
 
 import java.io.IOException;
 import java.util.Collections;
@@ -52,4 +57,19 @@ public IndexUpgradeInfoResponse getAssistance(IndexUpgradeInfoRequest request, R
         return restHighLevelClient.performRequestAndParseEntity(request, MigrationRequestConverters::getMigrationAssistance, options,
             IndexUpgradeInfoResponse::fromXContent, Collections.emptySet());
     }
+
+    public BulkByScrollResponse upgrade(IndexUpgradeRequest request, RequestOptions options) throws IOException {
+        return restHighLevelClient.performRequestAndParseEntity(request, MigrationRequestConverters::migrate, options,
+            BulkByScrollResponse::fromXContent, Collections.emptySet());
+    }
+
+    public TaskSubmissionResponse submitUpgradeTask(IndexUpgradeRequest request, RequestOptions options) throws IOException {
+        return restHighLevelClient.performRequestAndParseEntity(request, MigrationRequestConverters::submitMigrateTask, options,
+            TaskSubmissionResponse::fromXContent, Collections.emptySet());
+    }
+
+    public void upgradeAsync(IndexUpgradeRequest request, RequestOptions options, ActionListener<BulkByScrollResponse> listener)  {
+        restHighLevelClient.performRequestAsyncAndParseEntity(request, MigrationRequestConverters::migrate, options,
+            BulkByScrollResponse::fromXContent, listener, Collections.emptySet());
+    }
 }

client/rest-high-level/src/main/java/org/elasticsearch/client/MigrationRequestConverters.java

@@ -20,11 +20,14 @@
 package org.elasticsearch.client;
 
 import org.apache.http.client.methods.HttpGet;
+import org.apache.http.client.methods.HttpPost;
 import org.elasticsearch.client.migration.IndexUpgradeInfoRequest;
+import org.elasticsearch.client.migration.IndexUpgradeRequest;
 
 final class MigrationRequestConverters {
 
-    private MigrationRequestConverters() {}
+    private MigrationRequestConverters() {
+    }
 
     static Request getMigrationAssistance(IndexUpgradeInfoRequest indexUpgradeInfoRequest) {
         RequestConverters.EndpointBuilder endpointBuilder = new RequestConverters.EndpointBuilder()
@@ -36,4 +39,26 @@ static Request getMigrationAssistance(IndexUpgradeInfoRequest indexUpgradeInfoRe
         parameters.withIndicesOptions(indexUpgradeInfoRequest.indicesOptions());
         return request;
     }
+
+    static Request migrate(IndexUpgradeRequest indexUpgradeRequest) {
+        return prepareMigrateRequest(indexUpgradeRequest, true);
+    }
+
+    static Request submitMigrateTask(IndexUpgradeRequest indexUpgradeRequest) {
+        return prepareMigrateRequest(indexUpgradeRequest, false);
+    }
+
+    private static Request prepareMigrateRequest(IndexUpgradeRequest indexUpgradeRequest, boolean waitForCompletion) {
+        String endpoint = new RequestConverters.EndpointBuilder()
+            .addPathPartAsIs("_xpack", "migration", "upgrade")
+            .addPathPart(indexUpgradeRequest.index())
+            .build();
+
+        Request request = new Request(HttpPost.METHOD_NAME, endpoint);
+
+        RequestConverters.Params params = new RequestConverters.Params(request)
+            .withWaitForCompletion(waitForCompletion);
+
+        return request;
+    }
 }

client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java

@@ -1417,6 +1417,38 @@ private <Req, Resp> Resp internalPerformRequest(Req request,
             throw new IOException("Unable to parse response body for " + response, e);
         }
     }
+
+    /**
+     * Defines a helper method for requests that can 404 and in which case will return an empty Optional 
+     * otherwise tries to parse the response body
+     */
+    protected final <Req extends Validatable, Resp> Optional<Resp> performRequestAndParseOptionalEntity(Req request,
+                                                                  CheckedFunction<Req, Request, IOException> requestConverter,
+                                                                  RequestOptions options,
+                                                                  CheckedFunction<XContentParser, Resp, IOException> entityParser
+                                                                  ) throws IOException {
+        Optional<ValidationException> validationException = request.validate();
+        if (validationException != null && validationException.isPresent()) {
+            throw validationException.get();
+        }
+        Request req = requestConverter.apply(request);
+        req.setOptions(options);
+        Response response;
+        try {
+            response = client.performRequest(req);
+        } catch (ResponseException e) {
+            if (RestStatus.NOT_FOUND.getStatus() == e.getResponse().getStatusLine().getStatusCode()) {
+                return Optional.empty();
+            }
+            throw parseResponseException(e);
+        }
+
+        try {
+            return Optional.of(parseEntity(response.getEntity(), entityParser));
+        } catch (Exception e) {
+            throw new IOException("Unable to parse response body for " + response, e);
+        }
+    }      
 
     /**
      * @deprecated If creating a new HLRC ReST API call, consider creating new actions instead of reusing server actions. The Validation
@@ -1538,6 +1570,62 @@ public void onFailure(Exception exception) {
             }
         };
     }
+
+    /**
+     * Async request which returns empty Optionals in the case of 404s or parses entity into an Optional
+     */
+    protected final <Req extends Validatable, Resp> void performRequestAsyncAndParseOptionalEntity(Req request,
+            CheckedFunction<Req, Request, IOException> requestConverter,
+            RequestOptions options,
+            CheckedFunction<XContentParser, Resp, IOException> entityParser,
+            ActionListener<Optional<Resp>> listener) {
+        Optional<ValidationException> validationException = request.validate();
+        if (validationException != null && validationException.isPresent()) {
+            listener.onFailure(validationException.get());
+            return;
+        }
+        Request req;
+        try {
+            req = requestConverter.apply(request);
+        } catch (Exception e) {
+            listener.onFailure(e);
+            return;
+        }
+        req.setOptions(options);
+        ResponseListener responseListener = wrapResponseListener404sOptional(response -> parseEntity(response.getEntity(), 
+                entityParser), listener);
+        client.performRequestAsync(req, responseListener);        
+    }    
+
+    final <Resp> ResponseListener wrapResponseListener404sOptional(CheckedFunction<Response, Resp, IOException> responseConverter,
+            ActionListener<Optional<Resp>> actionListener) {
+        return new ResponseListener() {
+            @Override
+            public void onSuccess(Response response) {
+                try {
+                    actionListener.onResponse(Optional.of(responseConverter.apply(response)));
+                } catch (Exception e) {
+                    IOException ioe = new IOException("Unable to parse response body for " + response, e);
+                    onFailure(ioe);
+                }
+            }
+
+            @Override
+            public void onFailure(Exception exception) {
+                if (exception instanceof ResponseException) {
+                    ResponseException responseException = (ResponseException) exception;
+                    Response response = responseException.getResponse();
+                    if (RestStatus.NOT_FOUND.getStatus() == response.getStatusLine().getStatusCode()) {
+                            actionListener.onResponse(Optional.empty());
+                    } else {
+                        actionListener.onFailure(parseResponseException(responseException));
+                    }
+                } else {
+                    actionListener.onFailure(exception);
+                }
+            }
+        };
+    }    
 
     /**
      * Converts a {@link ResponseException} obtained from the low level REST client into an {@link ElasticsearchException}.