Merge remote-tracking branch 'es/6.x' into ccr-6.x

* es/6.x: (29 commits)
  [Docs] Add rank_eval size parameter k (#29218)
  Docs: Update docs/index_.asciidoc (#29172)
  Docs: Link C++ client lib elasticlient (#28949)
  Docs: HighLevelRestClient#multiSearch (#29144)
  [DOCS] Remove ignore_z_value parameter link
  Add Z value support to geo_shape
  Change BroadcastResponse from ToXContentFragment to ToXContentObject (#28878)
  REST : Split `RestUpgradeAction` into two actions (#29124)
  [DOCS] Unregister repository instead of deleting it (#29206)
  Remove type casts in logging in server component (#28807)
  Add error file docs to important settings
  Add note to low-level client docs for DNS caching (#29213)
  testShrinkAfterUpgrade should only set mapping.single_type if bwc version > 5.5.0
  Harden periodically check to avoid endless flush loop (#29125)
  REST high-level client: add force merge API (#28896)
  Remove license information from README.textile (#29198)
  Decouple more classes from XContentBuilder and make builder strict (#29197)
  Propagate mapping.single_type setting on shrinked index (#29202)
  [Docs] Fix missing closing block in cluster/misc.asciidoc
  RankEvalRequest should implement IndicesRequest (#29188)
  ...

README.textile

@@ -27,7 +27,6 @@ Elasticsearch is a distributed RESTful search engine built for the cloud. Featur
 ** All the power of Lucene easily exposed through simple configuration / plugins.
 * Per operation consistency
 ** Single document level operations are atomic, consistent, isolated and durable.
-* Open Source under the Apache License, version 2 ("ALv2")
 
 h2. Getting Started
 
@@ -217,23 +216,3 @@ Elasticsearch (1.x), it is required to perform a full cluster restart. Please
 see the "setup reference":
 https://www.elastic.co/guide/en/elasticsearch/reference/current/setup-upgrade.html
 for more details on the upgrade process.
-
-h1. License
-
-<pre>
-This software is licensed under the Apache License, version 2 ("ALv2"), quoted below.
-
-Copyright 2009-2016 Elasticsearch <https://www.elastic.co>
-
-Licensed under the Apache License, Version 2.0 (the "License"); you may not
-use this file except in compliance with the License. You may obtain a copy of
-the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-License for the specific language governing permissions and limitations under
-the License.
-</pre>

buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy

@@ -475,14 +475,18 @@ class BuildPlugin implements Plugin<Project> {
     }
 
     static void configureJavadoc(Project project) {
-        project.tasks.withType(Javadoc) {
-            executable = new File(project.compilerJavaHome, 'bin/javadoc')
+        // remove compiled classes from the Javadoc classpath: http://mail.openjdk.java.net/pipermail/javadoc-dev/2018-January/000400.html
+        final List<File> classes = new ArrayList<>()
+        project.tasks.withType(JavaCompile) { javaCompile ->
+            classes.add(javaCompile.destinationDir)
         }
-        configureJavadocJar(project)
-        if (project.compilerJavaVersion == JavaVersion.VERSION_1_10) {
-            project.tasks.withType(Javadoc) { it.enabled = false }
-            project.tasks.getByName('javadocJar').each { it.enabled = false }
+        project.tasks.withType(Javadoc) { javadoc ->
+            javadoc.executable = new File(project.compilerJavaHome, 'bin/javadoc')
+            javadoc.classpath = javadoc.getClasspath().filter { f ->
+                return classes.contains(f) == false
+            }
         }
+        configureJavadocJar(project)
     }
 
     /** Adds a javadocJar task to generate a jar containing javadocs. */

client/rest-high-level/src/main/java/org/elasticsearch/client/IndicesClient.java

@@ -34,15 +34,17 @@
 import org.elasticsearch.action.admin.indices.delete.DeleteIndexResponse;
 import org.elasticsearch.action.admin.indices.flush.FlushRequest;
 import org.elasticsearch.action.admin.indices.flush.FlushResponse;
+import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeRequest;
+import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeResponse;
 import org.elasticsearch.action.admin.indices.get.GetIndexRequest;
 import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest;
 import org.elasticsearch.action.admin.indices.mapping.put.PutMappingResponse;
 import org.elasticsearch.action.admin.indices.open.OpenIndexRequest;
 import org.elasticsearch.action.admin.indices.open.OpenIndexResponse;
-import org.elasticsearch.action.admin.indices.rollover.RolloverRequest;
-import org.elasticsearch.action.admin.indices.rollover.RolloverResponse;
 import org.elasticsearch.action.admin.indices.refresh.RefreshRequest;
 import org.elasticsearch.action.admin.indices.refresh.RefreshResponse;
+import org.elasticsearch.action.admin.indices.rollover.RolloverRequest;
+import org.elasticsearch.action.admin.indices.rollover.RolloverResponse;
 import org.elasticsearch.action.admin.indices.shrink.ResizeRequest;
 import org.elasticsearch.action.admin.indices.shrink.ResizeResponse;
 
@@ -261,6 +263,28 @@ public void flushAsync(FlushRequest flushRequest, ActionListener<FlushResponse>
                 listener, emptySet(), headers);
     }
 
+    /**
+     * Force merge one or more indices using the Force Merge API
+     * <p>
+     * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-forcemerge.html">
+     * Force Merge API on elastic.co</a>
+     */
+    public ForceMergeResponse forceMerge(ForceMergeRequest forceMergeRequest, Header... headers) throws IOException {
+        return restHighLevelClient.performRequestAndParseEntity(forceMergeRequest, Request::forceMerge, ForceMergeResponse::fromXContent,
+            emptySet(), headers);
+    }
+
+    /**
+     * Asynchronously force merge one or more indices using the Force Merge API
+     * <p>
+     * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-forcemerge.html">
+     * Force Merge API on elastic.co</a>
+     */
+    public void forceMergeAsync(ForceMergeRequest forceMergeRequest, ActionListener<ForceMergeResponse> listener, Header... headers) {
+        restHighLevelClient.performRequestAsyncAndParseEntity(forceMergeRequest, Request::forceMerge, ForceMergeResponse::fromXContent,
+                listener, emptySet(), headers);
+    }
+
     /**
      * Clears the cache of one or more indices using the Clear Cache API
      * <p>

client/rest-high-level/src/main/java/org/elasticsearch/client/Request.java

@@ -37,6 +37,7 @@
 import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
 import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest;
 import org.elasticsearch.action.admin.indices.flush.FlushRequest;
+import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeRequest;
 import org.elasticsearch.action.admin.indices.get.GetIndexRequest;
 import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest;
 import org.elasticsearch.action.admin.indices.open.OpenIndexRequest;
@@ -235,6 +236,17 @@ static Request flush(FlushRequest flushRequest) {
         return new Request(HttpPost.METHOD_NAME, endpoint, parameters.getParams(), null);
     }
 
+    static Request forceMerge(ForceMergeRequest forceMergeRequest) {
+        String[] indices = forceMergeRequest.indices() == null ? Strings.EMPTY_ARRAY : forceMergeRequest.indices();
+        String endpoint = endpoint(indices, "_forcemerge");
+        Params parameters = Params.builder();
+        parameters.withIndicesOptions(forceMergeRequest.indicesOptions());
+        parameters.putParam("max_num_segments", Integer.toString(forceMergeRequest.maxNumSegments()));
+        parameters.putParam("only_expunge_deletes", Boolean.toString(forceMergeRequest.onlyExpungeDeletes()));
+        parameters.putParam("flush", Boolean.toString(forceMergeRequest.flush()));
+        return new Request(HttpPost.METHOD_NAME, endpoint, parameters.getParams(), null);
+    }
+
     static Request clearCache(ClearIndicesCacheRequest clearIndicesCacheRequest) {
         String[] indices = clearIndicesCacheRequest.indices() == null ? Strings.EMPTY_ARRAY :clearIndicesCacheRequest.indices();
         String endpoint = endpoint(indices, "_cache/clear");
@@ -533,7 +545,7 @@ static Request existsAlias(GetAliasesRequest getAliasesRequest) {
     }
 
     static Request rankEval(RankEvalRequest rankEvalRequest) throws IOException {
-        String endpoint = endpoint(rankEvalRequest.getIndices(), Strings.EMPTY_ARRAY, "_rank_eval");
+        String endpoint = endpoint(rankEvalRequest.indices(), Strings.EMPTY_ARRAY, "_rank_eval");
         HttpEntity entity = createEntity(rankEvalRequest.getRankEvalSpec(), REQUEST_BODY_CONTENT_TYPE);
         return new Request(HttpGet.METHOD_NAME, endpoint, Collections.emptyMap(), entity);
     }

client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesClientIT.java

@@ -38,6 +38,8 @@
 import org.elasticsearch.action.admin.indices.delete.DeleteIndexResponse;
 import org.elasticsearch.action.admin.indices.flush.FlushRequest;
 import org.elasticsearch.action.admin.indices.flush.FlushResponse;
+import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeRequest;
+import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeResponse;
 import org.elasticsearch.action.admin.indices.get.GetIndexRequest;
 import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest;
 import org.elasticsearch.action.admin.indices.mapping.put.PutMappingResponse;
@@ -469,6 +471,32 @@ public void testClearCache() throws IOException {
         }
     }
 
+    public void testForceMerge() throws IOException {
+        {
+            String index = "index";
+            Settings settings = Settings.builder()
+                .put("number_of_shards", 1)
+                .put("number_of_replicas", 0)
+                .build();
+            createIndex(index, settings);
+            ForceMergeRequest forceMergeRequest = new ForceMergeRequest(index);
+            ForceMergeResponse forceMergeResponse =
+                execute(forceMergeRequest, highLevelClient().indices()::forceMerge, highLevelClient().indices()::forceMergeAsync);
+            assertThat(forceMergeResponse.getTotalShards(), equalTo(1));
+            assertThat(forceMergeResponse.getSuccessfulShards(), equalTo(1));
+            assertThat(forceMergeResponse.getFailedShards(), equalTo(0));
+            assertThat(forceMergeResponse.getShardFailures(), equalTo(BroadcastResponse.EMPTY));
+        }
+        {
+            String nonExistentIndex = "non_existent_index";
+            assertFalse(indexExists(nonExistentIndex));
+            ForceMergeRequest forceMergeRequest = new ForceMergeRequest(nonExistentIndex);
+            ElasticsearchException exception = expectThrows(ElasticsearchException.class,
+                () -> execute(forceMergeRequest, highLevelClient().indices()::forceMerge, highLevelClient().indices()::forceMergeAsync));
+            assertEquals(RestStatus.NOT_FOUND, exception.status());
+        }
+    }
+
     public void testExistsAlias() throws IOException {
         GetAliasesRequest getAliasesRequest = new GetAliasesRequest("alias");
         assertFalse(execute(getAliasesRequest, highLevelClient().indices()::existsAlias, highLevelClient().indices()::existsAliasAsync));

client/rest-high-level/src/test/java/org/elasticsearch/client/RequestTests.java

@@ -40,6 +40,7 @@
 import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
 import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest;
 import org.elasticsearch.action.admin.indices.flush.FlushRequest;
+import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeRequest;
 import org.elasticsearch.action.admin.indices.get.GetIndexRequest;
 import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest;
 import org.elasticsearch.action.admin.indices.open.OpenIndexRequest;
@@ -623,6 +624,43 @@ public void testFlush() {
         assertThat(request.getMethod(), equalTo(HttpPost.METHOD_NAME));
     }
 
+    public void testForceMerge() {
+        String[] indices = randomBoolean() ? null : randomIndicesNames(0, 5);
+        ForceMergeRequest forceMergeRequest;
+        if (randomBoolean()) {
+            forceMergeRequest = new ForceMergeRequest(indices);
+        } else {
+            forceMergeRequest = new ForceMergeRequest();
+            forceMergeRequest.indices(indices);
+        }
+
+        Map<String, String> expectedParams = new HashMap<>();
+        setRandomIndicesOptions(forceMergeRequest::indicesOptions, forceMergeRequest::indicesOptions, expectedParams);
+        if (randomBoolean()) {
+            forceMergeRequest.maxNumSegments(randomInt());
+        }
+        expectedParams.put("max_num_segments", Integer.toString(forceMergeRequest.maxNumSegments()));
+        if (randomBoolean()) {
+            forceMergeRequest.onlyExpungeDeletes(randomBoolean());
+        }
+        expectedParams.put("only_expunge_deletes", Boolean.toString(forceMergeRequest.onlyExpungeDeletes()));
+        if (randomBoolean()) {
+            forceMergeRequest.flush(randomBoolean());
+        }
+        expectedParams.put("flush", Boolean.toString(forceMergeRequest.flush()));
+
+        Request request = Request.forceMerge(forceMergeRequest);
+        StringJoiner endpoint = new StringJoiner("/", "/", "");
+        if (indices != null && indices.length > 0) {
+            endpoint.add(String.join(",", indices));
+        }
+        endpoint.add("_forcemerge");
+        assertThat(request.getEndpoint(), equalTo(endpoint.toString()));
+        assertThat(request.getParameters(), equalTo(expectedParams));
+        assertThat(request.getEntity(), nullValue());
+        assertThat(request.getMethod(), equalTo(HttpPost.METHOD_NAME));
+    }
+
     public void testClearCache() {
         String[] indices = randomBoolean() ? null : randomIndicesNames(0, 5);
         ClearIndicesCacheRequest clearIndicesCacheRequest;

client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IndicesClientDocumentationIT.java

@@ -37,6 +37,8 @@
 import org.elasticsearch.action.admin.indices.delete.DeleteIndexResponse;
 import org.elasticsearch.action.admin.indices.flush.FlushRequest;
 import org.elasticsearch.action.admin.indices.flush.FlushResponse;
+import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeRequest;
+import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeResponse;
 import org.elasticsearch.action.admin.indices.get.GetIndexRequest;
 import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest;
 import org.elasticsearch.action.admin.indices.mapping.put.PutMappingResponse;
@@ -771,6 +773,79 @@ public void onFailure(Exception e) {
         }
     }
 
+    public void testForceMergeIndex() throws Exception {
+        RestHighLevelClient client = highLevelClient();
+
+        {
+            createIndex("index", Settings.EMPTY);
+        }
+
+        {
+            // tag::force-merge-request
+            ForceMergeRequest request = new ForceMergeRequest("index1"); // <1>
+            ForceMergeRequest requestMultiple = new ForceMergeRequest("index1", "index2"); // <2>
+            ForceMergeRequest requestAll = new ForceMergeRequest(); // <3>
+            // end::force-merge-request
+
+            // tag::force-merge-request-indicesOptions
+            request.indicesOptions(IndicesOptions.lenientExpandOpen()); // <1>
+            // end::force-merge-request-indicesOptions
+
+            // tag::force-merge-request-segments-num
+            request.maxNumSegments(1); // <1>
+            // end::force-merge-request-segments-num
+
+            // tag::force-merge-request-only-expunge-deletes
+            request.onlyExpungeDeletes(true); // <1>
+            // end::force-merge-request-only-expunge-deletes
+
+            // tag::force-merge-request-flush
+            request.flush(true); // <1>
+            // end::force-merge-request-flush
+
+            // tag::force-merge-execute
+            ForceMergeResponse forceMergeResponse = client.indices().forceMerge(request);
+            // end::force-merge-execute
+
+            // tag::force-merge-response
+            int totalShards = forceMergeResponse.getTotalShards(); // <1>
+            int successfulShards = forceMergeResponse.getSuccessfulShards(); // <2>
+            int failedShards = forceMergeResponse.getFailedShards(); // <3>
+            DefaultShardOperationFailedException[] failures = forceMergeResponse.getShardFailures(); // <4>
+            // end::force-merge-response
+
+            // tag::force-merge-execute-listener
+            ActionListener<ForceMergeResponse> listener = new ActionListener<ForceMergeResponse>() {
+                @Override
+                public void onResponse(ForceMergeResponse forceMergeResponse) {
+                    // <1>
+                }
+
+                @Override
+                public void onFailure(Exception e) {
+                    // <2>
+                }
+            };
+            // end::force-merge-execute-listener
+
+            // tag::force-merge-execute-async
+            client.indices().forceMergeAsync(request, listener); // <1>
+            // end::force-merge-execute-async
+        }
+        {
+            // tag::force-merge-notfound
+            try {
+                ForceMergeRequest request = new ForceMergeRequest("does_not_exist");
+                client.indices().forceMerge(request);
+            } catch (ElasticsearchException exception) {
+                if (exception.status() == RestStatus.NOT_FOUND) {
+                    // <1>
+                }
+            }
+            // end::force-merge-notfound
+        }
+    }
+
     public void testClearCache() throws Exception {
         RestHighLevelClient client = highLevelClient();
 
@@ -855,7 +930,6 @@ public void onFailure(Exception e) {
         }
     }
 
-
     public void testCloseIndex() throws Exception {
         RestHighLevelClient client = highLevelClient();