Merge branch 'master' into index_routing_from_source_clean_2

elastic · Oct 27, 2021 · 673d8bf · 673d8bf
2 parents d7fd2d2 + a8258bd
commit 673d8bf
Show file tree

Hide file tree

Showing 47 changed files with 451 additions and 868 deletions.
diff --git a/.backportrc.json b/.backportrc.json
@@ -11,6 +11,6 @@
   "targetPRLabels": ["backport"],
   "branchLabelMapping": {
     "^v8.1.0$": "master",
-    "^v(\\d+).(\\d+).\\d+$": "$1.$2"
+    "^v(\\d+).(\\d+).\\d+(?:-(?:alpha|beta|rc)\\d+)?$": "$1.$2"
   }
 }
diff --git a/.ci/java-versions-fips.properties b/.ci/java-versions-fips.properties
@@ -0,0 +1,8 @@
+# This file is used when building pull requests with FIPS enabled
+
+# This .properties file defines the versions of Java with which to
+# build and test Elasticsearch for this branch. Valid Java versions
+# are 'java' or 'openjdk' followed by the major release number.
+
+ES_BUILD_JAVA=openjdk16
+ES_RUNTIME_JAVA=openjdk17
diff --git a/.ci/jobs.t/elastic+elasticsearch+multijob+platform-support-arm.yml b/.ci/jobs.t/elastic+elasticsearch+multijob+platform-support-arm.yml
@@ -23,6 +23,7 @@
             COMPOSE_HTTP_TIMEOUT=120
             JAVA_HOME=$HOME/.java/$ES_BUILD_JAVA
             RUNTIME_JAVA_HOME=$HOME/.java/$ES_RUNTIME_JAVA
+            JAVA11_HOME=$HOME/.java/java11
             JAVA15_HOME=$HOME/.java/jdk15
       - shell: |
           #!/usr/local/bin/runbld --redirect-stderr

diff --git a/.ci/jobs.t/elastic+elasticsearch+multijob+platform-support-darwin.yml b/.ci/jobs.t/elastic+elasticsearch+multijob+platform-support-darwin.yml
@@ -10,6 +10,7 @@
           properties-content: |
             JAVA_HOME=$HOME/.java/$ES_BUILD_JAVA
             RUNTIME_JAVA_HOME=$HOME/.java/$ES_RUNTIME_JAVA
+            JAVA11_HOME=$HOME/.java/java11
             JAVA15_HOME=$HOME/.java/openjdk15
       - shell: |
           #!/usr/local/bin/runbld --redirect-stderr

diff --git a/.ci/jobs.t/elastic+elasticsearch+multijob+platform-support-unix.yml b/.ci/jobs.t/elastic+elasticsearch+multijob+platform-support-unix.yml
@@ -32,6 +32,7 @@
           properties-content: |
             JAVA_HOME=$HOME/.java/$ES_BUILD_JAVA
             RUNTIME_JAVA_HOME=$HOME/.java/$ES_RUNTIME_JAVA
+            JAVA11_HOME=$HOME/.java/java11
             JAVA15_HOME=$HOME/.java/openjdk15
       - shell: |
           #!/usr/local/bin/runbld --redirect-stderr

diff --git a/.ci/jobs.t/elastic+elasticsearch+multijob+platform-support-windows.yml b/.ci/jobs.t/elastic+elasticsearch+multijob+platform-support-windows.yml
@@ -34,6 +34,7 @@
           properties-content: |
             JAVA_HOME=$USERPROFILE\\.java\\$ES_BUILD_JAVA
             RUNTIME_JAVA_HOME=$USERPROFILE\\.java\\$ES_RUNTIME_JAVA
+            JAVA11_HOME=$USERPROFILE\\.java\\java11
             JAVA15_HOME=$USERPROFILE\\.java\\openjdk15
       - batch: |
           del /f /s /q %USERPROFILE%\.gradle\init.d\*.*

diff --git a/.ci/jobs.t/elastic+elasticsearch+periodic+ear.yml b/.ci/jobs.t/elastic+elasticsearch+periodic+ear.yml
@@ -11,6 +11,7 @@
           properties-content: |
             JAVA_HOME=$HOME/.java/$ES_BUILD_JAVA
             RUNTIME_JAVA_HOME=$HOME/.java/$ES_RUNTIME_JAVA
+            JAVA11_HOME=$HOME/.java/java11
             JAVA15_HOME=$HOME/.java/openjdk15
       - shell: |
           #!/bin/bash

diff --git a/.ci/jobs.t/elastic+elasticsearch+periodic+release-tests.yml b/.ci/jobs.t/elastic+elasticsearch+periodic+release-tests.yml
@@ -11,6 +11,7 @@
           properties-content: |
             JAVA_HOME=$HOME/.java/$ES_BUILD_JAVA
             RUNTIME_JAVA_HOME=$HOME/.java/$ES_RUNTIME_JAVA
+            JAVA11_HOME=$HOME/.java/java11
             JAVA15_HOME=$HOME/.java/openjdk15
       - shell: |
           #!/usr/local/bin/runbld --redirect-stderr

diff --git a/.ci/jobs.t/elastic+elasticsearch+pull-request+part-1-fips.yml b/.ci/jobs.t/elastic+elasticsearch+pull-request+part-1-fips.yml
@@ -28,10 +28,10 @@
             - '>test-mute'
     builders:
       - inject:
-          properties-file: '.ci/java-versions.properties'
+          properties-file: '.ci/java-versions-fips.properties'
           properties-content: |
             JAVA_HOME=$HOME/.java/$ES_BUILD_JAVA
-            RUNTIME_JAVA_HOME=$HOME/.java/java11
+            RUNTIME_JAVA_HOME=$HOME/.java/$ES_RUNTIME_JAVA
             JAVA15_HOME=$HOME/.java/openjdk15
       - shell: |
           #!/usr/local/bin/runbld --redirect-stderr

diff --git a/.ci/jobs.t/elastic+elasticsearch+pull-request+part-2-fips.yml b/.ci/jobs.t/elastic+elasticsearch+pull-request+part-2-fips.yml
@@ -28,10 +28,10 @@
             - '>test-mute'
     builders:
       - inject:
-          properties-file: '.ci/java-versions.properties'
+          properties-file: '.ci/java-versions-fips.properties'
           properties-content: |
             JAVA_HOME=$HOME/.java/$ES_BUILD_JAVA
-            RUNTIME_JAVA_HOME=$HOME/.java/java11
+            RUNTIME_JAVA_HOME=$HOME/.java/$ES_RUNTIME_JAVA
             JAVA15_HOME=$HOME/.java/openjdk15
       - shell: |
           #!/usr/local/bin/runbld --redirect-stderr

diff --git a/.ci/templates.t/generic-gradle-unix.yml b/.ci/templates.t/generic-gradle-unix.yml
@@ -10,6 +10,7 @@
           properties-content: |
             JAVA_HOME=$HOME/.java/$ES_BUILD_JAVA
             RUNTIME_JAVA_HOME=$HOME/.java/$ES_RUNTIME_JAVA
+            JAVA11_HOME=$HOME/.java/java11
             JAVA15_HOME=$HOME/.java/openjdk15
       - shell: |
           #!/usr/local/bin/runbld --redirect-stderr

diff --git a/.ci/templates.t/matrix-gradle-unix.yml b/.ci/templates.t/matrix-gradle-unix.yml
@@ -25,6 +25,7 @@
           properties-content: |
             JAVA_HOME=$HOME/.java/$ES_BUILD_JAVA
             RUNTIME_JAVA_HOME=$HOME/.java/$ES_RUNTIME_JAVA
+            JAVA11_HOME=$HOME/.java/java11
             JAVA15_HOME=$HOME/.java/openjdk15
       - shell: |
           #!/usr/local/bin/runbld --redirect-stderr

diff --git a/...ain/java/org/elasticsearch/benchmark/search/fetch/subphase/FetchSourcePhaseBenchmark.java b/...ain/java/org/elasticsearch/benchmark/search/fetch/subphase/FetchSourcePhaseBenchmark.java
@@ -12,8 +12,8 @@
 import org.elasticsearch.xcontent.NamedXContentRegistry;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.XContentParser;
+import org.elasticsearch.xcontent.XContentParserConfiguration;
 import org.elasticsearch.xcontent.XContentType;
-import org.elasticsearch.xcontent.support.filtering.FilterPath;
 import org.openjdk.jmh.annotations.Benchmark;
 import org.openjdk.jmh.annotations.BenchmarkMode;
 import org.openjdk.jmh.annotations.Fork;
@@ -41,8 +41,7 @@ public class FetchSourcePhaseBenchmark {
     private FetchSourceContext fetchContext;
     private Set<String> includesSet;
     private Set<String> excludesSet;
-    private FilterPath[] includesFilters;
-    private FilterPath[] excludesFilters;
+    private XContentParserConfiguration parserConfig;
 
     @Param({ "tiny", "short", "one_4k_field", "one_4m_field" })
     private String source;
@@ -76,8 +75,7 @@ public void setup() throws IOException {
         );
         includesSet = Set.of(fetchContext.includes());
         excludesSet = Set.of(fetchContext.excludes());
-        includesFilters = FilterPath.compile(Set.of(fetchContext.includes()));
-        excludesFilters = FilterPath.compile(Set.of(fetchContext.excludes()));
+        parserConfig = XContentParserConfiguration.EMPTY.withFiltering(includesSet, excludesSet);
     }
 
     private BytesReference read300BytesExample() throws IOException {
@@ -102,16 +100,7 @@ public BytesReference filterObjects() throws IOException {
     public BytesReference filterXContentOnParser() throws IOException {
         BytesStreamOutput streamOutput = new BytesStreamOutput(Math.min(1024, sourceBytes.length()));
         XContentBuilder builder = new XContentBuilder(XContentType.JSON.xContent(), streamOutput);
-        try (
-            XContentParser parser = XContentType.JSON.xContent()
-                .createParser(
-                    NamedXContentRegistry.EMPTY,
-                    DeprecationHandler.THROW_UNSUPPORTED_OPERATION,
-                    sourceBytes.streamInput(),
-                    includesFilters,
-                    excludesFilters
-                )
-        ) {
+        try (XContentParser parser = XContentType.JSON.xContent().createParser(parserConfig, sourceBytes.streamInput())) {
             builder.copyCurrentStructure(parser);
             return BytesReference.bytes(builder);
         }

diff --git a/build.gradle b/build.gradle
@@ -115,6 +115,12 @@ tasks.register("verifyVersions") {
         throw new GradleException("No branch choice exists for development branch ${unreleasedVersion.branch} in .backportrc.json.")
       }
     }
+    String versionMapping = backportConfig.get("branchLabelMapping").fields().find { it.value.textValue() == 'master' }.key
+    String expectedMapping = "^v${versions.elasticsearch.replaceAll('-SNAPSHOT', '')}\$"
+    if (versionMapping != expectedMapping) {
+      throw new GradleException("Backport label mapping for branch 'master' is '${versionMapping}' but should be " +
+        "'${expectedMapping}'. Update .backportrc.json.")
+    }
   }
 }
 

diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java
@@ -55,7 +55,6 @@
 import org.elasticsearch.common.io.Streams;
 import org.elasticsearch.common.lucene.uid.Versions;
 import org.elasticsearch.common.xcontent.XContentHelper;
-import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.core.Tuple;
 import org.elasticsearch.index.VersionType;
@@ -1257,17 +1256,16 @@ public void testMultiSearch() throws IOException {
             requests.add(searchRequest);
         };
         MultiSearchRequest.readMultiLineFormat(
-            new BytesArray(EntityUtils.toByteArray(request.getEntity())),
             REQUEST_BODY_CONTENT_TYPE.xContent(),
+            parserConfig(),
+            new BytesArray(EntityUtils.toByteArray(request.getEntity())),
             consumer,
             null,
             multiSearchRequest.indicesOptions(),
             null,
             null,
             null,
-            xContentRegistry(),
-            true,
-            RestApiVersion.current()
+            true
         );
         assertEquals(requests, multiSearchRequest.requests());
     }

diff --git a/docs/reference/ml/df-analytics/apis/put-dfanalytics.asciidoc b/docs/reference/ml/df-analytics/apis/put-dfanalytics.asciidoc
@@ -657,7 +657,7 @@ PUT _ml/data_frame/analytics/loganalytics
   }
 }
 --------------------------------------------------
-// TEST[skip:"AwaitsFix https://github.com/elastic/elasticsearch/issues/79931"]
+// TEST[setup:setup_logdata]
 
 
 The API returns the following result:
@@ -692,7 +692,7 @@ The API returns the following result:
 }
 ----
 // TESTRESPONSE[s/1562265491319/$body.$_path/]
-// TESTRESPONSE[s/"version" : "8.0.0"/"version" : $body.version/]
+// TESTRESPONSE[s/"version": "8.0.0"/"version": $body.version/]
 
 
 [[ml-put-dfanalytics-example-r]]

diff --git a/libs/x-content/src/main/java/org/elasticsearch/xcontent/XContent.java b/libs/x-content/src/main/java/org/elasticsearch/xcontent/XContent.java
@@ -8,9 +8,6 @@
 
 package org.elasticsearch.xcontent;
 
-import org.elasticsearch.core.RestApiVersion;
-import org.elasticsearch.xcontent.support.filtering.FilterPath;
-
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
@@ -50,78 +47,67 @@ default XContentGenerator createGenerator(OutputStream os) throws IOException {
     /**
      * Creates a parser over the provided string content.
      */
-    XContentParser createParser(NamedXContentRegistry xContentRegistry, DeprecationHandler deprecationHandler, String content)
-        throws IOException;
+    XContentParser createParser(XContentParserConfiguration config, String content) throws IOException;
+
+    /**
+     * Creates a parser over the provided string content.
+     * @deprecated Use {@link #createParser(XContentParserConfiguration, InputStream)}
+     */
+    @Deprecated
+    default XContentParser createParser(NamedXContentRegistry registry, DeprecationHandler deprecationHandler, String content)
+        throws IOException {
+        return createParser(XContentParserConfiguration.EMPTY.withRegistry(registry).withDeprecationHandler(deprecationHandler), content);
+    }
 
     /**
      * Creates a parser over the provided input stream.
      */
-    XContentParser createParser(NamedXContentRegistry xContentRegistry, DeprecationHandler deprecationHandler, InputStream is)
-        throws IOException;
+    XContentParser createParser(XContentParserConfiguration config, InputStream is) throws IOException;
 
     /**
      * Creates a parser over the provided input stream.
+     * @deprecated Use {@link #createParser(XContentParserConfiguration, InputStream)}
      */
-    XContentParser createParser(
-        NamedXContentRegistry xContentRegistry,
-        DeprecationHandler deprecationHandler,
-        InputStream is,
-        FilterPath[] includes,
-        FilterPath[] excludes
-    ) throws IOException;
+    @Deprecated
+    default XContentParser createParser(NamedXContentRegistry registry, DeprecationHandler deprecationHandler, InputStream is)
+        throws IOException {
+        return createParser(XContentParserConfiguration.EMPTY.withRegistry(registry).withDeprecationHandler(deprecationHandler), is);
+    }
 
     /**
      * Creates a parser over the provided bytes.
      */
-    XContentParser createParser(NamedXContentRegistry xContentRegistry, DeprecationHandler deprecationHandler, byte[] data)
-        throws IOException;
+    default XContentParser createParser(XContentParserConfiguration config, byte[] data) throws IOException {
+        return createParser(config, data, 0, data.length);
+    }
 
     /**
      * Creates a parser over the provided bytes.
+     * @deprecated Use {@link #createParser(XContentParserConfiguration, byte[])}
      */
-    XContentParser createParser(
-        NamedXContentRegistry xContentRegistry,
-        DeprecationHandler deprecationHandler,
-        byte[] data,
-        int offset,
-        int length
-    ) throws IOException;
+    @Deprecated
+    default XContentParser createParser(NamedXContentRegistry registry, DeprecationHandler deprecationHandler, byte[] data)
+        throws IOException {
+        return createParser(XContentParserConfiguration.EMPTY.withRegistry(registry).withDeprecationHandler(deprecationHandler), data);
+    }
 
-    XContentParser createParser(
-        NamedXContentRegistry xContentRegistry,
-        DeprecationHandler deprecationHandler,
-        byte[] data,
-        int offset,
-        int length,
-        FilterPath[] includes,
-        FilterPath[] excludes
-    ) throws IOException;
+    /**
+     * Creates a parser over the provided bytes.
+     */
+    XContentParser createParser(XContentParserConfiguration config, byte[] data, int offset, int length) throws IOException;
 
     /**
      * Creates a parser over the provided reader.
      */
-    XContentParser createParser(NamedXContentRegistry xContentRegistry, DeprecationHandler deprecationHandler, Reader reader)
-        throws IOException;
+    XContentParser createParser(XContentParserConfiguration config, Reader reader) throws IOException;
 
     /**
-     * Creates a parser over the provided input stream and with the indication that a request is using REST compatible API.
-     *
-     * @param restApiVersion - indicates if the N-1 or N compatible XContent parsing logic will be used.
+     * Creates a parser over the provided reader.
+     * @deprecated Use {@link #createParser(XContentParserConfiguration, Reader)}
      */
-    XContentParser createParserForCompatibility(
-        NamedXContentRegistry xContentRegistry,
-        DeprecationHandler deprecationHandler,
-        InputStream is,
-        RestApiVersion restApiVersion
-    ) throws IOException;
-
-    XContentParser createParserForCompatibility(
-        NamedXContentRegistry xContentRegistry,
-        DeprecationHandler deprecationHandler,
-        byte[] data,
-        int offset,
-        int length,
-        RestApiVersion restApiVersion
-    ) throws IOException;
-
+    @Deprecated
+    default XContentParser createParser(NamedXContentRegistry registry, DeprecationHandler deprecationHandler, Reader reader)
+        throws IOException {
+        return createParser(XContentParserConfiguration.EMPTY.withRegistry(registry).withDeprecationHandler(deprecationHandler), reader);
+    }
 }