Merge remote-tracking branch 'elastic/master' into watcher-stats-cce

* elastic/master:
  Add pre-upgrade check to test cluster routing allocation is enabled (elastic#39340)
  Update logstash-management.json to use typeless template (elastic#38653)
  Small simplifications to mapping validation. (elastic#39777)
  Update distribution build instructions to reflect file names with OS/architecture classifiers. (elastic#39762)
  Give jspawnhelper execute permissions in bundled JDK (elastic#39787)
  Maintain step order for ILM trace logging (elastic#39522)
  [ML-DataFrame] fix wire serialization issues in data frame response objects (elastic#39790)
  fix index refresh in test within 20_mix_typeless_typeful (elastic#39198)
  Combine overriddenOps and skippedOps in translog (elastic#39771)

CONTRIBUTING.md

@@ -202,14 +202,14 @@ Run all build commands from within the root directory:
 cd elasticsearch/
 ```
 
-To build a tar distribution, run this command:
+To build a darwin-tar distribution, run this command:
 
 ```sh
-./gradlew -p distribution/archives/tar assemble --parallel
+./gradlew -p distribution/archives/darwin-tar assemble --parallel
 ```
 
 You will find the distribution under:
-`./distribution/archives/tar/build/distributions/`
+`./distribution/archives/darwin-tar/build/distributions/`
 
 To create all build artifacts (e.g., plugins and Javadocs) as well as
 distributions in all formats, run this command:
@@ -219,10 +219,10 @@ distributions in all formats, run this command:
 ```
 
 The package distributions (Debian and RPM) can be found under:
-`./distribution/packages/(deb|rpm)/build/distributions/`
+`./distribution/packages/(deb|rpm|oss-deb|oss-rpm)/build/distributions/`
 
 The archive distributions (tar and zip) can be found under:
-`./distribution/archives/(tar|zip)/build/distributions/`
+`./distribution/archives/(darwin-tar|linux-tar|windows-zip|oss-darwin-tar|oss-linux-tar|oss-windows-zip)/build/distributions/`
 
 ### Running The Full Test Suite
 

distribution/build.gradle

@@ -385,7 +385,7 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) {
       copySpec {
         from project(':distribution').tasks.getByName("extract${platform.capitalize()}Jdk")
         eachFile { FileCopyDetails details ->
-          if (details.relativePath.segments[-2] == 'bin') {
+          if (details.relativePath.segments[-2] == 'bin' || details.relativePath.segments[-1] == 'jspawnhelper') {
             details.mode = 0755
           }
         }

distribution/packages/build.gradle

@@ -145,7 +145,7 @@ Closure commonPackageConfig(String type, boolean oss) {
         String[] segments = fcp.relativePath.segments
         for (int i = segments.length - 2; i > 2; --i) {
           directory('/' + segments[0..i].join('/'), 0755)
-          if (segments[-2] == 'bin') {
+          if (segments[-2] == 'bin' || segments[-1] == 'jspawnhelper') {
             fcp.mode = 0755
           } else {
             fcp.mode = 0644

modules/parent-join/src/test/java/org/elasticsearch/join/mapper/ParentJoinFieldMapperTests.java

@@ -400,7 +400,7 @@ public void testMultipleJoinFields() throws Exception {
                 .endObject());
             IllegalArgumentException exc = expectThrows(IllegalArgumentException.class, () -> indexService.mapperService().merge("type",
                 new CompressedXContent(mapping), MapperService.MergeReason.MAPPING_UPDATE));
-            assertThat(exc.getMessage(), containsString("Field [_parent_join] is defined twice in [type]"));
+            assertThat(exc.getMessage(), containsString("Field [_parent_join] is defined twice."));
         }
 
         {
@@ -426,7 +426,7 @@ public void testMultipleJoinFields() throws Exception {
                 .endObject());
             IllegalArgumentException exc = expectThrows(IllegalArgumentException.class, () -> indexService.mapperService().merge("type",
                 new CompressedXContent(updateMapping), MapperService.MergeReason.MAPPING_UPDATE));
-            assertThat(exc.getMessage(), containsString("Field [_parent_join] is defined twice in [type]"));
+            assertThat(exc.getMessage(), containsString("Field [_parent_join] is defined twice."));
         }
     }
 

rest-api-spec/src/main/resources/rest-api-spec/test/indices.create/20_mix_typeless_typeful.yml

@@ -104,8 +104,8 @@
 "Implicitly create a typeless index while there is a typed template":
 
  - skip:
-      version: "all"
-      reason: "awaits fix in #39198"
+      version: " - 6.99.99"
+      reason: needs typeless index operations to work on typed indices
 
  - do:
       indices.put_template:
@@ -124,6 +124,11 @@
           index: test-1
           body: { bar: 42 }
 
+# ensures dynamic mapping update is visible to get_mapping
+ - do:
+     cluster.health:
+         wait_for_events: normal
+
  - do:
       indices.get_mapping:
           include_type_name: true

server/src/main/java/org/elasticsearch/index/mapper/MapperMergeValidator.java

@@ -37,37 +37,18 @@ class MapperMergeValidator {
      * duplicate fields are present, and if the provided fields have already been
      * defined with a different data type.
      *
-     * @param type The mapping type, for use in error messages.
      * @param objectMappers The newly added object mappers.
      * @param fieldMappers The newly added field mappers.
      * @param fieldAliasMappers The newly added field alias mappers.
-     * @param fullPathObjectMappers All object mappers, indexed by their full path.
-     * @param fieldTypes All field and field alias mappers, collected into a lookup structure.
      */
-    public static void validateMapperStructure(String type,
-                                               Collection<ObjectMapper> objectMappers,
+    public static void validateMapperStructure(Collection<ObjectMapper> objectMappers,
                                                Collection<FieldMapper> fieldMappers,
-                                               Collection<FieldAliasMapper> fieldAliasMappers,
-                                               Map<String, ObjectMapper> fullPathObjectMappers,
-                                               FieldTypeLookup fieldTypes) {
-        checkFieldUniqueness(type, objectMappers, fieldMappers,
-            fieldAliasMappers, fullPathObjectMappers, fieldTypes);
-        checkObjectsCompatibility(objectMappers, fullPathObjectMappers);
-    }
-
-    private static void checkFieldUniqueness(String type,
-                                             Collection<ObjectMapper> objectMappers,
-                                             Collection<FieldMapper> fieldMappers,
-                                             Collection<FieldAliasMapper> fieldAliasMappers,
-                                             Map<String, ObjectMapper> fullPathObjectMappers,
-                                             FieldTypeLookup fieldTypes) {
-
-        // first check within mapping
+                                               Collection<FieldAliasMapper> fieldAliasMappers) {
         Set<String> objectFullNames = new HashSet<>();
         for (ObjectMapper objectMapper : objectMappers) {
             String fullPath = objectMapper.fullPath();
             if (objectFullNames.add(fullPath) == false) {
-                throw new IllegalArgumentException("Object mapper [" + fullPath + "] is defined twice in mapping for type [" + type + "]");
+                throw new IllegalArgumentException("Object mapper [" + fullPath + "] is defined twice.");
             }
         }
 
@@ -76,38 +57,11 @@ private static void checkFieldUniqueness(String type,
             .forEach(mapper -> {
                 String name = mapper.name();
                 if (objectFullNames.contains(name)) {
-                    throw new IllegalArgumentException("Field [" + name + "] is defined both as an object and a field in [" + type + "]");
+                    throw new IllegalArgumentException("Field [" + name + "] is defined both as an object and a field.");
                 } else if (fieldNames.add(name) == false) {
-                    throw new IllegalArgumentException("Field [" + name + "] is defined twice in [" + type + "]");
+                    throw new IllegalArgumentException("Field [" + name + "] is defined twice.");
                 }
             });
-
-        // then check other types
-        for (String fieldName : fieldNames) {
-            if (fullPathObjectMappers.containsKey(fieldName)) {
-                throw new IllegalArgumentException("[" + fieldName + "] is defined as a field in mapping [" + type
-                    + "] but this name is already used for an object in other types");
-            }
-        }
-
-        for (String objectPath : objectFullNames) {
-            if (fieldTypes.get(objectPath) != null) {
-                throw new IllegalArgumentException("[" + objectPath + "] is defined as an object in mapping [" + type
-                    + "] but this name is already used for a field in other types");
-            }
-        }
-    }
-
-    private static void checkObjectsCompatibility(Collection<ObjectMapper> objectMappers,
-                                                  Map<String, ObjectMapper> fullPathObjectMappers) {
-        for (ObjectMapper newObjectMapper : objectMappers) {
-            ObjectMapper existingObjectMapper = fullPathObjectMappers.get(newObjectMapper.fullPath());
-            if (existingObjectMapper != null) {
-                // simulate a merge and ignore the result, we are just interested
-                // in exceptions here
-                existingObjectMapper.merge(newObjectMapper);
-            }
-        }
     }
 
     /**

server/src/main/java/org/elasticsearch/index/mapper/MapperService.java

@@ -471,8 +471,7 @@ private synchronized Map<String, DocumentMapper> internalMerge(@Nullable Documen
             Collections.addAll(fieldMappers, metadataMappers);
             MapperUtils.collect(newMapper.mapping().root(), objectMappers, fieldMappers, fieldAliasMappers);
 
-            MapperMergeValidator.validateMapperStructure(newMapper.type(), objectMappers, fieldMappers,
-                fieldAliasMappers, fullPathObjectMappers, fieldTypes);
+            MapperMergeValidator.validateMapperStructure(objectMappers, fieldMappers, fieldAliasMappers);
             checkPartitionedIndexConstraints(newMapper);
 
             // update lookup data-structures

server/src/main/java/org/elasticsearch/index/translog/MultiSnapshot.java

@@ -58,16 +58,7 @@ public int totalOperations() {
 
     @Override
     public int skippedOperations() {
-        int skippedOperations = overriddenOperations;
-        for (TranslogSnapshot translog : translogs) {
-            skippedOperations += translog.skippedOperations();
-        }
-        return skippedOperations;
-    }
-
-    @Override
-    public int overriddenOperations() {
-        return overriddenOperations;
+        return Arrays.stream(translogs).mapToInt(TranslogSnapshot::skippedOperations).sum() + overriddenOperations;
     }
 
     @Override

server/src/main/java/org/elasticsearch/index/translog/Translog.java

@@ -936,20 +936,12 @@ public interface Snapshot extends Closeable {
 
         /**
          * The number of operations have been skipped (overridden or trimmed) in the snapshot so far.
+         * Unlike {@link #totalOperations()}, this value is updated each time after {@link #next()}) is called.
          */
         default int skippedOperations() {
             return 0;
         }
 
-        /**
-         * The number of operations have been overridden (eg. superseded) in the snapshot so far.
-         * If two operations have the same sequence number, the operation with a lower term will be overridden by the operation
-         * with a higher term. Unlike {@link #totalOperations()}, this value is updated each time after {@link #next()}) is called.
-         */
-        default int overriddenOperations() {
-            return 0;
-        }
-
         /**
          * Returns the next operation in the snapshot or <code>null</code> if we reached the end.
          */
@@ -985,11 +977,6 @@ public int skippedOperations() {
             return filteredOpsCount + delegate.skippedOperations();
         }
 
-        @Override
-        public int overriddenOperations() {
-            return delegate.overriddenOperations();
-        }
-
         @Override
         public Operation next() throws IOException {
             Translog.Operation op;

server/src/test/java/org/elasticsearch/index/mapper/MapperMergeValidatorTests.java

@@ -36,13 +36,11 @@ public void testDuplicateFieldAliasAndObject() {
         FieldAliasMapper aliasMapper = new FieldAliasMapper("path", "some.path", "field");
 
         IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () ->
-            MapperMergeValidator.validateMapperStructure("type",
+            MapperMergeValidator.validateMapperStructure(
                 singletonList(objectMapper),
                 emptyList(),
-                singletonList(aliasMapper),
-                emptyMap(),
-                new FieldTypeLookup()));
-        assertEquals("Field [some.path] is defined both as an object and a field in [type]", e.getMessage());
+                singletonList(aliasMapper)));
+        assertEquals("Field [some.path] is defined both as an object and a field.", e.getMessage());
     }
 
     public void testFieldAliasWithNestedScope() {

server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java

@@ -880,7 +880,7 @@ public void testIndexPrefixMapping() throws IOException {
 
             IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () ->
                 indexService.mapperService().merge("type", new CompressedXContent(illegalMapping), MergeReason.MAPPING_UPDATE));
-            assertThat(e.getMessage(), containsString("Field [field._index_prefix] is defined twice in [type]"));
+            assertThat(e.getMessage(), containsString("Field [field._index_prefix] is defined twice."));
 
         }
 

server/src/test/java/org/elasticsearch/index/mapper/UpdateMappingTests.java

@@ -151,14 +151,14 @@ public void testReuseMetaField() throws IOException {
             mapperService.merge("type", new CompressedXContent(Strings.toString(mapping)), MapperService.MergeReason.MAPPING_UPDATE);
             fail();
         } catch (IllegalArgumentException e) {
-            assertTrue(e.getMessage().contains("Field [_id] is defined twice in [type]"));
+            assertTrue(e.getMessage().contains("Field [_id] is defined twice."));
         }
 
         try {
             mapperService.merge("type", new CompressedXContent(Strings.toString(mapping)), MapperService.MergeReason.MAPPING_UPDATE);
             fail();
         } catch (IllegalArgumentException e) {
-            assertTrue(e.getMessage().contains("Field [_id] is defined twice in [type]"));
+            assertTrue(e.getMessage().contains("Field [_id] is defined twice."));
         }
     }
 

server/src/test/java/org/elasticsearch/index/replication/IndexLevelReplicationTests.java

@@ -584,7 +584,6 @@ public void testSeqNoCollision() throws Exception {
                 assertThat(op2.seqNo(), equalTo(op1.seqNo()));
                 assertThat(op2.primaryTerm(), greaterThan(op1.primaryTerm()));
                 assertThat("Remaining of snapshot should contain init operations", snapshot, containsOperationsInAnyOrder(initOperations));
-                assertThat(snapshot.overriddenOperations(), equalTo(0));
                 assertThat(snapshot.skippedOperations(), equalTo(1));
             }
 

server/src/test/java/org/elasticsearch/index/translog/TranslogTests.java

@@ -715,7 +715,6 @@ public void testSeqNoFilterSnapshot() throws Exception {
             Translog.Snapshot filter = new Translog.SeqNoFilterSnapshot(snapshot, between(200, 300), between(300, 400)); // out range
             assertThat(filter, SnapshotMatchers.size(0));
             assertThat(filter.totalOperations(), equalTo(snapshot.totalOperations()));
-            assertThat(filter.overriddenOperations(), equalTo(snapshot.overriddenOperations()));
             assertThat(filter.skippedOperations(), equalTo(snapshot.totalOperations()));
         }
         try (Translog.Snapshot snapshot = translog.newSnapshot()) {
@@ -726,7 +725,6 @@ public void testSeqNoFilterSnapshot() throws Exception {
             Translog.Snapshot filter = new Translog.SeqNoFilterSnapshot(snapshot, fromSeqNo, toSeqNo);
             assertThat(filter, SnapshotMatchers.containsOperationsInAnyOrder(selectedOps));
             assertThat(filter.totalOperations(), equalTo(snapshot.totalOperations()));
-            assertThat(filter.overriddenOperations(), equalTo(snapshot.overriddenOperations()));
             assertThat(filter.skippedOperations(), equalTo(snapshot.skippedOperations() + operations.size() - selectedOps.size()));
         }
     }

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/action/DeleteDataFrameTransformAction.java

@@ -49,7 +49,7 @@ public Request(String id) {
             this.id = ExceptionsHelper.requireNonNull(id, DataFrameField.ID.getPreferredName());
         }
 
-        public Request() {
+        private Request() {
         }
 
         public Request(StreamInput in) throws IOException {
@@ -113,7 +113,7 @@ protected RequestBuilder(ElasticsearchClient client, DeleteDataFrameTransformAct
     public static class Response extends BaseTasksResponse implements Writeable, ToXContentObject {
         private boolean acknowledged;
         public Response(StreamInput in) throws IOException {
-            super(Collections.emptyList(), Collections.emptyList());
+            super(in);
             readFrom(in);
         }
 

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/action/GetDataFrameTransformsAction.java

@@ -63,7 +63,7 @@ public Request(String id) {
             }
         }
 
-        public Request() {}
+        private Request() {}
 
         public Request(StreamInput in) throws IOException {
             super(in);
@@ -149,7 +149,7 @@ public Response() {
         }
 
         public Response(StreamInput in) throws IOException {
-            super(Collections.emptyList(), Collections.emptyList());
+            super(in);
             readFrom(in);
         }
 
@@ -173,6 +173,7 @@ public void writeTo(StreamOutput out) throws IOException {
         public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
             List<String> invalidTransforms = new ArrayList<>();
             builder.startObject();
+            toXContentCommon(builder, params);
             builder.field(DataFrameField.COUNT.getPreferredName(), transformConfigurations.size());
             // XContentBuilder does not support passing the params object for Iterables
             builder.field(DataFrameField.TRANSFORMS.getPreferredName());

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/action/GetDataFrameTransformsStatsAction.java

@@ -55,7 +55,7 @@ public Request(String id) {
             }
         }
 
-        public Request() {}
+        private Request() {}
 
         public Request(StreamInput in) throws IOException {
             super(in);
@@ -138,7 +138,7 @@ public Response() {
         }
 
         public Response(StreamInput in) throws IOException {
-            super(Collections.emptyList(), Collections.emptyList());
+            super(in);
             readFrom(in);
         }
 
@@ -161,6 +161,7 @@ public void writeTo(StreamOutput out) throws IOException {
         @Override
         public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
             builder.startObject();
+            toXContentCommon(builder, params);
             builder.field(DataFrameField.COUNT.getPreferredName(), transformsStateAndStats.size());
             builder.field(DataFrameField.TRANSFORMS.getPreferredName(), transformsStateAndStats);
             builder.endObject();

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/action/StartDataFrameTransformAction.java

@@ -46,7 +46,7 @@ public Request(String id) {
             this.id = ExceptionsHelper.requireNonNull(id, DataFrameField.ID.getPreferredName());
         }
 
-        public Request() {
+        private Request() {
         }
 
         public Request(StreamInput in) throws IOException {
@@ -108,7 +108,7 @@ public Response() {
         }
 
         public Response(StreamInput in) throws IOException {
-            super(Collections.emptyList(), Collections.emptyList());
+            super(in);
             readFrom(in);
         }
 
@@ -136,6 +136,7 @@ public void writeTo(StreamOutput out) throws IOException {
         @Override
         public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
             builder.startObject();
+            toXContentCommon(builder, params);
             builder.field("started", started);
             builder.endObject();
             return builder;