diff --git a/docs/changelog/112444.yaml b/docs/changelog/112444.yaml new file mode 100644 index 0000000000000..bfa4fd693f0e0 --- /dev/null +++ b/docs/changelog/112444.yaml @@ -0,0 +1,6 @@ +pr: 112444 +summary: Full coverage of ECS by ecs@mappings when `date_detection` is disabled +area: Mapping +type: bug +issues: + - 112398 diff --git a/x-pack/plugin/core/template-resources/src/main/resources/ecs@mappings.json b/x-pack/plugin/core/template-resources/src/main/resources/ecs@mappings.json index 1951431859ffe..a373c261760e5 100644 --- a/x-pack/plugin/core/template-resources/src/main/resources/ecs@mappings.json +++ b/x-pack/plugin/core/template-resources/src/main/resources/ecs@mappings.json @@ -155,7 +155,11 @@ "ingested", "*.ingested", "*.start", - "*.end" + "*.end", + "*.indicator.first_seen", + "*.indicator.last_seen", + "*.indicator.modified_at", + "*threat.enrichments.matched.occurred" ], "unmatch_mapping_type": "object" } diff --git a/x-pack/plugin/stack/src/javaRestTest/java/org/elasticsearch/xpack/stack/EcsDynamicTemplatesIT.java b/x-pack/plugin/stack/src/javaRestTest/java/org/elasticsearch/xpack/stack/EcsDynamicTemplatesIT.java index 8bdf7b30a9997..eba239cd97e28 100644 --- a/x-pack/plugin/stack/src/javaRestTest/java/org/elasticsearch/xpack/stack/EcsDynamicTemplatesIT.java +++ b/x-pack/plugin/stack/src/javaRestTest/java/org/elasticsearch/xpack/stack/EcsDynamicTemplatesIT.java @@ -13,6 +13,7 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.network.NetworkAddress; import org.elasticsearch.common.time.DateFormatter; +import org.elasticsearch.common.time.FormatNames; import org.elasticsearch.core.Nullable; import org.elasticsearch.core.SuppressForbidden; import org.elasticsearch.test.cluster.ElasticsearchCluster; @@ -63,7 +64,7 @@ public class EcsDynamicTemplatesIT extends ESRestTestCase { private static Map ecsDynamicTemplates; private static Map> ecsFlatFieldDefinitions; - private static Map ecsFlatMultiFieldDefinitions; + private static Map> ecsFlatMultiFieldDefinitions; @BeforeClass public static void setupSuiteScopeCluster() throws Exception { @@ -142,12 +143,11 @@ private static void prepareEcsDefinitions() throws IOException { iterator.remove(); } - List> multiFields = (List>) definitions.get("multi_fields"); + List> multiFields = (List>) definitions.get("multi_fields"); if (multiFields != null) { multiFields.forEach(multiFieldsDefinitions -> { - String subfieldFlatName = Objects.requireNonNull(multiFieldsDefinitions.get("flat_name")); - String subfieldType = Objects.requireNonNull(multiFieldsDefinitions.get("type")); - ecsFlatMultiFieldDefinitions.put(subfieldFlatName, subfieldType); + String subfieldFlatName = (String) Objects.requireNonNull(multiFieldsDefinitions.get("flat_name")); + ecsFlatMultiFieldDefinitions.put(subfieldFlatName, multiFieldsDefinitions); }); } } @@ -166,6 +166,22 @@ public void testFlattenedFields() throws IOException { verifyEcsMappings(indexName); } + public void testFlattenedFieldsWithinAttributes() throws IOException { + String indexName = "test-flattened-attributes"; + createTestIndex(indexName); + Map flattenedFieldsMap = createTestDocument(true); + indexDocument(indexName, Map.of("attributes", flattenedFieldsMap)); + verifyEcsMappings(indexName, "attributes."); + } + + public void testFlattenedFieldsWithinResourceAttributes() throws IOException { + String indexName = "test-flattened-attributes"; + createTestIndex(indexName); + Map flattenedFieldsMap = createTestDocument(true); + indexDocument(indexName, Map.of("resource.attributes", flattenedFieldsMap)); + verifyEcsMappings(indexName, "resource.attributes."); + } + public void testFlattenedFieldsWithoutSubobjects() throws IOException { String indexName = "test_flattened_fields_subobjects_false"; createTestIndex(indexName, Map.of("subobjects", false)); @@ -191,7 +207,45 @@ public void testNumericMessage() throws IOException { verifyEcsMappings(indexName); } - private void assertType(String expectedType, Map actualMappings) throws IOException { + public void testDateFieldsWithDifferentFormats() throws IOException { + Map dateFieldsMap = ecsFlatFieldDefinitions.entrySet() + .stream() + .filter(entry -> "date".equals(entry.getValue().get("type"))) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + + // test with iso8601 format + String indexName = "test-date-fields-as-is8601"; + createTestIndex(indexName); + Map document = new HashMap<>(); + DateFormatter formatter = DateFormatter.forPattern(FormatNames.ISO8601.getName()); + for (String field : dateFieldsMap.keySet()) { + document.put(field, formatter.formatMillis(System.currentTimeMillis())); + } + verifyAllDateFields(indexName, document, dateFieldsMap); + + // test with milliseconds since epoch format + indexName = "test-date-fields-as-millis"; + createTestIndex(indexName); + document = new HashMap<>(); + for (String field : dateFieldsMap.keySet()) { + document.put(field, System.currentTimeMillis()); + } + verifyAllDateFields(indexName, document, dateFieldsMap); + } + + private void verifyAllDateFields(String indexName, Map document, Map dateFieldsMap) throws IOException { + indexDocument(indexName, document); + final Map rawMappings = getMappings(indexName); + final Map> flatFieldMappings = new HashMap<>(); + processRawMappingsSubtree(rawMappings, flatFieldMappings, new HashMap<>(), ""); + flatFieldMappings.forEach((fieldName, fieldMappings) -> { + if (dateFieldsMap.containsKey(fieldName)) { + assertType("date", fieldMappings); + } + }); + } + + private void assertType(String expectedType, Map actualMappings) { assertNotNull("expected to get non-null mappings for field", actualMappings); assertEquals(expectedType, actualMappings.get("type")); } @@ -297,6 +351,7 @@ private static void createTestIndex(String indexName, @Nullable Map { - return DateFormatter.forPattern("strict_date_optional_time").formatMillis(System.currentTimeMillis()); + return DateFormatter.forPattern(FormatNames.STRICT_DATE_OPTIONAL_TIME.getName()).formatMillis(System.currentTimeMillis()); } case "ip" -> { return NetworkAddress.format(randomIp(true)); @@ -395,12 +450,19 @@ private void processRawMappingsSubtree( } private void verifyEcsMappings(String indexName) throws IOException { + verifyEcsMappings(indexName, ""); + } + + private void verifyEcsMappings(String indexName, String fieldPrefix) throws IOException { final Map rawMappings = getMappings(indexName); final Map> flatFieldMappings = new HashMap<>(); final Map> flatMultiFieldsMappings = new HashMap<>(); processRawMappingsSubtree(rawMappings, flatFieldMappings, flatMultiFieldsMappings, ""); - Map> shallowFieldMapCopy = new HashMap<>(ecsFlatFieldDefinitions); + Map> shallowFieldMapCopy = ecsFlatFieldDefinitions.entrySet() + .stream() + .collect(Collectors.toMap(e -> fieldPrefix + e.getKey(), Map.Entry::getValue)); + logger.info("Testing mapping of {} ECS fields", shallowFieldMapCopy.size()); List nonEcsFields = new ArrayList<>(); Map fieldToWrongMappingType = new HashMap<>(); @@ -411,32 +473,35 @@ private void verifyEcsMappings(String indexName) throws IOException { if (expectedMappings == null) { nonEcsFields.add(fieldName); } else { - String expectedType = (String) expectedMappings.get("type"); - String actualMappingType = (String) actualMappings.get("type"); - if (actualMappingType.equals(expectedType) == false) { - fieldToWrongMappingType.put(fieldName, actualMappingType); - } - if (expectedMappings.get("index") != actualMappings.get("index")) { - wronglyIndexedFields.add(fieldName); - } - if (expectedMappings.get("doc_values") != actualMappings.get("doc_values")) { - wronglyDocValuedFields.add(fieldName); - } + compareExpectedToActualMappings( + fieldName, + actualMappings, + expectedMappings, + fieldToWrongMappingType, + wronglyIndexedFields, + wronglyDocValuedFields + ); } }); - Map shallowMultiFieldMapCopy = new HashMap<>(ecsFlatMultiFieldDefinitions); + Map> shallowMultiFieldMapCopy = ecsFlatMultiFieldDefinitions.entrySet() + .stream() + .collect(Collectors.toMap(e -> fieldPrefix + e.getKey(), Map.Entry::getValue)); logger.info("Testing mapping of {} ECS multi-fields", shallowMultiFieldMapCopy.size()); flatMultiFieldsMappings.forEach((fieldName, actualMappings) -> { - String expectedType = shallowMultiFieldMapCopy.remove(fieldName); - if (expectedType != null) { + Map expectedMultiFieldMappings = shallowMultiFieldMapCopy.remove(fieldName); + if (expectedMultiFieldMappings != null) { // not finding an entry in the expected multi-field mappings map is acceptable: our dynamic templates are required to // ensure multi-field mapping for all fields with such ECS definitions. However, the patterns in these templates may lead // to multi-field mapping for ECS fields for which such are not defined - String actualMappingType = (String) actualMappings.get("type"); - if (actualMappingType.equals(expectedType) == false) { - fieldToWrongMappingType.put(fieldName, actualMappingType); - } + compareExpectedToActualMappings( + fieldName, + actualMappings, + expectedMultiFieldMappings, + fieldToWrongMappingType, + wronglyIndexedFields, + wronglyDocValuedFields + ); } }); @@ -460,7 +525,13 @@ private void verifyEcsMappings(String indexName) throws IOException { ); }); fieldToWrongMappingType.forEach((fieldName, actualMappingType) -> { - String ecsExpectedType = (String) ecsFlatFieldDefinitions.get(fieldName).get("type"); + // if fieldPrefix is not null, we need to remove it from the field name for the ECS lookup + String ecsFieldName = fieldPrefix == null ? fieldName : fieldName.substring(fieldPrefix.length()); + Map fieldMappings = ecsFlatFieldDefinitions.get(ecsFieldName); + if (fieldMappings == null) { + fieldMappings = ecsFlatMultiFieldDefinitions.get(ecsFieldName); + } + String ecsExpectedType = (String) fieldMappings.get("type"); logger.error( "ECS field '{}' should be mapped to type '{}' but is mapped to type '{}'. Update {} accordingly.", fieldName, @@ -493,4 +564,25 @@ private void verifyEcsMappings(String indexName) throws IOException { wronglyDocValuedFields.isEmpty() ); } + + private static void compareExpectedToActualMappings( + String fieldName, + Map actualMappings, + Map expectedMappings, + Map fieldToWrongMappingType, + List wronglyIndexedFields, + List wronglyDocValuedFields + ) { + String expectedType = (String) expectedMappings.get("type"); + String actualMappingType = (String) actualMappings.get("type"); + if (actualMappingType.equals(expectedType) == false) { + fieldToWrongMappingType.put(fieldName, actualMappingType); + } + if (expectedMappings.get("index") != actualMappings.get("index")) { + wronglyIndexedFields.add(fieldName); + } + if (expectedMappings.get("doc_values") != actualMappings.get("doc_values")) { + wronglyDocValuedFields.add(fieldName); + } + } } diff --git a/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java b/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java index 6a9936f4f27d3..da51a379da20b 100644 --- a/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java +++ b/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java @@ -48,7 +48,7 @@ public class StackTemplateRegistry extends IndexTemplateRegistry { // The stack template registry version. This number must be incremented when we make changes // to built-in templates. - public static final int REGISTRY_VERSION = 12; + public static final int REGISTRY_VERSION = 14; public static final String TEMPLATE_VERSION_VARIABLE = "xpack.stack.template.version"; public static final Setting STACK_TEMPLATES_ENABLED = Setting.boolSetting(