Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Make ecs@mappings more exhaustive #113337

Closed
wants to merge 4 commits into from
Closed

Make ecs@mappings more exhaustive #113337

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
bd78549
Make ecs@mapping more exhaustive to enforce correct mapping for numer…
eyalkoren Sep 21, 2024
728cedc
Update docs/changelog/113337.yaml
eyalkoren Sep 21, 2024
5068434
Fix forbidden APIs
eyalkoren Sep 21, 2024
f7f4988
Merge remote-tracking branch 'eyalkoren/ecs-mappings-full-coverage' i…
eyalkoren Sep 21, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions docs/changelog/113337.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
pr: 113337
summary: Make `ecs@mappings` more exhaustive
area: Data streams
type: bug
issues:
- 113124
81 changes: 81 additions & 0 deletions x-pack/plugin/core/template-resources/src/main/resources/ecs@mappings.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,87 @@
"match_mapping_type": "string"
}
},
{
"ecs_explicit_keywords": {
"mapping": {
"ignore_above": 1024,
"type": "keyword"
},
"path_match": [
"*registry.data.bytes"
],
"match_mapping_type": "string"
}
},
Comment on lines +63 to +74
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This allows me to add *.bytes to the long-mapping dynamic template. Otherwise I would have to add:

"*.ingress.bytes",
"*.egress.bytes",
"*.write.bytes",
"*.read.bytes",
"*.body.bytes",
"*server.bytes",
"*client.bytes",
"*http.request.bytes",
"*http.response.bytes",
"*network.bytes",
"*destination.bytes",
"*source.bytes",

{
"ecs_boolean": {
"mapping": {
"type": "boolean"
},
"path_match": [
"*.code_signature.exists",
"*.code_signature.trusted",
"*.code_signature.valid",
"*.go_stripped",
"*.interactive",
"*.same_as_process",
"*faas.coldstart",
"*volume.removable",
"*tls.established",
"*container.security_context.privileged",
"*tls.resumed",
"*process.io.max_bytes_per_process_exceeded",
"*volume.writable"
],
"match_mapping_type": ["boolean", "string"]
}
},
{
"ecs_long": {
"mapping": {
"type": "long"
},
"path_match": [
"*entropy",
"*.size",
"*_size",
"*.as.number",
"*.bytes",
"*bytes_skipped*",
"*.pid",
"*.vpid",
"*.port",
"*.packets",
"*.char_device.major",
"*.char_device.minor",
"*.chi2",
"*.args_count",
"*.virtual_address",
"*.pgid",
"*.thread.id",
"*.header.entrypoint",
"*.uptime",
"*.scanner_stats",
"*.indicator.sightings",
"*process.io.total_bytes_captured",
"*process.tty.columns",
"*event.severity",
"*log.syslog.severity.code",
"*process.parent.exit_code",
"*http.response.status_code",
"*log.syslog.facility.code",
"*process.exit_code",
"*log.origin.file.line",
"*process.tty.rows",
"*event.duration",
"*event.sequence",
"*dns.answers.ttl",
"*log.syslog.priority"

],
"match_mapping_type": ["long", "string"]
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think that automatically mapping double values to long is not a good idea in general, but it's something we may consider for specific fields. For this initial version I decided not to include double in the match_mapping_type.

}
},
{
"ecs_wildcard": {
"mapping": {
Expand Down
126 changes: 101 additions & 25 deletions ...ugin/stack/src/javaRestTest/java/org/elasticsearch/xpack/stack/EcsDynamicTemplatesIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -208,10 +208,7 @@ public void testNumericMessage() throws IOException {
}

public void testDateFieldsWithDifferentFormats() throws IOException {
Map<String, Object> dateFieldsMap = ecsFlatFieldDefinitions.entrySet()
.stream()
.filter(entry -> "date".equals(entry.getValue().get("type")))
.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
Map<String, Object> dateFieldsMap = getFieldMappingsByType("date");

// test with iso8601 format
String indexName = "test-date-fields-as-is8601";
Expand All @@ -221,7 +218,7 @@ public void testDateFieldsWithDifferentFormats() throws IOException {
for (String field : dateFieldsMap.keySet()) {
document.put(field, formatter.formatMillis(System.currentTimeMillis()));
}
verifyAllDateFields(indexName, document, dateFieldsMap);
assertFieldsType("date", indexName, document, dateFieldsMap);

// test with milliseconds since epoch format
indexName = "test-date-fields-as-millis";
Expand All @@ -230,22 +227,91 @@ public void testDateFieldsWithDifferentFormats() throws IOException {
for (String field : dateFieldsMap.keySet()) {
document.put(field, System.currentTimeMillis());
}
verifyAllDateFields(indexName, document, dateFieldsMap);
assertFieldsType("date", indexName, document, dateFieldsMap);
}

public void testBooleanFieldsAsString() throws IOException {
Map<String, Object> dateFieldsMap = getFieldMappingsByType("boolean");
String indexName = "test-boolean-fields-as-string";
createTestIndex(indexName);
Map<String, Object> document = new HashMap<>();
for (String field : dateFieldsMap.keySet()) {
document.put(field, Boolean.toString(randomBoolean()));
}
assertFieldsType("boolean", indexName, document, dateFieldsMap);
}

public void testLongFieldsAsString() throws IOException {
Map<String, Object> dateFieldsMap = getFieldMappingsByType("long");
String indexName = "test-long-fields-as-string";
createTestIndex(indexName);
Map<String, Object> document = new HashMap<>();
for (String field : dateFieldsMap.keySet()) {
document.put(field, Long.toString(randomLong()));
}
assertFieldsType("long", indexName, document, dateFieldsMap);
}

private void verifyAllDateFields(String indexName, Map<String, Object> document, Map<String, Object> dateFieldsMap) throws IOException {
public void testFloatFieldsAsString() throws IOException {
Map<String, Object> dateFieldsMap = getFieldMappingsByType("float");
String indexName = "test-flot-fields-as-string";
createTestIndex(indexName);
Map<String, Object> document = new HashMap<>();
for (String field : dateFieldsMap.keySet()) {
document.put(field, Float.toString(randomFloat()));
}
assertFieldsType("float", indexName, document, dateFieldsMap);
}

private static Map<String, Object> getFieldMappingsByType(String type) {
Map<String, Object> ecsBaseFields = ecsFlatFieldDefinitions.entrySet()
.stream()
.filter(entry -> type.equals(entry.getValue().get("type")))
.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));

Map<String, Object> completeMapIncludingAttributes = new HashMap<>(ecsBaseFields);
ecsBaseFields.forEach((key, value) -> {
String newKey = "attributes." + key;
completeMapIncludingAttributes.put(newKey, value);
});

return completeMapIncludingAttributes;
}

private void assertFieldsType(String expectedType, String indexName, Map<String, Object> document, Map<String, Object> dateFieldsMap)
throws IOException {
indexDocument(indexName, document);
final Map<String, Object> rawMappings = getMappings(indexName);
final Map<String, Map<String, Object>> flatFieldMappings = new HashMap<>();
processRawMappingsSubtree(rawMappings, flatFieldMappings, new HashMap<>(), "");
final Map<String, String> wronglyTypedFields = new HashMap<>();
flatFieldMappings.forEach((fieldName, fieldMappings) -> {
if (dateFieldsMap.containsKey(fieldName)) {
assertType("date", fieldMappings);
String actualType = (String) fieldMappings.get("type");
if (expectedType.equals(actualType) == false) {
wronglyTypedFields.put(fieldName, actualType);
}
}
});
wronglyTypedFields.forEach((fieldName, actualType) -> {
Object ingestedValue = document.get(fieldName);
logger.error(
"ECS field '{}' is expected to be mapped to '{}', but when provided with a value of type '{}' it gets mapped to '{}'. "
+ "Fix {} accordingly.",
fieldName,
expectedType,
ingestedValue.getClass().getSimpleName().toLowerCase(Locale.ROOT),
actualType,
ECS_DYNAMIC_TEMPLATES_FILE
);
});
assertTrue(
"At least one field was not mapped with correctly according to its ECS definitions, see details above",
wronglyTypedFields.isEmpty()
);
}

private void assertType(String expectedType, Map<String, Object> actualMappings) {
private void assertFieldType(String expectedType, Map<String, Object> actualMappings) {
assertNotNull("expected to get non-null mappings for field", actualMappings);
assertEquals(expectedType, actualMappings.get("type"));
}
Expand All @@ -266,11 +332,11 @@ public void testUsage() throws IOException {
final Map<String, Object> rawMappings = getMappings(indexName);
final Map<String, Map<String, Object>> flatFieldMappings = new HashMap<>();
processRawMappingsSubtree(rawMappings, flatFieldMappings, new HashMap<>(), "");
assertType("scaled_float", flatFieldMappings.get("host.cpu.usage"));
assertType("scaled_float", flatFieldMappings.get("string.usage"));
assertType("long", flatFieldMappings.get("usage"));
assertType("long", flatFieldMappings.get("root.usage.long"));
assertType("float", flatFieldMappings.get("root.usage.float"));
assertFieldType("scaled_float", flatFieldMappings.get("host.cpu.usage"));
assertFieldType("scaled_float", flatFieldMappings.get("string.usage"));
assertFieldType("long", flatFieldMappings.get("usage"));
assertFieldType("long", flatFieldMappings.get("root.usage.long"));
assertFieldType("float", flatFieldMappings.get("root.usage.float"));
}

public void testOnlyMatchLeafFields() throws IOException {
Expand All @@ -291,14 +357,14 @@ public void testOnlyMatchLeafFields() throws IOException {
final Map<String, Object> rawMappings = getMappings(indexName);
final Map<String, Map<String, Object>> flatFieldMappings = new HashMap<>();
processRawMappingsSubtree(rawMappings, flatFieldMappings, new HashMap<>(), "");
assertType("long", flatFieldMappings.get("foo.message.bar"));
assertType("long", flatFieldMappings.get("foo.url.path.bar"));
assertType("long", flatFieldMappings.get("foo.url.full.bar"));
assertType("long", flatFieldMappings.get("foo.stack_trace.bar"));
assertType("long", flatFieldMappings.get("foo.user_agent.original.bar"));
assertType("long", flatFieldMappings.get("foo.created.bar"));
assertType("float", flatFieldMappings.get("foo._score.bar"));
assertType("long", flatFieldMappings.get("foo.structured_data"));
assertFieldType("long", flatFieldMappings.get("foo.message.bar"));
assertFieldType("long", flatFieldMappings.get("foo.url.path.bar"));
assertFieldType("long", flatFieldMappings.get("foo.url.full.bar"));
assertFieldType("long", flatFieldMappings.get("foo.stack_trace.bar"));
assertFieldType("long", flatFieldMappings.get("foo.user_agent.original.bar"));
assertFieldType("long", flatFieldMappings.get("foo.created.bar"));
assertFieldType("float", flatFieldMappings.get("foo._score.bar"));
assertFieldType("long", flatFieldMappings.get("foo.structured_data"));
}

private static void indexDocument(String indexName, Map<String, Object> flattenedFieldsMap) throws IOException {
Expand Down Expand Up @@ -376,9 +442,6 @@ private Object generateTestValue(String type) {
case "long" -> {
return randomLong();
}
case "int" -> {
return randomInt();
}
case "float", "scaled_float" -> {
return randomFloat();
}
Expand All @@ -401,6 +464,13 @@ private Object generateTestValue(String type) {
// creating multiple subfields
return Map.of("subfield1", randomAlphaOfLength(20), "subfield2", randomAlphaOfLength(20));
}

// TODO - REMOVE ONCE THE ERROR INTRODUCED WITH https://github.com/elastic/ecs/pull/2370 IS FIXED
case "string" -> {
logger.error("Field type 'string' is not supported by ECS and should be replaced with 'keyword' or 'text'");
return randomAlphaOfLength(20);
}

}
throw new IllegalArgumentException("Unknown field type: " + type);
}
Expand Down Expand Up @@ -576,6 +646,12 @@ private static void compareExpectedToActualMappings(
String expectedType = (String) expectedMappings.get("type");
String actualMappingType = (String) actualMappings.get("type");
if (actualMappingType.equals(expectedType) == false) {

// TODO - REMOVE ONCE THE ERROR INTRODUCED WITH https://github.com/elastic/ecs/pull/2370 IS FIXED
if (expectedType.equals("string") && actualMappingType.equals("keyword")) {
return;
}

fieldToWrongMappingType.put(fieldName, actualMappingType);
}
if (expectedMappings.get("index") != actualMappings.get("index")) {
Expand Down