From 777470c7249e31531c263d26af988498f0b86339 Mon Sep 17 00:00:00 2001 From: Ryland Herrick Date: Mon, 29 Nov 2021 12:39:55 -0600 Subject: [PATCH] [RAC,Security Solution]Update alerts mappings to ECS 1.12 (#118812) * Update output directory for generative script These files were moved in #98935 but the script has become out of date. * Update ECS fieldmap with ECS 1.12 This fieldmap was missing fields from ECS 1.11+. Notable ommissions were the threat.indicator and threat.enrichments fieldsets. * Remove non-additive mappings changes These are incompatible with the current alerts framework. * Add only necessary threat fields for CTI features This could probably be pared down further, as most of these fields are not critical for CTI features. Additionally, these additions now exceed the limit of 1000 fields and is causing an error in the ruleRegistry bootstrapping. * Remove file.pe threat fields * Remove geo threat indicator fields * Remove all threat.indicator mappings These are not relevant for alerts, which will only have enrichments. * increments index mappings total fields limit to 1200 Co-authored-by: Ece Ozalp Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- .../common/assets/field_maps/ecs_field_map.ts | 420 ++++++++++++++++++ .../scripts/generate_ecs_fieldmap/index.js | 2 +- .../resource_installer.ts | 2 +- 3 files changed, 422 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/rule_registry/common/assets/field_maps/ecs_field_map.ts b/x-pack/plugins/rule_registry/common/assets/field_maps/ecs_field_map.ts index 859070bd498e3..1ea85e5a5434e 100644 --- a/x-pack/plugins/rule_registry/common/assets/field_maps/ecs_field_map.ts +++ b/x-pack/plugins/rule_registry/common/assets/field_maps/ecs_field_map.ts @@ -2550,6 +2550,426 @@ export const ecsFieldMap = { array: false, required: false, }, + 'threat.enrichments': { + type: 'nested', + array: true, + required: false, + }, + 'threat.enrichments.indicator': { + type: 'object', + array: false, + required: false, + }, + 'threat.enrichments.indicator.as.number': { + type: 'long', + array: false, + required: false, + }, + 'threat.enrichments.indicator.as.organization.name': { + type: 'keyword', + array: false, + required: false, + }, + 'threat.enrichments.indicator.confidence': { + type: 'keyword', + array: false, + required: false, + }, + 'threat.enrichments.indicator.description': { + type: 'keyword', + array: false, + required: false, + }, + 'threat.enrichments.indicator.email.address': { + type: 'keyword', + array: false, + required: false, + }, + 'threat.enrichments.indicator.file.accessed': { + type: 'date', + array: false, + required: false, + }, + 'threat.enrichments.indicator.file.attributes': { + type: 'keyword', + array: true, + required: false, + }, + 'threat.enrichments.indicator.file.code_signature.digest_algorithm': { + type: 'keyword', + array: false, + required: false, + }, + 'threat.enrichments.indicator.file.code_signature.exists': { + type: 'boolean', + array: false, + required: false, + }, + 'threat.enrichments.indicator.file.code_signature.signing_id': { + type: 'keyword', + array: false, + required: false, + }, + 'threat.enrichments.indicator.file.code_signature.status': { + type: 'keyword', + array: false, + required: false, + }, + 'threat.enrichments.indicator.file.code_signature.subject_name': { + type: 'keyword', + array: false, + required: false, + }, + 'threat.enrichments.indicator.file.code_signature.team_id': { + type: 'keyword', + array: false, + required: false, + }, + 'threat.enrichments.indicator.file.code_signature.timestamp': { + type: 'date', + array: false, + required: false, + }, + 'threat.enrichments.indicator.file.code_signature.trusted': { + type: 'boolean', + array: false, + required: false, + }, + 'threat.enrichments.indicator.file.code_signature.valid': { + type: 'boolean', + array: false, + required: false, + }, + 'threat.enrichments.indicator.file.created': { + type: 'date', + array: false, + required: false, + }, + 'threat.enrichments.indicator.file.ctime': { + type: 'date', + array: false, + required: false, + }, + 'threat.enrichments.indicator.file.device': { + type: 'keyword', + array: false, + required: false, + }, + 'threat.enrichments.indicator.file.directory': { + type: 'keyword', + array: false, + required: false, + }, + 'threat.enrichments.indicator.file.drive_letter': { + type: 'keyword', + array: false, + required: false, + }, + 'threat.enrichments.indicator.file.extension': { + type: 'keyword', + array: false, + required: false, + }, + 'threat.enrichments.indicator.file.fork_name': { + type: 'keyword', + array: false, + required: false, + }, + 'threat.enrichments.indicator.file.gid': { + type: 'keyword', + array: false, + required: false, + }, + 'threat.enrichments.indicator.file.group': { + type: 'keyword', + array: false, + required: false, + }, + 'threat.enrichments.indicator.file.hash.md5': { + type: 'keyword', + array: false, + required: false, + }, + 'threat.enrichments.indicator.file.hash.sha1': { + type: 'keyword', + array: false, + required: false, + }, + 'threat.enrichments.indicator.file.hash.sha256': { + type: 'keyword', + array: false, + required: false, + }, + 'threat.enrichments.indicator.file.hash.sha512': { + type: 'keyword', + array: false, + required: false, + }, + 'threat.enrichments.indicator.file.hash.ssdeep': { + type: 'keyword', + array: false, + required: false, + }, + 'threat.enrichments.indicator.file.inode': { + type: 'keyword', + array: false, + required: false, + }, + 'threat.enrichments.indicator.file.mime_type': { + type: 'keyword', + array: false, + required: false, + }, + 'threat.enrichments.indicator.file.mode': { + type: 'keyword', + array: false, + required: false, + }, + 'threat.enrichments.indicator.file.mtime': { + type: 'date', + array: false, + required: false, + }, + 'threat.enrichments.indicator.file.name': { + type: 'keyword', + array: false, + required: false, + }, + 'threat.enrichments.indicator.file.owner': { + type: 'keyword', + array: false, + required: false, + }, + 'threat.enrichments.indicator.file.path': { + type: 'keyword', + array: false, + required: false, + }, + 'threat.enrichments.indicator.file.size': { + type: 'long', + array: false, + required: false, + }, + 'threat.enrichments.indicator.file.target_path': { + type: 'keyword', + array: false, + required: false, + }, + 'threat.enrichments.indicator.file.type': { + type: 'keyword', + array: false, + required: false, + }, + 'threat.enrichments.indicator.file.uid': { + type: 'keyword', + array: false, + required: false, + }, + 'threat.enrichments.indicator.first_seen': { + type: 'date', + array: false, + required: false, + }, + 'threat.enrichments.indicator.ip': { + type: 'ip', + array: false, + required: false, + }, + 'threat.enrichments.indicator.last_seen': { + type: 'date', + array: false, + required: false, + }, + 'threat.enrichments.indicator.marking.tlp': { + type: 'keyword', + array: false, + required: false, + }, + 'threat.enrichments.indicator.modified_at': { + type: 'date', + array: false, + required: false, + }, + 'threat.enrichments.indicator.port': { + type: 'long', + array: false, + required: false, + }, + 'threat.enrichments.indicator.provider': { + type: 'keyword', + array: false, + required: false, + }, + 'threat.enrichments.indicator.reference': { + type: 'keyword', + array: false, + required: false, + }, + 'threat.enrichments.indicator.registry.data.bytes': { + type: 'keyword', + array: false, + required: false, + }, + 'threat.enrichments.indicator.registry.data.strings': { + type: 'wildcard', + array: true, + required: false, + }, + 'threat.enrichments.indicator.registry.data.type': { + type: 'keyword', + array: false, + required: false, + }, + 'threat.enrichments.indicator.registry.hive': { + type: 'keyword', + array: false, + required: false, + }, + 'threat.enrichments.indicator.registry.key': { + type: 'keyword', + array: false, + required: false, + }, + 'threat.enrichments.indicator.registry.path': { + type: 'keyword', + array: false, + required: false, + }, + 'threat.enrichments.indicator.registry.value': { + type: 'keyword', + array: false, + required: false, + }, + 'threat.enrichments.indicator.scanner_stats': { + type: 'long', + array: false, + required: false, + }, + 'threat.enrichments.indicator.sightings': { + type: 'long', + array: false, + required: false, + }, + 'threat.enrichments.indicator.type': { + type: 'keyword', + array: false, + required: false, + }, + 'threat.enrichments.indicator.url.domain': { + type: 'keyword', + array: false, + required: false, + }, + 'threat.enrichments.indicator.url.extension': { + type: 'keyword', + array: false, + required: false, + }, + 'threat.enrichments.indicator.url.fragment': { + type: 'keyword', + array: false, + required: false, + }, + 'threat.enrichments.indicator.url.full': { + type: 'wildcard', + array: false, + required: false, + }, + 'threat.enrichments.indicator.url.original': { + type: 'wildcard', + array: false, + required: false, + }, + 'threat.enrichments.indicator.url.password': { + type: 'keyword', + array: false, + required: false, + }, + 'threat.enrichments.indicator.url.path': { + type: 'wildcard', + array: false, + required: false, + }, + 'threat.enrichments.indicator.url.port': { + type: 'long', + array: false, + required: false, + }, + 'threat.enrichments.indicator.url.query': { + type: 'keyword', + array: false, + required: false, + }, + 'threat.enrichments.indicator.url.registered_domain': { + type: 'keyword', + array: false, + required: false, + }, + 'threat.enrichments.indicator.url.scheme': { + type: 'keyword', + array: false, + required: false, + }, + 'threat.enrichments.indicator.url.subdomain': { + type: 'keyword', + array: false, + required: false, + }, + 'threat.enrichments.indicator.url.top_level_domain': { + type: 'keyword', + array: false, + required: false, + }, + 'threat.enrichments.indicator.url.username': { + type: 'keyword', + array: false, + required: false, + }, + 'threat.enrichments.matched.atomic': { + type: 'keyword', + array: false, + required: false, + }, + 'threat.enrichments.matched.field': { + type: 'keyword', + array: false, + required: false, + }, + 'threat.enrichments.matched.id': { + type: 'keyword', + array: false, + required: false, + }, + 'threat.enrichments.matched.index': { + type: 'keyword', + array: false, + required: false, + }, + 'threat.enrichments.matched.type': { + type: 'keyword', + array: false, + required: false, + }, + 'threat.group.alias': { + type: 'keyword', + array: true, + required: false, + }, + 'threat.group.id': { + type: 'keyword', + array: false, + required: false, + }, + 'threat.group.name': { + type: 'keyword', + array: false, + required: false, + }, + 'threat.group.reference': { + type: 'keyword', + array: false, + required: false, + }, 'threat.tactic.id': { type: 'keyword', array: true, diff --git a/x-pack/plugins/rule_registry/scripts/generate_ecs_fieldmap/index.js b/x-pack/plugins/rule_registry/scripts/generate_ecs_fieldmap/index.js index 6b10ca5f837d5..bbcf651bd6d69 100644 --- a/x-pack/plugins/rule_registry/scripts/generate_ecs_fieldmap/index.js +++ b/x-pack/plugins/rule_registry/scripts/generate_ecs_fieldmap/index.js @@ -19,7 +19,7 @@ const exec = util.promisify(execCb); const ecsDir = path.resolve(__dirname, '../../../../../../ecs'); const ecsYamlFilename = path.join(ecsDir, 'generated/ecs/ecs_flat.yml'); -const outputDir = path.join(__dirname, '../../common/field_map'); +const outputDir = path.join(__dirname, '../../common/assets/field_maps'); const outputFieldMapFilename = path.join(outputDir, 'ecs_field_map.ts'); diff --git a/x-pack/plugins/rule_registry/server/rule_data_plugin_service/resource_installer.ts b/x-pack/plugins/rule_registry/server/rule_data_plugin_service/resource_installer.ts index bfdec28a50987..bbfa17c5694f1 100644 --- a/x-pack/plugins/rule_registry/server/rule_data_plugin_service/resource_installer.ts +++ b/x-pack/plugins/rule_registry/server/rule_data_plugin_service/resource_installer.ts @@ -316,7 +316,7 @@ export class ResourceInstaller { // @ts-expect-error rollover_alias: primaryNamespacedAlias, }, - 'index.mapping.total_fields.limit': 1100, + 'index.mapping.total_fields.limit': 1200, }, mappings: { dynamic: false,