From 419fb711db67f2cede9c8219fd88991f134ed0b0 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Mon, 16 Jan 2023 12:18:31 +0100 Subject: [PATCH] Convert to yaml ecs mappings static file --- internal/builder/_static/ecs_mappings.json | 889 --------------------- internal/builder/_static/ecs_mappings.yaml | 453 +++++++++++ internal/builder/dynamic_mappings.go | 2 +- 3 files changed, 454 insertions(+), 890 deletions(-) delete mode 100644 internal/builder/_static/ecs_mappings.json create mode 100644 internal/builder/_static/ecs_mappings.yaml diff --git a/internal/builder/_static/ecs_mappings.json b/internal/builder/_static/ecs_mappings.json deleted file mode 100644 index f2f6f7660..000000000 --- a/internal/builder/_static/ecs_mappings.json +++ /dev/null @@ -1,889 +0,0 @@ -{ - "mappings": { - "dynamic_templates": [ - { - "ecs_timestamp": { - "path_match": "@timestamp", - "mapping": { - "type": "date", - "ignore_malformed": false - } - } - }, - { - "data_stream_to_constant": { - "path_match": "data_stream.*", - "mapping": { - "type": "constant_keyword" - } - } - }, - { - "resolved_ip_to_ip": { - "match": "resolved_ip", - "mapping": { - "type": "ip" - } - } - }, - { - "forwarded_ip_to_ip": { - "match_mapping_type": "string", - "match": "forwarded_ip", - "mapping": { - "type": "ip" - } - } - }, - { - "ip_to_ip": { - "match_mapping_type": "string", - "match": "ip", - "mapping": { - "type": "ip" - } - } - }, - { - "port_to_long": { - "match": "port", - "mapping": { - "type": "long" - } - } - }, - { - "thread_id_to_long": { - "path_match": "*.thread.id", - "mapping": { - "type": "long" - } - } - }, - { - "status_code_to_long": { - "match": "status_code", - "mapping": { - "type": "long" - } - } - }, - { - "line_to_long": { - "path_match": "*.file.line", - "mapping": { - "type": "long" - } - } - }, - { - "priority_to_long": { - "path_match": "log.syslog.priority", - "mapping": { - "type": "long" - } - } - }, - { - "code_to_long": { - "path_match": "*.facility.code", - "mapping": { - "type": "long" - } - } - }, - { - "code_to_long": { - "path_match": "*.severity.code", - "mapping": { - "type": "long" - } - } - }, - { - "bytes_to_long": { - "match": "bytes", - "path_unmatch": "*.data.bytes", - "mapping": { - "type": "long" - } - } - }, - { - "packets_to_long": { - "match": "packets", - "mapping": { - "type": "long" - } - } - }, - { - "public_key_exponent_to_long": { - "match": "public_key_exponent", - "mapping": { - "type": "long" - } - } - }, - { - "severity_to_long": { - "path_match": "event.severity", - "mapping": { - "type": "long" - } - } - }, - { - "duration_to_long": { - "path_match": "event.duration", - "mapping": { - "type": "long" - } - } - }, - { - "pid_to_long": { - "match": "pid", - "mapping": { - "type": "long" - } - } - }, - { - "uptime_to_long": { - "match": "uptime", - "mapping": { - "type": "long" - } - } - }, - { - "sequence_to_long": { - "match": "sequence", - "mapping": { - "type": "long" - } - } - }, - { - "entropy_to_long": { - "match": "*entropy", - "mapping": { - "type": "long" - } - } - }, - { - "size_to_long": { - "match": "*size", - "mapping": { - "type": "long" - } - } - }, - { - "entrypoint_to_long": { - "match": "entrypoint", - "mapping": { - "type": "long" - } - } - }, - { - "ttl_to_long": { - "match": "ttl", - "mapping": { - "type": "long" - } - } - }, - { - "major_to_long": { - "match": "major", - "mapping": { - "type": "long" - } - } - }, - { - "minor_to_long": { - "match": "minor", - "mapping": { - "type": "long" - } - } - }, - { - "as_number_to_long": { - "path_match": "*.as.number", - "mapping": { - "type": "long" - } - } - }, - { - "pgid_to_long": { - "match": "pgid", - "mapping": { - "type": "long" - } - } - }, - { - "exit_code_to_long": { - "match": "exit_code", - "mapping": { - "type": "long" - } - } - }, - { - "chi_to_long": { - "match": "chi2", - "mapping": { - "type": "long" - } - } - }, - { - "args_count_to_long": { - "match": "args_count", - "mapping": { - "type": "long" - } - } - }, - { - "virtual_address_to_long": { - "match": "virtual_address", - "mapping": { - "type": "long" - } - } - }, - { - "io_text_to_wildcard": { - "path_match": "*.io.text", - "mapping": { - "type": "wildcard" - } - } - }, - { - "strings_to_wildcard": { - "path_match": "registry.data.strings", - "mapping": { - "type": "wildcard" - } - } - }, - { - "path_to_wildcard": { - "path_match": "*url.path", - "mapping": { - "type": "wildcard" - } - } - }, - { - "message_id_to_wildcard": { - "match": "message_id", - "mapping": { - "type": "wildcard" - } - } - }, - { - "command_line_to_multifield": { - "match": "command_line", - "mapping": { - "type": "wildcard", - "fields": { - "text": { - "type": "match_only_text" - } - } - } - } - }, - { - "error_stack_trace_to_multifield": { - "match": "stack_trace", - "mapping": { - "type": "wildcard", - "fields": { - "text": { - "type": "match_only_text" - } - } - } - } - }, - { - "http_content_to_multifield": { - "path_match": "*.body.content", - "mapping": { - "type": "wildcard", - "fields": { - "text": { - "type": "match_only_text" - } - } - } - } - }, - { - "url_full_to_multifield": { - "path_match": "*.url.full", - "mapping": { - "type": "wildcard", - "fields": { - "text": { - "type": "match_only_text" - } - } - } - } - }, - { - "url_original_to_multifield": { - "path_match": "*.url.original", - "mapping": { - "type": "wildcard", - "fields": { - "text": { - "type": "match_only_text" - } - } - } - } - }, - { - "user_agent_original_to_multifield": { - "path_match": "user_agent.original", - "mapping": { - "type": "wildcard", - "fields": { - "text": { - "type": "match_only_text" - } - } - } - } - }, - { - "error_message_to_match_only": { - "path_match": "error.message", - "mapping": { - "type": "match_only_text" - } - } - }, - { - "message_match_only_text": { - "path_match": "message", - "mapping": { - "type": "match_only_text" - } - } - }, - { - "agent_name_to_keyword": { - "path_match": "agent.name", - "mapping": { - "type": "keyword" - } - } - }, - { - "service_name_to_keyword": { - "path_match": "*.service.name", - "mapping": { - "type": "keyword" - } - } - }, - { - "sections_name_to_keyword": { - "path_match": "*.sections.name", - "mapping": { - "type": "keyword" - } - } - }, - { - "resource_name_to_keyword": { - "path_match": "*.resource.name", - "mapping": { - "type": "keyword" - } - } - }, - { - "observer_name_to_keyword": { - "path_match": "observer.name", - "mapping": { - "type": "keyword" - } - } - }, - { - "question_name_to_keyword": { - "path_match": "*.question.name", - "mapping": { - "type": "keyword" - } - } - }, - { - "group_name_to_keyword": { - "path_match": "*.group.name", - "mapping": { - "type": "keyword" - } - } - }, - { - "geo_name_to_keyword": { - "path_match": "*.geo.name", - "mapping": { - "type": "keyword" - } - } - }, - { - "host_name_to_keyword": { - "path_match": "host.name", - "mapping": { - "type": "keyword" - } - } - }, - { - "severity_name_to_keyword": { - "path_match": "*.severity.name", - "mapping": { - "type": "keyword" - } - } - }, - { - "title_to_multifield": { - "match": "title", - "mapping": { - "type": "keyword", - "fields": { - "text": { - "type": "match_only_text" - } - } - } - } - }, - { - "executable_to_multifield": { - "match": "executable", - "mapping": { - "type": "keyword", - "fields": { - "text": { - "type": "match_only_text" - } - } - } - } - }, - { - "file_path_to_multifield": { - "path_match": "*.file.path", - "mapping": { - "type": "keyword", - "fields": { - "text": { - "type": "match_only_text" - } - } - } - } - }, - { - "file_target_path_to_multifield": { - "path_match": "*.file.target_path", - "mapping": { - "type": "keyword", - "fields": { - "text": { - "type": "match_only_text" - } - } - } - } - }, - { - "name_to_multifield": { - "match": "name", - "mapping": { - "type": "keyword", - "fields": { - "text": { - "type": "match_only_text" - } - } - } - } - }, - { - "full_name_to_multifield": { - "match": "full_name", - "mapping": { - "type": "keyword", - "fields": { - "text": { - "type": "match_only_text" - } - } - } - } - }, - { - "os_full_to_multifield": { - "path_match": "*.os.full", - "mapping": { - "type": "keyword", - "fields": { - "text": { - "type": "match_only_text" - } - } - } - } - }, - { - "working_directory_to_multifield": { - "match": "working_directory", - "mapping": { - "type": "keyword", - "fields": { - "text": { - "type": "match_only_text" - } - } - } - } - }, - { - "timestamp_to_date": { - "match": "timestamp", - "mapping": { - "type": "date" - } - } - }, - { - "delivery_timestamp_to_date": { - "match": "delivery_timestamp", - "mapping": { - "type": "date" - } - } - }, - { - "not_after_to_date": { - "match": "not_after", - "mapping": { - "type": "date" - } - } - }, - { - "not_before_to_date": { - "match": "not_before", - "mapping": { - "type": "date" - } - } - }, - { - "accessed_to_date": { - "match": "accessed", - "mapping": { - "type": "date" - } - } - }, - { - "origination_timestamp_to_date": { - "match": "origination_timestamp", - "mapping": { - "type": "date" - } - } - }, - { - "created_to_date": { - "match": "created", - "mapping": { - "type": "date" - } - } - }, - { - "installed_to_date": { - "match": "installed", - "mapping": { - "type": "date" - } - } - }, - { - "creation_date_to_date": { - "match": "creation_date", - "mapping": { - "type": "date" - } - } - }, - { - "ctime_to_date": { - "match": "ctime", - "mapping": { - "type": "date" - } - } - }, - { - "mtime_to_date": { - "match": "mtime", - "mapping": { - "type": "date" - } - } - }, - { - "ingested_to_date": { - "match": "ingested", - "mapping": { - "type": "date" - } - } - }, - { - "start_to_date": { - "match": "start", - "mapping": { - "type": "date" - } - } - }, - { - "end_to_date": { - "match": "end", - "mapping": { - "type": "date" - } - } - }, - { - "score_base_to_float": { - "path_match": "*.score.base", - "mapping": { - "type": "float" - } - } - }, - { - "score_temporal_to_float": { - "path_match": "*.score.temporal", - "mapping": { - "type": "float" - } - } - }, - { - "score_to_float": { - "match": "*_score", - "mapping": { - "type": "float" - } - } - }, - { - "score_norm_to_float": { - "match": "*_score_norm", - "mapping": { - "type": "float" - } - } - }, - { - "usage_to_float": { - "match": "usage", - "mapping": { - "type": "scaled_float", - "scaling_factor": 1000 - } - } - }, - { - "location_to_geo_point": { - "match": "location", - "mapping": { - "type": "geo_point" - } - } - }, - { - "same_as_process_to_boolean": { - "match": "same_as_process", - "mapping": { - "type": "boolean" - } - } - }, - { - "established_to_boolean": { - "match": "established", - "mapping": { - "type": "boolean" - } - } - }, - { - "resumed_to_boolean": { - "match": "resumed", - "mapping": { - "type": "boolean" - } - } - }, - { - "max_bytes_per_process_exceeded_to_boolean": { - "match": "max_bytes_per_process_exceeded", - "mapping": { - "type": "boolean" - } - } - }, - { - "interactive_to_boolean": { - "match": "interactive", - "mapping": { - "type": "boolean" - } - } - }, - { - "exists_to_boolean": { - "match": "exists", - "mapping": { - "type": "boolean" - } - } - }, - { - "trusted_to_boolean": { - "match": "trusted", - "mapping": { - "type": "boolean" - } - } - }, - { - "valid_to_boolean": { - "match": "valid", - "mapping": { - "type": "boolean" - } - } - }, - { - "go_stripped_to_boolean": { - "match": "go_stripped", - "mapping": { - "type": "boolean" - } - } - }, - { - "coldstart_to_boolean": { - "match": "coldstart", - "mapping": { - "type": "boolean" - } - } - }, - { - "exports_to_flattened": { - "match": "exports", - "mapping": { - "type": "flattened" - } - } - }, - { - "structured_data_to_flattened": { - "match": "structured_data", - "mapping": { - "type": "flattened" - } - } - }, - { - "imports_to_flattened": { - "match": "*imports", - "mapping": { - "type": "flattened" - } - } - }, - { - "attachments_to_nested": { - "match": "attachments", - "mapping": { - "type": "nested" - } - } - }, - { - "segments_to_nested": { - "match": "segments", - "mapping": { - "type": "nested" - } - } - }, - { - "elf_sections_to_nested": { - "path_match": "*.elf.sections", - "mapping": { - "type": "nested" - } - } - }, - { - "pe_sections_to_nested": { - "path_match": "*.pe.sections", - "mapping": { - "type": "nested" - } - } - }, - { - "macho_sections_to_nested": { - "path_match": "*.macho.sections", - "mapping": { - "type": "nested" - } - } - }, - { - "trigger_to_nested": { - "match": "trigger", - "mapping": { - "type": "nested" - } - } - } - ] - } -} diff --git a/internal/builder/_static/ecs_mappings.yaml b/internal/builder/_static/ecs_mappings.yaml new file mode 100644 index 000000000..7417d07a0 --- /dev/null +++ b/internal/builder/_static/ecs_mappings.yaml @@ -0,0 +1,453 @@ +mappings: + dynamic_templates: + - ecs_timestamp: + mapping: + ignore_malformed: false + type: date + path_match: '@timestamp' + - data_stream_to_constant: + mapping: + type: constant_keyword + path_match: data_stream.* + - resolved_ip_to_ip: + mapping: + type: ip + match: resolved_ip + - forwarded_ip_to_ip: + mapping: + type: ip + match: forwarded_ip + match_mapping_type: string + - ip_to_ip: + mapping: + type: ip + match: ip + match_mapping_type: string + - port_to_long: + mapping: + type: long + match: port + - thread_id_to_long: + mapping: + type: long + path_match: '*.thread.id' + - status_code_to_long: + mapping: + type: long + match: status_code + - line_to_long: + mapping: + type: long + path_match: '*.file.line' + - priority_to_long: + mapping: + type: long + path_match: log.syslog.priority + - code_to_long: + mapping: + type: long + path_match: '*.facility.code' + - code_to_long: + mapping: + type: long + path_match: '*.severity.code' + - bytes_to_long: + mapping: + type: long + match: bytes + path_unmatch: '*.data.bytes' + - packets_to_long: + mapping: + type: long + match: packets + - public_key_exponent_to_long: + mapping: + type: long + match: public_key_exponent + - severity_to_long: + mapping: + type: long + path_match: event.severity + - duration_to_long: + mapping: + type: long + path_match: event.duration + - pid_to_long: + mapping: + type: long + match: pid + - uptime_to_long: + mapping: + type: long + match: uptime + - sequence_to_long: + mapping: + type: long + match: sequence + - entropy_to_long: + mapping: + type: long + match: '*entropy' + - size_to_long: + mapping: + type: long + match: '*size' + - entrypoint_to_long: + mapping: + type: long + match: entrypoint + - ttl_to_long: + mapping: + type: long + match: ttl + - major_to_long: + mapping: + type: long + match: major + - minor_to_long: + mapping: + type: long + match: minor + - as_number_to_long: + mapping: + type: long + path_match: '*.as.number' + - pgid_to_long: + mapping: + type: long + match: pgid + - exit_code_to_long: + mapping: + type: long + match: exit_code + - chi_to_long: + mapping: + type: long + match: chi2 + - args_count_to_long: + mapping: + type: long + match: args_count + - virtual_address_to_long: + mapping: + type: long + match: virtual_address + - io_text_to_wildcard: + mapping: + type: wildcard + path_match: '*.io.text' + - strings_to_wildcard: + mapping: + type: wildcard + path_match: registry.data.strings + - path_to_wildcard: + mapping: + type: wildcard + path_match: '*url.path' + - message_id_to_wildcard: + mapping: + type: wildcard + match: message_id + - command_line_to_multifield: + mapping: + fields: + text: + type: match_only_text + type: wildcard + match: command_line + - error_stack_trace_to_multifield: + mapping: + fields: + text: + type: match_only_text + type: wildcard + match: stack_trace + - http_content_to_multifield: + mapping: + fields: + text: + type: match_only_text + type: wildcard + path_match: '*.body.content' + - url_full_to_multifield: + mapping: + fields: + text: + type: match_only_text + type: wildcard + path_match: '*.url.full' + - url_original_to_multifield: + mapping: + fields: + text: + type: match_only_text + type: wildcard + path_match: '*.url.original' + - user_agent_original_to_multifield: + mapping: + fields: + text: + type: match_only_text + type: wildcard + path_match: user_agent.original + - error_message_to_match_only: + mapping: + type: match_only_text + path_match: error.message + - message_match_only_text: + mapping: + type: match_only_text + path_match: message + - agent_name_to_keyword: + mapping: + type: keyword + path_match: agent.name + - service_name_to_keyword: + mapping: + type: keyword + path_match: '*.service.name' + - sections_name_to_keyword: + mapping: + type: keyword + path_match: '*.sections.name' + - resource_name_to_keyword: + mapping: + type: keyword + path_match: '*.resource.name' + - observer_name_to_keyword: + mapping: + type: keyword + path_match: observer.name + - question_name_to_keyword: + mapping: + type: keyword + path_match: '*.question.name' + - group_name_to_keyword: + mapping: + type: keyword + path_match: '*.group.name' + - geo_name_to_keyword: + mapping: + type: keyword + path_match: '*.geo.name' + - host_name_to_keyword: + mapping: + type: keyword + path_match: host.name + - severity_name_to_keyword: + mapping: + type: keyword + path_match: '*.severity.name' + - title_to_multifield: + mapping: + fields: + text: + type: match_only_text + type: keyword + match: title + - executable_to_multifield: + mapping: + fields: + text: + type: match_only_text + type: keyword + match: executable + - file_path_to_multifield: + mapping: + fields: + text: + type: match_only_text + type: keyword + path_match: '*.file.path' + - file_target_path_to_multifield: + mapping: + fields: + text: + type: match_only_text + type: keyword + path_match: '*.file.target_path' + - name_to_multifield: + mapping: + fields: + text: + type: match_only_text + type: keyword + match: name + - full_name_to_multifield: + mapping: + fields: + text: + type: match_only_text + type: keyword + match: full_name + - os_full_to_multifield: + mapping: + fields: + text: + type: match_only_text + type: keyword + path_match: '*.os.full' + - working_directory_to_multifield: + mapping: + fields: + text: + type: match_only_text + type: keyword + match: working_directory + - timestamp_to_date: + mapping: + type: date + match: timestamp + - delivery_timestamp_to_date: + mapping: + type: date + match: delivery_timestamp + - not_after_to_date: + mapping: + type: date + match: not_after + - not_before_to_date: + mapping: + type: date + match: not_before + - accessed_to_date: + mapping: + type: date + match: accessed + - origination_timestamp_to_date: + mapping: + type: date + match: origination_timestamp + - created_to_date: + mapping: + type: date + match: created + - installed_to_date: + mapping: + type: date + match: installed + - creation_date_to_date: + mapping: + type: date + match: creation_date + - ctime_to_date: + mapping: + type: date + match: ctime + - mtime_to_date: + mapping: + type: date + match: mtime + - ingested_to_date: + mapping: + type: date + match: ingested + - start_to_date: + mapping: + type: date + match: start + - end_to_date: + mapping: + type: date + match: end + - score_base_to_float: + mapping: + type: float + path_match: '*.score.base' + - score_temporal_to_float: + mapping: + type: float + path_match: '*.score.temporal' + - score_to_float: + mapping: + type: float + match: '*_score' + - score_norm_to_float: + mapping: + type: float + match: '*_score_norm' + - usage_to_float: + mapping: + scaling_factor: 1000 + type: scaled_float + match: usage + - location_to_geo_point: + mapping: + type: geo_point + match: location + - same_as_process_to_boolean: + mapping: + type: boolean + match: same_as_process + - established_to_boolean: + mapping: + type: boolean + match: established + - resumed_to_boolean: + mapping: + type: boolean + match: resumed + - max_bytes_per_process_exceeded_to_boolean: + mapping: + type: boolean + match: max_bytes_per_process_exceeded + - interactive_to_boolean: + mapping: + type: boolean + match: interactive + - exists_to_boolean: + mapping: + type: boolean + match: exists + - trusted_to_boolean: + mapping: + type: boolean + match: trusted + - valid_to_boolean: + mapping: + type: boolean + match: valid + - go_stripped_to_boolean: + mapping: + type: boolean + match: go_stripped + - coldstart_to_boolean: + mapping: + type: boolean + match: coldstart + - exports_to_flattened: + mapping: + type: flattened + match: exports + - structured_data_to_flattened: + mapping: + type: flattened + match: structured_data + - imports_to_flattened: + mapping: + type: flattened + match: '*imports' + - attachments_to_nested: + mapping: + type: nested + match: attachments + - segments_to_nested: + mapping: + type: nested + match: segments + - elf_sections_to_nested: + mapping: + type: nested + path_match: '*.elf.sections' + - pe_sections_to_nested: + mapping: + type: nested + path_match: '*.pe.sections' + - macho_sections_to_nested: + mapping: + type: nested + path_match: '*.macho.sections' + - trigger_to_nested: + mapping: + type: nested + match: trigger diff --git a/internal/builder/dynamic_mappings.go b/internal/builder/dynamic_mappings.go index 2cb85f0a1..ead4a1093 100644 --- a/internal/builder/dynamic_mappings.go +++ b/internal/builder/dynamic_mappings.go @@ -21,7 +21,7 @@ import ( "github.com/elastic/elastic-package/internal/packages/buildmanifest" ) -//go:embed _static/ecs_mappings.json +//go:embed _static/ecs_mappings.yaml var staticEcsMappings string const prefixMapping = "_embedded_ecs"