diff --git a/ecs-mappings.csv b/ecs-mappings.csv index ae03ff3..528eac3 100644 --- a/ecs-mappings.csv +++ b/ecs-mappings.csv @@ -1,4 +1,4 @@ -revision,Meta Class,Meta Concept,Meta Description,envisionName,nwName,format,flags,nullTokens,failureKey,failureMapping,ecsName,Proposed,nbOccurrences,shortMeta,name_noDot, old ecsType +revision,Meta Class,Meta Concept,Meta Description,envisionName,nwName,format,flags,nullTokens,failureKey,failureMapping,ecsName,Proposed,nbOccurrences,shortMeta,name_noDot, old ecsType,extra ecs 1,Reserved,Message,This key is used to capture the raw message that comes into the Log Decoder,msg,msg,Text,Transient,,,,rsa.internal.msg,log.original,272,internal,msg,text 1,,,,messageid,,,,,,,rsa.internal.messageid,event.code,270,internal,messageid,keyword 1,Time,Event Time,This key is used to capture the time mentioned in a raw session that represents the actual time an event occured in a standard normalized form,event_time,event.time,TimeT,None,,,,rsa.time.event_time,@timestamp,253,time,event_time,keyword @@ -28,16 +28,16 @@ revision,Meta Class,Meta Concept,Meta Description,envisionName,nwName,format,fla 1,Miscellaneous,Result Code,This key is used to capture the outcome/result numeric value of an action in a session,resultcode,result.code,Text,None,,,,rsa.misc.result_code,,112,misc,result_code,keyword 1,Miscellaneous,Category,This key is used to capture the category of an event given by the vendor in the session,category,category,Text,None,,,,rsa.misc.category,,105,misc,category,keyword 1,Miscellaneous,Object Name,This is used to capture name of object,obj_name,obj.name,Text,None,,,,rsa.misc.obj_name,,102,misc,obj_name,keyword -1,Network,Source Hostname,This key should only be used when it’s a Source Hostname.,shost,host.src,Text,None,,,,host.hostname,source.address,99,network,host_src,keyword +1,Network,Source Hostname,This key should only be used when it’s a Source Hostname.,shost,host.src,Text,None,,,,host.hostname,source.address,99,network,host_src,keyword,related.hosts 1,Miscellaneous,Object Type,This is used to capture type of object,obj_type,obj.type,Text,None,,,,rsa.misc.obj_type,,96,misc,obj_type,keyword 1,Web,URL,This key is used for capturing complete url,url,url,Text,Transient,,,,url.original,,93,web,url,keyword 1,Miscellaneous,Server Application,This key is used to capture the name of the server application only,application,server,Text,Transient,,,,network.application,,91,misc,server,keyword -1,Miscellaneous,Event Source,This key captures Source of the event that’s not a hostname,event_source,event.source,Text,None,,,,rsa.misc.event_source,,90,misc,event_source,keyword +1,Miscellaneous,Event Source,This key captures Source of the event that’s not a hostname,event_source,event.source,Text,None,,,,rsa.misc.event_source,related.hosts,90,misc,event_source,keyword 1,Network,Service Name,"This is used to capture descriptive service name, typically seen in Windows",service,service.name,Text,None,,,,service.name,,88,network,service_name,keyword -1,,,,domain,domain,Text,None,,,,server.domain,rsa.network.domain,84,network,domain,keyword +1,,,,domain,domain,Text,None,,,,server.domain,rsa.network.domain,84,network,domain,keyword,related.hosts 1,Miscellaneous,Event Session ID,This key is used to capture a sessionid from the session directly,sessionid,log.session.id,Text,Transient,,,,rsa.misc.log_session_id,,82,misc,log_session_id,keyword 1,Miscellaneous,Group Name,This key captures the Group Name value,group,group,Text,None,,,,rsa.misc.group,group.name,81,misc,group,keyword -1,Network,Destination Hostname,This key should only be used when it’s a Destination Hostname,dhost,host.dst,Text,None,,,,rsa.network.host_dst,destination.address,81,network,host_dst,keyword +1,Network,Destination Hostname,This key should only be used when it’s a Destination Hostname,dhost,host.dst,Text,None,,,,rsa.network.host_dst,destination.address,81,network,host_dst,keyword,related.hosts 1,Counters,Device class Counter 1,This is a generic counter key that should be used with the label dclass.c1.str only,dclass_counter1,dclass.c1,Int32,Transient,,,,rsa.counters.dclass_c1,,80,counters,dclass_c1,integer 1,Miscellaneous,Policy Name,This key is used to capture the Policy Name only.,policyname,policy.name,Text,None,,,,rsa.misc.policy_name,,80,misc,policy_name,keyword 1,Identity,Source User Account,This key should only be used to capture the Secondary/Source User in the event,c_username,user.src,Text,None,none|-,,,related.user,user.name,77,identity,user_src,keyword @@ -97,7 +97,7 @@ revision,Meta Class,Meta Concept,Meta Description,envisionName,nwName,format,fla 1,,Privilege,"Deprecated, use permissions",privilege,privilege,Text,Transient,,,,rsa.file.privilege,,31,file,privilege,keyword 1,Identity,User Role,This key is used to capture the Role of a user only,user_role,user.role,Text,Transient,,,,rsa.identity.user_role,,31,identity,user_role,keyword 1,Miscellaneous,Event Log Name,This key captures the Name of the event log,event_log,event.log,Text,Transient,,,,rsa.misc.event_log,,30,misc,event_log,keyword -1,Web,FQDN,Fully Qualified Domain Names,fqdn,fqdn,Text,None,,,,rsa.web.fqdn,,29,web,fqdn,keyword +1,Web,FQDN,Fully Qualified Domain Names,fqdn,fqdn,Text,None,,,,rsa.web.fqdn,related.hosts,29,web,fqdn,keyword 1,,User Account,"Deprecated, use user",administrator,username,Text,None,none|-,,,related.user,user.name,29,identity,username,keyword 1,,,,hostid,alias.host,Text,None,,,,rsa.network.alias_host,,28,network,alias_host,keyword 1,,,Deprecated key defined only in table map.,data,data,Text,Transient,,,,rsa.internal.data,,28,internal,data,keyword @@ -137,7 +137,7 @@ revision,Meta Class,Meta Concept,Meta Description,envisionName,nwName,format,fla 1,Cryptography,Cipher Name,This key is used to capture the Encryption Type or Encryption Key only,encryption_type,crypto,Text,Transient,,,,rsa.crypto.crypto,,18,crypto,crypto,keyword 1,Time,Recorded time,The event time as recorded by the system the event is collected from. The usage scenario is a multi-tier application where the management layer of the system records it's own timestamp at the time of collection from its child nodes. Must be in timestamp format.,recorded_time,recorded.time,TimeT,Transient,,,,rsa.time.recorded_time,,18,time,recorded_time,keyword 1,Miscellaneous,Virtual system name,This key captures Virtual System Name,vsys,vsys,Text,Transient,,,,rsa.misc.vsys,,18,misc,vsys,keyword -1,Web,Web request Domain,This key captures Domain name in the Web Request,web_domain,web.domain,Text,Transient,,,,url.domain,,18,web,web_domain,keyword +1,Web,Web request Domain,This key captures Domain name in the Web Request,web_domain,web.domain,Text,Transient,,,,url.domain,related.hosts,18,web,web_domain,keyword 1,Miscellaneous,Connection ID,This key captures the Connection ID,connectionid,connection.id,Text,Transient,,,,rsa.misc.connection_id,,17,misc,connection_id,keyword 1,Investigations,Vendor supplied Event Category,This is a vendor supplied category. This should be used in situations where the vendor has adopted their own event_category taxonomy.,vendor_event_cat,event.vcat,Text,Transient,,,,rsa.investigations.event_vcat,,17,investigations,event_vcat,keyword 1,Miscellaneous,Packets Total,"This key is the total number of packets sent/received in a session. Also, in cases where the Sent or Received context is not clear, this can be used.",packets,packets,UInt32,Transient,(null)|-,,,network.packets,,17,misc,packets,long @@ -197,7 +197,7 @@ revision,Meta Class,Meta Concept,Meta Description,envisionName,nwName,format,fla 1,Miscellaneous,Rule Unique ID,This key is the Unique Identifier for a rule.,rule_uid,rule.uid,Text,Transient,,,,rsa.misc.rule_uid,,9,misc,rule_uid,keyword 1,,Source Domain,"Deprecated, use domain.src",c_domain,sdomain,Text,Transient,,,,source.domain,,9,network,sdomain,keyword 1,Miscellaneous,Trigger Description,This key captures the Description of the trigger or threshold condition.,trigger_desc,trigger.desc,Text,Transient,,,,rsa.misc.trigger_desc,,9,misc,trigger_desc,keyword -1,,,,host,,,,,,,host.name,,9,network,host,keyword +1,,,,host,,,,,,,host.name,related.hosts,9,network,host,keyword 1,,,,inout,,,,,,,rsa.misc.inout,,9,misc,inout,keyword 1,,,,p_msgid,,,,,,,rsa.misc.p_msgid,,9,misc,p_msgid,keyword 1,,Child Pid,"Deprecated, use process.id",child_pid,child.pid,Int32,Transient,,child.pid.val,child_pid_val,process.pid,,8,misc,child_pid,long @@ -209,7 +209,7 @@ revision,Meta Class,Meta Concept,Meta Description,envisionName,nwName,format,fla 1,,,,process_src,process.src,Text,Transient,,,,process.parent.name,,8,misc,process_src,keyword 1,Network,Network mask Source,This key is used for capturing source Network Mask,smask,smask,Text,Transient,,,,rsa.network.smask,,8,network,smask,keyword 1,Database,SQL Transaction ID,This key captures the SQL transantion ID of the current session,trans_id,transact.id,Text,Transient,,,,rsa.db.transact_id,,8,db,transact_id,keyword -1,Web,Web referer Domain,Web referer's domain,web_ref_domain,web.ref.domain,Text,Transient,,,,rsa.web.web_ref_domain,,8,web,web_ref_domain,keyword +1,Web,Web referer Domain,Web referer's domain,web_ref_domain,web.ref.domain,Text,Transient,,,,rsa.web.web_ref_domain,related.hosts,8,web,web_ref_domain,keyword 1,,,,data_type,,,,,,,rsa.misc.data_type,,8,misc,data_type,keyword 1,,,,msgIdPart4,,,,,,,rsa.misc.msgIdPart4,,8,misc,msgIdPart4,keyword 1,Cryptography,Source (Server) Cipher size,This key captures Source (Client) Cipher Size,s_ciphersize,cipher.size.src,Int32,Transient,,,,rsa.crypto.cipher_size_src,,7,crypto,cipher_size_src,integer diff --git a/fields-merge.csv b/fields-merge.csv index c763a9e..eb8108c 100644 --- a/fields-merge.csv +++ b/fields-merge.csv @@ -7,6 +7,7 @@ host.name,by_prio,hostname,host destination.ip,append source.ip,append related.user,append +related.hosts,append event.action,by_prio,action,event_type host.ip,by_prio,hostip,hostip_v6,devicehostip,alias.ip,alias.ipv6 source.port,by_prio,sport,port.src,tcp.srcport,udp.srcport diff --git a/layout/module/__module__/__fileset__/config/input.yml.tpl b/layout/module/__module__/__fileset__/config/input.yml.tpl index 3f6fd3a..f09b160 100644 --- a/layout/module/__module__/__fileset__/config/input.yml.tpl +++ b/layout/module/__module__/__fileset__/config/input.yml.tpl @@ -31,7 +31,49 @@ processors: {{ if .community_id }} - community_id: ~ {{ end }} +- registered_domain: + ignore_missing: true + ignore_failure: true + field: dns.question.name + target_field: dns.question.registered_domain + target_subdomain_field: dns.question.subdomain + target_etld_field: dns.question.top_level_domain +- registered_domain: + ignore_missing: true + ignore_failure: true + field: client.domain + target_field: client.registered_domain + target_subdomain_field: client.subdomain + target_etld_field: client.top_level_domain +- registered_domain: + ignore_missing: true + ignore_failure: true + field: server.domain + target_field: server.registered_domain + target_subdomain_field: server.subdomain + target_etld_field: server.top_level_domain +- registered_domain: + ignore_missing: true + ignore_failure: true + field: destination.domain + target_field: destination.registered_domain + target_subdomain_field: destination.subdomain + target_etld_field: destination.top_level_domain +- registered_domain: + ignore_missing: true + ignore_failure: true + field: source.domain + target_field: source.registered_domain + target_subdomain_field: source.subdomain + target_etld_field: source.top_level_domain +- registered_domain: + ignore_missing: true + ignore_failure: true + field: url.domain + target_field: url.registered_domain + target_subdomain_field: url.subdomain + target_etld_field: url.top_level_domain - add_fields: target: '' fields: - ecs.version: 1.6.0 + ecs.version: 1.7.0 diff --git a/layout/module/__module__/__fileset__/ingest/pipeline.yml.tpl b/layout/module/__module__/__fileset__/ingest/pipeline.yml.tpl index c1019ae..3936c8f 100644 --- a/layout/module/__module__/__fileset__/ingest/pipeline.yml.tpl +++ b/layout/module/__module__/__fileset__/ingest/pipeline.yml.tpl @@ -53,6 +53,11 @@ processors: field: destination.as.organization_name target_field: destination.as.organization.name ignore_missing: true + - append: + field: related.hosts + value: '{{host.name}}' + allow_duplicates: false + if: ctx.host?.name != null && ctx.host?.name != '' on_failure: - append: field: error.message diff --git a/layout/package/__module__/__version__/data_stream/__fileset__/agent/stream/stream.yml.hbs.tpl b/layout/package/__module__/__version__/data_stream/__fileset__/agent/stream/stream.yml.hbs.tpl index 729ac81..ae3dcac 100644 --- a/layout/package/__module__/__version__/data_stream/__fileset__/agent/stream/stream.yml.hbs.tpl +++ b/layout/package/__module__/__version__/data_stream/__fileset__/agent/stream/stream.yml.hbs.tpl @@ -22,6 +22,48 @@ processors: ((- setvar "var_prefix" "" -)) ((- getvar "extra_processors" -)) - community_id: +- registered_domain: + ignore_missing: true + ignore_failure: true + field: dns.question.name + target_field: dns.question.registered_domain + target_subdomain_field: dns.question.subdomain + target_etld_field: dns.question.top_level_domain +- registered_domain: + ignore_missing: true + ignore_failure: true + field: client.domain + target_field: client.registered_domain + target_subdomain_field: client.subdomain + target_etld_field: client.top_level_domain +- registered_domain: + ignore_missing: true + ignore_failure: true + field: server.domain + target_field: server.registered_domain + target_subdomain_field: server.subdomain + target_etld_field: server.top_level_domain +- registered_domain: + ignore_missing: true + ignore_failure: true + field: destination.domain + target_field: destination.registered_domain + target_subdomain_field: destination.subdomain + target_etld_field: destination.top_level_domain +- registered_domain: + ignore_missing: true + ignore_failure: true + field: source.domain + target_field: source.registered_domain + target_subdomain_field: source.subdomain + target_etld_field: source.top_level_domain +- registered_domain: + ignore_missing: true + ignore_failure: true + field: url.domain + target_field: url.registered_domain + target_subdomain_field: url.subdomain + target_etld_field: url.top_level_domain - add_locale: ~ - add_fields: target: '' diff --git a/layout/package/__module__/__version__/data_stream/__fileset__/agent/stream/tcp.yml.hbs.tpl b/layout/package/__module__/__version__/data_stream/__fileset__/agent/stream/tcp.yml.hbs.tpl index f7b767d..5e13274 100644 --- a/layout/package/__module__/__version__/data_stream/__fileset__/agent/stream/tcp.yml.hbs.tpl +++ b/layout/package/__module__/__version__/data_stream/__fileset__/agent/stream/tcp.yml.hbs.tpl @@ -19,8 +19,50 @@ processors: ((- setvar "var_prefix" "" -)) ((- getvar "extra_processors" -)) - community_id: +- registered_domain: + ignore_missing: true + ignore_failure: true + field: dns.question.name + target_field: dns.question.registered_domain + target_subdomain_field: dns.question.subdomain + target_etld_field: dns.question.top_level_domain +- registered_domain: + ignore_missing: true + ignore_failure: true + field: client.domain + target_field: client.registered_domain + target_subdomain_field: client.subdomain + target_etld_field: client.top_level_domain +- registered_domain: + ignore_missing: true + ignore_failure: true + field: server.domain + target_field: server.registered_domain + target_subdomain_field: server.subdomain + target_etld_field: server.top_level_domain +- registered_domain: + ignore_missing: true + ignore_failure: true + field: destination.domain + target_field: destination.registered_domain + target_subdomain_field: destination.subdomain + target_etld_field: destination.top_level_domain +- registered_domain: + ignore_missing: true + ignore_failure: true + field: source.domain + target_field: source.registered_domain + target_subdomain_field: source.subdomain + target_etld_field: source.top_level_domain +- registered_domain: + ignore_missing: true + ignore_failure: true + field: url.domain + target_field: url.registered_domain + target_subdomain_field: url.subdomain + target_etld_field: url.top_level_domain - add_locale: ~ - add_fields: target: '' fields: - ecs.version: 1.6.0 + ecs.version: 1.7.0 diff --git a/layout/package/__module__/__version__/data_stream/__fileset__/agent/stream/udp.yml.hbs.tpl b/layout/package/__module__/__version__/data_stream/__fileset__/agent/stream/udp.yml.hbs.tpl index 35bf50f..1dbdc70 100644 --- a/layout/package/__module__/__version__/data_stream/__fileset__/agent/stream/udp.yml.hbs.tpl +++ b/layout/package/__module__/__version__/data_stream/__fileset__/agent/stream/udp.yml.hbs.tpl @@ -19,6 +19,48 @@ processors: ((- setvar "var_prefix" "" -)) ((- getvar "extra_processors" -)) - community_id: +- registered_domain: + ignore_missing: true + ignore_failure: true + field: dns.question.name + target_field: dns.question.registered_domain + target_subdomain_field: dns.question.subdomain + target_etld_field: dns.question.top_level_domain +- registered_domain: + ignore_missing: true + ignore_failure: true + field: client.domain + target_field: client.registered_domain + target_subdomain_field: client.subdomain + target_etld_field: client.top_level_domain +- registered_domain: + ignore_missing: true + ignore_failure: true + field: server.domain + target_field: server.registered_domain + target_subdomain_field: server.subdomain + target_etld_field: server.top_level_domain +- registered_domain: + ignore_missing: true + ignore_failure: true + field: destination.domain + target_field: destination.registered_domain + target_subdomain_field: destination.subdomain + target_etld_field: destination.top_level_domain +- registered_domain: + ignore_missing: true + ignore_failure: true + field: source.domain + target_field: source.registered_domain + target_subdomain_field: source.subdomain + target_etld_field: source.top_level_domain +- registered_domain: + ignore_missing: true + ignore_failure: true + field: url.domain + target_field: url.registered_domain + target_subdomain_field: url.subdomain + target_etld_field: url.top_level_domain - add_locale: ~ - add_fields: target: '' diff --git a/layout/package/__module__/__version__/data_stream/__fileset__/elasticsearch/ingest_pipeline/default.yml.tpl b/layout/package/__module__/__version__/data_stream/__fileset__/elasticsearch/ingest_pipeline/default.yml.tpl index c1019ae..3936c8f 100644 --- a/layout/package/__module__/__version__/data_stream/__fileset__/elasticsearch/ingest_pipeline/default.yml.tpl +++ b/layout/package/__module__/__version__/data_stream/__fileset__/elasticsearch/ingest_pipeline/default.yml.tpl @@ -53,6 +53,11 @@ processors: field: destination.as.organization_name target_field: destination.as.organization.name ignore_missing: true + - append: + field: related.hosts + value: '{{host.name}}' + allow_duplicates: false + if: ctx.host?.name != null && ctx.host?.name != '' on_failure: - append: field: error.message diff --git a/layout/package/__module__/__version__/data_stream/__fileset__/fields/ecs.yml b/layout/package/__module__/__version__/data_stream/__fileset__/fields/ecs.yml index ea38c07..6239f0e 100644 --- a/layout/package/__module__/__version__/data_stream/__fileset__/fields/ecs.yml +++ b/layout/package/__module__/__version__/data_stream/__fileset__/fields/ecs.yml @@ -168,6 +168,13 @@ ignore_above: 1024 description: All the user names seen on your event. default_field: false + - name: hosts + level: extended + type: keyword + ignore_above: 1024 + description: All hostnames or other host identifiers seen on your event. Example + identifiers include FQDNs, domain names, workstation names, or aliases. + default_field: false - name: ip level: extended type: ip @@ -276,6 +283,43 @@ type: keyword ignore_above: 1024 description: Source domain. + - name: registered_domain + level: extended + type: keyword + ignore_above: 1024 + description: 'The highest registered source domain, stripped of the subdomain. + + For example, the registered domain for "foo.example.com" is "example.com". + + This value can be determined precisely with a list like the public suffix + list (http://publicsuffix.org). Trying to approximate this by simply taking + the last two labels will not work well for TLDs such as "co.uk".' + example: example.com + - name: subdomain + level: extended + type: keyword + ignore_above: 1024 + description: 'The subdomain portion of a fully qualified domain name includes + all of the names except the host name under the registered_domain. In a partially + qualified domain, or if the the qualification level of the full name cannot + be determined, subdomain contains all of the names below the registered domain. + + For example the subdomain portion of "www.east.mydomain.co.uk" is "east". + If the domain has multiple levels of subdomain, such as "sub2.sub1.example.com", + the subdomain field should contain "sub2.sub1", with no trailing period.' + example: east + - name: top_level_domain + level: extended + type: keyword + ignore_above: 1024 + description: 'The effective top level domain (eTLD), also known as the domain + suffix, is the last part of the domain name. For example, the top level domain + for example.com is "com". + + This value can be determined precisely with a list like the public suffix + list (http://publicsuffix.org). Trying to approximate this by simply taking + the last label will not work well for effective TLDs such as "co.uk".' + example: co.uk - name: geo type: group fields: @@ -404,6 +448,43 @@ type: keyword ignore_above: 1024 description: Destination domain. + - name: registered_domain + level: extended + type: keyword + ignore_above: 1024 + description: 'The highest registered source domain, stripped of the subdomain. + + For example, the registered domain for "foo.example.com" is "example.com". + + This value can be determined precisely with a list like the public suffix + list (http://publicsuffix.org). Trying to approximate this by simply taking + the last two labels will not work well for TLDs such as "co.uk".' + example: example.com + - name: subdomain + level: extended + type: keyword + ignore_above: 1024 + description: 'The subdomain portion of a fully qualified domain name includes + all of the names except the host name under the registered_domain. In a partially + qualified domain, or if the the qualification level of the full name cannot + be determined, subdomain contains all of the names below the registered domain. + + For example the subdomain portion of "www.east.mydomain.co.uk" is "east". + If the domain has multiple levels of subdomain, such as "sub2.sub1.example.com", + the subdomain field should contain "sub2.sub1", with no trailing period.' + example: east + - name: top_level_domain + level: extended + type: keyword + ignore_above: 1024 + description: 'The effective top level domain (eTLD), also known as the domain + suffix, is the last part of the domain name. For example, the top level domain + for example.com is "com". + + This value can be determined precisely with a list like the public suffix + list (http://publicsuffix.org). Trying to approximate this by simply taking + the last label will not work well for effective TLDs such as "co.uk".' + example: co.uk - name: geo type: group fields: @@ -493,11 +574,17 @@ type: keyword ignore_above: 1024 description: "Direction of the network traffic.\nRecommended values are:\n *\ - \ inbound\n * outbound\n * internal\n * external\n * unknown\n\nWhen mapping\ - \ events from a host-based monitoring context, populate this field from the\ - \ host's point of view.\nWhen mapping events from a network or perimeter-based\ - \ monitoring context, populate this field from the point of view of your network\ - \ perimeter." + \ ingress\n * egress\n * inbound\n * outbound\n * internal\n * external\n\ + \ * unknown\n\nWhen mapping events from a host-based monitoring context, populate\ + \ this field from the host's point of view, using the values \"ingress\" or\ + \ \"egress\".\nWhen mapping events from a network or perimeter-based monitoring\ + \ context, populate this field from the point of view of the network perimeter,\ + \ using the values \"inbound\", \"outbound\", \"internal\" or \"external\".\n\ + Note that \"internal\" is not crossing perimeter boundaries, and is meant to\ + \ describe communication between two hosts within the perimeter. Note also that\ + \ \"external\" is meant to describe traffic between two hosts that are external\ + \ to the perimeter. This could for example be useful for ISPs or VPN service\ + \ providers." example: inbound - name: packets level: core @@ -600,7 +687,10 @@ level: extended type: keyword ignore_above: 1024 - description: File extension. + description: 'File extension, excluding the leading dot. + + Note that when the file name has multiple extensions (example.tar.gz), only + the last one should be captured ("gz", not "tar.gz").' example: png - name: path level: extended @@ -662,7 +752,10 @@ description: 'Domain of the url, such as "www.elastic.co". In some cases a URL may refer to an IP and/or port directly, without a domain - name. In this case, the IP address would go to the `domain` field.' + name. In this case, the IP address would go to the `domain` field. + + If the URL contains a literal IPv6 address enclosed by `[` and `]` (IETF RFC + 2732), the `[` and `]` characters should also be captured in the `domain` field.' example: www.elastic.co - name: path level: extended @@ -710,6 +803,51 @@ For Beats the `service.name` is by default a copy of the `service.type` field if no name is specified.' example: elasticsearch-metrics +- name: client + type: group + fields: + - name: domain + level: core + type: keyword + ignore_above: 1024 + description: Server domain. + - name: registered_domain + level: extended + type: keyword + ignore_above: 1024 + description: 'The highest registered source domain, stripped of the subdomain. + + For example, the registered domain for "foo.example.com" is "example.com". + + This value can be determined precisely with a list like the public suffix + list (http://publicsuffix.org). Trying to approximate this by simply taking + the last two labels will not work well for TLDs such as "co.uk".' + example: example.com + - name: subdomain + level: extended + type: keyword + ignore_above: 1024 + description: 'The subdomain portion of a fully qualified domain name includes + all of the names except the host name under the registered_domain. In a partially + qualified domain, or if the the qualification level of the full name cannot + be determined, subdomain contains all of the names below the registered domain. + + For example the subdomain portion of "www.east.mydomain.co.uk" is "east". + If the domain has multiple levels of subdomain, such as "sub2.sub1.example.com", + the subdomain field should contain "sub2.sub1", with no trailing period.' + example: east + - name: top_level_domain + level: extended + type: keyword + ignore_above: 1024 + description: 'The effective top level domain (eTLD), also known as the domain + suffix, is the last part of the domain name. For example, the top level domain + for example.com is "com". + + This value can be determined precisely with a list like the public suffix + list (http://publicsuffix.org). Trying to approximate this by simply taking + the last label will not work well for effective TLDs such as "co.uk".' + example: co.uk - name: server type: group fields: @@ -718,6 +856,43 @@ type: keyword ignore_above: 1024 description: Server domain. + - name: registered_domain + level: extended + type: keyword + ignore_above: 1024 + description: 'The highest registered source domain, stripped of the subdomain. + + For example, the registered domain for "foo.example.com" is "example.com". + + This value can be determined precisely with a list like the public suffix + list (http://publicsuffix.org). Trying to approximate this by simply taking + the last two labels will not work well for TLDs such as "co.uk".' + example: example.com + - name: subdomain + level: extended + type: keyword + ignore_above: 1024 + description: 'The subdomain portion of a fully qualified domain name includes + all of the names except the host name under the registered_domain. In a partially + qualified domain, or if the the qualification level of the full name cannot + be determined, subdomain contains all of the names below the registered domain. + + For example the subdomain portion of "www.east.mydomain.co.uk" is "east". + If the domain has multiple levels of subdomain, such as "sub2.sub1.example.com", + the subdomain field should contain "sub2.sub1", with no trailing period.' + example: east + - name: top_level_domain + level: extended + type: keyword + ignore_above: 1024 + description: 'The effective top level domain (eTLD), also known as the domain + suffix, is the last part of the domain name. For example, the top level domain + for example.com is "com". + + This value can be determined precisely with a list like the public suffix + list (http://publicsuffix.org). Trying to approximate this by simply taking + the last label will not work well for effective TLDs such as "co.uk".' + example: co.uk - name: group type: group fields: @@ -896,6 +1071,48 @@ ignore_above: 1024 description: The type of record being queried. example: AAAA + - name: domain + level: core + type: keyword + ignore_above: 1024 + description: Server domain. + - name: registered_domain + level: extended + type: keyword + ignore_above: 1024 + description: 'The highest registered source domain, stripped of the subdomain. + + For example, the registered domain for "foo.example.com" is "example.com". + + This value can be determined precisely with a list like the public suffix + list (http://publicsuffix.org). Trying to approximate this by simply taking + the last two labels will not work well for TLDs such as "co.uk".' + example: example.com + - name: subdomain + level: extended + type: keyword + ignore_above: 1024 + description: 'The subdomain portion of a fully qualified domain name includes + all of the names except the host name under the registered_domain. In a partially + qualified domain, or if the the qualification level of the full name cannot + be determined, subdomain contains all of the names below the registered domain. + + For example the subdomain portion of "www.east.mydomain.co.uk" is "east". + If the domain has multiple levels of subdomain, such as "sub2.sub1.example.com", + the subdomain field should contain "sub2.sub1", with no trailing period.' + example: east + - name: top_level_domain + level: extended + type: keyword + ignore_above: 1024 + description: 'The effective top level domain (eTLD), also known as the domain + suffix, is the last part of the domain name. For example, the top level domain + for example.com is "com". + + This value can be determined precisely with a list like the public suffix + list (http://publicsuffix.org). Trying to approximate this by simply taking + the last label will not work well for effective TLDs such as "co.uk".' + example: co.uk - name: answers type: group fields: @@ -928,3 +1145,6 @@ ignore_above: 1024 description: List of keywords used to tag each event. example: '["production", "env2"]' +- name: ecs.version + type: keyword + description: ECS version this event conforms to. diff --git a/output/javascript/liblogparser.js b/output/javascript/liblogparser.js index 6cdb48a..cec99a0 100644 --- a/output/javascript/liblogparser.js +++ b/output/javascript/liblogparser.js @@ -1012,7 +1012,7 @@ var ecs_mappings = { "ddomain": {to:[{field: "destination.domain", setter: fld_prio, prio: 0}]}, "devicehostip": {convert: to_ip, to:[{field: "host.ip", setter: fld_prio, prio: 2},{field: "related.ip", setter: fld_append}]}, "devicehostmac": {convert: to_mac, to:[{field: "host.mac", setter: fld_prio, prio: 0}]}, - "dhost": {to:[{field: "destination.address", setter: fld_set}]}, + "dhost": {to:[{field: "destination.address", setter: fld_set},{field: "related.hosts", setter: fld_append}]}, "dinterface": {to:[{field: "observer.egress.interface.name", setter: fld_set}]}, "direction": {to:[{field: "network.direction", setter: fld_set}]}, "directory": {to:[{field: "file.directory", setter: fld_set}]}, @@ -1020,7 +1020,7 @@ var ecs_mappings = { "dns.responsetype": {to:[{field: "dns.answers.type", setter: fld_set}]}, "dns.resptext": {to:[{field: "dns.answers.name", setter: fld_set}]}, "dns_querytype": {to:[{field: "dns.question.type", setter: fld_set}]}, - "domain": {to:[{field: "server.domain", setter: fld_prio, prio: 0}]}, + "domain": {to:[{field: "server.domain", setter: fld_prio, prio: 0},{field: "related.hosts", setter: fld_append}]}, "domain.dst": {to:[{field: "destination.domain", setter: fld_prio, prio: 1}]}, "domain.src": {to:[{field: "source.domain", setter: fld_prio, prio: 2}]}, "domain_id": {to:[{field: "user.domain", setter: fld_set}]}, @@ -1030,6 +1030,7 @@ var ecs_mappings = { "dtransport": {convert: to_long, to:[{field: "destination.nat.port", setter: fld_prio, prio: 0}]}, "ec_outcome": {to:[{field: "event.outcome", setter: fld_ecs_outcome}]}, "event_description": {to:[{field: "message", setter: fld_prio, prio: 0}]}, + "event_source": {to:[{field: "related.hosts", setter: fld_append}]}, "event_time": {convert: to_date, to:[{field: "@timestamp", setter: fld_set}]}, "event_type": {to:[{field: "event.action", setter: fld_prio, prio: 1}]}, "extension": {to:[{field: "file.extension", setter: fld_prio, prio: 1}]}, @@ -1038,9 +1039,10 @@ var ecs_mappings = { "filename_size": {convert: to_long, to:[{field: "file.size", setter: fld_set}]}, "filepath": {to:[{field: "file.path", setter: fld_set}]}, "filetype": {to:[{field: "file.type", setter: fld_set}]}, + "fqdn": {to:[{field: "related.hosts", setter: fld_append}]}, "group": {to:[{field: "group.name", setter: fld_set}]}, "groupid": {to:[{field: "group.id", setter: fld_set}]}, - "host": {to:[{field: "host.name", setter: fld_prio, prio: 1}]}, + "host": {to:[{field: "host.name", setter: fld_prio, prio: 1},{field: "related.hosts", setter: fld_append}]}, "hostip": {convert: to_ip, to:[{field: "host.ip", setter: fld_prio, prio: 0},{field: "related.ip", setter: fld_append}]}, "hostip_v6": {convert: to_ip, to:[{field: "host.ip", setter: fld_prio, prio: 1},{field: "related.ip", setter: fld_append}]}, "hostname": {to:[{field: "host.name", setter: fld_prio, prio: 0}]}, @@ -1094,7 +1096,7 @@ var ecs_mappings = { "service.name": {to:[{field: "service.name", setter: fld_prio, prio: 0}]}, "service_account": {to:[{field: "related.user", setter: fld_append},{field: "user.name", setter: fld_prio, prio: 7}]}, "severity": {to:[{field: "log.level", setter: fld_set}]}, - "shost": {to:[{field: "host.hostname", setter: fld_set},{field: "source.address", setter: fld_set}]}, + "shost": {to:[{field: "host.hostname", setter: fld_set},{field: "source.address", setter: fld_set},{field: "related.hosts", setter: fld_append}]}, "sinterface": {to:[{field: "observer.ingress.interface.name", setter: fld_set}]}, "sld": {to:[{field: "url.registered_domain", setter: fld_set}]}, "smacaddr": {convert: to_mac, to:[{field: "source.mac", setter: fld_set}]}, @@ -1119,9 +1121,10 @@ var ecs_mappings = { "user_id": {to:[{field: "user.id", setter: fld_prio, prio: 0}]}, "username": {to:[{field: "related.user", setter: fld_append},{field: "user.name", setter: fld_prio, prio: 1}]}, "version": {to:[{field: "observer.version", setter: fld_set}]}, - "web_domain": {to:[{field: "url.domain", setter: fld_prio, prio: 1}]}, + "web_domain": {to:[{field: "url.domain", setter: fld_prio, prio: 1},{field: "related.hosts", setter: fld_append}]}, "web_extension": {to:[{field: "file.extension", setter: fld_prio, prio: 0}]}, "web_query": {to:[{field: "url.query", setter: fld_prio, prio: 1}]}, + "web_ref_domain": {to:[{field: "related.hosts", setter: fld_append}]}, "web_referer": {to:[{field: "http.request.referrer", setter: fld_prio, prio: 0}]}, "web_root": {to:[{field: "url.path", setter: fld_set}]}, "webpage": {to:[{field: "file.name", setter: fld_prio, prio: 1}]}, @@ -2014,6 +2017,7 @@ function do_populate(evt, base, targets) { var mapping = targets[key]; if (mapping === undefined) continue; var value = base[key]; + if (value === "") continue; if (mapping.convert !== undefined) { value = mapping.convert(value); if (value === undefined) { diff --git a/scripts/gen-field-mappings.py b/scripts/gen-field-mappings.py index 6c32cfa..35024af 100644 --- a/scripts/gen-field-mappings.py +++ b/scripts/gen-field-mappings.py @@ -34,7 +34,7 @@ def by_prio(lst): def process_row(row): - lst = filter(str.__len__, [row[idx] for idx in [MAP, ALT]]) + lst = filter(str.__len__, [row[idx] if idx < len(row) else '' for idx in [MAP, ALT, EXTRA]]) typ = row[TYPE] if typ not in type_to_es: raise Exception('unsupported type: {}'.format(typ)) diff --git a/scripts/gen-field-yml.py b/scripts/gen-field-yml.py index d1caa12..c010968 100644 --- a/scripts/gen-field-yml.py +++ b/scripts/gen-field-yml.py @@ -99,7 +99,7 @@ def to_list(dct): es_type = type_to_es[typ] if es_type is None or es_type == 'mac': es_type = 'keyword' - for field in filter(str.__len__, [row[idx] for idx in [MAP, ALT]]): + for field in filter(str.__len__, [row[idx] if idx < len(row) else '' for idx in [MAP, ALT, EXTRA]]): if is_rsa_field(field): if field in rsa: if rsa[field]['type'] != es_type: diff --git a/scripts/shared.py b/scripts/shared.py index bd85cd6..30121ba 100644 --- a/scripts/shared.py +++ b/scripts/shared.py @@ -8,6 +8,7 @@ TYPE = 6 MAP = 11 ALT = 12 +EXTRA = 17 # Conversions type_to_es = {