Update Suricata for ECS DNS

This updates the Suricata module to populate the ECS DNS fields. It does not remove existing `suricata.eve.dns.*` fields to preserve backward compatibility.

This also enhances the pipeline to handle the Suricata detailed DNS format (aka version 2). It requires that when using EVE DNS `version: 2` that `formats: [detailed]` is used (`grouped` can be enabled too but it is ignored).

`log.original` is now populated with the original JSON log data.

`source.address` and `destination.address` are now populated.

`event.end` is populated with the `flow.end` value now and hence some events that did not contain `flow.end` no longer have an `event.end`.

Relates elastic#13320

CHANGELOG.next.asciidoc

@@ -274,6 +274,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add module for ingesting IBM MQ logs. {pull}8782[8782]
 - Add S3 input to retrieve logs from AWS S3 buckets. {pull}12640[12640] {issue}12582[12582]
 - Add aws module s3access metricset. {pull}13170[13170] {issue}12880[12880]
+- Update Suricata module to populate ECS DNS fields and handle EVE DNS version 2. {issue}13320[13320] {pull}13329[13329]
 
 *Heartbeat*
 

x-pack/filebeat/module/suricata/eve/config/eve.yml

@@ -6,15 +6,180 @@ paths:
 exclude_files: [".gz$"]
 tags: {{.tags}}
 
-json.keys_under_root: false
-
-{{ if .community_id }}
 processors:
-  - community_id:
+  - rename:
+      fields:
+        - {from: message, to: log.original}
+  - decode_json_fields:
+      fields: [log.original]
+      target: suricata.eve
+  - convert:
+      ignore_missing: true
+      ignore_failure: true
+      mode: rename
+      fields:
+        - {from: suricata.eve.src_ip, to: source.address}
+        - {from: suricata.eve.src_port, to: source.port, type: long}
+        - {from: suricata.eve.dest_ip, to: destination.address}
+        - {from: suricata.eve.dest_port, to: destination.port}
+        - {from: suricata.eve.proto, to: network.transport}
+  - convert:
+      ignore_missing: true
+      ignore_failure: true
+      mode: copy
       fields:
-        source_ip: json.src_ip
-        source_port: json.src_port
-        destination_ip: json.dest_ip
-        destination_port: json.dest_port
-        transport: json.proto
+        - {from: source.address, to: source.ip, type: ip}
+        - {from: destination.address, to: destination.ip, type: ip}
+        - {from: '@timestamp', to: event.created}
+  - timestamp:
+      field: suricata.eve.timestamp
+      layouts:
+        - '2006-01-02T15:04:05.999999999Z0700' # ISO8601
+  - drop_fields:
+      fields:
+        - suricata.eve.timestamp
+{{ if .community_id }}
+  - community_id:
 {{ end }}
+  - if:
+      equals.suricata.eve.event_type: dns
+    then:
+      - convert:
+          ignore_missing: true
+          ignore_failure: true
+          mode: copy
+          fields:
+            - {from: suricata.eve.dns.id, to: dns.id, type: string}
+            - {from: suricata.eve.dns.rcode, to: dns.response_code}
+            - {from: suricata.eve.dns.type, to: dns.type}
+      - convert:
+          when.equals.dns.type: query
+          ignore_missing: true
+          ignore_failure: true
+          mode: copy
+          fields:
+            - {from: suricata.eve.dns.rrname, to: dns.question.name}
+            - {from: suricata.eve.dns.rrtype, to: dns.question.type}
+      # Handle the version=1 EVE DNS answer format. Each JSON event contains
+      # a single resource record from the DNS response.
+      - script:
+          when.and:
+            - equals.dns.type: answer
+            - not.has_fields: [suricata.eve.dns.version]
+          id: suricata_dns_answers_v1
+          lang: javascript
+          source: >
+            function process(evt) {
+                var name = evt.Get("suricata.eve.dns.rrname");
+                var data = evt.Get("suricata.eve.dns.rdata");
+                var type = evt.Get("suricata.eve.dns.rrtype");
+                var ttl = evt.Get("suricata.eve.dns.ttl");
+
+                var answer = {};
+                if (name) {
+                    answer.name = name;
+                }
+                if (data) {
+                    answer.data = data;
+                }
+                if (type) {
+                    answer.type = type;
+                }
+                if (ttl) {
+                    answer.ttl = ttl;
+                }
+
+                if (Object.keys(answer).length === 0) {
+                    return;
+                }
+                evt.Put("dns.answers", [answer]);
+            }
+      # Handle the version=2 EVE DNS answer format.
+      - if:
+          and:
+            - equals.dns.type: answer
+            - equals.suricata.eve.dns.version: 2
+        then:
+          - convert:
+              ignore_missing: true
+              ignore_failure: true
+              mode: copy
+              fields:
+                - {from: suricata.eve.dns.rrname, to: dns.question.name}
+                - {from: suricata.eve.dns.rrtype, to: dns.question.type}
+          - script:
+              id: suricata_dns_answers_v2
+              lang: javascript
+              source: >
+                function transformDetailedAnswers(evt) {
+                    var answers = evt.Get("suricata.eve.dns.answers");
+                    if (!answers) {
+                        return;
+                    }
+                    evt.Delete("suricata.eve.dns.answers");
+
+                    var resolvedIps = [];
+                    for (var i = 0; i < answers.length; i++) {
+                        var answer = answers[i];
+
+                        // Rename properties.
+                        var name = answer["rrname"];
+                        delete answer["rrname"];
+                        var type = answer["rrtype"];
+                        delete answer["rrtype"];
+                        var data = answer["rdata"];
+                        delete answer["rdata"];
+
+                        answer["name"] = name;
+                        answer["type"] = type;
+                        answer["data"] = data;
+
+                        // Append IP addresses to dns.resolved_ip.
+                        if (type === "A" || type === "AAAA") {
+                            resolvedIps.push(data);
+                        }
+                    }
+                    evt.Put("dns.answers", answers);
+                    if (resolvedIps.length > 0) {
+                        evt.Put("dns.resolved_ip", resolvedIps);
+                    }
+                }
+
+                function addDnsHeaderFlags(evt) {
+                    var flag = evt.Get("suricata.eve.dns.aa");
+                    if (flag === true) {
+                        evt.AppendTo("dns.header_flags", "AA");
+                    }
+
+                    flag = evt.Get("suricata.eve.dns.tc");
+                    if (flag === true) {
+                        evt.AppendTo("dns.header_flags", "TC");
+                    }
+
+                    flag = evt.Get("suricata.eve.dns.rd");
+                    if (flag === true) {
+                        evt.AppendTo("dns.header_flags", "RD");
+                    }
+
+                    flag = evt.Get("suricata.eve.dns.ra");
+                    if (flag === true) {
+                        evt.AppendTo("dns.header_flags", "RA");
+                    }
+                }
+
+                function process(evt) {
+                    transformDetailedAnswers(evt);
+                    addDnsHeaderFlags(evt);
+                }
+      - drop_fields:
+          # TODO (andrewkroh 2019-08-22): Uncomment after ignore_missing is added to drop_fields.
+          #ignore_missing: true
+          fields:
+            - suricata.eve.dns.aa
+            - suricata.eve.dns.tc
+            - suricata.eve.dns.rd
+            - suricata.eve.dns.ra
+            - suricata.eve.dns.qr
+            - suricata.eve.dns.version
+            - suricata.eve.dns.flags
+            - suricata.eve.dns.grouped