From 3a69204281e49d2404b4b072df7e381a83c823ce Mon Sep 17 00:00:00 2001 From: Andrew Kroh Date: Tue, 27 Aug 2019 13:06:34 -0400 Subject: [PATCH] Update Zeek DNS pipeline with ECS DNS fields (#13324) This adds ECS DNS fields to the Zeek DNS fileset (but does not change or remove any existing ones). Use event.original Add registered_domain Add dns.resolved_ip Relates #13320 --- CHANGELOG.next.asciidoc | 1 + .../script/javascript/module/include.go | 1 + .../script/javascript/module/net/net.go | 68 ++++++++ .../script/javascript/module/net/net_test.go | 98 +++++++++++ .../filebeat/module/zeek/dns/config/dns.yml | 152 +++++++++++++++--- .../module/zeek/dns/ingest/pipeline.json | 100 ------------ .../module/zeek/dns/ingest/pipeline.yml | 52 ++++++ x-pack/filebeat/module/zeek/dns/manifest.yml | 2 +- .../zeek/dns/test/dns-json.log-expected.json | 31 +++- 9 files changed, 379 insertions(+), 126 deletions(-) create mode 100644 libbeat/processors/script/javascript/module/net/net.go create mode 100644 libbeat/processors/script/javascript/module/net/net_test.go delete mode 100644 x-pack/filebeat/module/zeek/dns/ingest/pipeline.json create mode 100644 x-pack/filebeat/module/zeek/dns/ingest/pipeline.yml diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 93f86daa2b9..279f598f5e2 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -280,6 +280,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Add aws module s3access metricset. {pull}13170[13170] {issue}12880[12880] - Update Suricata module to populate ECS DNS fields and handle EVE DNS version 2. {issue}13320[13320] {pull}13329[13329] - Update PAN-OS fileset to use the ECS NAT fields. {issue}13320[13320] {pull}13330[13330] +- Add fields to the Zeek DNS fileset for ECS DNS. {issue}13320[13320] {pull}13324[13324] *Heartbeat* diff --git a/libbeat/processors/script/javascript/module/include.go b/libbeat/processors/script/javascript/module/include.go index 96951e3008c..6b4608a51ea 100644 --- a/libbeat/processors/script/javascript/module/include.go +++ b/libbeat/processors/script/javascript/module/include.go @@ -20,6 +20,7 @@ package module import ( // Register javascript modules. _ "github.com/elastic/beats/libbeat/processors/script/javascript/module/console" + _ "github.com/elastic/beats/libbeat/processors/script/javascript/module/net" _ "github.com/elastic/beats/libbeat/processors/script/javascript/module/path" _ "github.com/elastic/beats/libbeat/processors/script/javascript/module/processor" _ "github.com/elastic/beats/libbeat/processors/script/javascript/module/require" diff --git a/libbeat/processors/script/javascript/module/net/net.go b/libbeat/processors/script/javascript/module/net/net.go new file mode 100644 index 00000000000..5551bce5b2a --- /dev/null +++ b/libbeat/processors/script/javascript/module/net/net.go @@ -0,0 +1,68 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package net + +import ( + "net" + + "github.com/dop251/goja" + "github.com/dop251/goja_nodejs/require" +) + +// Require registers the net module that provides utilities for working with IP +// addresses. It can be accessed using: +// +// // javascript +// var net = require('net'); +// +func Require(vm *goja.Runtime, module *goja.Object) { + o := module.Get("exports").(*goja.Object) + o.Set("isIP", isIP) + o.Set("isIPv4", isIPv4) + o.Set("isIPv6", isIPv6) +} + +func isIP(input string) int32 { + ip := net.ParseIP(input) + if ip == nil { + return 0 + } + + if ip.To4() != nil { + return 4 + } + + return 6 +} + +func isIPv4(input string) bool { + return 4 == isIP(input) +} + +func isIPv6(input string) bool { + return 6 == isIP(input) +} + +// Enable adds net to the given runtime. +func Enable(runtime *goja.Runtime) { + runtime.Set("net", require.Require(runtime, "net")) +} + +func init() { + require.RegisterNativeModule("net", Require) +} diff --git a/libbeat/processors/script/javascript/module/net/net_test.go b/libbeat/processors/script/javascript/module/net/net_test.go new file mode 100644 index 00000000000..f5401403643 --- /dev/null +++ b/libbeat/processors/script/javascript/module/net/net_test.go @@ -0,0 +1,98 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package net_test + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/elastic/beats/libbeat/beat" + "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/libbeat/processors/script/javascript" + + _ "github.com/elastic/beats/libbeat/processors/script/javascript/module/net" + _ "github.com/elastic/beats/libbeat/processors/script/javascript/module/require" +) + +func TestNetIsIP(t *testing.T) { + const script = ` +var net = require('net'); + +function process(evt) { + var ip = evt.Get("ip"); + var ipType = net.isIP(ip); + switch (ipType) { + case 4: + evt.Put("network.type", "ipv4"); + break + case 6: + evt.Put("network.type", "ipv6"); + break + } +} +` + + p, err := javascript.NewFromConfig(javascript.Config{Source: script}, nil) + if err != nil { + t.Fatal(err) + } + + for ip, typ := range map[string]interface{}{ + "192.168.0.1": "ipv4", + "::ffff:192.168.0.1": "ipv4", + "2001:0db8:0000:0000:0000:ff00:0042:8329": "ipv6", + "2001:db8:0:0:0:ff00:42:8329": "ipv6", + "2001:db8::ff00:42:8329": "ipv6", + "www.elastic.co": nil, + } { + evt, err := p.Run(&beat.Event{Fields: common.MapStr{"ip": ip}}) + if err != nil { + t.Fatal(err) + } + + fields := evt.Fields.Flatten() + assert.Equal(t, typ, fields["network.type"]) + } +} + +func TestNetIsIPvN(t *testing.T) { + const script = ` +var net = require('net'); + +function process(evt) { + if (net.isIPv4("192.168.0.1") !== true) { + throw "isIPv4 failed"; + } + + if (net.isIPv6("2001:db8::ff00:42:8329") !== true) { + throw "isIPv6 failed"; + } +} +` + + p, err := javascript.NewFromConfig(javascript.Config{Source: script}, nil) + if err != nil { + t.Fatal(err) + } + + _, err = p.Run(&beat.Event{Fields: common.MapStr{}}) + if err != nil { + t.Fatal(err) + } +} diff --git a/x-pack/filebeat/module/zeek/dns/config/dns.yml b/x-pack/filebeat/module/zeek/dns/config/dns.yml index f1b004ec296..aa755d756d9 100644 --- a/x-pack/filebeat/module/zeek/dns/config/dns.yml +++ b/x-pack/filebeat/module/zeek/dns/config/dns.yml @@ -6,39 +6,143 @@ paths: exclude_files: [".gz$"] tags: {{.tags}} -json.keys_under_root: false - processors: - - drop_fields: - fields: ["json.Z","json.auth","json.addl"] - rename: fields: - - from: "json" - to: "zeek.dns" + - {from: message, to: event.original} + - decode_json_fields: + fields: [event.original] + target: zeek.dns + - script: + lang: javascript + id: zeek_dns_flags + source: > + var net = require("net"); - - from: "zeek.dns.id.orig_h" - to: "source.address" + function addDnsHeaderFlags(evt) { + var flag = evt.Get("zeek.dns.AA"); + if (flag === true) { + evt.AppendTo("dns.header_flags", "AA"); + } + flag = evt.Get("zeek.dns.TC"); + if (flag === true) { + evt.AppendTo("dns.header_flags", "TC"); + } + flag = evt.Get("zeek.dns.RD"); + if (flag === true) { + evt.AppendTo("dns.header_flags", "RD"); + } + flag = evt.Get("zeek.dns.RA"); + if (flag === true) { + evt.AppendTo("dns.header_flags", "RA"); + } + } - - from: "zeek.dns.id.orig_p" - to: "source.port" + function addDnsQuestionClass(evt) { + var qclass = evt.Get("zeek.dns.qclass"); + if (!qclass) { + return; + } + switch (qclass) { + case 1: + qclass = "IN"; + break; + case 3: + qclass = "CH"; + break; + case 4: + qclass = "HS"; + break; + case 254: + qclass = "NONE"; + break; + case 255: + qclass = "ANY"; + break; + } + evt.Put("dns.question.class", qclass); + } - - from: "zeek.dns.id.resp_h" - to: "destination.address" + function addDnsAnswers(evt) { + var answers = evt.Get("zeek.dns.answers"); + var ttls = evt.Get("zeek.dns.TTLs"); + if (!answers || !ttls || answers.length != ttls.length) { + return; + } - - from: "zeek.dns.id.resp_p" - to: "destination.port" + var resolvedIps = []; + var answersObjs = []; + for (var i = 0; i < answers.length; i++) { + var answer = answers[i]; + answersObjs.push({ + data: answer, + ttl: ttls[i], + }) + if (net.isIP(answer)) { + resolvedIps.push(answer); + } + } + evt.Put("dns.answers", answersObjs); + if (resolvedIps.length > 0) { + evt.Put("dns.resolved_ip", resolvedIps); + } + } - - from: "zeek.dns.proto" - to: "network.transport" + function addEventDuration(evt) { + var rttSec = evt.Get("zeek.dns.rtt"); + if (!rttSec) { + return; + } + evt.Put("event.duration", rttSec * 1000000000); + } - - from: "zeek.dns.uid" - to: "zeek.session_id" - + function process(evt) { + addDnsHeaderFlags(evt); + addDnsQuestionClass(evt); + addDnsAnswers(evt); + addEventDuration(evt); + } + - convert: ignore_missing: true - fail_on_error: false -{{ if .community_id }} - - community_id: + ignore_failure: true + mode: rename fields: - source_ip: source.address - destination_ip: destination.address + - {from: zeek.dns.id.orig_h, to: source.address} + - {from: zeek.dns.id.orig_p, to: source.port, type: long} + - {from: zeek.dns.id.resp_h, to: destination.address} + - {from: zeek.dns.id.resp_p, to: destination.port, type: long} + - {from: zeek.dns.uid, to: zeek.session_id} + - {from: zeek.dns.proto, to: network.transport} + - convert: + ignore_missing: true + ignore_failure: true + mode: copy + fields: + - {from: source.address, to: source.ip, type: ip} + - {from: destination.address, to: destination.ip, type: ip} + - {from: zeek.session_id, to: event.id} + - {from: '@timestamp', to: event.created} + - {from: zeek.dns.trans_id, to: dns.id} + - {from: zeek.dns.query, to: dns.question.name} + - {from: zeek.dns.qtype_name, to: dns.question.type} + - {from: zeek.dns.rcode_name, to: dns.response_code} + - registered_domain: + ignore_missing: true + ignore_failure: true + field: dns.question.name + target_field: dns.question.registered_domain +{{ if .community_id }} + - community_id: ~ {{ end }} + - timestamp: + ignore_missing: true + field: zeek.dns.ts + layouts: + - UNIX + - drop_fields: + ignore_missing: true + fields: + - zeek.dns.Z + - zeek.dns.auth + - zeek.dns.addl + - zeek.dns.ts diff --git a/x-pack/filebeat/module/zeek/dns/ingest/pipeline.json b/x-pack/filebeat/module/zeek/dns/ingest/pipeline.json deleted file mode 100644 index 8cfd06c1ca5..00000000000 --- a/x-pack/filebeat/module/zeek/dns/ingest/pipeline.json +++ /dev/null @@ -1,100 +0,0 @@ -{ - "description": "Pipeline for normalizing Zeek dns.log", - "processors": [ - { - "script": { - "lang": "painless", - "source": "ctx.event.created = ctx['@timestamp']; ctx['@timestamp'] = (long)ctx['zeek']['dns']['ts'] * 1000; ctx.zeek.dns.remove('ts');" - } - }, - { - "set": { - "field": "event.id", - "value": "{{zeek.session_id}}", - "if": "ctx.zeek.session_id != null" - } - }, - { - "set": { - "field": "source.ip", - "value": "{{source.address}}" - } - }, - { - "set": { - "field": "destination.ip", - "value": "{{destination.address}}" - } - }, - { - "geoip": { - "field": "destination.ip", - "target_field": "destination.geo" - } - }, - { - "geoip": { - "field": "source.ip", - "target_field": "source.geo" - } - }, - { - "geoip": { - "database_file": "GeoLite2-ASN.mmdb", - "field": "source.ip", - "target_field": "source.as", - "properties": [ - "asn", - "organization_name" - ], - "ignore_missing": true - } - }, - { - "geoip": { - "database_file": "GeoLite2-ASN.mmdb", - "field": "destination.ip", - "target_field": "destination.as", - "properties": [ - "asn", - "organization_name" - ], - "ignore_missing": true - } - }, - { - "rename": { - "field": "source.as.asn", - "target_field": "source.as.number", - "ignore_missing": true - } - }, - { - "rename": { - "field": "source.as.organization_name", - "target_field": "source.as.organization.name", - "ignore_missing": true - } - }, - { - "rename": { - "field": "destination.as.asn", - "target_field": "destination.as.number", - "ignore_missing": true - } - }, - { - "rename": { - "field": "destination.as.organization_name", - "target_field": "destination.as.organization.name", - "ignore_missing": true - } - } - ], - "on_failure" : [{ - "set" : { - "field" : "error.message", - "value" : "{{ _ingest.on_failure_message }}" - } - }] -} diff --git a/x-pack/filebeat/module/zeek/dns/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/dns/ingest/pipeline.yml new file mode 100644 index 00000000000..db603d93dbb --- /dev/null +++ b/x-pack/filebeat/module/zeek/dns/ingest/pipeline.yml @@ -0,0 +1,52 @@ +--- +description: Pipeline for Filebeat Zeek dns.log + +processors: + # IP Geolocation Lookup + - geoip: + field: source.ip + target_field: source.geo + ignore_missing: true + - geoip: + field: destination.ip + target_field: destination.geo + ignore_missing: true + + # IP Autonomous System (AS) Lookup + - geoip: + database_file: GeoLite2-ASN.mmdb + field: source.ip + target_field: source.as + properties: + - asn + - organization_name + ignore_missing: true + - geoip: + database_file: GeoLite2-ASN.mmdb + field: destination.ip + target_field: destination.as + properties: + - asn + - organization_name + ignore_missing: true + - rename: + field: source.as.asn + target_field: source.as.number + ignore_missing: true + - rename: + field: source.as.organization_name + target_field: source.as.organization.name + ignore_missing: true + - rename: + field: destination.as.asn + target_field: destination.as.number + ignore_missing: true + - rename: + field: destination.as.organization_name + target_field: destination.as.organization.name + ignore_missing: true + +on_failure: + - set: + field: error.message + value: "{{ _ingest.on_failure_message }}" diff --git a/x-pack/filebeat/module/zeek/dns/manifest.yml b/x-pack/filebeat/module/zeek/dns/manifest.yml index 391a2a95463..0c81ed95c2d 100644 --- a/x-pack/filebeat/module/zeek/dns/manifest.yml +++ b/x-pack/filebeat/module/zeek/dns/manifest.yml @@ -13,7 +13,7 @@ var: - name: community_id default: true -ingest_pipeline: ingest/pipeline.json +ingest_pipeline: ingest/pipeline.yml input: config/dns.yml requires.processors: diff --git a/x-pack/filebeat/module/zeek/dns/test/dns-json.log-expected.json b/x-pack/filebeat/module/zeek/dns/test/dns-json.log-expected.json index 5bfe034dc39..8d1052a27b4 100644 --- a/x-pack/filebeat/module/zeek/dns/test/dns-json.log-expected.json +++ b/x-pack/filebeat/module/zeek/dns/test/dns-json.log-expected.json @@ -1,12 +1,41 @@ [ { - "@timestamp": 1547188415000, + "@timestamp": "2019-01-11T06:33:35.857Z", "destination.address": "192.168.86.1", "destination.ip": "192.168.86.1", "destination.port": 53, + "dns.answers": [ + { + "data": "proxy-production-us-west1.gcp.cloud.es.io", + "ttl": 119 + }, + { + "data": "proxy-production-us-west1-v1-009.gcp.cloud.es.io", + "ttl": 119 + }, + { + "data": "35.199.178.4", + "ttl": 59 + } + ], + "dns.header_flags": [ + "RD", + "RA" + ], + "dns.id": 15209, + "dns.question.class": "IN", + "dns.question.name": "dd625ffb4fc54735b281862aa1cd6cd4.us-west1.gcp.cloud.es.io", + "dns.question.registered_domain": "es.io", + "dns.question.type": "A", + "dns.resolved_ip": [ + "35.199.178.4" + ], + "dns.response_code": "NOERROR", "event.dataset": "zeek.dns", + "event.duration": 76967000, "event.id": "CAcJw21BbVedgFnYH3", "event.module": "zeek", + "event.original": "{\"ts\":1547188415.857497,\"uid\":\"CAcJw21BbVedgFnYH3\",\"id.orig_h\":\"192.168.86.167\",\"id.orig_p\":38339,\"id.resp_h\":\"192.168.86.1\",\"id.resp_p\":53,\"proto\":\"udp\",\"trans_id\":15209,\"rtt\":0.076967,\"query\":\"dd625ffb4fc54735b281862aa1cd6cd4.us-west1.gcp.cloud.es.io\",\"qclass\":1,\"qclass_name\":\"C_INTERNET\",\"qtype\":1,\"qtype_name\":\"A\",\"rcode\":0,\"rcode_name\":\"NOERROR\",\"AA\":false,\"TC\":false,\"RD\":true,\"RA\":true,\"Z\":0,\"answers\":[\"proxy-production-us-west1.gcp.cloud.es.io\",\"proxy-production-us-west1-v1-009.gcp.cloud.es.io\",\"35.199.178.4\"],\"TTLs\":[119.0,119.0,59.0],\"rejected\":false}", "fileset.name": "dns", "input.type": "log", "log.offset": 0,