From bda27ea7a12fe05f45d2f2c00f6d251de418318f Mon Sep 17 00:00:00 2001 From: Dan Kortschak <90160302+efd6@users.noreply.github.com> Date: Fri, 8 Dec 2023 08:08:45 +1030 Subject: [PATCH] infoblox_nios: fix handling of messages containing view field (#8675) --- packages/infoblox_nios/changelog.yml | 5 + .../log/_dev/test/pipeline/test-dns.log | 2 + .../test/pipeline/test-dns.log-expected.json | 145 ++++++++++++++++++ .../ingest_pipeline/pipeline_dns.yml | 21 +-- .../data_stream/log/fields/fields.yml | 2 + packages/infoblox_nios/docs/README.md | 1 + packages/infoblox_nios/manifest.yml | 2 +- 7 files changed, 168 insertions(+), 10 deletions(-) diff --git a/packages/infoblox_nios/changelog.yml b/packages/infoblox_nios/changelog.yml index 4cb3a8c2844..852c40cea09 100644 --- a/packages/infoblox_nios/changelog.yml +++ b/packages/infoblox_nios/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: 1.19.1 + changes: + - description: Fix handling of messages containing view field. + type: bugfix + link: https://github.com/elastic/integrations/pull/8675 - version: 1.19.0 changes: - description: ECS version updated to 8.11.0. diff --git a/packages/infoblox_nios/data_stream/log/_dev/test/pipeline/test-dns.log b/packages/infoblox_nios/data_stream/log/_dev/test/pipeline/test-dns.log index 0947643cb1c..a941a3bf213 100644 --- a/packages/infoblox_nios/data_stream/log/_dev/test/pipeline/test-dns.log +++ b/packages/infoblox_nios/data_stream/log/_dev/test/pipeline/test-dns.log @@ -24,3 +24,5 @@ <30>Apr 14 16:16:05 10.50.1.227 named[2588]: query-errors: client @0x7f97e40eb500 192.168.1.90#64727 (ocsp.digicert.com): query failed (REFUSED) for ocsp.digicert.com/IN/A at query.c:10288 <30>Oct 4 10:18:07 a1.foo.com 89.160.20.112 named[10750]: 04-Oct-2022 10:18:07.834 client 89.160.20.128#59605: UDP: query: 89.160.20.128.a1.foo.com IN PTR response: NOERROR + 89.160.20.128.a1.foo.com. 21801 IN PTR 089.160.20.112.a1.foo.com.; <30>May 9 11:54:36 a1.foo.com 89.160.20.112 named[12261]: 09-May-2023 11:54:36.185 client 89.160.20.128#59605: view 12: UDP: query: settings-win.data.microsoft.com IN TXT response: NOERROR + settings-win.data.microsoft.com. 3600 IN TXT "k=rsa; p=abc" "def" "ghi" "jkl" "AB"; +<30>Nov 27 13:03:52 81.2.69.144 named[27014]: client @0x7f1dd4114af0 89.160.20.128#24602 (abugtera.tun.p2.42): view 1: query: abugtera.tun.p2.42 IN A + (81.2.69.144) +<30>Nov 27 11:53:09 192.168.0.1 named[15242]: client @0x7fec7c11dab0 10.4.71.204#40026 (version.bind): query: version.bind CH TXT +T (192.168.0.1) diff --git a/packages/infoblox_nios/data_stream/log/_dev/test/pipeline/test-dns.log-expected.json b/packages/infoblox_nios/data_stream/log/_dev/test/pipeline/test-dns.log-expected.json index a0fa0fd759b..26e6587b1ef 100644 --- a/packages/infoblox_nios/data_stream/log/_dev/test/pipeline/test-dns.log-expected.json +++ b/packages/infoblox_nios/data_stream/log/_dev/test/pipeline/test-dns.log-expected.json @@ -1605,6 +1605,151 @@ "tags": [ "preserve_original_event" ] + }, + { + "@timestamp": "2023-11-27T13:03:52.000Z", + "client": { + "as": { + "number": 29518, + "organization": { + "name": "Bredband2 AB" + } + }, + "domain": "abugtera.tun.p2.42", + "geo": { + "city_name": "Linköping", + "continent_name": "Europe", + "country_iso_code": "SE", + "country_name": "Sweden", + "location": { + "lat": 58.4167, + "lon": 15.6167 + }, + "region_iso_code": "SE-E", + "region_name": "Östergötland County" + }, + "ip": "89.160.20.128", + "port": 24602 + }, + "dns": { + "header_flags": [ + "RD" + ], + "question": { + "class": "IN", + "name": "abugtera.tun.p2.42", + "type": "A" + } + }, + "ecs": { + "version": "8.11.0" + }, + "event": { + "created": "2023-11-27T13:03:52.000Z", + "original": "<30>Nov 27 13:03:52 81.2.69.144 named[27014]: client @0x7f1dd4114af0 89.160.20.128#24602 (abugtera.tun.p2.42): view 1: query: abugtera.tun.p2.42 IN A + (81.2.69.144)" + }, + "host": { + "ip": [ + "81.2.69.144" + ] + }, + "infoblox_nios": { + "log": { + "dns": { + "header_flags": "+" + }, + "service_name": "named", + "type": "DNS", + "view": "1" + } + }, + "log": { + "syslog": { + "priority": 30 + } + }, + "message": "client @0x7f1dd4114af0 89.160.20.128#24602 (abugtera.tun.p2.42): view 1: query: abugtera.tun.p2.42 IN A + (81.2.69.144)", + "process": { + "pid": 27014 + }, + "related": { + "hosts": [ + "abugtera.tun.p2.42" + ], + "ip": [ + "89.160.20.128", + "81.2.69.144" + ] + }, + "server": { + "ip": "81.2.69.144" + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "@timestamp": "2023-11-27T11:53:09.000Z", + "client": { + "domain": "version.bind", + "ip": "10.4.71.204", + "port": 40026 + }, + "dns": { + "header_flags": [ + "RD" + ], + "question": { + "class": "CH", + "name": "version.bind", + "type": "TXT" + } + }, + "ecs": { + "version": "8.11.0" + }, + "event": { + "created": "2023-11-27T11:53:09.000Z", + "original": "<30>Nov 27 11:53:09 192.168.0.1 named[15242]: client @0x7fec7c11dab0 10.4.71.204#40026 (version.bind): query: version.bind CH TXT +T (192.168.0.1)" + }, + "host": { + "ip": [ + "192.168.0.1" + ] + }, + "infoblox_nios": { + "log": { + "dns": { + "header_flags": "+T" + }, + "service_name": "named", + "type": "DNS" + } + }, + "log": { + "syslog": { + "priority": 30 + } + }, + "message": "client @0x7fec7c11dab0 10.4.71.204#40026 (version.bind): query: version.bind CH TXT +T (192.168.0.1)", + "process": { + "pid": 15242 + }, + "related": { + "hosts": [ + "version.bind" + ], + "ip": [ + "10.4.71.204", + "192.168.0.1" + ] + }, + "server": { + "ip": "192.168.0.1" + }, + "tags": [ + "preserve_original_event" + ] } ] } \ No newline at end of file diff --git a/packages/infoblox_nios/data_stream/log/elasticsearch/ingest_pipeline/pipeline_dns.yml b/packages/infoblox_nios/data_stream/log/elasticsearch/ingest_pipeline/pipeline_dns.yml index 841082609b8..f9629381c69 100644 --- a/packages/infoblox_nios/data_stream/log/elasticsearch/ingest_pipeline/pipeline_dns.yml +++ b/packages/infoblox_nios/data_stream/log/elasticsearch/ingest_pipeline/pipeline_dns.yml @@ -7,17 +7,20 @@ processors: - "^zone %{DATA:dns.question.name}/%{DATA:dns.question.class}: notify from %{IP:client.ip}#%{NUMBER:client.port:long}:? %{GREEDYDATA:infoblox_nios.log.dns.message}$" - "^transfer of '%{DATA:dns.question.name}/%{DATA:dns.question.class}' from %{IP:client.ip}#%{NUMBER:client.port:long}:? %{GREEDYDATA:infoblox_nios.log.dns.message}$" - "^validating %{DATA:dns.question.name}/%{WORD:dns.question.type}: %{GREEDYDATA:infoblox_nios.log.dns.message}$" - - "^(%{NOTSPACE:infoblox_nios.log.dns.category}:)?\\s*client %{DATA} %{IP:client.ip}#%{NUMBER:client.port:long}:? updating zone '%{DATA:dns.question.name}/%{DATA:dns.question.class}': %{GREEDYDATA:infoblox_nios.log.dns.message}$" - - "^(%{NOTSPACE:infoblox_nios.log.dns.category}:)?\\s*client %{DATA} %{IP:client.ip}#%{NUMBER:client.port:long}:? \\(%{DATA:client.domain}\\): query failed %{GREEDYDATA:infoblox_nios.log.dns.message}$" - - "^(%{NOTSPACE:infoblox_nios.log.dns.category}:)?\\s*client %{DATA} %{IP:client.ip}#%{NUMBER:client.port:long}:? \\(%{DATA:infoblox_nios.log.dns.before_query}\\): rewriting query name %{DATA} to '%{DATA:infoblox_nios.log.dns.after_query}', type %{DATA:dns.question.type}$" - - "^(%{NOTSPACE:infoblox_nios.log.dns.category}:)?\\s*client %{DATA} %{IP:client.ip}#%{NUMBER:client.port:long}:? \\(%{DATA:client.domain}\\): query: %{DATA:dns.question.name} %{DATA:dns.question.class} %{WORD:dns.question.type} %{DATA:infoblox_nios.log.dns.header_flags} \\(%{IP:server.ip}\\)$" - - "^(%{NOTSPACE:infoblox_nios.log.dns.category}:)?\\s*client %{IP:client.ip}#%{NUMBER:client.port:long}:? %{DATA:network.transport}: query: %{DATA:dns.question.name} %{DATA:dns.question.class} %{WORD:dns.question.type} response: %{DATA:dns.response_code} %{DATA:infoblox_nios.log.dns.header_flags}$" - - "^(%{NOTSPACE:infoblox_nios.log.dns.category}:)?\\s*client %{DATA} %{IP:client.ip}#%{NUMBER:client.port:long}:? \\(%{DATA:client.domain}\\): transfer of '%{DATA:dns.question.name}/%{DATA:dns.question.class}': %{GREEDYDATA:infoblox_nios.log.dns.message}$" + - "^(%{NOTSPACE:infoblox_nios.log.dns.category}:)?\\s*%{CLIENT} updating zone '%{DATA:dns.question.name}/%{DATA:dns.question.class}': %{GREEDYDATA:infoblox_nios.log.dns.message}$" + - "^(%{NOTSPACE:infoblox_nios.log.dns.category}:)?\\s*%{CLIENT} \\(%{DATA:client.domain}\\): %{VIEW}?query failed %{GREEDYDATA:infoblox_nios.log.dns.message}$" + - "^(%{NOTSPACE:infoblox_nios.log.dns.category}:)?\\s*%{CLIENT} \\(%{DATA:infoblox_nios.log.dns.before_query}\\): rewriting query name %{DATA} to '%{DATA:infoblox_nios.log.dns.after_query}', type %{DATA:dns.question.type}$" + - "^(%{NOTSPACE:infoblox_nios.log.dns.category}:)?\\s*%{CLIENT} \\(%{DATA:client.domain}\\): %{VIEW}?query: %{DATA:dns.question.name} %{DATA:dns.question.class} %{WORD:dns.question.type} %{DATA:infoblox_nios.log.dns.header_flags} \\(%{IP:server.ip}\\)$" + - "^(%{NOTSPACE:infoblox_nios.log.dns.category}:)?\\s*%{CLIENT} %{DATA:network.transport}: %{VIEW}?query: %{DATA:dns.question.name} %{DATA:dns.question.class} %{WORD:dns.question.type} response: %{DATA:dns.response_code} %{DATA:infoblox_nios.log.dns.header_flags}$" + - "^(%{NOTSPACE:infoblox_nios.log.dns.category}:)?\\s*%{CLIENT} \\(%{DATA:client.domain}\\): transfer of '%{DATA:dns.question.name}/%{DATA:dns.question.class}': %{GREEDYDATA:infoblox_nios.log.dns.message}$" - "^(%{NOTSPACE:infoblox_nios.log.dns.category}:)?\\s*CEF:0\\|Infoblox\\|NIOS\\|%{GREEDYDATA:infoblox_nios.log.dns.version}\\|RPZ-%{DATA:dns.answers.type}\\|%{DATA:infoblox_nios.log.dns.answers_policy}\\|\\d+\\|app=DNS dst=%{IP:server.ip} src=%{IP:client.ip} spt=%{NUMBER:client.port:long} view=%{DATA:infoblox_nios.log.dns.view_name} qtype=%{WORD:dns.question.type} msg=%{GREEDYDATA:infoblox_nios.log.dns.message}$" - - "^(%{NOTSPACE:infoblox_nios.log.dns.category}:)?\\s*%{GREEDYDATA:timestamp} client %{IP:client.ip}#%{NUMBER:client.port:long}:? %{DATA:network.transport}: query: %{DATA:dns.question.name} %{DATA:dns.question.class} %{WORD:dns.question.type} response: %{DATA:dns.response_code} %{DATA:infoblox_nios.log.dns.header_flags} %{GREEDYDATA:repeat_message}$" - - "^(%{NOTSPACE:infoblox_nios.log.dns.category}:)?\\s*%{GREEDYDATA:timestamp} client %{IP:client.ip}#%{NUMBER:client.port:long}:? %{DATA:network.transport}: query: %{DATA:dns.question.name} %{DATA:dns.question.class} %{WORD:dns.question.type} response: %{DATA:dns.response_code} %{DATA:infoblox_nios.log.dns.header_flags}$" - - "^(%{NOTSPACE:infoblox_nios.log.dns.category}:)?\\s*client %{DATA} %{IP:client.ip}#%{NUMBER:client.port:long}:? %{GREEDYDATA:infoblox_nios.log.dns.message}$" + - "^(%{NOTSPACE:infoblox_nios.log.dns.category}:)?\\s*%{GREEDYDATA:timestamp} %{CLIENT} %{DATA:network.transport}: %{VIEW}?query: %{DATA:dns.question.name} %{DATA:dns.question.class} %{WORD:dns.question.type} response: %{DATA:dns.response_code} %{DATA:infoblox_nios.log.dns.header_flags} %{GREEDYDATA:repeat_message}$" + - "^(%{NOTSPACE:infoblox_nios.log.dns.category}:)?\\s*%{GREEDYDATA:timestamp} %{CLIENT} %{DATA:network.transport}: %{VIEW}?query: %{DATA:dns.question.name} %{DATA:dns.question.class} %{WORD:dns.question.type} response: %{DATA:dns.response_code} %{DATA:infoblox_nios.log.dns.header_flags}$" + - "^(%{NOTSPACE:infoblox_nios.log.dns.category}:)?\\s*%{CLIENT} %{GREEDYDATA:infoblox_nios.log.dns.message}$" - "^%{GREEDYDATA:infoblox_nios.log.dns.message}$" + pattern_definitions: + CLIENT: 'client (?:%{DATA} )?%{IP:client.ip}#%{NUMBER:client.port:long}:?' + VIEW: 'view %{DATA:infoblox_nios.log.view}: ' - date: field: timestamp if: ctx.timestamp != null && ctx.event?.timezone != null diff --git a/packages/infoblox_nios/data_stream/log/fields/fields.yml b/packages/infoblox_nios/data_stream/log/fields/fields.yml index 4dd6578e402..02001a06247 100644 --- a/packages/infoblox_nios/data_stream/log/fields/fields.yml +++ b/packages/infoblox_nios/data_stream/log/fields/fields.yml @@ -139,3 +139,5 @@ type: keyword - name: type type: keyword + - name: view + type: keyword diff --git a/packages/infoblox_nios/docs/README.md b/packages/infoblox_nios/docs/README.md index 1ea9781dfc6..561786d79de 100644 --- a/packages/infoblox_nios/docs/README.md +++ b/packages/infoblox_nios/docs/README.md @@ -350,6 +350,7 @@ An example event for `log` looks as following: | infoblox_nios.log.dns.view_name | | text | | infoblox_nios.log.service_name | | keyword | | infoblox_nios.log.type | | keyword | +| infoblox_nios.log.view | | keyword | | input.type | Input type | keyword | | log.file.path | Full path to the log file this event came from, including the file name. It should include the drive letter, when appropriate. If the event wasn't read from a log file, do not populate this field. | keyword | | log.offset | Log offset | long | diff --git a/packages/infoblox_nios/manifest.yml b/packages/infoblox_nios/manifest.yml index 83cbe105b00..6c14512ae65 100644 --- a/packages/infoblox_nios/manifest.yml +++ b/packages/infoblox_nios/manifest.yml @@ -1,7 +1,7 @@ format_version: "3.0.0" name: infoblox_nios title: Infoblox NIOS -version: "1.19.0" +version: "1.19.1" description: Collect logs from Infoblox NIOS with Elastic Agent. type: integration categories: