infoblox_nios: fix handling of messages containing view field (elasti…

…c#8675)
miltonhultgren · Dec 7, 2023 · bda27ea · bda27ea
1 parent f8404d7
commit bda27ea
Show file tree

Hide file tree

Showing 7 changed files with 168 additions and 10 deletions.
diff --git a/packages/infoblox_nios/changelog.yml b/packages/infoblox_nios/changelog.yml
@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: 1.19.1
+  changes:
+    - description: Fix handling of messages containing view field.
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/8675
 - version: 1.19.0
   changes:
     - description: ECS version updated to 8.11.0.

diff --git a/packages/infoblox_nios/data_stream/log/_dev/test/pipeline/test-dns.log b/packages/infoblox_nios/data_stream/log/_dev/test/pipeline/test-dns.log
@@ -24,3 +24,5 @@
 <30>Apr 14 16:16:05 10.50.1.227 named[2588]: query-errors: client @0x7f97e40eb500 192.168.1.90#64727 (ocsp.digicert.com): query failed (REFUSED) for ocsp.digicert.com/IN/A at query.c:10288
 <30>Oct 4 10:18:07 a1.foo.com 89.160.20.112 named[10750]: 04-Oct-2022 10:18:07.834 client 89.160.20.128#59605: UDP: query: 89.160.20.128.a1.foo.com IN PTR response: NOERROR + 89.160.20.128.a1.foo.com. 21801 IN PTR 089.160.20.112.a1.foo.com.;
 <30>May 9 11:54:36 a1.foo.com 89.160.20.112 named[12261]: 09-May-2023 11:54:36.185 client 89.160.20.128#59605: view 12: UDP: query: settings-win.data.microsoft.com IN TXT response: NOERROR + settings-win.data.microsoft.com. 3600 IN TXT "k=rsa; p=abc" "def" "ghi" "jkl" "AB";
+<30>Nov 27 13:03:52 81.2.69.144 named[27014]: client @0x7f1dd4114af0 89.160.20.128#24602 (abugtera.tun.p2.42): view 1: query: abugtera.tun.p2.42 IN A + (81.2.69.144)
+<30>Nov 27 11:53:09 192.168.0.1 named[15242]: client @0x7fec7c11dab0 10.4.71.204#40026 (version.bind): query: version.bind CH TXT +T (192.168.0.1)
diff --git a/packages/infoblox_nios/data_stream/log/_dev/test/pipeline/test-dns.log-expected.json b/packages/infoblox_nios/data_stream/log/_dev/test/pipeline/test-dns.log-expected.json
@@ -1605,6 +1605,151 @@
             "tags": [
                 "preserve_original_event"
             ]
+        },
+        {
+            "@timestamp": "2023-11-27T13:03:52.000Z",
+            "client": {
+                "as": {
+                    "number": 29518,
+                    "organization": {
+                        "name": "Bredband2 AB"
+                    }
+                },
+                "domain": "abugtera.tun.p2.42",
+                "geo": {
+                    "city_name": "Linköping",
+                    "continent_name": "Europe",
+                    "country_iso_code": "SE",
+                    "country_name": "Sweden",
+                    "location": {
+                        "lat": 58.4167,
+                        "lon": 15.6167
+                    },
+                    "region_iso_code": "SE-E",
+                    "region_name": "Östergötland County"
+                },
+                "ip": "89.160.20.128",
+                "port": 24602
+            },
+            "dns": {
+                "header_flags": [
+                    "RD"
+                ],
+                "question": {
+                    "class": "IN",
+                    "name": "abugtera.tun.p2.42",
+                    "type": "A"
+                }
+            },
+            "ecs": {
+                "version": "8.11.0"
+            },
+            "event": {
+                "created": "2023-11-27T13:03:52.000Z",
+                "original": "<30>Nov 27 13:03:52 81.2.69.144 named[27014]: client @0x7f1dd4114af0 89.160.20.128#24602 (abugtera.tun.p2.42): view 1: query: abugtera.tun.p2.42 IN A + (81.2.69.144)"
+            },
+            "host": {
+                "ip": [
+                    "81.2.69.144"
+                ]
+            },
+            "infoblox_nios": {
+                "log": {
+                    "dns": {
+                        "header_flags": "+"
+                    },
+                    "service_name": "named",
+                    "type": "DNS",
+                    "view": "1"
+                }
+            },
+            "log": {
+                "syslog": {
+                    "priority": 30
+                }
+            },
+            "message": "client @0x7f1dd4114af0 89.160.20.128#24602 (abugtera.tun.p2.42): view 1: query: abugtera.tun.p2.42 IN A + (81.2.69.144)",
+            "process": {
+                "pid": 27014
+            },
+            "related": {
+                "hosts": [
+                    "abugtera.tun.p2.42"
+                ],
+                "ip": [
+                    "89.160.20.128",
+                    "81.2.69.144"
+                ]
+            },
+            "server": {
+                "ip": "81.2.69.144"
+            },
+            "tags": [
+                "preserve_original_event"
+            ]
+        },
+        {
+            "@timestamp": "2023-11-27T11:53:09.000Z",
+            "client": {
+                "domain": "version.bind",
+                "ip": "10.4.71.204",
+                "port": 40026
+            },
+            "dns": {
+                "header_flags": [
+                    "RD"
+                ],
+                "question": {
+                    "class": "CH",
+                    "name": "version.bind",
+                    "type": "TXT"
+                }
+            },
+            "ecs": {
+                "version": "8.11.0"
+            },
+            "event": {
+                "created": "2023-11-27T11:53:09.000Z",
+                "original": "<30>Nov 27 11:53:09 192.168.0.1 named[15242]: client @0x7fec7c11dab0 10.4.71.204#40026 (version.bind): query: version.bind CH TXT +T (192.168.0.1)"
+            },
+            "host": {
+                "ip": [
+                    "192.168.0.1"
+                ]
+            },
+            "infoblox_nios": {
+                "log": {
+                    "dns": {
+                        "header_flags": "+T"
+                    },
+                    "service_name": "named",
+                    "type": "DNS"
+                }
+            },
+            "log": {
+                "syslog": {
+                    "priority": 30
+                }
+            },
+            "message": "client @0x7fec7c11dab0 10.4.71.204#40026 (version.bind): query: version.bind CH TXT +T (192.168.0.1)",
+            "process": {
+                "pid": 15242
+            },
+            "related": {
+                "hosts": [
+                    "version.bind"
+                ],
+                "ip": [
+                    "10.4.71.204",
+                    "192.168.0.1"
+                ]
+            },
+            "server": {
+                "ip": "192.168.0.1"
+            },
+            "tags": [
+                "preserve_original_event"
+            ]
         }
     ]
 }
diff --git a/packages/infoblox_nios/data_stream/log/elasticsearch/ingest_pipeline/pipeline_dns.yml b/packages/infoblox_nios/data_stream/log/elasticsearch/ingest_pipeline/pipeline_dns.yml
@@ -7,17 +7,20 @@ processors:
         - "^zone %{DATA:dns.question.name}/%{DATA:dns.question.class}: notify from %{IP:client.ip}#%{NUMBER:client.port:long}:? %{GREEDYDATA:infoblox_nios.log.dns.message}$"
         - "^transfer of '%{DATA:dns.question.name}/%{DATA:dns.question.class}' from %{IP:client.ip}#%{NUMBER:client.port:long}:? %{GREEDYDATA:infoblox_nios.log.dns.message}$"
         - "^validating %{DATA:dns.question.name}/%{WORD:dns.question.type}: %{GREEDYDATA:infoblox_nios.log.dns.message}$"
-        - "^(%{NOTSPACE:infoblox_nios.log.dns.category}:)?\\s*client %{DATA} %{IP:client.ip}#%{NUMBER:client.port:long}:? updating zone '%{DATA:dns.question.name}/%{DATA:dns.question.class}': %{GREEDYDATA:infoblox_nios.log.dns.message}$"
-        - "^(%{NOTSPACE:infoblox_nios.log.dns.category}:)?\\s*client %{DATA} %{IP:client.ip}#%{NUMBER:client.port:long}:? \\(%{DATA:client.domain}\\): query failed %{GREEDYDATA:infoblox_nios.log.dns.message}$"
-        - "^(%{NOTSPACE:infoblox_nios.log.dns.category}:)?\\s*client %{DATA} %{IP:client.ip}#%{NUMBER:client.port:long}:? \\(%{DATA:infoblox_nios.log.dns.before_query}\\): rewriting query name %{DATA} to '%{DATA:infoblox_nios.log.dns.after_query}', type %{DATA:dns.question.type}$"
-        - "^(%{NOTSPACE:infoblox_nios.log.dns.category}:)?\\s*client %{DATA} %{IP:client.ip}#%{NUMBER:client.port:long}:? \\(%{DATA:client.domain}\\): query: %{DATA:dns.question.name} %{DATA:dns.question.class} %{WORD:dns.question.type} %{DATA:infoblox_nios.log.dns.header_flags} \\(%{IP:server.ip}\\)$"
-        - "^(%{NOTSPACE:infoblox_nios.log.dns.category}:)?\\s*client %{IP:client.ip}#%{NUMBER:client.port:long}:? %{DATA:network.transport}: query: %{DATA:dns.question.name} %{DATA:dns.question.class} %{WORD:dns.question.type} response: %{DATA:dns.response_code} %{DATA:infoblox_nios.log.dns.header_flags}$"
-        - "^(%{NOTSPACE:infoblox_nios.log.dns.category}:)?\\s*client %{DATA} %{IP:client.ip}#%{NUMBER:client.port:long}:? \\(%{DATA:client.domain}\\): transfer of '%{DATA:dns.question.name}/%{DATA:dns.question.class}': %{GREEDYDATA:infoblox_nios.log.dns.message}$"
+        - "^(%{NOTSPACE:infoblox_nios.log.dns.category}:)?\\s*%{CLIENT} updating zone '%{DATA:dns.question.name}/%{DATA:dns.question.class}': %{GREEDYDATA:infoblox_nios.log.dns.message}$"
+        - "^(%{NOTSPACE:infoblox_nios.log.dns.category}:)?\\s*%{CLIENT} \\(%{DATA:client.domain}\\): %{VIEW}?query failed %{GREEDYDATA:infoblox_nios.log.dns.message}$"
+        - "^(%{NOTSPACE:infoblox_nios.log.dns.category}:)?\\s*%{CLIENT} \\(%{DATA:infoblox_nios.log.dns.before_query}\\): rewriting query name %{DATA} to '%{DATA:infoblox_nios.log.dns.after_query}', type %{DATA:dns.question.type}$"
+        - "^(%{NOTSPACE:infoblox_nios.log.dns.category}:)?\\s*%{CLIENT} \\(%{DATA:client.domain}\\): %{VIEW}?query: %{DATA:dns.question.name} %{DATA:dns.question.class} %{WORD:dns.question.type} %{DATA:infoblox_nios.log.dns.header_flags} \\(%{IP:server.ip}\\)$"
+        - "^(%{NOTSPACE:infoblox_nios.log.dns.category}:)?\\s*%{CLIENT} %{DATA:network.transport}: %{VIEW}?query: %{DATA:dns.question.name} %{DATA:dns.question.class} %{WORD:dns.question.type} response: %{DATA:dns.response_code} %{DATA:infoblox_nios.log.dns.header_flags}$"
+        - "^(%{NOTSPACE:infoblox_nios.log.dns.category}:)?\\s*%{CLIENT} \\(%{DATA:client.domain}\\): transfer of '%{DATA:dns.question.name}/%{DATA:dns.question.class}': %{GREEDYDATA:infoblox_nios.log.dns.message}$"
         - "^(%{NOTSPACE:infoblox_nios.log.dns.category}:)?\\s*CEF:0\\|Infoblox\\|NIOS\\|%{GREEDYDATA:infoblox_nios.log.dns.version}\\|RPZ-%{DATA:dns.answers.type}\\|%{DATA:infoblox_nios.log.dns.answers_policy}\\|\\d+\\|app=DNS dst=%{IP:server.ip} src=%{IP:client.ip} spt=%{NUMBER:client.port:long} view=%{DATA:infoblox_nios.log.dns.view_name} qtype=%{WORD:dns.question.type} msg=%{GREEDYDATA:infoblox_nios.log.dns.message}$"
-        - "^(%{NOTSPACE:infoblox_nios.log.dns.category}:)?\\s*%{GREEDYDATA:timestamp} client %{IP:client.ip}#%{NUMBER:client.port:long}:? %{DATA:network.transport}: query: %{DATA:dns.question.name} %{DATA:dns.question.class} %{WORD:dns.question.type} response: %{DATA:dns.response_code} %{DATA:infoblox_nios.log.dns.header_flags} %{GREEDYDATA:repeat_message}$"
-        - "^(%{NOTSPACE:infoblox_nios.log.dns.category}:)?\\s*%{GREEDYDATA:timestamp} client %{IP:client.ip}#%{NUMBER:client.port:long}:? %{DATA:network.transport}: query: %{DATA:dns.question.name} %{DATA:dns.question.class} %{WORD:dns.question.type} response: %{DATA:dns.response_code} %{DATA:infoblox_nios.log.dns.header_flags}$"
-        - "^(%{NOTSPACE:infoblox_nios.log.dns.category}:)?\\s*client %{DATA} %{IP:client.ip}#%{NUMBER:client.port:long}:? %{GREEDYDATA:infoblox_nios.log.dns.message}$"
+        - "^(%{NOTSPACE:infoblox_nios.log.dns.category}:)?\\s*%{GREEDYDATA:timestamp} %{CLIENT} %{DATA:network.transport}: %{VIEW}?query: %{DATA:dns.question.name} %{DATA:dns.question.class} %{WORD:dns.question.type} response: %{DATA:dns.response_code} %{DATA:infoblox_nios.log.dns.header_flags} %{GREEDYDATA:repeat_message}$"
+        - "^(%{NOTSPACE:infoblox_nios.log.dns.category}:)?\\s*%{GREEDYDATA:timestamp} %{CLIENT} %{DATA:network.transport}: %{VIEW}?query: %{DATA:dns.question.name} %{DATA:dns.question.class} %{WORD:dns.question.type} response: %{DATA:dns.response_code} %{DATA:infoblox_nios.log.dns.header_flags}$"
+        - "^(%{NOTSPACE:infoblox_nios.log.dns.category}:)?\\s*%{CLIENT} %{GREEDYDATA:infoblox_nios.log.dns.message}$"
         - "^%{GREEDYDATA:infoblox_nios.log.dns.message}$"
+      pattern_definitions:
+        CLIENT: 'client (?:%{DATA} )?%{IP:client.ip}#%{NUMBER:client.port:long}:?'
+        VIEW: 'view %{DATA:infoblox_nios.log.view}: '
   - date:
       field: timestamp
       if: ctx.timestamp != null && ctx.event?.timezone != null

diff --git a/packages/infoblox_nios/data_stream/log/fields/fields.yml b/packages/infoblox_nios/data_stream/log/fields/fields.yml
@@ -139,3 +139,5 @@
       type: keyword
     - name: type
       type: keyword
+    - name: view
+      type: keyword
diff --git a/packages/infoblox_nios/docs/README.md b/packages/infoblox_nios/docs/README.md
@@ -350,6 +350,7 @@ An example event for `log` looks as following:
 | infoblox_nios.log.dns.view_name |  | text |
 | infoblox_nios.log.service_name |  | keyword |
 | infoblox_nios.log.type |  | keyword |
+| infoblox_nios.log.view |  | keyword |
 | input.type | Input type | keyword |
 | log.file.path | Full path to the log file this event came from, including the file name. It should include the drive letter, when appropriate. If the event wasn't read from a log file, do not populate this field. | keyword |
 | log.offset | Log offset | long |

diff --git a/packages/infoblox_nios/manifest.yml b/packages/infoblox_nios/manifest.yml
@@ -1,7 +1,7 @@
 format_version: "3.0.0"
 name: infoblox_nios
 title: Infoblox NIOS
-version: "1.19.0"
+version: "1.19.1"
 description: Collect logs from Infoblox NIOS with Elastic Agent.
 type: integration
 categories: