Fix timezone parsing in iptables, mssql and panw modules (#13926)

elastic · Oct 8, 2019 · 05cc502 · 05cc502
1 parent 0133b7e
commit 05cc502
Show file tree

Hide file tree

Showing 15 changed files with 861 additions and 850 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -157,6 +157,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fix conditions and error checking of date processors in ingest pipelines that use `event.timezone` to parse dates. {pull}13883[13883]
 - Fix timezone parsing of logstash module ingest pipelines. {pull}13890[13890]
 - cisco asa and ftd filesets: Fix parsing of message 106001. {issue}13891[13891] {pull}13903[13903]
+- Fix timezone parsing of iptables, mssql and panw module ingest pipelines. {pull}13926[13926]
 - Fix merging of fields specified in global scope with fields specified under an input's scope. {issue}3628[3628] {pull}13909[13909]
 - Fix delay in enforcing close_renamed and close_removed options. {issue}13488[13488] {pull}13907[13907]
 

diff --git a/x-pack/filebeat/module/iptables/log/ingest/pipeline.json b/x-pack/filebeat/module/iptables/log/ingest/pipeline.json
@@ -62,19 +62,23 @@
             },
             {
                 "date": {
+                    "if": "ctx.event.timezone == null",
                     "field": "iptables.raw_date",
-                    "ignore_failure": true,
                     "formats": [
                         "MMM  d HH:mm:ss",
                         "MMM dd HH:mm:ss"
-                    ]
+                    ],
+                    "on_failure": [{"append": {"field": "error.message", "value": "{{ _ingest.on_failure_message }}"}}]
                 }
             },
             {
                 "date": {
                     "if": "ctx.event.timezone != null",
-                    "field": "@timestamp",
-                    "formats": ["ISO8601"],
+                    "field": "iptables.raw_date",
+                    "formats": [
+                        "MMM  d HH:mm:ss",
+                        "MMM dd HH:mm:ss"
+                    ],
                     "timezone": "{{ event.timezone }}",
                     "on_failure": [{"append": {"field": "error.message", "value": "{{ _ingest.on_failure_message }}"}}]
                 }

diff --git a/x-pack/filebeat/module/iptables/log/test/geo.log-expected.json b/x-pack/filebeat/module/iptables/log/test/geo.log-expected.json
@@ -1,6 +1,6 @@
 [
     {
-        "@timestamp": "2019-10-10T05:25:12.000-02:00",
+        "@timestamp": "2019-10-10T07:25:12.000-02:00",
         "destination.ip": "10.4.0.5",
         "destination.mac": "90:10:20:76:8d:20",
         "destination.port": 443,

diff --git a/x-pack/filebeat/module/iptables/log/test/icmp.log-expected.json b/x-pack/filebeat/module/iptables/log/test/icmp.log-expected.json
@@ -1,6 +1,6 @@
 [
     {
-        "@timestamp": "2019-01-08T01:37:09.000-02:00",
+        "@timestamp": "2019-01-08T03:37:09.000-02:00",
         "destination.ip": "192.0.2.83",
         "destination.mac": "90:10:28:5f:62:24",
         "event.dataset": "iptables.log",

diff --git a/x-pack/filebeat/module/iptables/log/test/iptables.log-expected.json b/x-pack/filebeat/module/iptables/log/test/iptables.log-expected.json
@@ -1,6 +1,6 @@
 [
     {
-        "@timestamp": "2019-01-08T01:37:09.000-02:00",
+        "@timestamp": "2019-01-08T03:37:09.000-02:00",
         "destination.ip": "172.16.54.114",
         "destination.mac": "90:10:35:5a:1e:3a",
         "destination.port": 445,
@@ -35,7 +35,7 @@
         ]
     },
     {
-        "@timestamp": "2019-01-08T01:37:57.000-02:00",
+        "@timestamp": "2019-01-08T03:37:57.000-02:00",
         "destination.ip": "172.16.54.114",
         "destination.mac": "90:10:35:5a:1e:3a",
         "destination.port": 1433,
@@ -69,7 +69,7 @@
         ]
     },
     {
-        "@timestamp": "2019-01-08T01:38:45.000-02:00",
+        "@timestamp": "2019-01-08T03:38:45.000-02:00",
         "destination.ip": "172.16.54.114",
         "destination.mac": "90:10:35:5a:1e:3a",
         "destination.port": 445,
@@ -104,7 +104,7 @@
         ]
     },
     {
-        "@timestamp": "2019-01-08T01:39:25.000-02:00",
+        "@timestamp": "2019-01-08T03:39:25.000-02:00",
         "destination.ip": "172.16.54.114",
         "destination.mac": "90:10:35:5a:1e:3a",
         "destination.port": 80,
@@ -139,7 +139,7 @@
         ]
     },
     {
-        "@timestamp": "2019-01-08T01:40:21.000-02:00",
+        "@timestamp": "2019-01-08T03:40:21.000-02:00",
         "destination.ip": "172.16.54.114",
         "destination.mac": "90:10:35:5a:1e:3a",
         "destination.port": 445,
@@ -174,7 +174,7 @@
         ]
     },
     {
-        "@timestamp": "2019-01-08T01:40:25.000-02:00",
+        "@timestamp": "2019-01-08T03:40:25.000-02:00",
         "destination.ip": "172.16.54.114",
         "destination.mac": "90:10:35:5a:1e:3a",
         "destination.port": 445,
@@ -208,7 +208,7 @@
         ]
     },
     {
-        "@timestamp": "2019-01-08T01:41:17.000-02:00",
+        "@timestamp": "2019-01-08T03:41:17.000-02:00",
         "destination.ip": "172.16.54.114",
         "destination.mac": "90:10:35:5a:1e:3a",
         "destination.port": 445,
@@ -243,7 +243,7 @@
         ]
     },
     {
-        "@timestamp": "2019-01-08T01:41:23.000-02:00",
+        "@timestamp": "2019-01-08T03:41:23.000-02:00",
         "destination.ip": "172.16.54.114",
         "destination.mac": "90:10:35:5a:1e:3a",
         "destination.port": 445,
@@ -278,7 +278,7 @@
         ]
     },
     {
-        "@timestamp": "2019-01-08T01:43:18.000-02:00",
+        "@timestamp": "2019-01-08T03:43:18.000-02:00",
         "destination.ip": "172.16.54.114",
         "destination.mac": "90:10:35:5a:1e:3a",
         "destination.port": 139,
@@ -312,7 +312,7 @@
         ]
     },
     {
-        "@timestamp": "2019-01-08T01:43:42.000-02:00",
+        "@timestamp": "2019-01-08T03:43:42.000-02:00",
         "destination.ip": "172.16.54.114",
         "destination.mac": "90:10:35:5a:1e:3a",
         "destination.port": 8088,

diff --git a/x-pack/filebeat/module/iptables/log/test/ipv6.log-expected.json b/x-pack/filebeat/module/iptables/log/test/ipv6.log-expected.json
@@ -1,6 +1,6 @@
 [
     {
-        "@timestamp": "2019-01-22T07:05:05.000-02:00",
+        "@timestamp": "2019-01-22T09:05:05.000-02:00",
         "destination.ip": "2001:0db8:0000:0000:0000:0000:0000:0002",
         "event.dataset": "iptables.log",
         "event.module": "iptables",
@@ -28,7 +28,7 @@
         ]
     },
     {
-        "@timestamp": "2019-01-22T07:05:05.000-02:00",
+        "@timestamp": "2019-01-22T09:05:05.000-02:00",
         "destination.ip": "2001:0db8:0000:0000:0000:0000:0000:0002",
         "event.dataset": "iptables.log",
         "event.module": "iptables",
@@ -56,7 +56,7 @@
         ]
     },
     {
-        "@timestamp": "2019-01-22T07:05:06.000-02:00",
+        "@timestamp": "2019-01-22T09:05:06.000-02:00",
         "destination.ip": "2001:0db8:0000:0000:0000:0000:0000:0002",
         "event.dataset": "iptables.log",
         "event.module": "iptables",
@@ -84,7 +84,7 @@
         ]
     },
     {
-        "@timestamp": "2019-01-22T07:05:06.000-02:00",
+        "@timestamp": "2019-01-22T09:05:06.000-02:00",
         "destination.ip": "2001:0db8:0000:0000:0000:0000:0000:0002",
         "event.dataset": "iptables.log",
         "event.module": "iptables",
@@ -112,7 +112,7 @@
         ]
     },
     {
-        "@timestamp": "2019-01-22T07:05:07.000-02:00",
+        "@timestamp": "2019-01-22T09:05:07.000-02:00",
         "destination.ip": "2001:0db8:0000:0000:0000:0000:0000:0002",
         "event.dataset": "iptables.log",
         "event.module": "iptables",
@@ -140,7 +140,7 @@
         ]
     },
     {
-        "@timestamp": "2019-01-22T07:05:07.000-02:00",
+        "@timestamp": "2019-01-22T09:05:07.000-02:00",
         "destination.ip": "2001:0db8:0000:0000:0000:0000:0000:0002",
         "event.dataset": "iptables.log",
         "event.module": "iptables",
@@ -168,7 +168,7 @@
         ]
     },
     {
-        "@timestamp": "2019-01-22T07:05:08.000-02:00",
+        "@timestamp": "2019-01-22T09:05:08.000-02:00",
         "destination.ip": "2001:0db8:0000:0000:0000:0000:0000:0002",
         "event.dataset": "iptables.log",
         "event.module": "iptables",
@@ -196,7 +196,7 @@
         ]
     },
     {
-        "@timestamp": "2019-01-22T07:05:08.000-02:00",
+        "@timestamp": "2019-01-22T09:05:08.000-02:00",
         "destination.ip": "2001:0db8:0000:0000:0000:0000:0000:0002",
         "event.dataset": "iptables.log",
         "event.module": "iptables",
@@ -224,7 +224,7 @@
         ]
     },
     {
-        "@timestamp": "2019-01-22T07:05:09.000-02:00",
+        "@timestamp": "2019-01-22T09:05:09.000-02:00",
         "destination.ip": "2001:0db8:0000:0000:0000:0000:0000:0002",
         "event.dataset": "iptables.log",
         "event.module": "iptables",
@@ -252,7 +252,7 @@
         ]
     },
     {
-        "@timestamp": "2019-01-22T07:05:09.000-02:00",
+        "@timestamp": "2019-01-22T09:05:09.000-02:00",
         "destination.ip": "2001:0db8:0000:0000:0000:0000:0000:0002",
         "event.dataset": "iptables.log",
         "event.module": "iptables",
@@ -280,7 +280,7 @@
         ]
     },
     {
-        "@timestamp": "2019-01-22T08:52:34.000-02:00",
+        "@timestamp": "2019-01-22T10:52:34.000-02:00",
         "destination.ip": "ff02:0000:0000:0000:0000:0000:0000:0016",
         "destination.mac": "90:10:12:34:56:78",
         "event.dataset": "iptables.log",

diff --git a/x-pack/filebeat/module/iptables/log/test/ubiquiti.log-expected.json b/x-pack/filebeat/module/iptables/log/test/ubiquiti.log-expected.json
@@ -1,6 +1,6 @@
 [
     {
-        "@timestamp": "2019-01-05T18:17:05.000-02:00",
+        "@timestamp": "2019-01-05T20:17:05.000-02:00",
         "destination.ip": "255.55.174.225",
         "destination.mac": "90:10:92:6e:ea:a7",
         "destination.port": 48689,
@@ -35,7 +35,7 @@
         ]
     },
     {
-        "@timestamp": "2019-01-05T18:17:01.000-02:00",
+        "@timestamp": "2019-01-05T20:17:01.000-02:00",
         "destination.ip": "192.0.2.25",
         "destination.mac": "90:10:20:76:8d:20",
         "destination.port": 443,
@@ -73,7 +73,7 @@
         ]
     },
     {
-        "@timestamp": "2019-01-05T18:17:01.000-02:00",
+        "@timestamp": "2019-01-05T20:17:01.000-02:00",
         "destination.ip": "192.0.2.25",
         "destination.mac": "90:10:20:76:8d:20",
         "destination.port": 1443,
@@ -113,7 +113,7 @@
         ]
     },
     {
-        "@timestamp": "2019-01-05T18:17:01.000-02:00",
+        "@timestamp": "2019-01-05T20:17:01.000-02:00",
         "destination.ip": "192.0.2.25",
         "destination.mac": "90:10:20:76:8d:20",
         "destination.port": 1443,
@@ -151,7 +151,7 @@
         ]
     },
     {
-        "@timestamp": "2019-01-05T18:17:01.000-02:00",
+        "@timestamp": "2019-01-05T20:17:01.000-02:00",
         "destination.ip": "192.0.2.25",
         "destination.mac": "90:10:20:76:8d:20",
         "destination.port": 1443,

diff --git a/x-pack/filebeat/module/mssql/log/ingest/pipeline.json b/x-pack/filebeat/module/mssql/log/ingest/pipeline.json
@@ -12,17 +12,17 @@
         },
         {
             "date": {
+                "if": "ctx.event.timezone == null",
                 "field": "date",
-                "target_field": "@timestamp",
                 "formats": ["yyyy-MM-dd HH:mm:ss.SS"],
-                "ignore_failure": true
+                "on_failure": [{"append": {"field": "error.message", "value": "{{ _ingest.on_failure_message }}"}}]
             }
         },
         {
             "date": {
                 "if": "ctx.event.timezone != null",
-                "field": "@timestamp",
-                "formats": ["ISO8601"],
+                "field": "date",
+                "formats": ["yyyy-MM-dd HH:mm:ss.SS"],
                 "timezone": "{{ event.timezone }}",
                 "on_failure": [{"append": {"field": "error.message", "value": "{{ _ingest.on_failure_message }}"}}]
             }