[7.6] Add pattern to handle logs from newer ES versions (#17406) (#17423

)

* Add pattern to handle logs from newer ES versions (#17406)

* Add pattern to handle logs from newer ES versions

* Adding CHANGELOG entry

* Removing fields not available in 7.6

CHANGELOG.next.asciidoc

@@ -50,6 +50,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fixed a mapping exception when ingesting CEF logs that used the spriv or dpriv extensions. {issue}17216[17216] {pull}17220[17220]
 - Fix `elasticsearch.gc` fileset to not collect _all_ logs when Elasticsearch is running in Docker. {issue}13164[13164] {issue}16583[16583] {pull}17164[17164]
 - Fixed a mapping exception when ingesting Logstash plain logs (7.4+) with pipeline ids containing non alphanumeric chars. {issue}17242[17242] {pull}17243[17243]
+- Fix `elasticsearch.audit` data ingest pipeline to be more forgiving with date formats found in Elasticsearch audit logs. {pull}17406[17406]
 
 *Heartbeat*
 

filebeat/module/elasticsearch/audit/ingest/pipeline-json.json

@@ -24,7 +24,8 @@
                 "field": "elasticsearch.audit.@timestamp",
                 "target_field": "elasticsearch.audit.@timestamp",
                 "formats": [
-                    "yyyy-MM-dd'T'HH:mm:ss,SSS"
+                    "yyyy-MM-dd'T'HH:mm:ss,SSS",
+                    "yyyy-MM-dd'T'HH:mm:ss,SSSZ"
                 ],
                 "timezone": "{{ event.timezone }}"
             }

filebeat/module/elasticsearch/audit/test/test-audit-761.log

@@ -0,0 +1 @@
+{"@timestamp":"2020-04-01T11:21:06,725+0200", "node.id":"vvj136QVQ2Ci2aXmrhyi3Q", "event.type":"transport", "event.action":"access_granted", "user.name":"logstash_manager", "user.realm":"native1", "user.roles":["logstash_admin","cluster_monitor"], "origin.type":"rest", "origin.address":"10.54.25.111:52148", "request.id":"rLBMfPM2Q9q-DQEB_g30ww", "action":"indices:data/read/mget[shard]", "request.name":"MultiGetShardRequest", "indices":[".logstash",".logstash",".logstash",".logstash",".logstash",".logstash",".logstash",".logstash"]}

filebeat/module/elasticsearch/audit/test/test-audit-761.log-expected.json

@@ -0,0 +1,39 @@
+[
+    {
+        "@timestamp": "2020-04-01T13:21:06.725Z",
+        "elasticsearch.audit.action": "indices:data/read/mget[shard]",
+        "elasticsearch.audit.indices": [
+            ".logstash",
+            ".logstash",
+            ".logstash",
+            ".logstash",
+            ".logstash",
+            ".logstash",
+            ".logstash",
+            ".logstash"
+        ],
+        "elasticsearch.audit.layer": "transport",
+        "elasticsearch.audit.origin.type": "rest",
+        "elasticsearch.audit.request.id": "rLBMfPM2Q9q-DQEB_g30ww",
+        "elasticsearch.audit.request.name": "MultiGetShardRequest",
+        "elasticsearch.audit.user.realm": "native1",
+        "elasticsearch.audit.user.roles": [
+            "logstash_admin",
+            "cluster_monitor"
+        ],
+        "elasticsearch.node.id": "vvj136QVQ2Ci2aXmrhyi3Q",
+        "event.action": "access_granted",
+        "event.dataset": "elasticsearch.audit",
+        "event.module": "elasticsearch",
+        "event.timezone": "-02:00",
+        "fileset.name": "audit",
+        "input.type": "log",
+        "log.offset": 0,
+        "message": "{\"@timestamp\":\"2020-04-01T11:21:06,725+0200\", \"node.id\":\"vvj136QVQ2Ci2aXmrhyi3Q\", \"event.type\":\"transport\", \"event.action\":\"access_granted\", \"user.name\":\"logstash_manager\", \"user.realm\":\"native1\", \"user.roles\":[\"logstash_admin\",\"cluster_monitor\"], \"origin.type\":\"rest\", \"origin.address\":\"10.54.25.111:52148\", \"request.id\":\"rLBMfPM2Q9q-DQEB_g30ww\", \"action\":\"indices:data/read/mget[shard]\", \"request.name\":\"MultiGetShardRequest\", \"indices\":[\".logstash\",\".logstash\",\".logstash\",\".logstash\",\".logstash\",\".logstash\",\".logstash\",\".logstash\"]}",
+        "service.type": "elasticsearch",
+        "source.address": "10.54.25.111:52148",
+        "source.ip": "10.54.25.111",
+        "source.port": 52148,
+        "user.name": "logstash_manager"
+    }
+]