From 0dcf9e8ccf54292a064f6d9b4c57436b8fd15fb3 Mon Sep 17 00:00:00 2001 From: Marius Iversen Date: Tue, 1 Jun 2021 19:34:31 +0200 Subject: [PATCH 1/4] update nats ECS version and add event.original options --- .../pipeline/test-log-sample.log-config.yml | 2 + .../test-log-sample.log-expected.json | 306 ++++++++------ .../data_stream/log/agent/stream/log.yml.hbs | 14 +- .../elasticsearch/ingest_pipeline/default.yml | 396 +++++++++--------- packages/nats/data_stream/log/fields/ecs.yml | 6 + packages/nats/data_stream/log/manifest.yml | 16 + packages/nats/docs/README.md | 1 + 7 files changed, 425 insertions(+), 316 deletions(-) diff --git a/packages/nats/data_stream/log/_dev/test/pipeline/test-log-sample.log-config.yml b/packages/nats/data_stream/log/_dev/test/pipeline/test-log-sample.log-config.yml index 1ecfca2ec4d..074df9e6338 100644 --- a/packages/nats/data_stream/log/_dev/test/pipeline/test-log-sample.log-config.yml +++ b/packages/nats/data_stream/log/_dev/test/pipeline/test-log-sample.log-config.yml @@ -2,5 +2,7 @@ fields: "@timestamp": "2020-04-28T11:07:58.223Z" ecs: version: "1.9.0" + tags: + - preserve_original_event dynamic_fields: event.ingested: ".*" diff --git a/packages/nats/data_stream/log/_dev/test/pipeline/test-log-sample.log-expected.json b/packages/nats/data_stream/log/_dev/test/pipeline/test-log-sample.log-expected.json index d3228f03d37..6fed9d0b69c 100644 --- a/packages/nats/data_stream/log/_dev/test/pipeline/test-log-sample.log-expected.json +++ b/packages/nats/data_stream/log/_dev/test/pipeline/test-log-sample.log-expected.json @@ -11,20 +11,24 @@ }, "@timestamp": "2019-02-06T07:19:40.624Z", "ecs": { - "version": "1.9.0" + "version": "1.10.0" }, "log": { "level": "info" }, "event": { - "ingested": "2021-04-23T12:54:11.211749185Z", + "ingested": "2021-06-01T17:33:56.028520100Z", + "original": "[1] 2019/02/06 07:19:40.624334 [INF] Starting nats-server version 1.3.0", "type": [ "info" ], "created": "2020-04-28T11:07:58.223Z", "kind": "event" }, - "message": "Starting nats-server version 1.3.0" + "message": "Starting nats-server version 1.3.0", + "tags": [ + "preserve_original_event" + ] }, { "nats": { @@ -37,20 +41,24 @@ }, "@timestamp": "2019-02-06T07:19:40.624Z", "ecs": { - "version": "1.9.0" + "version": "1.10.0" }, "log": { "level": "info" }, "event": { - "ingested": "2021-04-23T12:54:11.211755386Z", + "ingested": "2021-06-01T17:33:56.028546700Z", + "original": "[1] 2019/02/06 07:19:40.624547 [INF] Git commit [eed4fbc]", "type": [ "info" ], "created": "2020-04-28T11:07:58.223Z", "kind": "event" }, - "message": "Git commit [eed4fbc]" + "message": "Git commit [eed4fbc]", + "tags": [ + "preserve_original_event" + ] }, { "nats": { @@ -63,20 +71,24 @@ }, "@timestamp": "2019-02-06T07:19:40.624Z", "ecs": { - "version": "1.9.0" + "version": "1.10.0" }, "log": { "level": "info" }, "event": { - "ingested": "2021-04-23T12:54:11.211757374Z", + "ingested": "2021-06-01T17:33:56.028554600Z", + "original": "[1] 2019/02/06 07:19:40.624674 [INF] Listening for client connections on 0.0.0.0:4222", "type": [ "info" ], "created": "2020-04-28T11:07:58.223Z", "kind": "event" }, - "message": "Listening for client connections on 0.0.0.0:4222" + "message": "Listening for client connections on 0.0.0.0:4222", + "tags": [ + "preserve_original_event" + ] }, { "nats": { @@ -89,20 +101,24 @@ }, "@timestamp": "2019-02-06T07:19:40.624Z", "ecs": { - "version": "1.9.0" + "version": "1.10.0" }, "log": { "level": "info" }, "event": { - "ingested": "2021-04-23T12:54:11.211759010Z", + "ingested": "2021-06-01T17:33:56.028582700Z", + "original": "[1] 2019/02/06 07:19:40.624690 [INF] Server is ready", "type": [ "info" ], "created": "2020-04-28T11:07:58.223Z", "kind": "event" }, - "message": "Server is ready" + "message": "Server is ready", + "tags": [ + "preserve_original_event" + ] }, { "nats": { @@ -116,31 +132,35 @@ "process": { "pid": 1 }, + "log": { + "level": "debug" + }, + "message": "Client connection created", + "tags": [ + "preserve_original_event" + ], "@timestamp": "2019-02-06T07:20:08.508Z", "ecs": { - "version": "1.9.0" + "version": "1.10.0" }, "related": { "ip": [ "172.18.0.1" ] }, - "log": { - "level": "debug" - }, "client": { "port": 38630, "ip": "172.18.0.1" }, "event": { - "ingested": "2021-04-23T12:54:11.211760707Z", + "ingested": "2021-06-01T17:33:56.028589800Z", + "original": "[1] 2019/02/06 07:20:08.508891 [DBG] 172.18.0.1:38630 - cid:1 - Client connection created", "type": [ "info" ], "created": "2020-04-28T11:07:58.223Z", "kind": "event" - }, - "message": "Client connection created" + } }, { "nats": { @@ -160,12 +180,15 @@ "level": "trace" }, "message": "{\"verbose\":false,\"pedantic\":false,\"tls_required\":false,\"name\":\"NATS Benchmark\",\"lang\":\"go\",\"version\":\"1.7.0\",\"protocol\":1,\"echo\":true}", + "tags": [ + "preserve_original_event" + ], "network": { "direction": "outbound" }, "@timestamp": "2019-02-06T07:20:08.510Z", "ecs": { - "version": "1.9.0" + "version": "1.10.0" }, "related": { "ip": [ @@ -177,7 +200,8 @@ "ip": "172.18.0.1" }, "event": { - "ingested": "2021-04-23T12:54:11.211762349Z", + "ingested": "2021-06-01T17:33:56.028596400Z", + "original": "[1] 2019/02/06 07:20:08.510296 [TRC] 172.18.0.1:38630 - cid:1 - -\u003e\u003e [CONNECT {\"verbose\":false,\"pedantic\":false,\"tls_required\":false,\"name\":\"NATS Benchmark\",\"lang\":\"go\",\"version\":\"1.7.0\",\"protocol\":1,\"echo\":true}]", "type": [ "info" ], @@ -201,32 +225,36 @@ "process": { "pid": 1 }, + "log": { + "level": "trace" + }, + "tags": [ + "preserve_original_event" + ], + "network": { + "direction": "outbound" + }, "@timestamp": "2019-02-06T07:20:08.512Z", "ecs": { - "version": "1.9.0" + "version": "1.10.0" }, "related": { "ip": [ "172.18.0.1" ] }, - "log": { - "level": "trace" - }, "client": { "port": 38630, "ip": "172.18.0.1" }, "event": { - "ingested": "2021-04-23T12:54:11.211764111Z", + "ingested": "2021-06-01T17:33:56.028603Z", + "original": "[1] 2019/02/06 07:20:08.512052 [TRC] 172.18.0.1:38630 - cid:1 - -\u003e\u003e [SUB foo 1]", "type": [ "info" ], "created": "2020-04-28T11:07:58.223Z", "kind": "event" - }, - "network": { - "direction": "outbound" } }, { @@ -243,32 +271,36 @@ "process": { "pid": 1 }, + "log": { + "level": "trace" + }, + "tags": [ + "preserve_original_event" + ], + "network": { + "direction": "outbound" + }, "@timestamp": "2019-02-06T07:20:08.512Z", "ecs": { - "version": "1.9.0" + "version": "1.10.0" }, "related": { "ip": [ "172.18.0.1" ] }, - "log": { - "level": "trace" - }, "client": { "port": 38630, "ip": "172.18.0.1" }, "event": { - "ingested": "2021-04-23T12:54:11.211765728Z", + "ingested": "2021-06-01T17:33:56.028609Z", + "original": "[1] 2019/02/06 07:20:08.512128 [TRC] 172.18.0.1:38630 - cid:1 - -\u003e\u003e [PING]", "type": [ "info" ], "created": "2020-04-28T11:07:58.223Z", "kind": "event" - }, - "network": { - "direction": "outbound" } }, { @@ -285,32 +317,36 @@ "process": { "pid": 1 }, + "log": { + "level": "trace" + }, + "tags": [ + "preserve_original_event" + ], + "network": { + "direction": "inbound" + }, "@timestamp": "2019-02-06T07:20:08.512Z", "ecs": { - "version": "1.9.0" + "version": "1.10.0" }, "related": { "ip": [ "172.18.0.1" ] }, - "log": { - "level": "trace" - }, "client": { "port": 38630, "ip": "172.18.0.1" }, "event": { - "ingested": "2021-04-23T12:54:11.211767288Z", + "ingested": "2021-06-01T17:33:56.028614500Z", + "original": "[1] 2019/02/06 07:20:08.512153 [TRC] 172.18.0.1:38630 - cid:1 - \u003c\u003c- [PONG]", "type": [ "info" ], "created": "2020-04-28T11:07:58.223Z", "kind": "event" - }, - "network": { - "direction": "inbound" } }, { @@ -330,32 +366,36 @@ "process": { "pid": 1 }, + "log": { + "level": "trace" + }, + "tags": [ + "preserve_original_event" + ], + "network": { + "direction": "outbound" + }, "@timestamp": "2019-02-04T15:40:02.717Z", "ecs": { - "version": "1.9.0" + "version": "1.10.0" }, "related": { "ip": [ "50.39.246.116" ] }, - "log": { - "level": "trace" - }, "client": { "port": 62388, "ip": "50.39.246.116" }, "event": { - "ingested": "2021-04-23T12:54:11.211768835Z", + "ingested": "2021-06-01T17:33:56.028619800Z", + "original": "[1] 2019/02/04 15:40:02.717819 [TRC] 50.39.246.116:62388 - cid:3 - -\u003e\u003e [PUB aiuser.platinum1.pingpeer _INBOX.e3hAUbP4r5wbjw3Hudw42r.udigGiHn 20]", "type": [ "info" ], "created": "2020-04-28T11:07:58.223Z", "kind": "event" - }, - "network": { - "direction": "outbound" } }, { @@ -372,32 +412,36 @@ "process": { "pid": 1 }, + "log": { + "level": "trace" + }, + "tags": [ + "preserve_original_event" + ], + "network": { + "direction": "outbound" + }, "@timestamp": "2019-02-04T15:40:02.717Z", "ecs": { - "version": "1.9.0" + "version": "1.10.0" }, "related": { "ip": [ "50.39.246.116" ] }, - "log": { - "level": "trace" - }, "client": { "port": 62388, "ip": "50.39.246.116" }, "event": { - "ingested": "2021-04-23T12:54:11.211770442Z", + "ingested": "2021-06-01T17:33:56.028626200Z", + "original": "[1] 2019/02/04 15:40:02.717825 [TRC] 50.39.246.116:62388 - cid:3 - -\u003e\u003e MSG_PAYLOAD: [peer, are you alive?]", "type": [ "info" ], "created": "2020-04-28T11:07:58.223Z", "kind": "event" - }, - "network": { - "direction": "outbound" } }, { @@ -418,32 +462,36 @@ "process": { "pid": 1 }, + "log": { + "level": "trace" + }, + "tags": [ + "preserve_original_event" + ], + "network": { + "direction": "inbound" + }, "@timestamp": "2019-02-04T15:40:02.717Z", "ecs": { - "version": "1.9.0" + "version": "1.10.0" }, "related": { "ip": [ "192.168.176.11" ] }, - "log": { - "level": "trace" - }, "client": { "port": 36262, "ip": "192.168.176.11" }, "event": { - "ingested": "2021-04-23T12:54:11.211772364Z", + "ingested": "2021-06-01T17:33:56.028631500Z", + "original": "[1] 2019/02/04 15:40:02.717832 [TRC] 192.168.176.11:36262 - cid:4 - \u003c\u003c- [MSG aiuser.platinum1.pingpeer 1 _INBOX.e3hAUbP4r5wbjw3Hudw42r.udigGiHn 20]", "type": [ "info" ], "created": "2020-04-28T11:07:58.223Z", "kind": "event" - }, - "network": { - "direction": "inbound" } }, { @@ -462,32 +510,36 @@ "process": { "pid": 1 }, + "log": { + "level": "trace" + }, + "tags": [ + "preserve_original_event" + ], + "network": { + "direction": "outbound" + }, "@timestamp": "2019-02-04T15:40:02.718Z", "ecs": { - "version": "1.9.0" + "version": "1.10.0" }, "related": { "ip": [ "192.168.176.11" ] }, - "log": { - "level": "trace" - }, "client": { "port": 36262, "ip": "192.168.176.11" }, "event": { - "ingested": "2021-04-23T12:54:11.211773909Z", + "ingested": "2021-06-01T17:33:56.028636700Z", + "original": "[1] 2019/02/04 15:40:02.718007 [TRC] 192.168.176.11:36262 - cid:4 - -\u003e\u003e [PUB _INBOX.e3hAUbP4r5wbjw3Hudw42r.udigGiHn 17]", "type": [ "info" ], "created": "2020-04-28T11:07:58.223Z", "kind": "event" - }, - "network": { - "direction": "outbound" } }, { @@ -504,32 +556,36 @@ "process": { "pid": 1 }, + "log": { + "level": "trace" + }, + "tags": [ + "preserve_original_event" + ], + "network": { + "direction": "outbound" + }, "@timestamp": "2019-02-04T15:40:02.718Z", "ecs": { - "version": "1.9.0" + "version": "1.10.0" }, "related": { "ip": [ "192.168.176.11" ] }, - "log": { - "level": "trace" - }, "client": { "port": 36262, "ip": "192.168.176.11" }, "event": { - "ingested": "2021-04-23T12:54:11.211775453Z", + "ingested": "2021-06-01T17:33:56.028641800Z", + "original": "[1] 2019/02/04 15:40:02.718023 [TRC] 192.168.176.11:36262 - cid:4 - -\u003e\u003e MSG_PAYLOAD: [I am fine, agent!]", "type": [ "info" ], "created": "2020-04-28T11:07:58.223Z", "kind": "event" - }, - "network": { - "direction": "outbound" } }, { @@ -549,32 +605,36 @@ "process": { "pid": 1 }, + "log": { + "level": "trace" + }, + "tags": [ + "preserve_original_event" + ], + "network": { + "direction": "inbound" + }, "@timestamp": "2019-02-04T15:40:02.718Z", "ecs": { - "version": "1.9.0" + "version": "1.10.0" }, "related": { "ip": [ "50.39.246.116" ] }, - "log": { - "level": "trace" - }, "client": { "port": 62388, "ip": "50.39.246.116" }, "event": { - "ingested": "2021-04-23T12:54:11.211776985Z", + "ingested": "2021-06-01T17:33:56.028647Z", + "original": "[1] 2019/02/04 15:40:02.718044 [TRC] 50.39.246.116:62388 - cid:3 - \u003c\u003c- [MSG _INBOX.e3hAUbP4r5wbjw3Hudw42r.udigGiHn 11 17]", "type": [ "info" ], "created": "2020-04-28T11:07:58.223Z", "kind": "event" - }, - "network": { - "direction": "inbound" } }, { @@ -593,32 +653,36 @@ "process": { "pid": 1 }, + "log": { + "level": "trace" + }, + "tags": [ + "preserve_original_event" + ], + "network": { + "direction": "outbound" + }, "@timestamp": "2019-02-04T15:40:02.717Z", "ecs": { - "version": "1.9.0" + "version": "1.10.0" }, "related": { "ip": [ "50.39.246.116" ] }, - "log": { - "level": "trace" - }, "client": { "port": 62388, "ip": "50.39.246.116" }, "event": { - "ingested": "2021-04-23T12:54:11.211778556Z", + "ingested": "2021-06-01T17:33:56.028652200Z", + "original": "[1] 2019/02/04 15:40:02.717600 [TRC] 50.39.246.116:62388 - cid:3 - -\u003e\u003e [PUB aiuser.platinum1.appstats 1583]", "type": [ "info" ], "created": "2020-04-28T11:07:58.223Z", "kind": "event" - }, - "network": { - "direction": "outbound" } }, { @@ -638,32 +702,36 @@ "process": { "pid": 1 }, + "log": { + "level": "trace" + }, + "tags": [ + "preserve_original_event" + ], + "network": { + "direction": "inbound" + }, "@timestamp": "2019-02-04T15:40:02.717Z", "ecs": { - "version": "1.9.0" + "version": "1.10.0" }, "related": { "ip": [ "192.168.176.11" ] }, - "log": { - "level": "trace" - }, "client": { "port": 36262, "ip": "192.168.176.11" }, "event": { - "ingested": "2021-04-23T12:54:11.211780553Z", + "ingested": "2021-06-01T17:33:56.028657400Z", + "original": "[1] 2019/02/04 15:40:02.717811 [TRC] 192.168.176.11:36262 - cid:4 - \u003c\u003c- [MSG aiuser.platinum1.appstats 6 1583]", "type": [ "info" ], "created": "2020-04-28T11:07:58.223Z", "kind": "event" - }, - "network": { - "direction": "inbound" } }, { @@ -680,32 +748,36 @@ "process": { "pid": 1 }, + "log": { + "level": "trace" + }, + "tags": [ + "preserve_original_event" + ], + "network": { + "direction": "inbound" + }, "@timestamp": "2019-02-16T07:20:08.512Z", "ecs": { - "version": "1.9.0" + "version": "1.10.0" }, "related": { "ip": [ "172.18.0.1" ] }, - "log": { - "level": "trace" - }, "client": { "port": 38630, "ip": "172.18.0.1" }, "event": { - "ingested": "2021-04-23T12:54:11.211782830Z", + "ingested": "2021-06-01T17:33:56.028669400Z", + "original": "[1] 2019/02/16 07:20:08.512153 [TRC] 172.18.0.1:38630 - cid:1 - \u003c\u003c- [OK]", "type": [ "info" ], "created": "2020-04-28T11:07:58.223Z", "kind": "event" - }, - "network": { - "direction": "inbound" } } ] diff --git a/packages/nats/data_stream/log/agent/stream/log.yml.hbs b/packages/nats/data_stream/log/agent/stream/log.yml.hbs index 3d44dac9891..69d278b8ff4 100644 --- a/packages/nats/data_stream/log/agent/stream/log.yml.hbs +++ b/packages/nats/data_stream/log/agent/stream/log.yml.hbs @@ -2,9 +2,11 @@ paths: {{#each paths as |path i|}} - {{path}} {{/each}} -exclude_files: [".gz$"] -processors: - - add_fields: - target: '' - fields: - ecs.version: 1.9.0 +tags: +{{#if preserve_original_event}} + - preserve_original_event +{{/if}} +{{#each tags as |tag i|}} + - {{tag}} +{{/each}} +exclude_files: [".gz$"] \ No newline at end of file diff --git a/packages/nats/data_stream/log/elasticsearch/ingest_pipeline/default.yml b/packages/nats/data_stream/log/elasticsearch/ingest_pipeline/default.yml index 7685d33c6da..416e83f78de 100644 --- a/packages/nats/data_stream/log/elasticsearch/ingest_pipeline/default.yml +++ b/packages/nats/data_stream/log/elasticsearch/ingest_pipeline/default.yml @@ -1,197 +1,207 @@ --- description: Pipeline for parsing nats log logs processors: -- set: - field: event.ingested - value: '{{_ingest.timestamp}}' -- grok: - field: message - patterns: - - \[%{POSINT:process.pid}\]( %{NATSTIME:nats.log.timestamp})? \[%{NATSLOGLEVEL:log.level}\] - %{GREEDYDATA:nats.log.info} - pattern_definitions: - NATSTIME: '%{YEAR}/%{MONTHNUM}/%{MONTHDAY} %{TIME}' - NATSLOGLEVEL: (INF|DBG|WRN|ERR|FTL|TRC) - ignore_missing: true -- grok: - field: nats.log.info - patterns: - - '%{IPV4:client.ip}:%{POSINT:client.port} - cid:%{POSINT:nats.log.client.id} - - %{GREEDYDATA:nats.log.msg.info}' - - '%{GREEDYDATA:nats.log.msg.data}' - ignore_missing: true -- grok: - field: nats.log.msg.info - patterns: - - '%{NATSDIRECTION:network.direction} %{NATSPAYLOAD:nats.log.msg.type}: \[%{GREEDYDATA:nats.log.msg.payload}\]' - - '%{NATSDIRECTION:network.direction} \[%{NATSNOINFO:nats.log.msg.type}\]' - - '%{NATSDIRECTION:network.direction} \[%{NATSUNSUB:nats.log.msg.type}\s+%{POSINT:nats.log.msg.sid}(\s+%{POSINT:nats.log.msg.max_messages})?\]' - - '%{NATSDIRECTION:network.direction} \[%{NATSPUB:nats.log.msg.type}\s+%{NOTSPACE:nats.log.msg.subject}(\s+%{NOTSPACE:nats.log.msg.reply_to})?\s+%{POSINT:nats.log.msg.bytes}\]' - - '%{NATSDIRECTION:network.direction} \[%{NATSSUB:nats.log.msg.type}\s+%{NOTSPACE:nats.log.msg.subject}(\s+%{NOTSPACE:nats.log.msg.queue_group})?\s+%{POSINT:nats.log.msg.sid}\]' - - '%{NATSDIRECTION:network.direction} \[%{NATSMSG:nats.log.msg.type}\s+%{NOTSPACE:nats.log.msg.subject}\s+%{POSINT:nats.log.msg.sid}(\s+%{NOTSPACE:nats.log.msg.reply_to})?\s+%{POSINT:nats.log.msg.bytes}\]' - - '%{NATSDIRECTION:network.direction} \[%{NATSCONNECTION:nats.log.msg.type}\s+%{GREEDYDATA:nats.log.msg.data}\]' - - '%{NATSDIRECTION:network.direction} \[%{NATSERROR:nats.log.msg.type}\s+%{GREEDYDATA:nats.log.msg.error\]' - - '%{GREEDYDATA:nats.log.msg.data}' - pattern_definitions: - NATSDIRECTION: (<<-|->>) - NATSMSG: MSG - NATSPUB: PUB - NATSSUB: SUB - NATSUNSUB: UNSUB - NATSPAYLOAD: MSG_PAYLOAD - NATSERROR: -ERROR - NATSPING: PING - NATSPONG: PONG - NATSOK: OK - NATSCONNECT: CONNECT - NATSINFO: INFO - NATSCONNECTION: (?:%{NATSCONNECT}|%{NATSINFO}) - NATSNOINFO: (?:%{NATSPING}|%{NATSPONG}|%{NATSOK}) - ignore_missing: true -- remove: - field: nats.log.info -- remove: - field: nats.log.msg.info - ignore_missing: true -- remove: - field: nats.log.msg.payload - ignore_missing: true -- remove: - field: message -- rename: - field: nats.log.msg.data - target_field: message - ignore_missing: true -- script: - lang: painless - source: |- - if (ctx.log.level == params.inf) { - ctx.log.level = params.info; - } else if (ctx.log.level == params.dbg) { - ctx.log.level = params.debug; - } else if (ctx.log.level == params.wrn) { - ctx.log.level = params.warning; - } else if (ctx.log.level == params.err) { - ctx.log.level = params.error; - } else if (ctx.log.level == params.ftl) { - ctx.log.level = params.fatal; - } else if (ctx.log.level == params.trc) { - ctx.log.level = params.trace; - } - params: - inf: INF - info: info - dbg: DBG - debug: debug - wrn: WRN - warning: warning - err: ERR - error: error - ftl: FTL - fatal: fatal - trc: TRC - trace: trace -- script: - lang: painless - source: |- - if (ctx.nats.log.msg.type == params.msg) { - ctx.nats.log.msg.type = params.message; - } else if (ctx.nats.log.msg.type == params.pub) { - ctx.nats.log.msg.type = params.publish; - } else if (ctx.nats.log.msg.type == params.sub) { - ctx.nats.log.msg.type = params.subscribe; - } else if (ctx.nats.log.msg.type == params.unsub) { - ctx.nats.log.msg.type = params.unsubscribe; - } else if (ctx.nats.log.msg.type == params.msg_payload) { - ctx.nats.log.msg.type = params.payload; - } else if (ctx.nats.log.msg.type == params.err) { - ctx.nats.log.msg.type = params.error; - } else if (ctx.nats.log.msg.type == params.pi) { - ctx.nats.log.msg.type = params.ping; - } else if (ctx.nats.log.msg.type == params.po) { - ctx.nats.log.msg.type = params.pong; - } else if (ctx.nats.log.msg.type == params.ok) { - ctx.nats.log.msg.type = params.acknowledge; - } else if (ctx.nats.log.msg.type == params.connect) { - ctx.nats.log.msg.type = params.connection; - } else if (ctx.nats.log.msg.type == params.info) { - ctx.nats.log.msg.type = params.information; - } - params: - msg: MSG - message: message - pub: PUB - publish: publish - sub: SUB - subscribe: subscribe - unsub: UNSUB - unsubscribe: unsubscribe - msg_payload: MSG_PAYLOAD - payload: payload - err: -ERROR - error: error - pi: PING - ping: ping - po: PONG - pong: pong - ok: OK - acknowledge: acknowledge - connect: CONNECT - connection: connection - info: INFO - information: information - if: ctx.nats.log.msg?.type != null -- script: - lang: painless - source: |- - if (ctx.network.direction == params.in) { - ctx.network.direction = params.inbound; - } else if (ctx.network.direction == params.out) { - ctx.network.direction = params.outbound; - } - params: - in: <<- - inbound: inbound - out: ->> - outbound: outbound - if: ctx.network?.direction != null -- rename: - field: '@timestamp' - target_field: event.created -- date: - field: nats.log.timestamp - target_field: '@timestamp' - formats: - - yyyy/MM/dd HH:mm:ss.SSSSSS -- remove: - field: nats.log.timestamp -- set: - field: event.kind - value: event -- append: - field: event.type - value: info -- append: - field: event.type - value: error - if: "ctx?.log?.level != null && (ctx.log.level == 'error' || ctx.log.level == 'fatal')" -- append: - field: related.ip - value: "{{client.ip}}" - if: "ctx?.client?.ip != null" -- convert: - ignore_missing: true - field: process.pid - type: long -- convert: - ignore_missing: true - field: client.port - type: long -- convert: - ignore_missing: true - field: nats.log.msg.bytes - type: long + - set: + field: event.ingested + value: '{{_ingest.timestamp}}' + - set: + field: ecs.version + value: "1.10.0" + - rename: + field: message + target_field: event.original + ignore_missing: true + - grok: + field: event.original + patterns: + - \[%{POSINT:process.pid}\]( %{NATSTIME:nats.log.timestamp})? \[%{NATSLOGLEVEL:log.level}\] + %{GREEDYDATA:nats.log.info} + pattern_definitions: + NATSTIME: '%{YEAR}/%{MONTHNUM}/%{MONTHDAY} %{TIME}' + NATSLOGLEVEL: (INF|DBG|WRN|ERR|FTL|TRC) + ignore_missing: true + - grok: + field: nats.log.info + patterns: + - '%{IPV4:client.ip}:%{POSINT:client.port} - cid:%{POSINT:nats.log.client.id} + - %{GREEDYDATA:nats.log.msg.info}' + - '%{GREEDYDATA:nats.log.msg.data}' + ignore_missing: true + - grok: + field: nats.log.msg.info + patterns: + - '%{NATSDIRECTION:network.direction} %{NATSPAYLOAD:nats.log.msg.type}: \[%{GREEDYDATA:nats.log.msg.payload}\]' + - '%{NATSDIRECTION:network.direction} \[%{NATSNOINFO:nats.log.msg.type}\]' + - '%{NATSDIRECTION:network.direction} \[%{NATSUNSUB:nats.log.msg.type}\s+%{POSINT:nats.log.msg.sid}(\s+%{POSINT:nats.log.msg.max_messages})?\]' + - '%{NATSDIRECTION:network.direction} \[%{NATSPUB:nats.log.msg.type}\s+%{NOTSPACE:nats.log.msg.subject}(\s+%{NOTSPACE:nats.log.msg.reply_to})?\s+%{POSINT:nats.log.msg.bytes}\]' + - '%{NATSDIRECTION:network.direction} \[%{NATSSUB:nats.log.msg.type}\s+%{NOTSPACE:nats.log.msg.subject}(\s+%{NOTSPACE:nats.log.msg.queue_group})?\s+%{POSINT:nats.log.msg.sid}\]' + - '%{NATSDIRECTION:network.direction} \[%{NATSMSG:nats.log.msg.type}\s+%{NOTSPACE:nats.log.msg.subject}\s+%{POSINT:nats.log.msg.sid}(\s+%{NOTSPACE:nats.log.msg.reply_to})?\s+%{POSINT:nats.log.msg.bytes}\]' + - '%{NATSDIRECTION:network.direction} \[%{NATSCONNECTION:nats.log.msg.type}\s+%{GREEDYDATA:nats.log.msg.data}\]' + - '%{NATSDIRECTION:network.direction} \[%{NATSERROR:nats.log.msg.type}\s+%{GREEDYDATA:nats.log.msg.error\]' + - '%{GREEDYDATA:nats.log.msg.data}' + pattern_definitions: + NATSDIRECTION: (<<-|->>) + NATSMSG: MSG + NATSPUB: PUB + NATSSUB: SUB + NATSUNSUB: UNSUB + NATSPAYLOAD: MSG_PAYLOAD + NATSERROR: -ERROR + NATSPING: PING + NATSPONG: PONG + NATSOK: OK + NATSCONNECT: CONNECT + NATSINFO: INFO + NATSCONNECTION: (?:%{NATSCONNECT}|%{NATSINFO}) + NATSNOINFO: (?:%{NATSPING}|%{NATSPONG}|%{NATSOK}) + ignore_missing: true + - remove: + field: nats.log.info + - remove: + field: nats.log.msg.info + ignore_missing: true + - remove: + field: nats.log.msg.payload + ignore_missing: true + - rename: + field: nats.log.msg.data + target_field: message + ignore_missing: true + - script: + lang: painless + source: |- + if (ctx.log.level == params.inf) { + ctx.log.level = params.info; + } else if (ctx.log.level == params.dbg) { + ctx.log.level = params.debug; + } else if (ctx.log.level == params.wrn) { + ctx.log.level = params.warning; + } else if (ctx.log.level == params.err) { + ctx.log.level = params.error; + } else if (ctx.log.level == params.ftl) { + ctx.log.level = params.fatal; + } else if (ctx.log.level == params.trc) { + ctx.log.level = params.trace; + } + params: + inf: INF + info: info + dbg: DBG + debug: debug + wrn: WRN + warning: warning + err: ERR + error: error + ftl: FTL + fatal: fatal + trc: TRC + trace: trace + - script: + lang: painless + source: |- + if (ctx.nats.log.msg.type == params.msg) { + ctx.nats.log.msg.type = params.message; + } else if (ctx.nats.log.msg.type == params.pub) { + ctx.nats.log.msg.type = params.publish; + } else if (ctx.nats.log.msg.type == params.sub) { + ctx.nats.log.msg.type = params.subscribe; + } else if (ctx.nats.log.msg.type == params.unsub) { + ctx.nats.log.msg.type = params.unsubscribe; + } else if (ctx.nats.log.msg.type == params.msg_payload) { + ctx.nats.log.msg.type = params.payload; + } else if (ctx.nats.log.msg.type == params.err) { + ctx.nats.log.msg.type = params.error; + } else if (ctx.nats.log.msg.type == params.pi) { + ctx.nats.log.msg.type = params.ping; + } else if (ctx.nats.log.msg.type == params.po) { + ctx.nats.log.msg.type = params.pong; + } else if (ctx.nats.log.msg.type == params.ok) { + ctx.nats.log.msg.type = params.acknowledge; + } else if (ctx.nats.log.msg.type == params.connect) { + ctx.nats.log.msg.type = params.connection; + } else if (ctx.nats.log.msg.type == params.info) { + ctx.nats.log.msg.type = params.information; + } + params: + msg: MSG + message: message + pub: PUB + publish: publish + sub: SUB + subscribe: subscribe + unsub: UNSUB + unsubscribe: unsubscribe + msg_payload: MSG_PAYLOAD + payload: payload + err: -ERROR + error: error + pi: PING + ping: ping + po: PONG + pong: pong + ok: OK + acknowledge: acknowledge + connect: CONNECT + connection: connection + info: INFO + information: information + if: ctx.nats.log.msg?.type != null + - script: + lang: painless + source: |- + if (ctx.network.direction == params.in) { + ctx.network.direction = params.inbound; + } else if (ctx.network.direction == params.out) { + ctx.network.direction = params.outbound; + } + params: + in: <<- + inbound: inbound + out: ->> + outbound: outbound + if: ctx.network?.direction != null + - rename: + field: '@timestamp' + target_field: event.created + - date: + field: nats.log.timestamp + target_field: '@timestamp' + formats: + - yyyy/MM/dd HH:mm:ss.SSSSSS + - remove: + field: nats.log.timestamp + - set: + field: event.kind + value: event + - append: + field: event.type + value: info + - append: + field: event.type + value: error + if: "ctx?.log?.level != null && (ctx.log.level == 'error' || ctx.log.level == 'fatal')" + - append: + field: related.ip + value: "{{client.ip}}" + if: "ctx?.client?.ip != null" + - convert: + ignore_missing: true + field: process.pid + type: long + - convert: + ignore_missing: true + field: client.port + type: long + - convert: + ignore_missing: true + field: nats.log.msg.bytes + type: long + - remove: + field: event.original + if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))" + ignore_failure: true + ignore_missing: true on_failure: -- set: - field: error.message - value: '{{ _ingest.on_failure_message }}' + - set: + field: error.message + value: '{{ _ingest.on_failure_message }}' diff --git a/packages/nats/data_stream/log/fields/ecs.yml b/packages/nats/data_stream/log/fields/ecs.yml index a5e79097564..77a1a78e4cb 100644 --- a/packages/nats/data_stream/log/fields/ecs.yml +++ b/packages/nats/data_stream/log/fields/ecs.yml @@ -69,3 +69,9 @@ If multiple messages exist, they can be combined into one message.' example: Hello World +- name: tags + level: core + type: keyword + ignore_above: 1024 + description: List of keywords used to tag each event. + example: '["production", "env2"]' \ No newline at end of file diff --git a/packages/nats/data_stream/log/manifest.yml b/packages/nats/data_stream/log/manifest.yml index 0cdeea6b28b..90f17e47e0e 100644 --- a/packages/nats/data_stream/log/manifest.yml +++ b/packages/nats/data_stream/log/manifest.yml @@ -12,6 +12,22 @@ streams: show_user: true default: - /var/log/nats/nats.log* + - name: tags + type: text + title: Tags + multi: true + required: true + show_user: false + default: + - nats-log + - name: preserve_original_event + required: true + show_user: true + title: Preserve original event + description: Preserves a raw copy of the original event, added to the field `event.original` + type: bool + multi: false + default: false template_path: log.yml.hbs title: NATS logs (log) description: Collect NATS logs using log input diff --git a/packages/nats/docs/README.md b/packages/nats/docs/README.md index efc9c21a37d..54154ce91f9 100644 --- a/packages/nats/docs/README.md +++ b/packages/nats/docs/README.md @@ -140,6 +140,7 @@ An example event for `log` looks as following: | network.direction | Direction of the network traffic. Recommended values are: * inbound * outbound * internal * external * unknown When mapping events from a host-based monitoring context, populate this field from the host's point of view. When mapping events from a network or perimeter-based monitoring context, populate this field from the point of view of your network perimeter. | keyword | | process.pid | Process id. | long | | related.ip | All of the IPs seen on your event. | ip | +| tags | List of keywords used to tag each event. | keyword | ## Metrics From 3497cd32149cc458a8e67d8df9603099cd4afca2 Mon Sep 17 00:00:00 2001 From: Marius Iversen Date: Tue, 1 Jun 2021 19:37:24 +0200 Subject: [PATCH 2/4] update manifest, changelog and linting --- packages/nats/changelog.yml | 5 +++ .../test-log-sample.log-expected.json | 36 +++++++++---------- packages/nats/data_stream/log/fields/ecs.yml | 2 +- packages/nats/manifest.yml | 2 +- 4 files changed, 25 insertions(+), 20 deletions(-) diff --git a/packages/nats/changelog.yml b/packages/nats/changelog.yml index 4032bc36b69..cab99b48344 100644 --- a/packages/nats/changelog.yml +++ b/packages/nats/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "0.2.1" + changes: + - description: update to ECS 1.9.0 and add event.original options + type: enhancement + link: https://github.com/elastic/integrations/pull/996 - version: "0.2.0" changes: - description: Fix stack compatability diff --git a/packages/nats/data_stream/log/_dev/test/pipeline/test-log-sample.log-expected.json b/packages/nats/data_stream/log/_dev/test/pipeline/test-log-sample.log-expected.json index 6fed9d0b69c..7be225bad69 100644 --- a/packages/nats/data_stream/log/_dev/test/pipeline/test-log-sample.log-expected.json +++ b/packages/nats/data_stream/log/_dev/test/pipeline/test-log-sample.log-expected.json @@ -17,7 +17,7 @@ "level": "info" }, "event": { - "ingested": "2021-06-01T17:33:56.028520100Z", + "ingested": "2021-06-01T17:37:06.246940600Z", "original": "[1] 2019/02/06 07:19:40.624334 [INF] Starting nats-server version 1.3.0", "type": [ "info" @@ -47,7 +47,7 @@ "level": "info" }, "event": { - "ingested": "2021-06-01T17:33:56.028546700Z", + "ingested": "2021-06-01T17:37:06.246966900Z", "original": "[1] 2019/02/06 07:19:40.624547 [INF] Git commit [eed4fbc]", "type": [ "info" @@ -77,7 +77,7 @@ "level": "info" }, "event": { - "ingested": "2021-06-01T17:33:56.028554600Z", + "ingested": "2021-06-01T17:37:06.246974700Z", "original": "[1] 2019/02/06 07:19:40.624674 [INF] Listening for client connections on 0.0.0.0:4222", "type": [ "info" @@ -107,7 +107,7 @@ "level": "info" }, "event": { - "ingested": "2021-06-01T17:33:56.028582700Z", + "ingested": "2021-06-01T17:37:06.246991700Z", "original": "[1] 2019/02/06 07:19:40.624690 [INF] Server is ready", "type": [ "info" @@ -153,7 +153,7 @@ "ip": "172.18.0.1" }, "event": { - "ingested": "2021-06-01T17:33:56.028589800Z", + "ingested": "2021-06-01T17:37:06.246997400Z", "original": "[1] 2019/02/06 07:20:08.508891 [DBG] 172.18.0.1:38630 - cid:1 - Client connection created", "type": [ "info" @@ -200,7 +200,7 @@ "ip": "172.18.0.1" }, "event": { - "ingested": "2021-06-01T17:33:56.028596400Z", + "ingested": "2021-06-01T17:37:06.247002500Z", "original": "[1] 2019/02/06 07:20:08.510296 [TRC] 172.18.0.1:38630 - cid:1 - -\u003e\u003e [CONNECT {\"verbose\":false,\"pedantic\":false,\"tls_required\":false,\"name\":\"NATS Benchmark\",\"lang\":\"go\",\"version\":\"1.7.0\",\"protocol\":1,\"echo\":true}]", "type": [ "info" @@ -248,7 +248,7 @@ "ip": "172.18.0.1" }, "event": { - "ingested": "2021-06-01T17:33:56.028603Z", + "ingested": "2021-06-01T17:37:06.247008Z", "original": "[1] 2019/02/06 07:20:08.512052 [TRC] 172.18.0.1:38630 - cid:1 - -\u003e\u003e [SUB foo 1]", "type": [ "info" @@ -294,7 +294,7 @@ "ip": "172.18.0.1" }, "event": { - "ingested": "2021-06-01T17:33:56.028609Z", + "ingested": "2021-06-01T17:37:06.247012800Z", "original": "[1] 2019/02/06 07:20:08.512128 [TRC] 172.18.0.1:38630 - cid:1 - -\u003e\u003e [PING]", "type": [ "info" @@ -340,7 +340,7 @@ "ip": "172.18.0.1" }, "event": { - "ingested": "2021-06-01T17:33:56.028614500Z", + "ingested": "2021-06-01T17:37:06.247017500Z", "original": "[1] 2019/02/06 07:20:08.512153 [TRC] 172.18.0.1:38630 - cid:1 - \u003c\u003c- [PONG]", "type": [ "info" @@ -389,7 +389,7 @@ "ip": "50.39.246.116" }, "event": { - "ingested": "2021-06-01T17:33:56.028619800Z", + "ingested": "2021-06-01T17:37:06.247022Z", "original": "[1] 2019/02/04 15:40:02.717819 [TRC] 50.39.246.116:62388 - cid:3 - -\u003e\u003e [PUB aiuser.platinum1.pingpeer _INBOX.e3hAUbP4r5wbjw3Hudw42r.udigGiHn 20]", "type": [ "info" @@ -435,7 +435,7 @@ "ip": "50.39.246.116" }, "event": { - "ingested": "2021-06-01T17:33:56.028626200Z", + "ingested": "2021-06-01T17:37:06.247027300Z", "original": "[1] 2019/02/04 15:40:02.717825 [TRC] 50.39.246.116:62388 - cid:3 - -\u003e\u003e MSG_PAYLOAD: [peer, are you alive?]", "type": [ "info" @@ -485,7 +485,7 @@ "ip": "192.168.176.11" }, "event": { - "ingested": "2021-06-01T17:33:56.028631500Z", + "ingested": "2021-06-01T17:37:06.247032300Z", "original": "[1] 2019/02/04 15:40:02.717832 [TRC] 192.168.176.11:36262 - cid:4 - \u003c\u003c- [MSG aiuser.platinum1.pingpeer 1 _INBOX.e3hAUbP4r5wbjw3Hudw42r.udigGiHn 20]", "type": [ "info" @@ -533,7 +533,7 @@ "ip": "192.168.176.11" }, "event": { - "ingested": "2021-06-01T17:33:56.028636700Z", + "ingested": "2021-06-01T17:37:06.247037200Z", "original": "[1] 2019/02/04 15:40:02.718007 [TRC] 192.168.176.11:36262 - cid:4 - -\u003e\u003e [PUB _INBOX.e3hAUbP4r5wbjw3Hudw42r.udigGiHn 17]", "type": [ "info" @@ -579,7 +579,7 @@ "ip": "192.168.176.11" }, "event": { - "ingested": "2021-06-01T17:33:56.028641800Z", + "ingested": "2021-06-01T17:37:06.247041800Z", "original": "[1] 2019/02/04 15:40:02.718023 [TRC] 192.168.176.11:36262 - cid:4 - -\u003e\u003e MSG_PAYLOAD: [I am fine, agent!]", "type": [ "info" @@ -628,7 +628,7 @@ "ip": "50.39.246.116" }, "event": { - "ingested": "2021-06-01T17:33:56.028647Z", + "ingested": "2021-06-01T17:37:06.247046300Z", "original": "[1] 2019/02/04 15:40:02.718044 [TRC] 50.39.246.116:62388 - cid:3 - \u003c\u003c- [MSG _INBOX.e3hAUbP4r5wbjw3Hudw42r.udigGiHn 11 17]", "type": [ "info" @@ -676,7 +676,7 @@ "ip": "50.39.246.116" }, "event": { - "ingested": "2021-06-01T17:33:56.028652200Z", + "ingested": "2021-06-01T17:37:06.247053200Z", "original": "[1] 2019/02/04 15:40:02.717600 [TRC] 50.39.246.116:62388 - cid:3 - -\u003e\u003e [PUB aiuser.platinum1.appstats 1583]", "type": [ "info" @@ -725,7 +725,7 @@ "ip": "192.168.176.11" }, "event": { - "ingested": "2021-06-01T17:33:56.028657400Z", + "ingested": "2021-06-01T17:37:06.247057900Z", "original": "[1] 2019/02/04 15:40:02.717811 [TRC] 192.168.176.11:36262 - cid:4 - \u003c\u003c- [MSG aiuser.platinum1.appstats 6 1583]", "type": [ "info" @@ -771,7 +771,7 @@ "ip": "172.18.0.1" }, "event": { - "ingested": "2021-06-01T17:33:56.028669400Z", + "ingested": "2021-06-01T17:37:06.247062600Z", "original": "[1] 2019/02/16 07:20:08.512153 [TRC] 172.18.0.1:38630 - cid:1 - \u003c\u003c- [OK]", "type": [ "info" diff --git a/packages/nats/data_stream/log/fields/ecs.yml b/packages/nats/data_stream/log/fields/ecs.yml index 77a1a78e4cb..505b266d102 100644 --- a/packages/nats/data_stream/log/fields/ecs.yml +++ b/packages/nats/data_stream/log/fields/ecs.yml @@ -74,4 +74,4 @@ type: keyword ignore_above: 1024 description: List of keywords used to tag each event. - example: '["production", "env2"]' \ No newline at end of file + example: '["production", "env2"]' diff --git a/packages/nats/manifest.yml b/packages/nats/manifest.yml index 0906b98bcb7..f99e3da0ae9 100644 --- a/packages/nats/manifest.yml +++ b/packages/nats/manifest.yml @@ -1,6 +1,6 @@ name: nats title: NATS -version: 0.2.0 +version: 0.2.1 release: experimental description: NATS Integration type: integration From 5aa9229d76f3a3a0df6b453e48556a3efdc13206 Mon Sep 17 00:00:00 2001 From: Marius Iversen Date: Mon, 7 Jun 2021 16:31:04 +0200 Subject: [PATCH 3/4] Linting and updating manifest --- packages/nats/changelog.yml | 6 ++-- .../test-log-sample.log-expected.json | 36 +++++++++---------- .../data_stream/log/agent/stream/log.yml.hbs | 9 ++++- packages/nats/data_stream/log/manifest.yml | 9 +++++ packages/nats/manifest.yml | 2 +- 5 files changed, 39 insertions(+), 23 deletions(-) diff --git a/packages/nats/changelog.yml b/packages/nats/changelog.yml index cab99b48344..f741206c8dd 100644 --- a/packages/nats/changelog.yml +++ b/packages/nats/changelog.yml @@ -1,9 +1,9 @@ # newer versions go on top -- version: "0.2.1" +- version: "1.0.0" changes: - - description: update to ECS 1.9.0 and add event.original options + - description: update to ECS 1.10.0 and add event.original options type: enhancement - link: https://github.com/elastic/integrations/pull/996 + link: https://github.com/elastic/integrations/pull/1061 - version: "0.2.0" changes: - description: Fix stack compatability diff --git a/packages/nats/data_stream/log/_dev/test/pipeline/test-log-sample.log-expected.json b/packages/nats/data_stream/log/_dev/test/pipeline/test-log-sample.log-expected.json index 7be225bad69..dcb68fda155 100644 --- a/packages/nats/data_stream/log/_dev/test/pipeline/test-log-sample.log-expected.json +++ b/packages/nats/data_stream/log/_dev/test/pipeline/test-log-sample.log-expected.json @@ -17,7 +17,7 @@ "level": "info" }, "event": { - "ingested": "2021-06-01T17:37:06.246940600Z", + "ingested": "2021-06-07T14:30:17.150062400Z", "original": "[1] 2019/02/06 07:19:40.624334 [INF] Starting nats-server version 1.3.0", "type": [ "info" @@ -47,7 +47,7 @@ "level": "info" }, "event": { - "ingested": "2021-06-01T17:37:06.246966900Z", + "ingested": "2021-06-07T14:30:17.150103900Z", "original": "[1] 2019/02/06 07:19:40.624547 [INF] Git commit [eed4fbc]", "type": [ "info" @@ -77,7 +77,7 @@ "level": "info" }, "event": { - "ingested": "2021-06-01T17:37:06.246974700Z", + "ingested": "2021-06-07T14:30:17.150112300Z", "original": "[1] 2019/02/06 07:19:40.624674 [INF] Listening for client connections on 0.0.0.0:4222", "type": [ "info" @@ -107,7 +107,7 @@ "level": "info" }, "event": { - "ingested": "2021-06-01T17:37:06.246991700Z", + "ingested": "2021-06-07T14:30:17.150120800Z", "original": "[1] 2019/02/06 07:19:40.624690 [INF] Server is ready", "type": [ "info" @@ -153,7 +153,7 @@ "ip": "172.18.0.1" }, "event": { - "ingested": "2021-06-01T17:37:06.246997400Z", + "ingested": "2021-06-07T14:30:17.150127400Z", "original": "[1] 2019/02/06 07:20:08.508891 [DBG] 172.18.0.1:38630 - cid:1 - Client connection created", "type": [ "info" @@ -200,7 +200,7 @@ "ip": "172.18.0.1" }, "event": { - "ingested": "2021-06-01T17:37:06.247002500Z", + "ingested": "2021-06-07T14:30:17.150133300Z", "original": "[1] 2019/02/06 07:20:08.510296 [TRC] 172.18.0.1:38630 - cid:1 - -\u003e\u003e [CONNECT {\"verbose\":false,\"pedantic\":false,\"tls_required\":false,\"name\":\"NATS Benchmark\",\"lang\":\"go\",\"version\":\"1.7.0\",\"protocol\":1,\"echo\":true}]", "type": [ "info" @@ -248,7 +248,7 @@ "ip": "172.18.0.1" }, "event": { - "ingested": "2021-06-01T17:37:06.247008Z", + "ingested": "2021-06-07T14:30:17.150138800Z", "original": "[1] 2019/02/06 07:20:08.512052 [TRC] 172.18.0.1:38630 - cid:1 - -\u003e\u003e [SUB foo 1]", "type": [ "info" @@ -294,7 +294,7 @@ "ip": "172.18.0.1" }, "event": { - "ingested": "2021-06-01T17:37:06.247012800Z", + "ingested": "2021-06-07T14:30:17.150144300Z", "original": "[1] 2019/02/06 07:20:08.512128 [TRC] 172.18.0.1:38630 - cid:1 - -\u003e\u003e [PING]", "type": [ "info" @@ -340,7 +340,7 @@ "ip": "172.18.0.1" }, "event": { - "ingested": "2021-06-01T17:37:06.247017500Z", + "ingested": "2021-06-07T14:30:17.150149400Z", "original": "[1] 2019/02/06 07:20:08.512153 [TRC] 172.18.0.1:38630 - cid:1 - \u003c\u003c- [PONG]", "type": [ "info" @@ -389,7 +389,7 @@ "ip": "50.39.246.116" }, "event": { - "ingested": "2021-06-01T17:37:06.247022Z", + "ingested": "2021-06-07T14:30:17.150154500Z", "original": "[1] 2019/02/04 15:40:02.717819 [TRC] 50.39.246.116:62388 - cid:3 - -\u003e\u003e [PUB aiuser.platinum1.pingpeer _INBOX.e3hAUbP4r5wbjw3Hudw42r.udigGiHn 20]", "type": [ "info" @@ -435,7 +435,7 @@ "ip": "50.39.246.116" }, "event": { - "ingested": "2021-06-01T17:37:06.247027300Z", + "ingested": "2021-06-07T14:30:17.150161700Z", "original": "[1] 2019/02/04 15:40:02.717825 [TRC] 50.39.246.116:62388 - cid:3 - -\u003e\u003e MSG_PAYLOAD: [peer, are you alive?]", "type": [ "info" @@ -485,7 +485,7 @@ "ip": "192.168.176.11" }, "event": { - "ingested": "2021-06-01T17:37:06.247032300Z", + "ingested": "2021-06-07T14:30:17.150167300Z", "original": "[1] 2019/02/04 15:40:02.717832 [TRC] 192.168.176.11:36262 - cid:4 - \u003c\u003c- [MSG aiuser.platinum1.pingpeer 1 _INBOX.e3hAUbP4r5wbjw3Hudw42r.udigGiHn 20]", "type": [ "info" @@ -533,7 +533,7 @@ "ip": "192.168.176.11" }, "event": { - "ingested": "2021-06-01T17:37:06.247037200Z", + "ingested": "2021-06-07T14:30:17.150172700Z", "original": "[1] 2019/02/04 15:40:02.718007 [TRC] 192.168.176.11:36262 - cid:4 - -\u003e\u003e [PUB _INBOX.e3hAUbP4r5wbjw3Hudw42r.udigGiHn 17]", "type": [ "info" @@ -579,7 +579,7 @@ "ip": "192.168.176.11" }, "event": { - "ingested": "2021-06-01T17:37:06.247041800Z", + "ingested": "2021-06-07T14:30:17.150177800Z", "original": "[1] 2019/02/04 15:40:02.718023 [TRC] 192.168.176.11:36262 - cid:4 - -\u003e\u003e MSG_PAYLOAD: [I am fine, agent!]", "type": [ "info" @@ -628,7 +628,7 @@ "ip": "50.39.246.116" }, "event": { - "ingested": "2021-06-01T17:37:06.247046300Z", + "ingested": "2021-06-07T14:30:17.150183300Z", "original": "[1] 2019/02/04 15:40:02.718044 [TRC] 50.39.246.116:62388 - cid:3 - \u003c\u003c- [MSG _INBOX.e3hAUbP4r5wbjw3Hudw42r.udigGiHn 11 17]", "type": [ "info" @@ -676,7 +676,7 @@ "ip": "50.39.246.116" }, "event": { - "ingested": "2021-06-01T17:37:06.247053200Z", + "ingested": "2021-06-07T14:30:17.150188100Z", "original": "[1] 2019/02/04 15:40:02.717600 [TRC] 50.39.246.116:62388 - cid:3 - -\u003e\u003e [PUB aiuser.platinum1.appstats 1583]", "type": [ "info" @@ -725,7 +725,7 @@ "ip": "192.168.176.11" }, "event": { - "ingested": "2021-06-01T17:37:06.247057900Z", + "ingested": "2021-06-07T14:30:17.150193800Z", "original": "[1] 2019/02/04 15:40:02.717811 [TRC] 192.168.176.11:36262 - cid:4 - \u003c\u003c- [MSG aiuser.platinum1.appstats 6 1583]", "type": [ "info" @@ -771,7 +771,7 @@ "ip": "172.18.0.1" }, "event": { - "ingested": "2021-06-01T17:37:06.247062600Z", + "ingested": "2021-06-07T14:30:17.150198600Z", "original": "[1] 2019/02/16 07:20:08.512153 [TRC] 172.18.0.1:38630 - cid:1 - \u003c\u003c- [OK]", "type": [ "info" diff --git a/packages/nats/data_stream/log/agent/stream/log.yml.hbs b/packages/nats/data_stream/log/agent/stream/log.yml.hbs index 69d278b8ff4..a75dc37eed9 100644 --- a/packages/nats/data_stream/log/agent/stream/log.yml.hbs +++ b/packages/nats/data_stream/log/agent/stream/log.yml.hbs @@ -9,4 +9,11 @@ tags: {{#each tags as |tag i|}} - {{tag}} {{/each}} -exclude_files: [".gz$"] \ No newline at end of file +{{#contains tags "forwarded"}} +publisher_pipeline.disable_host: true +{{/contains}} +exclude_files: [".gz$"] +{{#if processors}} +processors: +{{processors}} +{{/if}} \ No newline at end of file diff --git a/packages/nats/data_stream/log/manifest.yml b/packages/nats/data_stream/log/manifest.yml index 90f17e47e0e..8ef7c819e6d 100644 --- a/packages/nats/data_stream/log/manifest.yml +++ b/packages/nats/data_stream/log/manifest.yml @@ -28,6 +28,15 @@ streams: type: bool multi: false default: false + - name: processors + type: yaml + title: Processors + multi: false + required: false + show_user: false + description: > + Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details. + template_path: log.yml.hbs title: NATS logs (log) description: Collect NATS logs using log input diff --git a/packages/nats/manifest.yml b/packages/nats/manifest.yml index f99e3da0ae9..5d0a2682642 100644 --- a/packages/nats/manifest.yml +++ b/packages/nats/manifest.yml @@ -1,6 +1,6 @@ name: nats title: NATS -version: 0.2.1 +version: 1.0.0 release: experimental description: NATS Integration type: integration From eade9c0613e42c65ef3a4790209d6d77590f8455 Mon Sep 17 00:00:00 2001 From: Marius Iversen Date: Wed, 9 Jun 2021 14:07:38 +0200 Subject: [PATCH 4/4] update changelog and linting --- packages/nats/changelog.yml | 2 +- .../test-log-sample.log-expected.json | 36 +++++++++---------- packages/nats/docs/README.md | 14 ++++---- packages/nats/manifest.yml | 2 +- 4 files changed, 27 insertions(+), 27 deletions(-) diff --git a/packages/nats/changelog.yml b/packages/nats/changelog.yml index f741206c8dd..5286015396e 100644 --- a/packages/nats/changelog.yml +++ b/packages/nats/changelog.yml @@ -1,5 +1,5 @@ # newer versions go on top -- version: "1.0.0" +- version: "0.3.0" changes: - description: update to ECS 1.10.0 and add event.original options type: enhancement diff --git a/packages/nats/data_stream/log/_dev/test/pipeline/test-log-sample.log-expected.json b/packages/nats/data_stream/log/_dev/test/pipeline/test-log-sample.log-expected.json index dcb68fda155..a3f8d9f8653 100644 --- a/packages/nats/data_stream/log/_dev/test/pipeline/test-log-sample.log-expected.json +++ b/packages/nats/data_stream/log/_dev/test/pipeline/test-log-sample.log-expected.json @@ -17,7 +17,7 @@ "level": "info" }, "event": { - "ingested": "2021-06-07T14:30:17.150062400Z", + "ingested": "2021-06-09T12:07:01.961493300Z", "original": "[1] 2019/02/06 07:19:40.624334 [INF] Starting nats-server version 1.3.0", "type": [ "info" @@ -47,7 +47,7 @@ "level": "info" }, "event": { - "ingested": "2021-06-07T14:30:17.150103900Z", + "ingested": "2021-06-09T12:07:01.961517200Z", "original": "[1] 2019/02/06 07:19:40.624547 [INF] Git commit [eed4fbc]", "type": [ "info" @@ -77,7 +77,7 @@ "level": "info" }, "event": { - "ingested": "2021-06-07T14:30:17.150112300Z", + "ingested": "2021-06-09T12:07:01.961527100Z", "original": "[1] 2019/02/06 07:19:40.624674 [INF] Listening for client connections on 0.0.0.0:4222", "type": [ "info" @@ -107,7 +107,7 @@ "level": "info" }, "event": { - "ingested": "2021-06-07T14:30:17.150120800Z", + "ingested": "2021-06-09T12:07:01.961570100Z", "original": "[1] 2019/02/06 07:19:40.624690 [INF] Server is ready", "type": [ "info" @@ -153,7 +153,7 @@ "ip": "172.18.0.1" }, "event": { - "ingested": "2021-06-07T14:30:17.150127400Z", + "ingested": "2021-06-09T12:07:01.961576800Z", "original": "[1] 2019/02/06 07:20:08.508891 [DBG] 172.18.0.1:38630 - cid:1 - Client connection created", "type": [ "info" @@ -200,7 +200,7 @@ "ip": "172.18.0.1" }, "event": { - "ingested": "2021-06-07T14:30:17.150133300Z", + "ingested": "2021-06-09T12:07:01.961583100Z", "original": "[1] 2019/02/06 07:20:08.510296 [TRC] 172.18.0.1:38630 - cid:1 - -\u003e\u003e [CONNECT {\"verbose\":false,\"pedantic\":false,\"tls_required\":false,\"name\":\"NATS Benchmark\",\"lang\":\"go\",\"version\":\"1.7.0\",\"protocol\":1,\"echo\":true}]", "type": [ "info" @@ -248,7 +248,7 @@ "ip": "172.18.0.1" }, "event": { - "ingested": "2021-06-07T14:30:17.150138800Z", + "ingested": "2021-06-09T12:07:01.961588800Z", "original": "[1] 2019/02/06 07:20:08.512052 [TRC] 172.18.0.1:38630 - cid:1 - -\u003e\u003e [SUB foo 1]", "type": [ "info" @@ -294,7 +294,7 @@ "ip": "172.18.0.1" }, "event": { - "ingested": "2021-06-07T14:30:17.150144300Z", + "ingested": "2021-06-09T12:07:01.961593400Z", "original": "[1] 2019/02/06 07:20:08.512128 [TRC] 172.18.0.1:38630 - cid:1 - -\u003e\u003e [PING]", "type": [ "info" @@ -340,7 +340,7 @@ "ip": "172.18.0.1" }, "event": { - "ingested": "2021-06-07T14:30:17.150149400Z", + "ingested": "2021-06-09T12:07:01.961599200Z", "original": "[1] 2019/02/06 07:20:08.512153 [TRC] 172.18.0.1:38630 - cid:1 - \u003c\u003c- [PONG]", "type": [ "info" @@ -389,7 +389,7 @@ "ip": "50.39.246.116" }, "event": { - "ingested": "2021-06-07T14:30:17.150154500Z", + "ingested": "2021-06-09T12:07:01.961605200Z", "original": "[1] 2019/02/04 15:40:02.717819 [TRC] 50.39.246.116:62388 - cid:3 - -\u003e\u003e [PUB aiuser.platinum1.pingpeer _INBOX.e3hAUbP4r5wbjw3Hudw42r.udigGiHn 20]", "type": [ "info" @@ -435,7 +435,7 @@ "ip": "50.39.246.116" }, "event": { - "ingested": "2021-06-07T14:30:17.150161700Z", + "ingested": "2021-06-09T12:07:01.961622700Z", "original": "[1] 2019/02/04 15:40:02.717825 [TRC] 50.39.246.116:62388 - cid:3 - -\u003e\u003e MSG_PAYLOAD: [peer, are you alive?]", "type": [ "info" @@ -485,7 +485,7 @@ "ip": "192.168.176.11" }, "event": { - "ingested": "2021-06-07T14:30:17.150167300Z", + "ingested": "2021-06-09T12:07:01.961630600Z", "original": "[1] 2019/02/04 15:40:02.717832 [TRC] 192.168.176.11:36262 - cid:4 - \u003c\u003c- [MSG aiuser.platinum1.pingpeer 1 _INBOX.e3hAUbP4r5wbjw3Hudw42r.udigGiHn 20]", "type": [ "info" @@ -533,7 +533,7 @@ "ip": "192.168.176.11" }, "event": { - "ingested": "2021-06-07T14:30:17.150172700Z", + "ingested": "2021-06-09T12:07:01.961636900Z", "original": "[1] 2019/02/04 15:40:02.718007 [TRC] 192.168.176.11:36262 - cid:4 - -\u003e\u003e [PUB _INBOX.e3hAUbP4r5wbjw3Hudw42r.udigGiHn 17]", "type": [ "info" @@ -579,7 +579,7 @@ "ip": "192.168.176.11" }, "event": { - "ingested": "2021-06-07T14:30:17.150177800Z", + "ingested": "2021-06-09T12:07:01.961642100Z", "original": "[1] 2019/02/04 15:40:02.718023 [TRC] 192.168.176.11:36262 - cid:4 - -\u003e\u003e MSG_PAYLOAD: [I am fine, agent!]", "type": [ "info" @@ -628,7 +628,7 @@ "ip": "50.39.246.116" }, "event": { - "ingested": "2021-06-07T14:30:17.150183300Z", + "ingested": "2021-06-09T12:07:01.961647100Z", "original": "[1] 2019/02/04 15:40:02.718044 [TRC] 50.39.246.116:62388 - cid:3 - \u003c\u003c- [MSG _INBOX.e3hAUbP4r5wbjw3Hudw42r.udigGiHn 11 17]", "type": [ "info" @@ -676,7 +676,7 @@ "ip": "50.39.246.116" }, "event": { - "ingested": "2021-06-07T14:30:17.150188100Z", + "ingested": "2021-06-09T12:07:01.961652500Z", "original": "[1] 2019/02/04 15:40:02.717600 [TRC] 50.39.246.116:62388 - cid:3 - -\u003e\u003e [PUB aiuser.platinum1.appstats 1583]", "type": [ "info" @@ -725,7 +725,7 @@ "ip": "192.168.176.11" }, "event": { - "ingested": "2021-06-07T14:30:17.150193800Z", + "ingested": "2021-06-09T12:07:01.961663200Z", "original": "[1] 2019/02/04 15:40:02.717811 [TRC] 192.168.176.11:36262 - cid:4 - \u003c\u003c- [MSG aiuser.platinum1.appstats 6 1583]", "type": [ "info" @@ -771,7 +771,7 @@ "ip": "172.18.0.1" }, "event": { - "ingested": "2021-06-07T14:30:17.150198600Z", + "ingested": "2021-06-09T12:07:01.961668200Z", "original": "[1] 2019/02/16 07:20:08.512153 [TRC] 172.18.0.1:38630 - cid:1 - \u003c\u003c- [OK]", "type": [ "info" diff --git a/packages/nats/docs/README.md b/packages/nats/docs/README.md index 54154ce91f9..d5bb5dac0d0 100644 --- a/packages/nats/docs/README.md +++ b/packages/nats/docs/README.md @@ -16,7 +16,7 @@ The `log` dataset collects the NATS logs. An example event for `log` looks as following: -```$json +```json { "nats": { "log": { @@ -156,7 +156,7 @@ metrics from a Nats instance. An example event for `stats` looks as following: -```$json +```json { "@timestamp": "2020-11-25T11:55:12.889Z", "agent": { @@ -291,7 +291,7 @@ metrics about connections from a Nats instance. An example event for `connections` looks as following: -```$json +```json { "@timestamp": "2020-11-25T11:55:32.849Z", "metricset": { @@ -384,7 +384,7 @@ metrics about routes from a Nats instance. An example event for `routes` looks as following: -```$json +```json { "@timestamp": "2020-11-25T11:54:52.887Z", "event": { @@ -477,7 +477,7 @@ metrics about subscriptions from a Nats instance. An example event for `subscriptions` looks as following: -```$json +```json { "@timestamp": "2020-11-25T11:56:12.814Z", "service": { @@ -584,7 +584,7 @@ metrics per connection from a Nats instance. An example event for `connection` looks as following: -```$json +```json { "@timestamp": "2020-11-25T11:55:52.814Z", "service": { @@ -696,7 +696,7 @@ metric per route from a Nats instance. An example event for `route` looks as following: -```$json +```json { "@timestamp": "2020-11-25T11:54:22.920Z", "service": { diff --git a/packages/nats/manifest.yml b/packages/nats/manifest.yml index 5d0a2682642..08facdd7acf 100644 --- a/packages/nats/manifest.yml +++ b/packages/nats/manifest.yml @@ -1,6 +1,6 @@ name: nats title: NATS -version: 1.0.0 +version: 0.3.0 release: experimental description: NATS Integration type: integration