diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 26fd2c0b2a55..80b16f10dd44 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -861,6 +861,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Use default add_locale for fortinet.firewall {issue}20300[20300] {pull}26524[26524] - Add new template functions and `value_type` parameter to `httpjson` transforms. {pull}26847[26847] - Add support to merge registry updates in the filestream input across multiple ACKed batches in case of backpressure in the registry or disk. {pull}25976[25976] +- Update Elasticsearch module's ingest pipeline for parsing new deprecation logs {issue}26857[26857] {pull}26880[26880] *Heartbeat* diff --git a/filebeat/module/elasticsearch/deprecation/ingest/pipeline-json.yml b/filebeat/module/elasticsearch/deprecation/ingest/pipeline-json.yml index 69ea1f0f59f1..08f044e68d5c 100644 --- a/filebeat/module/elasticsearch/deprecation/ingest/pipeline-json.yml +++ b/filebeat/module/elasticsearch/deprecation/ingest/pipeline-json.yml @@ -8,7 +8,7 @@ processors: field: message target_field: elasticsearch.deprecation - drop: - if: ctx.elasticsearch.deprecation.type != 'deprecation' + if: '!["deprecation", "deprecation.elasticsearch"].contains(ctx.elasticsearch.deprecation.type)' - remove: field: elasticsearch.deprecation.type - dot_expander: @@ -78,17 +78,20 @@ processors: - rename: field: elasticsearch.deprecation.message target_field: message -- rename: - field: elasticsearch.deprecation.@timestamp - target_field: '@timestamp' - ignore_missing: true -- rename: - field: elasticsearch.deprecation.timestamp - target_field: '@timestamp' - ignore_missing: true - date: - field: '@timestamp' - target_field: '@timestamp' + field: 'elasticsearch.deprecation.@timestamp' + formats: + - ISO8601 + ignore_failure: true + if: 'ctx.elasticsearch?.deprecation["@timestamp"] != null' +- date: + field: 'elasticsearch.deprecation.timestamp' formats: - ISO8601 ignore_failure: true + if: 'ctx.elasticsearch?.deprecation?.timestamp != null' +- remove: + field: + - elasticsearch.deprecation.timestamp + - elasticsearch.deprecation.@timestamp + ignore_missing: true diff --git a/filebeat/module/elasticsearch/deprecation/ingest/pipeline-plaintext.yml b/filebeat/module/elasticsearch/deprecation/ingest/pipeline-plaintext.yml index 433d6ba53dcb..aa5c6e40e6b5 100644 --- a/filebeat/module/elasticsearch/deprecation/ingest/pipeline-plaintext.yml +++ b/filebeat/module/elasticsearch/deprecation/ingest/pipeline-plaintext.yml @@ -11,8 +11,9 @@ processors: GREEDYMULTILINE: |- (.| )* + DEP_LOGLEVEL: '(%{LOGLEVEL}|DEPRECATION)' patterns: - - \[%{TIMESTAMP_ISO8601:elasticsearch.deprecation.timestamp}\]\[%{LOGLEVEL:log.level}%{SPACE}\]\[%{DATA:elasticsearch.component}%{SPACE}\] + - \[%{TIMESTAMP_ISO8601:elasticsearch.deprecation.timestamp}\]\[%{DEP_LOGLEVEL:log.level}(%{SPACE})?\]\[%{DATA:elasticsearch.component}(%{SPACE})?\](%{SPACE}\[%{HOSTNAME:elasticsearch.node.name}\])? %{GREEDYMULTILINE:message} - date: if: ctx.event.timezone == null diff --git a/filebeat/module/elasticsearch/deprecation/test/elasticsearch_deprecation.log b/filebeat/module/elasticsearch/deprecation/test/elasticsearch_deprecation.log index f797c1c20311..9a635bdaf50c 100644 --- a/filebeat/module/elasticsearch/deprecation/test/elasticsearch_deprecation.log +++ b/filebeat/module/elasticsearch/deprecation/test/elasticsearch_deprecation.log @@ -2,3 +2,6 @@ [2018-04-23T16:40:13,862][WARN ][o.e.d.a.a.i.t.p.PutIndexTemplateRequest] Deprecated field [template] used, replaced by [index_patterns] [2018-04-23T16:40:14,792][WARN ][o.e.d.a.a.i.t.p.PutIndexTemplateRequest] Deprecated field [template] used, replaced by [index_patterns] [2018-04-23T16:40:15,127][WARN ][o.e.d.a.a.i.t.p.PutIndexTemplateRequest] Deprecated field [template] used, replaced by [index_patterns] +[2021-07-09T10:33:11,962][DEPRECATION][o.e.d.r.RestController ] [ecenter.local] Legacy index templates are deprecated in favor of composable templates. +[2021-07-09T10:42:03,065][DEPRECATION][o.e.d.c.m.MetadataCreateIndexService] [ecenter.local] index name [.kibana-event-log-7.13.3-000001] starts with a dot '.', in the next major version, index names starting with a dot are reserved for hidden indices and system indices +[2021-07-09T10:48:46,329][DEPRECATION][o.e.d.c.m.IndexNameExpressionResolver] [ecenter.local] this request accesses system indices: [.apm-agent-configuration, .apm-custom-link, .async-search, .kibana_7.13.3_001, .kibana_task_manager_7.13.3_001], but in a future major version, direct access to system indices will be prevented by default diff --git a/filebeat/module/elasticsearch/deprecation/test/elasticsearch_deprecation.log-expected.json b/filebeat/module/elasticsearch/deprecation/test/elasticsearch_deprecation.log-expected.json index 78e49516f93c..e5399bed9331 100644 --- a/filebeat/module/elasticsearch/deprecation/test/elasticsearch_deprecation.log-expected.json +++ b/filebeat/module/elasticsearch/deprecation/test/elasticsearch_deprecation.log-expected.json @@ -62,5 +62,56 @@ "log.offset": 411, "message": "Deprecated field [template] used, replaced by [index_patterns]", "service.type": "elasticsearch" + }, + { + "@timestamp": "2021-07-09T10:33:11.962-02:00", + "elasticsearch.component": "o.e.d.r.RestController", + "elasticsearch.node.name": "ecenter.local", + "event.category": "database", + "event.dataset": "elasticsearch.deprecation", + "event.kind": "event", + "event.module": "elasticsearch", + "event.timezone": "-02:00", + "event.type": "info", + "fileset.name": "deprecation", + "input.type": "log", + "log.level": "DEPRECATION", + "log.offset": 548, + "message": "Legacy index templates are deprecated in favor of composable templates.", + "service.type": "elasticsearch" + }, + { + "@timestamp": "2021-07-09T10:42:03.065-02:00", + "elasticsearch.component": "o.e.d.c.m.MetadataCreateIndexService", + "elasticsearch.node.name": "ecenter.local", + "event.category": "database", + "event.dataset": "elasticsearch.deprecation", + "event.kind": "event", + "event.module": "elasticsearch", + "event.timezone": "-02:00", + "event.type": "info", + "fileset.name": "deprecation", + "input.type": "log", + "log.level": "DEPRECATION", + "log.offset": 702, + "message": "index name [.kibana-event-log-7.13.3-000001] starts with a dot '.', in the next major version, index names starting with a dot are reserved for hidden indices and system indices", + "service.type": "elasticsearch" + }, + { + "@timestamp": "2021-07-09T10:48:46.329-02:00", + "elasticsearch.component": "o.e.d.c.m.IndexNameExpressionResolver", + "elasticsearch.node.name": "ecenter.local", + "event.category": "database", + "event.dataset": "elasticsearch.deprecation", + "event.kind": "event", + "event.module": "elasticsearch", + "event.timezone": "-02:00", + "event.type": "info", + "fileset.name": "deprecation", + "input.type": "log", + "log.level": "DEPRECATION", + "log.offset": 973, + "message": "this request accesses system indices: [.apm-agent-configuration, .apm-custom-link, .async-search, .kibana_7.13.3_001, .kibana_task_manager_7.13.3_001], but in a future major version, direct access to system indices will be prevented by default", + "service.type": "elasticsearch" } ] \ No newline at end of file diff --git a/filebeat/module/elasticsearch/deprecation/test/test-json.log b/filebeat/module/elasticsearch/deprecation/test/test-json.log index c6852de59d29..de8c5841d5ee 100644 --- a/filebeat/module/elasticsearch/deprecation/test/test-json.log +++ b/filebeat/module/elasticsearch/deprecation/test/test-json.log @@ -11,3 +11,6 @@ {"type": "deprecation", "timestamp": "2019-01-30T14:17:17,546-0800", "level": "WARN", "component": "o.e.d.r.a.s.RestSearchAction", "cluster.name": "es1", "node.name": "es1_1", "cluster.uuid": "S4dWw65ZT1eu3SltmAr84A", "node.id": "gCoNXf3qSQ6a190zBKr7Bw", "message": "[types removal] Specifying types in search requests is deprecated." } {"type": "deprecation", "timestamp": "2019-01-30T14:18:33,367-0800", "level": "WARN", "component": "o.e.d.x.w.a.i.IndexAction", "cluster.name": "es1", "node.name": "es1_1", "cluster.uuid": "S4dWw65ZT1eu3SltmAr84A", "node.id": "gCoNXf3qSQ6a190zBKr7Bw", "message": "[types removal] Specifying types in a watcher index action is deprecated." } {"type": "deprecation", "timestamp": "2019-01-30T14:18:46,493-0800", "level": "WARN", "component": "o.e.d.i.q.QueryShardContext", "cluster.name": "es1", "node.name": "es1_1", "cluster.uuid": "S4dWw65ZT1eu3SltmAr84A", "node.id": "gCoNXf3qSQ6a190zBKr7Bw", "message": "[types removal] Using the _type field in queries and aggregations is deprecated, prefer to use a field instead." } +{"type": "deprecation.elasticsearch", "timestamp": "2021-07-09T10:33:11,962+10:00", "level": "DEPRECATION", "component": "o.e.d.r.RestController", "cluster.name": "elasticsearch", "node.name": "ecenter.local", "message": "Legacy index templates are deprecated in favor of composable templates.", "cluster.uuid": "k_63KpgPSDeFRu1NxnlDiw", "node.id": "VLed1qvSSOCitAo-UeM3Jg" } +{"type": "deprecation.elasticsearch", "timestamp": "2021-07-09T10:42:03,065+10:00", "level": "DEPRECATION", "component": "o.e.d.c.m.MetadataCreateIndexService", "cluster.name": "elasticsearch", "node.name": "ecenter.local", "message": "index name [.kibana-event-log-7.13.3-000001] starts with a dot '.', in the next major version, index names starting with a dot are reserved for hidden indices and system indices", "cluster.uuid": "k_63KpgPSDeFRu1NxnlDiw", "node.id": "VLed1qvSSOCitAo-UeM3Jg" } +{"type": "deprecation.elasticsearch", "timestamp": "2021-07-09T10:48:46,329+10:00", "level": "DEPRECATION", "component": "o.e.d.a.a.i.a.g.TransportGetAliasesAction", "cluster.name": "elasticsearch", "node.name": "ecenter.local", "message": "this request accesses system indices: [.kibana_task_manager_7.13.3_001, .apm-agent-configuration, .kibana_7.13.3_001, .apm-custom-link, .async-search], but in a future major version, direct access to system indices will be prevented by default", "cluster.uuid": "k_63KpgPSDeFRu1NxnlDiw", "node.id": "VLed1qvSSOCitAo-UeM3Jg" } diff --git a/filebeat/module/elasticsearch/deprecation/test/test-json.log-expected.json b/filebeat/module/elasticsearch/deprecation/test/test-json.log-expected.json index 446721df60c1..f7e46255f6b2 100644 --- a/filebeat/module/elasticsearch/deprecation/test/test-json.log-expected.json +++ b/filebeat/module/elasticsearch/deprecation/test/test-json.log-expected.json @@ -258,5 +258,65 @@ "log.offset": 4459, "message": "[types removal] Using the _type field in queries and aggregations is deprecated, prefer to use a field instead.", "service.type": "elasticsearch" + }, + { + "@timestamp": "2021-07-09T00:33:11.962Z", + "elasticsearch.cluster.name": "elasticsearch", + "elasticsearch.cluster.uuid": "k_63KpgPSDeFRu1NxnlDiw", + "elasticsearch.component": "o.e.d.r.RestController", + "elasticsearch.node.id": "VLed1qvSSOCitAo-UeM3Jg", + "elasticsearch.node.name": "ecenter.local", + "event.category": "database", + "event.dataset": "elasticsearch.deprecation", + "event.kind": "event", + "event.module": "elasticsearch", + "event.type": "info", + "fileset.name": "deprecation", + "host.id": "VLed1qvSSOCitAo-UeM3Jg", + "input.type": "log", + "log.level": "DEPRECATION", + "log.offset": 4842, + "message": "Legacy index templates are deprecated in favor of composable templates.", + "service.type": "elasticsearch" + }, + { + "@timestamp": "2021-07-09T00:42:03.065Z", + "elasticsearch.cluster.name": "elasticsearch", + "elasticsearch.cluster.uuid": "k_63KpgPSDeFRu1NxnlDiw", + "elasticsearch.component": "o.e.d.c.m.MetadataCreateIndexService", + "elasticsearch.node.id": "VLed1qvSSOCitAo-UeM3Jg", + "elasticsearch.node.name": "ecenter.local", + "event.category": "database", + "event.dataset": "elasticsearch.deprecation", + "event.kind": "event", + "event.module": "elasticsearch", + "event.type": "info", + "fileset.name": "deprecation", + "host.id": "VLed1qvSSOCitAo-UeM3Jg", + "input.type": "log", + "log.level": "DEPRECATION", + "log.offset": 5219, + "message": "index name [.kibana-event-log-7.13.3-000001] starts with a dot '.', in the next major version, index names starting with a dot are reserved for hidden indices and system indices", + "service.type": "elasticsearch" + }, + { + "@timestamp": "2021-07-09T00:48:46.329Z", + "elasticsearch.cluster.name": "elasticsearch", + "elasticsearch.cluster.uuid": "k_63KpgPSDeFRu1NxnlDiw", + "elasticsearch.component": "o.e.d.a.a.i.a.g.TransportGetAliasesAction", + "elasticsearch.node.id": "VLed1qvSSOCitAo-UeM3Jg", + "elasticsearch.node.name": "ecenter.local", + "event.category": "database", + "event.dataset": "elasticsearch.deprecation", + "event.kind": "event", + "event.module": "elasticsearch", + "event.type": "info", + "fileset.name": "deprecation", + "host.id": "VLed1qvSSOCitAo-UeM3Jg", + "input.type": "log", + "log.level": "DEPRECATION", + "log.offset": 5716, + "message": "this request accesses system indices: [.kibana_task_manager_7.13.3_001, .apm-agent-configuration, .kibana_7.13.3_001, .apm-custom-link, .async-search], but in a future major version, direct access to system indices will be prevented by default", + "service.type": "elasticsearch" } ] \ No newline at end of file