diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 193e83480c3..49950c1c3d6 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -60,6 +60,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Add shared_credential_file to cloudtrail config {issue}15652[15652] {pull}15656[15656] - Fix typos in zeek notice fileset config file. {issue}15764[15764] {pull}15765[15765] - Improve `elasticsearch/audit` fileset to handle timestamps correctly. {pull}15942[15942] +- Prevent Elasticsearch from spewing log warnings about redundant wildcards when setting up ingest pipelines for the `elasticsearch` module. {issue}15840[15840] {pull}15900[15900] *Heartbeat* diff --git a/filebeat/module/elasticsearch/deprecation/ingest/pipeline-plaintext.json b/filebeat/module/elasticsearch/deprecation/ingest/pipeline-plaintext.json index d9c4faada7c..c4276664547 100755 --- a/filebeat/module/elasticsearch/deprecation/ingest/pipeline-plaintext.json +++ b/filebeat/module/elasticsearch/deprecation/ingest/pipeline-plaintext.json @@ -16,7 +16,7 @@ "GREEDYMULTILINE": "(.|\n)*" }, "patterns": [ - "\\[%{TIMESTAMP_ISO8601:elasticsearch.deprecation.timestamp}\\]\\[%{LOGLEVEL:log.level}%{SPACE}*\\]\\[%{DATA:elasticsearch.component}%{SPACE}*\\] %{GREEDYMULTILINE:message}" + "\\[%{TIMESTAMP_ISO8601:elasticsearch.deprecation.timestamp}\\]\\[%{LOGLEVEL:log.level}%{SPACE}\\]\\[%{DATA:elasticsearch.component}%{SPACE}\\] %{GREEDYMULTILINE:message}" ] } }, diff --git a/filebeat/module/elasticsearch/gc/ingest/pipeline.json b/filebeat/module/elasticsearch/gc/ingest/pipeline.json index 9cdea8d5e34..3dbc83bdfe2 100644 --- a/filebeat/module/elasticsearch/gc/ingest/pipeline.json +++ b/filebeat/module/elasticsearch/gc/ingest/pipeline.json @@ -15,7 +15,7 @@ "pattern_definitions": { "GREEDYMULTILINE": "(.|\n)*", "JVM8HEADER": "%{TIMESTAMP_ISO8601:timestamp}: %{BASE10NUM:elasticsearch.gc.jvm_runtime_sec}:", - "JVM9HEADER": "\\[%{TIMESTAMP_ISO8601:timestamp}\\]\\[%{POSINT:process.pid}\\]\\[%{DATA:elasticsearch.gc.tags}%{SPACE}*\\]", + "JVM9HEADER": "\\[%{TIMESTAMP_ISO8601:timestamp}\\]\\[%{POSINT:process.pid}\\]\\[%{DATA:elasticsearch.gc.tags}%{SPACE}\\]", "PROCTIME": "\\[Times: user=%{BASE10NUM:elasticsearch.gc.phase.cpu_time.user_sec} sys=%{BASE10NUM:elasticsearch.gc.phase.cpu_time.sys_sec}, real=%{BASE10NUM:elasticsearch.gc.phase.cpu_time.real_sec} secs\\]" } } diff --git a/filebeat/module/elasticsearch/server/ingest/pipeline-plaintext.json b/filebeat/module/elasticsearch/server/ingest/pipeline-plaintext.json index b1752133d3a..0da534584b3 100755 --- a/filebeat/module/elasticsearch/server/ingest/pipeline-plaintext.json +++ b/filebeat/module/elasticsearch/server/ingest/pipeline-plaintext.json @@ -17,7 +17,7 @@ "INDEXNAME": "[a-zA-Z0-9_.-]*", "GC_ALL": "\\[gc\\]\\[%{NUMBER:elasticsearch.server.gc.overhead_seq}\\] overhead, spent \\[%{NUMBER:elasticsearch.server.gc.collection_duration.time:float}%{DATA:elasticsearch.server.gc.collection_duration.unit}\\] collecting in the last \\[%{NUMBER:elasticsearch.server.gc.observation_duration.time:float}%{DATA:elasticsearch.server.gc.observation_duration.unit}\\]", "GC_YOUNG": "\\[gc\\]\\[young\\]\\[%{NUMBER:elasticsearch.server.gc.young.one}\\]\\[%{NUMBER:elasticsearch.server.gc.young.two}\\]%{SPACE}%{GREEDYMULTILINE:message}", - "LOG_HEADER": "\\[%{TIMESTAMP_ISO8601:elasticsearch.server.timestamp}\\]\\[%{LOGLEVEL:log.level}%{SPACE}?\\]\\[%{DATA:elasticsearch.component}%{SPACE}\\](%{SPACE})?(\\[%{DATA:elasticsearch.node.name}\\])?(%{SPACE})?" + "LOG_HEADER": "\\[%{TIMESTAMP_ISO8601:elasticsearch.server.timestamp}\\]\\[%{LOGLEVEL:log.level}%{SPACE}\\]\\[%{DATA:elasticsearch.component}%{SPACE}\\](%{SPACE})?(\\[%{DATA:elasticsearch.node.name}\\])?(%{SPACE})?" }, "patterns": [ "%{LOG_HEADER}%{GC_ALL}", diff --git a/filebeat/module/elasticsearch/slowlog/ingest/pipeline-json.json b/filebeat/module/elasticsearch/slowlog/ingest/pipeline-json.json index ff0794d0089..d37d170cf43 100644 --- a/filebeat/module/elasticsearch/slowlog/ingest/pipeline-json.json +++ b/filebeat/module/elasticsearch/slowlog/ingest/pipeline-json.json @@ -116,7 +116,7 @@ "INDEXNAME": "[a-zA-Z0-9_.-]*" }, "patterns": [ - "(\\[%{INDEXNAME:elasticsearch.index.name}\\]\\[%{NUMBER:elasticsearch.shard.id}\\])?(%{SPACE})?(\\[%{INDEXNAME:elasticsearch.index.name}\\/%{DATA:elasticsearch.index.id}\\])?(%{SPACE})?%{SPACE}(took\\[%{DATA:elasticsearch.slowlog.took}\\],)?%{SPACE}(took_millis\\[%{NUMBER:elasticsearch.slowlog.duration:long}\\],)?%{SPACE}(type\\[%{DATA:elasticsearch.slowlog.type}\\],)?%{SPACE}(id\\[%{DATA:elasticsearch.slowlog.id}\\],)?%{SPACE}(routing\\[%{DATA:elasticsearch.slowlog.routing}\\],)?%{SPACE}(total_hits\\[%{NUMBER:elasticsearch.slowlog.total_hits:int}\\],)?%{SPACE}(types\\[%{DATA:elasticsearch.slowlog.types}\\],)?%{SPACE}(stats\\[%{DATA:elasticsearch.slowlog.stats}\\],)?%{SPACE}(search_type\\[%{DATA:elasticsearch.slowlog.search_type}\\],)?%{SPACE}(total_shards\\[%{NUMBER:elasticsearch.slowlog.total_shards:int}\\],)?%{SPACE}(source\\[%{GREEDYMULTILINE:elasticsearch.slowlog.source_query}\\])?,?%{SPACE}(extra_source\\[%{DATA:elasticsearch.slowlog.extra_source}\\])?,?", + "(\\[%{INDEXNAME:elasticsearch.index.name}\\]\\[%{NUMBER:elasticsearch.shard.id}\\])?(%{SPACE})(\\[%{INDEXNAME:elasticsearch.index.name}\\/%{DATA:elasticsearch.index.id}\\])?(%{SPACE})%{SPACE}(took\\[%{DATA:elasticsearch.slowlog.took}\\],)?%{SPACE}(took_millis\\[%{NUMBER:elasticsearch.slowlog.duration:long}\\],)?%{SPACE}(type\\[%{DATA:elasticsearch.slowlog.type}\\],)?%{SPACE}(id\\[%{DATA:elasticsearch.slowlog.id}\\],)?%{SPACE}(routing\\[%{DATA:elasticsearch.slowlog.routing}\\],)?%{SPACE}(total_hits\\[%{NUMBER:elasticsearch.slowlog.total_hits:int}\\],)?%{SPACE}(types\\[%{DATA:elasticsearch.slowlog.types}\\],)?%{SPACE}(stats\\[%{DATA:elasticsearch.slowlog.stats}\\],)?%{SPACE}(search_type\\[%{DATA:elasticsearch.slowlog.search_type}\\],)?%{SPACE}(total_shards\\[%{NUMBER:elasticsearch.slowlog.total_shards:int}\\],)?%{SPACE}(source\\[%{GREEDYMULTILINE:elasticsearch.slowlog.source_query}\\])?,?%{SPACE}(extra_source\\[%{DATA:elasticsearch.slowlog.extra_source}\\])?,?", "\\[%{INDEXNAME:elasticsearch.index.name}\\]\\[%{NUMBER:elasticsearch.shard.id}\\]" ] } diff --git a/filebeat/module/elasticsearch/slowlog/ingest/pipeline-plaintext.json b/filebeat/module/elasticsearch/slowlog/ingest/pipeline-plaintext.json index ae88869d0c4..e27d3ce0f81 100644 --- a/filebeat/module/elasticsearch/slowlog/ingest/pipeline-plaintext.json +++ b/filebeat/module/elasticsearch/slowlog/ingest/pipeline-plaintext.json @@ -9,7 +9,7 @@ "INDEXNAME": "[a-zA-Z0-9_.-]*" }, "patterns": [ - "\\[%{TIMESTAMP_ISO8601:elasticsearch.slowlog.timestamp}\\]\\[%{WORD:log.level}(%{SPACE})?\\]\\[%{DATA:elasticsearch.slowlog.logger}\\]%{SPACE}\\[%{DATA:elasticsearch.node.name}\\](%{SPACE})?(\\[%{INDEXNAME:elasticsearch.index.name}\\]\\[%{NUMBER:elasticsearch.shard.id}\\])?(%{SPACE})?(\\[%{INDEXNAME:elasticsearch.index.name}\\/%{DATA:elasticsearch.index.id}\\])?(%{SPACE})?%{SPACE}(took\\[%{DATA:elasticsearch.slowlog.took}\\],)?%{SPACE}(took_millis\\[%{NUMBER:elasticsearch.slowlog.duration:long}\\],)?%{SPACE}(type\\[%{DATA:elasticsearch.slowlog.type}\\],)?%{SPACE}(id\\[%{DATA:elasticsearch.slowlog.id}\\],)?%{SPACE}(routing\\[%{DATA:elasticsearch.slowlog.routing}\\],)?%{SPACE}(total_hits\\[%{NUMBER:elasticsearch.slowlog.total_hits:int}\\],)?%{SPACE}(types\\[%{DATA:elasticsearch.slowlog.types}\\],)?%{SPACE}(stats\\[%{DATA:elasticsearch.slowlog.stats}\\],)?%{SPACE}(search_type\\[%{DATA:elasticsearch.slowlog.search_type}\\],)?%{SPACE}(total_shards\\[%{NUMBER:elasticsearch.slowlog.total_shards:int}\\],)?%{SPACE}(source\\[%{GREEDYMULTILINE:elasticsearch.slowlog.source_query}\\])?,?%{SPACE}(extra_source\\[%{DATA:elasticsearch.slowlog.extra_source}\\])?,?" + "\\[%{TIMESTAMP_ISO8601:elasticsearch.slowlog.timestamp}\\]\\[%{WORD:log.level}(%{SPACE})\\]\\[%{DATA:elasticsearch.slowlog.logger}\\]%{SPACE}\\[%{DATA:elasticsearch.node.name}\\](%{SPACE})?(\\[%{INDEXNAME:elasticsearch.index.name}\\]\\[%{NUMBER:elasticsearch.shard.id}\\])?(%{SPACE})?(\\[%{INDEXNAME:elasticsearch.index.name}\\/%{DATA:elasticsearch.index.id}\\])?(%{SPACE})?%{SPACE}(took\\[%{DATA:elasticsearch.slowlog.took}\\],)?%{SPACE}(took_millis\\[%{NUMBER:elasticsearch.slowlog.duration:long}\\],)?%{SPACE}(type\\[%{DATA:elasticsearch.slowlog.type}\\],)?%{SPACE}(id\\[%{DATA:elasticsearch.slowlog.id}\\],)?%{SPACE}(routing\\[%{DATA:elasticsearch.slowlog.routing}\\],)?%{SPACE}(total_hits\\[%{NUMBER:elasticsearch.slowlog.total_hits:int}\\],)?%{SPACE}(types\\[%{DATA:elasticsearch.slowlog.types}\\],)?%{SPACE}(stats\\[%{DATA:elasticsearch.slowlog.stats}\\],)?%{SPACE}(search_type\\[%{DATA:elasticsearch.slowlog.search_type}\\],)?%{SPACE}(total_shards\\[%{NUMBER:elasticsearch.slowlog.total_shards:int}\\],)?%{SPACE}(source\\[%{GREEDYMULTILINE:elasticsearch.slowlog.source_query}\\])?,?%{SPACE}(extra_source\\[%{DATA:elasticsearch.slowlog.extra_source}\\])?,?" ] } },