From d7e586066c3a9f5704b425debdde726d548b371e Mon Sep 17 00:00:00 2001 From: sayden Date: Wed, 9 Dec 2020 17:21:36 +0100 Subject: [PATCH] Cleaning things up --- metricbeat/docs/fields.asciidoc | 263 ++------------ metricbeat/module/logstash/_meta/fields.yml | 127 ++----- metricbeat/module/logstash/fields.go | 2 +- .../module/logstash/node/_meta/data.json | 139 +++++++- .../module/logstash/node/_meta/fields.yml | 11 + metricbeat/module/logstash/node/data.go | 123 ++++++- metricbeat/module/logstash/node/data_test.go | 303 +++++++++++++++- metricbeat/module/logstash/node/data_xpack.go | 120 ------- .../module/logstash/node/data_xpack_test.go | 328 ------------------ metricbeat/module/logstash/node/node.go | 30 +- .../logstash/node_stats/_meta/data.json | 250 +++++++------ .../logstash/node_stats/_meta/fields.yml | 39 --- .../node_stats/_meta/test/node_stats.710.json | 192 ++++++++++ .../module/logstash/node_stats/data_test.go | 5 +- 14 files changed, 917 insertions(+), 1015 deletions(-) delete mode 100644 metricbeat/module/logstash/node/data_xpack.go delete mode 100644 metricbeat/module/logstash/node/data_xpack_test.go create mode 100644 metricbeat/module/logstash/node_stats/_meta/test/node_stats.710.json diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index 9348f91f3be6..e3483b7c66e2 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -33169,51 +33169,12 @@ type: nested -*`logstash_stats.os.cpu.load_average.15m`*:: -+ --- -type: alias - -alias to: logstash.node.stats.os.cpu.load_average.15m - --- - -*`logstash_stats.os.cpu.load_average.1m`*:: -+ --- -type: alias - -alias to: logstash.node.stats.os.cpu.load_average.1m - --- - -*`logstash_stats.os.cpu.load_average.5m`*:: -+ --- -type: alias - -alias to: logstash.node.stats.os.cpu.load_average.5m - --- - - -*`logstash_stats.os.cgroup.cpuacct.usage_nanos`*:: -+ --- -type: alias - -alias to: logstash.node.stats.os.cgroup.cpuacct.usage_nanos - --- - - - *`logstash_stats.os.cpu.stat.number_of_elapsed_periods`*:: + -- type: alias -alias to: logstash.node.stats.os.cgroup.stat.number_of_elapsed_periods +alias to: logstash.node.stats.os.cgroup.cpu.stat.number_of_elapsed_periods -- @@ -33222,7 +33183,7 @@ alias to: logstash.node.stats.os.cgroup.stat.number_of_elapsed_periods -- type: alias -alias to: logstash.node.stats.os.cgroup.stat.time_throttled_nanos +alias to: logstash.node.stats.os.cgroup.cpu.stat.time_throttled_nanos -- @@ -33231,215 +33192,114 @@ alias to: logstash.node.stats.os.cgroup.stat.time_throttled_nanos -- type: alias -alias to: logstash.node.stats.os.cgroup.stat.number_of_times_throttled +alias to: logstash.node.stats.os.cgroup.cpu.stat.number_of_times_throttled -- -*`logstash_stats.process.cpu.percent`*:: -+ --- -type: alias - -alias to: logstash.node.stats.process.cpu.percent - --- - -*`logstash_stats.queue.events_count`*:: -+ --- -type: alias - -alias to: logstash.node.stats.queue.events_count - --- - -[float] -=== logstash -TODO - - - - -*`logstash.stats.timestamp`*:: -+ --- -type: alias - -alias to: @timestamp - --- - - - -*`logstash.stats.jvm.mem.heap_used.bytes`*:: -+ --- -type: alias - -alias to: logstash.node.stats.jvm.mem.heap_used_in_bytes - --- - -*`logstash.stats.jvm.mem.heap_max.bytes`*:: +*`logstash_stats.os.cpu.load_average.15m`*:: + -- type: alias -alias to: logstash.node.stats.jvm.mem.heap_max_in_bytes +alias to: logstash.node.stats.os.cpu.load_average.15m -- -*`logstash.stats.jvm.uptime.ms`*:: +*`logstash_stats.os.cpu.load_average.1m`*:: + -- type: alias -alias to: logstash.node.stats.jvm.uptime_in_millis +alias to: logstash.node.stats.os.cpu.load_average.1m -- - -*`logstash.stats.events.in`*:: +*`logstash_stats.os.cpu.load_average.5m`*:: + -- type: alias -alias to: logstash.node.stats.events.in - --- +alias to: logstash.node.stats.os.cpu.load_average.5m -*`logstash.stats.events.out`*:: -+ -- -type: alias - -alias to: logstash.node.stats.events.out --- -*`logstash.stats.events.duration.ms`*:: +*`logstash_stats.os.cgroup.cpuacct.usage_nanos`*:: + -- type: alias -alias to: logstash.node.stats.events.duration_in_millis +alias to: logstash.node.stats.os.cgroup.cpuacct.usage_nanos -- - -*`logstash.stats.logstash.uuid`*:: +*`logstash_stats.process.cpu.percent`*:: + -- type: alias -alias to: logstash.node.stats.logstash.uuid +alias to: logstash.node.stats.process.cpu.percent -- -*`logstash.stats.logstash.version`*:: +*`logstash_stats.queue.events_count`*:: + -- type: alias -alias to: logstash.node.stats.logstash.version +alias to: logstash.node.stats.queue.events_count -- - - -*`logstash.stats.os.cpu.load_average.15m`*:: +*`logstash_state.pipeline.id`*:: + -- type: alias -alias to: logstash.node.stats.os.cpu.load_average.15m +alias to: logstash.node.state.pipeline.id -- -*`logstash.stats.os.cpu.load_average.1m`*:: +*`logstash_state.pipeline.hash`*:: + -- type: alias -alias to: logstash.node.stats.os.cpu.load_average.1m +alias to: logstash.node.state.pipeline.hash -- -*`logstash.stats.os.cpu.load_average.5m`*:: -+ --- -type: alias - -alias to: logstash.node.stats.os.cpu.load_average.5m - --- - - -*`logstash.stats.os.cgroup.cpuacct.usage_nanos`*:: -+ --- -type: alias - -alias to: logstash.node.stats.os.cgroup.cpuacct.usage_nanos - --- +[float] +=== node +node -*`logstash.stats.os.cpu.stat.number_of_elapsed_periods`*:: -+ --- -type: alias -alias to: logstash.node.stats.os.cgroup.stat.number_of_elapsed_periods --- -*`logstash.stats.os.cpu.stat.time_throttled_nanos`*:: +*`logstash.node.state.pipeline.id`*:: + -- -type: alias - -alias to: logstash.node.stats.os.cgroup.stat.time_throttled_nanos - --- +TODO -*`logstash.stats.os.cpu.stat.number_of_times_throttled`*:: -+ --- -type: alias -alias to: logstash.node.stats.os.cgroup.stat.number_of_times_throttled +type: keyword -- -*`logstash.stats.process.cpu.percent`*:: +*`logstash.node.state.pipeline.hash`*:: + -- -type: alias - -alias to: logstash.node.stats.process.cpu.percent - --- +TODO -*`logstash.stats.queue.events_count`*:: -+ --- -type: alias -alias to: logstash.node.stats.queue.events_count +type: keyword -- -[float] -=== node - -node - - - *`logstash.node.host`*:: + -- @@ -33504,9 +33364,6 @@ node_stats metrics. *`logstash.node.stats.jvm.uptime_in_millis`*:: + -- -TODO - - type: long -- @@ -33515,9 +33372,6 @@ type: long *`logstash.node.stats.jvm.mem.heap_used_in_bytes`*:: + -- -TODO - - type: long -- @@ -33525,9 +33379,6 @@ type: long *`logstash.node.stats.jvm.mem.heap_max_in_bytes`*:: + -- -TODO - - type: long -- @@ -33572,8 +33423,6 @@ type: long *`logstash.node.stats.events.duration_in_millis`*:: + -- -TODO - type: long -- @@ -33582,9 +33431,6 @@ type: long *`logstash.node.stats.logstash.uuid`*:: + -- -TODO - - type: keyword -- @@ -33592,9 +33438,6 @@ type: keyword *`logstash.node.stats.logstash.version`*:: + -- -TODO - - type: keyword -- @@ -33605,9 +33448,6 @@ type: keyword *`logstash.node.stats.os.cpu.load_average.15m`*:: + -- -TODO - - type: long -- @@ -33615,9 +33455,6 @@ type: long *`logstash.node.stats.os.cpu.load_average.1m`*:: + -- -TODO - - type: long -- @@ -33625,9 +33462,6 @@ type: long *`logstash.node.stats.os.cpu.load_average.5m`*:: + -- -TODO - - type: long -- @@ -33636,9 +33470,6 @@ type: long *`logstash.node.stats.os.cgroup.cpuact.usage_nanos`*:: + -- -TODO - - type: long -- @@ -33689,9 +33520,6 @@ type: double *`logstash.node.stats.pipelines.id`*:: + -- -TODO - - type: keyword -- @@ -33699,9 +33527,6 @@ type: keyword *`logstash.node.stats.pipelines.hash`*:: + -- -TODO - - type: keyword -- @@ -33710,8 +33535,6 @@ type: keyword *`logstash.node.stats.pipelines.queue.events_count`*:: + -- -TODO - type: long -- @@ -33719,8 +33542,6 @@ type: long *`logstash.node.stats.pipelines.queue.type`*:: + -- -TODO - type: keyword -- @@ -33728,8 +33549,6 @@ type: keyword *`logstash.node.stats.pipelines.queue.queue_size_in_bytes`*:: + -- -TODO - type: long -- @@ -33737,8 +33556,6 @@ type: long *`logstash.node.stats.pipelines.queue.max_queue_size_in_bytes`*:: + -- -TODO - type: long -- @@ -33747,8 +33564,6 @@ type: long *`logstash.node.stats.pipelines.events.out`*:: + -- -TODO - type: long -- @@ -33756,8 +33571,6 @@ type: long *`logstash.node.stats.pipelines.events.duration_in_millis`*:: + -- -TODO - type: long -- @@ -33766,8 +33579,6 @@ type: long *`logstash.node.stats.pipelines.vertices.duration_in_millis`*:: + -- -TODO - type: long -- @@ -33775,8 +33586,6 @@ type: long *`logstash.node.stats.pipelines.vertices.events_in`*:: + -- -TODO - type: long -- @@ -33823,8 +33632,6 @@ type: float *`logstash.node.stats.os.cpu.load_average.15m`*:: + -- -TODO - type: long -- @@ -33832,8 +33639,6 @@ type: long *`logstash.node.stats.os.cpu.load_average.1m`*:: + -- -TODO - type: long -- @@ -33841,8 +33646,6 @@ type: long *`logstash.node.stats.os.cpu.load_average.5m`*:: + -- -TODO - type: long -- @@ -33851,8 +33654,6 @@ type: long *`logstash.node.stats.os.cgroup.cpuacct.usage_nanos`*:: + -- -TODO - type: long -- @@ -33862,8 +33663,6 @@ type: long *`logstash.node.stats.os.cgroup.cpu.stat.number_of_elapsed_periods`*:: + -- -TODO - type: long -- @@ -33871,8 +33670,6 @@ type: long *`logstash.node.stats.os.cgroup.cpu.stat.time_throttled_nanos`*:: + -- -TODO - type: long -- @@ -33880,8 +33677,6 @@ type: long *`logstash.node.stats.os.cgroup.cpu.stat.number_of_times_throttled`*:: + -- -TODO - type: long -- @@ -33889,8 +33684,6 @@ type: long *`logstash.node.stats.queue.events_count`*:: + -- -TODO - type: long -- diff --git a/metricbeat/module/logstash/_meta/fields.yml b/metricbeat/module/logstash/_meta/fields.yml index 79307d9a8596..2b39900e3b30 100644 --- a/metricbeat/module/logstash/_meta/fields.yml +++ b/metricbeat/module/logstash/_meta/fields.yml @@ -4,6 +4,7 @@ Logstash module release: ga settings: ["ssl", "http"] + short_config: false fields: - name: logstash_stats type: group @@ -104,6 +105,18 @@ - name: cpu type: group fields: + - name: stat + type: group + fields: + - name: number_of_elapsed_periods + type: alias + path: logstash.node.stats.os.cgroup.cpu.stat.number_of_elapsed_periods + - name: time_throttled_nanos + type: alias + path: logstash.node.stats.os.cgroup.cpu.stat.time_throttled_nanos + - name: number_of_times_throttled + type: alias + path: logstash.node.stats.os.cgroup.cpu.stat.number_of_times_throttled - name: load_average type: group fields: @@ -122,117 +135,21 @@ - name: cpuacct.usage_nanos type: alias path: logstash.node.stats.os.cgroup.cpuacct.usage_nanos - - name: cpu - type: group - fields: - - name: stat - type: group - fields: - - name: number_of_elapsed_periods - type: alias - path: logstash.node.stats.os.cgroup.stat.number_of_elapsed_periods - - name: time_throttled_nanos - type: alias - path: logstash.node.stats.os.cgroup.stat.time_throttled_nanos - - name: number_of_times_throttled - type: alias - path: logstash.node.stats.os.cgroup.stat.number_of_times_throttled - name: process.cpu.percent type: alias path: logstash.node.stats.process.cpu.percent - name: queue.events_count type: alias path: logstash.node.stats.queue.events_count + - name: logstash_state + type: group + fields: + - name: pipeline.id + type: alias + path: logstash.node.state.pipeline.id + - name: pipeline.hash + type: alias + path: logstash.node.state.pipeline.hash - name: logstash type: group - description: > - TODO fields: - - name: stats - type: group - fields: - - name: timestamp - type: alias - path: "@timestamp" - - name: jvm - type: group - fields: - - name: mem - type: group - fields: - - name: heap_used.bytes - type: alias - path: logstash.node.stats.jvm.mem.heap_used_in_bytes - - name: heap_max.bytes - type: alias - path: logstash.node.stats.jvm.mem.heap_max_in_bytes - - name: uptime.ms - type: alias - path: logstash.node.stats.jvm.uptime_in_millis - - name: events - type: group - fields: - - name: in - type: alias - path: logstash.node.stats.events.in - - name: out - type: alias - path: logstash.node.stats.events.out - - name: duration.ms - type: alias - path: logstash.node.stats.events.duration_in_millis - - name: logstash - type: group - fields: - - name: uuid - type: alias - path: logstash.node.stats.logstash.uuid - - name: version - type: alias - path: logstash.node.stats.logstash.version - - name: os - type: group - fields: - - name: cpu - type: group - fields: - - name: load_average - type: group - fields: - - name: 15m - type: alias - path: logstash.node.stats.os.cpu.load_average.15m - - name: 1m - type: alias - path: logstash.node.stats.os.cpu.load_average.1m - - name: 5m - type: alias - path: logstash.node.stats.os.cpu.load_average.5m - - name: cgroup - type: group - fields: - - name: cpuacct.usage_nanos - type: alias - path: logstash.node.stats.os.cgroup.cpuacct.usage_nanos - - name: cpu - type: group - fields: - - name: stat - type: group - fields: - - name: number_of_elapsed_periods - type: alias - path: logstash.node.stats.os.cgroup.stat.number_of_elapsed_periods - - name: time_throttled_nanos - type: alias - path: logstash.node.stats.os.cgroup.stat.time_throttled_nanos - - name: number_of_times_throttled - type: alias - path: logstash.node.stats.os.cgroup.stat.number_of_times_throttled - - name: process.cpu.percent - type: alias - path: logstash.node.stats.process.cpu.percent - - name: queue.events_count - type: alias - path: logstash.node.stats.queue.events_count - \ No newline at end of file diff --git a/metricbeat/module/logstash/fields.go b/metricbeat/module/logstash/fields.go index d1628941a223..923fa1b54fd5 100644 --- a/metricbeat/module/logstash/fields.go +++ b/metricbeat/module/logstash/fields.go @@ -32,5 +32,5 @@ func init() { // AssetLogstash returns asset data. // This is the base64 encoded gzipped contents of module/logstash. func AssetLogstash() string { - return "eJzsWk+PpLYTvfenKPX5t5Z+h1z6EOWwiTJRoskh2ksUIQ/UdHsXMMGmM5NPH9HdMID/AjY9h+Gw0jJNvVdlu8x7+BN8w9cD5PwoJBWnHYBkMscD7H+93drvADIUac0qyXh5gO93AADdn6HgWZPjDqDGHKnAAxzpDkCglKw8igP8uRci3/8P9icpq/1fO4BnhnkmDpc4n6CkBb4xSISkUlz+BCBfqzZezZvqdmf46PBxyQoUkhZV/5fuaZozKgZ3KypPB9j/0D+xV4J9PRdKmCEJHZHh8wUWo/umGKY4w1gnpFXSCMwSViZPrxKF8lNTpuOMuwKTkmdILlUmX88FKbAgHiAjPgV9iU7HiNExaap2CNvfFCzP2RTKTMJOwBi2w8UzllKsmiCsDED2yoNMYnUQvJHhMKbBOpCsqWnbFIIOwg3TEnvaM1aNRdOwLADr/p4SrwM6Yy0YDzHy/b1pyA6qYhXmrER1lpYoJGa+0zREYXouxFCX03gE1+MoATukvxtsMFhnbp8M2Pze+F9oEm34USaJYP9ijD48peKD1u999CXZlp0PorF3w6opoDbZMIkZ2i7Mab1BmTjQBi1OslQZbUPr8anwRml2xF2JgjKXEmUvD8zIDDPt9glWJyywpnmidO7ApPwgJ5WKtVqmpbItm9iFsVTh2qOqRpySrWe1H3Tf1da93KZVE6zB5pxmCT1jTY/mvVYX2BZ8CPD/76ZSbRzbVH6wDgEXJK0aMmRPTEg9k82I2HlsVpAJUD99dMO5fAalVUPTVJJG0CMmJS15yLXW5nVhRVw4MVZHyyHKqiib4gnrhD8nmNNKYJZUWDOemQZ+7dS4lrD9P5kHPbR/EnmquZQ5ZsZhDszVG1Wt68V9ent2y7qaoPvXiZqnKK4LtsI6xVIq+4HeVNPuRpZoo52x271T3qwCNAQzeAbqWtF4ne31x+Pnx9t/TB7k0LnUx9Y9DQ4X0z4PLG4mGB1NMzkTQbC6m/Z4tpigczqJSTHaS9FdQexOhVhBX7bhZfVWx94nKUK7r1ZPNZKIDiqg9KYoRNPsM5R62NGaJ8w1TimsGjSNa7omG7N7Ck4HNQjuLXT3hLaIyua+6q00Sg916CU3iAsIvLQTeDVECKahwEdHRSPl5rR5oTSAVn0FQWafn9byyzmE5oINVpxBg7mDuwBghR6DQNNqsS6DZdosAu9Z6Et1WgTefhT8NJud3jLtBr76bSm4M+hgl+7HjWfoFGAGgddek+fHJ1q6yybgTlzMzb19hLT/tBEmvyjY8fp6dwBZK58uLYm0189cSFCCrvsaLbA+sxSVD88B6PYHir5oYi/RsQ68X778Bg/lM5/55uZ6B/2Gr//wWtcoHHzaS5c6jL70LH3p7payPoJ94DzJ/36FgIfP2nVJ/OwRx+q8Hg+DAmXNUkFWLdaQpojjMNJb8JyXx2X1HThQKn50U8ZulTiy88zQmKXCyHEALSqhhTaIA+7HSzTFQ4R1Bsry2fZQprxg5fGWJlx2YKzJIl9lOY3HRh75HBrPLJdYa9/U1nL56Rbam8uMz7w+rIwzcVt3Z9UOZ21iUfdWY/U+bJ0xmI+tY2mr4N9awdbvYZ6nszGj+9doKz/H387ZYPPXLzB3tjBj/lscHT8gHzBY6e6A7zSDeVMNXAsAljs774LzEmtnU+JzLZ2MN0/5TK1tBtUcHId1X/fu8SKhOVK+BazufDkE6cRWG2wI4tuDnZ3WcLodPMo4H8zvwHiELP1PqwcCt4pHCDJX9EIsEH+YK28C4TpOmEOQym2ZEHieJI8A53teHJasdf/gXqfD56fvCGfdlxZl7Ehv7rHvNw7POTe+iY4Y+EN8aF49WGjNew9Vew/deldlGkGafgjQ+AL0/cnLdyoejSrN83u7Bk+N/18AAAD//29X95w=" + return "eJzsWk2Po0YQvc+vKPmcRcohFx+iHDZRJko0OUR7iSLUA2XTu0B3+sPZya+PwAYD/YlpWzkMhzmA+71X1UXRVTUf4Au+7aFmR6mIrJ4AFFU17mH36+XW7gmgRFkIyhVl7R6+fwIAGB5Dw0pd4xOAwBqJxD0cyROARKVoe5R7+HMnZb37BnaVUnz3V/esYkLlBWsP9LiHA6llt/5AsS7lvkf/AC1p8Korl4oo2T8CUG+8YxFM88ud6dLpckUblIo0fHwyrCY1JXJylxNV7WH3w7hiZ4B9PjUGzFSETch0fYPN7L4Lw4UzxaqQ8FxLLHPa5q9vCqXxU5elc4sHB2ctKzHrvZx9PjVZg00WQTLT05Cvd5fj5BiUaN5tYfebhtY1XVK5RfgFOGEHXjxhq+SmAKFtArFnHdkCa6BgWqXjWIINJKUWpEsVSTfhwunBXuaMTXuhNS0TqB7vGXgD0QmFpCzFzo/3lpADFacca9qiGaUtSoVlbJimcMyoJXP4pZrv4HYeA3Bg+lujxmSZuVuZMPld9fcyMyv8zJJc0n/xHnl4KSWGbfz2ka/5Y9XFMDpzN2wKATPJpjHMkXZhTepNqiTANklxihbGbjtST4yHH2TmIDxkKBixlBvf8sSK3DTLbJ8jr7BBQercyNyJRcVRLjx1r7dl6Srfa3Nvx3i8cM5RXMsqf3RUx1GPWW3b4bbgOlmC7SxyusYG6AOdAre6eUWRs0OONeFd5cNRUFbaXA3BzQDvhjCZFb3WrOC6v5eto58WubmqBFOqxjJvScsepDea2fRvX2df1z7avyH6ay1BypycUJCj+1S3JeK+/W7ZFEhlO9fZVH3mYhqVPEyIX8fDHLIgGhOVbTtvz1UF16QoVKYlOaLzFbk1q8+C3MkzHgkEK1CeXcFRFNiqyMaY9YviQZt93YYvcMH0JkIHmLVXOLyt8b3C4QM5/1iv04iZDcVgqGzdiRs4RhxH6yPe+A49+Hm3dIGHa7F+3goeLt/xYG5ZsrfPc7D7gm//MGF77rFzuP54+fjiJLW0Lu5FO1Iyubax1y3Juj8dwuIXDT2ez4N7UMJojgSE/sykAgN0W79LojjRAo3WVgK54yDjkwXb3vsHb0wG+H759Bs8twe2MpLtlkOauLKZDrPMdWuBNHwk7Aj+jYsU//uZAp4/WtNaNh0dwa3J7TyAggaVoIXMNuW6NcEUCovAuOMKXrP26EQxR1N+VT5lsHZMFdBo4AXGTB64G/t9gRj8sUeDZZj5fOSc+AR9EfE+PLcFa2h7vJgJ/UkJReZU4evA3C7jRasjWyPjQGuFwloLbtXy0wU6WsuKVowzyixzKNj2rpszKQjm/y3fD2vrZ5sNZhvIj+fDhPhCPUwSIoKooh3CuQxiau6VOLfq8da8kGRj+ro0VP5GmOsPoLDWkF4I9xfjiWLIYEPfca7EGyLnKyJHTi9reWMTvqL3+L/RvLYJ+XDhce2aq6SS6dd6ZdXjJrX8kwA8ugbfVEn75vp+S3zWgDm5WvazTJJgNnP8t0DYSMPQ4GB9har4aX0A1HvghiR7YT+8rjA26qgXaadj0p3G0hRCIXJSvQImds4MUTG9SFzx4FFT5QizFgICcN4cd5PFAfPWjouvGg41c55qZgriKd7rAzvZe31wvqLrg/cCIUB45wLh/sf5Ox2+o+aMi/X/BQAA///nVv4e" } diff --git a/metricbeat/module/logstash/node/_meta/data.json b/metricbeat/module/logstash/node/_meta/data.json index a9e3bbb153a3..ae1ab625ce36 100644 --- a/metricbeat/module/logstash/node/_meta/data.json +++ b/metricbeat/module/logstash/node/_meta/data.json @@ -1,33 +1,134 @@ { - "@timestamp": "2017-10-12T08:05:34.853Z", + "@timestamp": "2020-10-05T10:50:11.757Z", + "@metadata": { + "beat": "metricbeat", + "type": "_doc", + "version": "8.0.0", + "_id": "afb1a50a-95f0-484a-b7d7-e683ddddc75a" + }, + "host": { + "name": "mcastro" + }, "agent": { - "hostname": "host.example.com", - "name": "host.example.com" + "ephemeral_id": "c4b22628-7b30-4a5d-8e28-7b6de81c9974", + "id": "803dfdba-e638-4590-a2de-80cb1cebe78d", + "name": "mcastro", + "type": "metricbeat", + "version": "8.0.0" }, "event": { + "duration": 9740086, "dataset": "logstash.node", - "duration": 115000, "module": "logstash" }, + "metricset": { + "name": "node", + "period": 10000 + }, + "service": { + "address": "localhost:9600", + "type": "logstash" + }, "logstash": { "node": { + "host": "2cb47f6e0eab", + "version": "8.0.0", "jvm": { - "version": "1.8.0_191" + "version": "11.0.5" + }, + "id": "4cc683ce-3ddc-46e3-bea3-aefbf37bc082", + "state": { + "pipeline": { + "hash": "3000c3abf87d4dfa4a57aaf6af0a1f5bee2e0fc1c48a8e8636e2a33d7d2e91dd", + "ephemeral_id": "afb1a50a-95f0-484a-b7d7-e683ddddc75a", + "representation": { + "graph": { + "edges": [ + { + "from": "1bf3a9cc73ceb7c3a9cbe885df249b23f3496c52a342a6d513153cc865d78182", + "id": "b3db599ec6ae0b9493158bd7024dcd922c8a3e76295c37fef0da440086bf3f8c", + "to": "__QUEUE__", + "type": "plain" + }, + { + "type": "plain", + "from": "71b91bc85b66ab25c5fb16e63db4dd7111c183f96d1f18e19078051ed5fc74f7", + "id": "9db20a77b3e1eb91229a50bd33388425d59725f9093e076a37e6565f8d5a20ad", + "to": "__QUEUE__" + }, + { + "id": "9b2bc571e978746fb9b55b83521a6603c3c940144cde0e3f4296298cea6585cf", + "to": "a339cb309b29181703c6adf321da3d639f5b60713de5a1e5519ebfea069556d8", + "type": "plain", + "from": "__QUEUE__" + } + ], + "vertices": [ + { + "config_name": "beats", + "explicit_id": false, + "id": "1bf3a9cc73ceb7c3a9cbe885df249b23f3496c52a342a6d513153cc865d78182", + "meta": { + "source": { + "line": 2, + "protocol": "file", + "column": 3, + "id": "/usr/share/logstash/pipeline/default.conf" + } + }, + "plugin_type": "input", + "type": "plugin" + }, + { + "plugin_type": "input", + "type": "plugin", + "config_name": "beats", + "explicit_id": false, + "id": "71b91bc85b66ab25c5fb16e63db4dd7111c183f96d1f18e19078051ed5fc74f7", + "meta": { + "source": { + "protocol": "file", + "column": 3, + "id": "/usr/share/logstash/pipeline/default.conf", + "line": 7 + } + } + }, + { + "explicit_id": false, + "id": "__QUEUE__", + "meta": null, + "type": "queue" + }, + { + "config_name": "elasticsearch", + "explicit_id": false, + "id": "a339cb309b29181703c6adf321da3d639f5b60713de5a1e5519ebfea069556d8", + "meta": { + "source": { + "id": "/usr/share/logstash/pipeline/default.conf", + "line": 17, + "protocol": "file", + "column": 3 + } + }, + "plugin_type": "output", + "type": "plugin" + } + ] + }, + "type": "lir", + "version": "0.0.0", + "hash": "3000c3abf87d4dfa4a57aaf6af0a1f5bee2e0fc1c48a8e8636e2a33d7d2e91dd" + }, + "batch_size": 125, + "workers": 12, + "id": "main" + } } } }, - "metricset": { - "name": "node" - }, - "process": { - "pid": 93559 - }, - "service": { - "address": "127.0.0.1:9600", - "hostname": "Shaunaks-MBP-2.attlocal.net", - "id": "7565df20-c3aa-4261-81d5-3b0ab8d15c16", - "name": "logstash", - "type": "logstash", - "version": "7.0.0" + "ecs": { + "version": "1.5.0" } -} \ No newline at end of file +} diff --git a/metricbeat/module/logstash/node/_meta/fields.yml b/metricbeat/module/logstash/node/_meta/fields.yml index 658825edb0be..76e61b2ebdd5 100644 --- a/metricbeat/module/logstash/node/_meta/fields.yml +++ b/metricbeat/module/logstash/node/_meta/fields.yml @@ -4,6 +4,17 @@ node release: ga fields: + - name: state.pipeline + type: group + fields: + - name: id + type: keyword + description: > + TODO + - name: hash + type: keyword + description: > + TODO - name: host type: alias path: host.hostname diff --git a/metricbeat/module/logstash/node/data.go b/metricbeat/module/logstash/node/data.go index b1a6ef97ae41..b781e0a643b0 100644 --- a/metricbeat/module/logstash/node/data.go +++ b/metricbeat/module/logstash/node/data.go @@ -42,22 +42,10 @@ var ( } ) -func eventMapping(r mb.ReporterV2, content []byte) error { - event := mb.Event{} +func commonFieldsMapping(event mb.Event, fields common.MapStr) error { event.RootFields = common.MapStr{} event.RootFields.Put("service.name", logstash.ModuleName) - var data map[string]interface{} - err := json.Unmarshal(content, &data) - if err != nil { - return errors.Wrap(err, "failure parsing Logstash Node API response") - } - - fields, err := schema.Apply(data) - if err != nil { - return errors.Wrap(err, "failure applying node schema") - } - // Set service ID serviceID, err := fields.GetValue("id") if err != nil { @@ -90,8 +78,113 @@ func eventMapping(r mb.ReporterV2, content []byte) error { event.RootFields.Put("process.pid", pid) fields.Delete("jvm.pid") - event.MetricSetFields = fields + return nil +} + +func eventMapping(r mb.ReporterV2, content []byte, pipelines []logstash.PipelineState, overrideClusterUUID string) error { + var data map[string]interface{} + err := json.Unmarshal(content, &data) + if err != nil { + return errors.Wrap(err, "failure parsing Logstash Node API response") + } + + fields, err := schema.Apply(data) + if err != nil { + return errors.Wrap(err, "failure applying node schema") + } + + pipelines = getUserDefinedPipelines(pipelines) + clusterToPipelinesMap := makeClusterToPipelinesMap(pipelines, overrideClusterUUID) + + for clusterUUID, pipelines := range clusterToPipelinesMap { + for _, pipeline := range pipelines { + removeClusterUUIDsFromPipeline(pipeline) + + // Rename key: graph -> representation + pipeline.Representation = pipeline.Graph + pipeline.Graph = nil + + logstashState := map[string]logstash.PipelineState{ + "pipeline": pipeline, + } + + event := mb.Event{ + MetricSetFields: common.MapStr{ + "state": logstashState, + }, + } + event.MetricSetFields.Update(fields) + + if err = commonFieldsMapping(event, fields); err != nil { + return err + } + + if clusterUUID != "" { + event.ModuleFields.Put("cluster.id", clusterUUID) + } + + event.ID = pipeline.EphemeralID + + r.Event(event) + } + } - r.Event(event) return nil } + +func makeClusterToPipelinesMap(pipelines []logstash.PipelineState, overrideClusterUUID string) map[string][]logstash.PipelineState { + var clusterToPipelinesMap map[string][]logstash.PipelineState + clusterToPipelinesMap = make(map[string][]logstash.PipelineState) + + if overrideClusterUUID != "" { + clusterToPipelinesMap[overrideClusterUUID] = pipelines + return clusterToPipelinesMap + } + + for _, pipeline := range pipelines { + clusterUUIDs := common.StringSet{} + for _, vertex := range pipeline.Graph.Graph.Vertices { + clusterUUID := logstash.GetVertexClusterUUID(vertex, overrideClusterUUID) + if clusterUUID != "" { + clusterUUIDs.Add(clusterUUID) + } + } + + // If no cluster UUID was found in this pipeline, assign it a blank one + if len(clusterUUIDs) == 0 { + clusterUUIDs.Add("") + } + + for clusterUUID := range clusterUUIDs { + clusterPipelines := clusterToPipelinesMap[clusterUUID] + if clusterPipelines == nil { + clusterToPipelinesMap[clusterUUID] = []logstash.PipelineState{} + } + + clusterToPipelinesMap[clusterUUID] = append(clusterPipelines, pipeline) + } + } + + return clusterToPipelinesMap +} + +func getUserDefinedPipelines(pipelines []logstash.PipelineState) []logstash.PipelineState { + userDefinedPipelines := []logstash.PipelineState{} + for _, pipeline := range pipelines { + if pipeline.ID[0] != '.' { + userDefinedPipelines = append(userDefinedPipelines, pipeline) + } + } + return userDefinedPipelines +} + +func removeClusterUUIDsFromPipeline(pipeline logstash.PipelineState) { + for _, vertex := range pipeline.Graph.Graph.Vertices { + _, exists := vertex["cluster_uuid"] + if !exists { + continue + } + + delete(vertex, "cluster_uuid") + } +} diff --git a/metricbeat/module/logstash/node/data_test.go b/metricbeat/module/logstash/node/data_test.go index 65539432d8a8..3de3848272ab 100644 --- a/metricbeat/module/logstash/node/data_test.go +++ b/metricbeat/module/logstash/node/data_test.go @@ -27,6 +27,7 @@ import ( "github.com/stretchr/testify/require" mbtest "github.com/elastic/beats/v7/metricbeat/mb/testing" + "github.com/elastic/beats/v7/metricbeat/module/logstash" ) func TestEventMapping(t *testing.T) { @@ -39,10 +40,310 @@ func TestEventMapping(t *testing.T) { require.NoError(t, err) reporter := &mbtest.CapturingReporterV2{} - err = eventMapping(reporter, input) + err = commonFieldsMapping(reporter, input) require.NoError(t, err, f) require.True(t, len(reporter.GetEvents()) >= 1, f) require.Equal(t, 0, len(reporter.GetErrors()), f) } } + +func TestMakeClusterToPipelinesMap(t *testing.T) { + tests := map[string]struct { + pipelines []logstash.PipelineState + overrideClusterUUID string + expectedMap map[string][]logstash.PipelineState + }{ + "no_vertex_cluster_id": { + pipelines: []logstash.PipelineState{ + { + ID: "test_pipeline", + Graph: &logstash.GraphContainer{ + Graph: &logstash.Graph{ + Vertices: []map[string]interface{}{ + { + "id": "vertex_1", + }, + { + "id": "vertex_2", + }, + { + "id": "vertex_3", + }, + }, + }, + }, + }, + }, + overrideClusterUUID: "prod_cluster_id", + expectedMap: map[string][]logstash.PipelineState{ + "prod_cluster_id": { + { + ID: "test_pipeline", + Graph: &logstash.GraphContainer{ + Graph: &logstash.Graph{ + Vertices: []map[string]interface{}{ + { + "id": "vertex_1", + }, + { + "id": "vertex_2", + }, + { + "id": "vertex_3", + }, + }, + }, + }, + }, + }, + }, + }, + "one_vertex_cluster_id": { + pipelines: []logstash.PipelineState{ + { + ID: "test_pipeline", + Graph: &logstash.GraphContainer{ + Graph: &logstash.Graph{ + Vertices: []map[string]interface{}{ + { + "id": "vertex_1", + "cluster_uuid": "es_1", + }, + { + "id": "vertex_2", + }, + { + "id": "vertex_3", + }, + }, + }, + }, + }, + }, + overrideClusterUUID: "prod_cluster_id", + expectedMap: map[string][]logstash.PipelineState{ + "prod_cluster_id": { + { + ID: "test_pipeline", + Graph: &logstash.GraphContainer{ + Graph: &logstash.Graph{ + Vertices: []map[string]interface{}{ + { + "id": "vertex_1", + "cluster_uuid": "es_1", + }, + { + "id": "vertex_2", + }, + { + "id": "vertex_3", + }, + }, + }, + }, + }, + }, + }, + }, + "two_pipelines": { + pipelines: []logstash.PipelineState{ + { + ID: "test_pipeline_1", + Graph: &logstash.GraphContainer{ + Graph: &logstash.Graph{ + Vertices: []map[string]interface{}{ + { + "id": "vertex_1_1", + "cluster_uuid": "es_1", + }, + { + "id": "vertex_1_2", + }, + { + "id": "vertex_1_3", + }, + }, + }, + }, + }, + { + ID: "test_pipeline_2", + Graph: &logstash.GraphContainer{ + Graph: &logstash.Graph{ + Vertices: []map[string]interface{}{ + { + "id": "vertex_2_1", + }, + { + "id": "vertex_2_2", + }, + { + "id": "vertex_2_3", + }, + }, + }, + }, + }, + }, + overrideClusterUUID: "prod_cluster_id", + expectedMap: map[string][]logstash.PipelineState{ + "prod_cluster_id": { + { + ID: "test_pipeline_1", + Graph: &logstash.GraphContainer{ + Graph: &logstash.Graph{ + Vertices: []map[string]interface{}{ + { + "id": "vertex_1_1", + "cluster_uuid": "es_1", + }, + { + "id": "vertex_1_2", + }, + { + "id": "vertex_1_3", + }, + }, + }, + }, + }, + { + ID: "test_pipeline_2", + Graph: &logstash.GraphContainer{ + Graph: &logstash.Graph{ + Vertices: []map[string]interface{}{ + { + "id": "vertex_2_1", + }, + { + "id": "vertex_2_2", + }, + { + "id": "vertex_2_3", + }, + }, + }, + }, + }, + }, + }, + }, + "no_override_cluster_id": { + pipelines: []logstash.PipelineState{ + { + ID: "test_pipeline_1", + Graph: &logstash.GraphContainer{ + Graph: &logstash.Graph{ + Vertices: []map[string]interface{}{ + { + "id": "vertex_1_1", + "cluster_uuid": "es_1", + }, + { + "id": "vertex_1_2", + "cluster_uuid": "es_2", + }, + { + "id": "vertex_1_3", + }, + }, + }, + }, + }, + { + ID: "test_pipeline_2", + Graph: &logstash.GraphContainer{ + Graph: &logstash.Graph{ + Vertices: []map[string]interface{}{ + { + "id": "vertex_2_1", + }, + { + "id": "vertex_2_2", + }, + { + "id": "vertex_2_3", + }, + }, + }, + }, + }, + }, + overrideClusterUUID: "", + expectedMap: map[string][]logstash.PipelineState{ + "es_1": { + { + ID: "test_pipeline_1", + Graph: &logstash.GraphContainer{ + Graph: &logstash.Graph{ + Vertices: []map[string]interface{}{ + { + "id": "vertex_1_1", + "cluster_uuid": "es_1", + }, + { + "id": "vertex_1_2", + "cluster_uuid": "es_2", + }, + { + "id": "vertex_1_3", + }, + }, + }, + }, + }, + }, + "es_2": { + { + ID: "test_pipeline_1", + Graph: &logstash.GraphContainer{ + Graph: &logstash.Graph{ + Vertices: []map[string]interface{}{ + { + "id": "vertex_1_1", + "cluster_uuid": "es_1", + }, + { + "id": "vertex_1_2", + "cluster_uuid": "es_2", + }, + { + "id": "vertex_1_3", + }, + }, + }, + }, + }, + }, + "": { + { + ID: "test_pipeline_2", + Graph: &logstash.GraphContainer{ + Graph: &logstash.Graph{ + Vertices: []map[string]interface{}{ + { + "id": "vertex_2_1", + }, + { + "id": "vertex_2_2", + }, + { + "id": "vertex_2_3", + }, + }, + }, + }, + }, + }, + }, + }, + } + + for name, test := range tests { + t.Run(name, func(t *testing.T) { + actualMap := makeClusterToPipelinesMap(test.pipelines, test.overrideClusterUUID) + require.Equal(t, test.expectedMap, actualMap) + }) + } +} diff --git a/metricbeat/module/logstash/node/data_xpack.go b/metricbeat/module/logstash/node/data_xpack.go deleted file mode 100644 index 66d3623c7de6..000000000000 --- a/metricbeat/module/logstash/node/data_xpack.go +++ /dev/null @@ -1,120 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package node - -import ( - "time" - - "github.com/elastic/beats/v7/libbeat/common" - "github.com/elastic/beats/v7/metricbeat/helper/elastic" - "github.com/elastic/beats/v7/metricbeat/mb" - "github.com/elastic/beats/v7/metricbeat/module/logstash" -) - -func eventMappingXPack(r mb.ReporterV2, m *MetricSet, pipelines []logstash.PipelineState, overrideClusterUUID string) error { - pipelines = getUserDefinedPipelines(pipelines) - clusterToPipelinesMap := makeClusterToPipelinesMap(pipelines, overrideClusterUUID) - for clusterUUID, pipelines := range clusterToPipelinesMap { - for _, pipeline := range pipelines { - removeClusterUUIDsFromPipeline(pipeline) - - // Rename key: graph -> representation - pipeline.Representation = pipeline.Graph - pipeline.Graph = nil - - logstashState := map[string]logstash.PipelineState{ - "pipeline": pipeline, - } - - event := mb.Event{} - event.RootFields = common.MapStr{ - "timestamp": common.Time(time.Now()), - "interval_ms": m.Module().Config().Period / time.Millisecond, - "type": "logstash_state", - "logstash_state": logstashState, - } - - if clusterUUID != "" { - event.RootFields["cluster_uuid"] = clusterUUID - } - - event.ID = pipeline.EphemeralID - event.Index = elastic.MakeXPackMonitoringIndexName(elastic.Logstash) - r.Event(event) - } - } - - return nil -} - -func makeClusterToPipelinesMap(pipelines []logstash.PipelineState, overrideClusterUUID string) map[string][]logstash.PipelineState { - var clusterToPipelinesMap map[string][]logstash.PipelineState - clusterToPipelinesMap = make(map[string][]logstash.PipelineState) - - if overrideClusterUUID != "" { - clusterToPipelinesMap[overrideClusterUUID] = pipelines - return clusterToPipelinesMap - } - - for _, pipeline := range pipelines { - clusterUUIDs := common.StringSet{} - for _, vertex := range pipeline.Graph.Graph.Vertices { - clusterUUID := logstash.GetVertexClusterUUID(vertex, overrideClusterUUID) - if clusterUUID != "" { - clusterUUIDs.Add(clusterUUID) - } - } - - // If no cluster UUID was found in this pipeline, assign it a blank one - if len(clusterUUIDs) == 0 { - clusterUUIDs.Add("") - } - - for clusterUUID := range clusterUUIDs { - clusterPipelines := clusterToPipelinesMap[clusterUUID] - if clusterPipelines == nil { - clusterToPipelinesMap[clusterUUID] = []logstash.PipelineState{} - } - - clusterToPipelinesMap[clusterUUID] = append(clusterPipelines, pipeline) - } - } - - return clusterToPipelinesMap -} - -func getUserDefinedPipelines(pipelines []logstash.PipelineState) []logstash.PipelineState { - userDefinedPipelines := []logstash.PipelineState{} - for _, pipeline := range pipelines { - if pipeline.ID[0] != '.' { - userDefinedPipelines = append(userDefinedPipelines, pipeline) - } - } - return userDefinedPipelines -} - -func removeClusterUUIDsFromPipeline(pipeline logstash.PipelineState) { - for _, vertex := range pipeline.Graph.Graph.Vertices { - _, exists := vertex["cluster_uuid"] - if !exists { - continue - } - - delete(vertex, "cluster_uuid") - } -} diff --git a/metricbeat/module/logstash/node/data_xpack_test.go b/metricbeat/module/logstash/node/data_xpack_test.go deleted file mode 100644 index 17ae0aaaf912..000000000000 --- a/metricbeat/module/logstash/node/data_xpack_test.go +++ /dev/null @@ -1,328 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -// +build !integration - -package node - -import ( - "testing" - - "github.com/stretchr/testify/require" - - "github.com/elastic/beats/v7/metricbeat/module/logstash" -) - -func TestMakeClusterToPipelinesMap(t *testing.T) { - tests := map[string]struct { - pipelines []logstash.PipelineState - overrideClusterUUID string - expectedMap map[string][]logstash.PipelineState - }{ - "no_vertex_cluster_id": { - pipelines: []logstash.PipelineState{ - { - ID: "test_pipeline", - Graph: &logstash.GraphContainer{ - Graph: &logstash.Graph{ - Vertices: []map[string]interface{}{ - { - "id": "vertex_1", - }, - { - "id": "vertex_2", - }, - { - "id": "vertex_3", - }, - }, - }, - }, - }, - }, - overrideClusterUUID: "prod_cluster_id", - expectedMap: map[string][]logstash.PipelineState{ - "prod_cluster_id": { - { - ID: "test_pipeline", - Graph: &logstash.GraphContainer{ - Graph: &logstash.Graph{ - Vertices: []map[string]interface{}{ - { - "id": "vertex_1", - }, - { - "id": "vertex_2", - }, - { - "id": "vertex_3", - }, - }, - }, - }, - }, - }, - }, - }, - "one_vertex_cluster_id": { - pipelines: []logstash.PipelineState{ - { - ID: "test_pipeline", - Graph: &logstash.GraphContainer{ - Graph: &logstash.Graph{ - Vertices: []map[string]interface{}{ - { - "id": "vertex_1", - "cluster_uuid": "es_1", - }, - { - "id": "vertex_2", - }, - { - "id": "vertex_3", - }, - }, - }, - }, - }, - }, - overrideClusterUUID: "prod_cluster_id", - expectedMap: map[string][]logstash.PipelineState{ - "prod_cluster_id": { - { - ID: "test_pipeline", - Graph: &logstash.GraphContainer{ - Graph: &logstash.Graph{ - Vertices: []map[string]interface{}{ - { - "id": "vertex_1", - "cluster_uuid": "es_1", - }, - { - "id": "vertex_2", - }, - { - "id": "vertex_3", - }, - }, - }, - }, - }, - }, - }, - }, - "two_pipelines": { - pipelines: []logstash.PipelineState{ - { - ID: "test_pipeline_1", - Graph: &logstash.GraphContainer{ - Graph: &logstash.Graph{ - Vertices: []map[string]interface{}{ - { - "id": "vertex_1_1", - "cluster_uuid": "es_1", - }, - { - "id": "vertex_1_2", - }, - { - "id": "vertex_1_3", - }, - }, - }, - }, - }, - { - ID: "test_pipeline_2", - Graph: &logstash.GraphContainer{ - Graph: &logstash.Graph{ - Vertices: []map[string]interface{}{ - { - "id": "vertex_2_1", - }, - { - "id": "vertex_2_2", - }, - { - "id": "vertex_2_3", - }, - }, - }, - }, - }, - }, - overrideClusterUUID: "prod_cluster_id", - expectedMap: map[string][]logstash.PipelineState{ - "prod_cluster_id": { - { - ID: "test_pipeline_1", - Graph: &logstash.GraphContainer{ - Graph: &logstash.Graph{ - Vertices: []map[string]interface{}{ - { - "id": "vertex_1_1", - "cluster_uuid": "es_1", - }, - { - "id": "vertex_1_2", - }, - { - "id": "vertex_1_3", - }, - }, - }, - }, - }, - { - ID: "test_pipeline_2", - Graph: &logstash.GraphContainer{ - Graph: &logstash.Graph{ - Vertices: []map[string]interface{}{ - { - "id": "vertex_2_1", - }, - { - "id": "vertex_2_2", - }, - { - "id": "vertex_2_3", - }, - }, - }, - }, - }, - }, - }, - }, - "no_override_cluster_id": { - pipelines: []logstash.PipelineState{ - { - ID: "test_pipeline_1", - Graph: &logstash.GraphContainer{ - Graph: &logstash.Graph{ - Vertices: []map[string]interface{}{ - { - "id": "vertex_1_1", - "cluster_uuid": "es_1", - }, - { - "id": "vertex_1_2", - "cluster_uuid": "es_2", - }, - { - "id": "vertex_1_3", - }, - }, - }, - }, - }, - { - ID: "test_pipeline_2", - Graph: &logstash.GraphContainer{ - Graph: &logstash.Graph{ - Vertices: []map[string]interface{}{ - { - "id": "vertex_2_1", - }, - { - "id": "vertex_2_2", - }, - { - "id": "vertex_2_3", - }, - }, - }, - }, - }, - }, - overrideClusterUUID: "", - expectedMap: map[string][]logstash.PipelineState{ - "es_1": { - { - ID: "test_pipeline_1", - Graph: &logstash.GraphContainer{ - Graph: &logstash.Graph{ - Vertices: []map[string]interface{}{ - { - "id": "vertex_1_1", - "cluster_uuid": "es_1", - }, - { - "id": "vertex_1_2", - "cluster_uuid": "es_2", - }, - { - "id": "vertex_1_3", - }, - }, - }, - }, - }, - }, - "es_2": { - { - ID: "test_pipeline_1", - Graph: &logstash.GraphContainer{ - Graph: &logstash.Graph{ - Vertices: []map[string]interface{}{ - { - "id": "vertex_1_1", - "cluster_uuid": "es_1", - }, - { - "id": "vertex_1_2", - "cluster_uuid": "es_2", - }, - { - "id": "vertex_1_3", - }, - }, - }, - }, - }, - }, - "": { - { - ID: "test_pipeline_2", - Graph: &logstash.GraphContainer{ - Graph: &logstash.Graph{ - Vertices: []map[string]interface{}{ - { - "id": "vertex_2_1", - }, - { - "id": "vertex_2_2", - }, - { - "id": "vertex_2_3", - }, - }, - }, - }, - }, - }, - }, - }, - } - - for name, test := range tests { - t.Run(name, func(t *testing.T) { - actualMap := makeClusterToPipelinesMap(test.pipelines, test.overrideClusterUUID) - require.Equal(t, test.expectedMap, actualMap) - }) - } -} diff --git a/metricbeat/module/logstash/node/node.go b/metricbeat/module/logstash/node/node.go index 025d013e7566..ac608b981c0a 100644 --- a/metricbeat/module/logstash/node/node.go +++ b/metricbeat/module/logstash/node/node.go @@ -65,32 +65,22 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { // It returns the event which is then forward to the output. In case of an error, a // descriptive error must be returned. func (m *MetricSet) Fetch(r mb.ReporterV2) error { - if !m.MetricSet.XPack { - content, err := m.HTTP.FetchContent() - if err != nil { - return err - } - - return eventMapping(r, content) - } - - pipelinesContent, overrideClusterUUID, err := logstash.GetPipelines(m.MetricSet) + content, err := m.HTTP.FetchContent() if err != nil { - m.Logger().Error(err) - return nil + return err } + //commonFields, err := commonFieldsMapping(content) + //if err != nil { + // return err + //} - err = eventMappingXPack(r, m, pipelinesContent, overrideClusterUUID) + pipelinesContent, overrideClusterUUID, err := logstash.GetPipelines(m.MetricSet) if err != nil { - m.Logger().Error(err) + return err } - return nil -} - -func (m *MetricSet) init() error { - if m.XPack { - return m.CheckPipelineGraphAPIsAvailable() + if err = eventMapping(r, content, pipelinesContent, overrideClusterUUID); err != nil { + return err } return nil diff --git a/metricbeat/module/logstash/node_stats/_meta/data.json b/metricbeat/module/logstash/node_stats/_meta/data.json index 8d9623ef81be..4b24bae61b20 100644 --- a/metricbeat/module/logstash/node_stats/_meta/data.json +++ b/metricbeat/module/logstash/node_stats/_meta/data.json @@ -1,140 +1,130 @@ { - "@timestamp": "2020-10-30T12:41:12.793Z", - "@metadata": { - "beat": "metricbeat", - "type": "_doc", - "version": "8.0.0" - }, - "metricset": { - "name": "node_stats", - "period": 10000 + "@timestamp": "2017-10-12T08:05:34.853Z", + "event": { + "dataset": "logstash.node.stats", + "duration": 115000, + "module": "logstash" }, "logstash": { - "node": { - "stats": { - "events": { - "duration_in_millis": 0, - "in": 0, - "filtered": 0, - "out": 0 - }, - "reloads": { - "failures": 0, - "successes": 0 - }, - "queue": { - "events_count": 0 - }, - "os": { - "cpu": { - "percent": 0, - "load_average": { - "1m": 2.814453125 - } - }, - "cgroup": { - "cpuacct": null, - "cpu": { - "stat": null, - "control_group": "" - } - } - }, - "pipelines": [ - { - "id": "main", - "hash": "4dbe1b282a0265b7c90ae9212a3b4bef55168d267004e5124a3486f9151d4188", - "ephemeral_id": "5fd09014-09f9-449a-8b19-692c82f5494e", - "events": { - "filtered": 0, - "in": 0, - "out": 0, - "queue_push_duration_in_millis": 0, - "duration_in_millis": 0 - }, - "reloads": { - "successes": 0, - "failures": 0 - }, - "queue": { - "events_count": 0, - "max_queue_size_in_bytes": 0, - "queue_size_in_bytes": 0, - "type": "memory" - }, - "vertices": null - } - ], - "jvm": { - "gc": { - "collectors": { - "old": { - "collection_count": 1, - "collection_time_in_millis": 198 + "node": { + "stats": { + "events": { + "duration_in_millis": 0, + "in": 0, + "filtered": 0, + "out": 0 + }, + "jvm": { + "gc": { + "collectors": { + "old": { + "collection_count": 3, + "collection_time_in_millis": 449 + }, + "young": { + "collection_count": 7, + "collection_time_in_millis": 231 + } + } + }, + "mem": { + "heap_max_in_bytes": 1037959168, + "heap_used_in_bytes": 371211952, + "heap_used_percent": 35 + }, + "uptime_in_millis": 33546 + }, + "reloads": { + "failures": 0, + "successes": 0 + }, + "queue": { + "events_count": 0 }, - "young": { - "collection_time_in_millis": 298, - "collection_count": 10 - } - } - }, - "mem": { - "heap_max_in_bytes": 1.073741822e+09, - "heap_used_in_bytes": 3.18658432e+08, - "heap_used_percent": 29 - }, - "uptime_in_millis": 57904 - }, - "process": { - "open_file_descriptors": 93, - "max_file_descriptors": 10240, - "cpu": { - "percent": 0 + "process": { + "open_file_descriptors": 103, + "max_file_descriptors": 524288, + "cpu": { + "percent": 1 + } + }, + "os": { + "cpu": { + "percent": 0, + "load_average": { + "15m": 1.7, + "1m": 2.52, + "5m": 1.8 + } + }, + "cgroup": { + "cpuacct": { + "control_group": "/user.slice", + "usage_nanos": 24041075965919 + }, + "cpu": { + "stat": { + "number_of_elapsed_periods": 0, + "number_of_times_throttled": 0, + "time_throttled_nanos": 0 + }, + "control_group": "/user.slice" + } + } + }, + "pipelines": [ + { + "id": "main", + "hash": "6984380a58d40b7ebe6ba7ba8fc3134b95ffadfcef9ff328f0864ae002943792", + "ephemeral_id": "eb977feb-0052-4651-a995-39e545213723", + "events": { + "duration_in_millis": 0, + "filtered": 0, + "in": 0, + "out": 0, + "queue_push_duration_in_millis": 0 + }, + "reloads": { + "successes": 0, + "failures": 0 + }, + "queue": { + "events_count": 0, + "max_queue_size_in_bytes": 0, + "queue_size_in_bytes": 0, + "type": "memory" + }, + "vertices": null + } + ], + "logstash": { + "uuid": "2fc9524f-a36b-4611-82e4-5246d3ac4714", + "ephemeral_id": "10311adf-a5ba-4920-ada2-8dadc54618fe", + "name": "anonymous", + "host": "anonymous", + "version": "7.8.1", + "snapshot": false, + "status": "green", + "http_address": "127.0.0.1:9600", + "pipeline": { + "batch_size": 125, + "workers": 12 + } + }, + "timestamp": "2020-12-09T16:16:17.796Z" } - }, - "logstash": { - "uuid": "33f30cf2-d580-48fb-ad7c-2fbaf85db604", - "http_address": "127.0.0.1:9600", - "pipeline": { - "batch_size": 125, - "workers": 4 - }, - "status": "green", - "ephemeral_id": "c2f01c4f-5c09-4b65-a922-15fdf3634df3", - "name": "mcastro", - "host": "mcastro", - "version": "7.9.3", - "snapshot": false - }, - "timestamp": "2020-10-30T12:41:13.799Z" } - } - }, - "ecs": { - "version": "1.5.0" }, - "host": { - "name": "mcastro" - }, - "agent": { - "name": "mcastro", - "type": "metricbeat", - "version": "8.0.0", - "ephemeral_id": "fb36ebe3-f38d-4e5f-b68e-8d6f578ed471", - "id": "abfbaa55-b79d-4ae5-b412-fd35b933c6d3" + "metricset": { + "name": "node_stats", + "period": 10000 }, "service": { - "name": "logstash", - "id": "", - "hostname": "mcastro", - "version": "7.9.3", - "address": "localhost:9600", - "type": "logstash" - }, - "event": { - "module": "logstash", - "duration": 1005737788, - "dataset": "logstash.node.stats" + "address": "127.0.0.1:33437", + "hostname": "anonymous", + "id": "", + "name": "logstash", + "type": "logstash", + "version": "7.8.1" } - } - \ No newline at end of file +} \ No newline at end of file diff --git a/metricbeat/module/logstash/node_stats/_meta/fields.yml b/metricbeat/module/logstash/node_stats/_meta/fields.yml index 94aa32623985..b951c8d9b6c5 100644 --- a/metricbeat/module/logstash/node_stats/_meta/fields.yml +++ b/metricbeat/module/logstash/node_stats/_meta/fields.yml @@ -9,19 +9,13 @@ fields: - name: uptime_in_millis type: long - description: > - TODO - name: mem type: group fields: - name: heap_used_in_bytes type: long - description: > - TODO - name: heap_max_in_bytes type: long - description: > - TODO - name: events type: group description: > @@ -41,18 +35,13 @@ Filtered events counter. - name: duration_in_millis type: long - description: TODO - name: logstash type: group fields: - name: uuid type: keyword - description: > - TODO - name: version type: keyword - description: > - TODO - name: os type: group fields: @@ -64,23 +53,15 @@ fields: - name: 15m type: long - description: > - TODO - name: 1m type: long - description: > - TODO - name: 5m type: long - description: > - TODO - name: cgroup type: group fields: - name: cpuact.usage_nanos type: long - description: > - TODO - name: cpu type: group fields: @@ -108,45 +89,33 @@ fields: - name: id type: keyword - description: > - TODO - name: hash type: keyword - description: > - TODO - name: queue type: group fields: - name: events_count type: long - description: TODO - name: type type: keyword - description: TODO - name: queue_size_in_bytes type: long - description: TODO - name: max_queue_size_in_bytes type: long - description: TODO - name: events type: group fields: - name: out type: long - description: TODO - name: duration_in_millis type: long - description: TODO - name: vertices type: group fields: - name: duration_in_millis type: long - description: TODO - name: events_in type: long - description: TODO - name: pipeline_ephemeral_id type: keyword description: pipeline_ephemeral_id @@ -170,19 +139,15 @@ fields: - name: 15m type: long - description: TODO - name: 1m type: long - description: TODO - name: 5m type: long - description: TODO - name: cgroup type: group fields: - name: cpuacct.usage_nanos type: long - description: TODO - name: cpu type: group fields: @@ -191,13 +156,9 @@ fields: - name: number_of_elapsed_periods type: long - description: TODO - name: time_throttled_nanos type: long - description: TODO - name: number_of_times_throttled type: long - description: TODO - name: queue.events_count type: long - description: TODO diff --git a/metricbeat/module/logstash/node_stats/_meta/test/node_stats.710.json b/metricbeat/module/logstash/node_stats/_meta/test/node_stats.710.json new file mode 100644 index 000000000000..6b8afa8022bb --- /dev/null +++ b/metricbeat/module/logstash/node_stats/_meta/test/node_stats.710.json @@ -0,0 +1,192 @@ +{ + "host": "anonymous", + "version": "7.8.1", + "http_address": "127.0.0.1:9600", + "id": "2fc9524f-a36b-4611-82e4-5246d3ac4714", + "name": "anonymous", + "ephemeral_id": "10311adf-a5ba-4920-ada2-8dadc54618fe", + "status": "green", + "snapshot": false, + "pipeline": { + "workers": 12, + "batch_size": 125, + "batch_delay": 50 + }, + "jvm": { + "threads": { + "count": 45, + "peak_count": 45 + }, + "mem": { + "heap_used_percent": 35, + "heap_committed_in_bytes": 1037959168, + "heap_max_in_bytes": 1037959168, + "heap_used_in_bytes": 371211952, + "non_heap_used_in_bytes": 148921496, + "non_heap_committed_in_bytes": 169435136, + "pools": { + "young": { + "committed_in_bytes": 286326784, + "max_in_bytes": 286326784, + "used_in_bytes": 236814520, + "peak_used_in_bytes": 286326784, + "peak_max_in_bytes": 286326784 + }, + "old": { + "committed_in_bytes": 715849728, + "max_in_bytes": 715849728, + "used_in_bytes": 102245112, + "peak_used_in_bytes": 107311296, + "peak_max_in_bytes": 715849728 + }, + "survivor": { + "committed_in_bytes": 35782656, + "max_in_bytes": 35782656, + "used_in_bytes": 32152320, + "peak_used_in_bytes": 35782656, + "peak_max_in_bytes": 35782656 + } + } + }, + "gc": { + "collectors": { + "young": { + "collection_time_in_millis": 231, + "collection_count": 7 + }, + "old": { + "collection_time_in_millis": 449, + "collection_count": 3 + } + } + }, + "uptime_in_millis": 33546 + }, + "process": { + "open_file_descriptors": 103, + "peak_open_file_descriptors": 103, + "max_file_descriptors": 524288, + "mem": { + "total_virtual_in_bytes": 7189766144 + }, + "cpu": { + "total_in_millis": 68000, + "percent": 1, + "load_average": { + "1m": 2.52, + "5m": 1.8, + "15m": 1.7 + } + } + }, + "events": { + "in": 0, + "filtered": 0, + "out": 0, + "duration_in_millis": 0, + "queue_push_duration_in_millis": 0 + }, + "pipelines": { + "main": { + "events": { + "filtered": 0, + "out": 0, + "queue_push_duration_in_millis": 0, + "in": 0, + "duration_in_millis": 0 + }, + "plugins": { + "inputs": [ + { + "id": "adc9aa83962ecb7f9e68a9b25d9ef5473a197dbb8f6cd7d6025742b2e2bcadcf", + "events": { + "out": 0, + "queue_push_duration_in_millis": 0 + }, + "name": "file" + } + ], + "codecs": [ + { + "id": "plain_6013fcda-72f3-4a64-8a6c-2cbab83a1cd0", + "decode": { + "out": 0, + "duration_in_millis": 0, + "writes_in": 0 + }, + "encode": { + "duration_in_millis": 0, + "writes_in": 0 + }, + "name": "plain" + }, + { + "id": "rubydebug_e8db9697-3c57-4b27-be5a-6ecbdf34defb", + "decode": { + "out": 0, + "duration_in_millis": 0, + "writes_in": 0 + }, + "encode": { + "duration_in_millis": 5, + "writes_in": 0 + }, + "name": "rubydebug" + } + ], + "filters": [], + "outputs": [ + { + "id": "488688673ecd34b96c74c843d265da79aa11629216d2cb76ca008548316e310d", + "events": { + "out": 0, + "in": 0, + "duration_in_millis": 17 + }, + "name": "stdout" + } + ] + }, + "reloads": { + "last_error": null, + "failures": 0, + "last_success_timestamp": null, + "last_failure_timestamp": null, + "successes": 0 + }, + "queue": { + "type": "memory", + "events_count": 0, + "queue_size_in_bytes": 0, + "max_queue_size_in_bytes": 0 + }, + "hash": "6984380a58d40b7ebe6ba7ba8fc3134b95ffadfcef9ff328f0864ae002943792", + "ephemeral_id": "eb977feb-0052-4651-a995-39e545213723" + } + }, + "reloads": { + "failures": 0, + "successes": 0 + }, + "os": { + "cgroup": { + "cpu": { + "cfs_quota_micros": -1, + "cfs_period_micros": 100000, + "control_group": "/user.slice", + "stat": { + "time_throttled_nanos": 0, + "number_of_times_throttled": 0, + "number_of_elapsed_periods": 0 + } + }, + "cpuacct": { + "usage_nanos": 24041075965919, + "control_group": "/user.slice" + } + } + }, + "queue": { + "events_count": 0 + } +} diff --git a/metricbeat/module/logstash/node_stats/data_test.go b/metricbeat/module/logstash/node_stats/data_test.go index 1a30c773f733..ff69a8f20cee 100644 --- a/metricbeat/module/logstash/node_stats/data_test.go +++ b/metricbeat/module/logstash/node_stats/data_test.go @@ -20,13 +20,14 @@ package node_stats import ( - "github.com/elastic/beats/v7/metricbeat/module/logstash" "io/ioutil" "net/http" "net/http/httptest" "path/filepath" "testing" + "github.com/elastic/beats/v7/metricbeat/module/logstash" + "github.com/stretchr/testify/require" mbtest "github.com/elastic/beats/v7/metricbeat/mb/testing" @@ -63,7 +64,7 @@ func TestData(t *testing.T) { mux.Handle("/_node/stats", http.HandlerFunc( func(w http.ResponseWriter, r *http.Request) { - input, _ := ioutil.ReadFile("./_meta/test/node_stats.700.json") + input, _ := ioutil.ReadFile("./_meta/test/node_stats.710.json") w.Write(input) }))