diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 3664320db88c..946b7c01ced6 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -168,6 +168,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - auditd: Fix spelling of anomaly in `event.category`. - auditd: Fix typo in `event.action` of `removed-user-role-from`. {pull}19300[19300] - auditd: Fix typo in `event.action` of `used-suspicious-link`. {pull}19300[19300] +- system/socket: Fix kprobe grouping to allow running more than one instance. {pull}20325[20325] *Filebeat* @@ -225,9 +226,10 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Ignore missing in Zeek module when dropping unecessary fields. {pull}19984[19984] - Fix auditd module syscall table for ppc64 and ppc64le. {pull}20052[20052] - Fix Filebeat OOMs on very long lines {issue}19500[19500], {pull}19552[19552] -- Fix s3 input parsing json file without expand_event_list_from_field. {issue}19902[19902] {pull}19962[19962] +- Fix s3 input parsing json file without expand_event_list_from_field. {issue}19902[19902] {pull}19962[19962] {pull}20370[20370] - Fix millisecond timestamp normalization issues in CrowdStrike module {issue}20035[20035], {pull}20138[20138] - Fix support for message code 106100 in Cisco ASA and FTD. {issue}19350[19350] {pull}20245[20245] +- Fix `fortinet` setting `event.timezone` to the system one when no `tz` field present {pull}20273[20273] *Heartbeat* @@ -298,6 +300,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Add missing info about the rest of the azure metricsets in the documentation. {pull}19601[19601] - Fix k8s scheduler compatibility issue. {pull}19699[19699] - Fix SQL module mapping NULL values as string {pull}18955[18955] {issue}18898[18898 +- Modify doc for app_insights metricset to contain example of config. {pull}20185[20185] *Packetbeat* diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc index 9921f76167ac..5cad750548f8 100644 --- a/filebeat/docs/fields.asciidoc +++ b/filebeat/docs/fields.asciidoc @@ -45379,7 +45379,7 @@ Roles to which the principal belongs type: keyword -example: ['kibana_user', 'beats_admin'] +example: ['kibana_admin', 'beats_admin'] -- diff --git a/filebeat/module/elasticsearch/audit/_meta/fields.yml b/filebeat/module/elasticsearch/audit/_meta/fields.yml index 2461668157db..ceb94c00dd55 100644 --- a/filebeat/module/elasticsearch/audit/_meta/fields.yml +++ b/filebeat/module/elasticsearch/audit/_meta/fields.yml @@ -24,7 +24,7 @@ type: keyword - name: user.roles description: "Roles to which the principal belongs" - example: [ "kibana_user", "beats_admin" ] + example: [ "kibana_admin", "beats_admin" ] type: keyword - name: action description: "The name of the action that was executed" diff --git a/filebeat/module/elasticsearch/fields.go b/filebeat/module/elasticsearch/fields.go index 4e0467c018c1..4f44e586a19d 100644 --- a/filebeat/module/elasticsearch/fields.go +++ b/filebeat/module/elasticsearch/fields.go @@ -32,5 +32,5 @@ func init() { // AssetElasticsearch returns asset data. // This is the base64 encoded gzipped contents of module/elasticsearch. func AssetElasticsearch() string { - return "eJzUmltv2zj2wN/7KQi//GeARH/nMtmJgR1g6qZJil7SOEm34wbCMXUksaZIhaTseIp+9wUp2ZFlSb5s2+36JZF4Ob9z4eGhpH0yxlmPIAdtGNUIisbPCDHMcOyRzln5fucZIQFqqlhqmBQ98sczQsjyWPJGBhnHZ4SEDHmge67LPhGQ4KoY+zOzFHskUjJLizs1MpanK09JZZJKgcIsWioTLOvw1J+ESiZkGqNCYmIkXEYEJ7ZBKhYxAQaDTmlSfIQkdUaRHnrUS7w3aOAFGOgrBIOXIsDHAaoJo1gel+s3xtlUqmAVn2faoPKyjAWNGtzeXr4gMnSYxYB6svNkokYX/O0NG9x9ZFfh7+PH6DTansZeNdK8hQQ3ogkkHaPar+nTTiFkgF6LOZ6MYXvWy34xYB/ozQxv4g/m9l+vn5++6j5/M92SYWMzNHNMPrx9pf862lwws2HULtlFmuteLzNkHEcIZt+gNvtMpJnZVn6b9Z101rA24N159GI6ur0O+3e//ePPAX0Y9aMt7K5jUEGr+GBudNe1nqK7uUDIAmZWepfT0QrDH6WGaloqT81hhmqpparMjc07ttc8GTEaExMzvZKJekShNnvEKBA6lcq2EZb6IeOVtbVsCTuq2lpvkDK5k+7bfmvxbSfrkRzYxGCIpDRTyjKDkGKWyEz7QClq7QcoGAZ7BDITozCMgp3KD4Fxd7vSK7+MFAhjr6kUAqkbUXdvPsxAkqLCwFf4kDmrqUz4UJqouM4HNBtvWf72Zszd562144fFJlQQrzie/LLakscMkOuzwQ358+pyPvjXcpQsxk1BE4UU2QQDIoWT9tSNxiAE8l/3CJcUuG8TGvkl3xYpcJfgCNM6w6DM+Wuz7Z7m2d5uCoEnayNvOYbyQQ6u0mA1nwBngTMaRMDE6poowDt228IQMm7s0tqBPdOovM0UsF3/T9fqsUdYWG5ojNKOC1PDJugHTCE1Us12hZYcdSv0te1BjFwkKiSpYoKyFDgZIZci0o0RMSSdMRuBAN9K6+yRjt2jtA9BwkSH3G8NDW7Vr7WyKO3T+ZA8S9mwwEekWbNxe6RTFC+9RApmpPr/BJjYwb6KeykoSNbY167k2+tL4vqiQdVszs4Xa0Y7/T8/Ax0LRuPDr51a6UwEjK5x7WXep0i5GJDRrLBWm0NDKfcPuwenXvfA6x5bny7dOVq5c7KLo4tks1wVrKpwK9hDhiSvDosxzeb78Pdrfzw6uRtM3sV/PnTN9Gpy8e79Lrkqh6tUbKt45c1ynpS3CMQ+R1ADqiTn1/W6bczqj2Qwqx0MnEE1TlIwcY/ExqTeXFc73qNSmOVjl/0lLFKQa2xUhi3bog9BoFBXxa0D0TJTFD2W7iA4U2xLaXbhFjsu30HgIjtuK1avHsI2lZmg1hBVR+YSDT6alQo4wFRhvvl8uzp4PnlEd5+TkPM+sbuwRlMI8DYsvdMYdL0JqtLXENjfSyeI6BQpCxm1G+B5PxfhVTrXMZW5anxKWpfuRoD2Vz6OnvcJlZznZXE9aMn9WR5SvkbaiBZyCdWlviFYv0KyEGj3GqkCJiJrUcv9CiZAJkyZDDhJgMZMtIBrqrKRr2fJSHLfwIijb1iC30sPcgWZRmJFECaIRipFoAnlCMLqkKUkZyGORa8FN4qJ6AeAb8DtUNZyTxHGvsJQ+6mStkxw/N+R/MYy69QeLJ8kOgyiMESFwpYsT0o1o9uCinPkvkJNQfwo6pK9E1BjS8/ZBIkcfUZqtK2jORJIUz6v/pkm2sg0xaBZGcpBaz8TXELwozTJpbl4EZmtDx3EhtanaeY4GxnrkvKGjFd5YJD+1W0e40W8oAqlSizwUyqsQWxO2aRyPmowMllr6A0Vsb+KEjIzmgX5k4ExKoG8ToFSYpnp/wIlE1VI0kppD7g/AvNGGuAEOaQ2XivQRro3ARxNTl7aL91jFW1AuV4hE0zHXm2V8XmS+CoTDUuwWZE1CriTgkV1JK/u3hQ0WVpabXsENIF8ehvlqWTCEJElI1T1tCZWCIH2jbWLb7NMU/LYmfwc1AiiJWsWUomT6nJb4Ya6pLEIZJsC3e4yZ/7WJrYIRsqxdXEOVXC2chmI6s8r9aXbOmv1CZdRlG+9UYPIGKGaGXcuZC8QUgKcy2KzARHM/cL+3rqWtWP88agxqTNhMFp5AL4BJlksXqu8k2MDf8y4HM1MW4Vid6bvhnRr04gjaoZZHHF54EdYfTi1s+Pe8YBEKLAonCWlWQqCzn5+DzrnydAapKzBT+DORpuu9+5MZiL6lv79aCf8H/fwrKrDT+DjFrvW0y3shmqyJHT5Wd7ANbtPBqpvCepjYNVPT1sd0LFRQJer45K8To8MbCfiellwao/RMiSolFTLG5J7fdsjIfCl5x+1j2OqWuX70fJjxaaQbnv44iKhbQF0cr+c95sfd9Y/3KxbWvVLYJGIxeqpY5mlKqmNYs7B5YqCixJhKn+EwIV+E1QxQuBrfGg1+QAfMnteLkrERssfHR+fnp4e1pq/keKp3vPnT3e8Ne86lk/J5/09+ydhnLOiAmskPDjpdjesAxdWGtkFDdsBuuzmalVr5OKdV6mynYIuJsZgC/rfN6JfpAcup1xGzZkob89fv+v8xLDy0dYKRGd42D34fb97sn94enPQ7XVPegfHe6dHR/fDy7cv35H7Yf4ZSD6FV0B4Dxmq2T0ZTvy7V/Hnu3syTNAoRt3HJifekdfdt/N63RPv8OR+2L13Jfbw2Pst0fd77sLPjTQ8dtf2IBIzo4cHp8dHv9lbsxT18H7PpkWT/+MQ3LcIw/e3Z9cf/ZuLs7f+y7Ob/sViDvcpiB4e2P7u/cDwy6eOo/3U6X351EnA0NgHzvPLkZTafOr0Drzu169f7/f+k/xtK/jK9rTsodeuw8rnOmVv1Bo7RLPsveazxiL3SDluIXFLjpnFuad46eTOv85YTXxH3W6it0Sxjmxjse1N8rYT5UKlRdTAtucebZToWg+2lPsUmW3S888Oba8m4dWw3hLDBbzvHNjGweW03ctbLJntCPHRKPBzzhbCM9utUIcwEUqVwOoL6F2j5CnZtEVlfupkpilQjg93EJpnp7VirfEZBvl3bU0Ah9sBKJkZVtm0q990uB5NRtbdg4u/Dt8/H59+nh5HJoKXRmxn+Mpb+yXpl8G38W37ErxpWXuBpLsst2Zpgzx+ZUgCSbNk8VGcrRZcnsegRd6/AwAA//9xEEfG" + return "eJzUmltv2zj2wN/7KQi//GeARH/nMtmJgR1g6qZJil7SOEm34wbCMXUksaZIhaTseIp+9wUp2ZFlSb5s2+36JZF4Ob9z4eGhpH0yxlmPIAdtGNUIisbPCDHMcOyRzln5fucZIQFqqlhqmBQ98sczQsjyWPJGBhnHZ4SEDHmge67LPhGQ4KoY+zOzFHskUjJLizs1MpanK09JZZJKgcIsWioTLOvw1J+ESiZkGqNCYmIkXEYEJ7ZBKhYxAQaDTmlSfIQkdUaRHnrUS7w3aOAFGOgrBIOXIsDHAaoJo1gel+s3xtlUqmAVn2faoPKyjAWNGtzeXr4gMnSYxYB6svNkokYX/O0NG9x9ZFfh7+PH6DTansZeNdK8hQQ3ogkkHaPar+nTTiFkgF6LOZ6MYXvWy34xYB/ozQxv4g/m9l+vn5++6j5/M92SYWMzNHNMPrx9pf862lwws2HULtlFmuteLzNkHEcIZt+gNvtMpJnZVn6b9Z101rA24N159GI6ur0O+3e//ePPAX0Y9aMt7K5jUEGr+GBudNe1nqK7uUDIAmZWepfT0QrDH6WGaloqT81hhmqpparMjc07ttc8GTEaExMzvZKJekShNnvEKBA6lcq2EZb6IeOVtbVsCTuq2lpvkDK5k+7bfmvxbSfrkRzYxGCIpDRTyjKDkGKWyEz7QClq7QcoGAZ7BDITozCMgp3KD4Fxd7vSK7+MFAhjr6kUAqkbUXdvPsxAkqLCwFf4kDmrqUz4UJqouM4HNBtvWf72Zszd562144fFJlQQrzie/LLakscMkOuzwQ358+pyPvjXcpQsxk1BE4UU2QQDIoWT9tSNxiAE8l/3CJcUuG8TGvkl3xYpcJfgCNM6w6DM+Wuz7Z7m2d5uCoEnayNvOYbyQQ6u0mA1nwBngTMaRMDE6poowDt228IQMm7s0tqBPdOovM0UsF3/T9fqsUdYWG5ojNKOC1PDJugHTCE1Us12hZYcdSv0te1BjFwkKiSpYoKyFDgZIZci0o0RMSSdMRuBAB+ChInOHunYTUoXl+R+a2pwy36tmUVpo86H5GnKxgU+Is2ardsjnaJ66SVSMCPV/yfAxA4GVtxLQUGyxsB2Kd9eXxLXFw2qZnt2vliv2en/+RnoWDAaH37t1EpnImB0jW8v8z5FzsWAjGaFtdo8Gkq5f9g9OPW6B1732Pp06c7Ryp2TXRxdZJvlsmBVhVvBHjIkeXlYjGk234e/X/vj0cndYPIu/vOha6ZXk4t373dJVjlcpWRbxSvvlvOsvEUg9jmCGlAlOb+u121jVn8kg1ntYOAMqnGSgol7JDYm9ea62vEelcIsn7vsL2GRglxjozJs2Rd9CAKFuipuHYiWmaLosXQHwZliW0qzC7fYcvkOAhfpcVuxevUUtqnMBLWGqDoyl2jw0ayUwAGmCvPd59sVwvPJI7r7nISc94ndhjWaQoC3Ye2dxqDrTVCVvobA/l46QUSnSFnIqN0Bz/u5CK/SuY6pzFXjU9K6dDcCtL/yefS8T6jkPK+L60FL7s/ykPI10ka0kEuoLvUNwfoVkoVAu9dIFTARWYta7lcwATJhymTASQI0ZqIFXFOVjXw9S0aS+wZGHH3DEvxeepAryDQSK4IwQTRSKQJNKEcQVocsJTkLcSx6LbhRTEQ/AHwDboeylnuKMPYVhtpPlbRlguP/juQ3llmn9mT5JNFhEIUhKhS2ZHlSqhndFlScI/cVagriR1GX7J2AGlt6ziZI5OgzUqNtIc2RQJryefnPNNFGpikGzcpQDlr7meASgh+lSS7NxYvIbH3oIDa0Pk0zx9nIWJeUN2S8ygOD9K9u8xgv4gVVKFVigZ9SYQ1ic8omlQNSg5HJWkNvqIj9VZSQmdEsyB8NjFEJ5HUKlBLLTP8XKJmoQpJWSnvC/RGYN9IAJ8ghtfFagTbSvQrgaHLy0n7pnqtoA8r1CplgOvZqq4zPk8RXmWhYgs2KrFHAnRQsqiN5dfemoMnS0mrbI6AJ5NPbKE8lE4aILBmhqqc1sUIItG+sXXybZZqSx87k56BGEC1Zs5BKnFSX2wo31CWNRSDbFOh2lznztzaxRTBSjq2Lc6iCs5XLQFR/Xqkv3dZZq0+4jKJ8640aRMYI1cy4cyF7gZAS4FwWmw2IYO4X9vfWtawd449HjUmdCYPRyhPwDTDJYvFa5Z0cG/hjxuVoZtoqFLszfTekW5tGHFEzzOKIywM/wurDqZ0d944HJEKBReEsKc1SEHT283vQOU+G1iBlDX4CdzbadL13ZzIT0bf070c74f+4h2dVHX4CH7fYtZ5uYTdUkyWhy8/yBq7ZfTNQfU1QHwOrfnra6oCOjQK6XB2X5HV6ZGA7EdfLglN7jJYhQaWkWt6Q3PvbHgmBLz3/qH0cU9Uq34+WHys2hXTbwxcXCW0LoJP75bzf/Liz/uFm3dKqXwKLRCxWTx3LLFVJbRRzDi5XFFyUCFP5IwQu9JugihECX+NDq8kH+JDZ83JRIjZa/uj4+PT09LDW/I0UT/WeP3+6461517F8Sj7v79k/CeOcFRVYI+HBSbe7YR24sNLILmjYDtBlN1erWiMXL71Kle0UdDExBlvQ/74R/SI9cDnlMmrORHl7/v5d5yeGla+2ViA6w8Puwe/73ZP9w9Obg26ve9I7ON47PTq6H16+ffmO3A/z70DyKbwCwnvIUM3uyXDi372KP9/dk2GCRjHqvjY58Y687r6d1+ueeIcn98PuvSuxh8feb4m+33MXfm6k4bG7tgeRmBk9PDg9PvrN3pqlqIf3ezYtmvwfh+A+Rhi+vz27/ujfXJy99V+e3fQvFnO4b0H08MD2d+8Hhl8+dRztp07vy6dOAobGPnCeX46k1OZTp3fgdb9+/Xq/95/kb1vBV7anZQ+9dh1Wvtcpe6PW2CGaZe81nzUWuUfKcQuJW3LMLM49xUsnd/51xmriO+p2E70linVkG4ttb5K3nSgXKi2iBrY992ijRNd6sKXcp8hsk55/d2h7NQmvhvWWGC7gfefANg4up+1e3mLJbEeIj0aBn3O2EJ7ZboU6hIlQqgRWX0DvGiVPyaYtKvNTJzNNgXJ8uIPQPDutFWuNzzDIP2xrAjjcDkDJzLDKpl39qMP1aDKy7h5c/HX4/vn49PP0ODIRvDRiO8NX3tovSb8Mvo1v25fgTcvaCyTdZbk1Sxvk8StDEkiaJYuv4my14PI8Bi3y/h0AAP//rvZIEA==" } diff --git a/libbeat/docs/security/users.asciidoc b/libbeat/docs/security/users.asciidoc index dbf731bc4943..5886b2a68fcc 100644 --- a/libbeat/docs/security/users.asciidoc +++ b/libbeat/docs/security/users.asciidoc @@ -169,7 +169,7 @@ users who need to monitor {beatname_uc}: |==== |Role | Purpose -|`kibana_user` +|`kibana_admin` |Use {kib} |`monitoring_user` @@ -231,7 +231,7 @@ endif::serverless[] Users who publish events to {es} need to create and write to {beatname_uc} indices. To minimize the privileges required by the writer role, use the <> to pre-load dependencies. This section -assumes that you've pre-loaded dependencies. +assumes that you've run the setup. ifndef::no_ilm[] When using ILM, turn off the ILM setup check in the {beatname_uc} config file before diff --git a/libbeat/docs/tab-widgets/set-connection.asciidoc b/libbeat/docs/tab-widgets/set-connection.asciidoc index a9dad76d7e3c..571e9cec5702 100644 --- a/libbeat/docs/tab-widgets/set-connection.asciidoc +++ b/libbeat/docs/tab-widgets/set-connection.asciidoc @@ -58,8 +58,8 @@ include the scheme and port: `http://mykibanahost:5601/path`. <2> The `username` and `password` settings for {kib} are optional. If you don't specify credentials for {kib}, {beatname_uc} uses the `username` and `password` specified for the {es} output. -<3> To use the pre-built Kibana dashboards, this user must have the -`kibana_user` {ref}/built-in-roles.html[built-in role] or equivalent -privileges. +<3> To use the pre-built {kib} dashboards, this user must be authorized to +view dashboards or have the +`kibana_admin` {ref}/built-in-roles.html[built-in role]. // end::self-managed[] diff --git a/metricbeat/module/system/test_system.py b/metricbeat/module/system/test_system.py index cc6b731b1136..82e59e71af44 100644 --- a/metricbeat/module/system/test_system.py +++ b/metricbeat/module/system/test_system.py @@ -392,41 +392,85 @@ def test_process(self): self.assertGreater(len(output), 0) found_cmdline = False - found_env = False + for evt in output: + process = evt["system"]["process"] + found_cmdline |= "cmdline" in process + + # Remove 'env' prior to checking documented fields because its keys are dynamic. + process.pop("env", None) + self.assert_fields_are_documented(evt) + + # Remove optional keys. + process.pop("cgroup", None) + process.pop("fd", None) + process.pop("cmdline", None) + + self.assertCountEqual(SYSTEM_PROCESS_FIELDS, process.keys()) + + self.assertTrue(found_cmdline, "cmdline not found in any process events") + + @unittest.skipUnless(re.match("(?i)linux|darwin|freebsd", sys.platform), "os") + def test_process_unix(self): + """ + Test system/process output for fields specific of unix systems. + """ + import getpass + + self.render_config_template( + modules=[{ + "name": "system", + "metricsets": ["process"], + "period": "5s", + "extras": { + "process.env.whitelist": ["PATH"], + "process.include_cpu_ticks": True, + + # Remove 'percpu' prior to checking documented fields because its keys are dynamic. + "process.include_per_cpu": False, + }, + }], + # Some info is only guaranteed in processes with permissions, check + # only on own processes. + processors=[{ + "drop_event": { + "when": "not.equals.user.name: " + getpass.getuser(), + }, + }], + ) + proc = self.start_beat() + self.wait_until(lambda: self.output_lines() > 0) + proc.check_kill_and_wait() + self.assert_no_logged_warnings() + + output = self.read_output_json() + self.assertGreater(len(output), 0) + found_fd = False + found_env = False found_cwd = not sys.platform.startswith("linux") for evt in output: + found_cwd |= "working_directory" in evt["process"] + process = evt["system"]["process"] + found_fd |= "fd" in process + found_env |= "env" in process # Remove 'env' prior to checking documented fields because its keys are dynamic. env = process.pop("env", None) - if env is not None: - found_env = True - self.assert_fields_are_documented(evt) # Remove optional keys. process.pop("cgroup", None) - cmdline = process.pop("cmdline", None) - if cmdline is not None: - found_cmdline = True - fd = process.pop("fd", None) - if fd is not None: - found_fd = True - cwd = process.pop("cwd", None) - if cwd is not None: - found_cwd = True + process.pop("cmdline", None) + process.pop("fd", None) self.assertCountEqual(SYSTEM_PROCESS_FIELDS, process.keys()) - self.assertTrue(found_cmdline, "cmdline not found in any process events") - - if sys.platform.startswith("linux") or sys.platform.startswith("freebsd"): + if not sys.platform.startswith("darwin"): self.assertTrue(found_fd, "fd not found in any process events") - if sys.platform.startswith("linux") or sys.platform.startswith("freebsd")\ - or sys.platform.startswith("darwin"): - self.assertTrue(found_env, "env not found in any process events") + self.assertTrue(found_env, "env not found in any process events") + self.assertTrue(found_cwd, "working_directory not found in any process events") @unittest.skipUnless(re.match("(?i)win|linux|darwin|freebsd", sys.platform), "os") def test_process_metricbeat(self): diff --git a/x-pack/auditbeat/module/system/socket/socket_linux.go b/x-pack/auditbeat/module/system/socket/socket_linux.go index 4c2cfd7e7828..78fdd8ae4cae 100644 --- a/x-pack/auditbeat/module/system/socket/socket_linux.go +++ b/x-pack/auditbeat/module/system/socket/socket_linux.go @@ -9,8 +9,11 @@ package socket import ( "context" "fmt" + "os" + "path/filepath" "sort" "strconv" + "strings" "sync/atomic" "syscall" "time" @@ -36,17 +39,18 @@ import ( ) const ( - moduleName = "system" - metricsetName = "socket" - fullName = moduleName + "/" + metricsetName - namespace = "system.audit.socket" - detailSelector = metricsetName + "detailed" - auditbeatGroup = "auditbeat" + moduleName = "system" + metricsetName = "socket" + fullName = moduleName + "/" + metricsetName + namespace = "system.audit.socket" + detailSelector = metricsetName + "detailed" + groupNamePrefix = "auditbeat_" // Magic value to detect clock-sync events generated by the metricset. clockSyncMagic uint64 = 0x42DEADBEEFABCDEF ) var ( + groupName = fmt.Sprintf("%s%d", groupNamePrefix, os.Getpid()) kernelVersion string eventCount uint64 ) @@ -290,7 +294,7 @@ func (m *MetricSet) Setup() (err error) { extra = WithFilterPort(22) } m.installer = newProbeInstaller(traceFS, - WithGroup(auditbeatGroup), + WithGroup(groupName), WithTemplates(m.templateVars), extra) defer func() { @@ -300,10 +304,18 @@ func (m *MetricSet) Setup() (err error) { }() // - // remove existing KProbes from Auditbeat + // remove dangling KProbes from terminated Auditbeat processes. + // Not a fatal error if they can't be removed. // - if err = m.installer.UninstallIf(isOwnProbe); err != nil { - return errors.Wrap(err, "unable to delete existing KProbes. Is Auditbeat already running?") + if err = m.installer.UninstallIf(isDeadAuditbeat); err != nil { + m.log.Debugf("Removing existing probes from terminated instances: %+v", err) + } + + // + // remove existing Auditbeat KProbes that match the current PID. + // + if err = m.installer.UninstallIf(isThisAuditbeat); err != nil { + return errors.Wrapf(err, "unable to delete existing KProbes for group %s", groupName) } // @@ -409,7 +421,7 @@ func (m *MetricSet) Cleanup() { } } if m.installer != nil { - if err := m.installer.UninstallIf(isOwnProbe); err != nil { + if err := m.installer.UninstallIf(isThisAuditbeat); err != nil { m.log.Warnf("Failed to remove KProbes on exit: %v", err) } } @@ -468,8 +480,28 @@ func triggerClockSync() { unix.Uname(&buf) } -func isOwnProbe(probe tracing.Probe) bool { - return probe.Group == auditbeatGroup +func isRunningAuditbeat(pid int) bool { + path := fmt.Sprintf("/proc/%d/exe", pid) + exePath, err := os.Readlink(path) + if err != nil { + // Not a running process + return false + } + exeName := filepath.Base(exePath) + return strings.HasPrefix(exeName, "auditbeat") +} + +func isDeadAuditbeat(probe tracing.Probe) bool { + if strings.HasPrefix(probe.Group, groupNamePrefix) && probe.Group != groupName { + if pid, err := strconv.Atoi(probe.Group[len(groupNamePrefix):]); err == nil && !isRunningAuditbeat(pid) { + return true + } + } + return false +} + +func isThisAuditbeat(probe tracing.Probe) bool { + return probe.Group == groupName } type mountPoint struct { diff --git a/x-pack/elastic-agent/pkg/agent/control/addr.go b/x-pack/elastic-agent/pkg/agent/control/addr.go new file mode 100644 index 000000000000..3416480a6a05 --- /dev/null +++ b/x-pack/elastic-agent/pkg/agent/control/addr.go @@ -0,0 +1,20 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +// +build !windows + +package control + +import ( + "fmt" + "path/filepath" + + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/application/paths" +) + +// Address returns the address to connect to Elastic Agent daemon. +func Address() string { + data := paths.Data() + return fmt.Sprintf("unix://%s", filepath.Join(data, "agent.sock")) +} diff --git a/x-pack/elastic-agent/pkg/agent/control/addr_windows.go b/x-pack/elastic-agent/pkg/agent/control/addr_windows.go new file mode 100644 index 000000000000..1123eec941bf --- /dev/null +++ b/x-pack/elastic-agent/pkg/agent/control/addr_windows.go @@ -0,0 +1,22 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +// +build windows + +package control + +import ( + "crypto/sha256" + "fmt" + + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/application/paths" +) + +// Address returns the address to connect to Elastic Agent daemon. +func Address() string { + data = paths.Data() + // entire string cannot be longer than 256 characters, this forces the + // length to always be 87 characters (but unique per data path) + return fmt.Sprintf(`\\.\pipe\elastic-agent-%s`, sha256.Sum256(data)) +} diff --git a/x-pack/elastic-agent/pkg/agent/control/client/client.go b/x-pack/elastic-agent/pkg/agent/control/client/client.go new file mode 100644 index 000000000000..bcd8eccdb82d --- /dev/null +++ b/x-pack/elastic-agent/pkg/agent/control/client/client.go @@ -0,0 +1,188 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package client + +import ( + "context" + "encoding/json" + "fmt" + "sync" + "time" + + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/control" + + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/control/proto" +) + +// Status is the status of the Elastic Agent +type Status = proto.Status + +const ( + // Starting is when the it is still starting. + Starting Status = proto.Status_STARTING + // Configuring is when it is configuring. + Configuring Status = proto.Status_CONFIGURING + // Healthy is when it is healthy. + Healthy Status = proto.Status_HEALTHY + // Degraded is when it is degraded. + Degraded Status = proto.Status_DEGRADED + // Failed is when it is failed. + Failed Status = proto.Status_FAILED + // Stopping is when it is stopping. + Stopping Status = proto.Status_STOPPING + // Upgrading is when it is upgrading. + Upgrading Status = proto.Status_UPGRADING +) + +// Version is the current running version of the daemon. +type Version struct { + Version string + Commit string + BuildTime time.Time + Snapshot bool +} + +// ApplicationStatus is a status of an application inside of Elastic Agent. +type ApplicationStatus struct { + ID string + Name string + Status Status + Message string + Payload map[string]interface{} +} + +// AgentStatus is the current status of the Elastic Agent. +type AgentStatus struct { + Status Status + Message string + Applications []*ApplicationStatus +} + +// Client communicates to Elastic Agent through the control protocol. +type Client interface { + // Start starts the client. + Start(ctx context.Context) error + // Stop stops the client. + Stop() + // Version returns the current version of the running agent. + Version(ctx context.Context) (Version, error) + // Status returns the current status of the running agent. + Status(ctx context.Context) (*AgentStatus, error) + // Restart triggers restarting the current running daemon. + Restart(ctx context.Context) error + // Upgrade triggers upgrade of the current running daemon. + Upgrade(ctx context.Context, version string, sourceURI string) (string, error) +} + +// client manages the state and communication to the Elastic Agent. +type client struct { + ctx context.Context + cancel context.CancelFunc + wg sync.WaitGroup + client proto.ElasticAgentClient + cfgLock sync.RWMutex + obsLock sync.RWMutex +} + +// New creates a client connection to Elastic Agent. +func New() Client { + return &client{} +} + +// Start starts the connection to Elastic Agent. +func (c *client) Start(ctx context.Context) error { + c.ctx, c.cancel = context.WithCancel(ctx) + conn, err := dialContext(ctx) + if err != nil { + return err + } + c.client = proto.NewElasticAgentClient(conn) + return nil +} + +// Stop stops the connection to Elastic Agent. +func (c *client) Stop() { + if c.cancel != nil { + c.cancel() + c.wg.Wait() + c.ctx = nil + c.cancel = nil + } +} + +// Version returns the current version of the running agent. +func (c *client) Version(ctx context.Context) (Version, error) { + res, err := c.client.Version(ctx, &proto.Empty{}) + if err != nil { + return Version{}, err + } + bt, err := time.Parse(control.TimeFormat(), res.BuildTime) + if err != nil { + return Version{}, err + } + return Version{ + Version: res.Version, + Commit: res.Commit, + BuildTime: bt, + Snapshot: res.Snapshot, + }, nil +} + +// Status returns the current status of the running agent. +func (c *client) Status(ctx context.Context) (*AgentStatus, error) { + res, err := c.client.Status(ctx, &proto.Empty{}) + if err != nil { + return nil, err + } + s := &AgentStatus{ + Status: res.Status, + Message: res.Message, + Applications: make([]*ApplicationStatus, len(res.Applications)), + } + for i, appRes := range res.Applications { + var payload map[string]interface{} + if appRes.Payload != "" { + err := json.Unmarshal([]byte(appRes.Payload), &payload) + if err != nil { + return nil, err + } + } + s.Applications[i] = &ApplicationStatus{ + ID: appRes.Id, + Name: appRes.Name, + Status: appRes.Status, + Message: appRes.Message, + Payload: payload, + } + } + return s, nil +} + +// Restart triggers restarting the current running daemon. +func (c *client) Restart(ctx context.Context) error { + res, err := c.client.Restart(ctx, &proto.Empty{}) + if err != nil { + return err + } + if res.Status == proto.ActionStatus_FAILURE { + return fmt.Errorf(res.Error) + } + return nil +} + +// Upgrade triggers upgrade of the current running daemon. +func (c *client) Upgrade(ctx context.Context, version string, sourceURI string) (string, error) { + res, err := c.client.Upgrade(ctx, &proto.UpgradeRequest{ + Version: version, + SourceURI: sourceURI, + }) + if err != nil { + return "", err + } + if res.Status == proto.ActionStatus_FAILURE { + return "", fmt.Errorf(res.Error) + } + return res.Version, nil +} diff --git a/x-pack/elastic-agent/pkg/agent/control/client/dial.go b/x-pack/elastic-agent/pkg/agent/control/client/dial.go new file mode 100644 index 000000000000..56313b12c82a --- /dev/null +++ b/x-pack/elastic-agent/pkg/agent/control/client/dial.go @@ -0,0 +1,26 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +// +build !windows + +package client + +import ( + "context" + "net" + "strings" + + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/control" + + "google.golang.org/grpc" +) + +func dialContext(ctx context.Context) (*grpc.ClientConn, error) { + return grpc.DialContext(ctx, strings.TrimPrefix(control.Address(), "unix://"), grpc.WithInsecure(), grpc.WithContextDialer(dialer)) +} + +func dialer(ctx context.Context, addr string) (net.Conn, error) { + var d net.Dialer + return d.DialContext(ctx, "unix", addr) +} diff --git a/x-pack/elastic-agent/pkg/agent/control/client/dial_windows.go b/x-pack/elastic-agent/pkg/agent/control/client/dial_windows.go new file mode 100644 index 000000000000..58b36c180435 --- /dev/null +++ b/x-pack/elastic-agent/pkg/agent/control/client/dial_windows.go @@ -0,0 +1,26 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +// +build windows + +package client + +import ( + "context" + "net" + + "google.golang.org/grpc" + + "github.com/elastic/beats/v7/libbeat/api/npipe" + + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/control" +) + +func dialContext(ctx context.Context) (*grpc.ClientConn, error) { + return grpc.DialContext(ctx, control.Address(), grpc.WithInsecure(), grpc.WithContextDialer(dialer)) +} + +func dialer(ctx context.Context, addr string) (net.Conn, error) { + return npipe.DialContext(arr)(ctx, "", "") +} diff --git a/x-pack/elastic-agent/pkg/agent/control/control_test.go b/x-pack/elastic-agent/pkg/agent/control/control_test.go new file mode 100644 index 000000000000..13d32420258d --- /dev/null +++ b/x-pack/elastic-agent/pkg/agent/control/control_test.go @@ -0,0 +1,53 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package control_test + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/elastic/beats/v7/libbeat/logp" + + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/control/client" + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/control/server" + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/core/logger" + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/release" +) + +func TestServerClient_Version(t *testing.T) { + srv := server.New(newErrorLogger(t)) + err := srv.Start() + require.NoError(t, err) + defer srv.Stop() + + c := client.New() + err = c.Start(context.Background()) + require.NoError(t, err) + defer c.Stop() + + ver, err := c.Version(context.Background()) + require.NoError(t, err) + + assert.Equal(t, client.Version{ + Version: release.Version(), + Commit: release.Commit(), + BuildTime: release.BuildTime(), + Snapshot: release.Snapshot(), + }, ver) +} + +func newErrorLogger(t *testing.T) *logger.Logger { + t.Helper() + + loggerCfg := logger.DefaultLoggingConfig() + loggerCfg.Level = logp.ErrorLevel + + log, err := logger.NewFromConfig("", loggerCfg) + require.NoError(t, err) + return log +} diff --git a/x-pack/elastic-agent/pkg/agent/control/server/listener.go b/x-pack/elastic-agent/pkg/agent/control/server/listener.go new file mode 100644 index 000000000000..2dd5d54a46fe --- /dev/null +++ b/x-pack/elastic-agent/pkg/agent/control/server/listener.go @@ -0,0 +1,38 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +// +build !windows + +package server + +import ( + "net" + "os" + "path/filepath" + "strings" + + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/control" +) + +func createListener() (net.Listener, error) { + path := strings.TrimPrefix(control.Address(), "unix://") + dir := filepath.Dir(path) + if _, err := os.Stat(dir); os.IsNotExist(err) { + err = os.MkdirAll(dir, 0755) + if err != nil { + return nil, err + } + } + lis, err := net.Listen("unix", path) + if err != nil { + return nil, err + } + err = os.Chmod(path, 0700) + if err != nil { + // failed to set permissions (close listener) + lis.Close() + return nil, err + } + return lis, err +} diff --git a/x-pack/elastic-agent/pkg/agent/control/server/listener_windows.go b/x-pack/elastic-agent/pkg/agent/control/server/listener_windows.go new file mode 100644 index 000000000000..d2d2866b98a1 --- /dev/null +++ b/x-pack/elastic-agent/pkg/agent/control/server/listener_windows.go @@ -0,0 +1,29 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +// +build windows + +package server + +import ( + "net" + "os/user" + + "github.com/elastic/beats/v7/libbeat/api/npipe" + + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/control" +) + +// createListener creates a named pipe listener on Windows +func createListener() (net.Listener, error) { + u, err := user.Current() + if err != nil { + return nil, err + } + sd, err := npipe.DefaultSD(u.Username) + if err != nil { + return nil, err + } + return npipe.NewListener(control.Address(), sd) +} diff --git a/x-pack/elastic-agent/pkg/agent/control/server/server.go b/x-pack/elastic-agent/pkg/agent/control/server/server.go new file mode 100644 index 000000000000..c9a750808fcd --- /dev/null +++ b/x-pack/elastic-agent/pkg/agent/control/server/server.go @@ -0,0 +1,106 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package server + +import ( + "context" + "net" + + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/control" + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/release" + + "google.golang.org/grpc" + + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/control/proto" + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/core/logger" +) + +// Server is the daemon side of the control protocol. +type Server struct { + logger *logger.Logger + listener net.Listener + server *grpc.Server +} + +// New creates a new control protocol server. +func New(log *logger.Logger) *Server { + return &Server{ + logger: log, + } +} + +// Start starts the GRPC endpoint and accepts new connections. +func (s *Server) Start() error { + if s.server != nil { + // already started + return nil + } + + lis, err := createListener() + if err != nil { + return err + } + s.listener = lis + s.server = grpc.NewServer() + proto.RegisterElasticAgentServer(s.server, s) + + // start serving GRPC connections + go func() { + err := s.server.Serve(lis) + if err != nil { + s.logger.Errorf("error listening for GRPC: %s", err) + } + }() + + return nil +} + +// Stop stops the GRPC endpoint. +func (s *Server) Stop() { + if s.server != nil { + s.server.Stop() + s.server = nil + s.listener = nil + } +} + +// Version returns the currently running version. +func (s *Server) Version(_ context.Context, _ *proto.Empty) (*proto.VersionResponse, error) { + return &proto.VersionResponse{ + Version: release.Version(), + Commit: release.Commit(), + BuildTime: release.BuildTime().Format(control.TimeFormat()), + Snapshot: release.Snapshot(), + }, nil +} + +// Status returns the overall status of the agent. +func (s *Server) Status(_ context.Context, _ *proto.Empty) (*proto.StatusResponse, error) { + // not implemented + return &proto.StatusResponse{ + Status: proto.Status_HEALTHY, + Message: "not implemented", + Applications: nil, + }, nil +} + +// Restart performs re-exec. +func (s *Server) Restart(_ context.Context, _ *proto.Empty) (*proto.RestartResponse, error) { + // not implemented + return &proto.RestartResponse{ + Status: proto.ActionStatus_FAILURE, + Error: "not implemented", + }, nil +} + +// Upgrade performs the upgrade operation. +func (s *Server) Upgrade(ctx context.Context, request *proto.UpgradeRequest) (*proto.UpgradeResponse, error) { + // not implemented + return &proto.UpgradeResponse{ + Status: proto.ActionStatus_FAILURE, + Version: "", + Error: "not implemented", + }, nil +} diff --git a/x-pack/elastic-agent/pkg/agent/control/time.go b/x-pack/elastic-agent/pkg/agent/control/time.go new file mode 100644 index 000000000000..c87902bbc371 --- /dev/null +++ b/x-pack/elastic-agent/pkg/agent/control/time.go @@ -0,0 +1,10 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package control + +// TimeFormat returns the time format shared between the protocol. +func TimeFormat() string { + return "2006-01-02 15:04:05 -0700 MST" +} diff --git a/x-pack/filebeat/input/s3/input.go b/x-pack/filebeat/input/s3/input.go index 15f9384b7cf6..65984dace451 100644 --- a/x-pack/filebeat/input/s3/input.go +++ b/x-pack/filebeat/input/s3/input.go @@ -526,10 +526,11 @@ func (p *s3Input) decodeJSON(decoder *json.Decoder, objectHash string, s3Info s3 return nil } - offset, err = p.jsonFieldsType(jsonFields, offset, objectHash, s3Info, s3Ctx) + offsetNew, err := p.jsonFieldsType(jsonFields, offset, objectHash, s3Info, s3Ctx) if err != nil { return err } + offset = offsetNew } } @@ -554,6 +555,27 @@ func (p *s3Input) jsonFieldsType(jsonFields interface{}, offset int, objectHash return offset, nil } case map[string]interface{}: + if p.config.ExpandEventListFromField != "" { + textValues, ok := f[p.config.ExpandEventListFromField] + if !ok { + err := errors.Errorf("key '%s' not found", p.config.ExpandEventListFromField) + p.logger.Error(err) + return offset, err + } + + valuesConverted := textValues.([]interface{}) + for _, textValue := range valuesConverted { + offsetNew, err := p.convertJSONToEvent(textValue, offset, objectHash, s3Info, s3Ctx) + if err != nil { + err = errors.Wrapf(err, "convertJSONToEvent failed for '%s' from S3 bucket '%s'", s3Info.key, s3Info.name) + p.logger.Error(err) + return offset, err + } + offset = offsetNew + } + return offset, nil + } + offset, err := p.convertJSONToEvent(f, offset, objectHash, s3Info, s3Ctx) if err != nil { err = errors.Wrapf(err, "convertJSONToEvent failed for '%s' from S3 bucket '%s'", s3Info.key, s3Info.name) diff --git a/x-pack/filebeat/module/fortinet/firewall/config/firewall.yml b/x-pack/filebeat/module/fortinet/firewall/config/firewall.yml index 6af16945317c..1154d83947f2 100644 --- a/x-pack/filebeat/module/fortinet/firewall/config/firewall.yml +++ b/x-pack/filebeat/module/fortinet/firewall/config/firewall.yml @@ -24,7 +24,6 @@ tags: {{.tags | tojson}} publisher_pipeline.disable_host: {{ inList .tags "forwarded" }} processors: - - add_locale: ~ - add_fields: target: '' fields: diff --git a/x-pack/filebeat/module/fortinet/firewall/ingest/pipeline.yml b/x-pack/filebeat/module/fortinet/firewall/ingest/pipeline.yml index 60ada5b7f085..2aaf7065ec1d 100644 --- a/x-pack/filebeat/module/fortinet/firewall/ingest/pipeline.yml +++ b/x-pack/filebeat/module/fortinet/firewall/ingest/pipeline.yml @@ -178,4 +178,4 @@ processors: on_failure: - set: field: error.message - value: '{{ _ingest.on_failure_message }}' \ No newline at end of file + value: '{{ _ingest.on_failure_message }}' diff --git a/x-pack/filebeat/module/fortinet/firewall/test/fortinet.log-expected.json b/x-pack/filebeat/module/fortinet/firewall/test/fortinet.log-expected.json index 73ad332c40d5..bf1b5de3fd0b 100644 --- a/x-pack/filebeat/module/fortinet/firewall/test/fortinet.log-expected.json +++ b/x-pack/filebeat/module/fortinet/firewall/test/fortinet.log-expected.json @@ -96,7 +96,6 @@ "event.module": "fortinet", "event.outcome": "success", "event.start": "2020-06-24T01:16:08.000Z", - "event.timezone": "-02:00", "event.type": [ "connection", "end" diff --git a/x-pack/metricbeat/module/azure/app_insights/_meta/docs.asciidoc b/x-pack/metricbeat/module/azure/app_insights/_meta/docs.asciidoc index 2ba1150078b0..2b587acbbdd6 100644 --- a/x-pack/metricbeat/module/azure/app_insights/_meta/docs.asciidoc +++ b/x-pack/metricbeat/module/azure/app_insights/_meta/docs.asciidoc @@ -45,17 +45,15 @@ This value is only valid when segment is specified. `filter`:: (_string_) An expression used to filter the results. This value should be a valid OData filter expression where the keys of each clause should be applicable dimensions for the metric you are retrieving. -Users can select the options to retrieve all metrics from a specific namespace using the following: +Example configuration: ["source","yaml"] ---- - metrics: - - id: ["*"] - timespan: "Microsoft.Storage/storageAccounts" +metrics: + - id: ["requests/count", "requests/failed"] + segment: "request/name" + aggregation: ["sum"] ---- -A default non configurable timegrain of 5 min is set so users are advised to configure an interval of 300s or a multiply of it. - -