From 0e0093ea2e60754921d634f80cd79561b133f262 Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Tue, 17 Aug 2021 15:08:43 -0600 Subject: [PATCH 1/7] Move AWS Filebeat module to GA (#27428) --- CHANGELOG.next.asciidoc | 1 + filebeat/docs/modules/aws.asciidoc | 2 -- x-pack/filebeat/module/aws/_meta/docs.asciidoc | 2 -- x-pack/filebeat/module/aws/_meta/fields.yml | 2 +- x-pack/filebeat/module/aws/cloudtrail/_meta/fields.yml | 2 +- x-pack/filebeat/module/aws/cloudwatch/_meta/fields.yml | 2 +- x-pack/filebeat/module/aws/ec2/_meta/fields.yml | 2 +- x-pack/filebeat/module/aws/fields.go | 2 +- x-pack/filebeat/module/aws/vpcflow/_meta/fields.yml | 2 +- 9 files changed, 7 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 2a3eff527f9f..3a7b369a1654 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -758,6 +758,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Update Elasticsearch module's ingest pipeline for parsing new deprecation logs {issue}26857[26857] {pull}26880[26880] - Add new `hmac` template function for httpjson input {pull}27168[27168] - Update `tags` and `threatintel.indicator.provider` fields in `threatintel.anomali` ingest pipeline {issue}24746[24746] {pull}27141[27141] +- Move AWS module and filesets to GA. {pull}27428[27428] - update ecs.version to ECS 1.11.0. {pull}27107[27107] *Heartbeat* diff --git a/filebeat/docs/modules/aws.asciidoc b/filebeat/docs/modules/aws.asciidoc index 78cdc5c961cc..3f9c2c11653c 100644 --- a/filebeat/docs/modules/aws.asciidoc +++ b/filebeat/docs/modules/aws.asciidoc @@ -12,8 +12,6 @@ This file is generated! See scripts/docs_collector.py == AWS module -beta[] - This is a module for aws logs. It uses filebeat s3 input to get log files from AWS S3 buckets with SQS notification or directly polling list of S3 objects in an S3 bucket. The use of SQS notification is preferred: polling list of S3 objects is expensive diff --git a/x-pack/filebeat/module/aws/_meta/docs.asciidoc b/x-pack/filebeat/module/aws/_meta/docs.asciidoc index 2e90084b1f05..f286b9251dc3 100644 --- a/x-pack/filebeat/module/aws/_meta/docs.asciidoc +++ b/x-pack/filebeat/module/aws/_meta/docs.asciidoc @@ -7,8 +7,6 @@ == AWS module -beta[] - This is a module for aws logs. It uses filebeat s3 input to get log files from AWS S3 buckets with SQS notification or directly polling list of S3 objects in an S3 bucket. The use of SQS notification is preferred: polling list of S3 objects is expensive diff --git a/x-pack/filebeat/module/aws/_meta/fields.yml b/x-pack/filebeat/module/aws/_meta/fields.yml index 42e845dae7da..2d582f91e4bc 100644 --- a/x-pack/filebeat/module/aws/_meta/fields.yml +++ b/x-pack/filebeat/module/aws/_meta/fields.yml @@ -1,6 +1,6 @@ - key: aws title: AWS - release: beta + release: ga description: > Module for handling logs from AWS. fields: diff --git a/x-pack/filebeat/module/aws/cloudtrail/_meta/fields.yml b/x-pack/filebeat/module/aws/cloudtrail/_meta/fields.yml index 230268476feb..b5ec63224619 100644 --- a/x-pack/filebeat/module/aws/cloudtrail/_meta/fields.yml +++ b/x-pack/filebeat/module/aws/cloudtrail/_meta/fields.yml @@ -1,6 +1,6 @@ - name: cloudtrail type: group - release: beta + release: ga default_field: false description: > Fields for AWS CloudTrail logs. diff --git a/x-pack/filebeat/module/aws/cloudwatch/_meta/fields.yml b/x-pack/filebeat/module/aws/cloudwatch/_meta/fields.yml index 7d80e27ed153..5b6f7d9fc94c 100644 --- a/x-pack/filebeat/module/aws/cloudwatch/_meta/fields.yml +++ b/x-pack/filebeat/module/aws/cloudwatch/_meta/fields.yml @@ -1,6 +1,6 @@ - name: cloudwatch type: group - release: beta + release: ga default_field: false description: > Fields for AWS CloudWatch logs. diff --git a/x-pack/filebeat/module/aws/ec2/_meta/fields.yml b/x-pack/filebeat/module/aws/ec2/_meta/fields.yml index f6c21a4d7b6e..0572163d03b6 100644 --- a/x-pack/filebeat/module/aws/ec2/_meta/fields.yml +++ b/x-pack/filebeat/module/aws/ec2/_meta/fields.yml @@ -1,6 +1,6 @@ - name: ec2 type: group - release: beta + release: ga default_field: false description: > Fields for AWS EC2 logs in CloudWatch. diff --git a/x-pack/filebeat/module/aws/fields.go b/x-pack/filebeat/module/aws/fields.go index 032ca552e682..11cbba99909b 100644 --- a/x-pack/filebeat/module/aws/fields.go +++ b/x-pack/filebeat/module/aws/fields.go @@ -19,5 +19,5 @@ func init() { // AssetAws returns asset data. // This is the base64 encoded zlib format compressed contents of module/aws. func AssetAws() string { - return "eJzcXN9z47Zzf7+/Yicv8c1I7nwvmU7HnXRG5/M1apyLa+mS9omByBWFGgIYAJROmf7xnQXAHxJBWbaoXKZ+SHwmCXx2sb+xwBiecHcDbGveAFhuBd7A5LfZGwCNApnBG1igZW8AMjSp5oXlSt7Av70BAPhZZaVAWCoNKyYzwWUOQuUGllqtaZjrNwBLjiIzN+6DMUi2xmo6+rG7Am8g16oswl8i89DPRzdMPbKb5zo8bU/RniYVqsysZlzUj2Iz0s8htdVPhktWCpu4KW5gyYTBvcdRsG3ASju8t4RlTlj2oMfgt0nADUqbbFAbruTeGxUlT7jbKp0dPDsCjH7mK2wjCuODWoJdIQH0ExP6NbPXUWilQZ3wDKXldheFdsjkLrBxFBmNPA0DAwpcE5RUScu4NJChZVwYYAtVWoeXZgO17Iw1nfwMFUCwK2ZhzTJ0n2j8o0RjR8BkBtsVT1eQanTvMmFgixo7w5UGs2uYLsHiulCa6V3nG/fOyM1Q4TYrtTWwUlv6a2fMzgBqQVRidn3wakxI2qtBPOg8PC4j3eWIvOBXJHDYEdaz5C3t1oeS+nIkXcGooEzW7E8l4RGNKnWK8ImtEa4mj5/eVgALzWXKCyYO1jxlQhyytYU6TdGY5Al3CY/hGwq/n4cGgukHj3DLjBMcsAoMz2VbQvsBGzSktAkpBn6xvZBjWngq4OmyjcUBdezccrtqqYHBtNQxkYB9ESd1qxXDkV5oteEZGuDS2xoyQ41mBxqj49asSzUyi5kztXalDLanjHzap0pt5q6XLGGlXdEoKY0efft5qTiV0RCkY8NEicANWE3/D+xXyjqjCEo7o+Z+3xKpvYNFLVNgUbOgTBjleLhHq19eFmc7/fz8cQIZbniK/wrKrlBvucGR945dgW3z1a0VSW3GbB94z9MjL7yEoTSMM/KWrxG2K/Ta1ZXdLse4MWXXEO/TUymhe1cfJahPD19C0RD6CK/Xyd7xgns73Z1VP8d0EZ53b9XPKWoIL+AzBOkJPiY4luNCMwJTpqujQzIDj0rZESnxZ4N6RAr9qESP0rQZUDu1uHe6NCO4tKglE+SzAjfacVXbg+XYLyewL3vPkx2PJS5N7eTxU0VlkIArlqaqlH7pnP11a6eVwLdHh4ux5xlBOoErHszXEYUwuadMbaW5nDRU9HK5UU+YJYuYRRsqMKOpqlWnjM2gJg/XlziQsgOLxRdQxah3t+9gUloFs5S55DjkgneCGctTeI9MGsvEUzzBQq2VTlKVHVq+0xO/eH7Vps5NUgcawa9otKWWxnkGen4M3xqNYfmQEKfHwfj0qjVIbZD6oYaxkoJptkaL+nDdzmVpM/CImMnkbhR0gbygId/qfXR/ZL8uheXJs3nesVC/5+EzRYw2m0yhpMEkxANDc6kav443KB5lKX1jKj17QkhXTOZo4MpH9qNuJl5QWOcscIYCKcLzg7z9GzKVZRkncEwkrqiSsb0K07l8ndTDU6zLWsmTL+HUFlkqS2Jqg7p0Bqr0jdxaWKe/ITcrTe64vnNl02dejlu+wrHkaPb01UeEwW4ukMu8W85hQmAGOUrUzLrvufFD99hQV9+LRLdnGdB9/FXxpiUPFcCsJSgaU6WzOExW8LOLkM/inDxM60okM0alvElG3fOtmRT8lgnRGclRMCc6j/B6zSTLnd3xijikEsJ7pQQy2SNG2xVSmtziNjdwaAWghdC/1efMWJYoKeJF17OXosHKDaiC5IRWhAC7qcc0dfOgD6MPnOMe5DWF4QkIbpzxqscOtTTMgMuGtS+tnF6uXFnXKCePn7qB4knB/BAwJiFgb1K3ioMUvkeqFoPWlJ9hjTNOtaboaiV94cHtRd3csK0ZB7s7dshuyNON6VP37x4JTHnBSdl7GXyOwjxioZHiOm+7WMNjp/saU+QbZ1+5OabMgS5vkZKwt3E5G1uH/TTdCLhMRZlRarIl1FbzPEft3ULcyPpampehUvwdg1izYhqzwNBB1/zfP08/tFznYtfeQ7MKSsn/KFHsKnluP49zM2xoupWh9JMyMx/KBhdifO5gFWR8uURN//D7s/s/Qf5MXMg2RZqgzArFh2bJgXj9+nAL1USkyn5nLQRQoSToUmlHdtcB0vdWAZOuqtxOVOuEu0quZ98di6hSZjFX+tUu8n+7pM7cRl7jwasp6pUqgye6V+qpLO78+lJE2FqVrmCP4aPSHc9vfJZbbwc0zyPVIT+Ei/6jH9OT3s+m0vB8ZU38U+6fxlmdKmmUwESonMfjwtc4+rBxbgpM+ZKnJA+3fqJ7mifgfKmTfz4JO466izzua5tszAkAfKBVoTT3eRqO0dGmZa0WXGBPPL5PycJHpT3vnLzr0A0O96ih7NIptEcGx5BVRDiZSaw6Cn+YfbXPj/edFTjO4CVLSJm/DmudC4UlS63S7X05cgrbnoIjhPYEyEpN3ryX1IrEpWDWouzQ+Hq1vZs1gzqL4yI74rta/A+m1hGofRHVlAsv6MA0ym8tPEm1lWQ7WbZhMu0UHgZV7z7au1Q+p+LnFVzgtKLLC6qYr6GxvyA+QE3z9NLisNCHKjTCi4qNL4vqh6N4wBgfTorzKzIznqOJ11HO8PzOldzWvXvwwc0C9yp/qdcXKk+WXHSqEA1MiSbe3HFSYl2XJO5V7uaputSakoRn0RFJsUzbxPJ1f6bd0wlxqi64GWj5P89vfReEJqn3atBABCIAUkXOO64Ilj3RMMwXglwyknp77katw32hcvDcWLFNXMQWiJK0iW+cJO5lUv28QpldlFMos/8XfDLfJYsyfYruvF5kt7TKyMBPSyG7J9H1hJRad6utYbYWS1fMHNB7lEIfVVyQwoYqP5Vr2bsKyd4oSnh0LKF88Fb3JwaGtInvp1Tillx+2Ce4oPDXUh9QrpXxZawqJQW2ViTWQhwLaEJZI1i/RsRPNIdKZF+HXD9xD6X9+Rq8mtJC44ar0iR/hbYe19AKyp469qUaJ2hnTdqKmVXCRK40t6v1X2SNaFKoJ+12pLjnfeIbY8URQsuF4KnrGV5ymaMuNI9auqHoXOEXlmHK10wAylRlmEFr5roD2uFyBovIjw63ZjZdhYix0HzDLLoPDhqR+YmsoNeZLTX+JevdLG+nb7oXbdNM5EpaR2vtfRH5c/h8jTDey1zKDLXYUWgQYnBD68VkXYKL14ibyqc3Tn7jZOSbgFnu37DMcmN5akZuG5Uo7QYn/qxDZCfSVmEiJTkos0qOjgNr2Lt34GZLgvXmkKNf58DNb4TlRQdujrUzRXYmnqkX7IOoBu8yDtN3X59jd7fv/PEtLlvAT2UcLxKWZRrN63evOuxrOk/RQhi92cN0FQfUEW6KxenczC/Dy/v3LxK7zjbm2Xxru0ShWAYLJphMsadD7qwWlCiA9pmhPQCOSZt3cE9/fB/+2LNzZZnO0SZu9a67O/VnQmz1+fqJvJg0pxl7S1t1hYEb8hOHe+hn4nLSE0bu7CdT8iklpv3NF4VWVqXqcM/xTFDVqPE1vVpZW5D7sGnx9pkmTK1SNIbL3EX41wbTHiesOrHLKXKnLBN1qm0wVTIzYHiVhDfc81VXX4DlpuFxKS0XwPe2XymVz2lJXB7O0ieUPb1S4eHfiMwWGfQkAATLhdj7gwsBTCieZlzmvb09vpT7lSmsi7zttav30/eo3F9Lxx7Be3syGtZ5utZxlyZUpwPwZVRV9YDWSklYcyF4IHYUqPXwVeH2VloEpUKZw0SstpyC8i+ZmRV7wsvSUZ0qm9/PoJ6SGJ2qdeEq5gd0gYpIaV3/QWPZQnCz6iOtUj9+WGA+08JNHw6DjEqIGkn3KdFzFrhCWCj9+jbDuBVW2lYlhnPRkcm+rredKIcozXnnDaKQ/cBAA3fV86r9tKug/gPyh6S/VmlfWf+9F/rvUYqNEUnKi9XQjno2uwc/bt39Mb+f/ZP7c70IPaENYbqMp6bpa2/9YlzpShmUSYraXjTi8vMAzcOX7uAphJY6n9G3hOC18A1qzgZmrh8TZLleoL4wLVymau3cqzAJEzi0MaHcKkcd2n3U0llwN0/Loy52sRC+0mNPldsgDhT3uyP6LUtiO4Nn0kGw3eA+fu9BoFmK53S/Rad2p++lrT3G7/81nqz/lOM5zTaeZr/DClnWl3X5IlyW6FJQOMVV5NTu2WG7H7VZZBc7laI+2eYg7B9u40sKeOmd8LjnWIAPkvALpmV3G/VM4KEPshrcn93ec3XNGbGrpdJbprMRLPkXzMaVZxjtHWy/vr5+ew1TCymT1U4tGNygZsKzp0cPNWZcY2qTUg9sTT4/3gcL7Tge5nGFw7Tq+KlZcOQY3rVGZoa+IcafDfQjV0cE6+UI+JaMi94Q1CfvwwZB92HLvQnW0LjKJk1j6ljUz23qE+3hoICLj45m9AH0RaKhCnsr5ml1AFdCa/ZrEn1+TjBjvLMZ/Gqg/bEdTzM0O5nCmlueHzn3sf9lcgmpPAAXxJN42a0Emu/8EZGvXQ6cfee6clBX17+8pDbot+yS2FmNc1nJpJI8ZcJvJjQHRNxcB0fgPYyejCK2gTlgAbPataxaMdoXYDTKzXLGZZ9ea1wri0lPztj58ykOqiiY9iHdySVq6NbD/rJFredzvWwMxk48S7l/90vVrH+8hjdkGDUBY13X6t7RimbLOnqyAllzsCAOtT6hNix7mxNxrkKbwcrfFQMZpoJRfsoMzH6ZPFzXb47g8W42v/5xPn9I1mhXKruuTmC5o58j+O3u/Ww6vzv2itLwfjK//fH6w9393fzu+pf3/3F3O4+T/oQDB5HfPOHum3YzaxMqUgQTtqIdyG/G31SxQsOqTKHvh7XsCYG5jea6vfS4pJWaD0vLox94/PlxukcR8b42LJ2duja0lbVFCBAGLKjJco2apx5HuyrSHNqL9AcPeFlEPGGv1fDORYO3KsP2OksVwkSVur6mvpLdzqJJTN+p21dzLOTgdSnYzePqSCPAL1Xjq2Nps3+xQU1JWZuMP1GrHjPimr8Sw/+Mc/ac6ikNWrtc32TGJTgJ7I2w3Jex1qhhOLlXpj10tVzCUvB8ZVvH1lxY862BArUpKHfZ9EioLbVMmFZlvIvzIvCZbQmwKchZtzLInSr1cR/i2jf10B56LxN3svkY5gnFgufLKe7yTdf8cWlonw3q8YRmOlrJCGd+Bq+vVBcCTD9UVe/a85zqbMIQ0+w5l7NSA0c2RMCXMVv/OebZ+J27/qeWRvxiUWZNwAXTDz114rqz6iLXwNbDV3wawYznvzq09Mv3o24LXTti3DMSr44rfRE9MSXvdHWeWwzHlIibKcogDNyzHWq4ms3u31aV++bADubK8vouSRL/WYw0etBTENs7rnXevR5xvxGaKuoLHPfPh/nrZyelXf3odNWfe9l/x2uxGcF/lqh3Mx9603t/0L+rWPyq0Dgm2cCMQry3r19ap1V+0oHLU9VB40osQz2dfn3mkLAV5jLaNNdMGrdH5wVtVt3vdjW/n72trVlL0kJ5/XA/unV0eynU9vQCxqW6w359uAVC8qLSxUV4TEg+EpJ7lZtqCncx806VJAzhgjBHeLhJIhxrDuznBt7VH/iDxjtgkJbGqnXfFz2iNMDtEvHA210FUN8qUdU1qyXo2zCyqJeX2OdoyggS7Vbpp2Yuh63pJreaLZc8DV0ZSmfHdw+GhXlweULsGqGAbwST29u7h7m7gvOuP5cWKj+W670aqVB5ToY2ZHqBudXyjuCXn0bw6ZcPk/nEeeKfpg/0e29Ls2XyoqteTeFY+22Xs6+QilEVutVjc+Mqj84o7lTZ0932ZBOjU5ZlcX/ymlJewSg6GAvcoIArpXnOJRNvq9JntzEkkNOPMDP2L0GYUa4ovWdvway3QY7h3BTpBSXG3eVBeljf3D+o9TDlQuLwZrfB7ye4JAk2LZKlYJ1jrWeSsOB2zcxTyOVqx6GEUFuyOPPbB3DT3sC7H2b//Wn0j3+h/40ntz+N/vHDx+mn0fc/PM7mxyEnTGs2YJWx2hprgft2yeUIzE6OQFO+V5jVCFj6NIJS59/G4V2ui9kv6g1MHzbfj+i//+wy0LuPk+s3/xcAAP//jpR9HA==" + return "eJzcXN9z47Zzf7+/Yicv8c1I7nwvmU7HnXRG5/M1bpyLa+mS9omByRWFGgIYAJROmf7xnQXAHxJBWraoXKZ+SHwmif3sYhf7AwtM4Ql3V8C25g2A5VbgFcx+m78B0CiQGbyCnL0ByNCkmheWK3kF//YGAOBnlZUCYak0rJjMBJc5CJUbWGq1pkEu3wAsOYrMXLkPpiDZGiti9GN3BRHQqizCXyJ06OejG6Ye2dG5DE/bJNpkUqHKzGrGRf0oRpF+9nmtfjJcslLYxBG4giUTBvceR6G24Srt0F4TkgUh2QMeA99mADcobbJBbbiSe29UfDzhbqt0dvBsABj9LFbYRhTGB7UEu0IC6AkT+jWzl1FopUGd8Ayl5XYXhXYo4i6waRQZjXwbBgYUuCYoqZKWcWkgQ8u4MMAeVWkdXqIGatkZ63b2M1QAwa6YhTXL0H2i8Y8SjZ0AkxlsVzxdQarRvcuEgS1q7AxXGswu4XYJFteF0kzvOt+4dyaOQoXbrNTWwEpt6a+dMTsDqEfiErPLg1djStKeDZJB5+GwjnSnI/KCn5EgYcdYz5S3bFsfaurLkXQVo4IyW7M/lYQHNKrUKcIntka4mD18elsBLDSXKS+YOJjzlAlxKNYW6jRFY5In3CU8hm8s/J4ODQS3HzzCLTNOccAqMDyXbQ3tB2zQkNEmZBj4xfZCjlnhsYBvl20sDqgT55bbVcsMDKaljqkE7Ks4mVttGI71QqsNz9AAl36toWWosezAY3TcWnSpRmYxc0utXSmDbZKRT/tMqS3c9ZIlrLQrGiWl0aNvP68VxwoagnZsmCgRuAGr6f9B/EpZtyiC0m5Rc79vidXewaIrUxBRM6FMGOVkuMern14WFzv9/PxxBhlueIr/CsquUG+5wYn3jl2FbcvVzRVpbcZsH3gv04EXXiJQGsYt8pavEbYr9NbV1d2uxLgxZXch3uenMkL3rh5kqM8OX8LRGPYIr7fJ3vGCezvenVU/Q7YIz7u36ucYM4QXyBmC9gQfExzLsNJMwJTpanBIZuBBKTshI/5sUE/IoB+U6DGatgBqpxb3TucWBJcWtWSCfFaQRjuuanuwHPv1BPZ173m247HEubmdPXyquAwacMHSVJXST51bf93caSXw7eBwMfE8o0hHSMWD+TqqEIh7ztRWmvNpQ8Uvlxv1hFnyGFvRxgrMiFQ165SxGdTk4foSBzJ2YLH4AqoY9eb6HcxKq2CeMpcah1zwRjBjeQrvkUljmXiKJ1iotdJJqrLDle/4xC+eX7W5c0TqQCP4FY221NI4z0DPh/Ct0RiWjwnxdhiMT69ag9QLUj/UMFZSMM3WaFEfztupIm0GnpAwmdxNgi2QFzTkW72P7o/s16WwPHk2zxsK9XsePlPEaIvJFEoaTEI8MLaUqvHreIPiUZbSN6aysyeEdMVkjgYufGQ/6WbiBYV1bgXOUCBFeH6Qt39DobIs4wSOicQVVTJm2YhyndXDU6zLWsmTL+HUK7JUltTUBnPpDFTZG7m1ME9/Q2lWltxxfafqps+8nLR8hWPJ0ezZq48Iw7r5iFzm3XIOEwIzyFGiZtZ9z40fumcNdfW9SHR70gK6j78q3rT0oQKYtRRFY6p0FofJCn5yEfJZnLP727oSyYxRKW+SUfd8a2YFv2ZCdEZyHCyIzwFZr5lkuVt3vCGOaYTwXimBTPao0XaFlCa3pM0NHK4C0ELo3+pzZixLlBTxouvJU9Fg5QZUQXpCM0KAHekpkW4e9GH0gXPcg7ymMDwDwY1bvOqxQy0NM+CyEe1LK6fnK1fWNcrZw6duoHhUMD8GjFkI2JvUrZIghe+RqsWoNeVnROMWp9pSdDWTvvDg9qGurtjWTMO6O3XIrsjTTelT9+8eDUx5wcnYewV8isE8YKGR4jq/drFGxs72NabIN2595WbImANffkVKwt7G+dbYOuwnchPgMhVlRqnJllBbzfMctXcL8UXW19K8DpXi7xjEmhXTmAWBjjrn//759kPLdT7u2ntoVkEp+R8lil2lz+3ncWmG7Uw3M5R+UmbmQ9ngQozPHayCjC+XqOkffm92/yfon4kr2aZIE5RZofjYIjlQr1/vr6EiRKbsd9ZCABVKgi6Vdmx3HSB9bxUw6arK7US1Trir5Hr+3VBElTKLudKvdpH/22V17jbyGg9ekahnqgye6E6pp7K48fNLEWFrVrqKPYWPSnc8v/FZbr0d0DyPVIf8EC76j35MT3o/u5WG5ytr4p9y/zQu6lRJowQmQuU8Hhe+xtGHjXNTYMqXPCV9uPaE7ohOwPlSJ/98EjaMuos87mubbMwpAHygWaE093kehvho87JWj1xgTzy+z8mjj0p73jl616EbHO5xQ9mlM2iPDIaQVUw4nUmsGoQ/zr7a54e7zgwMC3jJEjLmryNa50JhyVKrdHtfjpzCtqfgCKE9AbJSkzfvZbVicSmYtSg7PL7ebG/mzaBuxXGRHcldPf4PptYxqH0R1ZSPXtGBaZTfWniSaitp7WTZhsm0U3gY1bz7eO9y+ZyJn1ZwgeOKLi+oYr6Gx/6C+Ag1zeNLi+NCH6vQCC8qNr4sqh+P4xFjfDgqzq/YzHiOJl5HOcHzO1dyXXfuwQdHBe5U/lKvL1SeLLnoVCEamBJNvLnjqMS6LkncqdzRqbrUmpKEF9GAplimbWL5uj/T7umEONYWHAWa/s+La98FoUnrvRk0EIEYgFSR844bgmVPNAzzhSCXjKR+PXej1uG+UDl4aazYJq5ij4iSrIlvnCbuZVL9skKZnVVSKLP/F3Iy3yWPZfoU3Xk9y25plZGBJ0shu2fR9YSUWnerrYFaS6QrZg74HeTQRxVn5LDhypNyLXsXIdmbRBmPjiWUD97q/sQgkDbz/ZxK3JLLD/sEZ1T+WusDyrUyvoxVpaTA1orUWoihgCaUNcLq16j4kcuhEtnXYdcT7uG0P1+DV3NaaNxwVZrkr7DWYQutoOyZY1+qcYR11qytmFklTORKc7ta/0WrERGFmmi3I8U971PfmCgGGC0fBU9dz/CSyxx1oXl0pRuLzxV+YRmmfM0EoExVhhm0KNcd0A6XW7CI/ehwa2bTVYgYC803zKL74KARmR8pCnqd2VLjXzLfzfR2+qZ70TbNRK6kNVhr74vIn8Pna4TxXuZSZqjFjkKDEIMbmi8m6xJcvEbcVD794uQ3Tia+CZjl/g3LLDeWp2bitlGJ025w4s86RHYibRUmUpKDMqv0aBhYI9694zZbUqw3hxL9GsdtfiMkLzpuM9TMFNmXeKZasA+iGrwrNkzffW153Vy/8we3uGzBPlZsvEhYlmk0r9+56giv6TpFC2H0Zv/SVRtQR2QpHr+6LO/ev0jpOluYJ8ut7Q6FYhk8MsFkij3dcSe1n0QBtM8L7QFwQtq8gzv64/vwx55dK8t0jjZxs3fZ3aU/EWKrx9cT8mrSnGPsLWvV1QVuyEcc7p+fiMtpTxi5s5dMiaeUmPY3XhRaWZWqw/3GE0FVo8bn9GJlbUGuw6bF22caMLVK0RgucxfdXxpMexyw6sQtx+idskzUabbBVMnMgOFVAt5Iz1dcffGVm0bGpbRcAN/beqU0PqcpcTk4S59Q9vRJhYd/IzZbbNCTABAsF2LvD879m1A4zbjMe/t6fBn3K3NYF3jbc1fvpe9xuT+XTjyC9/ZjNKLzfK3jLk2oTvffy7iqagGtmZKw5kLwwOwkcOvhq8Ltq7QYSoUyh0lYvXIKyr1kZlbsCc/LR3WibHE3h5okCTpV68JVyw/4AhXR0rr2g8ayR8HNqo+1yvz4YXH5xBXu9v4wyKiUqNF0nw49twJXCAulX99iGF+FlbZVeeFUdLRkX9ZbTpQ/lOa0swZRyH5goIG75nnRfto1UP8B+UOyX6u0r6r/3gv99yjHxogk5cVqbEc9n9+BH7fu/Fjczf/J/bmehJ7QhjCdx1MT+dpbvxhXulIGZZKitmeNuDwdIDp86Q6dQmin89l8SwleC9+g5mxk4foxQZbrR9Rn5oXLVK2dexUmYQLHXkwot8pRh1YftXQruKPT8qiPu1gIX9mx58ptDgeO+90R/ZYlsV3BE/kg2G5wH7/3INAsxVM636Kk3cl7aWuP8ft/TWfrP+V0QdSmt9nvsEKW9WVdvgCXJboUFE5xFTmxe3LY7kdtJtnFTqWoT7U5CPsH2/iSAl56JzzuORLggyT8gmnZ3UI9EXjogawG9+e291xdcz7sYqn0lulsAkv+BbNp5Rkme4faLy8v317CrYWUyWqXFgxuUDPhxdNjhxozrjG1SalHXk0+P9yFFdpJPNBxRcO06vapRTBwBO9SIzNj3w7jzwX6kavjgfV0BHxLxkVvCOqT93GDoLuw3d4Ea2hcVZPImDoW9bRNfZo9HBJw8dFgRh9AnyUaqrC3Yp5W92+ltGa/JtHn5wQzxjub0a8F2h/byTRDs5MprLnl+cCZj/0vk3No5QG4oJ4ky24l0Hznj4d87XLg/DvXkYO6uvrlJbVBv12XxM5pnCpKJpXkKRN+I6E5HOJoHRx/9zB6MorY5uWIBcxqx7Jqw2hfftEYN8sZl312rXGtLCY9OWPnz8c4qKJg2od0R5eooVsP+8smtabn+tgYTJ16lnL/3peqUX+4hjdmGDUDY13H6t6xima7OnqqAllzqCAOtT6dNq54m9NwrkKbwcrfEwMZpoJRfsoMzH+Z3V/Wb07g4Wa+uPxxsbhP1mhXKrusTl+5Y58T+O3m/fx2cTP0itLwfra4/vHyw83dzeLm8pf3/3FzvYiz/oQjB5HfPOHum3YjaxMqUgQTtqEdyG+m31SxQiOqTKHvhbXsCYG5Tea6tXRY00rNx+XlwQ88/fxwu8cRyb5eWDr7dG1oK2uLECCMWFCT5Ro1Tz2OdlWkObAX6Q0e8aKIeMJem+GNiwavVYbteZYqhIkqdT1NfSW7nUWTmL4Tt6+WWMjB61Kwo+PqSBPAL1XTqxNps3+xQU1JWZuNP1GrnmXENX4lhv8Zl+wp1VMatHa5vsGMS3Aa2BthuS9jbVHjSHKvTHvoarmEpeD5yraOrLmw5lsDBWpTUO6y6dFQW2qZMK3KeAfnWeAz21JgU5CzbmWQO1XqYR/iWjf12B56LxN3uvkQ6IRiwfPlFHfxpmv8ODe0zwb1dEaUBisZ4bzP6PWV6jKA2w9V1bv2PMc6mzDEbfacy1mpkSMbYuDLlK3/nPJs+s5d/VNrI36xKLMm4ILbDz114rqr6ixXwNbDV3KawJznvzq09Mv3k277XDti3FskXh1X+iJ6Ykre6eg8tRiOKTE3V5RBGLhjO9RwMZ/fva0q981hHcyV5fU9kqT+8xhr9KCnILZ3VOu0Oz3ifiM0VdSXN+6fDfNXz85Ku/rR2ao/87L/jrdiM4H/LFHv5j70pvf+oH9XsfhFoXFKuoEZhXhvXz+1zqo80ZHLU9Uh40otQz2dfn3mgLAV5jzWtNBMGrdH5xVtXt3tdrG4m7+tV7OWpoXy+uF+dOvY9lKo7dcuYPx6fw2E40WFi7NImJB8JCR3KjcVCXcl806VpArhajDHeLhDIhxoDsLnBt7VH/gjxjtgkJbGqnXfFz2KNMK9EvGw210CUN8nUVU1qyno2y6yqJfn2OVoiggS7Vbpp4aWw9b0kVvNlkuehp4MpbPhvYNxYR5cmxC7QCjgm8Ds+vrmfuEu37zpz6SFyocyvVcjFSrPaZkNeV4QbjW9E/jlpwl8+uXDbDFzfvin23v6vbeZ2TJ51lmvSDjRftuV7Cu0YlIFbvXY3Li6o1sSd6rs6W17sonRKcuyuDd5TSGvYBQbTAVuUMCF0jznkom3VeGz2xYS2OlHmBn7lyDMKFOU3q+3YNabIEM4N0V6Ro1xt3iQHdZ39o+6epjyUeL4y26D3xM4Jws2LZKlYJ0DrSey8MjtmpmnkMnVjkMJoba04iyu78GRvYJ3P8z/+9PkH/9C/5vOrn+a/OOHj7efJt//8DBfDENOmNZsxBpjtTHWAvftkssJmJ2cgKZsrzCrCbD0aQKlzr+NwztfD7Of1Cu4vd98P6H//rPLP28+zi7f/F8AAAD//5/wePg=" } diff --git a/x-pack/filebeat/module/aws/vpcflow/_meta/fields.yml b/x-pack/filebeat/module/aws/vpcflow/_meta/fields.yml index f1c658b8a450..fe63ea9bf07d 100644 --- a/x-pack/filebeat/module/aws/vpcflow/_meta/fields.yml +++ b/x-pack/filebeat/module/aws/vpcflow/_meta/fields.yml @@ -1,6 +1,6 @@ - name: vpcflow type: group - release: beta + release: ga default_field: false description: > Fields for AWS VPC flow logs. From 53e5d236a48fb826236ba79dbca759dac3515844 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20DERACO?= Date: Tue, 17 Aug 2021 23:16:10 +0200 Subject: [PATCH 2/7] Fix small typo "promtheus" -> "prometheus" (#26515) --- metricbeat/module/prometheus/query/_meta/docs.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metricbeat/module/prometheus/query/_meta/docs.asciidoc b/metricbeat/module/prometheus/query/_meta/docs.asciidoc index 60025f0d96da..73b8d5226d21 100644 --- a/metricbeat/module/prometheus/query/_meta/docs.asciidoc +++ b/metricbeat/module/prometheus/query/_meta/docs.asciidoc @@ -1,4 +1,4 @@ -This is the `query` metricset to query from https://prometheus.io/docs/prometheus/latest/querying/api/#expression-queries[querying API of Promtheus]. +This is the `query` metricset to query from https://prometheus.io/docs/prometheus/latest/querying/api/#expression-queries[querying API of Prometheus]. [float] From feac0a949ad77efd521b7800e6e760b48dc47af1 Mon Sep 17 00:00:00 2001 From: EamonnTP Date: Wed, 18 Aug 2021 10:00:15 +0100 Subject: [PATCH 3/7] [DOCS] Remove PKI mentions (#27422) * Remove pki mentions * Review edit --- libbeat/docs/shared-kibana-config.asciidoc | 35 +++++++++++----------- libbeat/docs/shared-ssl-config.asciidoc | 20 ++++++------- 2 files changed, 28 insertions(+), 27 deletions(-) diff --git a/libbeat/docs/shared-kibana-config.asciidoc b/libbeat/docs/shared-kibana-config.asciidoc index 6cb3a1e9e29f..9f05939aca3e 100644 --- a/libbeat/docs/shared-kibana-config.asciidoc +++ b/libbeat/docs/shared-kibana-config.asciidoc @@ -10,14 +10,15 @@ ////////////////////////////////////////////////////////////////////////// [[setup-kibana-endpoint]] -== Configure the Kibana endpoint +== Configure the {kib} endpoint ++++ -Kibana endpoint +{kib} endpoint ++++ -Starting with {beatname_uc} 6.0.0, the Kibana dashboards are loaded into Kibana -via the Kibana API. This requires a Kibana endpoint configuration. +{kib} dashboards are loaded into {kib} +via the {kib} API. This requires a {kib} endpoint configuration. For details on +authenticating to the {kib} API, see {kibana-ref}/api.html#api-authentication[Authentication]. You configure the endpoint in the `setup.kibana` section of the +{beatname_lc}.yml+ config file. @@ -38,7 +39,7 @@ You can specify the following options in the `setup.kibana` section of the [float] ==== `setup.kibana.host` -The Kibana host where the dashboards will be loaded. The default is +The {kib} host where the dashboards will be loaded. The default is `127.0.0.1:5601`. The value of `host` can be a `URL` or `IP:PORT`. For example: `http://192.15.3.2`, `192:15.3.2:5601` or `http://192.15.3.2:6701/path`. If no port is specified, `5601` is used. @@ -53,7 +54,7 @@ IPv6 addresses must be defined using the following format: [[kibana-protocol-option]] ==== `setup.kibana.protocol` -The name of the protocol Kibana is reachable on. The options are: `http` or +The name of the protocol {kib} is reachable on. The options are: `http` or `https`. The default is `http`. However, if you specify a URL for host, the value of `protocol` is overridden by whatever scheme you specify in the URL. @@ -70,23 +71,23 @@ setup.kibana.path: /kibana [float] ==== `setup.kibana.username` -The basic authentication username for connecting to Kibana. If you don't +The basic authentication username for connecting to {kib}. If you don't specify a value for this setting, {beatname_uc} uses the `username` specified -for the Elasticsearch output. +for the {es} output. [float] ==== `setup.kibana.password` -The basic authentication password for connecting to Kibana. If you don't +The basic authentication password for connecting to {kib}. If you don't specify a value for this setting, {beatname_uc} uses the `password` specified -for the Elasticsearch output. +for the {es} output. [float] [[kibana-path-option]] ==== `setup.kibana.path` An HTTP path prefix that is prepended to the HTTP API calls. This is useful for -the cases where Kibana listens behind an HTTP reverse proxy that exports the API +the cases where {kib} listens behind an HTTP reverse proxy that exports the API under a custom prefix. [float] @@ -94,13 +95,13 @@ under a custom prefix. ==== `setup.kibana.space.id` The {kibana-ref}/xpack-spaces.html[Kibana space] ID to use. If specified, -{beatname_uc} loads Kibana assets into this Kibana space. Omit this option to +{beatname_uc} loads {kib} assets into this {kib} space. Omit this option to use the default space. [float] ===== `setup.kibana.headers` -Custom HTTP headers to add to each request sent to Kibana. +Custom HTTP headers to add to each request sent to {kib}. Example: [source,yaml] @@ -112,7 +113,7 @@ setup.kibana.headers: [float] ==== `setup.kibana.ssl.enabled` -Enables {beatname_uc} to use SSL settings when connecting to Kibana via HTTPS. +Enables {beatname_uc} to use SSL settings when connecting to {kib} via HTTPS. If you configure {beatname_uc} to connect over HTTPS, this setting defaults to `true` and {beatname_uc} uses the default SSL settings. @@ -122,9 +123,9 @@ Example configuration: ---- setup.kibana.host: "https://192.0.2.255:5601" setup.kibana.ssl.enabled: true -setup.kibana.ssl.certificate_authorities: ["/etc/pki/root/ca.pem"] -setup.kibana.ssl.certificate: "/etc/pki/client/cert.pem" -setup.kibana.ssl.key: "/etc/pki/client/cert.key +setup.kibana.ssl.certificate_authorities: ["/etc/client/ca.pem"] +setup.kibana.ssl.certificate: "/etc/client/cert.pem" +setup.kibana.ssl.key: "/etc/client/cert.key ---- See <> for more information. diff --git a/libbeat/docs/shared-ssl-config.asciidoc b/libbeat/docs/shared-ssl-config.asciidoc index ec0690397a53..1c20e6b85658 100644 --- a/libbeat/docs/shared-ssl-config.asciidoc +++ b/libbeat/docs/shared-ssl-config.asciidoc @@ -32,9 +32,9 @@ Example output config with SSL enabled: [source,yaml] ---- output.elasticsearch.hosts: ["https://192.168.1.42:9200"] -output.elasticsearch.ssl.certificate_authorities: ["/etc/pki/root/ca.pem"] -output.elasticsearch.ssl.certificate: "/etc/pki/client/cert.pem" -output.elasticsearch.ssl.key: "/etc/pki/client/cert.key" +output.elasticsearch.ssl.certificate_authorities: ["/etc/client/ca.pem"] +output.elasticsearch.ssl.certificate: "/etc/client/cert.pem" +output.elasticsearch.ssl.key: "/etc/client/cert.key" ---- ifndef::no-output-logstash[] @@ -48,9 +48,9 @@ Example Kibana endpoint config with SSL enabled: ---- setup.kibana.host: "https://192.0.2.255:5601" setup.kibana.ssl.enabled: true -setup.kibana.ssl.certificate_authorities: ["/etc/pki/root/ca.pem"] -setup.kibana.ssl.certificate: "/etc/pki/client/cert.pem" -setup.kibana.ssl.key: "/etc/pki/client/cert.key" +setup.kibana.ssl.certificate_authorities: ["/etc/client/ca.pem"] +setup.kibana.ssl.certificate: "/etc/client/cert.pem" +setup.kibana.ssl.key: "/etc/client/cert.key" ---- endif::no_kibana[] @@ -264,7 +264,7 @@ certificate_authorities: [float] [[client-certificate]] -==== `certificate: "/etc/pki/client/cert.pem"` +==== `certificate: "/etc/client/cert.pem"` The path to the certificate for SSL client authentication is only required if `client_authentication` is specified. If the certificate @@ -302,7 +302,7 @@ certificate: | [float] [[client-key]] -==== `key: "/etc/pki/client/cert.key"` +==== `key: "/etc/client/cert.key"` The client certificate key used for client authentication and is only required if `client_authentication` is configured. The key option support embedding of the private key: @@ -423,7 +423,7 @@ certificate_authorities: [float] [[server-certificate]] -==== `certificate: "/etc/pki/server/cert.pem"` +==== `certificate: "/etc/server/cert.pem"` For server authentication, the path to the SSL authentication certificate must be specified for TLS. If the certificate is not specified, startup will fail. @@ -457,7 +457,7 @@ certificate: | [float] [[server-key]] -==== `key: "/etc/pki/server/cert.key"` +==== `key: "/etc/server/cert.key"` The server certificate key used for authentication is required. The key option support embedding of the private key: From 47c0c726b0fa1d0af72088216e7f04301196ae43 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?No=C3=A9mi=20V=C3=A1nyi?= Date: Wed, 18 Aug 2021 14:15:42 +0200 Subject: [PATCH 4/7] Do not overwrite content type if it is multipart in Kibana loader (#27424) ## What does this PR do? This PR fixes dashboard loading. Kibana loader overwrites every content type to application/json. However, for multipart file transfer it has to be set to multipart/formdata and application/ndjson. Otherwise, loading the assets fails. Also, a module file has not been updated with new dashboard names. --- libbeat/kibana/client.go | 2 ++ libbeat/kibana/client_test.go | 27 +++++++++++++++++++ x-pack/filebeat/module/netflow/dashboards.yml | 16 +++++------ 3 files changed, 37 insertions(+), 8 deletions(-) diff --git a/libbeat/kibana/client.go b/libbeat/kibana/client.go index a4a8eb4d38c7..2798443e90c2 100644 --- a/libbeat/kibana/client.go +++ b/libbeat/kibana/client.go @@ -24,6 +24,7 @@ import ( "fmt" "io" "io/ioutil" + "mime" "mime/multipart" "net/http" "net/textproto" @@ -214,6 +215,7 @@ func (conn *Connection) SendWithContext(ctx context.Context, method, extraPath s addHeaders(req.Header, headers) contentType := req.Header.Get("Content-Type") + contentType, _, _ = mime.ParseMediaType(contentType) if contentType != "multipart/form-data" && contentType != "application/ndjson" { req.Header.Set("Content-Type", "application/json") } diff --git a/libbeat/kibana/client_test.go b/libbeat/kibana/client_test.go index 24d8921bb041..c69592b8ef5c 100644 --- a/libbeat/kibana/client_test.go +++ b/libbeat/kibana/client_test.go @@ -119,3 +119,30 @@ headers: assert.Equal(t, []string{"1"}, requests[1].Header.Values("kbn-xsrf")) } + +func TestNewKibanaClientWithMultipartData(t *testing.T) { + var requests []*http.Request + kibanaTs := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests = append(requests, r) + if r.URL.Path == "/api/status" { + w.Write([]byte(`{"version":{"number":"1.2.3-beta","build_snapshot":true}}`)) + } + })) + defer kibanaTs.Close() + + client, err := NewKibanaClient(common.MustNewConfigFrom(fmt.Sprintf(` +protocol: http +host: %s +headers: + content-type: multipart/form-data; boundary=46bea21be603a2c2ea6f51571a5e1baf5ea3be8ebd7101199320607b36ff + accept: text/plain + kbn-xsrf: 0 +`, kibanaTs.Listener.Addr().String()))) + require.NoError(t, err) + require.NotNil(t, client) + + client.Request(http.MethodPost, "/foo", url.Values{}, http.Header{"key": []string{"another_value"}}, nil) + + assert.Equal(t, []string{"multipart/form-data; boundary=46bea21be603a2c2ea6f51571a5e1baf5ea3be8ebd7101199320607b36ff"}, requests[1].Header.Values("Content-Type")) + +} diff --git a/x-pack/filebeat/module/netflow/dashboards.yml b/x-pack/filebeat/module/netflow/dashboards.yml index 313bfb6fc4b2..1ba44d0e402e 100644 --- a/x-pack/filebeat/module/netflow/dashboards.yml +++ b/x-pack/filebeat/module/netflow/dashboards.yml @@ -2,25 +2,25 @@ dashboards: - id: 77326664-23be-4bf1-a126-6d7e60cfc024 - file: filebeat-netflow-geo-location.json + file: filebeat-netflow-geo-location.ndjson - id: 38012abe-c611-4124-8497-381fcd85acc8 - file: filebeat-netflow-traffic-analysis.json + file: filebeat-netflow-traffic-analysis.ndjson - id: c64665f9-d222-421e-90b0-c7310d944b8a - file: filebeat-netflow-autonomous-systems.json + file: filebeat-netflow-autonomous-systems.ndjson - id: acd7a630-0c71-4840-bc9e-4a3801374a32 - file: filebeat-netflow-conversation-partners.json + file: filebeat-netflow-conversation-partners.ndjson - id: 34e26884-161a-4448-9556-43b5bf2f62a2 - file: filebeat-netflow-overview.json + file: filebeat-netflow-overview.ndjson - id: feebb4e6-b13e-4e4e-b9fc-d3a178276425 - file: filebeat-netflow-flow-exporters.json + file: filebeat-netflow-flow-exporters.ndjson - id: 94972700-de4a-4272-9143-2fa8d4981365 - file: filebeat-netflow-flow-records.json + file: filebeat-netflow-flow-records.ndjson - id: 14387a13-53bc-43a4-b9cd-63977aa8d87c - file: filebeat-netflow-top-n.json + file: filebeat-netflow-top-n.ndjson From b4ecc29bb5031e1329f12bd4358e353a15385de6 Mon Sep 17 00:00:00 2001 From: Andrea Spacca Date: Wed, 18 Aug 2021 15:09:24 +0200 Subject: [PATCH 5/7] Add vars in modules.d/aws.yml.disabled (#27454) * Add vars in modules.d/aws.yml.disabled * missing metadata * rename bucket to bucket_arn --- filebeat/docs/modules/aws.asciidoc | 12 ++--- x-pack/filebeat/filebeat.reference.yml | 54 +++++++++++++++++++ x-pack/filebeat/input/awss3/acker.go | 2 +- x-pack/filebeat/input/awss3/config.go | 16 +++--- x-pack/filebeat/input/awss3/config_test.go | 22 ++++---- x-pack/filebeat/input/awss3/input.go | 7 +-- .../input/awss3/input_benchmark_test.go | 2 +- .../input/awss3/input_integration_test.go | 2 +- x-pack/filebeat/input/awss3/s3.go | 5 ++ x-pack/filebeat/input/awss3/s3_objects.go | 4 +- x-pack/filebeat/input/awss3/s3_test.go | 4 +- x-pack/filebeat/module/aws/_meta/config.yml | 54 +++++++++++++++++++ .../filebeat/module/aws/_meta/docs.asciidoc | 12 ++--- .../module/aws/cloudtrail/config/aws-s3.yml | 4 +- .../module/aws/cloudtrail/manifest.yml | 2 +- .../module/aws/cloudwatch/config/aws-s3.yml | 4 +- .../module/aws/cloudwatch/manifest.yml | 2 +- .../filebeat/module/aws/ec2/config/aws-s3.yml | 4 +- x-pack/filebeat/module/aws/ec2/manifest.yml | 2 +- .../filebeat/module/aws/elb/config/aws-s3.yml | 4 +- x-pack/filebeat/module/aws/elb/manifest.yml | 2 +- .../module/aws/s3access/config/aws-s3.yml | 4 +- .../filebeat/module/aws/s3access/manifest.yml | 2 +- .../module/aws/vpcflow/config/input.yml | 4 +- .../filebeat/module/aws/vpcflow/manifest.yml | 2 +- x-pack/filebeat/modules.d/aws.yml.disabled | 54 +++++++++++++++++++ 26 files changed, 227 insertions(+), 59 deletions(-) diff --git a/filebeat/docs/modules/aws.asciidoc b/filebeat/docs/modules/aws.asciidoc index 3f9c2c11653c..881d12481446 100644 --- a/filebeat/docs/modules/aws.asciidoc +++ b/filebeat/docs/modules/aws.asciidoc @@ -47,7 +47,7 @@ Example config: cloudtrail: enabled: false #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue - #var.bucket: 'arn:aws:s3:::mybucket' + #var.bucket_arn: 'arn:aws:s3:::mybucket' #var.bucket_list_interval: 300s #var.number_of_workers: 5 #var.shared_credential_file: /etc/filebeat/aws_credentials @@ -64,7 +64,7 @@ Example config: cloudwatch: enabled: false #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue - #var.bucket: 'arn:aws:s3:::mybucket' + #var.bucket_arn: 'arn:aws:s3:::mybucket' #var.bucket_list_interval: 300s #var.number_of_workers: 5 #var.shared_credential_file: /etc/filebeat/aws_credentials @@ -81,7 +81,7 @@ Example config: ec2: enabled: false #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue - #var.bucket: 'arn:aws:s3:::mybucket' + #var.bucket_arn: 'arn:aws:s3:::mybucket' #var.bucket_list_interval: 300s #var.number_of_workers: 5 #var.shared_credential_file: /etc/filebeat/aws_credentials @@ -98,7 +98,7 @@ Example config: elb: enabled: false #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue - #var.bucket: 'arn:aws:s3:::mybucket' + #var.bucket_arn: 'arn:aws:s3:::mybucket' #var.bucket_list_interval: 300s #var.number_of_workers: 5 #var.shared_credential_file: /etc/filebeat/aws_credentials @@ -115,7 +115,7 @@ Example config: s3access: enabled: false #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue - #var.bucket: 'arn:aws:s3:::mybucket' + #var.bucket_arn: 'arn:aws:s3:::mybucket' #var.bucket_list_interval: 300s #var.number_of_workers: 5 #var.shared_credential_file: /etc/filebeat/aws_credentials @@ -132,7 +132,7 @@ Example config: vpcflow: enabled: false #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue - #var.bucket: 'arn:aws:s3:::mybucket' + #var.bucket_arn: 'arn:aws:s3:::mybucket' #var.bucket_list_interval: 300s #var.number_of_workers: 5 #var.shared_credential_file: /etc/filebeat/aws_credentials diff --git a/x-pack/filebeat/filebeat.reference.yml b/x-pack/filebeat/filebeat.reference.yml index a482caf1b31f..31f7554f6b3a 100644 --- a/x-pack/filebeat/filebeat.reference.yml +++ b/x-pack/filebeat/filebeat.reference.yml @@ -102,6 +102,15 @@ filebeat.modules: # AWS SQS queue url #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue + # AWS S3 bucket arn + #var.bucket_arn: 'arn:aws:s3:::mybucket' + + # Bucket list interval on S3 bucket + #var.bucket_list_interval: 300s + + # Number of workers on S3 bucket + #var.number_of_workers: 5 + # Process CloudTrail logs # default is true, set to false to skip Cloudtrail logs # var.process_cloudtrail_logs: false @@ -154,6 +163,15 @@ filebeat.modules: # AWS SQS queue url #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue + # AWS S3 bucket arn + #var.bucket_arn: 'arn:aws:s3:::mybucket' + + # Bucket list interval on S3 bucket + #var.bucket_list_interval: 300s + + # Number of workers on S3 bucket + #var.number_of_workers: 5 + # Filename of AWS credential file # If not set "$HOME/.aws/credentials" is used on Linux/Mac # "%UserProfile%\.aws\credentials" is used on Windows @@ -194,6 +212,15 @@ filebeat.modules: # AWS SQS queue url #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue + # AWS S3 bucket arn + #var.bucket_arn: 'arn:aws:s3:::mybucket' + + # Bucket list interval on S3 bucket + #var.bucket_list_interval: 300s + + # Number of workers on S3 bucket + #var.number_of_workers: 5 + # Filename of AWS credential file # If not set "$HOME/.aws/credentials" is used on Linux/Mac # "%UserProfile%\.aws\credentials" is used on Windows @@ -234,6 +261,15 @@ filebeat.modules: # AWS SQS queue url #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue + # AWS S3 bucket arn + #var.bucket_arn: 'arn:aws:s3:::mybucket' + + # Bucket list interval on S3 bucket + #var.bucket_list_interval: 300s + + # Number of workers on S3 bucket + #var.number_of_workers: 5 + # Filename of AWS credential file # If not set "$HOME/.aws/credentials" is used on Linux/Mac # "%UserProfile%\.aws\credentials" is used on Windows @@ -274,6 +310,15 @@ filebeat.modules: # AWS SQS queue url #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue + # AWS S3 bucket arn + #var.bucket_arn: 'arn:aws:s3:::mybucket' + + # Bucket list interval on S3 bucket + #var.bucket_list_interval: 300s + + # Number of workers on S3 bucket + #var.number_of_workers: 5 + # Filename of AWS credential file # If not set "$HOME/.aws/credentials" is used on Linux/Mac # "%UserProfile%\.aws\credentials" is used on Windows @@ -314,6 +359,15 @@ filebeat.modules: # AWS SQS queue url #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue + # AWS S3 bucket arn + #var.bucket_arn: 'arn:aws:s3:::mybucket' + + # Bucket list interval on S3 bucket + #var.bucket_list_interval: 300s + + # Number of workers on S3 bucket + #var.number_of_workers: 5 + # Filename of AWS credential file # If not set "$HOME/.aws/credentials" is used on Linux/Mac # "%UserProfile%\.aws\credentials" is used on Windows diff --git a/x-pack/filebeat/input/awss3/acker.go b/x-pack/filebeat/input/awss3/acker.go index ba80601997fe..db88c23f7d18 100644 --- a/x-pack/filebeat/input/awss3/acker.go +++ b/x-pack/filebeat/input/awss3/acker.go @@ -52,7 +52,7 @@ func (a *eventACKTracker) ACK() { // Wait waits for the number of pending ACKs to be zero. // Wait must be called sequentially only after every expected -// Add call are made. Failing to do so could reset the pendingACKs +// `Add` calls are made. Failing to do so could reset the pendingACKs // property to 0 and would results in Wait returning after additional // calls to `Add` are made without a corresponding `ACK` call. func (a *eventACKTracker) Wait() { diff --git a/x-pack/filebeat/input/awss3/config.go b/x-pack/filebeat/input/awss3/config.go index 9fef4a9fc600..4e887003477f 100644 --- a/x-pack/filebeat/input/awss3/config.go +++ b/x-pack/filebeat/input/awss3/config.go @@ -26,7 +26,7 @@ type config struct { FIPSEnabled bool `config:"fips_enabled"` MaxNumberOfMessages int `config:"max_number_of_messages"` QueueURL string `config:"queue_url"` - Bucket string `config:"bucket"` + BucketARN string `config:"bucket_arn"` BucketListInterval time.Duration `config:"bucket_list_interval"` NumberOfWorkers int `config:"number_of_workers"` AWSConfig awscommon.ConfigAWS `config:",inline"` @@ -49,20 +49,20 @@ func defaultConfig() config { } func (c *config) Validate() error { - if c.QueueURL == "" && c.Bucket == "" { - return fmt.Errorf("queue_url or bucket must provided") + if c.QueueURL == "" && c.BucketARN == "" { + return fmt.Errorf("queue_url or bucket_arn must provided") } - if c.QueueURL != "" && c.Bucket != "" { - return fmt.Errorf("queue_url <%v> and bucket <%v> "+ - "cannot be set at the same time", c.QueueURL, c.Bucket) + if c.QueueURL != "" && c.BucketARN != "" { + return fmt.Errorf("queue_url <%v> and bucket_arn <%v> "+ + "cannot be set at the same time", c.QueueURL, c.BucketARN) } - if c.Bucket != "" && c.BucketListInterval <= 0 { + if c.BucketARN != "" && c.BucketListInterval <= 0 { return fmt.Errorf("bucket_list_interval <%v> must be greater than 0", c.BucketListInterval) } - if c.Bucket != "" && c.NumberOfWorkers <= 0 { + if c.BucketARN != "" && c.NumberOfWorkers <= 0 { return fmt.Errorf("number_of_workers <%v> must be greater than 0", c.NumberOfWorkers) } diff --git a/x-pack/filebeat/input/awss3/config_test.go b/x-pack/filebeat/input/awss3/config_test.go index c0fd94e2aee6..9fdf4c1dffbc 100644 --- a/x-pack/filebeat/input/awss3/config_test.go +++ b/x-pack/filebeat/input/awss3/config_test.go @@ -29,7 +29,7 @@ func TestConfig(t *testing.T) { require.NoError(t, parserConf.Unpack(common.MustNewConfigFrom(""))) return config{ QueueURL: quequeURL, - Bucket: s3Bucket, + BucketARN: s3Bucket, APITimeout: 120 * time.Second, VisibilityTimeout: 300 * time.Second, SQSMaxReceiveCount: 5, @@ -69,7 +69,7 @@ func TestConfig(t *testing.T) { "", s3Bucket, common.MapStr{ - "bucket": s3Bucket, + "bucket_arn": s3Bucket, "number_of_workers": 5, }, "", @@ -109,10 +109,10 @@ func TestConfig(t *testing.T) { "", "", common.MapStr{ - "queue_url": "", - "bucket": "", + "queue_url": "", + "bucket_arn": "", }, - "queue_url or bucket must provided", + "queue_url or bucket_arn must provided", nil, }, { @@ -120,10 +120,10 @@ func TestConfig(t *testing.T) { queueURL, s3Bucket, common.MapStr{ - "queue_url": queueURL, - "bucket": s3Bucket, + "queue_url": queueURL, + "bucket_arn": s3Bucket, }, - "queue_url and bucket cannot be set at the same time", + "queue_url and bucket_arn cannot be set at the same time", nil, }, { @@ -164,7 +164,7 @@ func TestConfig(t *testing.T) { "", s3Bucket, common.MapStr{ - "bucket": s3Bucket, + "bucket_arn": s3Bucket, "bucket_list_interval": "0", }, "bucket_list_interval <0s> must be greater than 0", @@ -175,7 +175,7 @@ func TestConfig(t *testing.T) { "", s3Bucket, common.MapStr{ - "bucket": s3Bucket, + "bucket_arn": s3Bucket, "number_of_workers": "0", }, "number_of_workers <0> must be greater than 0", @@ -231,7 +231,7 @@ func TestConfig(t *testing.T) { "", s3Bucket, common.MapStr{ - "bucket": s3Bucket, + "bucket_arn": s3Bucket, "expand_event_list_from_field": "Records", "content_type": "text/plain", }, diff --git a/x-pack/filebeat/input/awss3/input.go b/x-pack/filebeat/input/awss3/input.go index d12b2a24cee5..6662984e060e 100644 --- a/x-pack/filebeat/input/awss3/input.go +++ b/x-pack/filebeat/input/awss3/input.go @@ -138,7 +138,7 @@ func (in *s3Input) Run(inputContext v2.Context, pipeline beat.Pipeline) error { } } - if in.config.Bucket != "" { + if in.config.BucketARN != "" { // Create S3 receiver and S3 notification processor. poller, err := in.createS3Lister(inputContext, client, persistentStore, states) if err != nil { @@ -203,7 +203,7 @@ func (in *s3Input) createS3Lister(ctx v2.Context, client beat.Client, persistent client: s3.New(awscommon.EnrichAWSConfigWithEndpoint(in.config.AWSConfig.Endpoint, s3ServiceName, in.awsConfig.Region, in.awsConfig)), } - log := ctx.Logger.With("s3_bucket", in.config.Bucket) + log := ctx.Logger.With("bucket_arn", in.config.BucketARN) log.Infof("number_of_workers is set to %v.", in.config.NumberOfWorkers) log.Infof("bucket_list_interval is set to %v.", in.config.BucketListInterval) log.Infof("AWS region is set to %v.", in.awsConfig.Region) @@ -223,7 +223,8 @@ func (in *s3Input) createS3Lister(ctx v2.Context, client beat.Client, persistent s3EventHandlerFactory, states, persistentStore, - in.config.Bucket, + in.config.BucketARN, + in.awsConfig.Region, in.config.NumberOfWorkers, in.config.BucketListInterval) diff --git a/x-pack/filebeat/input/awss3/input_benchmark_test.go b/x-pack/filebeat/input/awss3/input_benchmark_test.go index 0c7df7e012b0..00540479d5c5 100644 --- a/x-pack/filebeat/input/awss3/input_benchmark_test.go +++ b/x-pack/filebeat/input/awss3/input_benchmark_test.go @@ -277,7 +277,7 @@ func benchmarkInputS3(t *testing.T, numberOfWorkers int) testing.BenchmarkResult } s3EventHandlerFactory := newS3ObjectProcessorFactory(log.Named("s3"), metrics, s3API, client, conf.FileSelectors) - s3Poller := newS3Poller(logp.NewLogger(inputName), metrics, s3API, s3EventHandlerFactory, newStates(inputCtx), store, "bucket", numberOfWorkers, time.Second) + s3Poller := newS3Poller(logp.NewLogger(inputName), metrics, s3API, s3EventHandlerFactory, newStates(inputCtx), store, "bucket", "region", numberOfWorkers, time.Second) ctx, cancel := context.WithCancel(context.Background()) b.Cleanup(cancel) diff --git a/x-pack/filebeat/input/awss3/input_integration_test.go b/x-pack/filebeat/input/awss3/input_integration_test.go index 1ce157c553f8..4b8a3115db69 100644 --- a/x-pack/filebeat/input/awss3/input_integration_test.go +++ b/x-pack/filebeat/input/awss3/input_integration_test.go @@ -75,7 +75,7 @@ func getTerraformOutputs(t *testing.T) terraformOutputData { func makeTestConfigS3(s3bucket string) *common.Config { return common.MustNewConfigFrom(fmt.Sprintf(`--- -bucket: aws:s3:::%s +bucket_arn: aws:s3:::%s number_of_workers: 1 file_selectors: - diff --git a/x-pack/filebeat/input/awss3/s3.go b/x-pack/filebeat/input/awss3/s3.go index f1c08d2fb81f..313a71211e70 100644 --- a/x-pack/filebeat/input/awss3/s3.go +++ b/x-pack/filebeat/input/awss3/s3.go @@ -41,6 +41,7 @@ type s3ObjectPayload struct { type s3Poller struct { numberOfWorkers int bucket string + region string bucketPollInterval time.Duration workerSem *sem s3 s3API @@ -60,6 +61,7 @@ func newS3Poller(log *logp.Logger, states *states, store *statestore.Store, bucket string, + awsRegion string, numberOfWorkers int, bucketPollInterval time.Duration) *s3Poller { if metrics == nil { @@ -68,6 +70,7 @@ func newS3Poller(log *logp.Logger, return &s3Poller{ numberOfWorkers: numberOfWorkers, bucket: bucket, + region: awsRegion, bucketPollInterval: bucketPollInterval, workerSem: newSem(numberOfWorkers), s3: s3, @@ -178,7 +181,9 @@ func (p *s3Poller) GetS3Objects(ctx context.Context, s3ObjectPayloadChan chan<- p.states.Update(state, "") event := s3EventV2{} + event.AWSRegion = p.region event.S3.Bucket.Name = bucketName + event.S3.Bucket.ARN = p.bucket event.S3.Object.Key = filename acker := newEventACKTracker(ctx) diff --git a/x-pack/filebeat/input/awss3/s3_objects.go b/x-pack/filebeat/input/awss3/s3_objects.go index f486fbac0321..3b5c37ff0f67 100644 --- a/x-pack/filebeat/input/awss3/s3_objects.go +++ b/x-pack/filebeat/input/awss3/s3_objects.go @@ -76,8 +76,8 @@ func (f *s3ObjectProcessorFactory) findReaderConfig(key string) *readerConfig { // match the S3 object key. func (f *s3ObjectProcessorFactory) Create(ctx context.Context, log *logp.Logger, ack *eventACKTracker, obj s3EventV2) s3ObjectHandler { log = log.With( - "s3_bucket", obj.S3.Bucket.Name, - "s3_object", obj.S3.Object.Key) + "bucket_arn", obj.S3.Bucket.Name, + "object_key", obj.S3.Object.Key) readerConfig := f.findReaderConfig(obj.S3.Object.Key) if readerConfig == nil { diff --git a/x-pack/filebeat/input/awss3/s3_test.go b/x-pack/filebeat/input/awss3/s3_test.go index a02f3a58495c..dc87356ba655 100644 --- a/x-pack/filebeat/input/awss3/s3_test.go +++ b/x-pack/filebeat/input/awss3/s3_test.go @@ -133,7 +133,7 @@ func TestS3Poller(t *testing.T) { Return(nil, errFakeConnectivityFailure) s3ObjProc := newS3ObjectProcessorFactory(logp.NewLogger(inputName), nil, mockAPI, mockPublisher, nil) - receiver := newS3Poller(logp.NewLogger(inputName), nil, mockAPI, s3ObjProc, newStates(inputCtx), store, bucket, numberOfWorkers, pollInterval) + receiver := newS3Poller(logp.NewLogger(inputName), nil, mockAPI, s3ObjProc, newStates(inputCtx), store, bucket, "region", numberOfWorkers, pollInterval) require.Error(t, context.DeadlineExceeded, receiver.Poll(ctx)) assert.Equal(t, numberOfWorkers, receiver.workerSem.available) }) @@ -263,7 +263,7 @@ func TestS3Poller(t *testing.T) { Return(nil, errFakeConnectivityFailure) s3ObjProc := newS3ObjectProcessorFactory(logp.NewLogger(inputName), nil, mockAPI, mockPublisher, nil) - receiver := newS3Poller(logp.NewLogger(inputName), nil, mockAPI, s3ObjProc, newStates(inputCtx), store, bucket, numberOfWorkers, pollInterval) + receiver := newS3Poller(logp.NewLogger(inputName), nil, mockAPI, s3ObjProc, newStates(inputCtx), store, bucket, "region", numberOfWorkers, pollInterval) require.Error(t, context.DeadlineExceeded, receiver.Poll(ctx)) assert.Equal(t, numberOfWorkers, receiver.workerSem.available) }) diff --git a/x-pack/filebeat/module/aws/_meta/config.yml b/x-pack/filebeat/module/aws/_meta/config.yml index 881138ed9f45..c0bc21854fac 100644 --- a/x-pack/filebeat/module/aws/_meta/config.yml +++ b/x-pack/filebeat/module/aws/_meta/config.yml @@ -5,6 +5,15 @@ # AWS SQS queue url #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue + # AWS S3 bucket arn + #var.bucket_arn: 'arn:aws:s3:::mybucket' + + # Bucket list interval on S3 bucket + #var.bucket_list_interval: 300s + + # Number of workers on S3 bucket + #var.number_of_workers: 5 + # Process CloudTrail logs # default is true, set to false to skip Cloudtrail logs # var.process_cloudtrail_logs: false @@ -57,6 +66,15 @@ # AWS SQS queue url #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue + # AWS S3 bucket arn + #var.bucket_arn: 'arn:aws:s3:::mybucket' + + # Bucket list interval on S3 bucket + #var.bucket_list_interval: 300s + + # Number of workers on S3 bucket + #var.number_of_workers: 5 + # Filename of AWS credential file # If not set "$HOME/.aws/credentials" is used on Linux/Mac # "%UserProfile%\.aws\credentials" is used on Windows @@ -97,6 +115,15 @@ # AWS SQS queue url #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue + # AWS S3 bucket arn + #var.bucket_arn: 'arn:aws:s3:::mybucket' + + # Bucket list interval on S3 bucket + #var.bucket_list_interval: 300s + + # Number of workers on S3 bucket + #var.number_of_workers: 5 + # Filename of AWS credential file # If not set "$HOME/.aws/credentials" is used on Linux/Mac # "%UserProfile%\.aws\credentials" is used on Windows @@ -137,6 +164,15 @@ # AWS SQS queue url #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue + # AWS S3 bucket arn + #var.bucket_arn: 'arn:aws:s3:::mybucket' + + # Bucket list interval on S3 bucket + #var.bucket_list_interval: 300s + + # Number of workers on S3 bucket + #var.number_of_workers: 5 + # Filename of AWS credential file # If not set "$HOME/.aws/credentials" is used on Linux/Mac # "%UserProfile%\.aws\credentials" is used on Windows @@ -177,6 +213,15 @@ # AWS SQS queue url #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue + # AWS S3 bucket arn + #var.bucket_arn: 'arn:aws:s3:::mybucket' + + # Bucket list interval on S3 bucket + #var.bucket_list_interval: 300s + + # Number of workers on S3 bucket + #var.number_of_workers: 5 + # Filename of AWS credential file # If not set "$HOME/.aws/credentials" is used on Linux/Mac # "%UserProfile%\.aws\credentials" is used on Windows @@ -217,6 +262,15 @@ # AWS SQS queue url #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue + # AWS S3 bucket arn + #var.bucket_arn: 'arn:aws:s3:::mybucket' + + # Bucket list interval on S3 bucket + #var.bucket_list_interval: 300s + + # Number of workers on S3 bucket + #var.number_of_workers: 5 + # Filename of AWS credential file # If not set "$HOME/.aws/credentials" is used on Linux/Mac # "%UserProfile%\.aws\credentials" is used on Windows diff --git a/x-pack/filebeat/module/aws/_meta/docs.asciidoc b/x-pack/filebeat/module/aws/_meta/docs.asciidoc index f286b9251dc3..02abf381ea64 100644 --- a/x-pack/filebeat/module/aws/_meta/docs.asciidoc +++ b/x-pack/filebeat/module/aws/_meta/docs.asciidoc @@ -42,7 +42,7 @@ Example config: cloudtrail: enabled: false #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue - #var.bucket: 'arn:aws:s3:::mybucket' + #var.bucket_arn: 'arn:aws:s3:::mybucket' #var.bucket_list_interval: 300s #var.number_of_workers: 5 #var.shared_credential_file: /etc/filebeat/aws_credentials @@ -59,7 +59,7 @@ Example config: cloudwatch: enabled: false #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue - #var.bucket: 'arn:aws:s3:::mybucket' + #var.bucket_arn: 'arn:aws:s3:::mybucket' #var.bucket_list_interval: 300s #var.number_of_workers: 5 #var.shared_credential_file: /etc/filebeat/aws_credentials @@ -76,7 +76,7 @@ Example config: ec2: enabled: false #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue - #var.bucket: 'arn:aws:s3:::mybucket' + #var.bucket_arn: 'arn:aws:s3:::mybucket' #var.bucket_list_interval: 300s #var.number_of_workers: 5 #var.shared_credential_file: /etc/filebeat/aws_credentials @@ -93,7 +93,7 @@ Example config: elb: enabled: false #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue - #var.bucket: 'arn:aws:s3:::mybucket' + #var.bucket_arn: 'arn:aws:s3:::mybucket' #var.bucket_list_interval: 300s #var.number_of_workers: 5 #var.shared_credential_file: /etc/filebeat/aws_credentials @@ -110,7 +110,7 @@ Example config: s3access: enabled: false #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue - #var.bucket: 'arn:aws:s3:::mybucket' + #var.bucket_arn: 'arn:aws:s3:::mybucket' #var.bucket_list_interval: 300s #var.number_of_workers: 5 #var.shared_credential_file: /etc/filebeat/aws_credentials @@ -127,7 +127,7 @@ Example config: vpcflow: enabled: false #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue - #var.bucket: 'arn:aws:s3:::mybucket' + #var.bucket_arn: 'arn:aws:s3:::mybucket' #var.bucket_list_interval: 300s #var.number_of_workers: 5 #var.shared_credential_file: /etc/filebeat/aws_credentials diff --git a/x-pack/filebeat/module/aws/cloudtrail/config/aws-s3.yml b/x-pack/filebeat/module/aws/cloudtrail/config/aws-s3.yml index 5f4b3da66d21..97cde2469aea 100644 --- a/x-pack/filebeat/module/aws/cloudtrail/config/aws-s3.yml +++ b/x-pack/filebeat/module/aws/cloudtrail/config/aws-s3.yml @@ -2,8 +2,8 @@ type: aws-s3 {{ if .queue_url }} queue_url: {{ .queue_url }} {{ end }} -{{ if .bucket }} -bucket: {{ .bucket }} +{{ if .bucket_arn }} +bucket_arn: {{ .bucket_arn }} {{ end }} {{ if .number_of_workers }} diff --git a/x-pack/filebeat/module/aws/cloudtrail/manifest.yml b/x-pack/filebeat/module/aws/cloudtrail/manifest.yml index 9d40124d846a..6d2c9cdebe02 100644 --- a/x-pack/filebeat/module/aws/cloudtrail/manifest.yml +++ b/x-pack/filebeat/module/aws/cloudtrail/manifest.yml @@ -4,7 +4,7 @@ var: - name: input default: aws-s3 - name: queue_url - - name: bucket + - name: bucket_arn - name: number_of_workers - name: bucket_list_interval - name: shared_credential_file diff --git a/x-pack/filebeat/module/aws/cloudwatch/config/aws-s3.yml b/x-pack/filebeat/module/aws/cloudwatch/config/aws-s3.yml index 52861fdba122..617e9a46bc13 100644 --- a/x-pack/filebeat/module/aws/cloudwatch/config/aws-s3.yml +++ b/x-pack/filebeat/module/aws/cloudwatch/config/aws-s3.yml @@ -2,8 +2,8 @@ type: aws-s3 {{ if .queue_url }} queue_url: {{ .queue_url }} {{ end }} -{{ if .bucket }} -bucket: {{ .bucket }} +{{ if .bucket_arn }} +bucket_arn: {{ .bucket_arn }} {{ end }} {{ if .number_of_workers }} diff --git a/x-pack/filebeat/module/aws/cloudwatch/manifest.yml b/x-pack/filebeat/module/aws/cloudwatch/manifest.yml index 415d3f42718b..7634f73d8d27 100644 --- a/x-pack/filebeat/module/aws/cloudwatch/manifest.yml +++ b/x-pack/filebeat/module/aws/cloudwatch/manifest.yml @@ -4,7 +4,7 @@ var: - name: input default: aws-s3 - name: queue_url - - name: bucket + - name: bucket_arn - name: number_of_workers - name: bucket_list_interval - name: shared_credential_file diff --git a/x-pack/filebeat/module/aws/ec2/config/aws-s3.yml b/x-pack/filebeat/module/aws/ec2/config/aws-s3.yml index 52861fdba122..617e9a46bc13 100644 --- a/x-pack/filebeat/module/aws/ec2/config/aws-s3.yml +++ b/x-pack/filebeat/module/aws/ec2/config/aws-s3.yml @@ -2,8 +2,8 @@ type: aws-s3 {{ if .queue_url }} queue_url: {{ .queue_url }} {{ end }} -{{ if .bucket }} -bucket: {{ .bucket }} +{{ if .bucket_arn }} +bucket_arn: {{ .bucket_arn }} {{ end }} {{ if .number_of_workers }} diff --git a/x-pack/filebeat/module/aws/ec2/manifest.yml b/x-pack/filebeat/module/aws/ec2/manifest.yml index 415d3f42718b..7634f73d8d27 100644 --- a/x-pack/filebeat/module/aws/ec2/manifest.yml +++ b/x-pack/filebeat/module/aws/ec2/manifest.yml @@ -4,7 +4,7 @@ var: - name: input default: aws-s3 - name: queue_url - - name: bucket + - name: bucket_arn - name: number_of_workers - name: bucket_list_interval - name: shared_credential_file diff --git a/x-pack/filebeat/module/aws/elb/config/aws-s3.yml b/x-pack/filebeat/module/aws/elb/config/aws-s3.yml index 52861fdba122..617e9a46bc13 100644 --- a/x-pack/filebeat/module/aws/elb/config/aws-s3.yml +++ b/x-pack/filebeat/module/aws/elb/config/aws-s3.yml @@ -2,8 +2,8 @@ type: aws-s3 {{ if .queue_url }} queue_url: {{ .queue_url }} {{ end }} -{{ if .bucket }} -bucket: {{ .bucket }} +{{ if .bucket_arn }} +bucket_arn: {{ .bucket_arn }} {{ end }} {{ if .number_of_workers }} diff --git a/x-pack/filebeat/module/aws/elb/manifest.yml b/x-pack/filebeat/module/aws/elb/manifest.yml index 128fcbf735e7..128dc59791ed 100644 --- a/x-pack/filebeat/module/aws/elb/manifest.yml +++ b/x-pack/filebeat/module/aws/elb/manifest.yml @@ -4,7 +4,7 @@ var: - name: input default: aws-s3 - name: queue_url - - name: bucket + - name: bucket_arn - name: number_of_workers - name: bucket_list_interval - name: shared_credential_file diff --git a/x-pack/filebeat/module/aws/s3access/config/aws-s3.yml b/x-pack/filebeat/module/aws/s3access/config/aws-s3.yml index 52861fdba122..617e9a46bc13 100644 --- a/x-pack/filebeat/module/aws/s3access/config/aws-s3.yml +++ b/x-pack/filebeat/module/aws/s3access/config/aws-s3.yml @@ -2,8 +2,8 @@ type: aws-s3 {{ if .queue_url }} queue_url: {{ .queue_url }} {{ end }} -{{ if .bucket }} -bucket: {{ .bucket }} +{{ if .bucket_arn }} +bucket_arn: {{ .bucket_arn }} {{ end }} {{ if .number_of_workers }} diff --git a/x-pack/filebeat/module/aws/s3access/manifest.yml b/x-pack/filebeat/module/aws/s3access/manifest.yml index 415d3f42718b..7634f73d8d27 100644 --- a/x-pack/filebeat/module/aws/s3access/manifest.yml +++ b/x-pack/filebeat/module/aws/s3access/manifest.yml @@ -4,7 +4,7 @@ var: - name: input default: aws-s3 - name: queue_url - - name: bucket + - name: bucket_arn - name: number_of_workers - name: bucket_list_interval - name: shared_credential_file diff --git a/x-pack/filebeat/module/aws/vpcflow/config/input.yml b/x-pack/filebeat/module/aws/vpcflow/config/input.yml index 3c25c1a9254e..12c57c500926 100644 --- a/x-pack/filebeat/module/aws/vpcflow/config/input.yml +++ b/x-pack/filebeat/module/aws/vpcflow/config/input.yml @@ -4,8 +4,8 @@ type: aws-s3 {{ if .queue_url }} queue_url: {{ .queue_url }} {{ end }} -{{ if .bucket }} -bucket: {{ .bucket }} +{{ if .bucket_arn }} +bucket_arn: {{ .bucket_arn }} {{ end }} {{ if .number_of_workers }} diff --git a/x-pack/filebeat/module/aws/vpcflow/manifest.yml b/x-pack/filebeat/module/aws/vpcflow/manifest.yml index d3122493b8cd..cd16451fcba7 100644 --- a/x-pack/filebeat/module/aws/vpcflow/manifest.yml +++ b/x-pack/filebeat/module/aws/vpcflow/manifest.yml @@ -4,7 +4,7 @@ var: - name: input default: aws-s3 - name: queue_url - - name: bucket + - name: bucket_arn - name: number_of_workers - name: bucket_list_interval - name: shared_credential_file diff --git a/x-pack/filebeat/modules.d/aws.yml.disabled b/x-pack/filebeat/modules.d/aws.yml.disabled index b79a4212c936..7b8339c79847 100644 --- a/x-pack/filebeat/modules.d/aws.yml.disabled +++ b/x-pack/filebeat/modules.d/aws.yml.disabled @@ -8,6 +8,15 @@ # AWS SQS queue url #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue + # AWS S3 bucket arn + #var.bucket_arn: 'arn:aws:s3:::mybucket' + + # Bucket list interval on S3 bucket + #var.bucket_list_interval: 300s + + # Number of workers on S3 bucket + #var.number_of_workers: 5 + # Process CloudTrail logs # default is true, set to false to skip Cloudtrail logs # var.process_cloudtrail_logs: false @@ -60,6 +69,15 @@ # AWS SQS queue url #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue + # AWS S3 bucket arn + #var.bucket_arn: 'arn:aws:s3:::mybucket' + + # Bucket list interval on S3 bucket + #var.bucket_list_interval: 300s + + # Number of workers on S3 bucket + #var.number_of_workers: 5 + # Filename of AWS credential file # If not set "$HOME/.aws/credentials" is used on Linux/Mac # "%UserProfile%\.aws\credentials" is used on Windows @@ -100,6 +118,15 @@ # AWS SQS queue url #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue + # AWS S3 bucket arn + #var.bucket_arn: 'arn:aws:s3:::mybucket' + + # Bucket list interval on S3 bucket + #var.bucket_list_interval: 300s + + # Number of workers on S3 bucket + #var.number_of_workers: 5 + # Filename of AWS credential file # If not set "$HOME/.aws/credentials" is used on Linux/Mac # "%UserProfile%\.aws\credentials" is used on Windows @@ -140,6 +167,15 @@ # AWS SQS queue url #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue + # AWS S3 bucket arn + #var.bucket_arn: 'arn:aws:s3:::mybucket' + + # Bucket list interval on S3 bucket + #var.bucket_list_interval: 300s + + # Number of workers on S3 bucket + #var.number_of_workers: 5 + # Filename of AWS credential file # If not set "$HOME/.aws/credentials" is used on Linux/Mac # "%UserProfile%\.aws\credentials" is used on Windows @@ -180,6 +216,15 @@ # AWS SQS queue url #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue + # AWS S3 bucket arn + #var.bucket_arn: 'arn:aws:s3:::mybucket' + + # Bucket list interval on S3 bucket + #var.bucket_list_interval: 300s + + # Number of workers on S3 bucket + #var.number_of_workers: 5 + # Filename of AWS credential file # If not set "$HOME/.aws/credentials" is used on Linux/Mac # "%UserProfile%\.aws\credentials" is used on Windows @@ -220,6 +265,15 @@ # AWS SQS queue url #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue + # AWS S3 bucket arn + #var.bucket_arn: 'arn:aws:s3:::mybucket' + + # Bucket list interval on S3 bucket + #var.bucket_list_interval: 300s + + # Number of workers on S3 bucket + #var.number_of_workers: 5 + # Filename of AWS credential file # If not set "$HOME/.aws/credentials" is used on Linux/Mac # "%UserProfile%\.aws\credentials" is used on Windows From 7399f2c798684332af8b04c6b1072acb87c9bfc7 Mon Sep 17 00:00:00 2001 From: Elastic Machine Date: Wed, 18 Aug 2021 23:16:02 +1000 Subject: [PATCH 6/7] chore: add-backport-next (#27449) --- .backportrc.json | 2 +- .mergify.yml | 15 ++++++++++++++- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/.backportrc.json b/.backportrc.json index 95d8ec392376..9e00228ad68f 100644 --- a/.backportrc.json +++ b/.backportrc.json @@ -1,6 +1,6 @@ { "upstream": "elastic/beats", - "branches": [ { "name": "7.x", "checked": true }, "7.14", "7.13", "7.12" ], + "branches": [ { "name": "7.x", "checked": true }, "7.15", "7.14", "7.13", "7.12" ], "labels": ["backport"], "autoAssign": true, "prTitle": "Cherry-pick to {targetBranch}: {commitMessages}" diff --git a/.mergify.yml b/.mergify.yml index b2c82c4d4508..37c08b853ce0 100644 --- a/.mergify.yml +++ b/.mergify.yml @@ -15,7 +15,7 @@ pull_request_rules: - name: backport patches to 7.x branch conditions: - merged - - label=backport-v7.15.0 + - label=backport-v7.16.0 actions: backport: assignees: @@ -151,3 +151,16 @@ pull_request_rules: - files~=^\.mergify\.yml$ actions: delete_head_branch: + - name: backport patches to 7.15 branch + conditions: + - merged + - label=backport-v7.15.0 + actions: + backport: + assignees: + - "{{ author }}" + branches: + - "7.15" + labels: + - "backport" + title: "[{{ destination_branch }}](backport #{{ number }}) {{ title }}" From 9defc4acc11c3d6dd3539ce8289fbf938ade760f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?No=C3=A9mi=20V=C3=A1nyi?= Date: Wed, 18 Aug 2021 17:39:17 +0200 Subject: [PATCH 7/7] Be more lenient when checking for Saved Objects API in Kibana (#27460) ## What does this PR do? This PR sets the minimum required version for Kibana to 7.14 for using the Saved Objects API. The change lets module developers use the last released version for exporting dashboards. The PR also updates the testing environment to 7.14, so dashboard exporting and importing can be tested using e2e tests. ## Why is it important? Support module developers and run e2e tests to catch issues. --- libbeat/kibana/client.go | 4 +++- metricbeat/tests/system/test_base.py | 2 +- testing/environments/latest.yml | 6 +++--- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/libbeat/kibana/client.go b/libbeat/kibana/client.go index 2798443e90c2..2d2b9dc313a9 100644 --- a/libbeat/kibana/client.go +++ b/libbeat/kibana/client.go @@ -39,7 +39,9 @@ import ( ) var ( - MinimumRequiredVersionSavedObjects = common.MustNewVersion("7.15.0") + // We started using Saved Objects API in 7.15. But to help integration + // developers migrate their dashboards we are more lenient. + MinimumRequiredVersionSavedObjects = common.MustNewVersion("7.14.0") ) type Connection struct { diff --git a/metricbeat/tests/system/test_base.py b/metricbeat/tests/system/test_base.py index a0a88f855f3f..a3105492b9c0 100644 --- a/metricbeat/tests/system/test_base.py +++ b/metricbeat/tests/system/test_base.py @@ -121,7 +121,7 @@ def kibana_dir(self): def is_saved_object_api_available(self): kibana_semver = semver.VersionInfo.parse(self.get_version()) - return kibana_semver.major == 7 and kibana_semver.minor < 15 or kibana_semver.major >= 8 + return semver.VersionInfo.parse("7.14.0") <= kibana_semver def get_version(self): url = self.get_kibana_url() + "/api/status" diff --git a/testing/environments/latest.yml b/testing/environments/latest.yml index 7925b8b68383..24ded5163ca8 100644 --- a/testing/environments/latest.yml +++ b/testing/environments/latest.yml @@ -3,7 +3,7 @@ version: '2.3' services: elasticsearch: - image: docker.elastic.co/elasticsearch/elasticsearch:7.12.0 + image: docker.elastic.co/elasticsearch/elasticsearch:7.14.0 healthcheck: test: ["CMD-SHELL", "curl -s http://localhost:9200/_cat/health?h=status | grep -q green"] retries: 300 @@ -20,7 +20,7 @@ services: - "script.context.template.cache_max_size=2000" logstash: - image: docker.elastic.co/logstash/logstash:7.12.0 + image: docker.elastic.co/logstash/logstash:7.14.0 healthcheck: test: ["CMD", "curl", "-f", "http://localhost:9600/_node/stats"] retries: 300 @@ -30,7 +30,7 @@ services: - ./docker/logstash/pki:/etc/pki:ro kibana: - image: docker.elastic.co/kibana/kibana:7.12.0 + image: docker.elastic.co/kibana/kibana:7.14.0 healthcheck: test: ["CMD", "curl", "-f", "http://localhost:5601"] retries: 300