Merge remote-tracking branch 'upstream/master' into docker-memory-sta…

…ts-fix

.backportrc.json

@@ -1,6 +1,6 @@
 {
   "upstream": "elastic/beats",
-  "branches": [ { "name": "7.x", "checked": true }, "7.14", "7.13", "7.12" ],
+  "branches": [ { "name": "7.x", "checked": true }, "7.15", "7.14", "7.13", "7.12" ],
   "labels": ["backport"],
   "autoAssign": true,
   "prTitle": "Cherry-pick to {targetBranch}: {commitMessages}"

.mergify.yml

@@ -15,7 +15,7 @@ pull_request_rules:
   - name: backport patches to 7.x branch
     conditions:
       - merged
-      - label=backport-v7.15.0
+      - label=backport-v7.16.0
     actions:
       backport:
         assignees:
@@ -151,3 +151,16 @@ pull_request_rules:
         - files~=^\.mergify\.yml$
     actions:
       delete_head_branch:
+  - name: backport patches to 7.15 branch
+    conditions:
+      - merged
+      - label=backport-v7.15.0
+    actions:
+      backport:
+        assignees:
+          - "{{ author }}"
+        branches:
+          - "7.15"
+        labels:
+          - "backport"
+        title: "[{{ destination_branch }}](backport #{{ number }}) {{ title }}"

CHANGELOG.next.asciidoc

@@ -758,6 +758,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Update Elasticsearch module's ingest pipeline for parsing new deprecation logs {issue}26857[26857] {pull}26880[26880]
 - Add new `hmac` template function for httpjson input {pull}27168[27168]
 - Update `tags` and `threatintel.indicator.provider` fields in `threatintel.anomali` ingest pipeline {issue}24746[24746] {pull}27141[27141]
+- Move AWS module and filesets to GA. {pull}27428[27428]
 - update ecs.version to ECS 1.11.0. {pull}27107[27107]
 
 *Heartbeat*

filebeat/docs/modules/aws.asciidoc

@@ -12,8 +12,6 @@ This file is generated! See scripts/docs_collector.py
 
 == AWS module
 
-beta[]
-
 This is a module for aws logs. It uses filebeat s3 input to get log files from
 AWS S3 buckets with SQS notification or directly polling list of S3 objects in an S3 bucket.
 The use of SQS notification is preferred: polling list of S3 objects is expensive
@@ -49,7 +47,7 @@ Example config:
   cloudtrail:
     enabled: false
     #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue
-    #var.bucket: 'arn:aws:s3:::mybucket'
+    #var.bucket_arn: 'arn:aws:s3:::mybucket'
     #var.bucket_list_interval: 300s
     #var.number_of_workers: 5
     #var.shared_credential_file: /etc/filebeat/aws_credentials
@@ -66,7 +64,7 @@ Example config:
   cloudwatch:
     enabled: false
     #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue
-    #var.bucket: 'arn:aws:s3:::mybucket'
+    #var.bucket_arn: 'arn:aws:s3:::mybucket'
     #var.bucket_list_interval: 300s
     #var.number_of_workers: 5
     #var.shared_credential_file: /etc/filebeat/aws_credentials
@@ -83,7 +81,7 @@ Example config:
   ec2:
     enabled: false
     #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue
-    #var.bucket: 'arn:aws:s3:::mybucket'
+    #var.bucket_arn: 'arn:aws:s3:::mybucket'
     #var.bucket_list_interval: 300s
     #var.number_of_workers: 5
     #var.shared_credential_file: /etc/filebeat/aws_credentials
@@ -100,7 +98,7 @@ Example config:
   elb:
     enabled: false
     #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue
-    #var.bucket: 'arn:aws:s3:::mybucket'
+    #var.bucket_arn: 'arn:aws:s3:::mybucket'
     #var.bucket_list_interval: 300s
     #var.number_of_workers: 5
     #var.shared_credential_file: /etc/filebeat/aws_credentials
@@ -117,7 +115,7 @@ Example config:
   s3access:
     enabled: false
     #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue
-    #var.bucket: 'arn:aws:s3:::mybucket'
+    #var.bucket_arn: 'arn:aws:s3:::mybucket'
     #var.bucket_list_interval: 300s
     #var.number_of_workers: 5
     #var.shared_credential_file: /etc/filebeat/aws_credentials
@@ -134,7 +132,7 @@ Example config:
   vpcflow:
     enabled: false
     #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue
-    #var.bucket: 'arn:aws:s3:::mybucket'
+    #var.bucket_arn: 'arn:aws:s3:::mybucket'
     #var.bucket_list_interval: 300s
     #var.number_of_workers: 5
     #var.shared_credential_file: /etc/filebeat/aws_credentials

libbeat/docs/shared-kibana-config.asciidoc

@@ -10,14 +10,15 @@
 //////////////////////////////////////////////////////////////////////////
 
 [[setup-kibana-endpoint]]
-== Configure the Kibana endpoint
+== Configure the {kib} endpoint
 
 ++++
-<titleabbrev>Kibana endpoint</titleabbrev>
+<titleabbrev>{kib} endpoint</titleabbrev>
 ++++
 
-Starting with {beatname_uc} 6.0.0, the Kibana dashboards are loaded into Kibana
-via the Kibana API. This requires a Kibana endpoint configuration.
+{kib} dashboards are loaded into {kib}
+via the {kib} API. This requires a {kib} endpoint configuration. For details on
+authenticating to the {kib} API, see {kibana-ref}/api.html#api-authentication[Authentication].
 
 You configure the endpoint in the `setup.kibana` section of the
 +{beatname_lc}.yml+ config file.
@@ -38,7 +39,7 @@ You can specify the following options in the `setup.kibana` section of the
 [float]
 ==== `setup.kibana.host`
 
-The Kibana host where the dashboards will be loaded. The default is
+The {kib} host where the dashboards will be loaded. The default is
 `127.0.0.1:5601`. The value of `host` can be a `URL` or `IP:PORT`. For example: `http://192.15.3.2`, `192:15.3.2:5601` or `http://192.15.3.2:6701/path`. If no
 port is specified, `5601` is used.
 
@@ -53,7 +54,7 @@ IPv6 addresses must be defined using the following format:
 [[kibana-protocol-option]]
 ==== `setup.kibana.protocol`
 
-The name of the protocol Kibana is reachable on. The options are: `http` or
+The name of the protocol {kib} is reachable on. The options are: `http` or
 `https`. The default is `http`. However, if you specify a URL for host, the
 value of `protocol` is overridden by whatever scheme you specify in the URL.
 
@@ -70,37 +71,37 @@ setup.kibana.path: /kibana
 [float]
 ==== `setup.kibana.username`
 
-The basic authentication username for connecting to Kibana. If you don't
+The basic authentication username for connecting to {kib}. If you don't
 specify a value for this setting, {beatname_uc} uses the `username` specified
-for the Elasticsearch output.
+for the {es} output.
 
 [float]
 ==== `setup.kibana.password`
 
-The basic authentication password for connecting to Kibana. If you don't
+The basic authentication password for connecting to {kib}. If you don't
 specify a value for this setting, {beatname_uc} uses the `password` specified
-for the Elasticsearch output.
+for the {es} output.
 
 [float]
 [[kibana-path-option]]
 ==== `setup.kibana.path`
 
 An HTTP path prefix that is prepended to the HTTP API calls. This is useful for
-the cases where Kibana listens behind an HTTP reverse proxy that exports the API
+the cases where {kib} listens behind an HTTP reverse proxy that exports the API
 under a custom prefix.
 
 [float]
 [[kibana-space-id-option]]
 ==== `setup.kibana.space.id`
 
 The {kibana-ref}/xpack-spaces.html[Kibana space] ID to use. If specified,
-{beatname_uc} loads Kibana assets into this Kibana space. Omit this option to
+{beatname_uc} loads {kib} assets into this {kib} space. Omit this option to
 use the default space.
 
 [float]
 ===== `setup.kibana.headers`
 
-Custom HTTP headers to add to each request sent to Kibana.
+Custom HTTP headers to add to each request sent to {kib}.
 Example:
 
 [source,yaml]
@@ -112,7 +113,7 @@ setup.kibana.headers:
 [float]
 ==== `setup.kibana.ssl.enabled`
 
-Enables {beatname_uc} to use SSL settings when connecting to Kibana via HTTPS.
+Enables {beatname_uc} to use SSL settings when connecting to {kib} via HTTPS.
 If you configure {beatname_uc} to connect over HTTPS, this setting defaults to
 `true` and {beatname_uc} uses the default SSL settings.
 
@@ -122,9 +123,9 @@ Example configuration:
 ----
 setup.kibana.host: "https://192.0.2.255:5601"
 setup.kibana.ssl.enabled: true
-setup.kibana.ssl.certificate_authorities: ["/etc/pki/root/ca.pem"]
-setup.kibana.ssl.certificate: "/etc/pki/client/cert.pem"
-setup.kibana.ssl.key: "/etc/pki/client/cert.key
+setup.kibana.ssl.certificate_authorities: ["/etc/client/ca.pem"]
+setup.kibana.ssl.certificate: "/etc/client/cert.pem"
+setup.kibana.ssl.key: "/etc/client/cert.key
 ----
 
 See <<configuration-ssl>> for more information.

libbeat/docs/shared-ssl-config.asciidoc

@@ -32,9 +32,9 @@ Example output config with SSL enabled:
 [source,yaml]
 ----
 output.elasticsearch.hosts: ["https://192.168.1.42:9200"]
-output.elasticsearch.ssl.certificate_authorities: ["/etc/pki/root/ca.pem"]
-output.elasticsearch.ssl.certificate: "/etc/pki/client/cert.pem"
-output.elasticsearch.ssl.key: "/etc/pki/client/cert.key"
+output.elasticsearch.ssl.certificate_authorities: ["/etc/client/ca.pem"]
+output.elasticsearch.ssl.certificate: "/etc/client/cert.pem"
+output.elasticsearch.ssl.key: "/etc/client/cert.key"
 ----
 
 ifndef::no-output-logstash[]
@@ -48,9 +48,9 @@ Example Kibana endpoint config with SSL enabled:
 ----
 setup.kibana.host: "https://192.0.2.255:5601"
 setup.kibana.ssl.enabled: true
-setup.kibana.ssl.certificate_authorities: ["/etc/pki/root/ca.pem"]
-setup.kibana.ssl.certificate: "/etc/pki/client/cert.pem"
-setup.kibana.ssl.key: "/etc/pki/client/cert.key"
+setup.kibana.ssl.certificate_authorities: ["/etc/client/ca.pem"]
+setup.kibana.ssl.certificate: "/etc/client/cert.pem"
+setup.kibana.ssl.key: "/etc/client/cert.key"
 ----
 endif::no_kibana[]
 
@@ -264,7 +264,7 @@ certificate_authorities:
 
 [float]
 [[client-certificate]]
-==== `certificate: "/etc/pki/client/cert.pem"`
+==== `certificate: "/etc/client/cert.pem"`
 
 The path to the certificate for SSL client authentication is only required if
 `client_authentication` is specified. If the certificate
@@ -302,7 +302,7 @@ certificate: |
 
 [float]
 [[client-key]]
-==== `key: "/etc/pki/client/cert.key"`
+==== `key: "/etc/client/cert.key"`
 
 The client certificate key used for client authentication and is only required
 if `client_authentication` is configured. The key option support embedding of the private key:
@@ -423,7 +423,7 @@ certificate_authorities:
 
 [float]
 [[server-certificate]]
-==== `certificate: "/etc/pki/server/cert.pem"`
+==== `certificate: "/etc/server/cert.pem"`
 
 For server authentication, the path to the SSL authentication certificate must
 be specified for TLS. If the certificate is not specified, startup will fail.
@@ -457,7 +457,7 @@ certificate: |
 
 [float]
 [[server-key]]
-==== `key: "/etc/pki/server/cert.key"`
+==== `key: "/etc/server/cert.key"`
 
 The server certificate key used for authentication is required.
 The key option support embedding of the private key:

libbeat/kibana/client.go

@@ -24,6 +24,7 @@ import (
 	"fmt"
 	"io"
 	"io/ioutil"
+	"mime"
 	"mime/multipart"
 	"net/http"
 	"net/textproto"
@@ -38,7 +39,9 @@ import (
 )
 
 var (
-	MinimumRequiredVersionSavedObjects = common.MustNewVersion("7.15.0")
+	// We started using Saved Objects API in 7.15. But to help integration
+	// developers migrate their dashboards we are more lenient.
+	MinimumRequiredVersionSavedObjects = common.MustNewVersion("7.14.0")
 )
 
 type Connection struct {
@@ -214,6 +217,7 @@ func (conn *Connection) SendWithContext(ctx context.Context, method, extraPath s
 	addHeaders(req.Header, headers)
 
 	contentType := req.Header.Get("Content-Type")
+	contentType, _, _ = mime.ParseMediaType(contentType)
 	if contentType != "multipart/form-data" && contentType != "application/ndjson" {
 		req.Header.Set("Content-Type", "application/json")
 	}

libbeat/kibana/client_test.go

@@ -119,3 +119,30 @@ headers:
 	assert.Equal(t, []string{"1"}, requests[1].Header.Values("kbn-xsrf"))
 
 }
+
+func TestNewKibanaClientWithMultipartData(t *testing.T) {
+	var requests []*http.Request
+	kibanaTs := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		requests = append(requests, r)
+		if r.URL.Path == "/api/status" {
+			w.Write([]byte(`{"version":{"number":"1.2.3-beta","build_snapshot":true}}`))
+		}
+	}))
+	defer kibanaTs.Close()
+
+	client, err := NewKibanaClient(common.MustNewConfigFrom(fmt.Sprintf(`
+protocol: http
+host: %s
+headers:
+  content-type: multipart/form-data; boundary=46bea21be603a2c2ea6f51571a5e1baf5ea3be8ebd7101199320607b36ff
+  accept: text/plain
+  kbn-xsrf: 0
+`, kibanaTs.Listener.Addr().String())))
+	require.NoError(t, err)
+	require.NotNil(t, client)
+
+	client.Request(http.MethodPost, "/foo", url.Values{}, http.Header{"key": []string{"another_value"}}, nil)
+
+	assert.Equal(t, []string{"multipart/form-data; boundary=46bea21be603a2c2ea6f51571a5e1baf5ea3be8ebd7101199320607b36ff"}, requests[1].Header.Values("Content-Type"))
+
+}

metricbeat/module/prometheus/query/_meta/docs.asciidoc

@@ -1,4 +1,4 @@
-This is the `query` metricset to query from https://prometheus.io/docs/prometheus/latest/querying/api/#expression-queries[querying API of Promtheus].
+This is the `query` metricset to query from https://prometheus.io/docs/prometheus/latest/querying/api/#expression-queries[querying API of Prometheus].
 
 
 [float]

metricbeat/tests/system/test_base.py

@@ -121,7 +121,7 @@ def kibana_dir(self):
 
     def is_saved_object_api_available(self):
         kibana_semver = semver.VersionInfo.parse(self.get_version())
-        return kibana_semver.major == 7 and kibana_semver.minor < 15 or kibana_semver.major >= 8
+        return semver.VersionInfo.parse("7.14.0") <= kibana_semver
 
     def get_version(self):
         url = self.get_kibana_url() + "/api/status"

testing/environments/latest.yml

@@ -3,7 +3,7 @@
 version: '2.3'
 services:
   elasticsearch:
-    image: docker.elastic.co/elasticsearch/elasticsearch:7.12.0
+    image: docker.elastic.co/elasticsearch/elasticsearch:7.14.0
     healthcheck:
       test: ["CMD-SHELL", "curl -s http://localhost:9200/_cat/health?h=status | grep -q green"]
       retries: 300
@@ -20,7 +20,7 @@ services:
       - "script.context.template.cache_max_size=2000"
 
   logstash:
-    image: docker.elastic.co/logstash/logstash:7.12.0
+    image: docker.elastic.co/logstash/logstash:7.14.0
     healthcheck:
       test: ["CMD", "curl", "-f", "http://localhost:9600/_node/stats"]
       retries: 300
@@ -30,7 +30,7 @@ services:
       - ./docker/logstash/pki:/etc/pki:ro
 
   kibana:
-    image: docker.elastic.co/kibana/kibana:7.12.0
+    image: docker.elastic.co/kibana/kibana:7.14.0
     healthcheck:
       test: ["CMD", "curl", "-f", "http://localhost:5601"]
       retries: 300