Merge branch 'master' into 26857-elasticsearch-deprecation-logs

.go-version

@@ -1 +1 @@
-1.16.5
+1.16.6

CHANGELOG.asciidoc

@@ -3,6 +3,27 @@
 :issue: https://github.com/elastic/beats/issues/
 :pull: https://github.com/elastic/beats/pull/
 
+[[release-notes-7.13.4]]
+=== Beats version 7.13.4
+https://github.com/elastic/beats/compare/v7.13.3...v7.13.4[View commits]
+
+==== Bugfixes
+
+*Affecting all Beats*
+
+- Fix `add_process_metadata` processor complaining about valid pid fields not being valid integers. {pull}26829[26829] {issue}26830[26830]
+
+*Auditbeat*
+
+- Do not close filestream harvester if an unexpected error is returned when `close.on_state_change.*` is enabled. {pull}26411[26411]
+
+*Filebeat*
+
+- Fix Elasticsearch compatibility for modules that use `type: ip` with `convert` processors. {issue}26629[26629] {pull}26676[26676]
+- Fix Elasticsearch compatibility for modules that use the `network_direction` processor. {issue}26629[26629] {pull}26676[26676]
+- Fix Elasticsearch compatibility for modules that use the `registered_domain` processor. {issue}26629[26629] {pull}26676[26676]
+
+
 [[release-notes-7.13.3]]
 === Beats version 7.13.3
 https://github.com/elastic/beats/compare/v7.13.2...v7.13.3[View commits]

CHANGELOG.next.asciidoc

@@ -107,6 +107,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Change logging in logs input to structure logging. Some log message formats have changed. {pull}25299[25299]
 - Change source field for `event.action` in `fortinet.firewall` module to `fortinet.firewall.action` instead of `fortinet.firewall.eventtype`. {pull}24816[24816]
 - threatintel module: Changed the type of `threatintel.indicator.first_seen` from `keyword` to `date`. {pull}26765[26765]
+- Remove all alias fields pointing to ECS fields from modules. This affects the Suricata and Traefik modules. {issue}10535[10535] {pull}26627[26627]
 
 *Heartbeat*
 - Add support for screenshot blocks and use newer synthetics flags that only works in newer synthetics betas. {pull}25808[25808]
@@ -139,6 +140,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Added `statsd.mappings` configuration for Statsd module {pull}26220[26220]
 - Added Airflow lightweight module {pull}26220[26220]
 - Add state_job metricset to Kubernetes module{pull}26479[26479]
+- Recover service.address field in vsphere module {issue}26902[26902] {pull}26904[26904]
 
 *Packetbeat*
 
@@ -243,7 +245,6 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Omit full index template from errors that occur while loading the template. {pull}25743[25743]
 - In the script processor, the `decode_xml` and `decode_xml_wineventlog` processors are now available as `DecodeXML` and `DecodeXMLWineventlog` respectively.
 - Fix encoding errors when using the disk queue on nested data with multi-byte characters {pull}26484[26484]
-- Fix `add_process_metadata` processor complaining about valid pid fields not being valid integers. {pull}26829[26829] {issue}26830[26830]
 
 *Auditbeat*
 
@@ -282,7 +283,6 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fix CredentialsJSON unpacking for `gcp-pubsub` and `httpjson` inputs. {pull}23277[23277]
 - Fix issue with m365_defender, when parsing incidents that has no alerts attached: {pull}25421[25421]
 - Fix default config template values for paths on oracle module: {pull}26276[26276]
-- Do not close filestream harvester if an unexpected error is returned when close.on_state_change.* is enabled. {pull}26411[26411]
 - Fix Elasticsearch compatibility for modules that use `copy_from` in `set` processors. {issue}26629[26629]
 - Change type of max_bytes in all configs to be cfgtype.ByteSize {pull}26699[26699]
 
@@ -394,11 +394,11 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fix `kibana.log` pipeline when `event.duration` calculation becomes a Long. {issue}24556[24556] {pull}25675[25675]
 - Removed incorrect `http.request.referrer` field from `aws.elb` module. {issue}26435[26435] {pull}26441[26441]
 - Fix `threatintel.indicator.url.full` not being populated. {issue}26351[26351] {pull}26508[26508]
-- Fix Elasticsearch compatibility for modules that use `type: ip` with `convert` processors. {issue}26629[26629] {pull}26676[26676]
-- Fix Elasticsearch compatibility for modules that use the `network_direction` processor. {issue}26629[26629] {pull}26676[26676]
-- Fix Elasticsearch compatibility for modules that use the `registered_domain` processor. {issue}26629[26629] {pull}26676[26676]
 - Fix Suricata metadata fields breaking visualizations, moved out of flattened datatype. {pull}26710[26710]
 - Fix `httpjson` template data key for `url.params`. {pull}26848[26848]
+- Cisco asa/ftd: Fix reversed usage of observer ingress and egress interfaces. {pull}26265[26265]
+- Fix `aws.s3access` pipeline when remote IP is a `-`. {issue}26913[26913] {pull}26940[26940]
+- Fix service name in aws-cloudwatch input from cloudwatchlogs to logs. {pull}27007[27007]
 
 *Heartbeat*
 
@@ -510,6 +510,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fix GCP Project ID being ingested as `cloud.account.id` in `gcp.billing` module {issue}26357[26357] {pull}26412[26412]
 - Fix memory leak in SQL module when database is not available. {issue}25840[25840] {pull}26607[26607]
 - Fix aws metric tags with resourcegroupstaggingapi paginator.  {issue}26385[26385] {pull}26443[26443]
+- Fix quoting in GCP billing table name  {issue}26855[26855] {pull}26870[26870]
 
 *Packetbeat*
 
@@ -527,6 +528,8 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 
 *Functionbeat*
 
+- Expose region in AWS configuration so Functionbeat can deploy the Lamba in the correct place. {pull}26523[26523]
+
 *Elastic Logging Plugin*
 
 
@@ -611,6 +614,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add orchestrator.cluster.name/url fields as k8s metadata {pull}26056[26056]
 - Libbeat: report beat version to monitoring. {pull}26214[26214]
 - Ensure common proxy settings support in HTTP clients: proxy_disabled, proxy_url, proxy_headers and typical environment variables HTTP_PROXY, HTTPS_PROXY, NOPROXY. {pull}25219[25219]
+- `add_process_metadata` processor enrich process information with owner name and id. {issue}21068[21068] {pull}21111[21111]
 
 *Auditbeat*
 
@@ -856,6 +860,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Update `fortinet` ingest pipelines. {issue}22136[22136] {issue}25254[25254] {pull}24816[24816]
 - Use default add_locale for fortinet.firewall {issue}20300[20300] {pull}26524[26524]
 - Add new template functions and `value_type` parameter to `httpjson` transforms. {pull}26847[26847]
+- Add support to merge registry updates in the filestream input across multiple ACKed batches in case of backpressure in the registry or disk. {pull}25976[25976]
 - Update Elasticsearch module's ingest pipeline for parsing new deprecation logs {issue}26857[26857] {pull}26880[26880]
 
 *Heartbeat*
@@ -997,6 +1002,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add total CPU to vSphere virtual machine metrics. {pull}26167[26167]
 - Add AWS Kinesis metricset. {pull}25989[25989]
 - Move openmetrics module to oss. {pull}26561[26561]
+- Add Cluster filter on ECS Kubernetes overview dashboard and corresponding section on Kubernetes module documentation page. {pull}26919[26919]
 
 *Packetbeat*
 

Jenkinsfile

@@ -399,10 +399,13 @@ def publishPackages(beatsFolder){
 * @param beatsFolder the beats folder.
 */
 def uploadPackages(bucketUri, beatsFolder){
-  googleStorageUploadExt(bucket: bucketUri,
-    credentialsId: "${JOB_GCS_EXT_CREDENTIALS}",
-    pattern: "${beatsFolder}/build/distributions/**/*",
-    sharedPublicly: true)
+  // sometimes google storage reports ResumableUploadException: 503 Server Error
+  retryWithSleep(retries: 3, seconds: 5, backoff: true) {
+    googleStorageUploadExt(bucket: bucketUri,
+      credentialsId: "${JOB_GCS_EXT_CREDENTIALS}",
+      pattern: "${beatsFolder}/build/distributions/**/*",
+      sharedPublicly: true)
+  }
 }
 
 /**

auditbeat/Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.16.5
+FROM golang:1.16.6
 
 RUN \
     apt-get update \

auditbeat/docs/fields.asciidoc

@@ -12676,6 +12676,39 @@ alias to: process.executable
 
 --
 
+[float]
+=== owner
+
+Process owner information.
+
+
+*`process.owner.id`*::
++
+--
+Unique identifier of the user.
+
+type: keyword
+
+--
+
+*`process.owner.name`*::
++
+--
+Short name or login of the user.
+
+type: keyword
+
+example: albert
+
+--
+
+*`process.owner.name.text`*::
++
+--
+type: text
+
+--
+
 [[exported-fields-system]]
 == System fields
 

dev-tools/mage/pytest.go

@@ -179,6 +179,9 @@ func PythonTestForModule(params PythonTestArgs) error {
 		params.Files = []string{
 			fmt.Sprintf("module/%s/test_*.py", module),
 			fmt.Sprintf("module/%s/*/test_*.py", module),
+
+			// Run always the base tests, that include tests for module dashboards.
+			"tests/system/test*_base.py",
 		}
 		params.TestName += "-" + module
 	}

docs/devguide/create-metricset.asciidoc

@@ -78,14 +78,12 @@ modify this part of the file if your implementation requires more imports.
 The init method registers the metricset with the central registry. In Go the `init()` function is called
 before the execution of all other code. This means the module will be automatically registered with the global registry.
 
-The `New` method, which is passed to `AddMetricSet`, will be called after the setup of the module and before starting to fetch data. You normally don't need to change this part of the file.
+The `New` method, which is passed to `MustAddMetricSet`, will be called after the setup of the module and before starting to fetch data. You normally don't need to change this part of the file.
 
 [source,go]
 ----
 func init() {
-	if err := mb.Registry.AddMetricSet("{module}", "{metricset}", New); err != nil {
-		panic(err)
-	}
+	mb.Registry.MustAddMetricSet("{module}", "{metricset}", New)
 }
 ----
 

filebeat/Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.16.5
+FROM golang:1.16.6
 
 RUN \
     apt-get update \