Merge branch '7.x' into backport_17862_7.x

CHANGELOG.next.asciidoc

@@ -102,6 +102,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fix `setup.dashboards.index` setting not working. {pull}17749[17749]
 - Fix goroutine leak and Elasticsearch output file descriptor leak when output reloading is in use. {issue}10491[10491] {pull}17381[17381]
 - Fix Elasticsearch license endpoint URL referenced in error message. {issue}17880[17880] {pull}18030[18030]
+- Fix panic when assigning a key to a `nil` value in an event. {pull}18143[18143]
 
 *Auditbeat*
 
@@ -447,6 +448,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add static mapping for metricsets under aws module. {pull}17614[17614] {pull}17650[17650]
 - Collect new `bulk` indexing metrics from Elasticsearch when `xpack.enabled:true` is set. {issue} {pull}17992[17992]
 - Update MSSQL module to fix some SSPI authentication and add brackets to USE statements {pull}17862[17862]]
+- Remove requirement to connect as sysdba in Oracle module {issue}15846[15846] {pull}18182[18182]
 
 *Packetbeat*
 

filebeat/tests/system/test_registrar.py

@@ -869,7 +869,7 @@ def test_clean_removed(self):
         # Make sure the last file in the registry is the correct one and has the correct offset
         assert data[0]["offset"] == self.input_logs.size(file2)
 
-    @unittest.skip('flaky test https://github.com/elastic/beats/issues/10606')
+    @unittest.skipIf(os.name == 'nt', 'flaky test https://github.com/elastic/beats/issues/10606')
     def test_clean_removed_with_clean_inactive(self):
         """
         Checks that files which were removed, the state is removed

generator/_templates/metricbeat/{beat}/magefile.go

@@ -46,6 +46,7 @@ func Package() {
 	defer func() { fmt.Println("package ran for", time.Since(start)) }()
 
 	devtools.UseCommunityBeatPackaging()
+	devtools.PackageKibanaDashboardsFromBuildDir()
 
 	mg.Deps(Update)
 	mg.Deps(build.CrossBuild, build.CrossBuildGoDaemon)

generator/common/Makefile

@@ -16,7 +16,7 @@ test: prepare-test
 	git config user.name "beats-jenkins" || exit 1 ; \
 	$(MAKE) check CHECK_HEADERS_DISABLED=y || exit 1 ; \
 	$(MAKE) || exit 1 ; \
-	$(MAKE) unit
+	mage test
 
 .PHONY: test-package
 test-package: test

libbeat/common/mapstr.go

@@ -96,20 +96,26 @@ func (m MapStr) deepUpdateMap(d MapStr, overwrite bool) {
 }
 
 func deepUpdateValue(old interface{}, val MapStr, overwrite bool) interface{} {
-	if old == nil {
-		return val
-	}
-
 	switch sub := old.(type) {
 	case MapStr:
+		if sub == nil {
+			return val
+		}
+
 		sub.deepUpdateMap(val, overwrite)
 		return sub
 	case map[string]interface{}:
+		if sub == nil {
+			return val
+		}
+
 		tmp := MapStr(sub)
 		tmp.deepUpdateMap(val, overwrite)
 		return tmp
 	default:
-		// This should never happen
+		// We reach the default branch if old is no map or if old == nil.
+		// In either case we return `val`, such that the old value is completely
+		// replaced when merging.
 		return val
 	}
 }

libbeat/common/mapstr_test.go

@@ -87,6 +87,11 @@ func TestMapStrDeepUpdate(t *testing.T) {
 			MapStr{"a.b": 1},
 			MapStr{"a": 1, "a.b": 1},
 		},
+		{
+			MapStr{"a": (MapStr)(nil)},
+			MapStr{"a": MapStr{"b": 1}},
+			MapStr{"a": MapStr{"b": 1}},
+		},
 	}
 
 	for i, test := range tests {

metricbeat/docs/modules/oracle.asciidoc

@@ -57,7 +57,7 @@ metricbeat.modules:
   metricsets: ["tablespace", "performance"]
   enabled: true
   period: 10s
-  hosts: ["oracle://user:pass@localhost:1521/ORCLPDB1.localdomain?sysdba=1"]
+  hosts: ["user:pass@0.0.0.0:1521/ORCLPDB1.localdomain"]
 
   # username: ""
   # password: ""

x-pack/elastic-agent/CHANGELOG.asciidoc

@@ -28,6 +28,7 @@
 - Use /tmp for default monitoring endpoint location for libbeat {pull}18131[18131]
 - Use default output by default {pull}18091[18091]
 - Fix panic and flaky tests for the Agent. {pull}18135[18135]
+- Fix default configuration after enroll {pull}18232[18232]
 
 ==== New features
 

x-pack/elastic-agent/_meta/common.p2.yml

@@ -23,6 +23,14 @@ datasources:
           - metricset: filesystem
             dataset: system.filesystem
 
+settings.monitoring:
+  # enabled turns on monitoring of running processes
+  enabled: true
+  # enables log monitoring
+  logs: true
+  # enables metrics monitoring
+  metrics: true
+
 # management:
 #   # Mode of management, the Elastic Agent support two modes of operation:
 #   #
@@ -112,14 +120,6 @@ datasources:
 #   # Default is false
 #   exponential: false
 
-# settings.monitoring:
-#   # enabled turns on monitoring of running processes
-#   enabled: false
-#   # enables log monitoring
-#   logs: false
-#   # enables metrics monitoring
-#   metrics: false
-
 # Sets log level. The default log level is info.
 # Available log levels are: error, warning, info, debug
 #logging.level: trace

x-pack/elastic-agent/_meta/elastic-agent.fleet.yml

@@ -1,27 +1,27 @@
-#================================ General =====================================
+# ================================ General =====================================
 # Beats is configured under Fleet, you can define most settings
 # from the Kibana UI. You can update this file to configure the settings that
 # are not supported by Fleet.
-# management:
-#   mode: "fleet"
+management:
+  mode: "fleet"
 
-#   # Check in frequency configure the time between calls to fleet to retrieve the new configuration.
-#   #
-#   # Default is 30s
-#   #checkin_frequency: 30s
+  # Check in frequency configure the time between calls to fleet to retrieve the new configuration.
+  #
+  # Default is 30s
+  #checkin_frequency: 30s
 
-#   # Add variance between API calls to better distribute the calls.
-#   #jitter: 5s
+  # Add variance between API calls to better distribute the calls.
+  #jitter: 5s
 
-#   # The Elastic Agent does Exponential backoff when an error happen.
-#   #
-#   #backoff:
-#   #
-#   # Initial time to wait before retrying the call.
-#   # init: 1s
-#   #
-#   # Maximum time to wait before retrying the call.
-#   # max: 10s
+  # The Elastic Agent does Exponential backoff when an error happen.
+  #
+  #backoff:
+  #
+  # Initial time to wait before retrying the call.
+  # init: 1s
+  #
+  # Maximum time to wait before retrying the call.
+  # max: 10s
 
 # download:
 #   # source of the artifacts, requires elastic like structure and naming of the binaries

x-pack/elastic-agent/elastic-agent.yml

@@ -28,6 +28,14 @@ datasources:
           - metricset: filesystem
             dataset: system.filesystem
 
+settings.monitoring:
+  # enabled turns on monitoring of running processes
+  enabled: true
+  # enables log monitoring
+  logs: true
+  # enables metrics monitoring
+  metrics: true
+
 # management:
 #   # Mode of management, the Elastic Agent support two modes of operation:
 #   #
@@ -117,14 +125,6 @@ datasources:
 #   # Default is false
 #   exponential: false
 
-# settings.monitoring:
-#   # enabled turns on monitoring of running processes
-#   enabled: false
-#   # enables log monitoring
-#   logs: false
-#   # enables metrics monitoring
-#   metrics: false
-
 # Sets log level. The default log level is info.
 # Available log levels are: error, warning, info, debug
 #logging.level: trace

x-pack/elastic-agent/pkg/agent/warn/warn.go

@@ -11,7 +11,7 @@ import (
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/core/logger"
 )
 
-const message = "The Elastic Agent is currently in Alpha and should not be used in production"
+const message = "The Elastic Agent is currently in Experimental and should not be used in production"
 
 // LogNotGA warns the users in the log that the Elastic Agent is not GA.
 func LogNotGA(log *logger.Logger) {

x-pack/metricbeat/metricbeat.reference.yml

@@ -993,7 +993,7 @@ metricbeat.modules:
   metricsets: ["tablespace", "performance"]
   enabled: true
   period: 10s
-  hosts: ["oracle://user:pass@localhost:1521/ORCLPDB1.localdomain?sysdba=1"]
+  hosts: ["user:pass@0.0.0.0:1521/ORCLPDB1.localdomain"]
 
   # username: ""
   # password: ""

x-pack/metricbeat/module/aws/aws.go

@@ -103,6 +103,7 @@ func NewMetricSet(base mb.BaseMetricSet) (*MetricSet, error) {
 		// collecting the first one.
 		if output.AccountAliases != nil {
 			metricSet.AccountName = output.AccountAliases[0]
+			base.Logger().Debug("AWS Credentials belong to account name: ", metricSet.AccountName)
 		}
 	}
 
@@ -115,6 +116,7 @@ func NewMetricSet(base mb.BaseMetricSet) (*MetricSet, error) {
 		base.Logger().Warn("failed to get caller identity, please check permission setting: ", err)
 	} else {
 		metricSet.AccountID = *outputIdentity.Account
+		base.Logger().Debug("AWS Credentials belong to account ID: ", metricSet.AccountID)
 	}
 
 	// Construct MetricSet with a full regions list

x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go

@@ -18,6 +18,7 @@ import (
 	"github.com/pkg/errors"
 
 	"github.com/elastic/beats/v7/libbeat/common"
+	"github.com/elastic/beats/v7/libbeat/logp"
 	"github.com/elastic/beats/v7/metricbeat/mb"
 	awscommon "github.com/elastic/beats/v7/x-pack/libbeat/common/aws"
 	"github.com/elastic/beats/v7/x-pack/metricbeat/module/aws"
@@ -51,6 +52,7 @@ func init() {
 // interface methods except for Fetch.
 type MetricSet struct {
 	*aws.MetricSet
+	logger            *logp.Logger
 	CloudwatchConfigs []Config `config:"metrics" validate:"nonzero,required"`
 }
 
@@ -113,6 +115,7 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) {
 
 	return &MetricSet{
 		MetricSet:         metricSet,
+		logger:            logp.NewLogger(metricsetName),
 		CloudwatchConfigs: config.CloudwatchMetrics,
 	}, nil
 }
@@ -132,10 +135,13 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error {
 
 	// Get listMetricDetailTotal and namespaceDetailTotal from configuration
 	listMetricDetailTotal, namespaceDetailTotal := m.readCloudwatchConfig()
+	m.logger.Debugf("listMetricDetailTotal = %s", listMetricDetailTotal)
+	m.logger.Debugf("namespaceDetailTotal = %s", namespaceDetailTotal)
 
 	// Create events based on listMetricDetailTotal from configuration
 	if len(listMetricDetailTotal.metricsWithStats) != 0 {
 		for _, regionName := range m.MetricSet.RegionsList {
+			m.logger.Debugf("Collecting metrics from AWS region %s", regionName)
 			awsConfig := m.MetricSet.AwsConfig.Copy()
 			awsConfig.Region = regionName
 
@@ -150,6 +156,8 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error {
 				return errors.Wrap(err, "createEvents failed for region "+regionName)
 			}
 
+			m.logger.Debugf("Collected metrics of metrics = %d", len(eventsWithIdentifier))
+
 			err = reportEvents(eventsWithIdentifier, report)
 			if err != nil {
 				return errors.Wrap(err, "reportEvents failed")
@@ -158,6 +166,7 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error {
 	}
 
 	for _, regionName := range m.MetricSet.RegionsList {
+		m.logger.Debugf("Collecting metrics from AWS region %s", regionName)
 		awsConfig := m.MetricSet.AwsConfig.Copy()
 		awsConfig.Region = regionName
 
@@ -169,9 +178,11 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error {
 
 		// Create events based on namespaceDetailTotal from configuration
 		for namespace, namespaceDetails := range namespaceDetailTotal {
+			m.logger.Debugf("Collected metrics from namespace %s", namespace)
+
 			listMetricsOutput, err := aws.GetListMetricsOutput(namespace, regionName, svcCloudwatch)
 			if err != nil {
-				m.Logger().Info(err.Error())
+				m.logger.Info(err.Error())
 				continue
 			}
 
@@ -189,6 +200,8 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error {
 				return errors.Wrap(err, "createEvents failed for region "+regionName)
 			}
 
+			m.logger.Debugf("Collected number of metrics = %d", len(eventsWithIdentifier))
+
 			err = reportEvents(eventsWithIdentifier, report)
 			if err != nil {
 				return errors.Wrap(err, "reportEvents failed")
@@ -434,12 +447,14 @@ func (m *MetricSet) createEvents(svcCloudwatch cloudwatchiface.ClientAPI, svcRes
 
 	// Construct metricDataQueries
 	metricDataQueries := createMetricDataQueries(listMetricWithStatsTotal, m.Period)
+	m.logger.Debugf("Number of MetricDataQueries = %d", len(metricDataQueries))
 	if len(metricDataQueries) == 0 {
 		return events, nil
 	}
 
 	// Use metricDataQueries to make GetMetricData API calls
 	metricDataResults, err := aws.GetMetricDataResults(metricDataQueries, svcCloudwatch, startTime, endTime)
+	m.logger.Debugf("Number of metricDataResults = %d", len(metricDataResults))
 	if err != nil {
 		return events, errors.Wrap(err, "GetMetricDataResults failed")
 	}
@@ -482,7 +497,7 @@ func (m *MetricSet) createEvents(svcCloudwatch cloudwatchiface.ClientAPI, svcRes
 		resourceTagMap, err := aws.GetResourcesTags(svcResourceAPI, []string{resourceType})
 		if err != nil {
 			// If GetResourcesTags failed, continue report event just without tags.
-			m.Logger().Info(errors.Wrap(err, "getResourcesTags failed, skipping region "+regionName))
+			m.logger.Info(errors.Wrap(err, "getResourcesTags failed, skipping region "+regionName))
 		}
 
 		if len(tagsFilter) != 0 && len(resourceTagMap) == 0 {

x-pack/metricbeat/module/aws/cloudwatch/cloudwatch_test.go

@@ -11,6 +11,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/elastic/beats/v7/libbeat/logp"
 	"github.com/elastic/beats/v7/metricbeat/mb"
 
 	awssdk "github.com/aws/aws-sdk-go-v2/aws"
@@ -1232,6 +1233,7 @@ func TestCreateEventsWithIdentifier(t *testing.T) {
 	m := MetricSet{}
 	m.CloudwatchConfigs = []Config{{Statistic: []string{"Average"}}}
 	m.MetricSet = &aws.MetricSet{Period: 5}
+	m.logger = logp.NewLogger("test")
 
 	mockTaggingSvc := &MockResourceGroupsTaggingClient{}
 	mockCloudwatchSvc := &MockCloudWatchClient{}
@@ -1272,6 +1274,7 @@ func TestCreateEventsWithoutIdentifier(t *testing.T) {
 	m := MetricSet{}
 	m.CloudwatchConfigs = []Config{{Statistic: []string{"Average"}}}
 	m.MetricSet = &aws.MetricSet{Period: 5, AccountID: accountID}
+	m.logger = logp.NewLogger("test")
 
 	mockTaggingSvc := &MockResourceGroupsTaggingClient{}
 	mockCloudwatchSvc := &MockCloudWatchClientWithoutDim{}

x-pack/metricbeat/module/oracle/_meta/config.yml

@@ -2,7 +2,7 @@
   metricsets: ["tablespace", "performance"]
   enabled: true
   period: 10s
-  hosts: ["oracle://user:pass@localhost:1521/ORCLPDB1.localdomain?sysdba=1"]
+  hosts: ["user:pass@0.0.0.0:1521/ORCLPDB1.localdomain"]
 
   # username: ""
   # password: ""

x-pack/metricbeat/module/oracle/connection.go

@@ -50,10 +50,6 @@ func NewConnection(c *ConnectionDetails) (*sql.DB, error) {
 		params.Password = c.Password
 	}
 
-	if params.IsSysDBA == false {
-		return nil, errors.New("a user with DBA permissions are required, check your connection details on field `hosts`")
-	}
-
 	db, err := sql.Open("godror", params.StringWithPassword())
 	if err != nil {
 		return nil, errors.Wrap(err, "could not open database")