diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index d4133c889dd5..fefbc6abecb8 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -102,6 +102,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Fix `setup.dashboards.index` setting not working. {pull}17749[17749] - Fix goroutine leak and Elasticsearch output file descriptor leak when output reloading is in use. {issue}10491[10491] {pull}17381[17381] - Fix Elasticsearch license endpoint URL referenced in error message. {issue}17880[17880] {pull}18030[18030] +- Fix panic when assigning a key to a `nil` value in an event. {pull}18143[18143] *Auditbeat* @@ -447,6 +448,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Add static mapping for metricsets under aws module. {pull}17614[17614] {pull}17650[17650] - Collect new `bulk` indexing metrics from Elasticsearch when `xpack.enabled:true` is set. {issue} {pull}17992[17992] - Update MSSQL module to fix some SSPI authentication and add brackets to USE statements {pull}17862[17862]] +- Remove requirement to connect as sysdba in Oracle module {issue}15846[15846] {pull}18182[18182] *Packetbeat* diff --git a/filebeat/tests/system/test_registrar.py b/filebeat/tests/system/test_registrar.py index 59999302ff3c..ed2e04d2c07c 100644 --- a/filebeat/tests/system/test_registrar.py +++ b/filebeat/tests/system/test_registrar.py @@ -869,7 +869,7 @@ def test_clean_removed(self): # Make sure the last file in the registry is the correct one and has the correct offset assert data[0]["offset"] == self.input_logs.size(file2) - @unittest.skip('flaky test https://github.com/elastic/beats/issues/10606') + @unittest.skipIf(os.name == 'nt', 'flaky test https://github.com/elastic/beats/issues/10606') def test_clean_removed_with_clean_inactive(self): """ Checks that files which were removed, the state is removed diff --git a/generator/_templates/metricbeat/{beat}/magefile.go b/generator/_templates/metricbeat/{beat}/magefile.go index 934276e633b8..76eedb6496a9 100644 --- a/generator/_templates/metricbeat/{beat}/magefile.go +++ b/generator/_templates/metricbeat/{beat}/magefile.go @@ -46,6 +46,7 @@ func Package() { defer func() { fmt.Println("package ran for", time.Since(start)) }() devtools.UseCommunityBeatPackaging() + devtools.PackageKibanaDashboardsFromBuildDir() mg.Deps(Update) mg.Deps(build.CrossBuild, build.CrossBuildGoDaemon) diff --git a/generator/common/Makefile b/generator/common/Makefile index 927da092a224..8d82a444e84c 100644 --- a/generator/common/Makefile +++ b/generator/common/Makefile @@ -16,7 +16,7 @@ test: prepare-test git config user.name "beats-jenkins" || exit 1 ; \ $(MAKE) check CHECK_HEADERS_DISABLED=y || exit 1 ; \ $(MAKE) || exit 1 ; \ - $(MAKE) unit + mage test .PHONY: test-package test-package: test diff --git a/libbeat/common/mapstr.go b/libbeat/common/mapstr.go index fbae24a39b49..c4474ecd2834 100644 --- a/libbeat/common/mapstr.go +++ b/libbeat/common/mapstr.go @@ -96,20 +96,26 @@ func (m MapStr) deepUpdateMap(d MapStr, overwrite bool) { } func deepUpdateValue(old interface{}, val MapStr, overwrite bool) interface{} { - if old == nil { - return val - } - switch sub := old.(type) { case MapStr: + if sub == nil { + return val + } + sub.deepUpdateMap(val, overwrite) return sub case map[string]interface{}: + if sub == nil { + return val + } + tmp := MapStr(sub) tmp.deepUpdateMap(val, overwrite) return tmp default: - // This should never happen + // We reach the default branch if old is no map or if old == nil. + // In either case we return `val`, such that the old value is completely + // replaced when merging. return val } } diff --git a/libbeat/common/mapstr_test.go b/libbeat/common/mapstr_test.go index de633ff162ea..e1ffb8f5ee42 100644 --- a/libbeat/common/mapstr_test.go +++ b/libbeat/common/mapstr_test.go @@ -87,6 +87,11 @@ func TestMapStrDeepUpdate(t *testing.T) { MapStr{"a.b": 1}, MapStr{"a": 1, "a.b": 1}, }, + { + MapStr{"a": (MapStr)(nil)}, + MapStr{"a": MapStr{"b": 1}}, + MapStr{"a": MapStr{"b": 1}}, + }, } for i, test := range tests { diff --git a/metricbeat/docs/modules/oracle.asciidoc b/metricbeat/docs/modules/oracle.asciidoc index d56abf8648ba..2492adcaaaf0 100644 --- a/metricbeat/docs/modules/oracle.asciidoc +++ b/metricbeat/docs/modules/oracle.asciidoc @@ -57,7 +57,7 @@ metricbeat.modules: metricsets: ["tablespace", "performance"] enabled: true period: 10s - hosts: ["oracle://user:pass@localhost:1521/ORCLPDB1.localdomain?sysdba=1"] + hosts: ["user:pass@0.0.0.0:1521/ORCLPDB1.localdomain"] # username: "" # password: "" diff --git a/x-pack/elastic-agent/CHANGELOG.asciidoc b/x-pack/elastic-agent/CHANGELOG.asciidoc index 0fa3020453f1..9ed0e43d43f4 100644 --- a/x-pack/elastic-agent/CHANGELOG.asciidoc +++ b/x-pack/elastic-agent/CHANGELOG.asciidoc @@ -28,6 +28,7 @@ - Use /tmp for default monitoring endpoint location for libbeat {pull}18131[18131] - Use default output by default {pull}18091[18091] - Fix panic and flaky tests for the Agent. {pull}18135[18135] +- Fix default configuration after enroll {pull}18232[18232] ==== New features diff --git a/x-pack/elastic-agent/_meta/common.p2.yml b/x-pack/elastic-agent/_meta/common.p2.yml index 15582908fe7b..0e7c950cdd5c 100644 --- a/x-pack/elastic-agent/_meta/common.p2.yml +++ b/x-pack/elastic-agent/_meta/common.p2.yml @@ -23,6 +23,14 @@ datasources: - metricset: filesystem dataset: system.filesystem +settings.monitoring: + # enabled turns on monitoring of running processes + enabled: true + # enables log monitoring + logs: true + # enables metrics monitoring + metrics: true + # management: # # Mode of management, the Elastic Agent support two modes of operation: # # @@ -112,14 +120,6 @@ datasources: # # Default is false # exponential: false -# settings.monitoring: -# # enabled turns on monitoring of running processes -# enabled: false -# # enables log monitoring -# logs: false -# # enables metrics monitoring -# metrics: false - # Sets log level. The default log level is info. # Available log levels are: error, warning, info, debug #logging.level: trace diff --git a/x-pack/elastic-agent/_meta/elastic-agent.fleet.yml b/x-pack/elastic-agent/_meta/elastic-agent.fleet.yml index b8b3eb2ac993..30c0a68431b3 100644 --- a/x-pack/elastic-agent/_meta/elastic-agent.fleet.yml +++ b/x-pack/elastic-agent/_meta/elastic-agent.fleet.yml @@ -1,27 +1,27 @@ -#================================ General ===================================== +# ================================ General ===================================== # Beats is configured under Fleet, you can define most settings # from the Kibana UI. You can update this file to configure the settings that # are not supported by Fleet. -# management: -# mode: "fleet" +management: + mode: "fleet" -# # Check in frequency configure the time between calls to fleet to retrieve the new configuration. -# # -# # Default is 30s -# #checkin_frequency: 30s + # Check in frequency configure the time between calls to fleet to retrieve the new configuration. + # + # Default is 30s + #checkin_frequency: 30s -# # Add variance between API calls to better distribute the calls. -# #jitter: 5s + # Add variance between API calls to better distribute the calls. + #jitter: 5s -# # The Elastic Agent does Exponential backoff when an error happen. -# # -# #backoff: -# # -# # Initial time to wait before retrying the call. -# # init: 1s -# # -# # Maximum time to wait before retrying the call. -# # max: 10s + # The Elastic Agent does Exponential backoff when an error happen. + # + #backoff: + # + # Initial time to wait before retrying the call. + # init: 1s + # + # Maximum time to wait before retrying the call. + # max: 10s # download: # # source of the artifacts, requires elastic like structure and naming of the binaries diff --git a/x-pack/elastic-agent/elastic-agent.yml b/x-pack/elastic-agent/elastic-agent.yml index 547053af6b26..0e447c849d9b 100644 --- a/x-pack/elastic-agent/elastic-agent.yml +++ b/x-pack/elastic-agent/elastic-agent.yml @@ -28,6 +28,14 @@ datasources: - metricset: filesystem dataset: system.filesystem +settings.monitoring: + # enabled turns on monitoring of running processes + enabled: true + # enables log monitoring + logs: true + # enables metrics monitoring + metrics: true + # management: # # Mode of management, the Elastic Agent support two modes of operation: # # @@ -117,14 +125,6 @@ datasources: # # Default is false # exponential: false -# settings.monitoring: -# # enabled turns on monitoring of running processes -# enabled: false -# # enables log monitoring -# logs: false -# # enables metrics monitoring -# metrics: false - # Sets log level. The default log level is info. # Available log levels are: error, warning, info, debug #logging.level: trace diff --git a/x-pack/elastic-agent/pkg/agent/application/configuration_embed.go b/x-pack/elastic-agent/pkg/agent/application/configuration_embed.go index 7ccc52a40a88..b6bc827cc828 100644 --- a/x-pack/elastic-agent/pkg/agent/application/configuration_embed.go +++ b/x-pack/elastic-agent/pkg/agent/application/configuration_embed.go @@ -15,7 +15,7 @@ var DefaultAgentFleetConfig []byte func init() { // Packed File // _meta/elastic-agent.fleet.yml - unpacked := packer.MustUnpack("eJycVlGPo7oVfu/P2PfeggmrS6U+xJnBmDthGnKDsV8qbGeBxE5QEyCm6n+vDAnJblfVVR9GGgX7nPN95zvf8b++/EPvr8Vf9qq4XGvx56Lcn66/fFP7/fUXo9WXv37BgxP8/fc/9offpeI6M5T05R+9M/9tzgf8HiuapwNGasAo6bhmDfMyw8imlMQ/shyXqb4plm8uOFKd3MIDJYuSoUwXxFcYuZ3wNqUAmSORapm3/g0bqIUOrjhKz2wLPwoSV1y/l9nu2OLwGUPm8EjzVNlzRb4umVYXtoUON/DAga8LIl2hd6VElcJRomSUNlzLwZ6n+cbmqex3rgMHR4krItiJU6rYCtb7LUQcZUquFvbclRK/YiC7MuI7n+X5gFfLkpPgyH4/l1izCyOZg8vzb9gsS7xalxtQKQouZWGxnmIl8kxxnfjfcbSFToFsferKtrBmeeoxkrUYJRVHN1tnh9E9tv0/ipWM4oblTN25aVkuSgqClml1knlcSaQ6Xi8edcz1pCjTNM8ucgUbYZbDOnp8Sw4Fyg4FUG0OmGF56DLiH/a/n8v1AIfVHSs2EDK0KaUOTUHCloJdSXXmSJApXkO4ez+WFIQXHq1LiX6dvqHMYJQ2wktNQWJXonsv7vg+HrFB08goVeJwLpN8/duqXpd4tTzgcMQYchQOEqkDfg9Pln+M0o7l6zLNq54DX/FT2lj9Uh0eChBoVkOvQJbHsMUoM0IHBqOqEhFU/81NUlOS/JMDph99xf2MOeZaOQUJL9hqh+xGbBKEjVzBmhHWWf0InTnipBquxQu+vp1xINUW+eYrfguHx2/zt3e32iN1lcR56MDq16NEORjFiunA6sSwPDV7olpmJs2MXK9mLCUn4eKzXt5sX21shgKP61tHyebrnCtKOpnHB7aFHavvcfLYKQirqJcOHys49r/IYyUMVDPv6NYUYFfaPkqSODKPleWWrWBLiTvi5oDesccNHzWSDU/8u5aZZSeBam1dwriLzzfa7rXqpzNw4F5mKMiy3fH4Fa/is4zSXgzn7gOERiKlKUkcYfy5pg+ddB8g7STwLxyER2GCmpHQEebX+okXVhKVU8/G2tJG6OwgUWD229EjHEpUWxD/ZDX1iGW9SyCrpfDE8rtOorQSWioZBkfLD/XSTpxsrfi498Y8LUOhQ3One3KfDvg5P05BXMW9zMGIdaKGR+7JlqOgYmjsa291SIlUd07m85/167zOPtfTPD1PHvFeiiir+dirq9pvoStAdrznASyPG3bX50tt1ucqcUobpsODfIlfEDp5ImINR7vSequIYiVA1srVT+Yxkg1HfSmj2J/qGXX3Y02Tp6PsQvPEKci6pcDOUlYLoBy2vWs5gicx5f05t+RWCS9tqPF75i3r1xkTXmp95ZtAoVOs4JHlyUFoVVuvmPp/6ygInYIELUaBfrljuZj5FybohQ5OQodXYRblNt+MHFByG7iZuH3mHTk2jITHUUcRNBxIQ4lTCg8qCpQuSDL58ml99wTY8FPiUHK75BO+n2ItiD9IZOccjzMtorijIBvEcH6Ze6tft+Ir2HMvdjDyXU5iJWqouYdtDZUE/ohP6ODA8mSYNY3chqtgvDd6x9uyXy9nvV5pXjU8z6543Iubkp+yK9XW00fMg0Chx3V2nDAnSniJEv1jN4aL3BvvfcVv697Gfs6l9dKsk/mmZKM3J5Y/Z5pDX0nzXa0tjuSZkcUTv7E7DLbcvhu87DD+BpjL0c1/xLIzTE9H23d7dp6lFy90rPdhFPRs3ptjXOsLQ0FSJSw29F4yHTY8ygzbTNjunH4b/ZoErnx72ZXTjvY/n3OvuA5rjmysVNmdaHs2ahJUyupy1I/19+2Yv+deMhQkvrDt4oe9DbXV4DwrKGupPUc2X3FkvXk362LcSVop4QYH7o1zOwi75wlrbO5nLwP9fFesW/yeKqZDl0eT5tfmuWN2456zez5wudUDoOV6C6+MhK3tiVxBryC3i91bc0xiz/nqsT8pYY59E46/IVYV5GbfaI+5GHmwuyMHSSftW6tevuyR1Pq/j6OkZyRpmMVn4NF6guWO1/AqzMs76n/hsv2ack73H++dOZdb7cPgEft77ki4KIjr8u3/m3vZfzcnYML1WS9dvpl1E+4j2HGdtRIpO99HlltN2n2bDeNM5OnZcslGTz2OsWU0vscnbzdQfR/j2csMKKdYjXtlvk+Jf8RvL9oZLCfLC36j/cdqeVsfXvj5ARPT4UWA3UOXC4GC1u4G+3b6rO+63Zz/9uXff/pPAAAA//+74FmX") + unpacked := packer.MustUnpack("eJycVlGPo7oVfu/P2PfeEhNWS6U+xMxgzE6Yhmxs7JcK21kgsRN0E0ig6n+vbDJJZnVVXfVhpBGxz/nOd77vHP/7y7/M9lz+bavL07mRfy2r7eH820+93Z5/G4z+8vcveFiE//zh/bm/NUw4DbQ08Qm//Mk7j7/veIARp7EnB9jKAe4ECExJ1UwasscJ6TgiA37lJ06J9xbBQPibioG4wyjX3OiOr+FZ+JmHk0yrJG+FUWPULCt+SHuxhl6JNtUa6IbRoMYxSd8ayO8xEnLhKPa4O6dHjHgr0KZS6FvFQNhxow+qSDVO8iNfw5EXuVfS4CAHez72bB5mf0dBryI4qgJehJ96nK4qdthXublqXqy6yMxqYeIDpzMtDquvUbOoBA33/MexwmaqDVfH7zha7PBrduQ0+x0j3WHEB17EM06D3Xb9zI3FWFtcraCbihniKUC0aOCO0evJ4hMDNAIRrSL3/8CLfCgpAfzGiTDExyjrheEt98nAirwVYN5ZbPhyrHC0rHJEDCvISUW2N4txmSwdRgZqzcC5FTo00pCroqRj/v4rflle5OWjjnjPIwhYkbaMBju+hg0vcp9T0uHXeLVef8Jqf/N4gSuO9KiStGUH4n1gZSA+iWQ5YQNtq5Jcy92xyorld/dtgBt7LqfXWvp5ywYIOSCdiuBeADLiVzKXKOw4DbySxieM0pqBc88NqxSoLW+1aKCWh7SXDTyyAl44feYiaxjNfheAm/fKcXOrUXdlkbeMXiqF9NniFQP0GdUeRqnmJhz4euJ+S3XHh6d6ovkUA9kYq6/4JR6fuf9B43lJZzOnYTqzve5xouqyWNl+G+FbXaZaJWlgNXnTqOP0bcJYCRrP35vF1fbNeQKFvjDXnlGrwWXl8idZr4rU9qfnzQ1fkXol5TXz8/Etgk6DZZFqOUAtUDwqpHcYXdsSbCrp54Oimed8guKOR7BjdNYKIysB2KRTlLbCxFZ/4z0v2nR8WPQK6M7iksNs/v7Cuq3Rl+kMHIXVJSBks99/xVF6VEl+keOxfwPxoJA2jGaeHII7pjeT9W8g7xUITgLEezmEzTRbvjWPemGtUOU8PmHLW2nITqFwuHnMY1R3jlP0iGU9LVG8K0F84MXyOx4WFU7yWhqlVRzuLT/Mz3t5sFjxfuu7PJ2dL6zw+gf3+Yir4w5HTreut8InHkbcam8vfNUJFNYcOb1crO4YVfrGyf38e3Pzo8URLStu9Imv4YUV+dFpBb1WMiGNcL066+0aziQg+1sewIu05Qft+vSETfMI1vKQt9zEO/UUv6SsEib0PmZkWSwrmaRaTj6L77p4tXMu8HCiWoEuldXnhMfp7ldMRprwjBE5sSLzSrrsGPhWKUAaCbTH1/Mbz/Agp7x/zO3d98GF+4tHr5HupJ/XAl1/ShR7ZQT3vMh20uiGF8tb/689A7FX0rDDKDRPdywXd/7lEF6kCQ/SxGc5zKt1sXIcMHodxTBx+8jrOB44jfeTN+EggBoY9SrpQ82ANiXNaoV0Lw52rrnetuKQeda/xVTfH9Za0mBUyM4P7Dwtk7RngIxyPN57JZx+Z7WIpn2EUTATNNWygUb42GKoFQhcfdKEO15k413TaGbnurvnZsfL4rJc3PV6ZkXdioKccQJ7eVhV4kDOzJDhVvMoUewLt7ttzZmWfqblZbpv51Hhu3tuV9jYD1/aXUZ6VawqOzcxyix/3uTDQKvhE9YOJ+rI6fxR//R+6IS/qqRPdu4b4DOBrsFHrI+9rFDdyifvPc1Cz84+jMILL9KpP839XTKWNNfuXYJeK27iViRk4KupthunP90eoOFMvRyr5QjHyHr9NqffH77XwsSNQDZWrhUig+2Z0ySotdWl04/dG2uX/yL8bCxpeuLr+S97GRqrwbtXEOmYPUdXX3FiZ/PmrgtpiCeN1nIW7oTvfDtKRHYl5a3N/ehlaG6YW8f1q31rxTORTJpfDvN7zI2Nedjb99JMWD0AVi3X8Mxp3NmeqAj6Jb3anavvMak9FzzeL5S73e++IV6X9GrfOB+++Hi7jAXIekUDN/seXsvt/A9wkl04zVpu6xvg3s4Ey51o4FkOT2+P/1WX7deUc7r/4/j0zeaa1ds4/Ij9mbvHvv4/cy8un3wCprrem8VMrO66ibcJ7IUhnULa+nvPC6tJu2/J6DxR5MfpHWVn6t7FVkmqWXGb7QPUn2M8ekmA9srI7ZX7fUaDPX550s5oOVmc8Au7vEWL63L3xM8vNXETnyTYfOjy0xvsvbnpdnX8x5f//OW/AQAA////ZUbx") raw, ok := unpacked["_meta/elastic-agent.fleet.yml"] if !ok { // ensure we have something loaded. diff --git a/x-pack/elastic-agent/pkg/agent/warn/warn.go b/x-pack/elastic-agent/pkg/agent/warn/warn.go index d9667f6f2846..c3b97079aa11 100644 --- a/x-pack/elastic-agent/pkg/agent/warn/warn.go +++ b/x-pack/elastic-agent/pkg/agent/warn/warn.go @@ -11,7 +11,7 @@ import ( "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/core/logger" ) -const message = "The Elastic Agent is currently in Alpha and should not be used in production" +const message = "The Elastic Agent is currently in Experimental and should not be used in production" // LogNotGA warns the users in the log that the Elastic Agent is not GA. func LogNotGA(log *logger.Logger) { diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml index d94015263569..927f21a89d9f 100644 --- a/x-pack/metricbeat/metricbeat.reference.yml +++ b/x-pack/metricbeat/metricbeat.reference.yml @@ -993,7 +993,7 @@ metricbeat.modules: metricsets: ["tablespace", "performance"] enabled: true period: 10s - hosts: ["oracle://user:pass@localhost:1521/ORCLPDB1.localdomain?sysdba=1"] + hosts: ["user:pass@0.0.0.0:1521/ORCLPDB1.localdomain"] # username: "" # password: "" diff --git a/x-pack/metricbeat/module/aws/aws.go b/x-pack/metricbeat/module/aws/aws.go index dcf047da87d1..5476f0dae5f7 100644 --- a/x-pack/metricbeat/module/aws/aws.go +++ b/x-pack/metricbeat/module/aws/aws.go @@ -103,6 +103,7 @@ func NewMetricSet(base mb.BaseMetricSet) (*MetricSet, error) { // collecting the first one. if output.AccountAliases != nil { metricSet.AccountName = output.AccountAliases[0] + base.Logger().Debug("AWS Credentials belong to account name: ", metricSet.AccountName) } } @@ -115,6 +116,7 @@ func NewMetricSet(base mb.BaseMetricSet) (*MetricSet, error) { base.Logger().Warn("failed to get caller identity, please check permission setting: ", err) } else { metricSet.AccountID = *outputIdentity.Account + base.Logger().Debug("AWS Credentials belong to account ID: ", metricSet.AccountID) } // Construct MetricSet with a full regions list diff --git a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go index 05c9b21cd3a6..5674a45eccd3 100644 --- a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go +++ b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go @@ -18,6 +18,7 @@ import ( "github.com/pkg/errors" "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/libbeat/logp" "github.com/elastic/beats/v7/metricbeat/mb" awscommon "github.com/elastic/beats/v7/x-pack/libbeat/common/aws" "github.com/elastic/beats/v7/x-pack/metricbeat/module/aws" @@ -51,6 +52,7 @@ func init() { // interface methods except for Fetch. type MetricSet struct { *aws.MetricSet + logger *logp.Logger CloudwatchConfigs []Config `config:"metrics" validate:"nonzero,required"` } @@ -113,6 +115,7 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { return &MetricSet{ MetricSet: metricSet, + logger: logp.NewLogger(metricsetName), CloudwatchConfigs: config.CloudwatchMetrics, }, nil } @@ -132,10 +135,13 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error { // Get listMetricDetailTotal and namespaceDetailTotal from configuration listMetricDetailTotal, namespaceDetailTotal := m.readCloudwatchConfig() + m.logger.Debugf("listMetricDetailTotal = %s", listMetricDetailTotal) + m.logger.Debugf("namespaceDetailTotal = %s", namespaceDetailTotal) // Create events based on listMetricDetailTotal from configuration if len(listMetricDetailTotal.metricsWithStats) != 0 { for _, regionName := range m.MetricSet.RegionsList { + m.logger.Debugf("Collecting metrics from AWS region %s", regionName) awsConfig := m.MetricSet.AwsConfig.Copy() awsConfig.Region = regionName @@ -150,6 +156,8 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error { return errors.Wrap(err, "createEvents failed for region "+regionName) } + m.logger.Debugf("Collected metrics of metrics = %d", len(eventsWithIdentifier)) + err = reportEvents(eventsWithIdentifier, report) if err != nil { return errors.Wrap(err, "reportEvents failed") @@ -158,6 +166,7 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error { } for _, regionName := range m.MetricSet.RegionsList { + m.logger.Debugf("Collecting metrics from AWS region %s", regionName) awsConfig := m.MetricSet.AwsConfig.Copy() awsConfig.Region = regionName @@ -169,9 +178,11 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error { // Create events based on namespaceDetailTotal from configuration for namespace, namespaceDetails := range namespaceDetailTotal { + m.logger.Debugf("Collected metrics from namespace %s", namespace) + listMetricsOutput, err := aws.GetListMetricsOutput(namespace, regionName, svcCloudwatch) if err != nil { - m.Logger().Info(err.Error()) + m.logger.Info(err.Error()) continue } @@ -189,6 +200,8 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error { return errors.Wrap(err, "createEvents failed for region "+regionName) } + m.logger.Debugf("Collected number of metrics = %d", len(eventsWithIdentifier)) + err = reportEvents(eventsWithIdentifier, report) if err != nil { return errors.Wrap(err, "reportEvents failed") @@ -434,12 +447,14 @@ func (m *MetricSet) createEvents(svcCloudwatch cloudwatchiface.ClientAPI, svcRes // Construct metricDataQueries metricDataQueries := createMetricDataQueries(listMetricWithStatsTotal, m.Period) + m.logger.Debugf("Number of MetricDataQueries = %d", len(metricDataQueries)) if len(metricDataQueries) == 0 { return events, nil } // Use metricDataQueries to make GetMetricData API calls metricDataResults, err := aws.GetMetricDataResults(metricDataQueries, svcCloudwatch, startTime, endTime) + m.logger.Debugf("Number of metricDataResults = %d", len(metricDataResults)) if err != nil { return events, errors.Wrap(err, "GetMetricDataResults failed") } @@ -482,7 +497,7 @@ func (m *MetricSet) createEvents(svcCloudwatch cloudwatchiface.ClientAPI, svcRes resourceTagMap, err := aws.GetResourcesTags(svcResourceAPI, []string{resourceType}) if err != nil { // If GetResourcesTags failed, continue report event just without tags. - m.Logger().Info(errors.Wrap(err, "getResourcesTags failed, skipping region "+regionName)) + m.logger.Info(errors.Wrap(err, "getResourcesTags failed, skipping region "+regionName)) } if len(tagsFilter) != 0 && len(resourceTagMap) == 0 { diff --git a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch_test.go b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch_test.go index 92016c1cbc95..0b8cc468c062 100644 --- a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch_test.go +++ b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch_test.go @@ -11,6 +11,7 @@ import ( "testing" "time" + "github.com/elastic/beats/v7/libbeat/logp" "github.com/elastic/beats/v7/metricbeat/mb" awssdk "github.com/aws/aws-sdk-go-v2/aws" @@ -1232,6 +1233,7 @@ func TestCreateEventsWithIdentifier(t *testing.T) { m := MetricSet{} m.CloudwatchConfigs = []Config{{Statistic: []string{"Average"}}} m.MetricSet = &aws.MetricSet{Period: 5} + m.logger = logp.NewLogger("test") mockTaggingSvc := &MockResourceGroupsTaggingClient{} mockCloudwatchSvc := &MockCloudWatchClient{} @@ -1272,6 +1274,7 @@ func TestCreateEventsWithoutIdentifier(t *testing.T) { m := MetricSet{} m.CloudwatchConfigs = []Config{{Statistic: []string{"Average"}}} m.MetricSet = &aws.MetricSet{Period: 5, AccountID: accountID} + m.logger = logp.NewLogger("test") mockTaggingSvc := &MockResourceGroupsTaggingClient{} mockCloudwatchSvc := &MockCloudWatchClientWithoutDim{} diff --git a/x-pack/metricbeat/module/oracle/_meta/config.yml b/x-pack/metricbeat/module/oracle/_meta/config.yml index b36667b5733a..8208685e033b 100644 --- a/x-pack/metricbeat/module/oracle/_meta/config.yml +++ b/x-pack/metricbeat/module/oracle/_meta/config.yml @@ -2,7 +2,7 @@ metricsets: ["tablespace", "performance"] enabled: true period: 10s - hosts: ["oracle://user:pass@localhost:1521/ORCLPDB1.localdomain?sysdba=1"] + hosts: ["user:pass@0.0.0.0:1521/ORCLPDB1.localdomain"] # username: "" # password: "" diff --git a/x-pack/metricbeat/module/oracle/connection.go b/x-pack/metricbeat/module/oracle/connection.go index f2fc870fb8d3..8beb7c936ad6 100644 --- a/x-pack/metricbeat/module/oracle/connection.go +++ b/x-pack/metricbeat/module/oracle/connection.go @@ -50,10 +50,6 @@ func NewConnection(c *ConnectionDetails) (*sql.DB, error) { params.Password = c.Password } - if params.IsSysDBA == false { - return nil, errors.New("a user with DBA permissions are required, check your connection details on field `hosts`") - } - db, err := sql.Open("godror", params.StringWithPassword()) if err != nil { return nil, errors.Wrap(err, "could not open database") diff --git a/x-pack/metricbeat/module/oracle/testing.go b/x-pack/metricbeat/module/oracle/testing.go index 5ffe9cd83f44..5a177f9d8234 100644 --- a/x-pack/metricbeat/module/oracle/testing.go +++ b/x-pack/metricbeat/module/oracle/testing.go @@ -28,7 +28,7 @@ func GetOracleEnvServiceName() string { serviceName := os.Getenv("ORACLE_SERVICE_NAME") if len(serviceName) == 0 { - serviceName = "ORCLPDB1.localdomain" + serviceName = "ORCLCDB.localdomain" } return serviceName } diff --git a/x-pack/metricbeat/modules.d/oracle.yml.disabled b/x-pack/metricbeat/modules.d/oracle.yml.disabled index dda3830d7155..46207ba0ae43 100644 --- a/x-pack/metricbeat/modules.d/oracle.yml.disabled +++ b/x-pack/metricbeat/modules.d/oracle.yml.disabled @@ -5,7 +5,7 @@ metricsets: ["tablespace", "performance"] enabled: true period: 10s - hosts: ["oracle://user:pass@localhost:1521/ORCLPDB1.localdomain?sysdba=1"] + hosts: ["user:pass@0.0.0.0:1521/ORCLPDB1.localdomain"] # username: "" # password: ""