diff --git a/CHANGELOG.md b/CHANGELOG.md index d4ab0e9776..7b7179155c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -64,6 +64,10 @@ This Splunk OpenTelemetry Collector release includes changes from the [opentelem - (Contrib) `splunkhecreceiver`: Fix reusing the same splunkhecreiver between logs and metrics ([#22848](https://github.com/open-telemetry/opentelemetry-collector-contrib/pull/22848)) - (Core) `connectors`: When replicating data to connectors, consider whether the next pipeline will mutate data ([#7776](https://github.com/open-telemetry/opentelemetry-collector/issues/7776)) +### 💡 Enhancements 💡 + +- (Splunk) `receiver/smartagent`: Add `scrapeFailureLogLevel` config field to `prometheus-exporter` and its sourcing monitors to determine the log level for reported scrape failures ([#3260](https://github.com/signalfx/splunk-otel-collector/pull/3260)) + ## v0.78.1 ### 🧰 Bug fixes 🧰 diff --git a/internal/signalfx-agent/pkg/monitors/prometheusexporter/prometheus.go b/internal/signalfx-agent/pkg/monitors/prometheusexporter/prometheus.go index 8c5e94d046..53d0018800 100644 --- a/internal/signalfx-agent/pkg/monitors/prometheusexporter/prometheus.go +++ b/internal/signalfx-agent/pkg/monitors/prometheusexporter/prometheus.go @@ -45,6 +45,12 @@ type Config struct { // (the default). MetricPath string `yaml:"metricPath" default:"/metrics"` + // Control the log level to use if a scrape failure occurs when scraping + // a target. Modifying this configuration is useful for less stable + // targets. All logrus log levels are supported. + ScrapeFailureLogLevel string `yaml:"scrapeFailureLogLevel" default:"error"` + scrapeFailureLogrusLevel logrus.Level + // Send all the metrics that come out of the Prometheus exporter without // any filtering. This option has no effect when using the prometheus // exporter monitor directly since there is no built-in filtering, only @@ -52,6 +58,15 @@ type Config struct { SendAllMetrics bool `yaml:"sendAllMetrics"` } +func (c *Config) Validate() error { + l, err := logrus.ParseLevel(c.ScrapeFailureLogLevel) + if err != nil { + return err + } + c.scrapeFailureLogrusLevel = l + return nil +} + func (c *Config) GetExtraMetrics() []string { // Maintain backwards compatibility with the config flag that existing // prior to the new filtering mechanism. @@ -136,7 +151,8 @@ func (m *Monitor) Configure(conf *Config) error { utils.RunOnInterval(ctx, func() { dps, err := fetchPrometheusMetrics(fetch) if err != nil { - m.logger.WithError(err).Error("Could not get prometheus metrics") + // The default log level is error, users can configure which level to use + m.logger.WithError(err).Log(conf.scrapeFailureLogrusLevel, "Could not get prometheus metrics") return } diff --git a/internal/signalfx-agent/pkg/monitors/prometheusexporter/prometheus_test.go b/internal/signalfx-agent/pkg/monitors/prometheusexporter/prometheus_test.go new file mode 100644 index 0000000000..caff3d6d7c --- /dev/null +++ b/internal/signalfx-agent/pkg/monitors/prometheusexporter/prometheus_test.go @@ -0,0 +1,66 @@ +package prometheusexporter + +import ( + "errors" + "testing" + + "github.com/stretchr/testify/assert" +) + +type testCase struct { + name string + scrapeFailureLogLevel string + expectedError error +} + +func TestConfigValidate(t *testing.T) { + testCases := []testCase{ + { + name: "Valid log level: debug", + scrapeFailureLogLevel: "debug", + expectedError: nil, + }, + { + name: "Valid log level: info", + scrapeFailureLogLevel: "info", + expectedError: nil, + }, + { + name: "Valid log level: warn", + scrapeFailureLogLevel: "warn", + expectedError: nil, + }, + { + name: "Valid log level: error", + scrapeFailureLogLevel: "error", + expectedError: nil, + }, + { + name: "Invalid log level", + scrapeFailureLogLevel: "badValue", + expectedError: errors.New("not a valid logrus Level: \"badValue\""), + }, + } + + for _, tc := range testCases { + tcc := tc + t.Run(tc.name, func(t *testing.T) { + config := &Config{ + ScrapeFailureLogLevel: tcc.scrapeFailureLogLevel, + } + + err := config.Validate() + + if tcc.expectedError != nil { + if err == nil { + t.Errorf("Expected error '%s', but got nil", tcc.expectedError.Error()) + } else if err.Error() != tcc.expectedError.Error() { + t.Errorf("Expected error '%s', but got '%s'", tcc.expectedError.Error(), err.Error()) + assert.EqualValues(t, "smartagentvalid", config.MonitorConfigCore().MonitorID) + } + } else if err != nil { + t.Errorf("Expected no error, but got '%s'", err.Error()) + } + }) + } +} diff --git a/pkg/receiver/smartagentreceiver/config_test.go b/pkg/receiver/smartagentreceiver/config_test.go index c0d2fc05f3..537bfc9724 100644 --- a/pkg/receiver/smartagentreceiver/config_test.go +++ b/pkg/receiver/smartagentreceiver/config_test.go @@ -132,9 +132,10 @@ func TestLoadConfig(t *testing.T) { HTTPConfig: httpclient.HTTPConfig{ HTTPTimeout: timeutil.Duration(10 * time.Second), }, - Host: "localhost", - Port: 5309, - MetricPath: "/metrics", + Host: "localhost", + Port: 5309, + MetricPath: "/metrics", + ScrapeFailureLogLevel: "error", }, acceptsEndpoints: true, }, etcdCfg) @@ -328,9 +329,10 @@ func TestLoadConfigWithEndpoints(t *testing.T) { HTTPConfig: httpclient.HTTPConfig{ HTTPTimeout: timeutil.Duration(10 * time.Second), }, - Host: "localhost", - Port: 5555, - MetricPath: "/metrics", + Host: "localhost", + Port: 5555, + MetricPath: "/metrics", + ScrapeFailureLogLevel: "error", }, acceptsEndpoints: true, }, etcdCfg) diff --git a/tests/receivers/smartagent/prometheus-exporter/testdata/httpd_metrics_config.yaml b/tests/receivers/smartagent/prometheus-exporter/testdata/httpd_metrics_config.yaml index e061bd3795..18c89bb48f 100644 --- a/tests/receivers/smartagent/prometheus-exporter/testdata/httpd_metrics_config.yaml +++ b/tests/receivers/smartagent/prometheus-exporter/testdata/httpd_metrics_config.yaml @@ -4,12 +4,12 @@ receivers: intervalSeconds: 1 host: "localhost" port: 8000 + scrapeFailureLogLevel: debug exporters: otlp: endpoint: "${OTLP_ENDPOINT}" tls: insecure: true - service: pipelines: metrics: diff --git a/tests/receivers/smartagent/prometheus-exporter/testdata/internal_metrics_config.yaml b/tests/receivers/smartagent/prometheus-exporter/testdata/internal_metrics_config.yaml index d1bc00c86f..77e97f4ac1 100644 --- a/tests/receivers/smartagent/prometheus-exporter/testdata/internal_metrics_config.yaml +++ b/tests/receivers/smartagent/prometheus-exporter/testdata/internal_metrics_config.yaml @@ -6,13 +6,12 @@ receivers: port: 8889 extraDimensions: foo: bar - + scrapeFailureLogLevel: error exporters: otlp: endpoint: "${OTLP_ENDPOINT}" tls: insecure: true - service: telemetry: metrics: