Add scrapeFailureLogLevel config to smartagent prometheus receivers

CHANGELOG.md

@@ -11,6 +11,10 @@
   that the translation in the prometheus receiver is a subject to possible future changes.
   ([#23229](https://github.com/open-telemetry/opentelemetry-collector-contrib/pull/23229))
 
+### 💡 Enhancements 💡
+
+- (Splunk) `receiver/smartagent`: Add scrapeFailureLogLevel config to smartagent prometheus receivers to support logging scrape failures at different levels ([#3260](https://github.com/signalfx/splunk-otel-collector/pull/3260))
+
 ## v0.78.1
 
 ### 🧰 Bug fixes 🧰

internal/signalfx-agent/pkg/monitors/prometheusexporter/prometheus.go

@@ -45,13 +45,26 @@ type Config struct {
 	// (the default).
 	MetricPath string `yaml:"metricPath" default:"/metrics"`
 
+	// Control the log level to use if a scrape failure occurs when scraping
+	// a target. Modifying this configuration is useful for less stable
+	// targets. Only the debug, info, warn, and error log levels are supported.
+	ScrapeFailureLogLevel string `yaml:"scrapeFailureLogLevel" default:"error"`
+
 	// Send all the metrics that come out of the Prometheus exporter without
 	// any filtering.  This option has no effect when using the prometheus
 	// exporter monitor directly since there is no built-in filtering, only
 	// when embedding it in other monitors.
 	SendAllMetrics bool `yaml:"sendAllMetrics"`
 }
 
+func (c *Config) Validate() error {
+	if _, err := logrus.ParseLevel(c.ScrapeFailureLogLevel); err != nil {
+		return err
+	} else {
+		return nil
+	}
+}
+
 func (c *Config) GetExtraMetrics() []string {
 	// Maintain backwards compatibility with the config flag that existing
 	// prior to the new filtering mechanism.
@@ -74,16 +87,18 @@ type Monitor struct {
 	// If true, IncludedMetrics is ignored and everything is sent.
 	SendAll bool
 
-	monitorName string
-	logger      logrus.FieldLogger
-	cancel      func()
+	monitorName        string
+	logger             logrus.FieldLogger
+	loggerFailureLevel logrus.Level
+	cancel             func()
 }
 
 type fetcher func() (io.ReadCloser, expfmt.Format, error)
 
 // Configure the monitor and kick off volume metric syncing
 func (m *Monitor) Configure(conf *Config) error {
 	m.logger = logrus.WithFields(logrus.Fields{"monitorType": m.monitorName, "monitorID": conf.MonitorID})
+	m.loggerFailureLevel, _ = logrus.ParseLevel(conf.ScrapeFailureLogLevel)
 
 	var bearerToken string
 
@@ -136,7 +151,8 @@ func (m *Monitor) Configure(conf *Config) error {
 	utils.RunOnInterval(ctx, func() {
 		dps, err := fetchPrometheusMetrics(fetch)
 		if err != nil {
-			m.logger.WithError(err).Error("Could not get prometheus metrics")
+			// The default log level is error, users can configure which level to use
+			m.logger.WithError(err).Log(m.loggerFailureLevel, "Could not get prometheus metrics")
 			return
 		}
 

internal/signalfx-agent/pkg/monitors/prometheusexporter/prometheus_test.go

@@ -0,0 +1,60 @@
+package prometheusexporter
+
+import (
+	"errors"
+	"testing"
+)
+
+func TestConfigValidate(t *testing.T) {
+	tests := []struct {
+		name               string
+		scrapeFailureLevel string
+		expectedError      error
+	}{
+		{
+			name:               "Valid log level: debug",
+			scrapeFailureLevel: "debug",
+			expectedError:      nil,
+		},
+		{
+			name:               "Valid log level: info",
+			scrapeFailureLevel: "info",
+			expectedError:      nil,
+		},
+		{
+			name:               "Valid log level: warn",
+			scrapeFailureLevel: "warn",
+			expectedError:      nil,
+		},
+		{
+			name:               "Valid log level: error",
+			scrapeFailureLevel: "error",
+			expectedError:      nil,
+		},
+		{
+			name:               "Invalid log level",
+			scrapeFailureLevel: "badValue",
+			expectedError:      errors.New("not a valid logrus Level: \"badValue\""),
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			config := &Config{
+				ScrapeFailureLogLevel: test.scrapeFailureLevel,
+			}
+
+			err := config.Validate()
+
+			if test.expectedError != nil {
+				if err == nil {
+					t.Errorf("Expected error '%s', but got nil", test.expectedError.Error())
+				} else if err.Error() != test.expectedError.Error() {
+					t.Errorf("Expected error '%s', but got '%s'", test.expectedError.Error(), err.Error())
+				}
+			} else if err != nil {
+				t.Errorf("Expected no error, but got '%s'", err.Error())
+			}
+		})
+	}
+}

pkg/receiver/smartagentreceiver/config_test.go

@@ -132,9 +132,10 @@ func TestLoadConfig(t *testing.T) {
 			HTTPConfig: httpclient.HTTPConfig{
 				HTTPTimeout: timeutil.Duration(10 * time.Second),
 			},
-			Host:       "localhost",
-			Port:       5309,
-			MetricPath: "/metrics",
+			Host:                  "localhost",
+			Port:                  5309,
+			MetricPath:            "/metrics",
+			ScrapeFailureLogLevel: "error",
 		},
 		acceptsEndpoints: true,
 	}, etcdCfg)
@@ -328,9 +329,10 @@ func TestLoadConfigWithEndpoints(t *testing.T) {
 			HTTPConfig: httpclient.HTTPConfig{
 				HTTPTimeout: timeutil.Duration(10 * time.Second),
 			},
-			Host:       "localhost",
-			Port:       5555,
-			MetricPath: "/metrics",
+			Host:                  "localhost",
+			Port:                  5555,
+			MetricPath:            "/metrics",
+			ScrapeFailureLogLevel: "error",
 		},
 		acceptsEndpoints: true,
 	}, etcdCfg)

tests/receivers/smartagent/prometheus-exporter/testdata/httpd_metrics_config.yaml

@@ -4,12 +4,12 @@ receivers:
     intervalSeconds: 1
     host: "localhost"
     port: 8000
+    scrapeFailureLogLevel: debug
 exporters:
   otlp:
     endpoint: "${OTLP_ENDPOINT}"
     tls:
       insecure: true
-
 service:
   pipelines:
     metrics:

tests/receivers/smartagent/prometheus-exporter/testdata/internal_metrics_config.yaml

@@ -6,13 +6,12 @@ receivers:
     port: 8889
     extraDimensions:
       foo: bar
-
+    scrapeFailureLogLevel: error
 exporters:
   otlp:
     endpoint: "${OTLP_ENDPOINT}"
     tls:
       insecure: true
-
 service:
   telemetry:
     metrics: