Add scrapeFailureLogLevel config to smartagent prometheus receivers (#…

…3260)

Add scrapeFailureLogLevel config to smartagent prometheus receivers to support logging scrape failures at different levels

CHANGELOG.md

@@ -64,6 +64,10 @@ This Splunk OpenTelemetry Collector release includes changes from the [opentelem
 - (Contrib) `splunkhecreceiver`: Fix reusing the same splunkhecreiver between logs and metrics ([#22848](https://github.com/open-telemetry/opentelemetry-collector-contrib/pull/22848))
 - (Core) `connectors`: When replicating data to connectors, consider whether the next pipeline will mutate data ([#7776](https://github.com/open-telemetry/opentelemetry-collector/issues/7776))
 
+### 💡 Enhancements 💡
+
+- (Splunk) `receiver/smartagent`: Add `scrapeFailureLogLevel` config field to `prometheus-exporter` and its sourcing monitors to determine the log level for reported scrape failures ([#3260](https://github.com/signalfx/splunk-otel-collector/pull/3260))
+
 ## v0.78.1
 
 ### 🧰 Bug fixes 🧰

internal/signalfx-agent/pkg/monitors/prometheusexporter/prometheus.go

@@ -45,13 +45,28 @@ type Config struct {
 	// (the default).
 	MetricPath string `yaml:"metricPath" default:"/metrics"`
 
+	// Control the log level to use if a scrape failure occurs when scraping
+	// a target. Modifying this configuration is useful for less stable
+	// targets. All logrus log levels are supported.
+	ScrapeFailureLogLevel    string `yaml:"scrapeFailureLogLevel" default:"error"`
+	scrapeFailureLogrusLevel logrus.Level
+
 	// Send all the metrics that come out of the Prometheus exporter without
 	// any filtering.  This option has no effect when using the prometheus
 	// exporter monitor directly since there is no built-in filtering, only
 	// when embedding it in other monitors.
 	SendAllMetrics bool `yaml:"sendAllMetrics"`
 }
 
+func (c *Config) Validate() error {
+	l, err := logrus.ParseLevel(c.ScrapeFailureLogLevel)
+	if err != nil {
+		return err
+	}
+	c.scrapeFailureLogrusLevel = l
+	return nil
+}
+
 func (c *Config) GetExtraMetrics() []string {
 	// Maintain backwards compatibility with the config flag that existing
 	// prior to the new filtering mechanism.
@@ -136,7 +151,8 @@ func (m *Monitor) Configure(conf *Config) error {
 	utils.RunOnInterval(ctx, func() {
 		dps, err := fetchPrometheusMetrics(fetch)
 		if err != nil {
-			m.logger.WithError(err).Error("Could not get prometheus metrics")
+			// The default log level is error, users can configure which level to use
+			m.logger.WithError(err).Log(conf.scrapeFailureLogrusLevel, "Could not get prometheus metrics")
 			return
 		}
 

internal/signalfx-agent/pkg/monitors/prometheusexporter/prometheus_test.go

@@ -0,0 +1,66 @@
+package prometheusexporter
+
+import (
+	"errors"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+type testCase struct {
+	name                  string
+	scrapeFailureLogLevel string
+	expectedError         error
+}
+
+func TestConfigValidate(t *testing.T) {
+	testCases := []testCase{
+		{
+			name:                  "Valid log level: debug",
+			scrapeFailureLogLevel: "debug",
+			expectedError:         nil,
+		},
+		{
+			name:                  "Valid log level: info",
+			scrapeFailureLogLevel: "info",
+			expectedError:         nil,
+		},
+		{
+			name:                  "Valid log level: warn",
+			scrapeFailureLogLevel: "warn",
+			expectedError:         nil,
+		},
+		{
+			name:                  "Valid log level: error",
+			scrapeFailureLogLevel: "error",
+			expectedError:         nil,
+		},
+		{
+			name:                  "Invalid log level",
+			scrapeFailureLogLevel: "badValue",
+			expectedError:         errors.New("not a valid logrus Level: \"badValue\""),
+		},
+	}
+
+	for _, tc := range testCases {
+		tcc := tc
+		t.Run(tc.name, func(t *testing.T) {
+			config := &Config{
+				ScrapeFailureLogLevel: tcc.scrapeFailureLogLevel,
+			}
+
+			err := config.Validate()
+
+			if tcc.expectedError != nil {
+				if err == nil {
+					t.Errorf("Expected error '%s', but got nil", tcc.expectedError.Error())
+				} else if err.Error() != tcc.expectedError.Error() {
+					t.Errorf("Expected error '%s', but got '%s'", tcc.expectedError.Error(), err.Error())
+					assert.EqualValues(t, "smartagentvalid", config.MonitorConfigCore().MonitorID)
+				}
+			} else if err != nil {
+				t.Errorf("Expected no error, but got '%s'", err.Error())
+			}
+		})
+	}
+}

pkg/receiver/smartagentreceiver/config_test.go

@@ -132,9 +132,10 @@ func TestLoadConfig(t *testing.T) {
 			HTTPConfig: httpclient.HTTPConfig{
 				HTTPTimeout: timeutil.Duration(10 * time.Second),
 			},
-			Host:       "localhost",
-			Port:       5309,
-			MetricPath: "/metrics",
+			Host:                  "localhost",
+			Port:                  5309,
+			MetricPath:            "/metrics",
+			ScrapeFailureLogLevel: "error",
 		},
 		acceptsEndpoints: true,
 	}, etcdCfg)
@@ -328,9 +329,10 @@ func TestLoadConfigWithEndpoints(t *testing.T) {
 			HTTPConfig: httpclient.HTTPConfig{
 				HTTPTimeout: timeutil.Duration(10 * time.Second),
 			},
-			Host:       "localhost",
-			Port:       5555,
-			MetricPath: "/metrics",
+			Host:                  "localhost",
+			Port:                  5555,
+			MetricPath:            "/metrics",
+			ScrapeFailureLogLevel: "error",
 		},
 		acceptsEndpoints: true,
 	}, etcdCfg)

tests/receivers/smartagent/prometheus-exporter/testdata/httpd_metrics_config.yaml

@@ -4,12 +4,12 @@ receivers:
     intervalSeconds: 1
     host: "localhost"
     port: 8000
+    scrapeFailureLogLevel: debug
 exporters:
   otlp:
     endpoint: "${OTLP_ENDPOINT}"
     tls:
       insecure: true
-
 service:
   pipelines:
     metrics:

tests/receivers/smartagent/prometheus-exporter/testdata/internal_metrics_config.yaml

@@ -6,13 +6,12 @@ receivers:
     port: 8889
     extraDimensions:
       foo: bar
-
+    scrapeFailureLogLevel: error
 exporters:
   otlp:
     endpoint: "${OTLP_ENDPOINT}"
     tls:
       insecure: true
-
 service:
   telemetry:
     metrics: