[processor/resourcedetection] introduce retry mechanism for detectors

.chloggen/docker-resource-failure.yaml

@@ -0,0 +1,27 @@
+# Use this changelog template to create an entry for release notes.
+
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: enhancement
+
+# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver)
+component: processor/resourcedetection
+
+# A brief description of the change.  Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: "Introduce retry logic for failed resource detection."
+
+# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists.
+issues: [34761]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext:
+
+# If your change doesn't affect end users or the exported elements of any package,
+# you should instead start your pull request title with [chore] or use the "Skip Changelog" label.
+# Optional: The change log or logs in which this entry should be included.
+# e.g. '[user]' or '[user, api]'
+# Include 'user' if the change is relevant to end users.
+# Include 'api' if there is a change to a library API.
+# Default: '[user]'
+change_logs: []

processor/resourcedetectionprocessor/README.md

@@ -615,7 +615,7 @@ resourcedetection:
 
 ## Ordering
 
-Note that if multiple detectors are inserting the same attribute name, the first detector to insert wins. For example if you had `detectors: [eks, ec2]` then `cloud.platform` will be `aws_eks` instead of `ec2`. The below ordering is recommended.
+By default, if multiple detectors are inserting the same attribute name, the first detector to insert wins. For example if you had `detectors: [eks, ec2]` then `cloud.platform` will be `aws_eks` instead of `ec2`. The below ordering is recommended.
 
 ### AWS
 
@@ -627,3 +627,14 @@ Note that if multiple detectors are inserting the same attribute name, the first
 
 The full list of settings exposed for this extension are documented in [config.go](./config.go)
 with detailed sample configurations in [testdata/config.yaml](./testdata/config.yaml).
+
+**Note:**
+
+If you want to disable the ordering of the detectors and instead have a non-blocking resource detection in case of a detection failure, set the `order` parameter to `false`. For example:
+
+```yaml
+processors:
+  resourcedetection:
+    detectors: [docker]
+    order: false
+```

processor/resourcedetectionprocessor/config.go

@@ -41,6 +41,9 @@ type Config struct {
 	// If a supplied attribute is not a valid attribute of a supplied detector it will be ignored.
 	// Deprecated: Please use detector's resource_attributes config instead
 	Attributes []string `mapstructure:"attributes"`
+	// Order is a parameter which perserves the order of detectors and the detection of data
+	// This also introduces a blocking behavior, if one of the detectors cannot detect the resource
+	Order bool `mapstructure:"order"`
 }
 
 // DetectorConfig contains user-specified configurations unique to all individual detectors

processor/resourcedetectionprocessor/config_test.go

@@ -73,6 +73,7 @@ func TestLoadConfig(t *testing.T) {
 				DetectorConfig: openshiftConfig,
 				ClientConfig:   cfg,
 				Override:       false,
+				Order:          true,
 			},
 		},
 		{
@@ -82,6 +83,7 @@ func TestLoadConfig(t *testing.T) {
 				ClientConfig:   cfg,
 				Override:       false,
 				DetectorConfig: detectorCreateDefaultConfig(),
+				Order:          true,
 			},
 		},
 		{
@@ -91,6 +93,7 @@ func TestLoadConfig(t *testing.T) {
 				DetectorConfig: ec2Config,
 				ClientConfig:   cfg,
 				Override:       false,
+				Order:          true,
 			},
 		},
 		{
@@ -101,6 +104,7 @@ func TestLoadConfig(t *testing.T) {
 				ClientConfig:   cfg,
 				Override:       false,
 				Attributes:     []string{"a", "b"},
+				Order:          true,
 			},
 		},
 		{
@@ -110,6 +114,7 @@ func TestLoadConfig(t *testing.T) {
 				ClientConfig:   cfg,
 				Override:       false,
 				DetectorConfig: detectorCreateDefaultConfig(),
+				Order:          true,
 			},
 		},
 		{
@@ -119,6 +124,7 @@ func TestLoadConfig(t *testing.T) {
 				ClientConfig:   cfg,
 				Override:       false,
 				DetectorConfig: detectorCreateDefaultConfig(),
+				Order:          true,
 			},
 		},
 		{
@@ -128,6 +134,7 @@ func TestLoadConfig(t *testing.T) {
 				ClientConfig:   cfg,
 				Override:       false,
 				DetectorConfig: resourceAttributesConfig,
+				Order:          true,
 			},
 		},
 		{

processor/resourcedetectionprocessor/factory.go

@@ -96,6 +96,7 @@ func createDefaultConfig() component.Config {
 		Override:       true,
 		Attributes:     nil,
 		DetectorConfig: detectorCreateDefaultConfig(),
+		Order:          true,
 		// TODO: Once issue(https://github.com/open-telemetry/opentelemetry-collector/issues/4001) gets resolved,
 		//		 Set the default value of 'hostname_source' here instead of 'system' detector
 	}
@@ -199,7 +200,7 @@ func (f *factory) getResourceDetectionProcessor(
 	if oCfg.Attributes != nil {
 		params.Logger.Warn("You are using deprecated `attributes` option that will be removed soon; use `resource_attributes` instead, details on configuration: https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/processor/resourcedetectionprocessor#migration-from-attributes-to-resource_attributes")
 	}
-	provider, err := f.getResourceProvider(params, oCfg.ClientConfig.Timeout, oCfg.Detectors, oCfg.DetectorConfig, oCfg.Attributes)
+	provider, err := f.getResourceProvider(params, oCfg.ClientConfig.Timeout, oCfg.Detectors, oCfg.DetectorConfig, oCfg.Attributes, oCfg.Order)
 	if err != nil {
 		return nil, err
 	}
@@ -218,6 +219,7 @@ func (f *factory) getResourceProvider(
 	configuredDetectors []string,
 	detectorConfigs DetectorConfig,
 	attributes []string,
+	order bool,
 ) (*internal.ResourceProvider, error) {
 	f.lock.Lock()
 	defer f.lock.Unlock()
@@ -231,7 +233,7 @@ func (f *factory) getResourceProvider(
 		detectorTypes = append(detectorTypes, internal.DetectorType(strings.TrimSpace(key)))
 	}
 
-	provider, err := f.resourceProviderFactory.CreateResourceProvider(params, timeout, attributes, &detectorConfigs, detectorTypes...)
+	provider, err := f.resourceProviderFactory.CreateResourceProvider(params, timeout, attributes, &detectorConfigs, order, detectorTypes...)
 	if err != nil {
 		return nil, err
 	}

processor/resourcedetectionprocessor/internal/resourcedetection.go

@@ -45,6 +45,7 @@ func (f *ResourceProviderFactory) CreateResourceProvider(
 	timeout time.Duration,
 	attributes []string,
 	detectorConfigs ResourceDetectorConfig,
+	order bool,
 	detectorTypes ...DetectorType,
 ) (*ResourceProvider, error) {
 	detectors, err := f.getDetectors(params, detectorConfigs, detectorTypes)
@@ -59,7 +60,7 @@ func (f *ResourceProviderFactory) CreateResourceProvider(
 		}
 	}
 
-	provider := NewResourceProvider(params.Logger, timeout, attributesToKeep, detectors...)
+	provider := NewResourceProvider(params.Logger, timeout, attributesToKeep, order, detectors...)
 	return provider, nil
 }
 
@@ -89,6 +90,7 @@ type ResourceProvider struct {
 	detectedResource *resourceResult
 	once             sync.Once
 	attributesToKeep map[string]struct{}
+	order            bool
 }
 
 type resourceResult struct {
@@ -97,26 +99,39 @@ type resourceResult struct {
 	err       error
 }
 
-func NewResourceProvider(logger *zap.Logger, timeout time.Duration, attributesToKeep map[string]struct{}, detectors ...Detector) *ResourceProvider {
+func NewResourceProvider(logger *zap.Logger, timeout time.Duration, attributesToKeep map[string]struct{}, order bool, detectors ...Detector) *ResourceProvider {
 	return &ResourceProvider{
 		logger:           logger,
 		timeout:          timeout,
 		detectors:        detectors,
 		attributesToKeep: attributesToKeep,
+		order:            order,
 	}
 }
 
-func (p *ResourceProvider) Get(ctx context.Context, client *http.Client) (resource pcommon.Resource, schemaURL string, err error) {
+func (p *ResourceProvider) Get(ctx context.Context, _ *http.Client) (resource pcommon.Resource, schemaURL string, err error) {
 	p.once.Do(func() {
-		var cancel context.CancelFunc
-		ctx, cancel = context.WithTimeout(ctx, client.Timeout)
-		defer cancel()
 		p.detectResource(ctx)
 	})
 
 	return p.detectedResource.resource, p.detectedResource.schemaURL, p.detectedResource.err
 }
 
+type detectResult struct {
+	r         pcommon.Resource
+	schemaURL string
+	err       error
+}
+
+func handleResult(res *pcommon.Resource, resultsChan chan detectResult, mergedSchemaURL string) string {
+	result := <-resultsChan
+	if result.err == nil {
+		mergedSchemaURL = MergeSchemaURL(mergedSchemaURL, result.schemaURL)
+		MergeResource(*res, result.r, false)
+	}
+	return mergedSchemaURL
+}
+
 func (p *ResourceProvider) detectResource(ctx context.Context) {
 	p.detectedResource = &resourceResult{}
 
@@ -125,13 +140,29 @@ func (p *ResourceProvider) detectResource(ctx context.Context) {
 
 	p.logger.Info("began detecting resource information")
 
+	resultsChan := make(chan detectResult, len(p.detectors))
 	for _, detector := range p.detectors {
-		r, schemaURL, err := detector.Detect(ctx)
-		if err != nil {
-			p.logger.Warn("failed to detect resource", zap.Error(err))
-		} else {
-			mergedSchemaURL = MergeSchemaURL(mergedSchemaURL, schemaURL)
-			MergeResource(res, r, false)
+		go func(detector Detector) {
+			sleep := 2 * time.Second
+			for {
+				r, schemaURL, err := detector.Detect(ctx)
+				if err != nil {
+					p.logger.Warn("failed to detect resource", zap.Error(err))
+					time.Sleep(sleep)
+					sleep *= 2
+				} else {
+					resultsChan <- detectResult{r: r, schemaURL: schemaURL, err: nil}
+					return
+				}
+			}
+		}(detector)
+		if p.order {
+			mergedSchemaURL = handleResult(&res, resultsChan, mergedSchemaURL)
+		}
+	}
+	if !p.order {
+		for range p.detectors {
+			mergedSchemaURL = handleResult(&res, resultsChan, mergedSchemaURL)
 		}
 	}