Extend Expand converter to support expanding values from any provider

There are still not 100% finalize designed choices:
1. This PR requires a change in the current Provider interface to support returning a config.Map (maybe a map[string]interface{}) or a string (maybe accept any interface{} and set that)
2. It is a bit unclear how to support watching for config updates, there are 2 options: a) extend the Converter to also accept a WatcherFunc; b) remove expand converter and embed this capability into our implementation of the ConfigProvider (allow both constructors NewConfigProvider and NewDefaultConfigProvider to allow this to be configured somehow).
3. The current "os.Expand" does not allow to return an error or to handle an error. It may require us to copy that or look for an alternative API.

Signed-off-by: Bogdan Drutu <bogdandrutu@gmail.com>

config/configmapprovider/retrieved.go

@@ -26,7 +26,7 @@ import (
 type Retrieved interface {
 	// Get returns the config Map. Should never be called after Close.
 	// Should never be called concurrently with itself or Close.
-	Get(ctx context.Context) (*config.Map, error)
+	Get(ctx context.Context) (interface{}, error)
 
 	// Close signals that the configuration for which it was used to retrieve values is
 	// no longer in use and should close and release any watchers that it may have created.

service/internal/configprovider/expand.go

@@ -17,44 +17,72 @@ package configprovider // import "go.opentelemetry.io/collector/service/internal
 import (
 	"context"
 	"os"
+	"strings"
 
 	"go.opentelemetry.io/collector/config"
+	"go.opentelemetry.io/collector/config/configmapprovider"
 )
 
-// NewExpandConverter returns a service.ConfigMapConverterFunc, that expands all environment variables for a given config.Map.
-func NewExpandConverter() config.MapConverterFunc {
-	return func(_ context.Context, cfgMap *config.Map) error {
+// NewExpandConverter returns a config.MapConverterFunc, that expands all environment variables for a given config.Map.
+//
+// The `configMapProviders` is a map of pairs <scheme,Provider>.
+func NewExpandConverter(configMapProviders map[string]configmapprovider.Provider) config.MapConverterFunc {
+	// TODO: Figure out how to support watching for changes in a Converter, or if expand should be a converter.
+	return func(ctx context.Context, cfgMap *config.Map) error {
 		for _, k := range cfgMap.AllKeys() {
-			cfgMap.Set(k, expandStringValues(cfgMap.Get(k)))
+			cfgMap.Set(k, expandStringValues(cfgMap.Get(k), func(location string) string {
+				// This allows escaping environment variable substitution via $$, e.g.
+				// - $FOO will be substituted with env var FOO
+				// - $$FOO will be replaced with $FOO
+				// - $$$FOO will be replaced with $ + substituted env var FOO
+				if location == "$" {
+					return "$"
+				}
+				scheme := "env"
+				if idx := strings.Index(location, ":"); idx != -1 {
+					scheme = location[:idx]
+				} else {
+					// MapProviders require the location to always have the scheme.
+					location = scheme + ":" + location
+				}
+				p, ok := configMapProviders[scheme]
+				// TODO: Figure out how to return an error when not registered.
+				if !ok {
+					return ""
+				}
+				// TODO: Figure out how to return an error when retrieve returns an error.
+				ret, _ := p.Retrieve(ctx, location, nil)
+				cfg, _ := ret.Get(ctx)
+				return cfg.(string)
+			}))
 		}
 		return nil
 	}
 }
 
-func expandStringValues(value interface{}) interface{} {
+// NewDefaultExpandConverter returns a config.MapConverterFunc, that expands all environment variables for a given config.Map.
+func NewDefaultExpandConverter() config.MapConverterFunc {
+	return NewExpandConverter(map[string]configmapprovider.Provider{
+		"file": configmapprovider.NewFile(),
+		"env":  configmapprovider.NewEnv(),
+	})
+}
+
+func expandStringValues(value interface{}, mapping func(string) string) interface{} {
 	switch v := value.(type) {
 	case string:
-		return expandEnv(v)
+		return expandEnv(v, mapping)
 	case []interface{}:
 		nslice := make([]interface{}, 0, len(v))
 		for _, vint := range v {
-			nslice = append(nslice, expandStringValues(vint))
+			nslice = append(nslice, expandStringValues(vint, mapping))
 		}
 		return nslice
 	default:
 		return v
 	}
 }
 
-func expandEnv(s string) string {
-	return os.Expand(s, func(str string) string {
-		// This allows escaping environment variable substitution via $$, e.g.
-		// - $FOO will be substituted with env var FOO
-		// - $$FOO will be replaced with $FOO
-		// - $$$FOO will be replaced with $ + substituted env var FOO
-		if str == "$" {
-			return "$"
-		}
-		return os.Getenv(str)
-	})
+func expandEnv(s string, mapping func(string) string) string {
+	return os.Expand(s, mapping)
 }