Add a config source for env vars (#348)

* Add a config source for env vars

(not complete yet)

* Add a config source for env vars

* Checks cleanup

* Make selector required by default

* Updates to the README.md

internal/configsource/envvarconfigsource/README.md

@@ -0,0 +1,80 @@
+# Environment Variable Config Source (Alpha)
+
+Use the environmental variable config source instead of direct references to
+environment variables in the config to inject YAML fragments or to have default
+values in case the selected environment variable is undefined. For simple environment
+variable expansion without support for YAML fragments or defaults see
+[Collector Configuration Environment Variables](https://opentelemetry.io/docs/collector/configuration/#configuration-environment-variables) 
+
+## Configuration
+
+Under the `config_sources:` use `env:` or `env/<name>:` to create an
+environment variable config source. The following parameters are available to
+customize environment variable config sources:
+
+```yaml
+config_sources:
+  env:
+    # defaults is used to create a set of fallbacks in case the original env var is
+    # undefined in the environment.
+    defaults:
+      MY_ENV_VAR: my env var value
+```
+
+By default, the config source will cause an error if it tries to inject an environment variable
+that is not defined or not specified on the `defaults` section. That behavior can be controlled
+via the `optional` parameters when invoking the config source, example:
+
+```yaml
+config_sources:
+  env:
+    defaults:
+      BACKED_BY_DEFAULTS_ENV_VAR: some_value
+
+components:
+  component_0:
+    # Not an error if ENV_VAR_NAME is undefined since 'optional' is set to true,
+    # the resulting value is "/data/token".
+    not_required_field: ${env:ENV_VAR_NAME?optional=true}/data/token 
+
+  component_1:
+    # It will be an error if ENV_VAR_NAME is undefined, the config will fail to load.
+    required_field: ${env:ENV_VAR_NAME}/data/token 
+
+  component_2:
+    # Not an error if BACKED_BY_DEFAULTS_ENV_VAR is undefined, because the 'defaults'
+    # of the config source.
+    required_field: ${env:BACKED_BY_DEFAULTS_ENV_VAR}/data/token 
+```
+
+## Injecting YAML Fragments
+
+The typical case to use the environment variable config source is when one wants
+to inject YAML fragments. The example below shows how this can be done on Linux and
+Windows.
+
+1. Define the environment variables:
+- Linux:
+```terminal
+export OTLP_PROTOCOLS="{ grpc: , http: , }"
+export JAEGER_PROTOCOLS="{ protocols: { grpc: , thrift_binary: , thrift_compact: , thrift_http: , } }"
+```
+- Windows:
+```terminal
+set OTLP_PROTOCOLS={ grpc: , http: , }
+set JAEGER_PROTOCOLS={ protocols: { grpc: , thrift_binary: , thrift_compact: , thrift_http: , } }
+```
+
+2. Use the environment variables on the configuration:
+```yaml
+config_sources:
+  env:
+
+receivers:
+  jaeger:
+    ${env:JAEGER_PROTOCOLS}
+  otlp:
+    protocols:
+      ${env:OTLP_PROTOCOLS}
+...
+```

internal/configsource/envvarconfigsource/config.go

@@ -0,0 +1,27 @@
+// Copyright Splunk, Inc.
+// Copyright The OpenTelemetry Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package envvarconfigsource
+
+import (
+	"github.com/signalfx/splunk-otel-collector/internal/configprovider"
+)
+
+// Config holds the configuration for the creation of environment variable config source objects.
+type Config struct {
+	*configprovider.Settings
+	// Defaults specify a map to fallback if a given environment variable is not defined.
+	Defaults map[string]interface{} `mapstructure:"defaults"`
+}

internal/configsource/envvarconfigsource/config_test.go

@@ -0,0 +1,71 @@
+// Copyright Splunk, Inc.
+// Copyright The OpenTelemetry Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package envvarconfigsource
+
+import (
+	"context"
+	"path"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+	"go.opentelemetry.io/collector/config"
+	"go.uber.org/zap"
+
+	"github.com/signalfx/splunk-otel-collector/internal/configprovider"
+)
+
+func TestEnvVarConfigSourceLoadConfig(t *testing.T) {
+	fileName := path.Join("testdata", "config.yaml")
+	v, err := config.NewParserFromFile(fileName)
+	require.NoError(t, err)
+
+	factories := map[config.Type]configprovider.Factory{
+		typeStr: NewFactory(),
+	}
+
+	actualSettings, err := configprovider.Load(context.Background(), v, factories)
+	require.NoError(t, err)
+
+	expectedSettings := map[string]configprovider.ConfigSettings{
+		"env": &Config{
+			Settings: &configprovider.Settings{
+				TypeVal: "env",
+				NameVal: "env",
+			},
+		},
+		"env/with_fallback": &Config{
+			Settings: &configprovider.Settings{
+				TypeVal: "env",
+				NameVal: "env/with_fallback",
+			},
+			Defaults: map[string]interface{}{
+				"k0": 42,
+				"m0": map[string]interface{}{
+					"k0": "v0",
+					"k1": "v1",
+				},
+			},
+		},
+	}
+
+	require.Equal(t, expectedSettings, actualSettings)
+
+	params := configprovider.CreateParams{
+		Logger: zap.NewNop(),
+	}
+	_, err = configprovider.Build(context.Background(), actualSettings, params, factories)
+	require.NoError(t, err)
+}

internal/configsource/envvarconfigsource/configsource.go

@@ -0,0 +1,43 @@
+// Copyright Splunk, Inc.
+// Copyright The OpenTelemetry Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package envvarconfigsource
+
+import (
+	"context"
+
+	"go.opentelemetry.io/collector/config/experimental/configsource"
+	"go.uber.org/zap"
+)
+
+type envVarConfigSource struct {
+	defaults map[string]interface{}
+}
+
+var _ configsource.ConfigSource = (*envVarConfigSource)(nil)
+
+func (e *envVarConfigSource) NewSession(context.Context) (configsource.Session, error) {
+	return newSession(e.defaults)
+}
+
+func newConfigSource(_ *zap.Logger, cfg *Config) (*envVarConfigSource, error) {
+	defaults := make(map[string]interface{})
+	if cfg.Defaults != nil {
+		defaults = cfg.Defaults
+	}
+	return &envVarConfigSource{
+		defaults: defaults,
+	}, nil
+}

internal/configsource/envvarconfigsource/configsource_test.go

@@ -0,0 +1,97 @@
+// Copyright Splunk, Inc.
+// Copyright The OpenTelemetry Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package envvarconfigsource
+
+import (
+	"context"
+	"os"
+	"path"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"go.opentelemetry.io/collector/component"
+	"go.opentelemetry.io/collector/config"
+	"go.uber.org/zap"
+
+	"github.com/signalfx/splunk-otel-collector/internal/configprovider"
+)
+
+func TestEnvVarConfigSourceNew(t *testing.T) {
+	tests := []struct {
+		config *Config
+		name   string
+	}{
+		{
+			name:   "minimal",
+			config: &Config{},
+		},
+		{
+			name: "with_defaults",
+			config: &Config{
+				Defaults: map[string]interface{}{
+					"k0": "v0",
+				},
+			},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			cfgSrc, err := newConfigSource(zap.NewNop(), tt.config)
+			require.NoError(t, err)
+			require.NotNil(t, cfgSrc)
+			require.NotNil(t, cfgSrc.defaults)
+		})
+	}
+}
+
+func TestEnvVarConfigSource_End2End(t *testing.T) {
+	require.NoError(t, os.Setenv("_TEST_ENV_VAR_CFG_SRC", "test_env_var"))
+	defer func() {
+		assert.NoError(t, os.Unsetenv("_TEST_ENV_VAR_CFG_SRC"))
+	}()
+
+	file := path.Join("testdata", "env_config_source_end_2_end.yaml")
+	p, err := config.NewParserFromFile(file)
+	require.NoError(t, err)
+	require.NotNil(t, p)
+
+	factories := configprovider.Factories{
+		"env": NewFactory(),
+	}
+	m, err := configprovider.NewManager(p, zap.NewNop(), component.DefaultApplicationStartInfo(), factories)
+	require.NoError(t, err)
+	require.NotNil(t, m)
+
+	ctx := context.Background()
+	r, err := m.Resolve(ctx, p)
+	require.NoError(t, err)
+	require.NotNil(t, r)
+
+	go func() {
+		_ = m.WatchForUpdate()
+	}()
+	m.WaitForWatcher()
+
+	assert.NoError(t, m.Close(ctx))
+
+	file = path.Join("testdata", "env_config_source_end_2_end_expected.yaml")
+	expected, err := config.NewParserFromFile(file)
+	require.NoError(t, err)
+	require.NotNil(t, expected)
+
+	assert.Equal(t, expected.ToStringMap(), r.ToStringMap())
+}

internal/configsource/envvarconfigsource/factory.go

@@ -0,0 +1,51 @@
+// Copyright Splunk, Inc.
+// Copyright The OpenTelemetry Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//       http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package envvarconfigsource
+
+import (
+	"context"
+
+	"go.opentelemetry.io/collector/config"
+	"go.opentelemetry.io/collector/config/experimental/configsource"
+
+	"github.com/signalfx/splunk-otel-collector/internal/configprovider"
+)
+
+const (
+	// The "type" of environment variable config sources in configuration.
+	typeStr = "env"
+)
+
+type envVarFactory struct{}
+
+func (e *envVarFactory) Type() config.Type {
+	return typeStr
+}
+
+func (e *envVarFactory) CreateDefaultConfig() configprovider.ConfigSettings {
+	return &Config{
+		Settings: configprovider.NewSettings(typeStr),
+	}
+}
+
+func (e *envVarFactory) CreateConfigSource(_ context.Context, params configprovider.CreateParams, cfg configprovider.ConfigSettings) (configsource.ConfigSource, error) {
+	return newConfigSource(params.Logger, cfg.(*Config))
+}
+
+// NewFactory creates a factory for Vault ConfigSource objects.
+func NewFactory() configprovider.Factory {
+	return &envVarFactory{}
+}