diff --git a/internal/configsource/envvarconfigsource/README.md b/internal/configsource/envvarconfigsource/README.md new file mode 100644 index 0000000000..78eab4b39c --- /dev/null +++ b/internal/configsource/envvarconfigsource/README.md @@ -0,0 +1,80 @@ +# Environment Variable Config Source (Alpha) + +Use the environmental variable config source instead of direct references to +environment variables in the config to inject YAML fragments or to have default +values in case the selected environment variable is undefined. For simple environment +variable expansion without support for YAML fragments or defaults see +[Collector Configuration Environment Variables](https://opentelemetry.io/docs/collector/configuration/#configuration-environment-variables) + +## Configuration + +Under the `config_sources:` use `env:` or `env/<name>:` to create an +environment variable config source. The following parameters are available to +customize environment variable config sources: + +```yaml +config_sources: + env: + # defaults is used to create a set of fallbacks in case the original env var is + # undefined in the environment. + defaults: + MY_ENV_VAR: my env var value +``` + +By default, the config source will cause an error if it tries to inject an environment variable +that is not defined or not specified on the `defaults` section. That behavior can be controlled +via the `optional` parameters when invoking the config source, example: + +```yaml +config_sources: + env: + defaults: + BACKED_BY_DEFAULTS_ENV_VAR: some_value + +components: + component_0: + # Not an error if ENV_VAR_NAME is undefined since 'optional' is set to true, + # the resulting value is "/data/token". + not_required_field: ${env:ENV_VAR_NAME?optional=true}/data/token + + component_1: + # It will be an error if ENV_VAR_NAME is undefined, the config will fail to load. + required_field: ${env:ENV_VAR_NAME}/data/token + + component_2: + # Not an error if BACKED_BY_DEFAULTS_ENV_VAR is undefined, because the 'defaults' + # of the config source. + required_field: ${env:BACKED_BY_DEFAULTS_ENV_VAR}/data/token +``` + +## Injecting YAML Fragments + +The typical case to use the environment variable config source is when one wants +to inject YAML fragments. The example below shows how this can be done on Linux and +Windows. + +1. Define the environment variables: +- Linux: +```terminal +export OTLP_PROTOCOLS="{ grpc: , http: , }" +export JAEGER_PROTOCOLS="{ protocols: { grpc: , thrift_binary: , thrift_compact: , thrift_http: , } }" +``` +- Windows: +```terminal +set OTLP_PROTOCOLS={ grpc: , http: , } +set JAEGER_PROTOCOLS={ protocols: { grpc: , thrift_binary: , thrift_compact: , thrift_http: , } } +``` + +2. Use the environment variables on the configuration: +```yaml +config_sources: + env: + +receivers: + jaeger: + ${env:JAEGER_PROTOCOLS} + otlp: + protocols: + ${env:OTLP_PROTOCOLS} +... +``` \ No newline at end of file diff --git a/internal/configsource/envvarconfigsource/config.go b/internal/configsource/envvarconfigsource/config.go new file mode 100644 index 0000000000..d4abc0fc47 --- /dev/null +++ b/internal/configsource/envvarconfigsource/config.go @@ -0,0 +1,27 @@ +// Copyright Splunk, Inc. +// Copyright The OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package envvarconfigsource + +import ( + "github.com/signalfx/splunk-otel-collector/internal/configprovider" +) + +// Config holds the configuration for the creation of environment variable config source objects. +type Config struct { + *configprovider.Settings + // Defaults specify a map to fallback if a given environment variable is not defined. + Defaults map[string]interface{} `mapstructure:"defaults"` +} diff --git a/internal/configsource/envvarconfigsource/config_test.go b/internal/configsource/envvarconfigsource/config_test.go new file mode 100644 index 0000000000..b611bbca49 --- /dev/null +++ b/internal/configsource/envvarconfigsource/config_test.go @@ -0,0 +1,71 @@ +// Copyright Splunk, Inc. +// Copyright The OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package envvarconfigsource + +import ( + "context" + "path" + "testing" + + "github.com/stretchr/testify/require" + "go.opentelemetry.io/collector/config" + "go.uber.org/zap" + + "github.com/signalfx/splunk-otel-collector/internal/configprovider" +) + +func TestEnvVarConfigSourceLoadConfig(t *testing.T) { + fileName := path.Join("testdata", "config.yaml") + v, err := config.NewParserFromFile(fileName) + require.NoError(t, err) + + factories := map[config.Type]configprovider.Factory{ + typeStr: NewFactory(), + } + + actualSettings, err := configprovider.Load(context.Background(), v, factories) + require.NoError(t, err) + + expectedSettings := map[string]configprovider.ConfigSettings{ + "env": &Config{ + Settings: &configprovider.Settings{ + TypeVal: "env", + NameVal: "env", + }, + }, + "env/with_fallback": &Config{ + Settings: &configprovider.Settings{ + TypeVal: "env", + NameVal: "env/with_fallback", + }, + Defaults: map[string]interface{}{ + "k0": 42, + "m0": map[string]interface{}{ + "k0": "v0", + "k1": "v1", + }, + }, + }, + } + + require.Equal(t, expectedSettings, actualSettings) + + params := configprovider.CreateParams{ + Logger: zap.NewNop(), + } + _, err = configprovider.Build(context.Background(), actualSettings, params, factories) + require.NoError(t, err) +} diff --git a/internal/configsource/envvarconfigsource/configsource.go b/internal/configsource/envvarconfigsource/configsource.go new file mode 100644 index 0000000000..ebb88ebf60 --- /dev/null +++ b/internal/configsource/envvarconfigsource/configsource.go @@ -0,0 +1,43 @@ +// Copyright Splunk, Inc. +// Copyright The OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package envvarconfigsource + +import ( + "context" + + "go.opentelemetry.io/collector/config/experimental/configsource" + "go.uber.org/zap" +) + +type envVarConfigSource struct { + defaults map[string]interface{} +} + +var _ configsource.ConfigSource = (*envVarConfigSource)(nil) + +func (e *envVarConfigSource) NewSession(context.Context) (configsource.Session, error) { + return newSession(e.defaults) +} + +func newConfigSource(_ *zap.Logger, cfg *Config) (*envVarConfigSource, error) { + defaults := make(map[string]interface{}) + if cfg.Defaults != nil { + defaults = cfg.Defaults + } + return &envVarConfigSource{ + defaults: defaults, + }, nil +} diff --git a/internal/configsource/envvarconfigsource/configsource_test.go b/internal/configsource/envvarconfigsource/configsource_test.go new file mode 100644 index 0000000000..5af24b9c1b --- /dev/null +++ b/internal/configsource/envvarconfigsource/configsource_test.go @@ -0,0 +1,97 @@ +// Copyright Splunk, Inc. +// Copyright The OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package envvarconfigsource + +import ( + "context" + "os" + "path" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.opentelemetry.io/collector/component" + "go.opentelemetry.io/collector/config" + "go.uber.org/zap" + + "github.com/signalfx/splunk-otel-collector/internal/configprovider" +) + +func TestEnvVarConfigSourceNew(t *testing.T) { + tests := []struct { + config *Config + name string + }{ + { + name: "minimal", + config: &Config{}, + }, + { + name: "with_defaults", + config: &Config{ + Defaults: map[string]interface{}{ + "k0": "v0", + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cfgSrc, err := newConfigSource(zap.NewNop(), tt.config) + require.NoError(t, err) + require.NotNil(t, cfgSrc) + require.NotNil(t, cfgSrc.defaults) + }) + } +} + +func TestEnvVarConfigSource_End2End(t *testing.T) { + require.NoError(t, os.Setenv("_TEST_ENV_VAR_CFG_SRC", "test_env_var")) + defer func() { + assert.NoError(t, os.Unsetenv("_TEST_ENV_VAR_CFG_SRC")) + }() + + file := path.Join("testdata", "env_config_source_end_2_end.yaml") + p, err := config.NewParserFromFile(file) + require.NoError(t, err) + require.NotNil(t, p) + + factories := configprovider.Factories{ + "env": NewFactory(), + } + m, err := configprovider.NewManager(p, zap.NewNop(), component.DefaultApplicationStartInfo(), factories) + require.NoError(t, err) + require.NotNil(t, m) + + ctx := context.Background() + r, err := m.Resolve(ctx, p) + require.NoError(t, err) + require.NotNil(t, r) + + go func() { + _ = m.WatchForUpdate() + }() + m.WaitForWatcher() + + assert.NoError(t, m.Close(ctx)) + + file = path.Join("testdata", "env_config_source_end_2_end_expected.yaml") + expected, err := config.NewParserFromFile(file) + require.NoError(t, err) + require.NotNil(t, expected) + + assert.Equal(t, expected.ToStringMap(), r.ToStringMap()) +} diff --git a/internal/configsource/envvarconfigsource/factory.go b/internal/configsource/envvarconfigsource/factory.go new file mode 100644 index 0000000000..48740abbc9 --- /dev/null +++ b/internal/configsource/envvarconfigsource/factory.go @@ -0,0 +1,51 @@ +// Copyright Splunk, Inc. +// Copyright The OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package envvarconfigsource + +import ( + "context" + + "go.opentelemetry.io/collector/config" + "go.opentelemetry.io/collector/config/experimental/configsource" + + "github.com/signalfx/splunk-otel-collector/internal/configprovider" +) + +const ( + // The "type" of environment variable config sources in configuration. + typeStr = "env" +) + +type envVarFactory struct{} + +func (e *envVarFactory) Type() config.Type { + return typeStr +} + +func (e *envVarFactory) CreateDefaultConfig() configprovider.ConfigSettings { + return &Config{ + Settings: configprovider.NewSettings(typeStr), + } +} + +func (e *envVarFactory) CreateConfigSource(_ context.Context, params configprovider.CreateParams, cfg configprovider.ConfigSettings) (configsource.ConfigSource, error) { + return newConfigSource(params.Logger, cfg.(*Config)) +} + +// NewFactory creates a factory for Vault ConfigSource objects. +func NewFactory() configprovider.Factory { + return &envVarFactory{} +} diff --git a/internal/configsource/envvarconfigsource/factory_test.go b/internal/configsource/envvarconfigsource/factory_test.go new file mode 100644 index 0000000000..d0b00458e6 --- /dev/null +++ b/internal/configsource/envvarconfigsource/factory_test.go @@ -0,0 +1,59 @@ +// Copyright Splunk, Inc. +// Copyright The OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package envvarconfigsource + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + "go.opentelemetry.io/collector/config" + "go.uber.org/zap" + + "github.com/signalfx/splunk-otel-collector/internal/configprovider" +) + +func TestEnvVarConfigSourceFactory_CreateConfigSource(t *testing.T) { + factory := NewFactory() + assert.Equal(t, config.Type("env"), factory.Type()) + createParams := configprovider.CreateParams{ + Logger: zap.NewNop(), + } + tests := []struct { + config *Config + name string + }{ + { + name: "no_defaults", + config: &Config{}, + }, + { + name: "with_defaults", + config: &Config{ + Defaults: map[string]interface{}{ + "k0": "v0", + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + actual, err := factory.CreateConfigSource(context.Background(), createParams, tt.config) + assert.NoError(t, err) + assert.NotNil(t, actual) + }) + } +} diff --git a/internal/configsource/envvarconfigsource/session.go b/internal/configsource/envvarconfigsource/session.go new file mode 100644 index 0000000000..81dc86e05b --- /dev/null +++ b/internal/configsource/envvarconfigsource/session.go @@ -0,0 +1,91 @@ +// Copyright Splunk, Inc. +// Copyright The OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package envvarconfigsource + +import ( + "context" + "fmt" + "os" + + "github.com/spf13/cast" + "go.opentelemetry.io/collector/config" + "go.opentelemetry.io/collector/config/experimental/configsource" + + "github.com/signalfx/splunk-otel-collector/internal/configprovider" +) + +// Private error types to help with testability. +type ( + errInvalidRetrieveParams struct{ error } + errMissingRequiredEnvVar struct{ error } +) + +type retrieveParams struct { + // Optional is used to change the default behavior when an environment variable + // requested via the config source is not defined. By default the value of this + // field is 'false' which will cause an error if the specified environment variable + // is not defined. Set it to 'true' to ignore not defined environment variables. + Optional bool `mapstructure:"optional"` +} + +// envVarSession implements the configsource.Session interface. +type envVarSession struct { + defaults map[string]interface{} +} + +var _ configsource.Session = (*envVarSession)(nil) + +func (e *envVarSession) Retrieve(_ context.Context, selector string, params interface{}) (configsource.Retrieved, error) { + actualParams := retrieveParams{} + if params != nil { + paramsParser := config.NewParserFromStringMap(cast.ToStringMap(params)) + if err := paramsParser.UnmarshalExact(&actualParams); err != nil { + return nil, &errInvalidRetrieveParams{fmt.Errorf("failed to unmarshall retrieve params: %w", err)} + } + } + + value, ok := os.LookupEnv(selector) + if ok { + // Environment variable found, everything is done. + return configprovider.NewRetrieved(value, configprovider.WatcherNotSupported), nil + } + + defaultValue, ok := e.defaults[selector] + if !ok { + if !actualParams.Optional { + return nil, &errMissingRequiredEnvVar{fmt.Errorf("env var %q is required but not defined and not present on defaults", selector)} + } + + // To keep with default behavior for env vars not defined set the value to empty string + defaultValue = "" + } + + return configprovider.NewRetrieved(defaultValue, configprovider.WatcherNotSupported), nil +} + +func (e *envVarSession) RetrieveEnd(context.Context) error { + return nil +} + +func (e *envVarSession) Close(context.Context) error { + return nil +} + +func newSession(defaults map[string]interface{}) (*envVarSession, error) { + return &envVarSession{ + defaults: defaults, + }, nil +} diff --git a/internal/configsource/envvarconfigsource/session_test.go b/internal/configsource/envvarconfigsource/session_test.go new file mode 100644 index 0000000000..960fde06f8 --- /dev/null +++ b/internal/configsource/envvarconfigsource/session_test.go @@ -0,0 +1,110 @@ +// Copyright Splunk, Inc. +// Copyright The OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package envvarconfigsource + +import ( + "context" + "os" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.opentelemetry.io/collector/config/experimental/configsource" +) + +func TestEnvVarConfigSource_Session(t *testing.T) { + const testEnvVarName = "_TEST_ENV_VAR_CFG_SRC" + const testEnvVarValue = "test_env_value" + + tests := []struct { + defaults map[string]interface{} + params map[string]interface{} + expected interface{} + wantErr error + name string + selector string + }{ + { + name: "simple", + selector: testEnvVarName, + expected: testEnvVarValue, + }, + { + name: "missing_not_required", + selector: "UNDEFINED_ENV_VAR", + params: map[string]interface{}{ + "optional": true, + }, + expected: "", // The default behavior for undefined env var is empty string. + }, + { + name: "invalid_param", + params: map[string]interface{}{ + "unknow_params_field": true, + }, + wantErr: &errInvalidRetrieveParams{}, + }, + { + name: "missing_required", + selector: "UNDEFINED_ENV_VAR", + wantErr: &errMissingRequiredEnvVar{}, + }, + { + name: "required_on_defaults", + defaults: map[string]interface{}{ + "FALLBACK_ENV_VAR": "fallback_env_var", + }, + selector: "FALLBACK_ENV_VAR", + expected: "fallback_env_var", + }, + } + + require.NoError(t, os.Setenv(testEnvVarName, testEnvVarValue)) + t.Cleanup(func() { + assert.NoError(t, os.Unsetenv(testEnvVarName)) + }) + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + defaults := tt.defaults + if defaults == nil { + defaults = make(map[string]interface{}) + } + + s, err := newSession(defaults) + require.NoError(t, err) + require.NotNil(t, s) + + ctx := context.Background() + defer func() { + assert.NoError(t, s.RetrieveEnd(ctx)) + assert.NoError(t, s.Close(ctx)) + }() + + r, err := s.Retrieve(ctx, tt.selector, tt.params) + if tt.wantErr != nil { + assert.Nil(t, r) + require.IsType(t, tt.wantErr, err) + return + } + + require.NoError(t, err) + require.NotNil(t, r) + assert.Equal(t, tt.expected, r.Value()) + assert.Equal(t, configsource.ErrWatcherNotSupported, r.WatchForUpdate()) + }) + } +} diff --git a/internal/configsource/envvarconfigsource/testdata/config.yaml b/internal/configsource/envvarconfigsource/testdata/config.yaml new file mode 100644 index 0000000000..c83973bbda --- /dev/null +++ b/internal/configsource/envvarconfigsource/testdata/config.yaml @@ -0,0 +1,10 @@ +config_sources: + # First environment config source doesn't have any defaults to fallback. + env: + # An environment config source with defaults to fallback. + env/with_fallback: + defaults: + k0: 42 + m0: + k0: v0 + k1: v1 diff --git a/internal/configsource/envvarconfigsource/testdata/env_config_source_end_2_end.yaml b/internal/configsource/envvarconfigsource/testdata/env_config_source_end_2_end.yaml new file mode 100644 index 0000000000..7e36120ae1 --- /dev/null +++ b/internal/configsource/envvarconfigsource/testdata/env_config_source_end_2_end.yaml @@ -0,0 +1,13 @@ +config_sources: + env: + defaults: + k0: 42 + cfg: + k0: a string value + k1: true + +config: + from_defined_env_var: ${env:_TEST_ENV_VAR_CFG_SRC} + from_undefined_env_var: ${env:_UNDEFINED_ENV_VAR?optional=true}/some/path + field_from_default: $env:k0 + map_from_default: $env:cfg?optional=true diff --git a/internal/configsource/envvarconfigsource/testdata/env_config_source_end_2_end_expected.yaml b/internal/configsource/envvarconfigsource/testdata/env_config_source_end_2_end_expected.yaml new file mode 100644 index 0000000000..b0442033ea --- /dev/null +++ b/internal/configsource/envvarconfigsource/testdata/env_config_source_end_2_end_expected.yaml @@ -0,0 +1,8 @@ +config: + from_defined_env_var: test_env_var + from_undefined_env_var: /some/path + field_from_default: 42 + map_from_default: + k0: a string value + k1: true + diff --git a/internal/configsources/configsources.go b/internal/configsources/configsources.go index 255fbf6111..ceb816ad4d 100644 --- a/internal/configsources/configsources.go +++ b/internal/configsources/configsources.go @@ -18,6 +18,7 @@ package configsources import ( "github.com/signalfx/splunk-otel-collector/internal/configprovider" + "github.com/signalfx/splunk-otel-collector/internal/configsource/envvarconfigsource" "github.com/signalfx/splunk-otel-collector/internal/configsource/etcd2configsource" "github.com/signalfx/splunk-otel-collector/internal/configsource/vaultconfigsource" "github.com/signalfx/splunk-otel-collector/internal/configsource/zookeeperconfigsource" @@ -26,6 +27,7 @@ import ( // Get returns the factories to all config sources available to the user. func Get() []configprovider.Factory { return []configprovider.Factory{ + envvarconfigsource.NewFactory(), etcd2configsource.NewFactory(), vaultconfigsource.NewFactory(), zookeeperconfigsource.NewFactory(), diff --git a/internal/configsources/configsources_test.go b/internal/configsources/configsources_test.go index cacb12a5db..81f39eff10 100644 --- a/internal/configsources/configsources_test.go +++ b/internal/configsources/configsources_test.go @@ -29,6 +29,7 @@ func TestConfigSourcesGet(t *testing.T) { tests := []struct { configSourceType config.Type }{ + {"env"}, {"etcd2"}, {"vault"}, {"zookeeper"},