Skip to content

Commit

Permalink
Introduce global and cascading configuration options (#97)
Browse files Browse the repository at this point in the history 
This introduces the concept of global configurations and cascades struct-level configuration options onto all child fields. For example, marking a struct as required will mark all child fields as required. Similarly, setting a custom delimiter on a struct tag propagates that delimiter to all child fields of the struct.
  • Loading branch information
@sethvargo
sethvargo authored Dec 20, 2023
1 parent d42f4a2 commit 9dce086
Show file tree
Hide file tree
Showing 4 changed files with 269 additions and 62 deletions.
49 changes: 23 additions & 26 deletions README.md
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
# Envconfig

[![GoDoc](https://img.shields.io/badge/go-documentation-blue.svg?style=flat-square)](https://pkg.go.dev/mod/github.com/sethvargo/go-envconfig)
[![GoDoc](https://img.shields.io/badge/go-documentation-blue.svg?style=flat-square)][godoc]

Envconfig populates struct field values based on environment variables or
arbitrary lookup functions. It supports pre-setting mutations, which is useful
Expand Down Expand Up @@ -289,8 +289,9 @@ type MyStruct struct {
```go
// Process variables, but look for the "APP_" prefix.
l := envconfig.PrefixLookuper("APP_", envconfig.OsLookuper())
if err := envconfig.ProcessWith(ctx, &c, l); err != nil {
if err := envconfig.ProcessWith(ctx, &c, &envconfig.Config{
Lookuper: envconfig.PrefixLookuper("APP_", envconfig.OsLookuper()),
}); err != nil {
panic(err)
}
```
Expand Down Expand Up @@ -372,7 +373,10 @@ func resolveSecretFunc(ctx context.Context, originalKey, resolvedKey, originalVa
}

var config Config
envconfig.ProcessWith(ctx, &config, envconfig.OsLookuper(), resolveSecretFunc)
envconfig.ProcessWith(ctx, &config, &envconfig.Config{
Lookuper: envconfig.OsLookuper(),
Mutators: []envconfig.Mutator{resolveSecretFunc},
})
```
Mutators are like middleware, and they have access to the initial and current
Expand Down Expand Up @@ -438,11 +442,13 @@ type Config struct {
}

var config Config
lookuper := envconfig.PrefixLookuper("REDIS_", envconfig.MapLookuper(map[string]string{
"PASSWORD": "original",
}))
mutators := []envconfig.Mutators{mutatorFunc1, mutatorFunc2, mutatorFunc3}
envconfig.ProcessWith(ctx, &config, lookuper, mutators...)
envconfig.ProcessWith(ctx, &config, &envconfig.Config{
Lookuper: envconfig.PrefixLookuper("REDIS_", envconfig.MapLookuper(map[string]string{
"PASSWORD": "original",
})),
Mutators: mutators,
})

func mutatorFunc1(ctx context.Context, originalKey, resolvedKey, originalValue, currentValue string) (string, bool, error) {
// originalKey is "PASSWORD"
Expand All @@ -467,6 +473,11 @@ func mutatorFunc3(ctx context.Context, originalKey, resolvedKey, originalValue,
```
## Advanced Processing
See the [godoc][] for examples.
## Testing
Relying on the environment in tests can be troublesome because environment
Expand All @@ -480,7 +491,9 @@ lookuper := envconfig.MapLookuper(map[string]string{
})

var config Config
envconfig.ProcessWith(ctx, &config, lookuper)
envconfig.ProcessWith(ctx, &config, &envconfig.Config{
Lookuper: lookuper,
})
```
Now you can parallelize all your tests by providing a map for the lookup
Expand All @@ -491,20 +504,4 @@ You can also combine multiple lookupers with `MultiLookuper`. See the GoDoc for
more information and examples.
## Inspiration
This library is conceptually similar to [kelseyhightower/envconfig](https://github.com/kelseyhightower/envconfig), with the following
major behavioral differences:
- Adds support for specifying a custom lookup function (such as a map), which
is useful for testing.
- Only populates fields if they contain zero or nil values if `overwrite` is
unset. This means you can pre-initialize a struct and any pre-populated
fields will not be overwritten during processing.
- Support for interpolation. The default value for a field can be the value of
another field.
- Support for arbitrary mutators that change/resolve data before type
conversion.
[godoc]: https://pkg.go.dev/mod/github.com/sethvargo/go-envconfig
161 changes: 126 additions & 35 deletions envconfig.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -89,9 +89,6 @@ const (
optPrefix = "prefix="
optRequired = "required"
optSeparator = "separator="

defaultDelimiter = ","
defaultSeparator = ":"
)

// Error is a custom error type for errors returned by envconfig.
Expand Down Expand Up @@ -230,21 +227,76 @@ type options struct {
Required bool
}

// Config represent inputs to the envconfig decoding.
type Config struct {
// Target is the destination structure to decode. This value is required.
Target any

// Lookuper is the lookuper implementation to use. If not provided, it
// defaults to the OS Lookuper.
Lookuper Lookuper

// DefaultDelimiter is the default value to use for the delimiter in maps and
// slices. This can be overridden on a per-field basis, which takes
// precedence. The default value is ",".
DefaultDelimiter string

// DefaultSeparator is the default value to use for the separator in maps.
// This can be overridden on a per-field basis, which takes precedence. The
// default value is ":".
DefaultSeparator string

// DefaultNoInit is the default value for skipping initialization of
// unprovided fields. The default value is false (deeply initialize all
// fields and nested structs).
DefaultNoInit bool

// DefaultOverwrite is the default value for overwriting an existing value set
// on the struct before processing. The default value is false.
DefaultOverwrite bool

// DefaultRequired is the default value for marking a field as required. The
// default value is false.
DefaultRequired bool

// Mutators is an optiona list of mutators to apply to lookups.
Mutators []Mutator
}

// Process processes the struct using the environment. See [ProcessWith] for a
// more customizable version.
func Process(ctx context.Context, i any, mus ...Mutator) error {
return ProcessWith(ctx, i, OsLookuper(), mus...)
return ProcessWith(ctx, &Config{
Target: i,
Mutators: mus,
})
}

// ProcessWith processes the given interface with the given lookuper. See the
// package-level documentation for specific examples and behaviors.
func ProcessWith(ctx context.Context, i any, l Lookuper, mus ...Mutator) error {
return processWith(ctx, i, l, false, mus...)
func ProcessWith(ctx context.Context, c *Config) error {
if c == nil {
c = new(Config)
}

// Deep copy the slice and remove any nil functions.
var mus []Mutator
for _, m := range c.Mutators {
if m != nil {
mus = append(mus, m)
}
}
c.Mutators = mus

return processWith(ctx, c)
}

// processWith is a helper that captures whether the parent wanted
// initialization.
func processWith(ctx context.Context, i any, l Lookuper, parentNoInit bool, mus ...Mutator) error {
func processWith(ctx context.Context, c *Config) error {
i := c.Target

l := c.Lookuper
if l == nil {
return ErrLookuperNil
}
Expand All @@ -261,6 +313,23 @@ func processWith(ctx context.Context, i any, l Lookuper, parentNoInit bool, mus

t := e.Type()

structDelimiter := c.DefaultDelimiter
if structDelimiter == "" {
structDelimiter = ","
}

structNoInit := c.DefaultNoInit

structSeparator := c.DefaultSeparator
if structSeparator == "" {
structSeparator = ":"
}

structOverwrite := c.DefaultOverwrite
structRequired := c.DefaultRequired

mutators := c.Mutators

for i := 0; i < t.NumField(); i++ {
ef := e.Field(i)
tf := t.Field(i)
Expand Down Expand Up @@ -291,7 +360,20 @@ func processWith(ctx context.Context, i any, l Lookuper, parentNoInit bool, mus
ef.Kind() != reflect.UnsafePointer {
return fmt.Errorf("%s: %w", tf.Name, ErrNoInitNotPtr)
}
shouldNotInit := opts.NoInit || parentNoInit

// Compute defaults from local tags.
delimiter := structDelimiter
if v := opts.Delimiter; v != "" {
delimiter = v
}
separator := structSeparator
if v := opts.Separator; v != "" {
separator = v
}

noInit := structNoInit || opts.NoInit
overwrite := structOverwrite || opts.Overwrite
required := structRequired || opts.Required

isNilStructPtr := false
setNilStruct := func(v reflect.Value) {
Expand All @@ -302,7 +384,7 @@ func processWith(ctx context.Context, i any, l Lookuper, parentNoInit bool, mus
// If a struct (after traversal) equals to the empty value, it means
// nothing was changed in any sub-fields. With the noinit opt, we skip
// setting the empty value to the original struct pointer (keep it nil).
if !reflect.DeepEqual(v.Interface(), empty) || !shouldNotInit {
if !reflect.DeepEqual(v.Interface(), empty) || !noInit {
origin.Set(v)
}
}
Expand Down Expand Up @@ -337,7 +419,7 @@ func processWith(ctx context.Context, i any, l Lookuper, parentNoInit bool, mus
// Lookup the value, ignoring an error if the key isn't defined. This is
// required for nested structs that don't declare their own `env` keys,
// but have internal fields with an `env` defined.
val, _, _, err := lookup(key, opts, l)
val, _, _, err := lookup(key, required, opts.Default, l)
if err != nil && !errors.Is(err, ErrMissingKey) {
return fmt.Errorf("%s: %w", tf.Name, err)
}
Expand All @@ -356,7 +438,16 @@ func processWith(ctx context.Context, i any, l Lookuper, parentNoInit bool, mus
plu = PrefixLookuper(opts.Prefix, l)
}

if err := processWith(ctx, ef.Interface(), plu, shouldNotInit, mus...); err != nil {
if err := processWith(ctx, &Config{
Target: ef.Interface(),
Lookuper: plu,
DefaultDelimiter: delimiter,
DefaultSeparator: separator,
DefaultNoInit: noInit,
DefaultOverwrite: overwrite,
DefaultRequired: required,
Mutators: mutators,
}); err != nil {
return fmt.Errorf("%s: %w", tf.Name, err)
}

Expand All @@ -377,11 +468,11 @@ func processWith(ctx context.Context, i any, l Lookuper, parentNoInit bool, mus

// The field already has a non-zero value and overwrite is false, do not
// overwrite.
if (pointerWasSet || !ef.IsZero()) && !opts.Overwrite {
if (pointerWasSet || !ef.IsZero()) && !overwrite {
continue
}

val, found, usedDefault, err := lookup(key, opts, l)
val, found, usedDefault, err := lookup(key, required, opts.Default, l)
if err != nil {
return fmt.Errorf("%s: %w", tf.Name, err)
}
Expand All @@ -405,11 +496,7 @@ func processWith(ctx context.Context, i any, l Lookuper, parentNoInit bool, mus
originalValue := val
stop := false

for _, mu := range mus {
if mu == nil {
continue
}

for _, mu := range mutators {
val, stop, err = mu.EnvMutate(ctx, originalKey, resolvedKey, originalValue, val)
if err != nil {
return fmt.Errorf("%s: %w", tf.Name, err)
Expand All @@ -420,29 +507,33 @@ func processWith(ctx context.Context, i any, l Lookuper, parentNoInit bool, mus
}
}

// If Delimiter is not defined set it to ","
if opts.Delimiter == "" {
opts.Delimiter = defaultDelimiter
}

// If Separator is not defined set it to ":"
if opts.Separator == "" {
opts.Separator = defaultSeparator
}

// Set value.
if err := processField(val, ef, opts.Delimiter, opts.Separator, opts.NoInit); err != nil {
if err := processField(val, ef, delimiter, separator, noInit); err != nil {
return fmt.Errorf("%s(%q): %w", tf.Name, val, err)
}
}

return nil
}

// SplitString splits the given string on the provided rune, unless the rune is
// escaped by the escape character.
func splitString(s string, on string, esc string) []string {
a := strings.Split(s, on)

for i := len(a) - 2; i >= 0; i-- {
if strings.HasSuffix(a[i], esc) {
a[i] = a[i][:len(a[i])-len(esc)] + on + a[i+1]
a = append(a[:i+1], a[i+2:]...)
}
}
return a
}

// keyAndOpts parses the given tag value (e.g. env:"foo,required") and
// returns the key name and options as a list.
func keyAndOpts(tag string) (string, *options, error) {
parts := strings.Split(tag, ",")
parts := splitString(tag, ",", "\\")
key, tagOpts := strings.TrimSpace(parts[0]), parts[1:]

if key != "" && !validateEnvName(key) {
Expand Down Expand Up @@ -485,34 +576,34 @@ LOOP:
// first boolean parameter indicates whether the value was found in the
// lookuper. The second boolean parameter indicates whether the default value
// was used.
func lookup(key string, opts *options, l Lookuper) (string, bool, bool, error) {
func lookup(key string, required bool, defaultValue string, l Lookuper) (string, bool, bool, error) {
if key == "" {
// The struct has something like `env:",required"`, which is likely a
// mistake. We could try to infer the envvar from the field name, but that
// feels too magical.
return "", false, false, ErrMissingKey
}

if opts.Required && opts.Default != "" {
if required && defaultValue != "" {
// Having a default value on a required value doesn't make sense.
return "", false, false, ErrRequiredAndDefault
}

// Lookup value.
val, found := l.Lookup(key)
if !found {
if opts.Required {
if required {
if keyer, ok := l.(KeyedLookuper); ok {
key = keyer.Key(key)
}

return "", false, false, fmt.Errorf("%w: %s", ErrMissingRequired, key)
}

if opts.Default != "" {
if defaultValue != "" {
// Expand the default value. This allows for a default value that maps to
// a different variable.
val = os.Expand(opts.Default, func(i string) string {
val = os.Expand(defaultValue, func(i string) string {
s, ok := l.Lookup(i)
if ok {
return s
Expand Down
Loading

0 comments on commit 9dce086

Please sign in to comment.