From d1721d7b7ee01d66e80c73d6ed64ec6267eda9bc Mon Sep 17 00:00:00 2001 From: Seth Vargo Date: Mon, 18 Dec 2023 15:56:35 -0500 Subject: [PATCH] Change MutatorFunc to be more flexible (#92) **:warning: BREAKING!** This changes the signature of the `MutatorFunc` to have more information about prior states. It will include the original environment variable names and values, as well as the currently resolved values. Additionally, the mutation chain can now be stopped without returning an error. --- README.md | 85 +++++++++++++++++++++++++++++++++++++++++++-- envconfig.go | 71 +++++++++++++++++++++++++++++++++----- envconfig_test.go | 88 +++++++++++++++++++++++++++++++++++++++++++++-- go.mod | 2 +- 4 files changed, 231 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index cbfa556..8e2919e 100644 --- a/README.md +++ b/README.md @@ -337,6 +337,8 @@ the tag on a different type will return an error. ## Extension +### Decoders + All built-in types are supported except `Func` and `Chan`. If you need to define a custom decoder, implement the `Decoder` interface: @@ -351,6 +353,8 @@ func (v *MyStruct) EnvDecode(val string) error { } ``` +### Mutators + If you need to modify environment variable values before processing, you can specify a custom `Mutator`: @@ -359,17 +363,92 @@ type Config struct { Password `env:"PASSWORD"` } -func resolveSecretFunc(ctx context.Context, key, value string) (string, error) { +func resolveSecretFunc(ctx context.Context, originalKey, resolvedKey, originalValue, resolvedValue string) (newValue string, stop bool, err error) { if strings.HasPrefix(value, "secret://") { - return secretmanager.Resolve(ctx, value) // example + v, err := secretmanager.Resolve(ctx, value) // example + if err != nil { + return resolvedValue, true, fmt.Errorf("failed to access secret: %w", err) + } + return v, false, nil } - return value, nil + return resolvedValue, false, nil } var config Config envconfig.ProcessWith(ctx, &config, envconfig.OsLookuper(), resolveSecretFunc) ``` +Mutator functions are like middleware, and they have access to the initial and +current state of the stack. Mutators only run when a value has been provided in +the environment. They execute _before_ any complex type processing, so all +inputs and outputs are strings. The parameters (in order) are: + +- `originalKey` is the unmodified environment variable name as it was defined + on the struct. + +- `resolvedKey` is the fully-resolved environment variable name, which may + include prefixes or modifications from processing. When there are no + modifications, this will be equivalent to `originalKey`. + +- `originalValue` is the unmodified environment variable's value before any + mutations were run. + +- `currentValue` is the currently-resolved value, which may have been modified + by previous mutators and may be modified by subsequent mutators in the + stack. + +The function returns (in order): + +- The new value to use in both future mutations and final processing. + +- A boolean which indicates whether future mutations in the stack should be + applied. + +- Any errors that occurred. + +> [!TIP] +> +> Users coming from the v0 series can wrap their mutator functions with +> `LegacyMutatorFunc` for an easier transition to this new syntax. + +Consider the following example to illustrate the difference between +`originalKey` and `resolvedKey`: + +```go +type Config struct { + Password `env:"PASSWORD"` +} + +var config Config +lookuper := envconfig.PrefixLookuper("REDIS_", envconfig.MapLookuper(map[string]string{ + "PASSWORD": "original", +})) +mutators := []envconfig.MutatorFunc{mutatorFunc1, mutatorFunc2, mutatorFunc3} +envconfig.ProcessWith(ctx, &config, lookuper, mutators...) + +func mutatorFunc1(ctx context.Context, originalKey, resolvedKey, originalValue, currentValue string) (string, bool, error) { + // originalKey is "PASSWORD" + // resolvedKey is "REDIS_PASSWORD" + // originalValue is "original" + // currentValue is "original" + return currentValue+"-modified", false, nil +} + +func mutatorFunc2(ctx context.Context, originalKey, resolvedKey, originalValue, currentValue string) (string, bool, error) { + // originalKey is "PASSWORD" + // resolvedKey is "REDIS_PASSWORD" + // originalValue is "original" + // currentValue is "original-modified" + return currentValue, true, nil +} + +func mutatorFunc3(ctx context.Context, originalKey, resolvedKey, originalValue, currentValue string) (string, bool, error) { + // This mutator will never run because mutatorFunc2 stopped the chain. + return "...", false, nil +} +``` + + ## Testing Relying on the environment in tests can be troublesome because environment diff --git a/envconfig.go b/envconfig.go index 2d52059..49eb26c 100644 --- a/envconfig.go +++ b/envconfig.go @@ -188,7 +188,11 @@ type prefixLookuper struct { } func (p *prefixLookuper) Lookup(key string) (string, bool) { - return p.l.Lookup(p.prefix + key) + return p.l.Lookup(p.Key(key)) +} + +func (p *prefixLookuper) Key(key string) string { + return p.prefix + key } // MultiLookuper wraps a collection of lookupers. It does not combine them, and @@ -197,6 +201,12 @@ func MultiLookuper(lookupers ...Lookuper) Lookuper { return &multiLookuper{ls: lookupers} } +// KeyedLookuper is an extension to the [Lookuper] interface that returns the +// underlying key (used by the [PrefixLookuper] or custom implementations). +type KeyedLookuper interface { + Key(key string) string +} + // Decoder is an interface that custom types/fields can implement to control how // decoding takes place. For example: // @@ -212,7 +222,37 @@ type Decoder interface { // MutatorFunc is a function that mutates a given value before it is passed // along for processing. This is useful if you want to mutate the environment // variable value before it's converted to the proper type. -type MutatorFunc func(ctx context.Context, k, v string) (string, error) +// +// - `originalKey` is the unmodified environment variable name as it was defined +// on the struct. +// +// - `resolvedKey` is the fully-resolved environment variable name, which may +// include prefixes or modifications from processing. When there are +// no modifications, this will be equivalent to `originalKey`. +// +// - `originalValue` is the unmodified environment variable's value before any +// mutations were run. +// +// - `currentValue` is the currently-resolved value, which may have been +// modified by previous mutators and may be modified in the future by +// subsequent mutators in the stack. +// +// It returns the new value, a boolean which indicates whether future mutations +// in the stack should be applied, and any errors that occurred. +type MutatorFunc func(ctx context.Context, originalKey, resolvedKey, originalValue, currentValue string) (newValue string, stop bool, err error) + +// LegacyMutatorFunc is a helper that eases the transition from the previous +// MutatorFunc signature. It wraps the previous-style mutator function and +// returns a new one. Since the former mutator function had less data, this is +// inherently lossy. +// +// DEPRECATED: Change type signatures to [MutatorFunc] instead. +func LegacyMutatorFunc(fn func(ctx context.Context, key, value string) (string, error)) MutatorFunc { + return func(ctx context.Context, originalKey, resolvedKey, originalValue, currentValue string) (newValue string, stop bool, err error) { + v, err := fn(ctx, originalKey, currentValue) + return v, true, err + } +} // options are internal options for decoding. type options struct { @@ -414,12 +454,25 @@ func processWith(ctx context.Context, i interface{}, l Lookuper, parentNoInit bo // type conversions. They always resolve to a string (or error), so we don't // call mutators when the environment variable was not set. if found || usedDefault { + originalKey := key + resolvedKey := originalKey + if keyer, ok := l.(KeyedLookuper); ok { + resolvedKey = keyer.Key(resolvedKey) + } + originalValue := val + stop := false + for _, fn := range fns { - if fn != nil { - val, err = fn(ctx, key, val) - if err != nil { - return fmt.Errorf("%s: %w", tf.Name, err) - } + if fn == nil { + continue + } + + val, stop, err = fn(ctx, originalKey, resolvedKey, originalValue, val) + if err != nil { + return fmt.Errorf("%s: %w", tf.Name, err) + } + if stop { + break } } } @@ -506,8 +559,8 @@ func lookup(key string, opts *options, l Lookuper) (string, bool, bool, error) { val, found := l.Lookup(key) if !found { if opts.Required { - if pl, ok := l.(*prefixLookuper); ok { - key = pl.prefix + key + if keyer, ok := l.(KeyedLookuper); ok { + key = keyer.Key(key) } return "", false, false, fmt.Errorf("%w: %s", ErrMissingRequired, key) diff --git a/envconfig_test.go b/envconfig_test.go index ae2cf04..0fb9e67 100644 --- a/envconfig_test.go +++ b/envconfig_test.go @@ -158,8 +158,8 @@ func (c *CustomTypeError) UnmarshalText(text []byte) error { } // valueMutatorFunc is used for testing mutators. -var valueMutatorFunc MutatorFunc = func(ctx context.Context, k, v string) (string, error) { - return fmt.Sprintf("MUTATED_%s", v), nil +var valueMutatorFunc MutatorFunc = func(ctx context.Context, oKey, rKey, oVal, rVal string) (string, bool, error) { + return fmt.Sprintf("MUTATED_%s", rVal), false, nil } // Electron > Lepton > Quark @@ -1837,6 +1837,90 @@ func TestProcessWith(t *testing.T) { }), mutators: []MutatorFunc{valueMutatorFunc}, }, + { + name: "mutate/stops", + input: &struct { + Field string `env:"FIELD"` + }{}, + exp: &struct { + Field string `env:"FIELD"` + }{ + Field: "value-1", + }, + lookuper: MapLookuper(map[string]string{ + "FIELD": "", + }), + mutators: []MutatorFunc{ + func(_ context.Context, oKey, rKey, oVal, cVal string) (string, bool, error) { + return "value-1", true, nil + }, + func(_ context.Context, oKey, rKey, oVal, cVal string) (string, bool, error) { + return "value-2", true, nil + }, + }, + }, + { + name: "mutate/original_and_resolved_keys", + input: &struct { + Field string `env:"FIELD"` + }{}, + exp: &struct { + Field string `env:"FIELD"` + }{ + Field: "oKey:FIELD, rKey:KEY_FIELD", + }, + lookuper: PrefixLookuper("KEY_", MapLookuper(map[string]string{ + "KEY_FIELD": "", + })), + mutators: []MutatorFunc{ + func(_ context.Context, oKey, rKey, oVal, cVal string) (string, bool, error) { + return fmt.Sprintf("oKey:%s, rKey:%s", oKey, rKey), false, nil + }, + }, + }, + { + name: "mutate/original_and_current_values", + input: &struct { + Field string `env:"FIELD"` + }{}, + exp: &struct { + Field string `env:"FIELD"` + }{ + Field: "oVal:old-value, cVal:new-value", + }, + lookuper: PrefixLookuper("KEY_", MapLookuper(map[string]string{ + "KEY_FIELD": "old-value", + })), + mutators: []MutatorFunc{ + func(_ context.Context, oKey, rKey, oVal, cVal string) (string, bool, error) { + return "new-value", false, nil + }, + func(_ context.Context, oKey, rKey, oVal, cVal string) (string, bool, error) { + return fmt.Sprintf("oVal:%s, cVal:%s", oVal, cVal), false, nil + }, + }, + }, + { + name: "mutate/halts_error", + input: &struct { + Field string `env:"FIELD"` + }{}, + exp: &struct { + Field string `env:"FIELD"` + }{}, + lookuper: MapLookuper(map[string]string{ + "FIELD": "", + }), + mutators: []MutatorFunc{ + func(_ context.Context, oKey, rKey, oVal, cVal string) (string, bool, error) { + return "", false, fmt.Errorf("error 1") + }, + func(_ context.Context, oKey, rKey, oVal, cVal string) (string, bool, error) { + return "", false, fmt.Errorf("error 2") + }, + }, + errMsg: "error 1", + }, // Nesting { diff --git a/go.mod b/go.mod index d2c2d9c..e1f253c 100644 --- a/go.mod +++ b/go.mod @@ -1,5 +1,5 @@ module github.com/sethvargo/go-envconfig -go 1.17 +go 1.21 require github.com/google/go-cmp v0.5.8