Add ability to override configuration settings using environment vari…

…ables (#2147)

* Add ability to override configuration settings using environment variables

Signed-off-by: Wing924 <weihe924stephen@gmail.com>

CHANGELOG.md

@@ -17,6 +17,8 @@
   * `--experimental.distributor.user-subring-size`
 * [FEATURE] Added flag `-experimental.ruler.enable-api` to enable the ruler api which implements the Prometheus API `/api/v1/rules` and `/api/v1/alerts` endpoints under the configured `-http.prefix`. #1999
 * [FEATURE] Added sharding support to compactor when using the experimental TSDB blocks storage. #2113
+* [FEATURE] Add ability to override YAML config file settings using environment variables. #2147
+  * `-config.expand-env`
 * [ENHANCEMENT] Add `status` label to `cortex_alertmanager_configs` metric to gauge the number of valid and invalid configs. #2125
 * [ENHANCEMENT] Cassandra Authentication: added the `custom_authenticators` config option that allows users to authenticate with cassandra clusters using password authenticators that are not approved by default in [gocql](https://github.com/gocql/gocql/blob/81b8263d9fe526782a588ef94d3fa5c6148e5d67/conn.go#L27) #2093
 * [ENHANCEMENT] Experimental TSDB: Export TSDB Syncer metrics from Compactor component, they are prefixed with `cortex_compactor_`. #2023

cmd/cortex/main.go

@@ -7,6 +7,7 @@ import (
 	"math/rand"
 	"os"
 	"runtime"
+	"strings"
 	"time"
 
 	"github.com/go-kit/kit/log/level"
@@ -25,7 +26,10 @@ func init() {
 	prometheus.MustRegister(version.NewCollector("cortex"))
 }
 
-const configFileOption = "config.file"
+const (
+	configFileOption = "config.file"
+	configExpandENV  = "config.expand-env"
+)
 
 var testMode = false
 
@@ -37,14 +41,14 @@ func main() {
 		mutexProfileFraction int
 	)
 
-	configFile := parseConfigFileParameter()
+	configFile, expandENV := parseConfigFileParameter(os.Args[1:])
 
 	// This sets default values from flags to the config.
 	// It needs to be called before parsing the config file!
 	flagext.RegisterFlags(&cfg)
 
 	if configFile != "" {
-		if err := LoadConfig(configFile, &cfg); err != nil {
+		if err := LoadConfig(configFile, expandENV, &cfg); err != nil {
 			fmt.Fprintf(os.Stderr, "error loading config from %s: %v\n", configFile, err)
 			if testMode {
 				return
@@ -53,8 +57,10 @@ func main() {
 		}
 	}
 
-	// Ignore -config.file here, since it was already parsed, but it's still present on command line.
+	// Ignore -config.file and -config.expand-env here, since it was already parsed, but it's still present on command line.
 	flagext.IgnoredFlag(flag.CommandLine, configFileOption, "Configuration file to load.")
+	flagext.IgnoredFlag(flag.CommandLine, configExpandENV, "Expands ${var} or $var in config according to the values of the environment variables.")
+
 	flag.IntVar(&eventSampleRate, "event.sample-rate", 0, "How often to sample observability events (0 = never).")
 	flag.IntVar(&ballastBytes, "mem-ballast-size-bytes", 0, "Size of memory ballast to allocate.")
 	flag.IntVar(&mutexProfileFraction, "debug.mutex-profile-fraction", 0, "Fraction at which mutex profile vents will be reported, 0 to disable")
@@ -108,37 +114,38 @@ func main() {
 	util.CheckFatal("initializing cortex", err)
 }
 
-// Parse -config.file option via separate flag set, to avoid polluting default one and calling flag.Parse on it twice.
-func parseConfigFileParameter() string {
-	var configFile = ""
+// Parse -config.file and -config.expand-env option via separate flag set, to avoid polluting default one and calling flag.Parse on it twice.
+func parseConfigFileParameter(args []string) (configFile string, expandEnv bool) {
 	// ignore errors and any output here. Any flag errors will be reported by main flag.Parse() call.
-	fs := flag.NewFlagSet(os.Args[0], flag.ContinueOnError)
+	fs := flag.NewFlagSet("", flag.ContinueOnError)
 	fs.SetOutput(ioutil.Discard)
-	fs.StringVar(&configFile, configFileOption, "", "") // usage not used in this function.
 
-	// Try to find -config.file option in the flags. As Parsing stops on the first error, eg. unknown flag, we simply
+	// usage not used in these functions.
+	fs.StringVar(&configFile, configFileOption, "", "")
+	fs.BoolVar(&expandEnv, configExpandENV, false, "")
+
+	// Try to find -config.file and -config.expand-env option in the flags. As Parsing stops on the first error, eg. unknown flag, we simply
 	// try remaining parameters until we find config flag, or there are no params left.
 	// (ContinueOnError just means that flag.Parse doesn't call panic or os.Exit, but it returns error, which we ignore)
-	args := os.Args[1:]
 	for len(args) > 0 {
 		_ = fs.Parse(args)
-		if configFile != "" {
-			// found (!)
-			break
-		}
 		args = args[1:]
 	}
 
-	return configFile
+	return
 }
 
 // LoadConfig read YAML-formatted config from filename into cfg.
-func LoadConfig(filename string, cfg *cortex.Config) error {
+func LoadConfig(filename string, expandENV bool, cfg *cortex.Config) error {
 	buf, err := ioutil.ReadFile(filename)
 	if err != nil {
 		return errors.Wrap(err, "Error reading config file")
 	}
 
+	if expandENV {
+		buf = expandEnv(buf)
+	}
+
 	err = yaml.UnmarshalStrict(buf, cfg)
 	if err != nil {
 		return errors.Wrap(err, "Error parsing config file")
@@ -155,3 +162,19 @@ func DumpYaml(cfg *cortex.Config) {
 		fmt.Printf("%s\n", out)
 	}
 }
+
+// expandEnv replaces ${var} or $var in config according to the values of the current environment variables.
+// The replacement is case-sensitive. References to undefined variables are replaced by the empty string.
+// A default value can be given by using the form ${var:default value}.
+func expandEnv(config []byte) []byte {
+	return []byte(os.Expand(string(config), func(key string) string {
+		keyAndDefault := strings.SplitN(key, ":", 2)
+		key = keyAndDefault[0]
+
+		v := os.Getenv(key)
+		if v == "" && len(keyAndDefault) == 2 {
+			v = keyAndDefault[1] // Set value to the default.
+		}
+		return v
+	}))
+}

cmd/cortex/main_test.go

@@ -6,9 +6,11 @@ import (
 	"io"
 	"io/ioutil"
 	"os"
+	"strings"
 	"sync"
 	"testing"
 
+	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
 
@@ -61,6 +63,17 @@ func TestFlagParsing(t *testing.T) {
 			stdoutMessage: "target: ingester\n",
 		},
 
+		"config without expand-env": {
+			yaml:          "target: $TARGET",
+			stderrMessage: "Error parsing config file: unrecognised module name: $TARGET\n",
+		},
+
+		"config with expand-env": {
+			arguments:     []string{"-config.expand-env"},
+			yaml:          "target: $TARGET",
+			stdoutMessage: "target: ingester\n",
+		},
+
 		"config with arguments override": {
 			yaml:          "target: ingester",
 			arguments:     []string{"-target=distributor"},
@@ -70,12 +83,14 @@ func TestFlagParsing(t *testing.T) {
 		// we cannot test the happy path, as cortex would then fully start
 	} {
 		t.Run(name, func(t *testing.T) {
+			_ = os.Setenv("TARGET", "ingester")
 			testSingle(t, tc.arguments, tc.yaml, []byte(tc.stdoutMessage), []byte(tc.stderrMessage))
 		})
 	}
 }
 
 func testSingle(t *testing.T, arguments []string, yaml string, stdoutMessage, stderrMessage []byte) {
+	t.Helper()
 	oldArgs, oldStdout, oldStderr, oldTestMode := os.Args, os.Stdout, os.Stderr, testMode
 	defer func() {
 		os.Stdout = oldStdout
@@ -167,3 +182,74 @@ func (co *capturedOutput) Done() (stdout []byte, stderr []byte) {
 
 	return co.stdoutBuf.Bytes(), co.stderrBuf.Bytes()
 }
+
+func TestExpandEnv(t *testing.T) {
+	var tests = []struct {
+		in  string
+		out string
+	}{
+		// Environment variables can be specified as ${env} or $env.
+		{"x$y", "xy"},
+		{"x${y}", "xy"},
+
+		// Environment variables are case-sensitive. Neither are replaced.
+		{"x$Y", "x"},
+		{"x${Y}", "x"},
+
+		// Defaults can only be specified when using braces.
+		{"x${Z:D}", "xD"},
+		{"x${Z:A B C D}", "xA B C D"}, // Spaces are allowed in the default.
+		{"x${Z:}", "x"},
+
+		// Defaults don't work unless braces are used.
+		{"x$y:D", "xy:D"},
+	}
+
+	for _, test := range tests {
+		test := test
+		t.Run(test.in, func(t *testing.T) {
+			_ = os.Setenv("y", "y")
+			output := expandEnv([]byte(test.in))
+			assert.Equal(t, test.out, string(output), "Input: %s", test.in)
+		})
+	}
+}
+
+func TestParseConfigFileParameter(t *testing.T) {
+	var tests = []struct {
+		args       string
+		configFile string
+		expandENV  bool
+	}{
+		{"", "", false},
+		{"--foo", "", false},
+		{"-f -a", "", false},
+
+		{"--config.file=foo", "foo", false},
+		{"--config.file foo", "foo", false},
+		{"--config.file=foo --config.expand-env", "foo", true},
+		{"--config.expand-env --config.file=foo", "foo", true},
+
+		{"--opt1 --config.file=foo", "foo", false},
+		{"--opt1 --config.file foo", "foo", false},
+		{"--opt1 --config.file=foo --config.expand-env", "foo", true},
+		{"--opt1 --config.expand-env --config.file=foo", "foo", true},
+
+		{"--config.file=foo --opt1", "foo", false},
+		{"--config.file foo --opt1", "foo", false},
+		{"--config.file=foo --config.expand-env --opt1", "foo", true},
+		{"--config.expand-env --config.file=foo --opt1", "foo", true},
+
+		{"--config.file=foo --opt1 --config.expand-env", "foo", true},
+		{"--config.expand-env --opt1 --config.file=foo", "foo", true},
+	}
+	for _, test := range tests {
+		test := test
+		t.Run(test.args, func(t *testing.T) {
+			args := strings.Split(test.args, " ")
+			configFile, expandENV := parseConfigFileParameter(args)
+			assert.Equal(t, test.configFile, configFile)
+			assert.Equal(t, test.expandENV, expandENV)
+		})
+	}
+}

docs/configuration/config-file-reference.md

@@ -11,7 +11,7 @@ Cortex can be configured using a YAML file - specified using the `-config.file`
 
 To specify which configuration file to load, pass the `-config.file` flag at the command line. The file is written in [YAML format](https://en.wikipedia.org/wiki/YAML), defined by the scheme below. Brackets indicate that a parameter is optional.
 
-Generic placeholders are defined as follows:
+### Generic placeholders
 
 * `<boolean>`: a boolean that can take the values `true` or `false`
 * `<int>`: any integer matching the regular expression `[1-9]+[0-9]*`
@@ -20,7 +20,30 @@ Generic placeholders are defined as follows:
 * `<url>`: an URL
 * `<prefix>`: a CLI flag prefix based on the context (look at the parent configuration block to see which CLI flags prefix should be used)
 
-Supported contents and default values of the config file:
+### Use environment variables in the configuration
+
+You can use environment variable references in the config file to set values that need to be configurable during deployment.
+To do this, use:
+
+```
+${VAR}
+```
+
+Where VAR is the name of the environment variable.
+
+Each variable reference is replaced at startup by the value of the environment variable.
+The replacement is case-sensitive and occurs before the YAML file is parsed.
+References to undefined variables are replaced by empty strings unless you specify a default value or custom error text.
+
+To specify a default value, use:
+
+```
+${VAR:default_value}
+```
+
+Where default_value is the value to use if the environment variable is undefined.
+
+### Supported contents and default values of the config file
 
 ```yaml
 # The Cortex service to run. Supported values are: all, distributor, ingester,

docs/configuration/config-file-reference.template

@@ -11,7 +11,7 @@ Cortex can be configured using a YAML file - specified using the `-config.file`
 
 To specify which configuration file to load, pass the `-config.file` flag at the command line. The file is written in [YAML format](https://en.wikipedia.org/wiki/YAML), defined by the scheme below. Brackets indicate that a parameter is optional.
 
-Generic placeholders are defined as follows:
+### Generic placeholders
 
 * `<boolean>`: a boolean that can take the values `true` or `false`
 * `<int>`: any integer matching the regular expression `[1-9]+[0-9]*`
@@ -20,5 +20,28 @@ Generic placeholders are defined as follows:
 * `<url>`: an URL
 * `<prefix>`: a CLI flag prefix based on the context (look at the parent configuration block to see which CLI flags prefix should be used)
 
-Supported contents and default values of the config file:
+### Use environment variables in the configuration
+
+You can use environment variable references in the config file to set values that need to be configurable during deployment.
+To do this, use:
+
+```
+${VAR}
+```
+
+Where VAR is the name of the environment variable.
+
+Each variable reference is replaced at startup by the value of the environment variable.
+The replacement is case-sensitive and occurs before the YAML file is parsed.
+References to undefined variables are replaced by empty strings unless you specify a default value or custom error text.
+
+To specify a default value, use:
+
+```
+${VAR:default_value}
+```
+
+Where default_value is the value to use if the environment variable is undefined.
+
+### Supported contents and default values of the config file