Allow to pass config overrides via the Settings Struct (elastic#8449)

* Allow to pass config overrides via the Settings Struct

Sometime a custom beat that get executed want to override system
defaults instead of relying on code defined by libbeat.

This is the case with beatless, the queue has limits and flush values
that make sense when the beat is run on the edge but not on on AWS lambda.

Note that settings is passed via liberal common.Config no type checking is
done. I went that route because many parts of beats doesn't expose the
config struct outside of the package.

This is similar to explicitely defining them in the yaml.

CHANGELOG-developer.asciidoc

@@ -52,3 +52,4 @@ The list below covers the major changes between 6.3.0 and master only.
 - Libbeat provides a new function `cmd.GenRootCmdWithSettings` that should be preferred over deprecated functions
   `cmd.GenRootCmd`, `cmd.GenRootCmdWithRunFlags`, and `cmd.GenRootCmdWithIndexPrefixWithRunFlags`. {pull}7850[7850]
 - Set current year in generator templates. {pull}8396[8396]
+- You can now override default settings of libbeat by using instance.Settings. {pull}8449[8449]

libbeat/cfgfile/cfgfile.go

@@ -37,7 +37,7 @@ var (
 	testConfig  = flag.Bool("configtest", false, "Test configuration and exit.")
 
 	// Additional default settings, that must be available for variable expansion
-	defaults = mustNewConfigFrom(map[string]interface{}{
+	defaults = common.MustNewConfigFrom(map[string]interface{}{
 		"path": map[string]interface{}{
 			"home":   ".", // to be initialized by beat
 			"config": "${path.home}",
@@ -63,14 +63,6 @@ func init() {
 	makePathFlag("path.logs", "Logs path")
 }
 
-func mustNewConfigFrom(from interface{}) *common.Config {
-	cfg, err := common.NewConfigFrom(from)
-	if err != nil {
-		panic(err)
-	}
-	return cfg
-}
-
 // ChangeDefaultCfgfileFlag replaces the value and default value for the `-c`
 // flag so that it reflects the beat name.
 func ChangeDefaultCfgfileFlag(beatName string) error {
@@ -105,7 +97,7 @@ func HandleFlags() error {
 // structure. If path is empty this method reads from the configuration
 // file specified by the '-c' command line flag.
 func Read(out interface{}, path string) error {
-	config, err := Load(path)
+	config, err := Load(path, nil)
 	if err != nil {
 		return err
 	}
@@ -116,7 +108,7 @@ func Read(out interface{}, path string) error {
 // Load reads the configuration from a YAML file structure. If path is empty
 // this method reads from the configuration file specified by the '-c' command
 // line flag.
-func Load(path string) (*common.Config, error) {
+func Load(path string, beatOverrides *common.Config) (*common.Config, error) {
 	var config *common.Config
 	var err error
 
@@ -142,13 +134,22 @@ func Load(path string) (*common.Config, error) {
 		return nil, err
 	}
 
-	config, err = common.MergeConfigs(
-		defaults,
-		config,
-		overwrites,
-	)
-	if err != nil {
-		return nil, err
+	if beatOverrides != nil {
+		config, err = common.MergeConfigs(
+			defaults,
+			beatOverrides,
+			config,
+			overwrites,
+		)
+		if err != nil {
+			return nil, err
+		}
+	} else {
+		config, err = common.MergeConfigs(
+			defaults,
+			config,
+			overwrites,
+		)
 	}
 
 	config.PrintDebugf("Complete configuration loaded:")

libbeat/cmd/instance/beat.go

@@ -229,8 +229,8 @@ func NewBeat(name, indexPrefix, v string) (*Beat, error) {
 	return &Beat{Beat: b}, nil
 }
 
-// init does initialization of things common to all actions (read confs, flags)
-func (b *Beat) Init() error {
+// InitWithSettings does initialization of things common to all actions (read confs, flags)
+func (b *Beat) InitWithSettings(settings Settings) error {
 	err := b.handleFlags()
 	if err != nil {
 		return err
@@ -240,13 +240,20 @@ func (b *Beat) Init() error {
 		return err
 	}
 
-	if err := b.configure(); err != nil {
+	if err := b.configure(settings); err != nil {
 		return err
 	}
 
 	return nil
 }
 
+// Init does initialization of things common to all actions (read confs, flags)
+//
+// Deprecated: use InitWithSettings
+func (b *Beat) Init() error {
+	return b.InitWithSettings(Settings{})
+}
+
 // BeatConfig returns config section for this beat
 func (b *Beat) BeatConfig() (*common.Config, error) {
 	configName := strings.ToLower(b.Info.Beat)
@@ -323,7 +330,7 @@ func (b *Beat) launch(settings Settings, bt beat.Creator) error {
 	defer logp.Sync()
 	defer logp.Info("%s stopped.", b.Info.Beat)
 
-	err := b.Init()
+	err := b.InitWithSettings(settings)
 	if err != nil {
 		return err
 	}
@@ -504,10 +511,10 @@ func (b *Beat) handleFlags() error {
 // config reads the configuration file from disk, parses the common options
 // defined in BeatConfig, initializes logging, and set GOMAXPROCS if defined
 // in the config. Lastly it invokes the Config method implemented by the beat.
-func (b *Beat) configure() error {
+func (b *Beat) configure(settings Settings) error {
 	var err error
 
-	cfg, err := cfgfile.Load("")
+	cfg, err := cfgfile.Load("", settings.ConfigOverrides)
 	if err != nil {
 		return fmt.Errorf("error loading config file: %v", err)
 	}

libbeat/cmd/instance/settings.go

@@ -20,14 +20,16 @@ package instance
 import (
 	"github.com/spf13/pflag"
 
+	"github.com/elastic/beats/libbeat/common"
 	"github.com/elastic/beats/libbeat/monitoring/report"
 )
 
 // Settings contains basic settings for any beat to pass into GenRootCmd
 type Settings struct {
-	Name        string
-	IndexPrefix string
-	Version     string
-	Monitoring  report.Settings
-	RunFlags    *pflag.FlagSet
+	Name            string
+	IndexPrefix     string
+	Version         string
+	Monitoring      report.Settings
+	RunFlags        *pflag.FlagSet
+	ConfigOverrides *common.Config
 }