Configuration: add a common config section for object storage (#4473)

* add common config for object storage Signed-off-by: Trevor Whitney <trevorjwhitney@gmail.com> * wip Signed-off-by: Trevor Whitney <trevorjwhitney@gmail.com> * add test to compactor config override Signed-off-by: Trevor Whitney <trevorjwhitney@gmail.com> * update changelog Signed-off-by: Trevor Whitney <trevorjwhitney@gmail.com> * changes from PR review * rename object_store -> storage * add local/filesytem support to common object storage config
grafana · Oct 21, 2021 · 2427fab · 2427fab
1 parent d8e6deb
commit 2427fab
Show file tree

Hide file tree

Showing 9 changed files with 619 additions and 5 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -4,6 +4,7 @@
 * [4440](https://github.com/grafana/loki/pull/4440) **DylanGuedes**: Config: Override distributor's default ring KV store
 * [4443](https://github.com/grafana/loki/pull/4443) **DylanGuedes**: Loki: Change how push API checks for contentType
 * [4415](https://github.com/grafana/loki/pull/4415) **DylanGuedes**: Change default limits to common values
+* [4473](https://github.com/grafana/loki/pull/4473) **trevorwhitney**: Config: add object storage configuration to common config
 
 # 2.3.0 (2021/08/06)
 

diff --git a/pkg/loki/common/common.go b/pkg/loki/common/common.go
@@ -1,6 +1,23 @@
 package common
 
+import (
+	"github.com/grafana/loki/pkg/storage/chunk/aws"
+	"github.com/grafana/loki/pkg/storage/chunk/azure"
+	"github.com/grafana/loki/pkg/storage/chunk/gcp"
+	"github.com/grafana/loki/pkg/storage/chunk/local"
+	"github.com/grafana/loki/pkg/storage/chunk/openstack"
+)
+
 // Config holds common config that can be shared between multiple other config sections
 type Config struct {
-	PathPrefix string `yaml:"path_prefix"`
+	PathPrefix string  `yaml:"path_prefix"`
+	Storage    Storage `yaml:"storage"`
+}
+
+type Storage struct {
+	S3       *aws.S3Config            `yaml:"s3"`
+	GCS      *gcp.GCSConfig           `yaml:"gcs"`
+	Azure    *azure.BlobStorageConfig `yaml:"azure"`
+	Swift    *openstack.SwiftConfig   `yaml:"swift"`
+	FSConfig *local.FSConfig          `yaml:"filesystem"`
 }
diff --git a/pkg/loki/config_wrapper.go b/pkg/loki/config_wrapper.go
@@ -3,10 +3,12 @@ package loki
 import (
 	"flag"
 	"fmt"
+	"reflect"
 
 	"github.com/grafana/dskit/flagext"
 	"github.com/pkg/errors"
 
+	"github.com/grafana/loki/pkg/storage/chunk/storage"
 	"github.com/grafana/loki/pkg/util/cfg"
 )
 
@@ -72,6 +74,7 @@ func (c *ConfigWrapper) ApplyDynamicConfig() cfg.Source {
 		}
 
 		applyMemberlistConfig(r)
+		applyStorageConfig(r, &defaults)
 
 		return nil
 	}
@@ -89,3 +92,97 @@ func applyMemberlistConfig(r *ConfigWrapper) {
 		r.Ruler.Ring.KVStore.Store = memberlistStr
 	}
 }
+
+// applyStorageConfig will attempt to apply a common storage config for either
+// s3, gcs, azure, or swift to all the places we create an object storage client.
+// If any specific configs for an object storage client have been provided elsewhere in the
+// configuration file, applyStorageConfig will not override them.
+// If multiple storage configurations are provided, applyStorageConfig will apply
+// all of them, and will set the value for the Ruler's StoreConfig `type` to the
+// last one (alphabetically) that was defined.
+func applyStorageConfig(cfg, defaults *ConfigWrapper) {
+	rulerStoreConfigsToApply := make([]func(*ConfigWrapper), 0, 4)
+	chunkStorageConfigsToApply := make([]func(*ConfigWrapper), 0, 4)
+
+	if cfg.Common.Storage.Azure != nil {
+		rulerStoreConfigsToApply = append(rulerStoreConfigsToApply, func(r *ConfigWrapper) {
+			r.Ruler.StoreConfig.Type = "azure"
+			r.Ruler.StoreConfig.Azure = r.Common.Storage.Azure.ToCortexAzureConfig()
+		})
+
+		chunkStorageConfigsToApply = append(chunkStorageConfigsToApply, func(r *ConfigWrapper) {
+			r.StorageConfig.AzureStorageConfig = *r.Common.Storage.Azure
+			r.CompactorConfig.SharedStoreType = storage.StorageTypeAzure
+		})
+	}
+
+	if cfg.Common.Storage.GCS != nil {
+		rulerStoreConfigsToApply = append(rulerStoreConfigsToApply, func(r *ConfigWrapper) {
+			r.Ruler.StoreConfig.Type = "gcs"
+			r.Ruler.StoreConfig.GCS = r.Common.Storage.GCS.ToCortexGCSConfig()
+		})
+
+		chunkStorageConfigsToApply = append(chunkStorageConfigsToApply, func(r *ConfigWrapper) {
+			r.StorageConfig.GCSConfig = *r.Common.Storage.GCS
+			r.CompactorConfig.SharedStoreType = storage.StorageTypeGCS
+		})
+	}
+
+	if cfg.Common.Storage.FSConfig != nil {
+		rulerStoreConfigsToApply = append(rulerStoreConfigsToApply, func(r *ConfigWrapper) {
+			r.Ruler.StoreConfig.Type = "local"
+			r.Ruler.StoreConfig.Local = r.Common.Storage.FSConfig.ToCortexLocalConfig()
+		})
+
+		chunkStorageConfigsToApply = append(chunkStorageConfigsToApply, func(r *ConfigWrapper) {
+			r.StorageConfig.FSConfig = *r.Common.Storage.FSConfig
+			r.CompactorConfig.SharedStoreType = storage.StorageTypeFileSystem
+		})
+	}
+
+	if cfg.Common.Storage.S3 != nil {
+		rulerStoreConfigsToApply = append(rulerStoreConfigsToApply, func(r *ConfigWrapper) {
+			r.Ruler.StoreConfig.Type = "s3"
+			r.Ruler.StoreConfig.S3 = r.Common.Storage.S3.ToCortexS3Config()
+		})
+
+		chunkStorageConfigsToApply = append(chunkStorageConfigsToApply, func(r *ConfigWrapper) {
+			r.StorageConfig.AWSStorageConfig.S3Config = *r.Common.Storage.S3
+			r.CompactorConfig.SharedStoreType = storage.StorageTypeS3
+		})
+	}
+
+	if cfg.Common.Storage.Swift != nil {
+		rulerStoreConfigsToApply = append(rulerStoreConfigsToApply, func(r *ConfigWrapper) {
+			r.Ruler.StoreConfig.Type = "swift"
+			r.Ruler.StoreConfig.Swift = r.Common.Storage.Swift.ToCortexSwiftConfig()
+		})
+
+		chunkStorageConfigsToApply = append(chunkStorageConfigsToApply, func(r *ConfigWrapper) {
+			r.StorageConfig.Swift = *r.Common.Storage.Swift
+			r.CompactorConfig.SharedStoreType = storage.StorageTypeSwift
+		})
+	}
+
+	// store change funcs in slices and apply all at once, because once we change the
+	// config we can no longer compare it to the default, this allows us to only
+	// do that comparison once
+	applyRulerStoreConfigs(cfg, defaults, rulerStoreConfigsToApply)
+	applyChunkStorageConfigs(cfg, defaults, chunkStorageConfigsToApply)
+}
+
+func applyRulerStoreConfigs(cfg, defaults *ConfigWrapper, apply []func(*ConfigWrapper)) {
+	if reflect.DeepEqual(cfg.Ruler.StoreConfig, defaults.Ruler.StoreConfig) {
+		for _, ap := range apply {
+			ap(cfg)
+		}
+	}
+}
+
+func applyChunkStorageConfigs(cfg, defaults *ConfigWrapper, apply []func(*ConfigWrapper)) {
+	if reflect.DeepEqual(cfg.StorageConfig, defaults.StorageConfig) {
+		for _, ap := range apply {
+			ap(cfg)
+		}
+	}
+}