From 6c3a22c7bc1806ddc4a29a63a397751685986dd6 Mon Sep 17 00:00:00 2001 From: Matthias Loibl Date: Fri, 22 Mar 2019 18:39:17 +0100 Subject: [PATCH 1/3] Add possibility to inline ServiceAccount into GCS config --- docs/storage.md | 21 +++++++++++++++++++++ go.mod | 1 + pkg/objstore/gcs/gcs.go | 23 ++++++++++++++++++++--- 3 files changed, 42 insertions(+), 3 deletions(-) diff --git a/docs/storage.md b/docs/storage.md index 8395d44568..6d194de935 100644 --- a/docs/storage.md +++ b/docs/storage.md @@ -168,6 +168,27 @@ Application credentials are configured via JSON file, the client looks for: You can read more on how to get application credential json file in [https://cloud.google.com/docs/authentication/production](https://cloud.google.com/docs/authentication/production) +Another possibility is to inline the ServiceAccount into the Thanos configuration and only maintain one file: + +```yaml +type: GCS +config: + bucket: "thanos" + service_account: |- + { + "type": "service_account", + "project_id": "project", + "private_key_id": "abcdefghijklmnopqrstuvwxyz12345678906666", + "private_key": "-----BEGIN PRIVATE KEY-----\...\n-----END PRIVATE KEY-----\n", + "client_email": "project@thanos.iam.gserviceaccount.com", + "client_id": "123456789012345678901", + "auth_uri": "https://accounts.google.com/o/oauth2/auth", + "token_uri": "https://oauth2.googleapis.com/token", + "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs", + "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/thanos%40gitpods.iam.gserviceaccount.com" + } +``` + ### GCS Policies For deployment: diff --git a/go.mod b/go.mod index 9932f12049..01b4034b4b 100644 --- a/go.mod +++ b/go.mod @@ -38,6 +38,7 @@ require ( github.com/prometheus/tsdb v0.4.0 go.opencensus.io v0.19.0 // indirect golang.org/x/net v0.0.0-20190213061140-3a22650c66bd + golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4 golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223 // indirect golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2 diff --git a/pkg/objstore/gcs/gcs.go b/pkg/objstore/gcs/gcs.go index 634ebc4741..6e54a77d0a 100644 --- a/pkg/objstore/gcs/gcs.go +++ b/pkg/objstore/gcs/gcs.go @@ -16,6 +16,7 @@ import ( "github.com/improbable-eng/thanos/pkg/objstore" "github.com/pkg/errors" "github.com/prometheus/common/version" + "golang.org/x/oauth2/google" "google.golang.org/api/iterator" "google.golang.org/api/option" yaml "gopkg.in/yaml.v2" @@ -26,7 +27,8 @@ const DirDelim = "/" // Config stores the configuration for gcs bucket. type Config struct { - Bucket string `yaml:"bucket"` + Bucket string `yaml:"bucket"` + ServiceAccount string `yaml:"service_account"` } // Bucket implements the store.Bucket and shipper.Bucket interfaces against GCS. @@ -47,8 +49,23 @@ func NewBucket(ctx context.Context, logger log.Logger, conf []byte, component st if gc.Bucket == "" { return nil, errors.New("missing Google Cloud Storage bucket name for stored blocks") } - gcsOptions := option.WithUserAgent(fmt.Sprintf("thanos-%s/%s (%s)", component, version.Version, runtime.Version())) - gcsClient, err := storage.NewClient(ctx, gcsOptions) + + var opts []option.ClientOption + + // If ServiceAccount provided inside configuration use it, otherwise fallback to defaults + if gc.ServiceAccount != "" { + credentials, err := google.CredentialsFromJSON(ctx, []byte(gc.ServiceAccount)) + if err != nil { + return nil, errors.Wrap(err, "failed to create credentials from JSON") + } + opts = append(opts, option.WithCredentials(credentials)) + } + + opts = append(opts, + option.WithUserAgent(fmt.Sprintf("thanos-%s/%s (%s)", component, version.Version, runtime.Version())), + ) + + gcsClient, err := storage.NewClient(ctx, opts...) if err != nil { return nil, err } From 21e0f6932a2d9c9a72f38fc42ac219c6488e5a20 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartek=20P=C5=82otka?= Date: Tue, 26 Mar 2019 13:41:24 +0100 Subject: [PATCH 2/3] Update comment in pkg/objstore/gcs/gcs.go Co-Authored-By: metalmatze --- pkg/objstore/gcs/gcs.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/objstore/gcs/gcs.go b/pkg/objstore/gcs/gcs.go index 6e54a77d0a..7743b9449f 100644 --- a/pkg/objstore/gcs/gcs.go +++ b/pkg/objstore/gcs/gcs.go @@ -52,7 +52,7 @@ func NewBucket(ctx context.Context, logger log.Logger, conf []byte, component st var opts []option.ClientOption - // If ServiceAccount provided inside configuration use it, otherwise fallback to defaults + // If ServiceAccount is provided, use them in GCS client, otherwise fallback to Google default logic. if gc.ServiceAccount != "" { credentials, err := google.CredentialsFromJSON(ctx, []byte(gc.ServiceAccount)) if err != nil { From 76f47f38ca6ce04942cdb053effa8f256bfad0d5 Mon Sep 17 00:00:00 2001 From: Matthias Loibl Date: Tue, 26 Mar 2019 14:38:20 +0100 Subject: [PATCH 3/3] Generate docs for GCS config --- docs/storage.md | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/docs/storage.md b/docs/storage.md index 6d194de935..3f8ae931bc 100644 --- a/docs/storage.md +++ b/docs/storage.md @@ -152,9 +152,13 @@ For example: type: GCS config: bucket: "" + service_account: "" ``` -Application credentials are configured via JSON file, the client looks for: +### Using GOOGLE_APPLICATION_CREDENTIALS + +Application credentials are configured via JSON file and only the bucket needs to be specified, +the client looks for: 1. A JSON file whose path is specified by the `GOOGLE_APPLICATION_CREDENTIALS` environment variable. @@ -168,7 +172,10 @@ Application credentials are configured via JSON file, the client looks for: You can read more on how to get application credential json file in [https://cloud.google.com/docs/authentication/production](https://cloud.google.com/docs/authentication/production) -Another possibility is to inline the ServiceAccount into the Thanos configuration and only maintain one file: +### Using inline a Service Account + +Another possibility is to inline the ServiceAccount into the Thanos configuration and only maintain one file. +This feature was added, so that the Prometheus Operator only needs to take care of one secret file. ```yaml type: GCS