Skip to content

Commit

Permalink
feat(storage): find GoogleAccessID when using impersonated creds (#6591)
Browse files Browse the repository at this point in the history 
`bucket.SignedURL` and `bucket.GenerateSignedPostPolicyV4` now automatically detect the GoogleAccessID when using impersonated credentials, i.e. when authenticated via `gcloud auth application-default login --impersonate-service-account ${SERVICE_ACCOUNT}`

This prevents users from having to manage a private key or pass in the service account to these methods in their code; see #5979 

Co-authored-by: Chris Cotter <cjcotter@google.com>
  • Loading branch information
@BrennaEpp @tritone
BrennaEpp and tritone authored Sep 14, 2022
1 parent ec1a190 commit a2d16a7
Show file tree
Hide file tree
Showing 3 changed files with 155 additions and 36 deletions.
65 changes: 32 additions & 33 deletions storage/bucket.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ import (
"errors"
"fmt"
"reflect"
"strings"
"time"

"cloud.google.com/go/compute/metadata"
Expand Down Expand Up @@ -159,22 +160,17 @@ func (b *BucketHandle) Update(ctx context.Context, uattrs BucketAttrsToUpdate) (
}

// SignedURL returns a URL for the specified object. Signed URLs allow anyone
// access to a restricted resource for a limited time without needing a
// Google account or signing in. For more information about signed URLs, see
// https://cloud.google.com/storage/docs/accesscontrol#signed_urls_query_string_authentication
// access to a restricted resource for a limited time without needing a Google
// account or signing in.
// For more information about signed URLs, see "[Overview of access control]."
//
// This method only requires the Method and Expires fields in the specified
// SignedURLOptions opts to be non-nil. If not provided, it attempts to fill the
// GoogleAccessID and PrivateKey from the GOOGLE_APPLICATION_CREDENTIALS environment variable.
// If you are authenticating with a custom HTTP client, Service Account based
// auto-detection will be hindered.
// This method requires the Method and Expires fields in the specified
// SignedURLOptions to be non-nil. You may need to set the GoogleAccessID and
// PrivateKey fields in some cases. Read more on the [automatic detection of credentials]
// for this method.
//
// If no private key is found, it attempts to use the GoogleAccessID to sign the URL.
// This requires the IAM Service Account Credentials API to be enabled
// (https://console.developers.google.com/apis/api/iamcredentials.googleapis.com/overview)
// and iam.serviceAccounts.signBlob permissions on the GoogleAccessID service account.
// If you do not want these fields set for you, you may pass them in through opts or use
// SignedURL(bucket, name string, opts *SignedURLOptions) instead.
// [Overview of access control]: https://cloud.google.com/storage/docs/accesscontrol#signed_urls_query_string_authentication
// [automatic detection of credentials]: https://pkg.go.dev/cloud.google.com/go/storage#hdr-Credential_requirements_for_[BucketHandle.SignedURL]_and_[BucketHandle.GenerateSignedPostPolicyV4]
func (b *BucketHandle) SignedURL(object string, opts *SignedURLOptions) (string, error) {
if opts.GoogleAccessID != "" && (opts.SignBytes != nil || len(opts.PrivateKey) > 0) {
return SignedURL(b.name, object, opts)
Expand Down Expand Up @@ -212,18 +208,11 @@ func (b *BucketHandle) SignedURL(object string, opts *SignedURLOptions) (string,
// GenerateSignedPostPolicyV4 generates a PostPolicyV4 value from bucket, object and opts.
// The generated URL and fields will then allow an unauthenticated client to perform multipart uploads.
//
// This method only requires the Expires field in the specified PostPolicyV4Options
// to be non-nil. If not provided, it attempts to fill the GoogleAccessID and PrivateKey
// from the GOOGLE_APPLICATION_CREDENTIALS environment variable.
// If you are authenticating with a custom HTTP client, Service Account based
// auto-detection will be hindered.
// This method requires the Expires field in the specified PostPolicyV4Options
// to be non-nil. You may need to set the GoogleAccessID and PrivateKey fields
// in some cases. Read more on the [automatic detection of credentials] for this method.
//
// If no private key is found, it attempts to use the GoogleAccessID to sign the URL.
// This requires the IAM Service Account Credentials API to be enabled
// (https://console.developers.google.com/apis/api/iamcredentials.googleapis.com/overview)
// and iam.serviceAccounts.signBlob permissions on the GoogleAccessID service account.
// If you do not want these fields set for you, you may pass them in through opts or use
// GenerateSignedPostPolicyV4(bucket, name string, opts *PostPolicyV4Options) instead.
// [automatic detection of credentials]: https://pkg.go.dev/cloud.google.com/go/storage#hdr-Credential_requirements_for_[BucketHandle.SignedURL]_and_[BucketHandle.GenerateSignedPostPolicyV4]
func (b *BucketHandle) GenerateSignedPostPolicyV4(object string, opts *PostPolicyV4Options) (*PostPolicyV4, error) {
if opts.GoogleAccessID != "" && (opts.SignRawBytes != nil || opts.SignBytes != nil || len(opts.PrivateKey) > 0) {
return GenerateSignedPostPolicyV4(b.name, object, opts)
Expand Down Expand Up @@ -263,17 +252,27 @@ func (b *BucketHandle) detectDefaultGoogleAccessID() (string, error) {

if b.c.creds != nil && len(b.c.creds.JSON) > 0 {
var sa struct {
ClientEmail string `json:"client_email"`
ClientEmail string `json:"client_email"`
SAImpersonationURL string `json:"service_account_impersonation_url"`
CredType string `json:"type"`
}

err := json.Unmarshal(b.c.creds.JSON, &sa)
if err == nil && sa.ClientEmail != "" {
return sa.ClientEmail, nil
} else if err != nil {
if err != nil {
returnErr = err
} else if sa.CredType == "impersonated_service_account" {
start, end := strings.LastIndex(sa.SAImpersonationURL, "/"), strings.LastIndex(sa.SAImpersonationURL, ":")

if end <= start {
returnErr = errors.New("error parsing impersonated service account credentials")
} else {
return sa.SAImpersonationURL[start+1 : end], nil
}
} else if sa.CredType == "service_account" && sa.ClientEmail != "" {
return sa.ClientEmail, nil
} else {
returnErr = errors.New("storage: empty client email in credentials")
returnErr = errors.New("unable to parse credentials; only service_account and impersonated_service_account credentials are supported")
}

}

// Don't error out if we can't unmarshal, fallback to GCE check.
Expand All @@ -284,11 +283,11 @@ func (b *BucketHandle) detectDefaultGoogleAccessID() (string, error) {
} else if err != nil {
returnErr = err
} else {
returnErr = errors.New("got empty email from GCE metadata service")
returnErr = errors.New("empty email from GCE metadata service")
}

}
return "", fmt.Errorf("storage: unable to detect default GoogleAccessID: %v", returnErr)
return "", fmt.Errorf("storage: unable to detect default GoogleAccessID: %w. Please provide the GoogleAccessID or use a supported means for autodetecting it (see https://pkg.go.dev/cloud.google.com/go/storage#hdr-Credential_requirements_for_[BucketHandle.SignedURL]_and_[BucketHandle.GenerateSignedPostPolicyV4])", returnErr)
}

func (b *BucketHandle) defaultSignBytesFunc(email string) func([]byte) ([]byte, error) {
Expand Down
95 changes: 95 additions & 0 deletions storage/bucket_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,12 +15,14 @@
package storage

import (
"fmt"
"testing"
"time"

"cloud.google.com/go/internal/testutil"
"github.com/google/go-cmp/cmp"
gax "github.com/googleapis/gax-go/v2"
"golang.org/x/oauth2/google"
"google.golang.org/api/googleapi"
raw "google.golang.org/api/storage/v1"
)
Expand Down Expand Up @@ -782,3 +784,96 @@ func TestBucketRetryer(t *testing.T) {
})
}
}

func TestDetectDefaultGoogleAccessID(t *testing.T) {
testCases := []struct {
name string
serviceAccount string
creds func(string) string
expectSuccess bool
}{
{
name: "impersonated creds",
serviceAccount: "default@my-project.iam.gserviceaccount.com",
creds: func(sa string) string {
return fmt.Sprintf(`{
"delegates": [],
"service_account_impersonation_url": "https://iamcredentials.googleapis.com/v1/projects/-/serviceAccounts/%s:generateAccessToken",
"source_credentials": {
"client_id": "id",
"client_secret": "secret",
"refresh_token": "token",
"type": "authorized_user"
},
"type": "impersonated_service_account"
}`, sa)
},
expectSuccess: true,
},
{
name: "gcloud ADC creds",
serviceAccount: "default@my-project.iam.gserviceaccount.com",
creds: func(sa string) string {
return fmt.Sprint(`{
"client_id": "my-id.apps.googleusercontent.com",
"client_secret": "secret",
"quota_project_id": "",
"refresh_token": "token",
"type": "authorized_user"
}`)
},
expectSuccess: false,
},
{
name: "ADC private key",
serviceAccount: "default@my-project.iam.gserviceaccount.com",
creds: func(sa string) string {
return fmt.Sprintf(`{
"type": "service_account",
"project_id": "my-project",
"private_key_id": "my1",
"private_key": "-----BEGIN PRIVATE KEY-----\nkey\n-----END PRIVATE KEY-----\n",
"client_email": "%s",
"client_id": "01",
"auth_uri": "https://accounts.google.com/o/oauth2/auth",
"token_uri": "https://oauth2.googleapis.com/token",
"auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs",
"client_x509_cert_url": "cert"
}`, sa)
},
expectSuccess: true,
},
{
name: "no creds",
creds: func(_ string) string {
return ""
},
expectSuccess: false,
},
}

for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
bucket := BucketHandle{
c: &Client{
creds: &google.Credentials{
JSON: []byte(tc.creds(tc.serviceAccount)),
},
},
name: "my-bucket",
}

id, err := bucket.detectDefaultGoogleAccessID()
if tc.expectSuccess {
if err != nil {
t.Fatal(err)
}
if id != tc.serviceAccount {
t.Errorf("service account not found correctly; got: %s, want: %s", id, tc.serviceAccount)
}
} else if err == nil {
t.Error("expected error but detectDefaultGoogleAccessID did not return one")
}
})
}
}
31 changes: 28 additions & 3 deletions storage/doc.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -214,8 +214,8 @@ since you read it. Here is how to express that:
You can obtain a URL that lets anyone read or write an object for a limited time.
Signing a URL requires credentials authorized to sign a URL. To use the same
authentication that was used when instantiating the Storage client, use the
BucketHandle.SignedURL method.
authentication that was used when instantiating the Storage client, use
[BucketHandle.SignedURL].
url, err := client.Bucket(bucketName).SignedURL(objectName, opts)
if err != nil {
Expand All @@ -224,7 +224,7 @@ BucketHandle.SignedURL method.
fmt.Println(url)
You can also sign a URL wihout creating a client. See the documentation of
SignedURL for details.
[SignedURL] for details.
url, err := storage.SignedURL(bucketName, "shared-object", opts)
if err != nil {
Expand All @@ -247,6 +247,26 @@ as the documentation of BucketHandle.GenerateSignedPostPolicyV4.
}
fmt.Printf("URL: %s\nFields; %v\n", pv4.URL, pv4.Fields)
# Credential requirements for [BucketHandle.SignedURL] and [BucketHandle.GenerateSignedPostPolicyV4]
If the GoogleAccessID and PrivateKey option fields are not provided, they will
be automatically detected if any of the following are true:
- you are authenticated to the Storage Client with a service account's
downloaded private key, either directly in code or by setting the
GOOGLE_APPLICATION_CREDENTIALS environment variable (see [Other Environments]),
- your application is running on Google Compute Engine (GCE), or
- you are logged into [gcloud using application default credentials]
with [impersonation enabled].
Detecting GoogleAccessID may not be possible if you are authenticated using a
token source or using [option.WithHTTPClient]. In this case, you can provide a
service account email for GoogleAccessID and the client will attempt to sign
the URL or Post Policy using that service account.
To generate the signature, you must have:
- iam.serviceAccounts.signBlob permissions on the GoogleAccessID service account, and
- the [IAM Service Account Credentials API] enabled (unless authenticating with a downloaded private key).
# Errors
Errors returned by this client are often of the type googleapi.Error.
Expand Down Expand Up @@ -296,5 +316,10 @@ client (using Client.SetRetry). For example:
if err := o.Delete(ctx); err != nil {
// Handle err.
}
[Other Environments]: https://cloud.google.com/storage/docs/authentication#libauth
[gcloud using application default credentials]: https://cloud.google.com/sdk/gcloud/reference/auth/application-default/login
[impersonation enabled]: https://cloud.google.com/sdk/gcloud/reference#--impersonate-service-account
[IAM Service Account Credentials API]: https://console.developers.google.com/apis/api/iamcredentials.googleapis.com/overview
*/
package storage // import "cloud.google.com/go/storage"

0 comments on commit a2d16a7

Please sign in to comment.