added security manager integration

README.md

@@ -22,8 +22,10 @@ The BigQuery driver accepts the following DSN
  * 'bigquery://projectID/[location/]datasetID?queryString'
 
     Where queryString can optionally configure the following option:
-      - credentialsFile
-      - credentialJSON (with url-base64 encoded json)
+      - credURL (url encoded) local location or  [Scy](https://github.com/viant/scy) secret manager URL
+      - credKey (url encoded)  SCY secret manager key
+      - credID [Scy](https://github.com/viant/scy) resource secret ID
+      - credJSON rawURL base64 encoded cred JSON (not recommended)
       - endpoint
       - userAgent
       - apiKey

dsn.go

@@ -1,17 +1,21 @@
 package bigquery
 
 import (
+	"context"
 	"encoding/base64"
 	"fmt"
+	"github.com/viant/scy"
 	"google.golang.org/api/option"
 	"net/url"
 	"strings"
 )
 
 const (
 	bigqueryScheme  = "bigquery"
-	credentialsFile = "credentialsFile"
-	credentialJSON  = "credentialJSON"
+	credentialsJSON = "credJSON"
+	credentialsURL  = "credURL"
+	credentialsKey  = "credKey"
+	credID          = "credID"
 	endpoint        = "endpoint"
 	userAgent       = "ua"
 	apiKey          = "apiKey"
@@ -28,7 +32,10 @@ type Config struct {
 	CredentialsFile string // Username
 	Endpoint        string
 	APIKey          string
-	CredentialJSON  string
+	CredentialJSON  []byte
+	CredentialsURL  string
+	CredID          string //scy secret resource ID
+	CredentialsKey  string
 	UserAgent       string
 	ProjectID       string // project ID
 	DatasetID       string
@@ -56,12 +63,8 @@ func (c *Config) options() []option.ClientOption {
 	if c.QuotaProject != "" {
 		result = append(result, option.WithQuotaProject(c.QuotaProject))
 	}
-	if c.CredentialJSON != "" {
-		JSON := []byte(c.CredentialJSON)
-		if raw, err := base64.RawURLEncoding.DecodeString(c.CredentialJSON); err == nil {
-			JSON = raw
-		}
-		result = append(result, option.WithCredentialsJSON(JSON))
+	if len(c.CredentialJSON) > 0 {
+		result = append(result, option.WithCredentialsJSON(c.CredentialJSON))
 	}
 
 	if len(c.Scopes) > 0 {
@@ -98,9 +101,6 @@ func ParseDSN(dsn string) (*Config, error) {
 		Values:    URL.Query(),
 	}
 	if len(cfg.Values) > 0 {
-		if _, ok := cfg.Values[credentialsFile]; ok {
-			cfg.CredentialsFile = cfg.Values.Get(credentialsFile)
-		}
 		if _, ok := cfg.Values[endpoint]; ok {
 			cfg.Endpoint = cfg.Values.Get(endpoint)
 		}
@@ -113,8 +113,17 @@ func ParseDSN(dsn string) (*Config, error) {
 		if _, ok := cfg.Values[app]; ok {
 			cfg.App = cfg.Values.Get(app)
 		}
-		if _, ok := cfg.Values[credentialJSON]; ok {
-			cfg.CredentialJSON = cfg.Values.Get(credentialJSON)
+		if _, ok := cfg.Values[credID]; ok {
+			cfg.CredID = cfg.Values.Get(credID)
+		}
+		if _, ok := cfg.Values[credentialsJSON]; ok {
+			cfg.CredentialJSON = []byte(cfg.Values.Get(credentialsJSON))
+		}
+		if _, ok := cfg.Values[credentialsKey]; ok {
+			cfg.CredentialsKey = cfg.Values.Get(credentialsKey)
+		}
+		if _, ok := cfg.Values[credentialsURL]; ok {
+			cfg.CredentialsURL = cfg.Values.Get(credentialsURL)
 		}
 		if _, ok := cfg.Values[quotaProject]; ok {
 			cfg.QuotaProject = cfg.Values.Get(quotaProject)
@@ -123,6 +132,17 @@ func ParseDSN(dsn string) (*Config, error) {
 			cfg.Scopes = cfg.Values[scopes]
 		}
 	}
+
+	if cfg.CredentialsKey != "" {
+		if URL, err := base64.RawURLEncoding.DecodeString(cfg.CredentialsKey); err == nil {
+			cfg.CredentialsKey = string(URL)
+		}
+	}
+
+	if err = cfg.initialiseSecrets(); err != nil {
+		return nil, err
+	}
+
 	if cfg.App == "" {
 		cfg.App = defaultApp
 	}
@@ -131,3 +151,50 @@ func ParseDSN(dsn string) (*Config, error) {
 	}
 	return cfg, nil
 }
+
+func (c *Config) initialiseSecrets() error {
+	if c.CredentialsURL != "" {
+		if URL, err := base64.RawURLEncoding.DecodeString(c.CredentialsURL); err == nil {
+			c.CredentialsURL = string(URL)
+		}
+	}
+	if c.CredentialsKey != "" {
+		if URL, err := base64.RawURLEncoding.DecodeString(c.CredentialsKey); err == nil {
+			c.CredentialsKey = string(URL)
+		}
+	}
+	if len(c.CredentialJSON) > 0 {
+		if raw, err := base64.RawURLEncoding.DecodeString(string(c.CredentialJSON)); err == nil {
+			c.CredentialJSON = raw
+		}
+	}
+	if c.CredID != "" {
+		resource := scy.Resources().Lookup(c.CredID)
+		if resource == nil {
+			return fmt.Errorf("failed to lookup secretID: %v", c.CredID)
+		}
+		secrets, err := c.loadSecret(resource)
+		if err != nil {
+			return err
+		}
+		c.CredentialJSON = []byte(secrets.String())
+	}
+
+	if c.CredentialsURL != "" {
+		secrets, err := c.loadSecret(&scy.Resource{URL: c.CredentialsURL, Key: c.CredentialsKey})
+		if err != nil {
+			return err
+		}
+		c.CredentialJSON = []byte(secrets.String())
+	}
+	return nil
+}
+
+func (c *Config) loadSecret(resource *scy.Resource) (*scy.Secret, error) {
+	secretsMgr := scy.New()
+	secrets, err := secretsMgr.Load(context.Background(), resource)
+	if err != nil {
+		return nil, fmt.Errorf("failed to load secret from :%v, %w", resource.URL, err)
+	}
+	return secrets, nil
+}

go.mod

@@ -4,30 +4,35 @@ go 1.17
 
 require (
 	github.com/francoispqt/gojay v1.2.13
-	github.com/stretchr/testify v1.7.0
-	github.com/viant/toolbox v0.24.0
+	github.com/stretchr/testify v1.7.1
+	github.com/viant/scy v0.3.0
 	github.com/viant/xunsafe v0.8.0
-	google.golang.org/api v0.63.0
+	google.golang.org/api v0.74.0
 )
 
 require (
-	cloud.google.com/go v0.99.0 // indirect
+	cloud.google.com/go/compute v1.5.0 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
-	github.com/go-errors/errors v1.0.1 // indirect
+	github.com/go-errors/errors v1.4.2 // indirect
 	github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect
 	github.com/golang/protobuf v1.5.2 // indirect
-	github.com/googleapis/gax-go/v2 v2.1.1 // indirect
-	github.com/pkg/errors v0.8.1 // indirect
+	github.com/googleapis/gax-go/v2 v2.2.0 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/viant/afs v1.16.1-0.20220601210902-dc23d64dda15 // indirect
+	github.com/viant/toolbox v0.34.5 // indirect
 	go.opencensus.io v0.23.0 // indirect
-	golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420 // indirect
-	golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8 // indirect
-	golang.org/x/sys v0.0.0-20211210111614-af8b64212486 // indirect
-	golang.org/x/text v0.3.6 // indirect
+	golang.org/x/crypto v0.0.0-20220507011949-2cf3adece122 // indirect
+	golang.org/x/net v0.0.0-20220325170049-de3da57026de // indirect
+	golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5 // indirect
+	golang.org/x/sys v0.0.0-20220328115105-d36c6a25d886 // indirect
+	golang.org/x/text v0.3.7 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
-	google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa // indirect
-	google.golang.org/grpc v1.40.1 // indirect
-	google.golang.org/protobuf v1.27.1 // indirect
-	gopkg.in/yaml.v2 v2.2.3 // indirect
+	google.golang.org/genproto v0.0.0-20220405205423-9d709892a2bf // indirect
+	google.golang.org/grpc v1.45.0 // indirect
+	google.golang.org/protobuf v1.28.0 // indirect
+	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c // indirect
 )
+
+replace github.com/viant/scy v0.3.0 => /Users/awitas/go/src/github.com/viant/scy

internal/query/call.go

@@ -38,7 +38,9 @@ func (c *ResultsCall) Context(ctx context.Context) {
 
 // Do runs a call
 func (c *ResultsCall) Do(opts ...googleapi.CallOption) (*Response, error) {
+
 	SetOptions(c.urlParams_, opts...)
+
 	res, err := c.doRequest("json")
 	if res.Body != nil {
 		data, err := io.ReadAll(res.Body)
@@ -65,9 +67,9 @@ func (c *ResultsCall) Do(opts ...googleapi.CallOption) (*Response, error) {
 	if err := googleapi.CheckResponse(res); err != nil {
 		return nil, err
 	}
+
 	ret := &Response{
 		session: c.session,
-
 		QueryResponse: bigquery.QueryResponse{
 			ServerResponse: googleapi.ServerResponse{
 				Header:         res.Header,
@@ -81,12 +83,6 @@ func (c *ResultsCall) Do(opts ...googleapi.CallOption) (*Response, error) {
 	if err != nil {
 		return nil, fmt.Errorf("failed to parseJSON: %w, %s", err, c.session.Data)
 	}
-
-	//for i, reg := range c.session.Rows {
-	//	rr := c.session.Data[reg.Begin:reg.End]
-	//	fmt.Printf("%v %s\n", i , rr)
-	//}
-
 	return ret, nil
 }