diff --git a/br/pkg/lightning/config/BUILD.bazel b/br/pkg/lightning/config/BUILD.bazel index 6d2cb32a9ccc7..61aeaae344a56 100644 --- a/br/pkg/lightning/config/BUILD.bazel +++ b/br/pkg/lightning/config/BUILD.bazel @@ -16,6 +16,7 @@ go_library( "//br/pkg/lightning/log", "//br/pkg/version/build", "//config", + "//parser/ast", "//parser/mysql", "//util", "//util/table-filter", diff --git a/br/pkg/lightning/config/config.go b/br/pkg/lightning/config/config.go index 87256290c9072..85991e7a79215 100644 --- a/br/pkg/lightning/config/config.go +++ b/br/pkg/lightning/config/config.go @@ -37,6 +37,7 @@ import ( "github.com/pingcap/tidb/br/pkg/lightning/common" "github.com/pingcap/tidb/br/pkg/lightning/log" tidbcfg "github.com/pingcap/tidb/config" + "github.com/pingcap/tidb/parser/ast" "github.com/pingcap/tidb/parser/mysql" "github.com/pingcap/tidb/util" filter "github.com/pingcap/tidb/util/table-filter" @@ -162,6 +163,16 @@ func (cfg *Config) String() string { return string(bytes) } +// Redact redacts the sensitive information. +func (cfg *Config) Redact() string { + originDir := cfg.Mydumper.SourceDir + defer func() { + cfg.Mydumper.SourceDir = originDir + }() + cfg.Mydumper.SourceDir = ast.RedactURL(cfg.Mydumper.SourceDir) + return cfg.String() +} + func (cfg *Config) ToTLS() (*common.TLS, error) { hostPort := net.JoinHostPort(cfg.TiDB.Host, strconv.Itoa(cfg.TiDB.StatusPort)) return common.NewTLS( diff --git a/br/pkg/lightning/config/config_test.go b/br/pkg/lightning/config/config_test.go index 63cdce382d73f..ee7f22676f43e 100644 --- a/br/pkg/lightning/config/config_test.go +++ b/br/pkg/lightning/config/config_test.go @@ -1019,3 +1019,30 @@ func TestCreateSeveralConfigsWithDifferentFilters(t *testing.T) { )) require.True(t, common.StringSliceEqual(config.GetDefaultFilter(), originalDefaultCfg)) } + +func TestRedactConfig(t *testing.T) { + tests := []struct { + origin string + redact string + }{ + {"", ""}, + {":", ":"}, + {"~/file", "~/file"}, + {"gs://bucket/file", "gs://bucket/file"}, + {"gs://bucket/file?access-key=123", "gs://bucket/file?access-key=123"}, + {"gs://bucket/file?secret-access-key=123", "gs://bucket/file?secret-access-key=123"}, + {"s3://bucket/file", "s3://bucket/file"}, + {"s3://bucket/file?other-key=123", "s3://bucket/file?other-key=123"}, + {"s3://bucket/file?access-key=123", "s3://bucket/file?access-key=xxxxxx"}, + {"s3://bucket/file?secret-access-key=123", "s3://bucket/file?secret-access-key=xxxxxx"}, + {"s3://bucket/file?access_key=123", "s3://bucket/file?access_key=xxxxxx"}, + {"s3://bucket/file?secret_access_key=123", "s3://bucket/file?secret_access_key=xxxxxx"}, + } + for _, tt := range tests { + cfg := config.NewConfig() + cfg.Mydumper.SourceDir = tt.origin + + require.Contains(t, cfg.Redact(), tt.redact) + require.Contains(t, cfg.String(), tt.origin) + } +} diff --git a/br/pkg/lightning/lightning.go b/br/pkg/lightning/lightning.go index 7c08a0082db90..1c4a4cfe1aa4b 100644 --- a/br/pkg/lightning/lightning.go +++ b/br/pkg/lightning/lightning.go @@ -381,7 +381,7 @@ var ( func (l *Lightning) run(taskCtx context.Context, taskCfg *config.Config, o *options) (err error) { build.LogInfo(build.Lightning) - o.logger.Info("cfg", zap.Stringer("cfg", taskCfg)) + o.logger.Info("cfg", zap.String("cfg", taskCfg.Redact())) utils.LogEnvVariables() diff --git a/parser/ast/misc.go b/parser/ast/misc.go index 9bd075a22f449..42c1754d20d34 100644 --- a/parser/ast/misc.go +++ b/parser/ast/misc.go @@ -3287,6 +3287,31 @@ func (n *BRIEStmt) Restore(ctx *format.RestoreCtx) error { return nil } +// RedactURL redacts the secret tokens in the URL. only S3 url need redaction for now. +// if the url is not a valid url, return the original string. +func RedactURL(str string) string { + // FIXME: this solution is not scalable, and duplicates some logic from BR. + u, err := url.Parse(str) + if err != nil { + return str + } + scheme := u.Scheme + switch strings.ToLower(scheme) { + case "s3", "ks3": + values := u.Query() + for k := range values { + // see below on why we normalize key + // https://github.com/pingcap/tidb/blob/a7c0d95f16ea2582bb569278c3f829403e6c3a7e/br/pkg/storage/parse.go#L163 + normalizedKey := strings.ToLower(strings.ReplaceAll(k, "_", "-")) + if normalizedKey == "access-key" || normalizedKey == "secret-access-key" || normalizedKey == "session-token" { + values[k] = []string{"xxxxxx"} + } + } + u.RawQuery = values.Encode() + } + return u.String() +} + // SecureText implements SensitiveStmtNode func (n *BRIEStmt) SecureText() string { // FIXME: this solution is not scalable, and duplicates some logic from BR.