Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

enable customizing mydumper options in CRD backup #2407

Merged
merged 16 commits into from
May 11, 2020
Merged

enable customizing mydumper options in CRD backup #2407

Show file tree
Hide file tree
Changes from 5 commits
Commits
Show all changes
16 commits
Select commit Hold shift + click to select a range
482b152
enable customizing mydumper options in CRD backup
May 8, 2020
8eced5f
Merge remote-tracking branch 'upstream/master' into customizing-mydum…
May 8, 2020
221a883
fix CI
May 9, 2020
22aa89b
fix CI
May 9, 2020
d9dc3f4
Merge branch 'master' into customizing-mydumper-options
DanielZhangQD May 9, 2020
0666b64
update api docs
May 9, 2020
5caffb6
Merge branch 'customizing-mydumper-options' of github.com:onlymellb/t…
May 9, 2020
20f93e4
Update pkg/apis/pingcap/v1alpha1/types.go
May 9, 2020
6f67e02
update api docs
May 9, 2020
aebcba8
Merge remote-tracking branch 'upstream/master' into customizing-mydum…
May 9, 2020
1b95ce7
update crd yaml
May 9, 2020
bf6a5f1
address comments
May 9, 2020
e383d59
add unit tests
May 9, 2020
5d269e1
simplify UT code
May 9, 2020
9c6e84d
Merge remote-tracking branch 'upstream/master' into customizing-mydum…
May 11, 2020
057bb5b
Merge remote-tracking branch 'upstream/master' into customizing-mydum…
May 11, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 3 additions & 6 deletions cmd/backup-manager/app/export/export.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ import (
"github.com/mholt/archiver"
"github.com/pingcap/tidb-operator/cmd/backup-manager/app/constants"
"github.com/pingcap/tidb-operator/cmd/backup-manager/app/util"
"github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1"
"k8s.io/klog"
)

Expand All @@ -48,7 +49,7 @@ func (bo *Options) getDestBucketURI(remotePath string) string {
return fmt.Sprintf("%s://%s", bo.StorageType, remotePath)
}

func (bo *Options) dumpTidbClusterData() (string, error) {
func (bo *Options) dumpTidbClusterData(backup *v1alpha1.Backup) (string, error) {
bfPath := bo.getBackupFullPath()
err := util.EnsureDirectoryExist(bfPath)
if err != nil {
Expand All @@ -60,12 +61,8 @@ func (bo *Options) dumpTidbClusterData() (string, error) {
fmt.Sprintf("--port=%d", bo.Port),
fmt.Sprintf("--user=%s", bo.User),
fmt.Sprintf("--password=%s", bo.Password),
"--long-query-guard=3600",
"--tidb-force-priority=LOW_PRIORITY",
"--verbose=3",
"--regex",
"^(?!(mysql|test|INFORMATION_SCHEMA|PERFORMANCE_SCHEMA|METRICS_SCHEMA|INSPECTION_SCHEMA))",
}
args = append(args, util.ConstructMydumperOptionsForBackup(backup)...)

output, err := exec.Command("/mydumper", args...).CombinedOutput()
if err != nil {
Expand Down
2 changes: 1 addition & 1 deletion cmd/backup-manager/app/export/manager.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -190,7 +190,7 @@ func (bm *BackupManager) performBackup(backup *v1alpha1.Backup, db *sql.DB) erro
klog.Infof("set cluster %s %s to %s success", bm, constants.TikvGCVariable, constants.TikvGCLifeTime)
}

backupFullPath, backupErr := bm.dumpTidbClusterData()
backupFullPath, backupErr := bm.dumpTidbClusterData(backup)
if oldTikvGCTimeDuration < tikvGCTimeDuration {
err = bm.SetTikvGCLifeTime(db, oldTikvGCTime)
if err != nil {
Expand Down
31 changes: 31 additions & 0 deletions cmd/backup-manager/app/util/util.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -135,6 +135,37 @@ func ConstructBRGlobalOptionsForBackup(backup *v1alpha1.Backup) ([]string, strin
return args, remotePath, nil
}

// ConstructMydumperOptionsForBackup constructs mydumper options for backup
func ConstructMydumperOptionsForBackup(backup *v1alpha1.Backup) []string {
var args []string
defaultOptions := []string{
"--long-query-guard=3600",
"--tidb-force-priority=LOW_PRIORITY",
"--verbose=3",
"--compress-protocol",
"--threads=16",
"--rows=10000",
"--skip-tz-utc",
}
defaultTableRegexOptions := []string{
"--regex",
"^(?!(mysql|test|INFORMATION_SCHEMA|PERFORMANCE_SCHEMA|METRICS_SCHEMA|INSPECTION_SCHEMA))",
}
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Move these to the var definitions at the beginning of the file?

config := backup.Spec.Mydumper
if config == nil {
onlymellb marked this conversation as resolved.
Show resolved Hide resolved
args = append(args, defaultOptions...)
args = append(args, defaultTableRegexOptions...)
return args
}
if len(config.Options) != 0 {
args = append(args, config.Options...)
DanielZhangQD marked this conversation as resolved.
Show resolved Hide resolved
}
if config.TableRegex != nil {
onlymellb marked this conversation as resolved.
Show resolved Hide resolved
args = append(args, "--regex", *config.TableRegex)
DanielZhangQD marked this conversation as resolved.
Show resolved Hide resolved
}
return args
}

// ConstructBRGlobalOptionsForRestore constructs BR global options for restore.
func ConstructBRGlobalOptionsForRestore(restore *v1alpha1.Restore) ([]string, error) {
var args []string
Expand Down
Loading