diff --git a/lightning/config/config.go b/lightning/config/config.go index 3a03674f4cfd7..9483ba8705eb7 100644 --- a/lightning/config/config.go +++ b/lightning/config/config.go @@ -33,9 +33,8 @@ type Config struct { PostRestore PostRestore `toml:"post-restore"` // command line flags - DoCompact string - DoChecksum string - file string + ConfigFile string + DoCompact bool } type Lightning struct { @@ -69,15 +68,14 @@ func LoadConfig(args []string) (*Config, error) { cfg.FlagSet = flag.NewFlagSet("lightning", flag.ContinueOnError) fs := cfg.FlagSet - fs.StringVar(&cfg.file, "c", "tidb-lightning.toml", "tidb-lightning configuration file") - fs.StringVar(&cfg.DoCompact, "compact", "", "do manual compact for tables which in comma separated format, like foo.bar1, foo.bar2") - fs.StringVar(&cfg.DoChecksum, "checksum", "", "do manual checksum for tables which in comma separated format, like foo.bar1,foo.bar2") + fs.StringVar(&cfg.ConfigFile, "c", "tidb-lightning.toml", "tidb-lightning configuration file") + fs.BoolVar(&cfg.DoCompact, "compact", false, "do manual compaction on the target cluster") if err := fs.Parse(args); err != nil { return nil, errors.Trace(err) } - data, err := ioutil.ReadFile(cfg.file) + data, err := ioutil.ReadFile(cfg.ConfigFile) if err != nil { return nil, errors.Trace(err) } diff --git a/lightning/lightning.go b/lightning/lightning.go index 9aaaaac11d783..81aa566d3af71 100644 --- a/lightning/lightning.go +++ b/lightning/lightning.go @@ -4,10 +4,8 @@ import ( "fmt" "net/http" "runtime" - "strings" "sync" - - "github.com/pingcap/tidb/tablecodec" + "time" "github.com/juju/errors" uuid "github.com/satori/go.uuid" @@ -61,24 +59,14 @@ func New(cfg *config.Config) *Lightning { func (l *Lightning) Run() { runtime.GOMAXPROCS(runtime.NumCPU()) - if l.cfg.DoCompact != "" { - tables := strings.Split(l.cfg.DoCompact, ",") - err := l.doCompact(tables) + if l.cfg.DoCompact { + err := l.doCompact() if err != nil { log.Errorf("compact error %s", errors.ErrorStack(err)) } return } - if l.cfg.DoChecksum != "" { - tables := strings.Split(l.cfg.DoChecksum, ",") - err := l.doChecksum(tables) - if err != nil { - log.Errorf("checksum error %s", errors.ErrorStack(err)) - } - return - } - l.wg.Add(1) go func() { defer l.wg.Done() @@ -102,56 +90,19 @@ func (l *Lightning) run() { return } -func (l *Lightning) doCompact(tables []string) error { +func (l *Lightning) doCompact() error { cli, err := kv.NewKVDeliverClient(context.Background(), uuid.Nil, l.cfg.ImportServer.Addr, l.cfg.TiDB.PdAddr) if err != nil { return errors.Trace(err) } defer cli.Close() - tidbMgr, err := restore.NewTiDBManager(l.cfg.TiDB.PdAddr) - if err != nil { - return errors.Trace(err) - } - defer tidbMgr.Close() - - for _, table := range tables { - log.Infof("begin compaction for table %s", table) - - // table must contains only one dot or we don't know how to split it. - if strings.Count(table, ".") != 1 { - log.Warnf("tables %s contains not dot or more than one dot which is not allowed", table) - continue - } - - split := strings.Split(table, ".") - tableInfo, err := tidbMgr.GetTableByName(split[0], split[1]) - if err != nil { - return errors.Trace(err) - } - - start := tablecodec.GenTablePrefix(tableInfo.ID) - end := tablecodec.GenTablePrefix(tableInfo.ID + 1) - if err := cli.Compact(start, end); err != nil { - return errors.Trace(err) - } - log.Infof("finished compaction for table %s", table) - } - - log.Info("compact done") - return nil -} - -func (l *Lightning) doChecksum(tables []string) error { - results, err := restore.DoChecksum(l.cfg.TiDB, tables) - if err != nil { + start := time.Now() + if err := cli.Compact([]byte{}, []byte{}); err != nil { return errors.Trace(err) } - for _, result := range results { - log.Infof("table %s.%s remote(from tidb) checksum %d, total_kvs, total_bytes %d", - result.Schema, result.Table, result.Checksum, result.TotalKVs, result.TotalBytes) - } + fmt.Println("compact takes", time.Since(start)) return nil } diff --git a/lightning/restore/restore.go b/lightning/restore/restore.go index 1493e721bd926..f46b72d2572bd 100644 --- a/lightning/restore/restore.go +++ b/lightning/restore/restore.go @@ -20,7 +20,6 @@ import ( "github.com/pingcap/tidb-lightning/lightning/mydump" verify "github.com/pingcap/tidb-lightning/lightning/verification" tidbcfg "github.com/pingcap/tidb/config" - "github.com/pingcap/tidb/tablecodec" ) var ( @@ -220,16 +219,8 @@ func (rc *RestoreControlloer) compact(ctx context.Context) error { } defer cli.Close() - for _, table := range rc.dbInfo.Tables { - timer := time.Now() - log.Infof("[%s] compact", table.Name) - start := tablecodec.GenTablePrefix(table.ID) - end := tablecodec.GenTablePrefix(table.ID + 1) - err = cli.Compact(start, end) - if err != nil { - return errors.Trace(err) - } - log.Infof("[%s] compact takes %v", table.Name, time.Since(timer)) + if err := cli.Compact([]byte{}, []byte{}); err != nil { + return errors.Trace(err) } return nil }