Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[to #67] remove unused code related to restore #76

Merged
merged 1 commit into from
Mar 29, 2022
Merged

[to #67] remove unused code related to restore #76

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
174 changes: 3 additions & 171 deletions br/cmd/br/debug.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,31 +3,18 @@
package main

import (
"bytes"
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"path"
"reflect"

"github.com/gogo/protobuf/proto"
"github.com/pingcap/errors"
backuppb "github.com/pingcap/kvproto/pkg/brpb"
"github.com/pingcap/kvproto/pkg/import_sstpb"
"github.com/pingcap/log"
"github.com/pingcap/tidb/parser/model"
"github.com/spf13/cobra"
berrors "github.com/tikv/migration/br/pkg/errors"
"github.com/tikv/migration/br/pkg/logutil"
"github.com/tikv/migration/br/pkg/metautil"
"github.com/tikv/migration/br/pkg/mock/mockid"
"github.com/tikv/migration/br/pkg/restore"
"github.com/tikv/migration/br/pkg/rtree"
"github.com/tikv/migration/br/pkg/task"
"github.com/tikv/migration/br/pkg/utils"
"github.com/tikv/migration/br/pkg/version/build"
"go.uber.org/zap"
)

// NewDebugCommand return a debug subcommand.
Expand Down Expand Up @@ -64,80 +51,7 @@ func newCheckSumCommand() *cobra.Command {
Short: "check the backup data",
Args: cobra.NoArgs,
RunE: func(cmd *cobra.Command, _ []string) error {
ctx, cancel := context.WithCancel(GetDefaultContext())
defer cancel()

var cfg task.Config
if err := cfg.ParseFromFlags(cmd.Flags()); err != nil {
return errors.Trace(err)
}

_, s, backupMeta, err := task.ReadBackupMeta(ctx, metautil.MetaFile, &cfg)
if err != nil {
return errors.Trace(err)
}

reader := metautil.NewMetaReader(backupMeta, s, &cfg.CipherInfo)
dbs, err := utils.LoadBackupTables(ctx, reader)
if err != nil {
return errors.Trace(err)
}

for _, schema := range backupMeta.Schemas {
dbInfo := &model.DBInfo{}
err = json.Unmarshal(schema.Db, dbInfo)
if err != nil {
return errors.Trace(err)
}
tblInfo := &model.TableInfo{}
err = json.Unmarshal(schema.Table, tblInfo)
if err != nil {
return errors.Trace(err)
}
tbl := dbs[dbInfo.Name.String()].GetTable(tblInfo.Name.String())

var calCRC64 uint64
var totalKVs uint64
var totalBytes uint64
for _, file := range tbl.Files {
calCRC64 ^= file.Crc64Xor
totalKVs += file.GetTotalKvs()
totalBytes += file.GetTotalBytes()
log.Info("file info", zap.Stringer("table", tblInfo.Name),
zap.String("file", file.GetName()),
zap.Uint64("crc64xor", file.GetCrc64Xor()),
zap.Uint64("totalKvs", file.GetTotalKvs()),
zap.Uint64("totalBytes", file.GetTotalBytes()),
zap.Uint64("startVersion", file.GetStartVersion()),
zap.Uint64("endVersion", file.GetEndVersion()),
logutil.Key("startKey", file.GetStartKey()),
logutil.Key("endKey", file.GetEndKey()),
)

var data []byte
data, err = s.ReadFile(ctx, file.Name)
if err != nil {
return errors.Trace(err)
}
s := sha256.Sum256(data)
if !bytes.Equal(s[:], file.Sha256) {
return errors.Annotatef(berrors.ErrBackupChecksumMismatch, `
backup data checksum failed: %s may be changed
calculated sha256 is %s,
origin sha256 is %s`,
file.Name, hex.EncodeToString(s[:]), hex.EncodeToString(file.Sha256))
}
}
log.Info("table info", zap.Stringer("table", tblInfo.Name),
zap.Uint64("CRC64", calCRC64),
zap.Uint64("totalKvs", totalKVs),
zap.Uint64("totalBytes", totalBytes),
zap.Uint64("schemaTotalKvs", schema.TotalKvs),
zap.Uint64("schemaTotalBytes", schema.TotalBytes),
zap.Uint64("schemaCRC64", schema.Crc64Xor))
}
cmd.Println("backup data checksum succeed!")
return nil
return errors.Errorf("checksum is unsupported")
},
}
command.Hidden = true
Expand All @@ -159,89 +73,7 @@ func newBackupMetaValidateCommand() *cobra.Command {
Use: "validate",
Short: "validate key range and rewrite rules of backupmeta",
RunE: func(cmd *cobra.Command, _ []string) error {
ctx, cancel := context.WithCancel(GetDefaultContext())
defer cancel()

tableIDOffset, err := cmd.Flags().GetUint64("offset")
if err != nil {
return errors.Trace(err)
}

var cfg task.Config
if err = cfg.ParseFromFlags(cmd.Flags()); err != nil {
return errors.Trace(err)
}
_, s, backupMeta, err := task.ReadBackupMeta(ctx, metautil.MetaFile, &cfg)
if err != nil {
log.Error("read backupmeta failed", zap.Error(err))
return errors.Trace(err)
}
reader := metautil.NewMetaReader(backupMeta, s, &cfg.CipherInfo)
dbs, err := utils.LoadBackupTables(ctx, reader)
if err != nil {
log.Error("load tables failed", zap.Error(err))
return errors.Trace(err)
}
files := make([]*backuppb.File, 0)
tables := make([]*metautil.Table, 0)
for _, db := range dbs {
for _, table := range db.Tables {
files = append(files, table.Files...)
}
tables = append(tables, db.Tables...)
}
// Check if the ranges of files overlapped
rangeTree := rtree.NewRangeTree()
for _, file := range files {
if out := rangeTree.InsertRange(rtree.Range{
StartKey: file.GetStartKey(),
EndKey: file.GetEndKey(),
}); out != nil {
log.Error(
"file ranges overlapped",
zap.Stringer("out", out),
logutil.File(file),
)
}
}

tableIDAllocator := mockid.NewIDAllocator()
// Advance table ID allocator to the offset.
for offset := uint64(0); offset < tableIDOffset; offset++ {
_, _ = tableIDAllocator.Alloc() // Ignore error
}
rewriteRules := &restore.RewriteRules{
Data: make([]*import_sstpb.RewriteRule, 0),
}
tableIDMap := make(map[int64]int64)
// Simulate to create table
for _, table := range tables {
indexIDAllocator := mockid.NewIDAllocator()
newTable := new(model.TableInfo)
tableID, _ := tableIDAllocator.Alloc()
newTable.ID = int64(tableID)
newTable.Name = table.Info.Name
newTable.Indices = make([]*model.IndexInfo, len(table.Info.Indices))
for i, indexInfo := range table.Info.Indices {
indexID, _ := indexIDAllocator.Alloc()
newTable.Indices[i] = &model.IndexInfo{
ID: int64(indexID),
Name: indexInfo.Name,
}
}
rules := restore.GetRewriteRules(newTable, table.Info, 0)
rewriteRules.Data = append(rewriteRules.Data, rules.Data...)
tableIDMap[table.Info.ID] = int64(tableID)
}
// Validate rewrite rules
for _, file := range files {
err = restore.ValidateFileRewriteRule(file, rewriteRules)
if err != nil {
return errors.Trace(err)
}
}
cmd.Println("Check backupmeta done")
return nil
return errors.Errorf("validate is unsupported")
},
}
command.Flags().Uint64("offset", 0, "the offset of table id alloctor")
Expand Down Expand Up @@ -376,7 +208,7 @@ func setPDConfigCommand() *cobra.Command {
return errors.Trace(err)
}

mgr, err := task.NewMgr(ctx, tidbGlue, cfg.PD, cfg.TLS, task.GetKeepalive(&cfg), cfg.CheckRequirements, false)
mgr, err := task.NewMgr(ctx, tidbGlue, cfg.PD, cfg.TLS, task.GetKeepalive(&cfg), cfg.CheckRequirements)
if err != nil {
return errors.Trace(err)
}
Expand Down
21 changes: 0 additions & 21 deletions br/pkg/conn/conn.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,6 @@ import (
backuppb "github.com/pingcap/kvproto/pkg/brpb"
"github.com/pingcap/kvproto/pkg/metapb"
"github.com/pingcap/log"
"github.com/pingcap/tidb/domain"
"github.com/pingcap/tidb/kv"
"github.com/tikv/client-go/v2/tikv"
"github.com/tikv/client-go/v2/txnkv/txnlock"
Expand Down Expand Up @@ -104,7 +103,6 @@ func NewConnPool(cap int, newConn func(ctx context.Context) (*grpc.ClientConn, e
type Mgr struct {
*pdutil.PdController
tlsConf *tls.Config
dom *domain.Domain
storage kv.Storage // Used to access SQL related interfaces.
tikvStore tikv.Storage // Used to access TiKV specific interfaces.
grpcClis struct {
Expand Down Expand Up @@ -222,7 +220,6 @@ func checkStoresAlive(ctx context.Context,

// NewMgr creates a new Mgr.
//
// Domain is optional for Backup, set `needDomain` to false to disable
// initializing Domain.
func NewMgr(
ctx context.Context,
Expand All @@ -233,7 +230,6 @@ func NewMgr(
keepalive keepalive.ClientParameters,
storeBehavior StoreBehavior,
checkRequirements bool,
needDomain bool,
) (*Mgr, error) {
if span := opentracing.SpanFromContext(ctx); span != nil && span.Tracer() != nil {
span1 := span.Tracer().StartSpan("conn.NewMgr", opentracing.ChildOf(span.Context()))
Expand Down Expand Up @@ -272,19 +268,10 @@ func NewMgr(
return nil, berrors.ErrKVNotTiKV
}

var dom *domain.Domain
if needDomain {
dom, err = g.GetDomain(storage)
if err != nil {
return nil, errors.Trace(err)
}
}

mgr := &Mgr{
PdController: controller,
storage: storage,
tikvStore: tikvStorage,
dom: dom,
tlsConf: tlsConf,
ownsStorage: g.OwnsStorage(),
grpcClis: struct {
Expand Down Expand Up @@ -418,11 +405,6 @@ func (mgr *Mgr) GetLockResolver() *txnlock.LockResolver {
return mgr.tikvStore.GetLockResolver()
}

// GetDomain returns a tikv storage.
func (mgr *Mgr) GetDomain() *domain.Domain {
return mgr.dom
}

// Close closes all client in Mgr.
func (mgr *Mgr) Close() {
mgr.grpcClis.mu.Lock()
Expand All @@ -437,9 +419,6 @@ func (mgr *Mgr) Close() {
// Gracefully shutdown domain so it does not affect other TiDB DDL.
// Must close domain before closing storage, otherwise it gets stuck forever.
if mgr.ownsStorage {
if mgr.dom != nil {
mgr.dom.Close()
}
tikv.StoreShuttingDown(1)
mgr.storage.Close()
}
Expand Down
8 changes: 0 additions & 8 deletions br/pkg/glue/glue.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,16 +5,12 @@ package glue
import (
"context"

"github.com/pingcap/tidb/domain"
"github.com/pingcap/tidb/kv"
"github.com/pingcap/tidb/parser/model"
pd "github.com/tikv/pd/client"
)

// Glue is an abstraction of TiDB function calls used in BR.
type Glue interface {
GetDomain(store kv.Storage) (*domain.Domain, error)
CreateSession(store kv.Storage) (Session, error)
Open(path string, option pd.SecurityOption) (kv.Storage, error)

// OwnsStorage returns whether the storage returned by Open() is owned
Expand All @@ -32,10 +28,6 @@ type Glue interface {

// Session is an abstraction of the session.Session interface.
type Session interface {
Execute(ctx context.Context, sql string) error
ExecuteInternal(ctx context.Context, sql string, args ...interface{}) error
CreateDatabase(ctx context.Context, schema *model.DBInfo) error
CreateTable(ctx context.Context, dbName model.CIStr, table *model.TableInfo) error
Close()
}

Expand Down
Loading