Skip to content

Commit

Permalink
test backup meets key locked error (pingcap#74)
Browse files Browse the repository at this point in the history
Signed-off-by: Neil Shen <overvenus@gmail.com>
  • Loading branch information
overvenus authored Dec 2, 2019
1 parent 428c603 commit e9bae2c
Show file tree
Hide file tree
Showing 13 changed files with 499 additions and 15 deletions.
4 changes: 4 additions & 0 deletions .golangci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -9,3 +9,7 @@ issues:
text: "Potential HTTP request made with variable url"
linters:
- gosec
- path: .go
text: "Use of weak random number generator"
linters:
- gosec
2 changes: 2 additions & 0 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,8 @@ build_for_integration_test:
GO111MODULE=on go test -c -cover -covermode=count \
-coverpkg=$(BR_PKG)/... \
-o bin/br.test
# build key locker
GO111MODULE=on go build -race -o bin/locker tests/br_key_locked/*.go

test:
GO111MODULE=on go test -race ./...
Expand Down
2 changes: 0 additions & 2 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -39,5 +39,3 @@ require (
)

replace github.com/golang/lint => golang.org/x/lint v0.0.0-20190930215403-16217165b5de

replace github.com/google/pprof => github.com/lonng/pprof v0.0.0-20191012154247-04dfd648ce8d
9 changes: 3 additions & 6 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@ github.com/cheggaaa/pb/v3 v3.0.1 h1:m0BngUk2LuSRYdx4fujDKNRXNDpbNCfptPfVT2m6OJY=
github.com/cheggaaa/pb/v3 v3.0.1/go.mod h1:SqqeMF/pMOIu3xgGoxtPYhMNQP258xE4x/XRTYua+KU=
github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
github.com/chzyer/readline v0.0.0-20171208011716-f6d7a1f6fbf3/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa h1:OaNxuTZr7kxeODyLWsRMC+OD03aFUH+mW6r2d+MWa5Y=
Expand Down Expand Up @@ -102,6 +101,8 @@ github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo=
github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/google/pprof v0.0.0-20190930153522-6ce02741cba3 h1:3CYI9xg87xNAD+es02gZxbX/ky4KQeoFBsNOzuoAQZg=
github.com/google/pprof v0.0.0-20190930153522-6ce02741cba3/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
github.com/google/shlex v0.0.0-20181106134648-c34317bd91bf/go.mod h1:RpwtwJQFrIEPstU94h88MWPXP2ektJZ8cZ0YntAmXiE=
github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY=
Expand All @@ -127,7 +128,6 @@ github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t
github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
github.com/jonboulle/clockwork v0.1.0 h1:VKV+ZcuP6l3yW9doeqz6ziZGgcynBVQO+obU0+0hcPo=
Expand All @@ -152,8 +152,6 @@ github.com/kr/pty v1.0.0/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/lonng/pprof v0.0.0-20191012154247-04dfd648ce8d h1:6Ike9EBxOFsCMMih14rQJmb7WPWdgRu4C0OLl6oRHwE=
github.com/lonng/pprof v0.0.0-20191012154247-04dfd648ce8d/go.mod h1:0vjxLpmyJvBwQbIQuxhHxmogQFpJvB9doVHvxFFfyoY=
github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
github.com/mattn/go-colorable v0.1.2 h1:/bC9yWikZXAL9uJdulbSfyVNIR3n3trXl+v8+1sx8mU=
Expand Down Expand Up @@ -360,9 +358,8 @@ golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5h
golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190909082730-f460065e899a h1:mIzbOulag9/gXacgxKlFVwpCOWSfBT3/pDyyCwGA9as=
golang.org/x/sys v0.0.0-20190909082730-f460065e899a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191010194322-b09406accb47 h1:/XfQ9z7ib8eEJX2hdgFTZJ/ntt0swNk5oYBziWeTCvY=
golang.org/x/sys v0.0.0-20191010194322-b09406accb47/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
Expand Down
2 changes: 1 addition & 1 deletion pkg/backup/client.go
Original file line number Diff line number Diff line change
Expand Up @@ -455,7 +455,7 @@ func (bc *Client) fineGrainedBackup(
ms int
mu sync.Mutex
}{}
wg := sync.WaitGroup{}
wg := new(sync.WaitGroup)
for i := 0; i < 4; i++ {
wg.Add(1)
fork, _ := bo.Fork()
Expand Down
2 changes: 1 addition & 1 deletion pkg/backup/push.go
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ func (push *pushDown) pushBackup(
updateCh chan<- struct{},
) (RangeTree, error) {
// Push down backup tasks to all tikv instances.
wg := sync.WaitGroup{}
wg := new(sync.WaitGroup)
for _, s := range stores {
storeID := s.GetId()
wg.Add(1)
Expand Down
3 changes: 2 additions & 1 deletion pkg/backup/schema.go
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ type Schemas struct {
schemas map[string]backup.Schema
backupSchemaCh chan backup.Schema
errCh chan error
wg sync.WaitGroup
wg *sync.WaitGroup
skipChecksum bool
}

Expand All @@ -40,6 +40,7 @@ func newBackupSchemas() *Schemas {
schemas: make(map[string]backup.Schema),
backupSchemaCh: make(chan backup.Schema),
errCh: make(chan error),
wg: new(sync.WaitGroup),
}
}

Expand Down
8 changes: 4 additions & 4 deletions pkg/restore/client.go
Original file line number Diff line number Diff line change
Expand Up @@ -257,7 +257,7 @@ func (rc *Client) RestoreTable(
zap.Array("files", files(table.Files)),
)
errCh := make(chan error, len(table.Files))
var wg sync.WaitGroup
wg := new(sync.WaitGroup)
defer close(errCh)
// We should encode the rewrite rewriteRules before using it to import files
encodedRules := encodeRewriteRules(rewriteRules)
Expand Down Expand Up @@ -309,7 +309,7 @@ func (rc *Client) RestoreDatabase(
log.Info("RestoreDatabase", zap.Stringer("db", db.Schema.Name), zap.Duration("take", elapsed))
}()
errCh := make(chan error, len(db.Tables))
var wg sync.WaitGroup
wg := new(sync.WaitGroup)
defer close(errCh)
for _, table := range db.Tables {
wg.Add(1)
Expand Down Expand Up @@ -345,7 +345,7 @@ func (rc *Client) RestoreAll(
log.Info("RestoreAll", zap.Duration("take", elapsed))
}()
errCh := make(chan error, len(rc.databases))
var wg sync.WaitGroup
wg := new(sync.WaitGroup)
defer close(errCh)
for _, db := range rc.databases {
wg.Add(1)
Expand Down Expand Up @@ -436,7 +436,7 @@ func (rc *Client) ValidateChecksum(
}()

log.Info("Start to validate checksum")
wg := sync.WaitGroup{}
wg := new(sync.WaitGroup)
errCh := make(chan error)
workers := utils.NewWorkerPool(defaultChecksumConcurrency, "RestoreChecksum")
go func() {
Expand Down
108 changes: 108 additions & 0 deletions tests/br_key_locked/codec.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,108 @@
// Copyright 2019 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.

// This file is copied from pingcap/tidb/store/tikv/pd_codec.go https://git.io/Je1Ww

package main

import (
"context"

"github.com/pingcap/errors"
"github.com/pingcap/kvproto/pkg/metapb"
pd "github.com/pingcap/pd/client"
"github.com/pingcap/tidb/util/codec"
)

type codecPDClient struct {
pd.Client
}

// GetRegion encodes the key before send requests to pd-server and decodes the
// returned StartKey && EndKey from pd-server.
func (c *codecPDClient) GetRegion(ctx context.Context, key []byte) (*metapb.Region, *metapb.Peer, error) {
encodedKey := codec.EncodeBytes(nil, key)
region, peer, err := c.Client.GetRegion(ctx, encodedKey)
return processRegionResult(region, peer, err)
}

func (c *codecPDClient) GetPrevRegion(ctx context.Context, key []byte) (*metapb.Region, *metapb.Peer, error) {
encodedKey := codec.EncodeBytes(nil, key)
region, peer, err := c.Client.GetPrevRegion(ctx, encodedKey)
return processRegionResult(region, peer, err)
}

// GetRegionByID encodes the key before send requests to pd-server and decodes the
// returned StartKey && EndKey from pd-server.
func (c *codecPDClient) GetRegionByID(ctx context.Context, regionID uint64) (*metapb.Region, *metapb.Peer, error) {
region, peer, err := c.Client.GetRegionByID(ctx, regionID)
return processRegionResult(region, peer, err)
}

func (c *codecPDClient) ScanRegions(
ctx context.Context,
startKey []byte,
endKey []byte,
limit int,
) ([]*metapb.Region, []*metapb.Peer, error) {
startKey = codec.EncodeBytes(nil, startKey)
if len(endKey) > 0 {
endKey = codec.EncodeBytes(nil, endKey)
}

regions, peers, err := c.Client.ScanRegions(ctx, startKey, endKey, limit)
if err != nil {
return nil, nil, errors.Trace(err)
}
for _, region := range regions {
if region != nil {
err = decodeRegionMetaKey(region)
if err != nil {
return nil, nil, errors.Trace(err)
}
}
}
return regions, peers, nil
}

func processRegionResult(region *metapb.Region, peer *metapb.Peer, err error) (*metapb.Region, *metapb.Peer, error) {
if err != nil {
return nil, nil, errors.Trace(err)
}
if region == nil {
return nil, nil, nil
}
err = decodeRegionMetaKey(region)
if err != nil {
return nil, nil, errors.Trace(err)
}
return region, peer, nil
}

func decodeRegionMetaKey(r *metapb.Region) error {
if len(r.StartKey) != 0 {
_, decoded, err := codec.DecodeBytes(r.StartKey, nil)
if err != nil {
return errors.Trace(err)
}
r.StartKey = decoded
}
if len(r.EndKey) != 0 {
_, decoded, err := codec.DecodeBytes(r.EndKey, nil)
if err != nil {
return errors.Trace(err)
}
r.EndKey = decoded
}
return nil
}
Loading

0 comments on commit e9bae2c

Please sign in to comment.