From f24ed201720ca8495d9dfd9647ebda460a7464f8 Mon Sep 17 00:00:00 2001 From: WangXiangUSTC Date: Tue, 9 Apr 2019 16:36:18 +0800 Subject: [PATCH 1/5] sync-diff-inspector: support checkpoint --- go.mod1 | 26 +- go.sum1 | 36 ++- pkg/dbutil/common.go | 136 ++++++++++ pkg/dbutil/common_test.go | 60 +++++ pkg/diff/checkpoint.go | 377 +++++++++++++++++++++++++++ pkg/diff/checkpoint_test.go | 124 +++++++++ pkg/diff/chunk.go | 216 ++++++++-------- pkg/diff/chunk_test.go | 81 +++--- pkg/diff/diff.go | 411 ++++++++++++++++++++++-------- pkg/diff/diff_test.go | 19 ++ pkg/diff/spliter_test.go | 9 +- pkg/diff/util.go | 22 -- pkg/utils/errors_test.go | 36 +++ sync_diff_inspector/config.go | 4 + sync_diff_inspector/config.toml | 3 + sync_diff_inspector/diff.go | 24 +- tidb-binlog/pump_client/client.go | 2 +- tidb-binlog/pump_client/pump.go | 2 +- 18 files changed, 1286 insertions(+), 302 deletions(-) create mode 100644 pkg/diff/checkpoint.go create mode 100644 pkg/diff/checkpoint_test.go create mode 100644 pkg/utils/errors_test.go diff --git a/go.mod1 b/go.mod1 index dd076978b..e1db8da71 100644 --- a/go.mod1 +++ b/go.mod1 @@ -36,7 +36,6 @@ require ( github.com/grpc-ecosystem/grpc-gateway v1.5.1 // indirect github.com/jonboulle/clockwork v0.1.0 // indirect github.com/klauspost/cpuid v0.0.0-20180405133222-e7e905edc00e // indirect - github.com/konsorten/go-windows-terminal-sequences v1.0.1 // indirect github.com/kr/pretty v0.1.0 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect github.com/montanaflynn/stats v0.0.0-20180911141734-db72e6cae808 // indirect @@ -46,15 +45,15 @@ require ( github.com/opentracing/basictracer-go v1.0.0 // indirect github.com/opentracing/opentracing-go v1.0.2 // indirect github.com/pierrec/lz4 v2.0.5+incompatible // indirect - github.com/pingcap/check v0.0.0-20171206051426-1c287c953996 + github.com/pingcap/check v0.0.0-20190102082844-67f458068fc8 github.com/pingcap/errors v0.11.0 github.com/pingcap/goleveldb v0.0.0-20171020122428-b9ff6c35079e // indirect github.com/pingcap/kvproto v0.0.0-20181028030329-855d2192cdc7 + github.com/pingcap/log v0.0.0-20190307075452-bd41d9273596 github.com/pingcap/parser v0.0.0-20181024082006-53ac409ed043 github.com/pingcap/pd v2.1.0-rc.4+incompatible github.com/pingcap/tidb v2.1.0-rc.3.0.20181101080907-32b1dbd8d59f+incompatible github.com/pingcap/tipb v0.0.0-20181012112600-11e33c750323 - github.com/pkg/errors v0.8.0 // indirect github.com/prometheus/client_golang v0.9.0 // indirect github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910 // indirect github.com/prometheus/common v0.0.0-20181020173914-7e9e6cabbd39 // indirect @@ -62,19 +61,18 @@ require ( github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a // indirect github.com/remyoudompheng/bigfft v0.0.0-20170806203942-52369c62f446 // indirect github.com/siddontang/go v0.0.0-20180604090527-bdc77568d726 - github.com/sirupsen/logrus v1.1.1 + github.com/siddontang/go-mysql v0.0.0-20190312052122-c6ab05a85eb8 + github.com/sirupsen/logrus v1.4.1 github.com/soheilhy/cmux v0.1.4 // indirect - github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72 // indirect - github.com/tmc/grpc-websocket-proxy v0.0.0-20171017195756-830351dc03c6 // indirect + github.com/spaolacci/murmur3 v1.1.0 // indirect + github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5 // indirect github.com/twinj/uuid v1.0.0 // indirect - github.com/uber/jaeger-client-go v2.15.0+incompatible // indirect - github.com/uber/jaeger-lib v1.5.0 // indirect - github.com/ugorji/go v1.1.1 // indirect - github.com/unrolled/render v0.0.0-20180914162206-b9786414de4d // indirect - github.com/xiang90/probing v0.0.0-20160813154853-07dd2e8dfe18 // indirect + github.com/uber/jaeger-client-go v2.16.0+incompatible // indirect + github.com/uber/jaeger-lib v2.0.0+incompatible // indirect + github.com/ugorji/go v1.1.4 // indirect + github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 // indirect go.uber.org/atomic v1.3.2 - go.uber.org/multierr v1.1.0 // indirect - go.uber.org/zap v1.8.0 // indirect + go.uber.org/zap v1.9.1 golang.org/x/crypto v0.0.0-20181030102418-4d3f4d9ffa16 // indirect golang.org/x/net v0.0.0-20181029044818-c44066c5c816 golang.org/x/sys v0.0.0-20181030150119-7e31e0c00fa0 // indirect @@ -84,6 +82,4 @@ require ( google.golang.org/grpc v1.16.0 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce // indirect - gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect - gopkg.in/yaml.v2 v2.2.1 // indirect ) diff --git a/go.sum1 b/go.sum1 index b459c82d7..b575206b4 100644 --- a/go.sum1 +++ b/go.sum1 @@ -25,6 +25,7 @@ github.com/cznic/mathutil v0.0.0-20181021201202-eba54fb065b7 h1:y+DH9ARrWiiNBV+6 github.com/cznic/mathutil v0.0.0-20181021201202-eba54fb065b7/go.mod h1:e6NPNENfs9mPDVNRekM7lKScauxd5kXTr1Mfyig6TDM= github.com/cznic/sortutil v0.0.0-20150617083342-4c7342852e65 h1:hxuZop6tSoOi0sxFzoGGYdRqNrPubyaIf9KoBG9tPiE= github.com/cznic/sortutil v0.0.0-20150617083342-4c7342852e65/go.mod h1:q2w6Bg5jeox1B+QkJ6Wp/+Vn0G/bo3f1uY7Fn3vivIQ= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= @@ -91,12 +92,15 @@ github.com/pierrec/lz4 v2.0.5+incompatible h1:2xWsjqPFWcplujydGg4WmhC/6fZqK42wMM github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pingcap/check v0.0.0-20171206051426-1c287c953996 h1:ZBdiJCMan6GSo/aPAM7gywcUKa0z58gczVrnG6TQnAQ= github.com/pingcap/check v0.0.0-20171206051426-1c287c953996/go.mod h1:B1+S9LNcuMyLH/4HMTViQOJevkGiik3wW2AN9zb2fNQ= +github.com/pingcap/check v0.0.0-20190102082844-67f458068fc8 h1:USx2/E1bX46VG32FIw034Au6seQ2fY9NEILmNh/UlQg= +github.com/pingcap/check v0.0.0-20190102082844-67f458068fc8/go.mod h1:B1+S9LNcuMyLH/4HMTViQOJevkGiik3wW2AN9zb2fNQ= github.com/pingcap/errors v0.11.0 h1:DCJQB8jrHbQ1VVlMFIrbj2ApScNNotVmkSNplu2yUt4= github.com/pingcap/errors v0.11.0/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= github.com/pingcap/goleveldb v0.0.0-20171020122428-b9ff6c35079e h1:P73/4dPCL96rGrobssy1nVy2VaVpNCuLpCbr+FEaTA8= github.com/pingcap/goleveldb v0.0.0-20171020122428-b9ff6c35079e/go.mod h1:O17XtbryoCJhkKGbT62+L2OlrniwqiGLSqrmdHCMzZw= github.com/pingcap/kvproto v0.0.0-20181028030329-855d2192cdc7 h1:CYssSnPvf90ZSbFdZpsZGSI7y+drG1EfKxqTOnKnHb0= github.com/pingcap/kvproto v0.0.0-20181028030329-855d2192cdc7/go.mod h1:0gwbe1F2iBIjuQ9AH0DbQhL+Dpr5GofU8fgYyXk+ykk= +github.com/pingcap/log v0.0.0-20190307075452-bd41d9273596/go.mod h1:WpHUKhNZ18v116SvGrmjkA9CBhYmuUTKL+p8JC9ANEw= github.com/pingcap/parser v0.0.0-20181024082006-53ac409ed043 h1:P9Osi8lei5j2fiRgsBi2Wch7qe4a3yWUOsS5vSan/JU= github.com/pingcap/parser v0.0.0-20181024082006-53ac409ed043/go.mod h1:1FNvfp9+J0wvc4kl8eGNh7Rqrxveg15jJoWo/a0uHwA= github.com/pingcap/pd v2.1.0-rc.4+incompatible h1:/buwGk04aHO5odk/+O8ZOXGs4qkUjYTJ2UpCJXna8NE= @@ -107,6 +111,7 @@ github.com/pingcap/tipb v0.0.0-20181012112600-11e33c750323 h1:mRKKzRjDNaUNPnAkPA github.com/pingcap/tipb v0.0.0-20181012112600-11e33c750323/go.mod h1:RtkHW8WbcNxj8lsbzjaILci01CtYnYbIkQhjyZWrWVI= github.com/pkg/errors v0.8.0 h1:WdK/asTD0HN+q6hsWO3/vpuAkAr+tw6aNJNDFFf0+qw= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v0.9.0 h1:tXuTFVHC03mW0D+Ua1Q2d1EAVqLTuggX50V0VLICCzY= github.com/prometheus/client_golang v0.9.0/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= @@ -120,34 +125,59 @@ github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a h1:9ZKAASQSHhD github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/remyoudompheng/bigfft v0.0.0-20170806203942-52369c62f446 h1:/NRJ5vAYoqz+7sG51ubIDHXeWO8DlTSrToPu6q11ziA= github.com/remyoudompheng/bigfft v0.0.0-20170806203942-52369c62f446/go.mod h1:uYEyJGbgTkfkS4+E/PavXkNJcbFIpEtjt2B0KDQ5+9M= -github.com/siddontang/go v0.0.0-20180604090527-bdc77568d726 h1:xT+JlYxNGqyT+XcU8iUrN18JYed2TvG9yN5ULG2jATM= +github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= +github.com/shirou/gopsutil v2.18.10+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= +github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= +github.com/shurcooL/httpfs v0.0.0-20171119174359-809beceb2371/go.mod h1:ZY1cvUeJuFPAdZ/B6v7RHavJWZn2YPVFQ1OSXhCGOkg= +github.com/shurcooL/vfsgen v0.0.0-20181020040650-a97a25d856ca/go.mod h1:TrYk7fJVaAttu97ZZKrO9UbRa8izdowaMIZcxYMbVaw= github.com/siddontang/go v0.0.0-20180604090527-bdc77568d726/go.mod h1:3yhqj7WBBfRhbBlzyOC3gUxftwsU0u8gqevxwIHQpMw= -github.com/sirupsen/logrus v1.1.1 h1:VzGj7lhU7KEB9e9gMpAV/v5XT2NVSvLJhJLCWbnkgXg= +github.com/siddontang/go-log v0.0.0-20180807004314-8d05993dda07 h1:oI+RNwuC9jF2g2lP0u0cVEEZrc/AYBCuFdvwrLWM/6Q= +github.com/siddontang/go-log v0.0.0-20180807004314-8d05993dda07/go.mod h1:yFdBgwXP24JziuRl2NMUahT7nGLNOKi1SIiFxMttVD4= +github.com/siddontang/go-mysql v0.0.0-20190312052122-c6ab05a85eb8 h1:8puKTg/UOIQ+ZiowY1ywmGsI08sWqrKD7HJ/j165CUM= +github.com/siddontang/go-mysql v0.0.0-20190312052122-c6ab05a85eb8/go.mod h1:/b8ZcWjAShCcHp2dWpjb1vTlNyiG03UeHEQr2jteOpI= +github.com/sirupsen/logrus v0.0.0-20170323161349-3bcb09397d6d/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= github.com/sirupsen/logrus v1.1.1/go.mod h1:zrgwTnHtNr00buQ1vSptGe8m1f/BbgsPukg8qsT7A+A= +github.com/sirupsen/logrus v1.4.1 h1:GL2rEmy6nsikmW0r8opw9JIRScdMF5hA8cOYLH7In1k= +github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= github.com/soheilhy/cmux v0.1.4 h1:0HKaf1o97UwFjHH9o5XsHUOF+tqmdA7KEzXLpiyaw0E= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72 h1:qLC7fQah7D6K1B0ujays3HV9gkFtllcxhzImRR7ArPQ= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= +github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/tmc/grpc-websocket-proxy v0.0.0-20171017195756-830351dc03c6 h1:lYIiVDtZnyTWlNwiAxLj0bbpTcx1BWCFhXjfsvmPdNc= github.com/tmc/grpc-websocket-proxy v0.0.0-20171017195756-830351dc03c6/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/twinj/uuid v1.0.0 h1:fzz7COZnDrXGTAOHGuUGYd6sG+JMq+AoE7+Jlu0przk= github.com/twinj/uuid v1.0.0/go.mod h1:mMgcE1RHFUFqe5AfiwlINXisXfDGro23fWdPUfOMjRY= github.com/uber/jaeger-client-go v2.15.0+incompatible h1:NP3qsSqNxh8VYr956ur1N/1C1PjvOJnJykCzcD5QHbk= github.com/uber/jaeger-client-go v2.15.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk= +github.com/uber/jaeger-client-go v2.16.0+incompatible h1:Q2Pp6v3QYiocMxomCaJuwQGFt7E53bPYqEgug/AoBtY= +github.com/uber/jaeger-client-go v2.16.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk= github.com/uber/jaeger-lib v1.5.0 h1:OHbgr8l656Ub3Fw5k9SWnBfIEwvoHQ+W2y+Aa9D1Uyo= github.com/uber/jaeger-lib v1.5.0/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U= +github.com/uber/jaeger-lib v2.0.0+incompatible h1:iMSCV0rmXEogjNWPh2D0xk9YVKvrtGoHJNe9ebLu/pw= +github.com/uber/jaeger-lib v2.0.0+incompatible/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U= github.com/ugorji/go v1.1.1 h1:gmervu+jDMvXTbcHQ0pd2wee85nEoE0BsVyEuzkfK8w= github.com/ugorji/go v1.1.1/go.mod h1:hnLbHMwcvSihnDhEfx2/BzKp2xb0Y+ErdfYcrs9tkJQ= +github.com/ugorji/go v1.1.4 h1:j4s+tAvLfL3bZyefP2SEWmhBzmuIlH/eqNuPdFPgngw= +github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= github.com/unrolled/render v0.0.0-20180914162206-b9786414de4d/go.mod h1:tu82oB5W2ykJRVioYsB+IQKcft7ryBr7w12qMBUPyXg= github.com/xiang90/probing v0.0.0-20160813154853-07dd2e8dfe18 h1:MPPkRncZLN9Kh4MEFmbnK4h3BD7AUmskWv2+EeZJCCs= github.com/xiang90/probing v0.0.0-20160813154853-07dd2e8dfe18/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= +github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8= +github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= go.uber.org/atomic v1.3.2 h1:2Oa65PReHzfn29GpvgsYwloV9AVFHPDk8tYxt2c2tr4= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/multierr v1.1.0 h1:HoEmRHQPVSqub6w2z2d2EOVs2fjyFRGyofhKuyDq0QI= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/zap v1.8.0 h1:r6Za1Rii8+EGOYRDLvpooNOF6kP3iyDnkpzbw67gCQ8= go.uber.org/zap v1.8.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181030102418-4d3f4d9ffa16 h1:y6ce7gCWtnH+m3dCjzQ1PCuwl28DDIc3VNnvY29DlIA= golang.org/x/crypto v0.0.0-20181030102418-4d3f4d9ffa16/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -181,4 +211,6 @@ gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXL gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= gopkg.in/yaml.v2 v2.2.1 h1:mUhvW9EsL+naU5Q3cakzfE91YhliOondGd6ZrsDBHQE= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/pkg/dbutil/common.go b/pkg/dbutil/common.go index d6a513c25..3f38898f6 100644 --- a/pkg/dbutil/common.go +++ b/pkg/dbutil/common.go @@ -20,12 +20,20 @@ import ( "os" "strconv" "strings" + "time" + "github.com/go-sql-driver/mysql" "github.com/pingcap/errors" "github.com/pingcap/parser/model" + tmysql "github.com/pingcap/parser/mysql" + "github.com/pingcap/parser/terror" "github.com/pingcap/tidb-tools/pkg/utils" + "github.com/pingcap/tidb/ddl" + "github.com/pingcap/tidb/infoschema" "github.com/pingcap/tidb/types" + gmysql "github.com/siddontang/go-mysql/mysql" log "github.com/sirupsen/logrus" + "go.uber.org/zap" ) const ( @@ -34,6 +42,15 @@ const ( // ImplicitColID is ID implicit column in TiDB ImplicitColID = -1 + + // DefaultRetryTime is the default retry time to execute sql + DefaultRetryTime = 10 + + // DefaultTimeout is the default timeout for execute sql + DefaultTimeout time.Duration = 5 * time.Second + + // SlowWarnLog defines the duration to log warn log of sql when exec time greater than + SlowWarnLog = 100 * time.Millisecond ) var ( @@ -616,3 +633,122 @@ func ReplacePlaceholder(str string, args []string) string { newStr := strings.Replace(str, "?", "'%s'", -1) return fmt.Sprintf(newStr, utils.StringsToInterfaces(args)...) } + +// ExecSQLWithRetry executes sql with retry +func ExecSQLWithRetry(ctx context.Context, db *sql.DB, sql string, args ...interface{}) (err error) { + for i := 0; i < DefaultRetryTime; i++ { + startTime := time.Now() + _, err = db.ExecContext(ctx, sql, args...) + takeDuration := time.Since(startTime) + if takeDuration > SlowWarnLog { + log.Warn("exec sql slow", zap.String("sql", sql), zap.Reflect("args", args), zap.Duration("take", takeDuration)) + } + if err == nil { + return nil + } + + if ignoreError(err) { + log.Debug("ignore execute sql error", zap.Error(err)) + return nil + } + + if !isRetryableError(err) { + return errors.Trace(err) + } + + log.Warn("exe sql failed, will try again", zap.String("sql", sql), zap.Reflect("args", args), zap.Error(err)) + + if i == DefaultRetryTime-1 { + break + } + + select { + case <-ctx.Done(): + return errors.Trace(ctx.Err()) + case <-time.After(10 * time.Millisecond): + } + } + + return errors.Trace(err) +} + +// ExecuteSQLs executes some sqls in one transaction +func ExecuteSQLs(ctx context.Context, db *sql.DB, sqls []string, args [][]interface{}) error { + txn, err := db.Begin() + if err != nil { + log.Error("exec sqls begin", zap.Error(err)) + return errors.Trace(err) + } + + for i := range sqls { + startTime := time.Now() + + _, err = txn.ExecContext(ctx, sqls[i], args[i]...) + if err != nil { + log.Error("exec sql", zap.String("sql", sqls[i]), zap.Reflect("args", args[i]), zap.Error(err)) + rerr := txn.Rollback() + if rerr != nil { + log.Error("rollback", zap.Error(err)) + } + return errors.Trace(err) + } + + takeDuration := time.Since(startTime) + if takeDuration > SlowWarnLog { + log.Warn("exec sql slow", zap.String("sql", sqls[i]), zap.Reflect("args", args[i]), zap.Duration("take", takeDuration)) + } + } + + err = txn.Commit() + if err != nil { + log.Error("exec sqls commit", zap.Error(err)) + return errors.Trace(err) + } + + return nil +} + +func isRetryableError(err error) bool { + err = errors.Cause(err) // check the original error + mysqlErr, ok := err.(*mysql.MySQLError) + if !ok { + return false + } + + switch mysqlErr.Number { + // ER_LOCK_DEADLOCK can retry to commit while meet deadlock + case tmysql.ErrUnknown, gmysql.ER_LOCK_DEADLOCK, tmysql.ErrPDServerTimeout, tmysql.ErrTiKVServerTimeout, tmysql.ErrTiKVServerBusy, tmysql.ErrResolveLockTimeout, tmysql.ErrRegionUnavailable: + return true + default: + return false + } +} + +func ignoreError(err error) bool { + // TODO: now only ignore some ddl error, add some dml error later + if ignoreDDLError(err) { + return true + } + + return false +} + +func ignoreDDLError(err error) bool { + err = errors.Cause(err) + mysqlErr, ok := err.(*mysql.MySQLError) + if !ok { + return false + } + + errCode := terror.ErrCode(mysqlErr.Number) + switch errCode { + case infoschema.ErrDatabaseExists.Code(), infoschema.ErrDatabaseDropExists.Code(), + infoschema.ErrTableExists.Code(), infoschema.ErrTableDropExists.Code(), + infoschema.ErrColumnExists.Code(), infoschema.ErrIndexExists.Code(): + return true + case ddl.ErrDupKeyName.Code(): + return true + default: + return false + } +} diff --git a/pkg/dbutil/common_test.go b/pkg/dbutil/common_test.go index 0552020b8..7b02f18bb 100644 --- a/pkg/dbutil/common_test.go +++ b/pkg/dbutil/common_test.go @@ -14,7 +14,14 @@ package dbutil import ( + "database/sql/driver" + + "github.com/go-sql-driver/mysql" . "github.com/pingcap/check" + "github.com/pingcap/errors" + tmysql "github.com/pingcap/parser/mysql" + "github.com/pingcap/tidb/infoschema" + gmysql "github.com/siddontang/go-mysql/mysql" ) func (*testDBSuite) TestReplacePlaceholder(c *C) { @@ -69,3 +76,56 @@ func (*testDBSuite) TestTableName(c *C) { c.Assert(tableName, Equals, testCase.expectTableName) } } + +func newMysqlErr(number uint16, message string) *mysql.MySQLError { + return &mysql.MySQLError{ + Number: number, + Message: message, + } +} + +func (s *testDBSuite) TestIsRetryableError(c *C) { + cases := []struct { + err error + isRetryable bool + }{ + {newMysqlErr(tmysql.ErrNoDB, "no db error"), false}, + {errors.New("unknown error"), false}, + {newMysqlErr(tmysql.ErrUnknown, "i/o timeout"), true}, + {newMysqlErr(tmysql.ErrDBCreateExists, "db already exists"), false}, + {driver.ErrBadConn, false}, + {newMysqlErr(gmysql.ER_LOCK_DEADLOCK, "Deadlock found when trying to get lock; try restarting transaction"), true}, + {newMysqlErr(tmysql.ErrPDServerTimeout, "pd server timeout"), true}, + {newMysqlErr(tmysql.ErrTiKVServerTimeout, "tikv server timeout"), true}, + {newMysqlErr(tmysql.ErrTiKVServerBusy, "tikv server busy"), true}, + {newMysqlErr(tmysql.ErrResolveLockTimeout, "resolve lock timeout"), true}, + {newMysqlErr(tmysql.ErrRegionUnavailable, "region unavailable"), true}, + } + + for _, t := range cases { + c.Logf("err %v, expected %v", t.err, t.isRetryable) + c.Assert(isRetryableError(t.err), Equals, t.isRetryable) + } +} + +func (s *testDBSuite) TestIsIgnoreError(c *C) { + cases := []struct { + err error + canIgnore bool + }{ + {newMysqlErr(uint16(infoschema.ErrDatabaseExists.Code()), "Can't create database, database exists"), true}, + {newMysqlErr(uint16(infoschema.ErrDatabaseDropExists.Code()), "Can't drop database, database doesn't exists"), true}, + {newMysqlErr(uint16(infoschema.ErrTableExists.Code()), "Can't create table, table exists"), true}, + {newMysqlErr(uint16(infoschema.ErrTableDropExists.Code()), "Can't drop table, table dosen't exists"), true}, + {newMysqlErr(uint16(infoschema.ErrColumnExists.Code()), "Duplicate column name"), true}, + {newMysqlErr(uint16(infoschema.ErrIndexExists.Code()), "Duplicate Index"), true}, + + {newMysqlErr(uint16(999), "fake error"), false}, + {errors.New("unknown error"), false}, + } + + for _, t := range cases { + c.Logf("err %v, expected %v", t.err, t.canIgnore) + c.Assert(ignoreError(t.err), Equals, t.canIgnore) + } +} diff --git a/pkg/diff/checkpoint.go b/pkg/diff/checkpoint.go new file mode 100644 index 000000000..5ecbb8ffd --- /dev/null +++ b/pkg/diff/checkpoint.go @@ -0,0 +1,377 @@ +// Copyright 2019 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package diff + +import ( + "context" + "database/sql" + "encoding/json" + "fmt" + "time" + + "github.com/pingcap/errors" + "github.com/pingcap/log" + "github.com/pingcap/tidb-tools/pkg/dbutil" + "go.uber.org/zap" +) + +var ( + // for chunk: means this chunk's data is equal + // for table: means this all chunk in this table is equal(except ignore chunk) + successState = "success" + + // for chunk: means this chunk's data is not equal + // for table: means some chunks' data is not equal or some chunk check failed in this table + failedState = "failed" + + // for chunk: means meet error when check, don't know the chunk's data is equal or not equal + // for table: don't have this state + errorState = "error" + + // for chunk: means this chunk is not in check + // for table: this table is checking but not finished + notCheckedState = "not_checked" + + // for chunk: means this chunk is checking + // for table: don't have this state + checkingState = "checking" + + // for chunk: this chunk is ignored. if sample is not 100%, will ignore some chunk + // for table: don't have this state + ignoreState = "ignore" + + checkpointSchemaName = "sync_diff_inspector" + + summaryTableName = "summary" + + chunkTableName = "chunk" +) + +// saveChunk saves the chunk's info to `chunk` table +func saveChunk(ctx context.Context, db *sql.DB, chunkID int, instanceID, schema, table, checksum string, chunk *ChunkRange) error { + chunkBytes, err := json.Marshal(chunk) + if err != nil { + return errors.Trace(err) + } + + sql := fmt.Sprintf("REPLACE INTO `%s`.`%s` VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?);", checkpointSchemaName, chunkTableName) + err = dbutil.ExecSQLWithRetry(ctx, db, sql, chunkID, instanceID, schema, table, chunk.Where, checksum, string(chunkBytes), chunk.State, time.Now()) + if err != nil { + log.Error("save chunk info failed", zap.Error(err)) + return errors.Trace(err) + } + return nil +} + +// getChunk gets chunk info from table `chunk` by chunkID +func getChunk(ctx context.Context, db *sql.DB, instanceID, schema, table string, chunkID int) (*ChunkRange, error) { + query := fmt.Sprintf("SELECT `chunk_str` FROM `%s`.`%s` WHERE `instance_id` = ? AND `schema` = ? AND `table` = ? AND `chunk_id` = ? limit 1", checkpointSchemaName, chunkTableName) + rows, err := db.QueryContext(ctx, query, instanceID, schema, table, chunkID) + if err != nil { + return nil, err + } + defer rows.Close() + + for rows.Next() { + fields, err1 := dbutil.ScanRow(rows) + if err1 != nil { + return nil, errors.Trace(err1) + } + + chunkStr := fields["chunk_str"].Data + chunk := new(ChunkRange) + err := json.Unmarshal(chunkStr, &chunk) + if err != nil { + return nil, err + } + return chunk, nil + } + + if rows.Err() != nil { + return nil, errors.Trace(rows.Err()) + } + + return nil, errors.NotFoundf("instanceID %d, schema %s, table %s, chunk %d", instanceID, schema, table, chunkID) +} + +// loadChunks loads chunk info from table `chunk` +func loadChunks(ctx context.Context, db *sql.DB, instanceID, schema, table string) ([]*ChunkRange, error) { + chunks := make([]*ChunkRange, 0, 100) + + query := fmt.Sprintf("SELECT `chunk_str` FROM `%s`.`%s` WHERE `instance_id` = ? AND `schema` = ? AND `table` = ?", checkpointSchemaName, chunkTableName) + rows, err := db.QueryContext(ctx, query, instanceID, schema, table) + if err != nil { + return nil, err + } + defer rows.Close() + + for rows.Next() { + fields, err1 := dbutil.ScanRow(rows) + if err1 != nil { + return nil, errors.Trace(err1) + } + + chunkStr := fields["chunk_str"].Data + chunk := new(ChunkRange) + err := json.Unmarshal(chunkStr, &chunk) + if err != nil { + return nil, err + } + chunks = append(chunks, chunk) + } + + return chunks, errors.Trace(rows.Err()) +} + +// getTableSummary returns a table's total chunk num, check success chunk num, check failed chunk num, check ignore chunk num and the state +func getTableSummary(ctx context.Context, db *sql.DB, schema, table string) (total int64, success int64, failed int64, ignore int64, state string, err error) { + query := fmt.Sprintf("SELECT `chunk_num`, `check_success_num`, `check_failed_num`, `check_ignore_num`, `state` FROM `%s`.`%s` WHERE `schema` = ? AND `table` = ? LIMIT 1", + checkpointSchemaName, summaryTableName) + rows, err := db.QueryContext(ctx, query, schema, table) + if err != nil { + return 0, 0, 0, 0, "", errors.Trace(err) + } + defer rows.Close() + + var totalNum, successNum, failedNum, ignoreNum sql.NullInt64 + var stateStr sql.NullString + for rows.Next() { + err1 := rows.Scan(&totalNum, &successNum, &failedNum, &ignoreNum, &stateStr) + if err1 != nil { + return 0, 0, 0, 0, "", errors.Trace(err1) + } + + if !totalNum.Valid || !successNum.Valid || !failedNum.Valid || !ignoreNum.Valid || !stateStr.Valid { + return 0, 0, 0, 0, "", errors.Errorf("some values are invalid, query: %s, args: %v", query, []interface{}{schema, table}) + } + + return totalNum.Int64, successNum.Int64, failedNum.Int64, ignoreNum.Int64, stateStr.String, nil + + } + if rows.Err() != nil { + return 0, 0, 0, 0, "", errors.Trace(rows.Err()) + } + + return 0, 0, 0, 0, "", errors.NotFoundf("schema %s, table %s summary info", schema, table) +} + +// getChunkSummary get the table's summary info from `chunk` table +func getChunkSummary(ctx context.Context, db *sql.DB, instanceID, schema, table string) (total, successNum, failedNum, ignoreNum int64, err error) { + query := fmt.Sprintf("SELECT `state`, COUNT(*) FROM `%s`.`%s` WHERE `instance_id` = ? AND `schema` = ? AND `table` = ? GROUP BY `state` ;", checkpointSchemaName, chunkTableName) + rows, err := db.QueryContext(ctx, query, instanceID, schema, table) + if err != nil { + return 0, 0, 0, 0, errors.Trace(err) + } + defer rows.Close() + + var chunkState sql.NullString + var num sql.NullInt64 + for rows.Next() { + err1 := rows.Scan(&chunkState, &num) + if err1 != nil { + return 0, 0, 0, 0, errors.Trace(err1) + } + + if !chunkState.Valid || !num.Valid { + return 0, 0, 0, 0, errors.Errorf("some values are invalid, query: %s, args: %v", query, []interface{}{instanceID, schema, table}) + } + + total += num.Int64 + switch chunkState.String { + case successState: + successNum += num.Int64 + case failedState, errorState: + failedNum += num.Int64 + case ignoreState: + ignoreNum += num.Int64 + case notCheckedState, checkingState: + } + } + if rows.Err() != nil { + return 0, 0, 0, 0, errors.Trace(rows.Err()) + } + + if total == 0 { + return 0, 0, 0, 0, errors.NotFoundf("chunks of instanceID %s schema %s table %s", instanceID, schema, table) + } + + return +} + +// initTableSummary initials a table's summary info in table `summary` +func initTableSummary(ctx context.Context, db *sql.DB, schema, table string, configHash string) error { + sql := fmt.Sprintf("REPLACE INTO `%s`.`%s`(`schema`, `table`, `state`, `config_hash`) VALUES(?, ?, ?, ?)", checkpointSchemaName, summaryTableName) + err := dbutil.ExecSQLWithRetry(ctx, db, sql, schema, table, notCheckedState, configHash) + if err != nil { + log.Error("save summary info failed", zap.Error(err)) + return errors.Trace(err) + } + + return nil +} + +// updateTableSummary gets summary info from `chunk` table, and then update `summary` table +func updateTableSummary(ctx context.Context, db *sql.DB, instanceID, schema, table string) error { + total, successNum, failedNum, ignoreNum, err := getChunkSummary(ctx, db, instanceID, schema, table) + if err != nil { + return errors.Trace(err) + } + log.Info("summary info", zap.String("instance_id", instanceID), zap.String("schema", schema), zap.String("table", table), zap.Int64("chunk num", total), zap.Int64("success num", successNum), zap.Int64("failed num", failedNum), zap.Int64("ignore num", ignoreNum)) + + state := notCheckedState + if total == successNum+failedNum+ignoreNum { + if total == successNum+ignoreNum { + state = successState + } else { + state = failedState + } + } + + updateSQL := fmt.Sprintf("UPDATE `%s`.`%s` SET `chunk_num` = ?, `check_success_num` = ?, `check_failed_num` = ?, `check_ignore_num` = ?, `state` = ? WHERE `schema` = ? AND `table` = ?", checkpointSchemaName, summaryTableName) + err = dbutil.ExecSQLWithRetry(ctx, db, updateSQL, total, successNum, failedNum, ignoreNum, state, schema, table) + if err != nil { + return errors.Trace(err) + } + + return nil +} + +// createCheckpointTable creates checkpoint tables, include `summary` and `chunk` +func createCheckpointTable(ctx context.Context, db *sql.DB) error { + createSchemaSQL := fmt.Sprintf("CREATE DATABASE IF NOT EXISTS `%s`;", checkpointSchemaName) + _, err := db.ExecContext(ctx, createSchemaSQL) + if err != nil { + log.Info("create schema", zap.Error(err)) + return errors.Trace(err) + } + + /* example + mysql> select * from sync_diff_inspector.summary; + +--------+-------+-----------+-------------------+------------------+------------------+---------+----------------------------------+---------------------+ + | schema | table | chunk_num | check_success_num | check_failed_num | check_ignore_num | state | config_hash | update_time | + +--------+-------+-----------+-------------------+------------------+------------------+---------+----------------------------------+---------------------+ + | diff | test | 112 | 104 | 0 | 8 | success | 91f302052783672b01af3e2b0e7d66ff | 2019-03-26 12:42:11 | + +--------+-------+-----------+-------------------+------------------+------------------+---------+----------------------------------+---------------------+ + + note: config_hash is the hash value for the config, if config is changed, will clear the history checkpoint. + */ + createSummaryTableSQL := + "CREATE TABLE IF NOT EXISTS `sync_diff_inspector`.`summary`(" + + "`schema` varchar(30), `table` varchar(30)," + + "`chunk_num` int not null default 0," + + "`check_success_num` int not null default 0," + + "`check_failed_num` int not null default 0," + + "`check_ignore_num` int not null default 0," + + "`state` enum('not_checked', 'checking', 'success', 'failed') DEFAULT 'not_checked'," + + "`config_hash` varchar(50)," + + "`update_time` datetime ON UPDATE CURRENT_TIMESTAMP," + + "PRIMARY KEY(`schema`, `table`));" + + _, err = db.ExecContext(ctx, createSummaryTableSQL) + if err != nil { + log.Error("create chunk table", zap.Error(err)) + return errors.Trace(err) + } + + /* example + mysql> select * from sync_diff_inspector.chunk where chunk_id = 2;; + +----------+-------------+--------+-------+---------------------------------+-------------+-----------+---------+---------------------+ + | chunk_id | instance_id | schema | table | range | checksum | chunk_str | state | update_time | + +----------+-------------+--------+-------+---------------------------------+-------------+-----------+---------+---------------------+ + | 2 | target-1 | diff | test1 | (`a` >= ? AND `a` < ? AND TRUE) | 91f3020527 | ..... | success | 2019-03-26 12:41:42 | + +----------+-------------+--------+-------+---------------------------------+-------------+-----------+---------+---------------------+ + */ + createChunkTableSQL := + "CREATE TABLE IF NOT EXISTS `sync_diff_inspector`.`chunk`(" + + "`chunk_id` int," + + "`instance_id` varchar(30)," + + "`schema` varchar(30)," + + "`table` varchar(30)," + + "`range` varchar(100)," + + "`checksum` varchar(20)," + + "`chunk_str` text," + + "`state` enum('not_checked', 'checking', 'success', 'failed', 'ignore', 'error') DEFAULT 'not_checked'," + + "`update_time` datetime ON UPDATE CURRENT_TIMESTAMP," + + "PRIMARY KEY(`schema`, `table`, `instance_id`, `chunk_id`));" + _, err = db.ExecContext(ctx, createChunkTableSQL) + if err != nil { + log.Error("create chunk table", zap.Error(err)) + return errors.Trace(err) + } + + return nil +} + +// cleanCheckpoint deletes the table's checkpoint info in table `summary` and `chunk` +func cleanCheckpoint(ctx context.Context, db *sql.DB, schema, table string) error { + deleteSummarySQL := fmt.Sprintf("DELETE FROM `%s`.`%s` WHERE `schema` = ? AND `table` = ?;", checkpointSchemaName, summaryTableName) + args1 := []interface{}{schema, table} + + deleteChunkSQL := fmt.Sprintf("DELETE FROM `%s`.`%s` WHERE `schema` = ? AND `table` = ?;", checkpointSchemaName, chunkTableName) + args2 := []interface{}{schema, table} + + sqls := []string{deleteSummarySQL, deleteChunkSQL} + args := [][]interface{}{args1, args2} + + return errors.Trace(dbutil.ExecuteSQLs(ctx, db, sqls, args)) +} + +// dropCheckpoint drops the database `sync_diff_inspector` +func dropCheckpoint(ctx context.Context, db *sql.DB) error { + dropSchemaSQL := fmt.Sprintf("DROP DATABASE IF EXISTS `%s`;", checkpointSchemaName) + _, err := db.ExecContext(ctx, dropSchemaSQL) + if err != nil { + log.Error("drop schema", zap.Error(err)) + return errors.Trace(err) + } + + return nil +} + +// loadFromCheckPoint returns true if we should use the history checkpoint +func loadFromCheckPoint(ctx context.Context, db *sql.DB, schema, table, configHash string) (bool, error) { + query := fmt.Sprintf("SELECT `state`, `config_hash` FROM `%s`.`%s` WHERE `schema` = ? AND `table` = ? LIMIT 1;", checkpointSchemaName, summaryTableName) + rows, err := db.QueryContext(ctx, query, schema, table) + if err != nil { + return false, errors.Trace(err) + } + defer rows.Close() + + var state, cfgHash sql.NullString + + for rows.Next() { + err1 := rows.Scan(&state, &cfgHash) + if err1 != nil { + return false, errors.Trace(err1) + } + + if cfgHash.Valid { + if configHash != cfgHash.String { + return false, nil + } + } + + if state.Valid { + // is state is success, will begin a new check for this table + // if state is not checked, the chunk info maybe not exists, so just return false + if state.String == successState || state.String == notCheckedState { + return false, nil + } + } + + return true, nil + } + + return false, errors.Trace(rows.Err()) +} diff --git a/pkg/diff/checkpoint_test.go b/pkg/diff/checkpoint_test.go new file mode 100644 index 000000000..4b5da542c --- /dev/null +++ b/pkg/diff/checkpoint_test.go @@ -0,0 +1,124 @@ +// Copyright 2019 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package diff + +import ( + "context" + "database/sql" + + sqlmock "github.com/DATA-DOG/go-sqlmock" + . "github.com/pingcap/check" +) + +var _ = Suite(&testCheckpointSuite{}) + +type testCheckpointSuite struct{} + +func (s *testCheckpointSuite) TestCheckpoint(c *C) { + db, err := createConn() + c.Assert(err, IsNil) + defer dropCheckpoint(context.Background(), db) + + s.testInitAndGetSummary(c, db) + s.testSaveAndLoadChunk(c, db) + s.testUpdateSummary(c, db) +} + +func (s *testCheckpointSuite) testInitAndGetSummary(c *C, db *sql.DB) { + err := createCheckpointTable(context.Background(), db) + c.Assert(err, IsNil) + + _, _, _, _, _, err = getTableSummary(context.Background(), db, "test", "checkpoint") + c.Log(err) + c.Assert(err, ErrorMatches, "*not found*") + + err = initTableSummary(context.Background(), db, "test", "checkpoint", "123") + c.Assert(err, IsNil) + + total, successNum, failedNum, ignoreNum, state, err := getTableSummary(context.Background(), db, "test", "checkpoint") + c.Assert(err, IsNil) + c.Assert(total, Equals, int64(0)) + c.Assert(successNum, Equals, int64(0)) + c.Assert(failedNum, Equals, int64(0)) + c.Assert(ignoreNum, Equals, int64(0)) + c.Assert(state, Equals, notCheckedState) +} + +func (s *testCheckpointSuite) testSaveAndLoadChunk(c *C, db *sql.DB) { + chunk := &ChunkRange{ + ID: 1, + Bounds: []*Bound{{Column: "a", Lower: "1", LowerSymbol: ">"}}, + Mode: normalMode, + State: successState, + } + + err := saveChunk(context.Background(), db, chunk.ID, "target", "test", "checkpoint", "", chunk) + c.Assert(err, IsNil) + + newChunk, err := getChunk(context.Background(), db, "target", "test", "checkpoint", chunk.ID) + c.Assert(err, IsNil) + c.Assert(newChunk, DeepEquals, chunk) + + chunks, err := loadChunks(context.Background(), db, "target", "test", "checkpoint") + c.Assert(err, IsNil) + c.Assert(chunks, HasLen, 1) + c.Assert(chunks[0], DeepEquals, chunk) +} + +func (s *testCheckpointSuite) testUpdateSummary(c *C, db *sql.DB) { + failedChunk := &ChunkRange{ + ID: 2, + State: failedState, + } + err := saveChunk(context.Background(), db, failedChunk.ID, "target", "test", "checkpoint", "", failedChunk) + c.Assert(err, IsNil) + + ignoreChunk := &ChunkRange{ + ID: 3, + State: ignoreState, + } + err = saveChunk(context.Background(), db, ignoreChunk.ID, "target", "test", "checkpoint", "", ignoreChunk) + c.Assert(err, IsNil) + + err = updateTableSummary(context.Background(), db, "target", "test", "checkpoint") + c.Assert(err, IsNil) + + total, successNum, failedNum, ignoreNum, state, err := getTableSummary(context.Background(), db, "test", "checkpoint") + c.Assert(err, IsNil) + c.Assert(total, Equals, int64(3)) + c.Assert(successNum, Equals, int64(1)) + c.Assert(failedNum, Equals, int64(1)) + c.Assert(ignoreNum, Equals, int64(1)) + c.Assert(state, Equals, failedState) +} + +func (s *testUtilSuite) TestloadFromCheckPoint(c *C) { + db, mock, err := sqlmock.New() + c.Assert(err, IsNil) + + rows := sqlmock.NewRows([]string{"state", "config_hash"}).AddRow("success", "123") + mock.ExpectQuery("SELECT").WillReturnRows(rows) + useCheckpoint, err := loadFromCheckPoint(context.Background(), db, "test", "test", "123") + c.Assert(useCheckpoint, Equals, false) + + rows = sqlmock.NewRows([]string{"state", "config_hash"}).AddRow("success", "123") + mock.ExpectQuery("SELECT").WillReturnRows(rows) + useCheckpoint, err = loadFromCheckPoint(context.Background(), db, "test", "test", "456") + c.Assert(useCheckpoint, Equals, false) + + rows = sqlmock.NewRows([]string{"state", "config_hash"}).AddRow("failed", "123") + mock.ExpectQuery("SELECT").WillReturnRows(rows) + useCheckpoint, err = loadFromCheckPoint(context.Background(), db, "test", "test", "123") + c.Assert(useCheckpoint, Equals, true) +} diff --git a/pkg/diff/chunk.go b/pkg/diff/chunk.go index 50475cca5..652696a04 100644 --- a/pkg/diff/chunk.go +++ b/pkg/diff/chunk.go @@ -16,14 +16,17 @@ package diff import ( "context" "database/sql" + "encoding/json" "fmt" "strings" + "time" "github.com/pingcap/errors" "github.com/pingcap/parser/model" "github.com/pingcap/tidb-tools/pkg/dbutil" "github.com/pingcap/tidb-tools/pkg/utils" log "github.com/sirupsen/logrus" + "go.uber.org/zap" ) var ( @@ -33,47 +36,67 @@ var ( gt = ">" gte = ">=" - bucketMode = "bucket" + bucketMode = "bucketMode" normalMode = "normalMode" ) -type bound struct { - column string - lower string - lowerSymbol string - upper string - upperSymbol string +// Bound represents a bound for a column +type Bound struct { + Column string `json:"column"` + Lower string `json:"lower"` + LowerSymbol string `json:"lower-symbol"` + Upper string `json:"upper"` + UpperSymbol string `json:"upper-symbol"` } -// chunkRange represents chunk range -type chunkRange struct { - bounds []*bound +// ChunkRange represents chunk range +type ChunkRange struct { + ID int `json:"id"` + Bounds []*Bound `json:"bounds"` + Mode string `json:"mode"` + + Where string `json:"where"` + Args []string `json:"args"` + + State string `json:"state"` } -// newChunkRange return a chunkRange. -func newChunkRange() *chunkRange { - return &chunkRange{ - bounds: make([]*bound, 0, 2), +// NewChunkRange return a ChunkRange. +func NewChunkRange(mode string) *ChunkRange { + return &ChunkRange{ + Bounds: make([]*Bound, 0, 2), + Mode: mode, } } -func (c *chunkRange) toString(mode string, collation string) (string, []string) { +// String returns the string of ChunkRange, used for log. +func (c *ChunkRange) String() string { + chunkBytes, err := json.Marshal(c) + if err != nil { + log.Warn("fail to encode chunk into string", zap.Error(err)) + return "" + } + + return string(chunkBytes) +} + +func (c *ChunkRange) toString(collation string) (string, []string) { if collation != "" { collation = fmt.Sprintf(" COLLATE '%s'", collation) } - if mode != bucketMode { + if c.Mode != bucketMode { conditions := make([]string, 0, 2) args := make([]string, 0, 2) - for _, bound := range c.bounds { - if len(bound.lower) != 0 { - conditions = append(conditions, fmt.Sprintf("`%s`%s %s ?", bound.column, collation, bound.lowerSymbol)) - args = append(args, bound.lower) + for _, bound := range c.Bounds { + if len(bound.Lower) != 0 { + conditions = append(conditions, fmt.Sprintf("`%s`%s %s ?", bound.Column, collation, bound.LowerSymbol)) + args = append(args, bound.Lower) } - if len(bound.upper) != 0 { - conditions = append(conditions, fmt.Sprintf("`%s`%s %s ?", bound.column, collation, bound.upperSymbol)) - args = append(args, bound.upper) + if len(bound.Upper) != 0 { + conditions = append(conditions, fmt.Sprintf("`%s`%s %s ?", bound.Column, collation, bound.UpperSymbol)) + args = append(args, bound.Upper) } } @@ -100,29 +123,29 @@ func (c *chunkRange) toString(mode string, collation string) (string, []string) preConditionArgsForLower := make([]string, 0, 1) preConditionArgsForUpper := make([]string, 0, 1) - for _, bound := range c.bounds { - if len(bound.lower) != 0 { + for _, bound := range c.Bounds { + if len(bound.Lower) != 0 { if len(preConditionForLower) > 0 { - lowerCondition = append(lowerCondition, fmt.Sprintf("(%s AND `%s`%s %s ?)", strings.Join(preConditionForLower, " AND "), bound.column, collation, bound.lowerSymbol)) - lowerArgs = append(append(lowerArgs, preConditionArgsForLower...), bound.lower) + lowerCondition = append(lowerCondition, fmt.Sprintf("(%s AND `%s`%s %s ?)", strings.Join(preConditionForLower, " AND "), bound.Column, collation, bound.LowerSymbol)) + lowerArgs = append(append(lowerArgs, preConditionArgsForLower...), bound.Lower) } else { - lowerCondition = append(lowerCondition, fmt.Sprintf("(`%s`%s %s ?)", bound.column, collation, bound.lowerSymbol)) - lowerArgs = append(lowerArgs, bound.lower) + lowerCondition = append(lowerCondition, fmt.Sprintf("(`%s`%s %s ?)", bound.Column, collation, bound.LowerSymbol)) + lowerArgs = append(lowerArgs, bound.Lower) } - preConditionForLower = append(preConditionForLower, fmt.Sprintf("`%s` = ?", bound.column)) - preConditionArgsForLower = append(preConditionArgsForLower, bound.lower) + preConditionForLower = append(preConditionForLower, fmt.Sprintf("`%s` = ?", bound.Column)) + preConditionArgsForLower = append(preConditionArgsForLower, bound.Lower) } - if len(bound.upper) != 0 { + if len(bound.Upper) != 0 { if len(preConditionForUpper) > 0 { - upperCondition = append(upperCondition, fmt.Sprintf("(%s AND `%s`%s %s ?)", strings.Join(preConditionForUpper, " AND "), bound.column, collation, bound.upperSymbol)) - upperArgs = append(append(upperArgs, preConditionArgsForUpper...), bound.upper) + upperCondition = append(upperCondition, fmt.Sprintf("(%s AND `%s`%s %s ?)", strings.Join(preConditionForUpper, " AND "), bound.Column, collation, bound.UpperSymbol)) + upperArgs = append(append(upperArgs, preConditionArgsForUpper...), bound.Upper) } else { - upperCondition = append(upperCondition, fmt.Sprintf("(`%s`%s %s ?)", bound.column, collation, bound.upperSymbol)) - upperArgs = append(upperArgs, bound.upper) + upperCondition = append(upperCondition, fmt.Sprintf("(`%s`%s %s ?)", bound.Column, collation, bound.UpperSymbol)) + upperArgs = append(upperArgs, bound.Upper) } - preConditionForUpper = append(preConditionForUpper, fmt.Sprintf("`%s` = ?", bound.column)) - preConditionArgsForUpper = append(preConditionArgsForUpper, bound.upper) + preConditionForUpper = append(preConditionForUpper, fmt.Sprintf("`%s` = ?", bound.Column)) + preConditionArgsForUpper = append(preConditionArgsForUpper, bound.Upper) } } @@ -142,37 +165,38 @@ func (c *chunkRange) toString(mode string, collation string) (string, []string) } -func (c *chunkRange) update(column, lower, lowerSymbol, upper, upperSymbol string) { - newBound := &bound{ - column: column, - lower: lower, - lowerSymbol: lowerSymbol, - upper: upper, - upperSymbol: upperSymbol, +func (c *ChunkRange) update(column, lower, lowerSymbol, upper, upperSymbol string) { + newBound := &Bound{ + Column: column, + Lower: lower, + LowerSymbol: lowerSymbol, + Upper: upper, + UpperSymbol: upperSymbol, } - for i, b := range c.bounds { - if b.column == column { + for i, b := range c.Bounds { + if b.Column == column { // update the bound - c.bounds[i] = newBound + c.Bounds[i] = newBound return } } // add a new bound - c.bounds = append(c.bounds, newBound) + c.Bounds = append(c.Bounds, newBound) } -func (c *chunkRange) copy() *chunkRange { - newChunk := &chunkRange{ - bounds: make([]*bound, len(c.bounds)), +func (c *ChunkRange) copy() *ChunkRange { + newChunk := &ChunkRange{ + Mode: c.Mode, + Bounds: make([]*Bound, len(c.Bounds)), } - copy(newChunk.bounds, c.bounds) + copy(newChunk.Bounds, c.Bounds) return newChunk } -func (c *chunkRange) copyAndUpdate(column, lower, lowerSymbol, upper, upperSymbol string) *chunkRange { +func (c *ChunkRange) copyAndUpdate(column, lower, lowerSymbol, upper, upperSymbol string) *ChunkRange { newChunk := c.copy() newChunk.update(column, lower, lowerSymbol, upper, upperSymbol) return newChunk @@ -180,7 +204,7 @@ func (c *chunkRange) copyAndUpdate(column, lower, lowerSymbol, upper, upperSymbo type spliter interface { // split splits a table's data to several chunks. - split(table *TableInstance, columns []*model.ColumnInfo, chunkSize int, limits string, collation string) ([]*chunkRange, error) + split(table *TableInstance, columns []*model.ColumnInfo, chunkSize int, limits string, collation string) ([]*ChunkRange, error) } type randomSpliter struct { @@ -190,7 +214,7 @@ type randomSpliter struct { collation string } -func (s *randomSpliter) split(table *TableInstance, columns []*model.ColumnInfo, chunkSize int, limits string, collation string) ([]*chunkRange, error) { +func (s *randomSpliter) split(table *TableInstance, columns []*model.ColumnInfo, chunkSize int, limits string, collation string) ([]*ChunkRange, error) { s.table = table s.chunkSize = chunkSize s.limits = limits @@ -203,7 +227,7 @@ func (s *randomSpliter) split(table *TableInstance, columns []*model.ColumnInfo, } chunkCnt := (int(cnt) + chunkSize - 1) / chunkSize - chunks, err := s.splitRange(table.Conn, newChunkRange(), chunkCnt, table.Schema, table.Table, columns) + chunks, err := s.splitRange(table.Conn, NewChunkRange(normalMode), chunkCnt, table.Schema, table.Table, columns) if err != nil { return nil, errors.Trace(err) } @@ -212,8 +236,8 @@ func (s *randomSpliter) split(table *TableInstance, columns []*model.ColumnInfo, } // splitRange splits a chunk to multiple chunks. -func (s *randomSpliter) splitRange(db *sql.DB, chunk *chunkRange, count int, schema string, table string, columns []*model.ColumnInfo) ([]*chunkRange, error) { - var chunks []*chunkRange +func (s *randomSpliter) splitRange(db *sql.DB, chunk *ChunkRange, count int, schema string, table string, columns []*model.ColumnInfo) ([]*ChunkRange, error) { + var chunks []*ChunkRange if count <= 1 { chunks = append(chunks, chunk) @@ -226,17 +250,17 @@ func (s *randomSpliter) splitRange(db *sql.DB, chunk *chunkRange, count int, sch useNewColumn bool ) - chunkLimits, args := chunk.toString(normalMode, s.collation) + chunkLimits, args := chunk.toString(s.collation) limitRange := fmt.Sprintf("%s AND %s", chunkLimits, s.limits) // if the last column's condition is not '=', continue use this column split data. - colNum := len(chunk.bounds) - if colNum != 0 && chunk.bounds[colNum-1].lowerSymbol != equal { - splitCol = chunk.bounds[colNum-1].column - min = chunk.bounds[colNum-1].lower - max = chunk.bounds[colNum-1].upper - symbolMin = chunk.bounds[colNum-1].lowerSymbol - symbolMax = chunk.bounds[colNum-1].upperSymbol + colNum := len(chunk.Bounds) + if colNum != 0 && chunk.Bounds[colNum-1].LowerSymbol != equal { + splitCol = chunk.Bounds[colNum-1].Column + min = chunk.Bounds[colNum-1].Lower + max = chunk.Bounds[colNum-1].Upper + symbolMin = chunk.Bounds[colNum-1].LowerSymbol + symbolMax = chunk.Bounds[colNum-1].UpperSymbol } else { if len(columns) <= colNum { log.Warnf("chunk %v can't be splited", chunk) @@ -373,7 +397,7 @@ type bucketSpliter struct { buckets map[string][]dbutil.Bucket } -func (s *bucketSpliter) split(table *TableInstance, columns []*model.ColumnInfo, chunkSize int, limits string, collation string) ([]*chunkRange, error) { +func (s *bucketSpliter) split(table *TableInstance, columns []*model.ColumnInfo, chunkSize int, limits string, collation string) ([]*ChunkRange, error) { s.table = table s.chunkSize = chunkSize s.limits = limits @@ -388,8 +412,8 @@ func (s *bucketSpliter) split(table *TableInstance, columns []*model.ColumnInfo, return s.getChunksByBuckets() } -func (s *bucketSpliter) getChunksByBuckets() ([]*chunkRange, error) { - chunks := make([]*chunkRange, 0, 1000) +func (s *bucketSpliter) getChunksByBuckets() ([]*ChunkRange, error) { + chunks := make([]*ChunkRange, 0, 1000) indices := dbutil.FindAllIndex(s.table.info) for _, index := range indices { @@ -418,7 +442,7 @@ func (s *bucketSpliter) getChunksByBuckets() ([]*chunkRange, error) { if bucket.Count-latestCount > int64(s.chunkSize) || i == len(buckets)-1 { // create a new chunk - chunk := newChunkRange() + chunk := NewChunkRange(bucketMode) var lower, upper, lowerSymbol, upperSymbol string for j, col := range index.Columns { if len(lowerValues) != 0 { @@ -447,12 +471,12 @@ func (s *bucketSpliter) getChunksByBuckets() ([]*chunkRange, error) { return chunks, nil } -func getChunksForTable(table *TableInstance, columns []*model.ColumnInfo, chunkSize int, limits string, collation string, useTiDBStatsInfo bool) ([]*chunkRange, string, error) { +func getChunksForTable(table *TableInstance, columns []*model.ColumnInfo, chunkSize int, limits string, collation string, useTiDBStatsInfo bool) ([]*ChunkRange, error) { if useTiDBStatsInfo { s := bucketSpliter{} chunks, err := s.split(table, columns, chunkSize, limits, collation) if err == nil && len(chunks) > 0 { - return chunks, bucketMode, nil + return chunks, nil } log.Warnf("use tidb bucket information to get chunks error: %v, chunks num: %d, will split chunk by random again", errors.Trace(err), len(chunks)) @@ -461,7 +485,7 @@ func getChunksForTable(table *TableInstance, columns []*model.ColumnInfo, chunkS // get chunks from tidb bucket information failed, use random. s := randomSpliter{} chunks, err := s.split(table, columns, chunkSize, limits, collation) - return chunks, normalMode, err + return chunks, err } // getSplitFields returns fields to split chunks, order by pk, uk, index, columns. @@ -494,20 +518,8 @@ func getSplitFields(table *model.TableInfo, splitFields []string) ([]*model.Colu return cols, nil } -// CheckJob is the struct of job for check -type CheckJob struct { - Schema string - Table string - Where string - Args []string -} - -// GenerateCheckJob generates some CheckJobs. -func GenerateCheckJob(table *TableInstance, splitFields, limits string, chunkSize int, collation string, useTiDBStatsInfo bool) ([]*CheckJob, error) { - jobBucket := make([]*CheckJob, 0, 10) - var jobCnt int - var err error - +// SplitChunks splits the table to some chunks. +func SplitChunks(ctx context.Context, table *TableInstance, splitFields, limits string, chunkSize int, collation string, useTiDBStatsInfo bool) (chunks []*ChunkRange, err error) { var splitFieldArr []string if len(splitFields) != 0 { splitFieldArr = strings.Split(splitFields, ",") @@ -522,28 +534,30 @@ func GenerateCheckJob(table *TableInstance, splitFields, limits string, chunkSiz return nil, errors.Trace(err) } - chunks, mode, err := getChunksForTable(table, fields, chunkSize, limits, collation, useTiDBStatsInfo) + chunks, err = getChunksForTable(table, fields, chunkSize, limits, collation, useTiDBStatsInfo) if err != nil { return nil, errors.Trace(err) } + if chunks == nil { return nil, nil } - jobCnt += len(chunks) + ctx1, cancel1 := context.WithTimeout(ctx, time.Duration(len(chunks))*dbutil.DefaultTimeout) + defer cancel1() + for i, chunk := range chunks { + conditions, args := chunk.toString(collation) - for _, chunk := range chunks { - conditions, args := chunk.toString(mode, collation) - where := fmt.Sprintf("(%s AND %s)", conditions, limits) + chunk.ID = i + chunk.Where = fmt.Sprintf("(%s AND %s)", conditions, limits) + chunk.Args = args + chunk.State = notCheckedState - log.Debugf("%s.%s create check job, where: %s, args: %v", table.Schema, table.Table, where, args) - jobBucket = append(jobBucket, &CheckJob{ - Schema: table.Schema, - Table: table.Table, - Where: where, - Args: args, - }) + err = saveChunk(ctx1, table.Conn, i, table.InstanceID, table.Schema, table.Table, "", chunk) + if err != nil { + return nil, err + } } - return jobBucket, nil + return chunks, nil } diff --git a/pkg/diff/chunk_test.go b/pkg/diff/chunk_test.go index 76cae680e..eb159ada9 100644 --- a/pkg/diff/chunk_test.go +++ b/pkg/diff/chunk_test.go @@ -26,9 +26,9 @@ var _ = Suite(&testChunkSuite{}) type testChunkSuite struct{} type chunkTestCase struct { - chunk *chunkRange + chunk *ChunkRange chunkCnt int64 - expectChunks []*chunkRange + expectChunks []*ChunkRange } func (*testChunkSuite) TestSplitRange(c *C) { @@ -79,13 +79,13 @@ func (*testChunkSuite) TestSplitRange(c *C) { // split chunks fields, err := getSplitFields(tableInstance.info, nil) c.Assert(err, IsNil) - chunks, mode, err := getChunksForTable(tableInstance, fields, 100, "TRUE", "", false) + chunks, err := getChunksForTable(tableInstance, fields, 100, "TRUE", "", false) c.Assert(err, IsNil) // get data count from every chunk, and the sum of them should equal to the table's count. chunkDataCount := 0 for _, chunk := range chunks { - conditions, args := chunk.toString(mode, "") + conditions, args := chunk.toString("") count, err := dbutil.GetRowCount(context.Background(), tableInstance.Conn, tableInstance.Schema, tableInstance.Table, dbutil.ReplacePlaceholder(conditions, args)) c.Assert(err, IsNil) chunkDataCount += int(count) @@ -94,22 +94,23 @@ func (*testChunkSuite) TestSplitRange(c *C) { } func (*testChunkSuite) TestChunkUpdate(c *C) { - chunk := &chunkRange{ - bounds: []*bound{ + chunk := &ChunkRange{ + Bounds: []*Bound{ { - column: "a", - lower: "1", - lowerSymbol: ">", - upper: "2", - upperSymbol: "<=", + Column: "a", + Lower: "1", + LowerSymbol: ">", + Upper: "2", + UpperSymbol: "<=", }, { - column: "b", - lower: "3", - lowerSymbol: ">=", - upper: "4", - upperSymbol: "<", + Column: "b", + Lower: "3", + LowerSymbol: ">=", + Upper: "4", + UpperSymbol: "<", }, }, + Mode: normalMode, } testCases := []struct { @@ -134,13 +135,13 @@ func (*testChunkSuite) TestChunkUpdate(c *C) { for _, cs := range testCases { newChunk := chunk.copyAndUpdate(cs.boundArgs[0], cs.boundArgs[1], cs.boundArgs[2], cs.boundArgs[3], cs.boundArgs[4]) - conditions, args := newChunk.toString(normalMode, "") + conditions, args := newChunk.toString("") c.Assert(conditions, Equals, cs.expectStr) c.Assert(args, DeepEquals, cs.expectArgs) } // the origin chunk is not changed - conditions, args := chunk.toString(normalMode, "") + conditions, args := chunk.toString("") c.Assert(conditions, Equals, "`a` > ? AND `a` <= ? AND `b` >= ? AND `b` < ?") expectArgs := []string{"1", "2", "3", "4"} for i, arg := range args { @@ -149,52 +150,54 @@ func (*testChunkSuite) TestChunkUpdate(c *C) { } func (*testChunkSuite) TestChunkToString(c *C) { - chunk := &chunkRange{ - bounds: []*bound{ + chunk := &ChunkRange{ + Bounds: []*Bound{ { - column: "a", - lower: "1", - lowerSymbol: ">", - upper: "2", - upperSymbol: "<", + Column: "a", + Lower: "1", + LowerSymbol: ">", + Upper: "2", + UpperSymbol: "<", }, { - column: "b", - lower: "3", - lowerSymbol: ">", - upper: "4", - upperSymbol: "<", + Column: "b", + Lower: "3", + LowerSymbol: ">", + Upper: "4", + UpperSymbol: "<", }, { - column: "c", - lower: "5", - lowerSymbol: ">", - upper: "6", - upperSymbol: "<", + Column: "c", + Lower: "5", + LowerSymbol: ">", + Upper: "6", + UpperSymbol: "<", }, }, + Mode: normalMode, } - conditions, args := chunk.toString(normalMode, "") + conditions, args := chunk.toString("") c.Assert(conditions, Equals, "`a` > ? AND `a` < ? AND `b` > ? AND `b` < ? AND `c` > ? AND `c` < ?") expectArgs := []string{"1", "2", "3", "4", "5", "6"} for i, arg := range args { c.Assert(arg, Equals, expectArgs[i]) } - conditions, args = chunk.toString(normalMode, "latin1") + conditions, args = chunk.toString("latin1") c.Assert(conditions, Equals, "`a` COLLATE 'latin1' > ? AND `a` COLLATE 'latin1' < ? AND `b` COLLATE 'latin1' > ? AND `b` COLLATE 'latin1' < ? AND `c` COLLATE 'latin1' > ? AND `c` COLLATE 'latin1' < ?") expectArgs = []string{"1", "2", "3", "4", "5", "6"} for i, arg := range args { c.Assert(arg, Equals, expectArgs[i]) } - conditions, args = chunk.toString(bucketMode, "") + chunk.Mode = bucketMode + conditions, args = chunk.toString("") c.Assert(conditions, Equals, "((`a` > ?) OR (`a` = ? AND `b` > ?) OR (`a` = ? AND `b` = ? AND `c` > ?)) AND ((`a` < ?) OR (`a` = ? AND `b` < ?) OR (`a` = ? AND `b` = ? AND `c` < ?))") expectArgs = []string{"1", "1", "3", "1", "3", "5", "2", "2", "4", "2", "4", "6"} for i, arg := range args { c.Assert(arg, Equals, expectArgs[i]) } - conditions, args = chunk.toString(bucketMode, "latin1") + conditions, args = chunk.toString("latin1") c.Assert(conditions, Equals, "((`a` COLLATE 'latin1' > ?) OR (`a` = ? AND `b` COLLATE 'latin1' > ?) OR (`a` = ? AND `b` = ? AND `c` COLLATE 'latin1' > ?)) AND ((`a` COLLATE 'latin1' < ?) OR (`a` = ? AND `b` COLLATE 'latin1' < ?) OR (`a` = ? AND `b` = ? AND `c` COLLATE 'latin1' < ?))") expectArgs = []string{"1", "1", "3", "1", "3", "5", "2", "2", "4", "2", "4", "6"} for i, arg := range args { diff --git a/pkg/diff/diff.go b/pkg/diff/diff.go index 18ce61ce3..c7165ca00 100644 --- a/pkg/diff/diff.go +++ b/pkg/diff/diff.go @@ -16,82 +16,105 @@ package diff import ( "container/heap" "context" + "crypto/md5" "database/sql" + "encoding/json" "fmt" + "math/rand" "strconv" "strings" "sync" + "time" "github.com/pingcap/errors" "github.com/pingcap/parser/model" "github.com/pingcap/tidb-tools/pkg/dbutil" "github.com/pingcap/tidb-tools/pkg/utils" log "github.com/sirupsen/logrus" + "go.uber.org/zap" ) // TableInstance record a table instance type TableInstance struct { - Conn *sql.DB - Schema string - Table string - info *model.TableInfo + Conn *sql.DB `json:"-"` + Schema string `json:"schema"` + Table string `json:"table"` + InstanceID string `json:"instance-id"` + info *model.TableInfo } // TableDiff saves config for diff table type TableDiff struct { // source tables - SourceTables []*TableInstance + SourceTables []*TableInstance `json:"source-tables"` // target table - TargetTable *TableInstance + TargetTable *TableInstance `json:"target-table"` // columns be ignored - IgnoreColumns []string + IgnoreColumns []string `json:"-"` // columns be removed - RemoveColumns []string + RemoveColumns []string `json:"-"` // field should be the primary key, unique key or field with index - Fields string + Fields string `json:"fields"` // select range, for example: "age > 10 AND age < 20" - Range string + Range string `json:"range"` // for example, the whole data is [1...100] // we can split these data to [1...10], [11...20], ..., [91...100] // the [1...10] is a chunk, and it's chunk size is 10 // size of the split chunk - ChunkSize int + ChunkSize int `json:"chunk-size"` // sampling check percent, for example 10 means only check 10% data - Sample int + Sample int `json:"sample"` // how many goroutines are created to check data - CheckThreadCount int + CheckThreadCount int `json:"-"` // set true if target-db and source-db all support tidb implicit column "_tidb_rowid" - UseRowID bool + UseRowID bool `json:"use-rowid"` // set false if want to comapre the data directly - UseChecksum bool + UseChecksum bool `json:"-"` // set true if just want compare data by checksum, will skip select data when checksum is not equal - OnlyUseChecksum bool + OnlyUseChecksum bool `json:"-"` // collation config in mysql/tidb, should corresponding to charset. - Collation string + Collation string `json:"collation"` // ignore check table's struct - IgnoreStructCheck bool + IgnoreStructCheck bool `json:"-"` // ignore check table's data - IgnoreDataCheck bool + IgnoreDataCheck bool `json:"-"` + + // set true will continue check from the latest checkpoint + UseCheckpoint bool `json:"use-checkpoint"` // get tidb statistics information from which table instance. if is nil, will split chunk by random. - TiDBStatsSource *TableInstance + TiDBStatsSource *TableInstance `json:"tidb-stats-source"` sqlCh chan string wg sync.WaitGroup + + configHash string +} + +func (t *TableDiff) setConfigHash() error { + jsonBytes, err := json.Marshal(t) + if err != nil { + return errors.Trace(err) + } + + t.configHash = fmt.Sprintf("%x", md5.Sum(jsonBytes)) + log.Debug("sync-diff-inspector config", zap.ByteString("config", jsonBytes), zap.String("hash", t.configHash)) + + return nil } // Equal tests whether two database have same data and schema. @@ -99,11 +122,9 @@ func (t *TableDiff) Equal(ctx context.Context, writeFixSQL func(string) error) ( t.adjustConfig() t.sqlCh = make(chan string) - t.wg.Add(1) - go func() { - t.WriteSqls(ctx, writeFixSQL) - t.wg.Done() - }() + + stopWriteSqlsCh := t.WriteSqls(ctx, writeFixSQL) + stopUpdateSummaryCh := t.UpdateSummaryInfo(ctx) err := t.getTableInfo(ctx) if err != nil { @@ -127,7 +148,9 @@ func (t *TableDiff) Equal(ctx context.Context, writeFixSQL func(string) error) ( } } - t.sqlCh <- "end" + stopWriteSqlsCh <- true + stopUpdateSummaryCh <- true + t.wg.Wait() return structEqual, dataEqual, nil } @@ -180,64 +203,70 @@ func (t *TableDiff) getTableInfo(ctx context.Context) error { } // CheckTableData checks table's data -func (t *TableDiff) CheckTableData(ctx context.Context) (bool, error) { - return t.EqualTableData(ctx) -} - -// EqualTableData checks data is equal or not. -func (t *TableDiff) EqualTableData(ctx context.Context) (equal bool, err error) { - var allJobs []*CheckJob - +func (t *TableDiff) CheckTableData(ctx context.Context) (equal bool, err error) { table := t.TargetTable - useTiDB := false + useTiDB := false if t.TiDBStatsSource != nil { table = t.TiDBStatsSource useTiDB = true } - allJobs, err = GenerateCheckJob(table, t.Fields, t.Range, t.ChunkSize, t.Collation, useTiDB) - + fromCheckpoint := true + chunks, err := t.LoadCheckpoint(ctx) if err != nil { return false, errors.Trace(err) } - checkNums := len(allJobs) * t.Sample / 100 - checkNumArr := getRandomN(len(allJobs), checkNums) - log.Infof("total has %d check jobs, check %d of them", len(allJobs), len(checkNumArr)) - if checkNums == 0 { + if len(chunks) == 0 { + log.Debug("don't have checkpoint info or config changed") + + fromCheckpoint = false + chunks, err = SplitChunks(ctx, table, t.Fields, t.Range, t.ChunkSize, t.Collation, useTiDB) + } + + if len(chunks) == 0 { + log.Warn("get 0 chunks, table is not checked", zap.String("table", dbutil.TableName(t.TargetTable.Schema, t.TargetTable.Table))) return true, nil } checkResultCh := make(chan bool, t.CheckThreadCount) defer close(checkResultCh) + checkWorkerCh := make([]chan *ChunkRange, 0, t.CheckThreadCount) for i := 0; i < t.CheckThreadCount; i++ { - checkJobs := make([]*CheckJob, 0, len(checkNumArr)) - for j := len(checkNumArr) * i / t.CheckThreadCount; j < len(checkNumArr)*(i+1)/t.CheckThreadCount && j < len(checkNumArr); j++ { - checkJobs = append(checkJobs, allJobs[checkNumArr[j]]) - } - go func(checkJobs []*CheckJob) { - eq, err := t.checkChunkDataEqual(ctx, checkJobs) - if err != nil { - log.Errorf("check chunk data equal failed, error %v", errors.ErrorStack(err)) - } - checkResultCh <- eq - }(checkJobs) + checkWorkerCh = append(checkWorkerCh, make(chan *ChunkRange, 10)) + go t.checkChunksDataEqual(ctx, t.Sample < 100 && !fromCheckpoint, checkWorkerCh[i], checkResultCh) } - num := 0 + go func() { + defer func() { + for _, ch := range checkWorkerCh { + close(ch) + } + }() + + for _, chunk := range chunks { + select { + case checkWorkerCh[chunk.ID%t.CheckThreadCount] <- chunk: + case <-ctx.Done(): + return + } + } + }() + + checkedNum := 0 equal = true CheckResult: for { select { case eq := <-checkResultCh: - num++ + checkedNum++ if !eq { equal = false } - if num == t.CheckThreadCount { + if len(chunks) == checkedNum { break CheckResult } case <-ctx.Done(): @@ -247,11 +276,58 @@ CheckResult: return equal, nil } -func (t *TableDiff) getSourceTableChecksum(ctx context.Context, job *CheckJob) (int64, error) { +// LoadCheckpoint do some prepare work before check data, like adjust config and create checkpoint table +func (t *TableDiff) LoadCheckpoint(ctx context.Context) ([]*ChunkRange, error) { + ctx1, cancel1 := context.WithTimeout(ctx, 5*dbutil.DefaultTimeout) + defer cancel1() + + err := t.setConfigHash() + if err != nil { + return nil, errors.Trace(err) + } + + err = createCheckpointTable(ctx1, t.TargetTable.Conn) + if err != nil { + return nil, errors.Trace(err) + } + + if t.UseCheckpoint { + useCheckpoint, err := loadFromCheckPoint(ctx1, t.TargetTable.Conn, t.TargetTable.Schema, t.TargetTable.Table, t.configHash) + if err != nil { + return nil, errors.Trace(err) + } + + if useCheckpoint { + log.Info("use checkpoint to load chunks") + chunks, err := loadChunks(ctx1, t.TargetTable.Conn, t.TargetTable.InstanceID, t.TargetTable.Schema, t.TargetTable.Table) + if err != nil { + log.Error("load chunks info", zap.Error(err)) + return nil, errors.Trace(err) + } + + return chunks, nil + } + } + + // clean old checkpoint infomation, and initial table summary + err = cleanCheckpoint(ctx1, t.TargetTable.Conn, t.TargetTable.Schema, t.TargetTable.Table) + if err != nil { + return nil, errors.Trace(err) + } + + err = initTableSummary(ctx1, t.TargetTable.Conn, t.TargetTable.Schema, t.TargetTable.Table, t.configHash) + if err != nil { + return nil, errors.Trace(err) + } + + return nil, nil +} + +func (t *TableDiff) getSourceTableChecksum(ctx context.Context, chunk *ChunkRange) (int64, error) { var checksum int64 for _, sourceTable := range t.SourceTables { - checksumTmp, err := dbutil.GetCRC32Checksum(ctx, sourceTable.Conn, sourceTable.Schema, sourceTable.Table, t.TargetTable.info, job.Where, utils.StringsToInterfaces(job.Args), utils.SliceToMap(t.IgnoreColumns)) + checksumTmp, err := dbutil.GetCRC32Checksum(ctx, sourceTable.Conn, sourceTable.Schema, sourceTable.Table, t.TargetTable.info, chunk.Where, utils.StringsToInterfaces(chunk.Args), utils.SliceToMap(t.IgnoreColumns)) if err != nil { return -1, errors.Trace(err) } @@ -261,68 +337,134 @@ func (t *TableDiff) getSourceTableChecksum(ctx context.Context, job *CheckJob) ( return checksum, nil } -func (t *TableDiff) checkChunkDataEqual(ctx context.Context, checkJobs []*CheckJob) (bool, error) { - equal := true - if len(checkJobs) == 0 { - return true, nil - } - - for _, job := range checkJobs { - if t.UseChecksum { - // first check the checksum is equal or not - sourceChecksum, err := t.getSourceTableChecksum(ctx, job) - if err != nil { - return false, errors.Trace(err) - } - - targetChecksum, err := dbutil.GetCRC32Checksum(ctx, t.TargetTable.Conn, t.TargetTable.Schema, t.TargetTable.Table, t.TargetTable.info, job.Where, utils.StringsToInterfaces(job.Args), utils.SliceToMap(t.IgnoreColumns)) - if err != nil { - return false, errors.Trace(err) +func (t *TableDiff) checkChunksDataEqual(ctx context.Context, filterByRand bool, chunks chan *ChunkRange, resultCh chan bool) { + for { + select { + case chunk, ok := <-chunks: + if !ok { + return } - if sourceChecksum == targetChecksum { - log.Infof("table: %s, range: %s, args: %v, checksum is equal, checksum: %d", dbutil.TableName(job.Schema, job.Table), job.Where, job.Args, sourceChecksum) + if chunk.State == successState || chunk.State == ignoreState { + resultCh <- true continue } - - log.Warnf("table: %s, range: %s, args: %v, checksum is not equal, one is %d, another is %d", dbutil.TableName(job.Schema, job.Table), job.Where, job.Args, sourceChecksum, targetChecksum) + eq, err := t.checkChunkDataEqual(ctx, filterByRand, chunk) + if err != nil { + log.Error("check chunk data equal failed", zap.String("chunk", chunk.String()), zap.Error(err)) + resultCh <- false + } else { + if !eq { + log.Warn("check chunk data not equal", zap.String("chunk", chunk.String())) + } + resultCh <- eq + } + case <-ctx.Done(): + return } + } +} - if t.UseChecksum && t.OnlyUseChecksum { - equal = false - continue +func (t *TableDiff) checkChunkDataEqual(ctx context.Context, filterByRand bool, chunk *ChunkRange) (equal bool, err error) { + update := func() { + ctx1, cancel1 := context.WithTimeout(ctx, dbutil.DefaultTimeout) + defer cancel1() + + err1 := saveChunk(ctx1, t.TargetTable.Conn, chunk.ID, t.TargetTable.InstanceID, t.TargetTable.Schema, t.TargetTable.Table, "", chunk) + if err1 != nil { + log.Warn("update chunk info", zap.Error(err1)) } + } - // if checksum is not equal or don't need compare checksum, compare the data - log.Infof("select data from %s for range (where: %s, args: %v) and then check data", dbutil.TableName(job.Schema, job.Table), job.Where, job.Args) - sourceRows := make(map[string][]map[string]*dbutil.ColumnData) - for i, sourceTable := range t.SourceTables { - rows, _, err := getChunkRows(ctx, sourceTable.Conn, sourceTable.Schema, sourceTable.Table, sourceTable.info, job.Where, utils.StringsToInterfaces(job.Args), utils.SliceToMap(t.IgnoreColumns), t.Collation) + defer func() { + if chunk.State != ignoreState { if err != nil { - return false, errors.Trace(err) + chunk.State = errorState + } else if equal { + chunk.State = successState + } else { + chunk.State = failedState } - sourceRows[fmt.Sprintf("source-%d", i)] = rows } + update() + }() - targetRows, orderKeyCols, err := getChunkRows(ctx, t.TargetTable.Conn, t.TargetTable.Schema, t.TargetTable.Table, t.TargetTable.info, job.Where, utils.StringsToInterfaces(job.Args), utils.SliceToMap(t.IgnoreColumns), t.Collation) - if err != nil { - return false, errors.Trace(err) + if filterByRand { + rand.Seed(time.Now().UnixNano()) + r := rand.Intn(100) + if r > t.Sample { + chunk.State = ignoreState + return true, nil } + } + + chunk.State = checkingState + update() - eq, err := t.compareRows(sourceRows, targetRows, orderKeyCols) + if t.UseChecksum { + // first check the checksum is equal or not + equal, err = t.compareChecksum(ctx, chunk) if err != nil { return false, errors.Trace(err) } - - // if equal is false, we continue check data, we should find all the different data just run once - if !eq { - equal = false + if equal { + return true, nil } } + if t.UseChecksum && t.OnlyUseChecksum { + return false, nil + } + + // if checksum is not equal or don't need compare checksum, compare the data + log.Info("select data and then check data", zap.String("table", dbutil.TableName(t.TargetTable.Schema, t.TargetTable.Table)), zap.String("where", chunk.Where), zap.Reflect("args", chunk.Args)) + + equal, err = t.compareRows(ctx, chunk) + if err != nil { + return false, errors.Trace(err) + } + return equal, nil } -func (t *TableDiff) compareRows(sourceRows map[string][]map[string]*dbutil.ColumnData, targetRows []map[string]*dbutil.ColumnData, orderKeyCols []*model.ColumnInfo) (bool, error) { +func (t *TableDiff) compareChecksum(ctx context.Context, chunk *ChunkRange) (bool, error) { + // first check the checksum is equal or not + sourceChecksum, err := t.getSourceTableChecksum(ctx, chunk) + if err != nil { + return false, errors.Trace(err) + } + + targetChecksum, err := dbutil.GetCRC32Checksum(ctx, t.TargetTable.Conn, t.TargetTable.Schema, t.TargetTable.Table, t.TargetTable.info, chunk.Where, utils.StringsToInterfaces(chunk.Args), utils.SliceToMap(t.IgnoreColumns)) + if err != nil { + return false, errors.Trace(err) + } + if sourceChecksum == targetChecksum { + log.Info("checksum is equal", zap.String("table", dbutil.TableName(t.TargetTable.Schema, t.TargetTable.Table)), zap.String("where", chunk.Where), zap.Reflect("args", chunk.Args), zap.Int64("checksum", sourceChecksum)) + return true, nil + } + + log.Warn("checksum is not equal", zap.String("table", dbutil.TableName(t.TargetTable.Schema, t.TargetTable.Table)), zap.String("where", chunk.Where), zap.Reflect("args", chunk.Args), zap.Int64("source checksum", sourceChecksum), zap.Int64("target checksum", targetChecksum)) + + return false, nil +} + +func (t *TableDiff) compareRows(ctx context.Context, chunk *ChunkRange) (bool, error) { + sourceRows := make(map[string][]map[string]*dbutil.ColumnData) + args := utils.StringsToInterfaces(chunk.Args) + ignoreCloumns := utils.SliceToMap(t.IgnoreColumns) + + for i, sourceTable := range t.SourceTables { + rows, _, err := getChunkRows(ctx, sourceTable.Conn, sourceTable.Schema, sourceTable.Table, sourceTable.info, chunk.Where, args, ignoreCloumns, t.Collation) + if err != nil { + return false, errors.Trace(err) + } + sourceRows[fmt.Sprintf("source-%d", i)] = rows + } + + targetRows, orderKeyCols, err := getChunkRows(ctx, t.TargetTable.Conn, t.TargetTable.Schema, t.TargetTable.Table, t.TargetTable.info, chunk.Where, args, ignoreCloumns, t.Collation) + if err != nil { + return false, errors.Trace(err) + } + var ( equal = true rowsData1 = make([]map[string]*dbutil.ColumnData, 0, 100) @@ -430,23 +572,78 @@ func (t *TableDiff) compareRows(sourceRows map[string][]map[string]*dbutil.Colum } // WriteSqls write sqls to file -func (t *TableDiff) WriteSqls(ctx context.Context, writeFixSQL func(string) error) { - for { - select { - case dml, ok := <-t.sqlCh: - if !ok || dml == "end" { +func (t *TableDiff) WriteSqls(ctx context.Context, writeFixSQL func(string) error) chan bool { + t.wg.Add(1) + stopWriteCh := make(chan bool) + + go func() { + defer t.wg.Done() + + stop := false + for { + select { + case dml, ok := <-t.sqlCh: + if !ok { + return + } + + err := writeFixSQL(fmt.Sprintf("%s\n", dml)) + if err != nil { + log.Error("write sql failed", zap.String("sql", dml), zap.Error(err)) + } + t.wg.Done() + case <-stopWriteCh: + stop = true + case <-ctx.Done(): return + default: + if stop { + return + } + + time.Sleep(100 * time.Millisecond) } + } + }() - err := writeFixSQL(fmt.Sprintf("%s\n", dml)) + return stopWriteCh +} + +func (t *TableDiff) UpdateSummaryInfo(ctx context.Context) chan bool { + t.wg.Add(1) + stopUpdateCh := make(chan bool) + + go func() { + update := func() { + ctx1, cancel1 := context.WithTimeout(ctx, dbutil.DefaultTimeout) + defer cancel1() + + err := updateTableSummary(ctx1, t.TargetTable.Conn, t.TargetTable.InstanceID, t.TargetTable.Schema, t.TargetTable.Table) if err != nil { - log.Errorf("write sql: %s failed, error: %v", dml, err) + log.Error("save table summary info failed", zap.String("schema", t.TargetTable.Schema), zap.String("table", t.TargetTable.Table), zap.Error(err)) } + } + defer func() { + update() t.wg.Done() - case <-ctx.Done(): - return + }() + + ticker := time.NewTicker(10 * time.Second) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + return + case <-stopUpdateCh: + return + case <-ticker.C: + update() + } } - } + }() + + return stopUpdateCh } func generateDML(tp string, data map[string]*dbutil.ColumnData, keys []*model.ColumnInfo, table *model.TableInfo, schema string) (sql string) { diff --git a/pkg/diff/diff_test.go b/pkg/diff/diff_test.go index f2ffa4fbf..aa676bb4e 100644 --- a/pkg/diff/diff_test.go +++ b/pkg/diff/diff_test.go @@ -332,3 +332,22 @@ func updateData(dbConn *sql.DB, table string) error { return nil } + +func (*testDiffSuite) TestConfigHash(c *C) { + tbDiff := &TableDiff{ + Range: "a > 1", + ChunkSize: 1000, + } + tbDiff.setConfigHash() + hash1 := tbDiff.configHash + + tbDiff.CheckThreadCount = 10 + tbDiff.setConfigHash() + hash2 := tbDiff.configHash + c.Assert(hash1, Equals, hash2) + + tbDiff.Range = "b < 10" + tbDiff.setConfigHash() + hash3 := tbDiff.configHash + c.Assert(hash1 == hash3, Equals, false) +} diff --git a/pkg/diff/spliter_test.go b/pkg/diff/spliter_test.go index 21dc28f7d..5c1c5fc9d 100644 --- a/pkg/diff/spliter_test.go +++ b/pkg/diff/spliter_test.go @@ -155,7 +155,7 @@ func (s *testSpliterSuite) TestRandomSpliter(c *C) { c.Assert(err, IsNil) for j, chunk := range chunks { - chunkStr, args := chunk.toString(normalMode, "") + chunkStr, args := chunk.toString("") c.Assert(chunkStr, Equals, expectResult[i][j].chunkStr) c.Assert(args, DeepEquals, expectResult[i][j].args) } @@ -178,11 +178,11 @@ func (s *testSpliterSuite) TestRandomSpliter(c *C) { table: tableInstance, } - oriChunk := newChunkRange().copyAndUpdate("a", "0", gt, "10", lt) + oriChunk := NewChunkRange(normalMode).copyAndUpdate("a", "0", gt, "10", lt) chunks, err := r.splitRange(db, oriChunk, 2, "test", "test", tableInfo.Columns) c.Assert(err, IsNil) for i, chunk := range chunks { - chunkStr, args := chunk.toString(normalMode, "") + chunkStr, args := chunk.toString("") c.Assert(chunkStr, Equals, expectChunks[i].chunkStr) c.Assert(args, DeepEquals, expectChunks[i].args) } @@ -282,14 +282,13 @@ func (s *testSpliterSuite) TestBucketSpliter(c *C) { } for i, chunkSize := range chunkSizes { - fmt.Println(i) createFakeResultForBucketSplit(mock) bSpliter := new(bucketSpliter) chunks, err := bSpliter.split(tableInstance, tableInfo.Columns, chunkSize, "TRUE", "") c.Assert(err, IsNil) for j, chunk := range chunks { - chunkStr, args := chunk.toString(bucketMode, "") + chunkStr, args := chunk.toString("") c.Assert(chunkStr, Equals, expectResult[i][j].chunkStr) c.Assert(args, DeepEquals, expectResult[i][j].args) } diff --git a/pkg/diff/util.go b/pkg/diff/util.go index 3e98762e4..2d4eb5627 100644 --- a/pkg/diff/util.go +++ b/pkg/diff/util.go @@ -14,13 +14,10 @@ package diff import ( - "math/rand" - "github.com/pingcap/parser/model" "github.com/pingcap/tidb-tools/pkg/dbutil" "github.com/pingcap/tidb-tools/pkg/utils" "github.com/pingcap/tidb/types" - log "github.com/sirupsen/logrus" ) func equalStrings(str1, str2 []string) bool { @@ -80,25 +77,6 @@ func getColumnsFromIndex(index *model.IndexInfo, tableInfo *model.TableInfo) []* return indexColumns } -func getRandomN(total, num int) []int { - if num > total { - log.Warnf("the num %d is greater than total %d", num, total) - num = total - } - - totalArray := make([]int, 0, total) - for i := 0; i < total; i++ { - totalArray = append(totalArray, i) - } - - for j := 0; j < num; j++ { - r := j + rand.Intn(total-j) - totalArray[j], totalArray[r] = totalArray[r], totalArray[j] - } - - return totalArray[:num] -} - func needQuotes(ft types.FieldType) bool { return !(dbutil.IsNumberType(ft.Tp) || dbutil.IsFloatType(ft.Tp)) } diff --git a/pkg/utils/errors_test.go b/pkg/utils/errors_test.go new file mode 100644 index 000000000..e2f9615d1 --- /dev/null +++ b/pkg/utils/errors_test.go @@ -0,0 +1,36 @@ +// Copyright 2019 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package utils + +import ( + . "github.com/pingcap/check" + "github.com/pingcap/errors" +) + +var _ = Suite(&testErrorsSuite{}) + +type testErrorsSuite struct{} + +func (s *testErrorsSuite) TestOriginError(c *C) { + c.Assert(OriginError(nil), IsNil) + + err1 := errors.New("err1") + c.Assert(OriginError(err1), DeepEquals, err1) + + err2 := errors.Trace(err1) + c.Assert(OriginError(err2), DeepEquals, err1) + + err3 := errors.Trace(err2) + c.Assert(OriginError(err3), DeepEquals, err1) +} diff --git a/sync_diff_inspector/config.go b/sync_diff_inspector/config.go index 9622f654b..1dbc8719f 100644 --- a/sync_diff_inspector/config.go +++ b/sync_diff_inspector/config.go @@ -201,6 +201,9 @@ type Config struct { // ignore check table's data IgnoreDataCheck bool `toml:"ignore-data-check" json:"ignore-data-check"` + // set true will continue check from the latest checkpoint + UseCheckpoint bool `toml:"use-checkpoint" json:"use-checkpoint"` + // use this tidb's statistics information to split chunk TiDBInstanceID string `toml:"tidb-instance-id" json:"tidb-instance-id"` @@ -228,6 +231,7 @@ func NewConfig() *Config { fs.BoolVar(&cfg.PrintVersion, "V", false, "print version of sync_diff_inspector") fs.BoolVar(&cfg.IgnoreDataCheck, "ignore-data-check", false, "ignore check table's data") fs.BoolVar(&cfg.IgnoreStructCheck, "ignore-struct-check", false, "ignore check table's struct") + fs.BoolVar(&cfg.UseCheckpoint, "use-checkpoint", true, "set true will continue check from the latest checkpoint") return cfg } diff --git a/sync_diff_inspector/config.toml b/sync_diff_inspector/config.toml index d175f2756..6b96dd668 100644 --- a/sync_diff_inspector/config.toml +++ b/sync_diff_inspector/config.toml @@ -24,6 +24,9 @@ use-checksum = true # set true if just want compare data by checksum, will skip select data when checksum is not equal. only-use-checksum = false +# set true will continue check from the latest checkpoint +use-checkpoint = true + # ignore check table's data ignore-data-check = false diff --git a/sync_diff_inspector/diff.go b/sync_diff_inspector/diff.go index 30b5c7175..615e406da 100644 --- a/sync_diff_inspector/diff.go +++ b/sync_diff_inspector/diff.go @@ -25,6 +25,7 @@ import ( router "github.com/pingcap/tidb-tools/pkg/table-router" "github.com/pingcap/tidb-tools/pkg/utils" log "github.com/sirupsen/logrus" + "go.uber.org/zap" ) // Diff contains two sql DB, used for comparing. @@ -36,6 +37,7 @@ type Diff struct { checkThreadCount int useRowID bool useChecksum bool + useCheckpoint bool onlyUseChecksum bool ignoreDataCheck bool ignoreStructCheck bool @@ -57,6 +59,7 @@ func NewDiff(ctx context.Context, cfg *Config) (diff *Diff, err error) { checkThreadCount: cfg.CheckThreadCount, useRowID: cfg.UseRowID, useChecksum: cfg.UseChecksum, + useCheckpoint: cfg.UseCheckpoint, onlyUseChecksum: cfg.OnlyUseChecksum, ignoreDataCheck: cfg.IgnoreDataCheck, ignoreStructCheck: cfg.IgnoreStructCheck, @@ -366,9 +369,10 @@ func (df *Diff) Equal() (err error) { sourceTables := make([]*diff.TableInstance, 0, len(table.SourceTables)) for _, sourceTable := range table.SourceTables { sourceTableInstance := &diff.TableInstance{ - Conn: df.sourceDBs[sourceTable.InstanceID].Conn, - Schema: sourceTable.Schema, - Table: sourceTable.Table, + Conn: df.sourceDBs[sourceTable.InstanceID].Conn, + Schema: sourceTable.Schema, + Table: sourceTable.Table, + InstanceID: sourceTable.InstanceID, } sourceTables = append(sourceTables, sourceTableInstance) @@ -378,9 +382,10 @@ func (df *Diff) Equal() (err error) { } targetTableInstance := &diff.TableInstance{ - Conn: df.targetDB.Conn, - Schema: table.Schema, - Table: table.Table, + Conn: df.targetDB.Conn, + Schema: table.Schema, + Table: table.Table, + InstanceID: df.targetDB.InstanceID, } if df.targetDB.InstanceID == df.tidbInstanceID { @@ -406,6 +411,7 @@ func (df *Diff) Equal() (err error) { CheckThreadCount: df.checkThreadCount, UseRowID: df.useRowID, UseChecksum: df.useChecksum, + UseCheckpoint: df.useCheckpoint, OnlyUseChecksum: df.onlyUseChecksum, IgnoreStructCheck: df.ignoreStructCheck, IgnoreDataCheck: df.ignoreDataCheck, @@ -414,11 +420,11 @@ func (df *Diff) Equal() (err error) { structEqual, dataEqual, err := td.Equal(df.ctx, func(dml string) error { _, err := df.fixSQLFile.WriteString(fmt.Sprintf("%s\n", dml)) - return err + return errors.Trace(err) }) if err != nil { - log.Errorf("check %s.%s equal failed, error %v", table.Schema, table.Table, errors.ErrorStack(err)) - return err + log.Error("check failed", zap.String("table", dbutil.TableName(table.Schema, table.Table)), zap.Error(err)) + return errors.Trace(err) } df.report.SetTableStructCheckResult(table.Schema, table.Table, structEqual) diff --git a/tidb-binlog/pump_client/client.go b/tidb-binlog/pump_client/client.go index 98aa2694d..cfb3bd1d9 100644 --- a/tidb-binlog/pump_client/client.go +++ b/tidb-binlog/pump_client/client.go @@ -297,7 +297,7 @@ func (c *PumpsClient) WriteBinlog(binlog *pb.Binlog) error { } else { if !isRetryableError(err) { // this kind of error is not retryable, return directly. - return err + return errors.Trace(err) } // make sure already retry every avaliable pump. diff --git a/tidb-binlog/pump_client/pump.go b/tidb-binlog/pump_client/pump.go index 9053d23ab..960cf91cb 100644 --- a/tidb-binlog/pump_client/pump.go +++ b/tidb-binlog/pump_client/pump.go @@ -105,7 +105,7 @@ func (p *PumpStatus) createGrpcClient() error { } if err != nil { atomic.AddInt64(&p.ErrNum, 1) - return err + return errors.Trace(err) } p.grpcConn = clientConn From da4054f1ed6d249594559aed11281c39d2b11e8b Mon Sep 17 00:00:00 2001 From: WangXiangUSTC Date: Wed, 10 Apr 2019 15:49:29 +0800 Subject: [PATCH 2/5] update go.mod and go.sum --- go.mod1 | 16 +++++++++------- go.sum1 | 20 ++------------------ 2 files changed, 11 insertions(+), 25 deletions(-) diff --git a/go.mod1 b/go.mod1 index e1db8da71..08c853e1d 100644 --- a/go.mod1 +++ b/go.mod1 @@ -36,6 +36,7 @@ require ( github.com/grpc-ecosystem/grpc-gateway v1.5.1 // indirect github.com/jonboulle/clockwork v0.1.0 // indirect github.com/klauspost/cpuid v0.0.0-20180405133222-e7e905edc00e // indirect + github.com/konsorten/go-windows-terminal-sequences v1.0.1 // indirect github.com/kr/pretty v0.1.0 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect github.com/montanaflynn/stats v0.0.0-20180911141734-db72e6cae808 // indirect @@ -62,15 +63,16 @@ require ( github.com/remyoudompheng/bigfft v0.0.0-20170806203942-52369c62f446 // indirect github.com/siddontang/go v0.0.0-20180604090527-bdc77568d726 github.com/siddontang/go-mysql v0.0.0-20190312052122-c6ab05a85eb8 - github.com/sirupsen/logrus v1.4.1 + github.com/sirupsen/logrus v1.1.1 github.com/soheilhy/cmux v0.1.4 // indirect - github.com/spaolacci/murmur3 v1.1.0 // indirect - github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5 // indirect + github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72 // indirect + github.com/tmc/grpc-websocket-proxy v0.0.0-20171017195756-830351dc03c6 // indirect github.com/twinj/uuid v1.0.0 // indirect - github.com/uber/jaeger-client-go v2.16.0+incompatible // indirect - github.com/uber/jaeger-lib v2.0.0+incompatible // indirect - github.com/ugorji/go v1.1.4 // indirect - github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 // indirect + github.com/uber/jaeger-client-go v2.15.0+incompatible // indirect + github.com/uber/jaeger-lib v1.5.0 // indirect + github.com/ugorji/go v1.1.1 // indirect + github.com/unrolled/render v0.0.0-20180914162206-b9786414de4d // indirect + github.com/xiang90/probing v0.0.0-20160813154853-07dd2e8dfe18 // indirect go.uber.org/atomic v1.3.2 go.uber.org/zap v1.9.1 golang.org/x/crypto v0.0.0-20181030102418-4d3f4d9ffa16 // indirect diff --git a/go.sum1 b/go.sum1 index b575206b4..35947aa51 100644 --- a/go.sum1 +++ b/go.sum1 @@ -126,51 +126,35 @@ github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqn github.com/remyoudompheng/bigfft v0.0.0-20170806203942-52369c62f446 h1:/NRJ5vAYoqz+7sG51ubIDHXeWO8DlTSrToPu6q11ziA= github.com/remyoudompheng/bigfft v0.0.0-20170806203942-52369c62f446/go.mod h1:uYEyJGbgTkfkS4+E/PavXkNJcbFIpEtjt2B0KDQ5+9M= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= -github.com/shirou/gopsutil v2.18.10+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= -github.com/shurcooL/httpfs v0.0.0-20171119174359-809beceb2371/go.mod h1:ZY1cvUeJuFPAdZ/B6v7RHavJWZn2YPVFQ1OSXhCGOkg= -github.com/shurcooL/vfsgen v0.0.0-20181020040650-a97a25d856ca/go.mod h1:TrYk7fJVaAttu97ZZKrO9UbRa8izdowaMIZcxYMbVaw= +github.com/siddontang/go v0.0.0-20180604090527-bdc77568d726 h1:xT+JlYxNGqyT+XcU8iUrN18JYed2TvG9yN5ULG2jATM= github.com/siddontang/go v0.0.0-20180604090527-bdc77568d726/go.mod h1:3yhqj7WBBfRhbBlzyOC3gUxftwsU0u8gqevxwIHQpMw= github.com/siddontang/go-log v0.0.0-20180807004314-8d05993dda07 h1:oI+RNwuC9jF2g2lP0u0cVEEZrc/AYBCuFdvwrLWM/6Q= github.com/siddontang/go-log v0.0.0-20180807004314-8d05993dda07/go.mod h1:yFdBgwXP24JziuRl2NMUahT7nGLNOKi1SIiFxMttVD4= github.com/siddontang/go-mysql v0.0.0-20190312052122-c6ab05a85eb8 h1:8puKTg/UOIQ+ZiowY1ywmGsI08sWqrKD7HJ/j165CUM= github.com/siddontang/go-mysql v0.0.0-20190312052122-c6ab05a85eb8/go.mod h1:/b8ZcWjAShCcHp2dWpjb1vTlNyiG03UeHEQr2jteOpI= -github.com/sirupsen/logrus v0.0.0-20170323161349-3bcb09397d6d/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= +github.com/sirupsen/logrus v1.1.1 h1:VzGj7lhU7KEB9e9gMpAV/v5XT2NVSvLJhJLCWbnkgXg= github.com/sirupsen/logrus v1.1.1/go.mod h1:zrgwTnHtNr00buQ1vSptGe8m1f/BbgsPukg8qsT7A+A= -github.com/sirupsen/logrus v1.4.1 h1:GL2rEmy6nsikmW0r8opw9JIRScdMF5hA8cOYLH7In1k= -github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= github.com/soheilhy/cmux v0.1.4 h1:0HKaf1o97UwFjHH9o5XsHUOF+tqmdA7KEzXLpiyaw0E= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72 h1:qLC7fQah7D6K1B0ujays3HV9gkFtllcxhzImRR7ArPQ= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= -github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= -github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/tmc/grpc-websocket-proxy v0.0.0-20171017195756-830351dc03c6 h1:lYIiVDtZnyTWlNwiAxLj0bbpTcx1BWCFhXjfsvmPdNc= github.com/tmc/grpc-websocket-proxy v0.0.0-20171017195756-830351dc03c6/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= -github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/twinj/uuid v1.0.0 h1:fzz7COZnDrXGTAOHGuUGYd6sG+JMq+AoE7+Jlu0przk= github.com/twinj/uuid v1.0.0/go.mod h1:mMgcE1RHFUFqe5AfiwlINXisXfDGro23fWdPUfOMjRY= github.com/uber/jaeger-client-go v2.15.0+incompatible h1:NP3qsSqNxh8VYr956ur1N/1C1PjvOJnJykCzcD5QHbk= github.com/uber/jaeger-client-go v2.15.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk= -github.com/uber/jaeger-client-go v2.16.0+incompatible h1:Q2Pp6v3QYiocMxomCaJuwQGFt7E53bPYqEgug/AoBtY= -github.com/uber/jaeger-client-go v2.16.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk= github.com/uber/jaeger-lib v1.5.0 h1:OHbgr8l656Ub3Fw5k9SWnBfIEwvoHQ+W2y+Aa9D1Uyo= github.com/uber/jaeger-lib v1.5.0/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U= -github.com/uber/jaeger-lib v2.0.0+incompatible h1:iMSCV0rmXEogjNWPh2D0xk9YVKvrtGoHJNe9ebLu/pw= -github.com/uber/jaeger-lib v2.0.0+incompatible/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U= github.com/ugorji/go v1.1.1 h1:gmervu+jDMvXTbcHQ0pd2wee85nEoE0BsVyEuzkfK8w= github.com/ugorji/go v1.1.1/go.mod h1:hnLbHMwcvSihnDhEfx2/BzKp2xb0Y+ErdfYcrs9tkJQ= -github.com/ugorji/go v1.1.4 h1:j4s+tAvLfL3bZyefP2SEWmhBzmuIlH/eqNuPdFPgngw= -github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= github.com/unrolled/render v0.0.0-20180914162206-b9786414de4d/go.mod h1:tu82oB5W2ykJRVioYsB+IQKcft7ryBr7w12qMBUPyXg= github.com/xiang90/probing v0.0.0-20160813154853-07dd2e8dfe18 h1:MPPkRncZLN9Kh4MEFmbnK4h3BD7AUmskWv2+EeZJCCs= github.com/xiang90/probing v0.0.0-20160813154853-07dd2e8dfe18/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= -github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8= -github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= go.uber.org/atomic v1.3.2 h1:2Oa65PReHzfn29GpvgsYwloV9AVFHPDk8tYxt2c2tr4= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/multierr v1.1.0 h1:HoEmRHQPVSqub6w2z2d2EOVs2fjyFRGyofhKuyDq0QI= From 731fd956bea8829da0cb4961eeeca7cf6eae2bdc Mon Sep 17 00:00:00 2001 From: WangXiangUSTC Date: Wed, 20 Mar 2019 19:01:57 +0800 Subject: [PATCH 3/5] update log to pingcap/log --- checker/main.go | 13 +- go.mod1 | 43 +----- go.sum1 | 179 ++++++++++++++-------- importer/main.go | 5 +- pkg/dbutil/common.go | 18 +-- pkg/ddl-checker/executable_checker.go | 7 +- pkg/diff/chunk.go | 13 +- pkg/diff/diff.go | 20 +-- pkg/diff/merge.go | 10 +- pkg/importer/db.go | 9 +- pkg/importer/importer.go | 13 +- pkg/importer/job.go | 12 +- pkg/importer/parser.go | 5 +- sync_diff_inspector/config.go | 11 +- sync_diff_inspector/diff.go | 6 +- sync_diff_inspector/main.go | 27 ++-- tidb-binlog/binlogctl/main.go | 8 +- tidb-binlog/binlogctl/meta.go | 17 +- tidb-binlog/binlogctl/nodes.go | 14 +- tidb-binlog/driver/example/mysql/mysql.go | 17 +- tidb-binlog/driver/example/print/print.go | 5 +- tidb-binlog/driver/reader/offset.go | 9 +- tidb-binlog/driver/reader/reader.go | 17 +- tidb-binlog/node/node.go | 11 ++ tidb-binlog/node/registry.go | 9 +- tidb-binlog/pump_client/client.go | 34 ++-- tidb-binlog/pump_client/pump.go | 5 +- tidb-binlog/pump_client/selector.go | 15 +- 28 files changed, 300 insertions(+), 252 deletions(-) diff --git a/checker/main.go b/checker/main.go index 4f89d93a8..d73a1e13d 100644 --- a/checker/main.go +++ b/checker/main.go @@ -20,10 +20,11 @@ import ( "os" _ "github.com/go-sql-driver/mysql" + "github.com/pingcap/log" "github.com/pingcap/tidb-tools/pkg/check" "github.com/pingcap/tidb-tools/pkg/dbutil" "github.com/pingcap/tidb-tools/pkg/utils" - log "github.com/sirupsen/logrus" + "go.uber.org/zap" ) var ( @@ -57,7 +58,7 @@ func main() { schema := flag.Args()[0] tables := flag.Args()[1:] checkTables(schema, tables) - log.Infof("complete checking!!") + log.Info("complete checking!!") } func checkTables(schema string, tables []string) { @@ -71,16 +72,16 @@ func checkTables(schema string, tables []string) { db, err := dbutil.OpenDB(*dbInfo) if err != nil { - log.Fatal("create database connection failed:", err) + log.Fatal("create database connection failed:", zap.Error(err)) } defer dbutil.CloseDB(db) result := check.NewTablesChecker(db, dbInfo, map[string][]string{schema: tables}).Check(context.Background()) if result.State == check.StateSuccess { - log.Infof("check schema %s successfully!", schema) + log.Info("check schema %s successfully!", zap.String("schema", schema)) } else if result.State == check.StateWarning { - log.Warningf("check schema %s and find warnings.\n%s", schema, result.ErrorMsg) + log.Warn("check schema find warnings.", zap.String("schema", schema), zap.String("error message", result.ErrorMsg)) } else { - log.Errorf("check schema %s and failed.\n%s", schema, result.ErrorMsg) + log.Error("check schema and failed.", zap.String("schema", schema), zap.String("error message", result.ErrorMsg)) } } diff --git a/go.mod1 b/go.mod1 index 08c853e1d..46ebd6639 100644 --- a/go.mod1 +++ b/go.mod1 @@ -3,30 +3,10 @@ module github.com/pingcap/tidb-tools require ( github.com/BurntSushi/toml v0.3.1 github.com/DATA-DOG/go-sqlmock v1.3.3 - github.com/Shopify/sarama v1.19.0 - github.com/Shopify/toxiproxy v2.1.3+incompatible // indirect - github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 // indirect - github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd // indirect - github.com/coreos/bbolt v1.3.1-coreos.6 // indirect + github.com/Shopify/sarama v1.21.0 github.com/coreos/etcd v3.3.10+incompatible - github.com/coreos/go-semver v0.2.0 // indirect - github.com/coreos/go-systemd v0.0.0-20181031085051-9002847aa142 // indirect - github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f // indirect - github.com/cznic/mathutil v0.0.0-20181021201202-eba54fb065b7 // indirect - github.com/cznic/sortutil v0.0.0-20150617083342-4c7342852e65 // indirect - github.com/dgrijalva/jwt-go v3.2.0+incompatible // indirect - github.com/dustin/go-humanize v1.0.0 // indirect - github.com/eapache/go-resiliency v1.1.0 // indirect - github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 // indirect - github.com/eapache/queue v0.0.0-20180227141424-093482f3f8ce // indirect - github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385 // indirect - github.com/etcd-io/gofail v0.0.0-20180808172546-51ce9a71510a // indirect - github.com/ghodss/yaml v1.0.0 // indirect github.com/go-sql-driver/mysql v1.4.0 - github.com/gogo/protobuf v1.1.1 // indirect - github.com/golang/groupcache v0.0.0-20181024230925-c65c006176ff // indirect github.com/golang/protobuf v1.2.0 - github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db // indirect github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c // indirect github.com/gorilla/context v1.1.1 // indirect github.com/gorilla/mux v1.6.2 // indirect @@ -34,51 +14,40 @@ require ( github.com/grpc-ecosystem/go-grpc-middleware v1.0.0 // indirect github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect github.com/grpc-ecosystem/grpc-gateway v1.5.1 // indirect - github.com/jonboulle/clockwork v0.1.0 // indirect github.com/klauspost/cpuid v0.0.0-20180405133222-e7e905edc00e // indirect github.com/konsorten/go-windows-terminal-sequences v1.0.1 // indirect github.com/kr/pretty v0.1.0 // indirect - github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect github.com/montanaflynn/stats v0.0.0-20180911141734-db72e6cae808 // indirect github.com/ngaut/log v0.0.0-20180314031856-b8e36e7ba5ac - github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 // indirect - github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef // indirect - github.com/opentracing/basictracer-go v1.0.0 // indirect - github.com/opentracing/opentracing-go v1.0.2 // indirect - github.com/pierrec/lz4 v2.0.5+incompatible // indirect github.com/pingcap/check v0.0.0-20190102082844-67f458068fc8 - github.com/pingcap/errors v0.11.0 + github.com/pingcap/errors v0.11.1 github.com/pingcap/goleveldb v0.0.0-20171020122428-b9ff6c35079e // indirect - github.com/pingcap/kvproto v0.0.0-20181028030329-855d2192cdc7 + github.com/pingcap/kvproto v0.0.0-20190226063853-f6c0b7ffff11 github.com/pingcap/log v0.0.0-20190307075452-bd41d9273596 - github.com/pingcap/parser v0.0.0-20181024082006-53ac409ed043 + github.com/pingcap/parser v0.0.0-20190328044348-9945885931bb github.com/pingcap/pd v2.1.0-rc.4+incompatible - github.com/pingcap/tidb v2.1.0-rc.3.0.20181101080907-32b1dbd8d59f+incompatible + github.com/pingcap/tidb v0.0.0-20190410081335-276699fbb7a6 github.com/pingcap/tipb v0.0.0-20181012112600-11e33c750323 github.com/prometheus/client_golang v0.9.0 // indirect github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910 // indirect github.com/prometheus/common v0.0.0-20181020173914-7e9e6cabbd39 // indirect github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d // indirect - github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a // indirect github.com/remyoudompheng/bigfft v0.0.0-20170806203942-52369c62f446 // indirect github.com/siddontang/go v0.0.0-20180604090527-bdc77568d726 github.com/siddontang/go-mysql v0.0.0-20190312052122-c6ab05a85eb8 - github.com/sirupsen/logrus v1.1.1 + github.com/sirupsen/logrus v0.0.0-20170323161349-3bcb09397d6d github.com/soheilhy/cmux v0.1.4 // indirect github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72 // indirect github.com/tmc/grpc-websocket-proxy v0.0.0-20171017195756-830351dc03c6 // indirect github.com/twinj/uuid v1.0.0 // indirect github.com/uber/jaeger-client-go v2.15.0+incompatible // indirect github.com/uber/jaeger-lib v1.5.0 // indirect - github.com/ugorji/go v1.1.1 // indirect github.com/unrolled/render v0.0.0-20180914162206-b9786414de4d // indirect - github.com/xiang90/probing v0.0.0-20160813154853-07dd2e8dfe18 // indirect go.uber.org/atomic v1.3.2 go.uber.org/zap v1.9.1 golang.org/x/crypto v0.0.0-20181030102418-4d3f4d9ffa16 // indirect golang.org/x/net v0.0.0-20181029044818-c44066c5c816 golang.org/x/sys v0.0.0-20181030150119-7e31e0c00fa0 // indirect - golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2 // indirect google.golang.org/appengine v1.2.0 // indirect google.golang.org/genproto v0.0.0-20181029155118-b69ba1387ce2 // indirect google.golang.org/grpc v1.16.0 diff --git a/go.sum1 b/go.sum1 index 35947aa51..a56e10259 100644 --- a/go.sum1 +++ b/go.sum1 @@ -1,129 +1,156 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/DATA-DOG/go-sqlmock v1.3.3 h1:CWUqKXe0s8A2z6qCgkP4Kru7wC11YoAnoupUKFDnH08= github.com/DATA-DOG/go-sqlmock v1.3.3/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= -github.com/Shopify/sarama v1.19.0 h1:9oksLxC6uxVPHPVYUmq6xhr1BOF/hHobWH2UzO67z1s= +github.com/DataDog/zstd v1.3.5 h1:DtpNbljikUepEPD16hD4LvIcmhnhdLTiW/5pHgbmp14= +github.com/DataDog/zstd v1.3.5/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= +github.com/Shopify/sarama v1.21.0 h1:0GKs+e8mn1RRUzfg9oUXv3v7ZieQLmOZF/bfnmmGhM8= +github.com/Shopify/sarama v1.21.0/go.mod h1:yuqtN/pe8cXRWG5zPaO7hCfNJp5MwmkoJEoLjkm5tCQ= github.com/Shopify/toxiproxy v2.1.3+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= -github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLMYoU8P317H5OQ+Via4RmuPwCS0= +github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= +github.com/StackExchange/wmi v0.0.0-20180725035823-b12b22c5341f/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= +github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/apache/thrift v0.0.0-20161221203622-b2a4d4ae21c7/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= +github.com/beorn7/perks v0.0.0-20160229213445-3ac7bf7a47d1/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/blacktear23/go-proxyprotocol v0.0.0-20171102103907-62e368e1c470/go.mod h1:VKt7CNAQxpFpSDz3sXyj9hY/GbVsQCr0sB3w59nE7lU= +github.com/blacktear23/go-proxyprotocol v0.0.0-20180807104634-af7a81e8dd0d/go.mod h1:VKt7CNAQxpFpSDz3sXyj9hY/GbVsQCr0sB3w59nE7lU= +github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd h1:qMd81Ts1T2OTKmB4acZcyKaMtRnY5Y44NuXGX2GFJ1w= +github.com/cockroachdb/cmux v0.0.0-20170110192607-30d10be49292/go.mod h1:qRiX68mZX1lGBkTWyp3CLcenw9I94W2dLeRvMzcn9N4= +github.com/codahale/hdrhistogram v0.0.0-20160425231609-f8ad88b59a58/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= -github.com/coreos/bbolt v1.3.1-coreos.6 h1:uTXKg9gY70s9jMAKdfljFQcuh4e/BXOM+V+d00KFj3A= +github.com/coreos/bbolt v1.3.0 h1:HIgH5xUWXT914HCI671AxuTTqjj64UOFr7pHn48LUTI= +github.com/coreos/bbolt v1.3.0/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/bbolt v1.3.1-coreos.6/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= -github.com/coreos/etcd v3.3.10+incompatible h1:jFneRYjIvLMLhDLCzuTuU4rSJUjRplcJQ7pD7MnhC04= +github.com/coreos/etcd v3.2.18+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= -github.com/coreos/go-semver v0.2.0 h1:3Jm3tLmsgAYcjC+4Up7hJrFBPr+n7rAqYeSw/SZazuY= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= -github.com/coreos/go-systemd v0.0.0-20181031085051-9002847aa142 h1:3jFq2xL4ZajGK4aZY8jz+DAF0FHjI51BXjjSwCzS1Dk= +github.com/coreos/go-systemd v0.0.0-20180202092358-40e2722dffea/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20181031085051-9002847aa142/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f h1:lBNOc5arjvs8E5mO2tbpBpLoyyu8B6e44T7hJy6potg= +github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= -github.com/cznic/mathutil v0.0.0-20181021201202-eba54fb065b7 h1:y+DH9ARrWiiNBV+6waYP2IPcsRbxdU1qsnycPfShF4c= +github.com/cznic/mathutil v0.0.0-20160613104831-78ad7f262603/go.mod h1:e6NPNENfs9mPDVNRekM7lKScauxd5kXTr1Mfyig6TDM= github.com/cznic/mathutil v0.0.0-20181021201202-eba54fb065b7/go.mod h1:e6NPNENfs9mPDVNRekM7lKScauxd5kXTr1Mfyig6TDM= -github.com/cznic/sortutil v0.0.0-20150617083342-4c7342852e65 h1:hxuZop6tSoOi0sxFzoGGYdRqNrPubyaIf9KoBG9tPiE= +github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548/go.mod h1:e6NPNENfs9mPDVNRekM7lKScauxd5kXTr1Mfyig6TDM= github.com/cznic/sortutil v0.0.0-20150617083342-4c7342852e65/go.mod h1:q2w6Bg5jeox1B+QkJ6Wp/+Vn0G/bo3f1uY7Fn3vivIQ= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/dustin/go-humanize v0.0.0-20180421182945-02af3965c54e/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= -github.com/eapache/go-resiliency v1.1.0 h1:1NtRmCAqadE2FN4ZcN6g90TP3uk8cg9rn9eNK2197aU= github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= -github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 h1:YEetp8/yCZMuEPMUDHG0CW/brkkEp8mzqk2+ODEitlw= github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= -github.com/eapache/queue v0.0.0-20180227141424-093482f3f8ce h1:wgJIjAWDwKCWtv+sEEuBOizA9xBmlOoMsP3Cs9ZDxKk= github.com/eapache/queue v0.0.0-20180227141424-093482f3f8ce/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= +github.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc= +github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385/go.mod h1:0vRUJqYpeSZifjYj7uP3BG/gKcuzL9xWVV/Y+cK33KM= github.com/etcd-io/gofail v0.0.0-20180808172546-51ce9a71510a/go.mod h1:49H/RkXP8pKaZy4h0d+NW16rSLhyVBt4o6VLJbmOqDE= -github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= -github.com/go-sql-driver/mysql v1.4.0 h1:7LxgVwFb2hIQtMm87NdgAVfXjnt4OePseqT1tKx+opk= +github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dTyBNF8= +github.com/go-sql-driver/mysql v0.0.0-20170715192408-3955978caca4/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= -github.com/gogo/protobuf v1.1.1 h1:72R+M5VuhED/KujmZVcIquuo8mBgX4oVda//DQb3PXo= +github.com/gogo/protobuf v0.0.0-20180717141946-636bf0302bc9/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20181024230925-c65c006176ff/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:tluoj9z5200jBnyusfRPU2LqT6J+DAorxEvtC7LHB+E= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM= +github.com/golang/protobuf v0.0.0-20180814211427-aa810b61a9c7/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.1.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db h1:woRePGFeVFfLKN/pOkfl+p/TAqKOfFu+7KPlMVpok/w= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c h1:964Od4U6p2jUkFxvCydnIczKteheJEzHRToSGK3Bnlw= +github.com/google/btree v0.0.0-20161217183710-316fb6d3f031/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/gorilla/context v0.0.0-20160226214623-1ea25387ff6f/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= +github.com/gorilla/mux v0.0.0-20170228224354-599cba5e7b61/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= -github.com/gorilla/websocket v1.2.0 h1:VJtLvh6VQym50czpZzx07z/kw9EgAxI3x1ZB8taTMQQ= github.com/gorilla/websocket v1.2.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= -github.com/grpc-ecosystem/go-grpc-middleware v1.0.0 h1:Iju5GlWwrvL6UBg4zJJt3btmonfrMlCDdsejg4CZE7c= +github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= +github.com/grpc-ecosystem/go-grpc-middleware v0.0.0-20171020063731-82921fcf811d/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= -github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho= +github.com/grpc-ecosystem/go-grpc-prometheus v0.0.0-20160910222444-6b7015e65d36/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= -github.com/grpc-ecosystem/grpc-gateway v1.5.1 h1:3scN4iuXkNOyP98jF55Lv8a9j1o/IwvnDIZ0LHJK1nk= +github.com/grpc-ecosystem/grpc-gateway v1.4.1/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw= github.com/grpc-ecosystem/grpc-gateway v1.5.1/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw= -github.com/jonboulle/clockwork v0.1.0 h1:VKV+ZcuP6l3yW9doeqz6ziZGgcynBVQO+obU0+0hcPo= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/cpuid v0.0.0-20180405133222-e7e905edc00e h1:+lIPJOWl+jSiJOc70QXJ07+2eg2Jy2EC7Mi11BWujeM= +github.com/klauspost/cpuid v0.0.0-20170728055534-ae7887de9fa5/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/klauspost/cpuid v0.0.0-20180405133222-e7e905edc00e/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/konsorten/go-windows-terminal-sequences v0.0.0-20180402223658-b729f2633dfe/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= +github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/montanaflynn/stats v0.0.0-20151014174947-eeaced052adb/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= github.com/montanaflynn/stats v0.0.0-20180911141734-db72e6cae808/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= github.com/ngaut/log v0.0.0-20180314031856-b8e36e7ba5ac h1:wyheT2lPXRQqYPWY2IVW5BTLrbqCsnhL61zK2R5goLA= github.com/ngaut/log v0.0.0-20180314031856-b8e36e7ba5ac/go.mod h1:ueVCjKQllPmX7uEvCYnZD5b8qjidGf1TCH61arVe4SU= -github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 h1:7KAv7KMGTTqSmYZtNdcNTgsos+vFzULLwyElndwn+5c= github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7/go.mod h1:iWMfgwqYW+e8n5lC/jjNEhwcjbRDpl5NT7n2h+4UNcI= -github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef h1:K0Fn+DoFqNqktdZtdV3bPQ/0cuYh2H4rkg0tytX/07k= github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef/go.mod h1:7WjlapSfwQyo6LNmIvEWzsW1hbBQfpUO4JWnuQRmva8= -github.com/opentracing/basictracer-go v1.0.0 h1:YyUAhaEfjoWXclZVJ9sGoNct7j4TVk7lZWlQw5UXuoo= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/gomega v1.4.1/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= +github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/opentracing/basictracer-go v1.0.0/go.mod h1:QfBfYuafItcjQuMwinw9GhYKwFXS9KnPs5lxoYwgW74= -github.com/opentracing/opentracing-go v1.0.2 h1:3jA2P6O1F9UOrWVpwrIo17pu01KWvNWg4X946/Y5Zwg= github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= -github.com/pierrec/lz4 v2.0.5+incompatible h1:2xWsjqPFWcplujydGg4WmhC/6fZqK42wMM8aXeqhl0I= +github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU= github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= -github.com/pingcap/check v0.0.0-20171206051426-1c287c953996 h1:ZBdiJCMan6GSo/aPAM7gywcUKa0z58gczVrnG6TQnAQ= github.com/pingcap/check v0.0.0-20171206051426-1c287c953996/go.mod h1:B1+S9LNcuMyLH/4HMTViQOJevkGiik3wW2AN9zb2fNQ= github.com/pingcap/check v0.0.0-20190102082844-67f458068fc8 h1:USx2/E1bX46VG32FIw034Au6seQ2fY9NEILmNh/UlQg= github.com/pingcap/check v0.0.0-20190102082844-67f458068fc8/go.mod h1:B1+S9LNcuMyLH/4HMTViQOJevkGiik3wW2AN9zb2fNQ= github.com/pingcap/errors v0.11.0 h1:DCJQB8jrHbQ1VVlMFIrbj2ApScNNotVmkSNplu2yUt4= github.com/pingcap/errors v0.11.0/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= -github.com/pingcap/goleveldb v0.0.0-20171020122428-b9ff6c35079e h1:P73/4dPCL96rGrobssy1nVy2VaVpNCuLpCbr+FEaTA8= +github.com/pingcap/errors v0.11.1 h1:BXFZ6MdDd2U1uJUa2sRAWTmm+nieEzuyYM0R4aUTcC8= +github.com/pingcap/errors v0.11.1/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= +github.com/pingcap/gofail v0.0.0-20181217135706-6a951c1e42c3/go.mod h1:DazNTg0PTldtpsQiT9I5tVJwV1onHMKBBgXzmJUlMns= +github.com/pingcap/goleveldb v0.0.0-20171020084629-8d44bfdf1030/go.mod h1:O17XtbryoCJhkKGbT62+L2OlrniwqiGLSqrmdHCMzZw= github.com/pingcap/goleveldb v0.0.0-20171020122428-b9ff6c35079e/go.mod h1:O17XtbryoCJhkKGbT62+L2OlrniwqiGLSqrmdHCMzZw= -github.com/pingcap/kvproto v0.0.0-20181028030329-855d2192cdc7 h1:CYssSnPvf90ZSbFdZpsZGSI7y+drG1EfKxqTOnKnHb0= github.com/pingcap/kvproto v0.0.0-20181028030329-855d2192cdc7/go.mod h1:0gwbe1F2iBIjuQ9AH0DbQhL+Dpr5GofU8fgYyXk+ykk= +github.com/pingcap/kvproto v0.0.0-20190226063853-f6c0b7ffff11/go.mod h1:0gwbe1F2iBIjuQ9AH0DbQhL+Dpr5GofU8fgYyXk+ykk= github.com/pingcap/log v0.0.0-20190307075452-bd41d9273596/go.mod h1:WpHUKhNZ18v116SvGrmjkA9CBhYmuUTKL+p8JC9ANEw= github.com/pingcap/parser v0.0.0-20181024082006-53ac409ed043 h1:P9Osi8lei5j2fiRgsBi2Wch7qe4a3yWUOsS5vSan/JU= github.com/pingcap/parser v0.0.0-20181024082006-53ac409ed043/go.mod h1:1FNvfp9+J0wvc4kl8eGNh7Rqrxveg15jJoWo/a0uHwA= -github.com/pingcap/pd v2.1.0-rc.4+incompatible h1:/buwGk04aHO5odk/+O8ZOXGs4qkUjYTJ2UpCJXna8NE= +github.com/pingcap/parser v0.0.0-20190227090929-c68bfe7e0257 h1:laWEk5dHgwrtECrDACHx6p58DFPlCNJQ7QRIrvF2Zuo= +github.com/pingcap/parser v0.0.0-20190227090929-c68bfe7e0257/go.mod h1:1FNvfp9+J0wvc4kl8eGNh7Rqrxveg15jJoWo/a0uHwA= +github.com/pingcap/parser v0.0.0-20190312024907-3f6280b08c8b h1:NlvTrxqezIJh6CD5Leky12IZ8E/GtpEEmzgNNb34wbw= +github.com/pingcap/parser v0.0.0-20190312024907-3f6280b08c8b/go.mod h1:1FNvfp9+J0wvc4kl8eGNh7Rqrxveg15jJoWo/a0uHwA= +github.com/pingcap/parser v0.0.0-20190328044348-9945885931bb h1:JCc0MMW4fqfhbaI8LS1dXRmql1PbpoKLxoYq6GUlSaQ= +github.com/pingcap/parser v0.0.0-20190328044348-9945885931bb/go.mod h1:1FNvfp9+J0wvc4kl8eGNh7Rqrxveg15jJoWo/a0uHwA= github.com/pingcap/pd v2.1.0-rc.4+incompatible/go.mod h1:nD3+EoYes4+aNNODO99ES59V83MZSI+dFbhyr667a0E= -github.com/pingcap/tidb v2.1.0-rc.3.0.20181101080907-32b1dbd8d59f+incompatible h1:CIZHlDunFGiLzenh8z5Rh05JgrK6DW2f5/nTRGiyF6I= -github.com/pingcap/tidb v2.1.0-rc.3.0.20181101080907-32b1dbd8d59f+incompatible/go.mod h1:I8C6jrPINP2rrVunTRd7C9fRRhQrtR43S1/CL5ix/yQ= -github.com/pingcap/tipb v0.0.0-20181012112600-11e33c750323 h1:mRKKzRjDNaUNPnAkPAHnRqpNmwNWBX1iA+hxlmvQ93I= +github.com/pingcap/tidb v0.0.0-20190228062803-4dc309315c84 h1:XQetfynT/fE/PR31FRBW9+ZrwiUas1Os8CwM/jfhxYM= +github.com/pingcap/tidb v0.0.0-20190228062803-4dc309315c84/go.mod h1:cS9CcsV5+WiqYOKn8NzkR3RKP+RCKJDN6y4tdVOYbaE= +github.com/pingcap/tidb v0.0.0-20190318101756-cb83d4cf1af2 h1:7ixvBx48UdivJqL2u3rWzCxcL8Jq64G8ZQ8HUui6IBU= +github.com/pingcap/tidb v0.0.0-20190318101756-cb83d4cf1af2/go.mod h1:6O4QBr5btg5/YtcvI+wUvHkgwqtuuJLeRcMb+I4ewvk= +github.com/pingcap/tidb v0.0.0-20190320062740-9071c7b5b9ed h1:hG3ByogjqGf/Yngmsf5RWZ4HrrwSA/cnApfKoslOlG8= +github.com/pingcap/tidb v0.0.0-20190320062740-9071c7b5b9ed/go.mod h1:FcgD4o1kq3YNk08MWtMRwNZXQJpM28bFdb/go9KpmEA= +github.com/pingcap/tidb v0.0.0-20190410081335-276699fbb7a6 h1:X6Mc9Ee+/sT7jfYD1EoplLRAO+mNQ7q+ATbIuYpONlk= +github.com/pingcap/tidb v0.0.0-20190410081335-276699fbb7a6/go.mod h1:/pDrwtisYlXaJ+Is5BNFUOn84EPvj/uwNsSHeiP+8CY= +github.com/pingcap/tidb-tools v2.1.3-0.20190116051332-34c808eef588+incompatible/go.mod h1:XGdcy9+yqlDSEMTpOXnwf3hiTeqrV6MN/u1se9N8yIM= +github.com/pingcap/tipb v0.0.0-20180910045846-371b48b15d93/go.mod h1:RtkHW8WbcNxj8lsbzjaILci01CtYnYbIkQhjyZWrWVI= github.com/pingcap/tipb v0.0.0-20181012112600-11e33c750323/go.mod h1:RtkHW8WbcNxj8lsbzjaILci01CtYnYbIkQhjyZWrWVI= -github.com/pkg/errors v0.8.0 h1:WdK/asTD0HN+q6hsWO3/vpuAkAr+tw6aNJNDFFf0+qw= +github.com/pingcap/tipb v0.0.0-20190107072121-abbec73437b7/go.mod h1:RtkHW8WbcNxj8lsbzjaILci01CtYnYbIkQhjyZWrWVI= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/prometheus/client_golang v0.9.0 h1:tXuTFVHC03mW0D+Ua1Q2d1EAVqLTuggX50V0VLICCzY= +github.com/pquerna/ffjson v0.0.0-20181028064349-e517b90714f7/go.mod h1:YARuvh7BUWHNhzDq2OM5tzR2RiCcN2D7sapiKyCel/M= +github.com/prometheus/client_golang v0.8.0/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.0/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= -github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910 h1:idejC8f05m9MGOsuEi1ATq9shN03HrxNkD/luQvxCv8= +github.com/prometheus/client_model v0.0.0-20171117100541-99fa1f4be8e5/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= -github.com/prometheus/common v0.0.0-20181020173914-7e9e6cabbd39 h1:Cto4X6SVMWRPBkJ/3YHn1iDGDGc/Z+sW+AEMKHMVvN4= +github.com/prometheus/common v0.0.0-20180426121432-d811d2e9bf89/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.0.0-20181020173914-7e9e6cabbd39/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= -github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d h1:GoAlyOgbOEIFdaDqxJVlbOQ1DtGmZWs/Qau0hIlk+WQ= +github.com/prometheus/procfs v0.0.0-20180408092902-8b1c2da0d56d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a h1:9ZKAASQSHhDYGoxY8uLVpewe1GDZ2vu2Tr/vTdVAkFQ= github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= -github.com/remyoudompheng/bigfft v0.0.0-20170806203942-52369c62f446 h1:/NRJ5vAYoqz+7sG51ubIDHXeWO8DlTSrToPu6q11ziA= github.com/remyoudompheng/bigfft v0.0.0-20170806203942-52369c62f446/go.mod h1:uYEyJGbgTkfkS4+E/PavXkNJcbFIpEtjt2B0KDQ5+9M= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= @@ -133,68 +160,88 @@ github.com/siddontang/go-log v0.0.0-20180807004314-8d05993dda07 h1:oI+RNwuC9jF2g github.com/siddontang/go-log v0.0.0-20180807004314-8d05993dda07/go.mod h1:yFdBgwXP24JziuRl2NMUahT7nGLNOKi1SIiFxMttVD4= github.com/siddontang/go-mysql v0.0.0-20190312052122-c6ab05a85eb8 h1:8puKTg/UOIQ+ZiowY1ywmGsI08sWqrKD7HJ/j165CUM= github.com/siddontang/go-mysql v0.0.0-20190312052122-c6ab05a85eb8/go.mod h1:/b8ZcWjAShCcHp2dWpjb1vTlNyiG03UeHEQr2jteOpI= +github.com/sirupsen/logrus v0.0.0-20170323161349-3bcb09397d6d/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= github.com/sirupsen/logrus v1.1.1 h1:VzGj7lhU7KEB9e9gMpAV/v5XT2NVSvLJhJLCWbnkgXg= github.com/sirupsen/logrus v1.1.1/go.mod h1:zrgwTnHtNr00buQ1vSptGe8m1f/BbgsPukg8qsT7A+A= -github.com/soheilhy/cmux v0.1.4 h1:0HKaf1o97UwFjHH9o5XsHUOF+tqmdA7KEzXLpiyaw0E= +github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= -github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72 h1:qLC7fQah7D6K1B0ujays3HV9gkFtllcxhzImRR7ArPQ= +github.com/spaolacci/murmur3 v0.0.0-20150829172844-0d12bf811670/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/tmc/grpc-websocket-proxy v0.0.0-20171017195756-830351dc03c6 h1:lYIiVDtZnyTWlNwiAxLj0bbpTcx1BWCFhXjfsvmPdNc= github.com/tmc/grpc-websocket-proxy v0.0.0-20171017195756-830351dc03c6/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= -github.com/twinj/uuid v1.0.0 h1:fzz7COZnDrXGTAOHGuUGYd6sG+JMq+AoE7+Jlu0przk= +github.com/twinj/uuid v0.0.0-20150629100731-70cac2bcd273/go.mod h1:mMgcE1RHFUFqe5AfiwlINXisXfDGro23fWdPUfOMjRY= github.com/twinj/uuid v1.0.0/go.mod h1:mMgcE1RHFUFqe5AfiwlINXisXfDGro23fWdPUfOMjRY= -github.com/uber/jaeger-client-go v2.15.0+incompatible h1:NP3qsSqNxh8VYr956ur1N/1C1PjvOJnJykCzcD5QHbk= +github.com/uber-go/atomic v1.3.2/go.mod h1:/Ct5t2lcmbJ4OSe/waGBoaVvVqtO0bmtfVNex1PFV8g= +github.com/uber/jaeger-client-go v2.8.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk= github.com/uber/jaeger-client-go v2.15.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk= -github.com/uber/jaeger-lib v1.5.0 h1:OHbgr8l656Ub3Fw5k9SWnBfIEwvoHQ+W2y+Aa9D1Uyo= +github.com/uber/jaeger-lib v1.1.0/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U= github.com/uber/jaeger-lib v1.5.0/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U= -github.com/ugorji/go v1.1.1 h1:gmervu+jDMvXTbcHQ0pd2wee85nEoE0BsVyEuzkfK8w= +github.com/ugorji/go v0.0.0-20171019201919-bdcc60b419d1 h1:UvhxfNjNqlZ/x3cDyqxMhoiUpemd3zXkVQApN6bM/lg= +github.com/ugorji/go v0.0.0-20171019201919-bdcc60b419d1/go.mod h1:hnLbHMwcvSihnDhEfx2/BzKp2xb0Y+ErdfYcrs9tkJQ= github.com/ugorji/go v1.1.1/go.mod h1:hnLbHMwcvSihnDhEfx2/BzKp2xb0Y+ErdfYcrs9tkJQ= +github.com/ugorji/go/codec v0.0.0-20181127175209-856da096dbdf h1:BLcwkDfQ8QPXNXBApZUATvuigovcYPXkHzez80QFGNg= +github.com/ugorji/go/codec v0.0.0-20181127175209-856da096dbdf/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= +github.com/unrolled/render v0.0.0-20171102162132-65450fb6b2d3/go.mod h1:tu82oB5W2ykJRVioYsB+IQKcft7ryBr7w12qMBUPyXg= github.com/unrolled/render v0.0.0-20180914162206-b9786414de4d/go.mod h1:tu82oB5W2ykJRVioYsB+IQKcft7ryBr7w12qMBUPyXg= -github.com/xiang90/probing v0.0.0-20160813154853-07dd2e8dfe18 h1:MPPkRncZLN9Kh4MEFmbnK4h3BD7AUmskWv2+EeZJCCs= github.com/xiang90/probing v0.0.0-20160813154853-07dd2e8dfe18/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= -go.uber.org/atomic v1.3.2 h1:2Oa65PReHzfn29GpvgsYwloV9AVFHPDk8tYxt2c2tr4= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= -go.uber.org/multierr v1.1.0 h1:HoEmRHQPVSqub6w2z2d2EOVs2fjyFRGyofhKuyDq0QI= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/zap v1.8.0 h1:r6Za1Rii8+EGOYRDLvpooNOF6kP3iyDnkpzbw67gCQ8= go.uber.org/zap v1.8.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +golang.org/x/crypto v0.0.0-20180503215945-1f94bef427e3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20181030102418-4d3f4d9ffa16 h1:y6ce7gCWtnH+m3dCjzQ1PCuwl28DDIc3VNnvY29DlIA= golang.org/x/crypto v0.0.0-20181030102418-4d3f4d9ffa16/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181029044818-c44066c5c816 h1:mVFkLpejdFLXVUv9E42f3XJVfMdqd0IVLVIVLjZWn5o= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181005035420-146acd28ed58/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181029044818-c44066c5c816/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181106065722-10aee1819953/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181030150119-7e31e0c00fa0 h1:biUuj9O+0+XckRUCDzjoOGm6yFV5c0IHbm1ODP3e4Zw= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181030150119-7e31e0c00fa0/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= +golang.org/x/sys v0.0.0-20190109145017-48ac38b7c8cb/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2 h1:+DCIGbF/swA92ohVg0//6X2IVY3KZs6p9mix0ziNYJM= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20181108054448-85acf8d2951c h1:fqgJT0MGcGpPgpWU7VRdRjuArfcOvC4AoJmILihzhDg= +golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181105230042-78dc5bac0cac/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190130214255-bb1329dc71a0/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/genproto v0.0.0-20180427144745-86e600f69ee4/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= -google.golang.org/genproto v0.0.0-20181029155118-b69ba1387ce2 h1:67iHsV9djwGdZpdZNbLuQj6FOzCaZe3w+vhLjn5AcFA= +google.golang.org/genproto v0.0.0-20181004005441-af9cb2a35e7f/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20181029155118-b69ba1387ce2/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= -google.golang.org/grpc v1.16.0 h1:dz5IJGuC2BB7qXR5AyHNwAUBhZscK2xVez7mznh72sY= +google.golang.org/genproto v0.0.0-20190108161440-ae2f86662275/go.mod h1:7Ep/1NZk928CDR8SjdVbjWNpdIf6nzjE3BTgJDr2Atg= +google.golang.org/grpc v0.0.0-20180607172857-7a6a684ca69e/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= +google.golang.org/grpc v1.12.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.16.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9MZMio= +google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= +gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA= -gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8= +gopkg.in/natefinch/lumberjack.v2 v2.0.0-20161104145732-dd45e6a67c53/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= -gopkg.in/yaml.v2 v2.2.1 h1:mUhvW9EsL+naU5Q3cakzfE91YhliOondGd6ZrsDBHQE= +gopkg.in/stretchr/testify.v1 v1.2.2/go.mod h1:QI5V/q6UbPmuhtm10CaFZxED9NreB8PnFYN9JcR6TxU= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +sourcegraph.com/sourcegraph/appdash v0.0.0-20180531100431-4c381bd170b4/go.mod h1:hI742Nqp5OhwiqlzhgfbWU4mW4yO10fP+LoT9WOswdU= +sourcegraph.com/sourcegraph/appdash-data v0.0.0-20151005221446-73f23eafcf67/go.mod h1:L5q+DGLGOQFpo1snNEkLOJT2d1YTW66rWNzatr3He1k= diff --git a/importer/main.go b/importer/main.go index d29a654b3..52c52858f 100644 --- a/importer/main.go +++ b/importer/main.go @@ -18,8 +18,9 @@ import ( "os" "github.com/pingcap/errors" + "github.com/pingcap/log" "github.com/pingcap/tidb-tools/pkg/importer" - log "github.com/sirupsen/logrus" + "go.uber.org/zap" ) func main() { @@ -30,7 +31,7 @@ func main() { case flag.ErrHelp: os.Exit(0) default: - log.Errorf("parse cmd flags err %s\n", err) + log.Error("parse cmd flags failed", zap.Error(err)) os.Exit(2) } diff --git a/pkg/dbutil/common.go b/pkg/dbutil/common.go index 3f38898f6..a87e30619 100644 --- a/pkg/dbutil/common.go +++ b/pkg/dbutil/common.go @@ -24,6 +24,7 @@ import ( "github.com/go-sql-driver/mysql" "github.com/pingcap/errors" + "github.com/pingcap/log" "github.com/pingcap/parser/model" tmysql "github.com/pingcap/parser/mysql" "github.com/pingcap/parser/terror" @@ -32,7 +33,6 @@ import ( "github.com/pingcap/tidb/infoschema" "github.com/pingcap/tidb/types" gmysql "github.com/siddontang/go-mysql/mysql" - log "github.com/sirupsen/logrus" "go.uber.org/zap" ) @@ -173,7 +173,7 @@ func GetRowCount(ctx context.Context, db *sql.DB, schemaName string, tableName s if len(where) > 0 { query += fmt.Sprintf(" WHERE %s", where) } - log.Debugf("get row count by sql %s", query) + log.Debug("get row count", zap.String("sql", query)) var cnt sql.NullInt64 err := db.QueryRowContext(ctx, query).Scan(&cnt) @@ -216,7 +216,7 @@ func GetRandomValues(ctx context.Context, db *sql.DB, schemaName, table, column query := fmt.Sprintf("SELECT %[1]s, COUNT(*) count FROM (SELECT %[1]s FROM %[2]s WHERE %[3]s ORDER BY RAND() LIMIT %[4]d)rand_tmp GROUP BY %[1]s ORDER BY %[1]s%[5]s", escapeName(column), TableName(schemaName, table), limitRange, num, collation) - log.Debugf("get random values sql: %s, args: %v", query, limitArgs) + log.Debug("get random values", zap.String("sql", query), zap.Reflect("args", limitArgs)) rows, err := db.QueryContext(ctx, query, limitArgs...) if err != nil { @@ -260,7 +260,7 @@ func GetMinMaxValue(ctx context.Context, db *sql.DB, schema, table, column strin query := fmt.Sprintf("SELECT /*!40001 SQL_NO_CACHE */ MIN(`%s`%s) as MIN, MAX(`%s`%s) as MAX FROM `%s`.`%s` WHERE %s", column, collation, column, collation, schema, table, limitRange) - log.Debugf("GetMinMaxValue query: %v, args: %v", query, limitArgs) + log.Debug("GetMinMaxValue", zap.String("sql", query), zap.Reflect("args", limitArgs)) var min, max sql.NullString rows, err := db.QueryContext(ctx, query, limitArgs...) @@ -380,7 +380,7 @@ func GetCRC32Checksum(ctx context.Context, db *sql.DB, schemaName, tableName str query := fmt.Sprintf("SELECT BIT_XOR(CAST(CRC32(CONCAT_WS(',', %s, CONCAT(%s)))AS UNSIGNED)) AS checksum FROM %s WHERE %s;", strings.Join(columnNames, ", "), strings.Join(columnIsNull, ", "), TableName(schemaName, tableName), limitRange) - log.Debugf("checksum sql: %s, args: %v", query, args) + log.Debug("checksum", zap.String("sql", query), zap.Reflect("args", args)) var checksum sql.NullInt64 err := db.QueryRowContext(ctx, query, args...).Scan(&checksum) @@ -389,7 +389,7 @@ func GetCRC32Checksum(ctx context.Context, db *sql.DB, schemaName, tableName str } if !checksum.Valid { // if don't have any data, the checksum will be `NULL` - log.Warnf("get empty checksum by query %s, args %v", query, args) + log.Warn("get empty checksum", zap.String("sql", query), zap.Reflect("args", args)) return 0, nil } @@ -417,7 +417,7 @@ func GetBucketsInfo(ctx context.Context, db *sql.DB, schema, table string, table */ buckets := make(map[string][]Bucket) query := "SHOW STATS_BUCKETS WHERE db_name= ? AND table_name= ?;" - log.Debugf("GetBucketsInfo query: %s", query) + log.Debug("GetBucketsInfo", zap.String("sql", query)) rows, err := db.QueryContext(ctx, query, schema, table) if err != nil { @@ -555,7 +555,7 @@ func GetTidbLatestTSO(ctx context.Context, db *sql.DB) (int64, error) { // SetSnapshot set the snapshot variable for tidb func SetSnapshot(ctx context.Context, db *sql.DB, snapshot string) error { sql := fmt.Sprintf("SET @@tidb_snapshot='%s'", snapshot) - log.Infof("set history snapshot: %s", sql) + log.Info("set history snapshot", zap.String("sql", sql)) _, err := db.ExecContext(ctx, sql) return errors.Trace(err) } @@ -606,7 +606,7 @@ func GetDBVersion(ctx context.Context, db *sql.DB) (string, error) { func IsTiDB(ctx context.Context, db *sql.DB) (bool, error) { version, err := GetDBVersion(ctx, db) if err != nil { - log.Errorf("get database's version meets error %v", err) + log.Error("get database's version failed", zap.Error(err)) return false, errors.Trace(err) } diff --git a/pkg/ddl-checker/executable_checker.go b/pkg/ddl-checker/executable_checker.go index d6a66f129..784795fcd 100644 --- a/pkg/ddl-checker/executable_checker.go +++ b/pkg/ddl-checker/executable_checker.go @@ -18,6 +18,7 @@ import ( "fmt" "github.com/pingcap/errors" + "github.com/pingcap/log" "github.com/pingcap/parser" "github.com/pingcap/parser/ast" "github.com/pingcap/tidb/session" @@ -35,8 +36,10 @@ type ExecutableChecker struct { // NewExecutableChecker creates a new ExecutableChecker func NewExecutableChecker() (*ExecutableChecker, error) { - logutil.InitLogger(&logutil.LogConfig{ - Level: "error", + logutil.InitZapLogger(&logutil.LogConfig{ + Config: log.Config{ + Level: "error", + }, }) mocktikv, err := mockstore.NewMockTikvStore() if err != nil { diff --git a/pkg/diff/chunk.go b/pkg/diff/chunk.go index 652696a04..2c4dbc1a8 100644 --- a/pkg/diff/chunk.go +++ b/pkg/diff/chunk.go @@ -22,10 +22,10 @@ import ( "time" "github.com/pingcap/errors" + "github.com/pingcap/log" "github.com/pingcap/parser/model" "github.com/pingcap/tidb-tools/pkg/dbutil" "github.com/pingcap/tidb-tools/pkg/utils" - log "github.com/sirupsen/logrus" "go.uber.org/zap" ) @@ -263,7 +263,7 @@ func (s *randomSpliter) splitRange(db *sql.DB, chunk *ChunkRange, count int, sch symbolMax = chunk.Bounds[colNum-1].UpperSymbol } else { if len(columns) <= colNum { - log.Warnf("chunk %v can't be splited", chunk) + log.Warn("chunk can't be splited", zap.Stringer("chunk", chunk)) return append(chunks, chunk), nil } @@ -274,7 +274,7 @@ func (s *randomSpliter) splitRange(db *sql.DB, chunk *ChunkRange, count int, sch min, max, err = dbutil.GetMinMaxValue(context.Background(), db, schema, table, splitCol, limitRange, utils.StringsToInterfaces(args), s.collation) if err != nil { if errors.Cause(err) == dbutil.ErrNoData { - log.Infof("no data found in %s.%s range %s, args %v", schema, table, limitRange, args) + log.Info("no data found", zap.String("table", dbutil.TableName(schema, table)), zap.String("range", limitRange), zap.Reflect("args", args)) return append(chunks, chunk), nil } return nil, errors.Trace(err) @@ -292,7 +292,7 @@ func (s *randomSpliter) splitRange(db *sql.DB, chunk *ChunkRange, count int, sch if err != nil { return nil, errors.Trace(err) } - log.Debugf("split chunk %v, get split values from GetRandomValues: %v", chunk, randomValues) + log.Debug("get split values by random values", zap.Stringer("chunk", chunk), zap.Reflect("random values", randomValues)) /* for examples: @@ -384,8 +384,7 @@ func (s *randomSpliter) splitRange(db *sql.DB, chunk *ChunkRange, count int, sch lowerSymbol = gte } - log.Debugf("getChunksForTable cut table: cnt=%d min=%s max=%s chunk=%d", count, min, max, len(chunks)) - + log.Debug("getChunksForTable cut table", zap.Int("count", count), zap.String("min", min), zap.String("max", max), zap.Int("chunk num", len(chunks))) return chunks, nil } @@ -479,7 +478,7 @@ func getChunksForTable(table *TableInstance, columns []*model.ColumnInfo, chunkS return chunks, nil } - log.Warnf("use tidb bucket information to get chunks error: %v, chunks num: %d, will split chunk by random again", errors.Trace(err), len(chunks)) + log.Warn("use tidb bucket information to get chunks failed, will split chunk by random again", zap.Int("get chunk", len(chunks)), zap.Error(err)) } // get chunks from tidb bucket information failed, use random. diff --git a/pkg/diff/diff.go b/pkg/diff/diff.go index c7165ca00..6ba30e5fd 100644 --- a/pkg/diff/diff.go +++ b/pkg/diff/diff.go @@ -27,10 +27,10 @@ import ( "time" "github.com/pingcap/errors" + "github.com/pingcap/log" "github.com/pingcap/parser/model" "github.com/pingcap/tidb-tools/pkg/dbutil" "github.com/pingcap/tidb-tools/pkg/utils" - log "github.com/sirupsen/logrus" "go.uber.org/zap" ) @@ -514,7 +514,7 @@ func (t *TableDiff) compareRows(ctx context.Context, chunk *ChunkRange) (bool, e // all the rowsData2's data should be deleted for ; index2 < len(rowsData2); index2++ { sql := generateDML("delete", rowsData2[index2], orderKeyCols, t.TargetTable.info, t.TargetTable.Schema) - log.Infof("[delete] sql: %v", sql) + log.Info("[delete]", zap.String("sql", sql)) t.wg.Add(1) t.sqlCh <- sql equal = false @@ -525,7 +525,7 @@ func (t *TableDiff) compareRows(ctx context.Context, chunk *ChunkRange) (bool, e // rowsData2 lack some data, should insert them for ; index1 < len(rowsData1); index1++ { sql := generateDML("replace", rowsData1[index1], orderKeyCols, t.TargetTable.info, t.TargetTable.Schema) - log.Infof("[insert] sql: %v", sql) + log.Info("[insert]", zap.String("sql", sql)) t.wg.Add(1) t.sqlCh <- sql equal = false @@ -546,21 +546,21 @@ func (t *TableDiff) compareRows(ctx context.Context, chunk *ChunkRange) (bool, e case 1: // delete sql := generateDML("delete", rowsData2[index2], orderKeyCols, t.TargetTable.info, t.TargetTable.Schema) - log.Infof("[delete] sql: %s", sql) + log.Info("[delete]", zap.String("sql", sql)) t.wg.Add(1) t.sqlCh <- sql index2++ case -1: // insert sql := generateDML("replace", rowsData1[index1], orderKeyCols, t.TargetTable.info, t.TargetTable.Schema) - log.Infof("[insert] sql: %s", sql) + log.Info("[insert]", zap.String("sql", sql)) t.wg.Add(1) t.sqlCh <- sql index1++ case 0: // update sql := generateDML("replace", rowsData1[index1], orderKeyCols, t.TargetTable.info, t.TargetTable.Schema) - log.Infof("[update] sql: %s", sql) + log.Info("[update]", zap.String("sql", sql)) t.wg.Add(1) t.sqlCh <- sql index1++ @@ -682,7 +682,7 @@ func generateDML(tp string, data map[string]*dbutil.ColumnData, keys []*model.Co } sql = fmt.Sprintf("DELETE FROM `%s`.`%s` WHERE %s;", schema, table.Name, strings.Join(kvs, " AND ")) default: - log.Errorf("unknown sql type %s", tp) + log.Error("unknown sql type", zap.String("type", tp)) } return @@ -706,9 +706,9 @@ func compareData(map1, map2 map[string]*dbutil.ColumnData, orderKeyCols []*model } equal = false if data1.IsNull == data2.IsNull { - log.Errorf("find difference data in column %s, data1: %s, data2: %s", key, data1.Data, data2.Data) + log.Error("find difference data", zap.String("column", key), zap.Reflect("data1", map1), zap.Reflect("data2", map2)) } else { - log.Errorf("find difference data in column %s, one of them is NULL, data1: %s, data2: %s", key, data1.Data, data2.Data) + log.Error("find difference data, one of them is NULL", zap.String("column", key), zap.Reflect("data1", map1), zap.Reflect("data2", map2)) } break } @@ -788,7 +788,7 @@ func getChunkRows(ctx context.Context, db *sql.DB, schema, table string, tableIn query := fmt.Sprintf("SELECT /*!40001 SQL_NO_CACHE */ %s FROM `%s`.`%s` WHERE %s ORDER BY %s%s", columns, schema, table, where, strings.Join(orderKeys, ","), collation) - log.Debugf("select data by sql %s, args: %v", query, args) + log.Debug("select data", zap.String("sql", query), zap.Reflect("args", args)) rows, err := db.QueryContext(ctx, query, args...) if err != nil { return nil, nil, errors.Trace(err) diff --git a/pkg/diff/merge.go b/pkg/diff/merge.go index 1e6b34557..ee38e0a0f 100644 --- a/pkg/diff/merge.go +++ b/pkg/diff/merge.go @@ -16,9 +16,10 @@ package diff import ( "strconv" + "github.com/pingcap/log" "github.com/pingcap/parser/model" "github.com/pingcap/tidb-tools/pkg/dbutil" - log "github.com/sirupsen/logrus" + "go.uber.org/zap" ) // RowData is the struct of rows selected from mysql/tidb @@ -60,9 +61,12 @@ func (r RowDatas) Less(i, j int) bool { return true } num1, err1 := strconv.ParseFloat(string(data1), 64) + if err1 != nil { + log.Fatal("convert string to float failed", zap.ByteString("data", data1), zap.Error(err1)) + } num2, err2 := strconv.ParseFloat(string(data2), 64) - if err1 != nil || err2 != nil { - log.Fatalf("convert %s, %s to float failed, err1: %v, err2: %v", string(data1), string(data2), err1, err2) + if err2 != nil { + log.Fatal("convert string to float failed", zap.ByteString("data", data2), zap.Error(err2)) } if num1 == num2 { diff --git a/pkg/importer/db.go b/pkg/importer/db.go index fed382f48..caa5ba1e3 100644 --- a/pkg/importer/db.go +++ b/pkg/importer/db.go @@ -21,9 +21,10 @@ import ( _ "github.com/go-sql-driver/mysql" // for mysql driver "github.com/pingcap/errors" + "github.com/pingcap/log" "github.com/pingcap/parser/mysql" "github.com/pingcap/tidb-tools/pkg/dbutil" - log "github.com/sirupsen/logrus" + "go.uber.org/zap" ) func intRangeValue(column *column, min int64, max int64) (int64, int64) { @@ -31,13 +32,13 @@ func intRangeValue(column *column, min int64, max int64) (int64, int64) { if len(column.min) > 0 { min, err = strconv.ParseInt(column.min, 10, 64) if err != nil { - log.Fatal(err) + log.Fatal("intRangeValue", zap.Error(err)) } if len(column.max) > 0 { max, err = strconv.ParseInt(column.max, 10, 64) if err != nil { - log.Fatal(err) + log.Fatal("intRangeValue", zap.Error(err)) } } } @@ -257,7 +258,7 @@ func closeDBs(dbs []*sql.DB) { for _, db := range dbs { err := closeDB(db) if err != nil { - log.Errorf("close db failed - %v", err) + log.Error("close db failed", zap.Error(err)) } } } diff --git a/pkg/importer/importer.go b/pkg/importer/importer.go index 3ba96b542..4d0ed7305 100644 --- a/pkg/importer/importer.go +++ b/pkg/importer/importer.go @@ -14,7 +14,8 @@ package importer import ( - log "github.com/sirupsen/logrus" + "github.com/pingcap/log" + "go.uber.org/zap" ) // DoProcess generates data. @@ -22,28 +23,28 @@ func DoProcess(cfg *Config) { table := newTable() err := parseTableSQL(table, cfg.TableSQL) if err != nil { - log.Fatal(err) + log.Fatal("parseTableSQL", zap.Error(err)) } err = parseIndexSQL(table, cfg.IndexSQL) if err != nil { - log.Fatal(err) + log.Fatal("parseIndexSQL", zap.Error(err)) } dbs, err := createDBs(cfg.DBCfg, cfg.WorkerCount) if err != nil { - log.Fatal(err) + log.Fatal("createDBs", zap.Error(err)) } defer closeDBs(dbs) err = execSQL(dbs[0], cfg.TableSQL) if err != nil { - log.Fatal(err) + log.Fatal("execSQL", zap.Error(err)) } err = execSQL(dbs[0], cfg.IndexSQL) if err != nil { - log.Fatal(err) + log.Fatal("execSQL", zap.Error(err)) } doProcess(table, dbs, cfg.JobCount, cfg.WorkerCount, cfg.Batch) diff --git a/pkg/importer/job.go b/pkg/importer/job.go index 497419337..12f73376e 100644 --- a/pkg/importer/job.go +++ b/pkg/importer/job.go @@ -18,8 +18,8 @@ import ( "fmt" "time" - "github.com/pingcap/errors" - log "github.com/sirupsen/logrus" + "github.com/pingcap/log" + "go.uber.org/zap" ) func addJobs(jobCount int, jobChan chan struct{}) { @@ -33,24 +33,24 @@ func addJobs(jobCount int, jobChan chan struct{}) { func doInsert(table *table, db *sql.DB, count int) { sqls, err := genRowDatas(table, count) if err != nil { - log.Fatalf(errors.ErrorStack(err)) + log.Fatal("genRowDatas", zap.Error(err)) } txn, err := db.Begin() if err != nil { - log.Fatalf(errors.ErrorStack(err)) + log.Fatal("begin transcation", zap.Error(err)) } for _, sql := range sqls { _, err = txn.Exec(sql) if err != nil { - log.Fatalf(errors.ErrorStack(err)) + log.Fatal("exec", zap.String("sql", sql), zap.Error(err)) } } err = txn.Commit() if err != nil { - log.Fatalf(errors.ErrorStack(err)) + log.Fatal("commit transcation", zap.Error(err)) } } diff --git a/pkg/importer/parser.go b/pkg/importer/parser.go index e5c4d4261..3ed34b6cb 100644 --- a/pkg/importer/parser.go +++ b/pkg/importer/parser.go @@ -19,11 +19,12 @@ import ( "strings" "github.com/pingcap/errors" + "github.com/pingcap/log" "github.com/pingcap/parser" "github.com/pingcap/parser/ast" "github.com/pingcap/parser/types" _ "github.com/pingcap/tidb/types/parser_driver" // for parser driver - log "github.com/sirupsen/logrus" + "go.uber.org/zap" ) type column struct { @@ -68,7 +69,7 @@ func (col *column) parseRule(kvs []string) { var err error col.step, err = strconv.ParseInt(value, 10, 64) if err != nil { - log.Fatal(err) + log.Fatal("parseRule", zap.Error(err)) } } else if key == "set" { fields := strings.Split(value, ",") diff --git a/sync_diff_inspector/config.go b/sync_diff_inspector/config.go index 1dbc8719f..4ba28a38b 100644 --- a/sync_diff_inspector/config.go +++ b/sync_diff_inspector/config.go @@ -20,10 +20,11 @@ import ( "github.com/BurntSushi/toml" "github.com/pingcap/errors" + "github.com/pingcap/log" "github.com/pingcap/parser/model" "github.com/pingcap/tidb-tools/pkg/dbutil" router "github.com/pingcap/tidb-tools/pkg/table-router" - log "github.com/sirupsen/logrus" + "go.uber.org/zap" ) const ( @@ -137,7 +138,7 @@ func (t *TableInstance) Valid() bool { } if _, ok := sourceInstanceMap[t.InstanceID]; !ok { - log.Errorf("unknown database instance id %s", t.InstanceID) + log.Error("unknown database instance id", zap.String("instance id", t.InstanceID)) return false } @@ -280,12 +281,12 @@ func (c *Config) configFromFile(path string) error { func (c *Config) checkConfig() bool { if c.Sample > percent100 || c.Sample < percent0 { - log.Errorf("sample must be greater than 0 and less than or equal to 100!") + log.Error("sample must be greater than 0 and less than or equal to 100!") return false } if c.CheckThreadCount <= 0 { - log.Errorf("check-thcount must greater than 0!") + log.Error("check-thcount must greater than 0!") return false } @@ -304,7 +305,7 @@ func (c *Config) checkConfig() bool { c.TargetDBCfg.InstanceID = "target" } if _, ok := sourceInstanceMap[c.TargetDBCfg.InstanceID]; ok { - log.Errorf("target has same instance id %s in source", c.TargetDBCfg.InstanceID) + log.Error("target has same instance id in source", zap.String("instance id", c.TargetDBCfg.InstanceID)) return false } diff --git a/sync_diff_inspector/diff.go b/sync_diff_inspector/diff.go index 615e406da..42708ded5 100644 --- a/sync_diff_inspector/diff.go +++ b/sync_diff_inspector/diff.go @@ -20,11 +20,11 @@ import ( "regexp" "github.com/pingcap/errors" + "github.com/pingcap/log" "github.com/pingcap/tidb-tools/pkg/dbutil" "github.com/pingcap/tidb-tools/pkg/diff" router "github.com/pingcap/tidb-tools/pkg/table-router" "github.com/pingcap/tidb-tools/pkg/utils" - log "github.com/sirupsen/logrus" "go.uber.org/zap" ) @@ -203,13 +203,13 @@ func (df *Diff) AdjustTableConfig(cfg *Config) (err error) { } if _, ok := df.tables[schemaTables.Schema][tableName]; ok { - log.Errorf("duplicate config for %s.%s", schemaTables.Schema, tableName) + log.Error("duplicate config for one table", zap.String("table", dbutil.TableName(schemaTables.Schema, tableName))) continue } sourceTables := make([]TableInstance, 0, 1) if _, ok := sourceTablesMap[schemaTables.Schema][tableName]; ok { - log.Infof("find matched source tables %v for %s.%s", sourceTablesMap[schemaTables.Schema][tableName], schemaTables.Schema, tableName) + log.Info("find matched source tables", zap.Reflect("source tables", sourceTablesMap[schemaTables.Schema][tableName]), zap.String("target schema", schemaTables.Schema), zap.String("table", tableName)) sourceTables = sourceTablesMap[schemaTables.Schema][tableName] } else { // use same database name and table name diff --git a/sync_diff_inspector/main.go b/sync_diff_inspector/main.go index b9888281f..39c42fbac 100644 --- a/sync_diff_inspector/main.go +++ b/sync_diff_inspector/main.go @@ -16,13 +16,15 @@ package main import ( "context" "flag" + "fmt" "os" "time" _ "github.com/go-sql-driver/mysql" "github.com/pingcap/errors" + "github.com/pingcap/log" "github.com/pingcap/tidb-tools/pkg/utils" - log "github.com/sirupsen/logrus" + "go.uber.org/zap" ) func main() { @@ -33,22 +35,21 @@ func main() { case flag.ErrHelp: os.Exit(0) default: - log.Errorf("parse cmd flags err %s\n", errors.ErrorStack(err)) + log.Error("parse cmd flags", zap.Error(err)) os.Exit(2) } if cfg.PrintVersion { - log.Infof("version: \n%s", utils.GetRawInfo("sync_diff_inspector")) + fmt.Printf("version: \n%s", utils.GetRawInfo("sync_diff_inspector")) return } - logLevel, err := log.ParseLevel(cfg.LogLevel) - if err != nil { - log.Warnf("invalide log level %s", cfg.LogLevel) - log.SetLevel(log.InfoLevel) - } else { - log.SetLevel(logLevel) + l := zap.NewAtomicLevel() + if err := l.UnmarshalText([]byte(cfg.LogLevel)); err != nil { + log.Error("invalide log level", zap.String("log level", cfg.LogLevel)) + return } + log.SetLevel(l.Level()) ok := cfg.checkConfig() if !ok { @@ -67,20 +68,20 @@ func main() { func checkSyncState(ctx context.Context, cfg *Config) bool { beginTime := time.Now() defer func() { - log.Infof("check data finished, all cost %v", time.Since(beginTime)) + log.Info("check data finished", zap.Duration("cost", time.Since(beginTime))) }() d, err := NewDiff(ctx, cfg) if err != nil { - log.Fatalf("fail to initialize diff process %v", errors.ErrorStack(err)) + log.Fatal("fail to initialize diff process", zap.Error(err)) } err = d.Equal() if err != nil { - log.Fatalf("check data difference error %v", errors.ErrorStack(err)) + log.Fatal("check data difference failed", zap.Error(err)) } - log.Info(d.report) + log.Info("check report", zap.Stringer("report", d.report)) return d.report.Result == Pass } diff --git a/tidb-binlog/binlogctl/main.go b/tidb-binlog/binlogctl/main.go index a8e59fc06..3c849017c 100644 --- a/tidb-binlog/binlogctl/main.go +++ b/tidb-binlog/binlogctl/main.go @@ -17,9 +17,9 @@ import ( "flag" "os" - "github.com/pingcap/errors" + "github.com/pingcap/log" "github.com/pingcap/tidb-tools/tidb-binlog/node" - log "github.com/sirupsen/logrus" + "go.uber.org/zap" ) const ( @@ -35,7 +35,7 @@ func main() { case flag.ErrHelp: os.Exit(0) default: - log.Errorf("parse cmd flags err %v", err) + log.Error("parse cmd flags", zap.Error(err)) os.Exit(2) } @@ -61,6 +61,6 @@ func main() { } if err != nil { - log.Fatalf("fail to execute %s error %v", cfg.Command, errors.ErrorStack(err)) + log.Fatal("fail to execute command", zap.String("command", cfg.Command), zap.Error(err)) } } diff --git a/tidb-binlog/binlogctl/meta.go b/tidb-binlog/binlogctl/meta.go index 9bfff5493..fea557199 100644 --- a/tidb-binlog/binlogctl/meta.go +++ b/tidb-binlog/binlogctl/meta.go @@ -16,16 +16,18 @@ package main import ( "bytes" "context" + "fmt" "os" "path" "time" "github.com/BurntSushi/toml" "github.com/pingcap/errors" + "github.com/pingcap/log" pd "github.com/pingcap/pd/client" "github.com/pingcap/tidb-tools/pkg/utils" "github.com/siddontang/go/ioutil2" - log "github.com/sirupsen/logrus" + "go.uber.org/zap" ) const physicalShiftBits = 18 @@ -40,7 +42,7 @@ func generateMetaInfo(cfg *Config) error { // get newest ts from pd commitTS, err := GetTSO(cfg) if err != nil { - log.Errorf("get tso failed: %s", err) + log.Error("get tso failed", zap.Error(err)) return errors.Trace(err) } @@ -70,7 +72,7 @@ func GetTSO(cfg *Config) (int64, error) { } dist := time.Since(now) if dist > slowDist { - log.Warnf("get timestamp too slow: %s", dist) + log.Warn("get timestamp too slow", zap.Duration("dist", dist)) } return int64(composeTS(physical, logical)), nil @@ -86,6 +88,11 @@ type Meta struct { CommitTS int64 `toml:"commitTS" json:"commitTS"` } +// String returns the string of Meta +func (m *Meta) String() string { + return fmt.Sprintf("commitTS: %d", m.CommitTS) +} + // saveMeta saves current tso in meta file. func saveMeta(metaFileName string, ts int64, timeZone string) error { meta := &Meta{CommitTS: ts} @@ -101,7 +108,7 @@ func saveMeta(metaFileName string, ts int64, timeZone string) error { t := utils.TSOToRoughTime(ts) location, err1 := time.LoadLocation(timeZone) if err1 != nil { - log.Warningf("fail to load location %s, error %v", timeZone, err1) + log.Warn("fail to load location", zap.String("time zone", timeZone), zap.Error(err1)) } else { buf.WriteString(t.UTC().String()) buf.WriteByte('\n') @@ -114,6 +121,6 @@ func saveMeta(metaFileName string, ts int64, timeZone string) error { return errors.Annotatef(err, "save meta %+v into %s", meta, metaFileName) } - log.Infof("meta: %+v", meta) + log.Info("save meta", zap.Stringer("meta", meta)) return nil } diff --git a/tidb-binlog/binlogctl/nodes.go b/tidb-binlog/binlogctl/nodes.go index 747488f37..bfb4fb00b 100644 --- a/tidb-binlog/binlogctl/nodes.go +++ b/tidb-binlog/binlogctl/nodes.go @@ -20,10 +20,11 @@ import ( "time" "github.com/pingcap/errors" + "github.com/pingcap/log" "github.com/pingcap/tidb-tools/pkg/etcd" "github.com/pingcap/tidb-tools/pkg/utils" "github.com/pingcap/tidb-tools/tidb-binlog/node" - log "github.com/sirupsen/logrus" + "go.uber.org/zap" ) var ( @@ -43,7 +44,7 @@ func queryNodesByKind(urls string, kind string) error { } for _, n := range nodes { - log.Infof("%s: %s", kind, formatNodeInfo(n)) + log.Info("query node", zap.String("type", kind), zap.Stringer("node", n)) } return nil @@ -113,14 +114,14 @@ func applyAction(urls, kind, nodeID string, action string) error { client := &http.Client{} url := fmt.Sprintf("http://%s/state/%s/%s", n.Addr, n.NodeID, action) - log.Debugf("send put http request %s", url) + log.Debug("send put http request", zap.String("url", url)) req, err := http.NewRequest("PUT", url, nil) if err != nil { return errors.Trace(err) } _, err = client.Do(req) if err == nil { - log.Infof("apply action %s on node %s success", action, n.NodeID) + log.Info("apply action on node success", zap.String("action", action), zap.String("NodeID", n.NodeID)) return nil } @@ -129,8 +130,3 @@ func applyAction(urls, kind, nodeID string, action string) error { return errors.NotFoundf("nodeID %s", nodeID) } - -func formatNodeInfo(status *node.Status) string { - updateTime := utils.TSOToRoughTime(status.UpdateTS) - return fmt.Sprintf("{NodeID: %s, Addr: %s, State: %s, MaxCommitTS: %d, UpdateTime: %v}", status.NodeID, status.Addr, status.State, status.MaxCommitTS, updateTime) -} diff --git a/tidb-binlog/driver/example/mysql/mysql.go b/tidb-binlog/driver/example/mysql/mysql.go index b64c4973f..d7982e609 100644 --- a/tidb-binlog/driver/example/mysql/mysql.go +++ b/tidb-binlog/driver/example/mysql/mysql.go @@ -21,12 +21,13 @@ import ( "github.com/Shopify/sarama" _ "github.com/go-sql-driver/mysql" + "github.com/pingcap/log" "github.com/pingcap/parser" "github.com/pingcap/parser/ast" "github.com/pingcap/tidb-tools/tidb-binlog/driver/reader" pb "github.com/pingcap/tidb-tools/tidb-binlog/slave_binlog_proto/go-binlog" _ "github.com/pingcap/tidb/types/parser_driver" // for parser driver - log "github.com/sirupsen/logrus" + "go.uber.org/zap" ) // a simple example to sync data to mysql @@ -77,26 +78,26 @@ func main() { for { select { case msg := <-breader.Messages(): - log.Debug("recv: ", msg.Binlog.String()) + log.Debug("recv", zap.Stringer("message", msg.Binlog)) binlog := msg.Binlog sqls, args := toSQL(binlog) tx, err := db.Begin() if err != nil { - log.Fatal(err) + log.Fatal("begin transcation failed", zap.Error(err)) } for i := 0; i < len(sqls); i++ { - log.Debug("exec: args: ", sqls[i], args[i]) + log.Debug("exec sql", zap.String("sql", sqls[i]), zap.Reflect("args", args[i])) _, err = tx.Exec(sqls[i], args[i]...) if err != nil { tx.Rollback() - log.Fatal(err) + log.Fatal("exec sql failed", zap.Error(err)) } } err = tx.Commit() if err != nil { - log.Fatal(err) + log.Fatal("commit transcation failed", zap.Error(err)) } } } @@ -257,7 +258,7 @@ func toSQL(binlog *pb.Binlog) ([]string, [][]interface{}) { ddl := binlog.DdlData isCreateDatabase, err := isCreateDatabase(string(ddl.DdlQuery)) if err != nil { - log.Fatal(err) + log.Fatal("parse ddl failed", zap.Error(err)) } if !isCreateDatabase { sql := fmt.Sprintf("use %s", ddl.GetSchemaName()) @@ -276,7 +277,7 @@ func toSQL(binlog *pb.Binlog) ([]string, [][]interface{}) { } default: - log.Fatal("unknown type: ", binlog.GetType()) + log.Fatal("unknown type", zap.Stringer("type", binlog.GetType())) } return allSQL, allArgs diff --git a/tidb-binlog/driver/example/print/print.go b/tidb-binlog/driver/example/print/print.go index 0cd3004b7..3c30c457a 100644 --- a/tidb-binlog/driver/example/print/print.go +++ b/tidb-binlog/driver/example/print/print.go @@ -17,8 +17,9 @@ import ( "flag" "github.com/Shopify/sarama" + "github.com/pingcap/log" "github.com/pingcap/tidb-tools/tidb-binlog/driver/reader" - log "github.com/sirupsen/logrus" + "go.uber.org/zap" ) var ( @@ -47,7 +48,7 @@ func main() { for { select { case msg := <-breader.Messages(): - log.Info("recv: ", msg.Binlog.String()) + log.Info("recv", zap.Stringer("message", msg.Binlog)) } } } diff --git a/tidb-binlog/driver/reader/offset.go b/tidb-binlog/driver/reader/offset.go index 0e3d4a19d..96e753d06 100644 --- a/tidb-binlog/driver/reader/offset.go +++ b/tidb-binlog/driver/reader/offset.go @@ -16,8 +16,9 @@ package reader import ( "github.com/Shopify/sarama" "github.com/pingcap/errors" + "github.com/pingcap/log" pb "github.com/pingcap/tidb-tools/tidb-binlog/slave_binlog_proto/go-binlog" - log "github.com/sirupsen/logrus" + "go.uber.org/zap" ) // KafkaSeeker seeks offset in kafka topics by given condition @@ -57,7 +58,7 @@ func (ks *KafkaSeeker) Seek(topic string, ts int64, partitions []int32) (offsets if len(partitions) == 0 { partitions, err = ks.consumer.Partitions(topic) if err != nil { - log.Errorf("get partitions from topic %s error %v", topic, err) + log.Error("get partitions from topic failed", zap.String("topic", topic), zap.Error(err)) return nil, errors.Trace(err) } } @@ -65,7 +66,7 @@ func (ks *KafkaSeeker) Seek(topic string, ts int64, partitions []int32) (offsets offsets, err = ks.seekOffsets(topic, partitions, ts) if err != nil { err = errors.Trace(err) - log.Errorf("seek offsets error %v", err) + log.Error("seek offsets failed", zap.Error(err)) } return } @@ -116,7 +117,7 @@ func (ks *KafkaSeeker) seekOffset(topic string, partition int32, start int64, en } if ts < startTS { - log.Warnf("given ts %v is smaller than oldest message's ts %v, some binlogs may lose", ts, startTS) + log.Warn("given ts is smaller than oldest message's ts, some binlogs may lose", zap.Int64("given ts", ts), zap.Int64("oldest ts", startTS)) offset = start return } else if ts == startTS { diff --git a/tidb-binlog/driver/reader/reader.go b/tidb-binlog/driver/reader/reader.go index 3d7e5a389..726f06484 100644 --- a/tidb-binlog/driver/reader/reader.go +++ b/tidb-binlog/driver/reader/reader.go @@ -16,8 +16,9 @@ package reader import ( "github.com/Shopify/sarama" "github.com/pingcap/errors" + "github.com/pingcap/log" pb "github.com/pingcap/tidb-tools/tidb-binlog/slave_binlog_proto/go-binlog" - log "github.com/sirupsen/logrus" + "go.uber.org/zap" ) func init() { @@ -96,7 +97,7 @@ func NewReader(cfg *Config) (r *Reader, err error) { r = nil return } - log.Debug("set offset to: ", r.cfg.Offset) + log.Debug("set offset to", zap.Int64("offset", r.cfg.Offset)) } go r.run() @@ -137,17 +138,17 @@ func (r *Reader) getOffsetByTS(ts int64) (offset int64, err error) { func (r *Reader) run() { offset := r.cfg.Offset - log.Debug("start at offset: ", offset) + log.Debug("start at", zap.Int64("offset", offset)) consumer, err := sarama.NewConsumerFromClient(r.client) if err != nil { - log.Fatal(err) + log.Fatal("create kafka consumer failed", zap.Error(err)) } defer consumer.Close() topic, partition := r.getTopic() partitionConsumer, err := consumer.ConsumePartition(topic, partition, offset) if err != nil { - log.Fatal(err) + log.Fatal("create kafka partition consumer failed", zap.Error(err)) } defer partitionConsumer.Close() @@ -159,15 +160,15 @@ func (r *Reader) run() { log.Info("reader stop to run") return case kmsg := <-partitionConsumer.Messages(): - log.Debug("get kmsg offset: ", kmsg.Offset) + log.Debug("get kafka message", zap.Int64("offset", kmsg.Offset)) binlog := new(pb.Binlog) err := binlog.Unmarshal(kmsg.Value) if err != nil { - log.Warn(err) + log.Warn("unmarshal binlog failed", zap.Error(err)) continue } if r.cfg.CommitTS > 0 && binlog.CommitTs <= r.cfg.CommitTS { - log.Warn("skip binlog CommitTs: ", binlog.CommitTs) + log.Warn("skip binlog CommitTs", zap.Int64("commitTS", binlog.CommitTs)) continue } diff --git a/tidb-binlog/node/node.go b/tidb-binlog/node/node.go index 961a2a0fe..94464128c 100644 --- a/tidb-binlog/node/node.go +++ b/tidb-binlog/node/node.go @@ -1,5 +1,11 @@ package node +import ( + "fmt" + + "github.com/pingcap/tidb-tools/pkg/utils" +) + var ( // DefaultRootPath is the root path of the keys stored in etcd, the `v1` is the tidb-binlog's version. DefaultRootPath = "/tidb-binlog/v1" @@ -68,3 +74,8 @@ type Status struct { // UpdateTS is the last update ts of node's status. UpdateTS int64 `json:"updateTS"` } + +func (s *Status) String() string { + updateTime := utils.TSOToRoughTime(s.UpdateTS) + return fmt.Sprintf("{NodeID: %s, Addr: %s, State: %s, MaxCommitTS: %d, UpdateTime: %v}", s.NodeID, s.Addr, s.State, s.MaxCommitTS, updateTime) +} diff --git a/tidb-binlog/node/registry.go b/tidb-binlog/node/registry.go index aeb0828ab..302d6352e 100644 --- a/tidb-binlog/node/registry.go +++ b/tidb-binlog/node/registry.go @@ -9,8 +9,9 @@ import ( "github.com/coreos/etcd/clientv3" "github.com/pingcap/errors" + "github.com/pingcap/log" "github.com/pingcap/tidb-tools/pkg/etcd" - log "github.com/sirupsen/logrus" + "go.uber.org/zap" ) // EtcdRegistry wraps the reactions with etcd @@ -80,7 +81,7 @@ func (r *EtcdRegistry) UpdateNode(pctx context.Context, prefix string, status *S return errors.Trace(err) } else if !exists { // not found then create a new node - log.Infof("node %s dosen't exist, will create one", status.NodeID) + log.Info("node dosen't exist, will create one", zap.String("NodeID", status.NodeID)) return r.createNode(ctx, prefix, status) } else { // found it, update status infomation of the node @@ -162,12 +163,12 @@ func AnalyzeNodeID(key string) string { nodeIDOffset = 2 } } else { - log.Errorf("can't get nodeID or node type from key %s", key) + log.Error("can't get nodeID or node type", zap.String("key", key)) return "" } if len(paths) < nodeIDOffset+1 { - log.Errorf("can't get nodeID or node type from key %s", key) + log.Error("can't get nodeID or node type", zap.String("key", key)) return "" } diff --git a/tidb-binlog/pump_client/client.go b/tidb-binlog/pump_client/client.go index cfb3bd1d9..07015a401 100644 --- a/tidb-binlog/pump_client/client.go +++ b/tidb-binlog/pump_client/client.go @@ -24,13 +24,14 @@ import ( "github.com/coreos/etcd/mvcc/mvccpb" "github.com/pingcap/errors" + "github.com/pingcap/log" pd "github.com/pingcap/pd/client" "github.com/pingcap/tidb-tools/pkg/etcd" "github.com/pingcap/tidb-tools/pkg/utils" "github.com/pingcap/tidb-tools/tidb-binlog/node" "github.com/pingcap/tidb/util/logutil" pb "github.com/pingcap/tipb/go-binlog" - log "github.com/sirupsen/logrus" + "go.uber.org/zap" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" ) @@ -50,9 +51,6 @@ const ( ) var ( - // Logger is ..., obsolete now. - Logger = log.New() - // ErrNoAvaliablePump means no avaliable pump to write binlog. ErrNoAvaliablePump = errors.New("no avaliable pump to write binlog") @@ -233,7 +231,7 @@ func (c *PumpsClient) getPumpStatus(pctx context.Context) (revision int64, err e } for _, status := range nodesStatus { - log.Debugf("[pumps client] get pump %v from pd", status) + log.Debug("[pumps client] get pump from pd", zap.Stringer("pump", status)) c.addPump(NewPumpStatus(status, c.Security), false) } @@ -287,7 +285,7 @@ func (c *PumpsClient) WriteBinlog(binlog *pb.Binlog) error { } meetError = true - log.Warnf("[pumps client] write binlog to pump %s (type: %s, start ts: %d, commit ts: %d, length: %d) error %v", pump.NodeID, binlog.Tp, binlog.StartTs, binlog.CommitTs, len(commitData), err) + log.Warn("[pumps client] write binlog to pump failed", zap.String("NodeID", pump.NodeID), zap.Stringer("binlog type", binlog.Tp), zap.Int64("start ts", binlog.StartTs), zap.Int64("commit ts", binlog.CommitTs), zap.Int("length", len(commitData)), zap.Error(err)) if binlog.Tp != pb.BinlogType_Prewrite { // only use one pump to write commit/rollback binlog, util write success or blocked for ten minutes. And will not return error to tidb. @@ -358,7 +356,7 @@ func (c *PumpsClient) backoffWriteBinlog(req *pb.WriteBinlogReq, binlogType pb.B err = errors.New(resp.Errmsg) } else { // if this pump can write binlog success, set this pump to avaliable. - log.Debugf("[pumps client] write binlog to unavaliable pump %s success, set this pump to avaliable", pump.NodeID) + log.Debug("[pumps client] write binlog to unavaliable pump success, set this pump to avaliable", zap.String("NodeID", pump.NodeID)) c.setPumpAvaliable(pump, true) return pump, nil } @@ -489,7 +487,7 @@ func (c *PumpsClient) watchStatus(revision int64) { case wresp := <-rch: if wresp.Err() != nil { // meet error, watch from the latest revision. - log.Warnf("[pumps client] watch status meet error %v", wresp.Err()) + log.Warn("[pumps client] watch status meet error", zap.Error(wresp.Err())) rch = c.EtcdRegistry.WatchNode(c.ctx, c.nodePath, revision) continue } @@ -500,28 +498,28 @@ func (c *PumpsClient) watchStatus(revision int64) { status := &node.Status{} err := json.Unmarshal(ev.Kv.Value, &status) if err != nil { - log.Errorf("[pumps client] unmarshal pump status %q failed", ev.Kv.Value) + log.Error("[pumps client] unmarshal pump status failed", zap.ByteString("value", ev.Kv.Value), zap.Error(err)) continue } switch ev.Type { case mvccpb.PUT: if !c.exist(status.NodeID) { - log.Infof("[pumps client] find a new pump %s", status.NodeID) + log.Info("[pumps client] find a new pump", zap.String("NodeID", status.NodeID)) c.addPump(NewPumpStatus(status, c.Security), true) continue } pump, avaliableChanged, avaliable := c.updatePump(status) if avaliableChanged { - log.Infof("[pumps client] pump %s's state is changed to %s", pump.Status.NodeID, status.State) + log.Info("[pumps client] pump's state is changed", zap.String("NodeID", pump.Status.NodeID), zap.String("state", status.State)) c.setPumpAvaliable(pump, avaliable) } case mvccpb.DELETE: // now will not delete pump node in fact, just for compatibility. nodeID := node.AnalyzeNodeID(string(ev.Kv.Key)) - log.Infof("[pumps client] remove pump %s", nodeID) + log.Info("[pumps client] remove pump", zap.String("NodeID", nodeID)) c.removePump(nodeID) } } @@ -540,7 +538,7 @@ func (c *PumpsClient) detect() { for { select { case <-c.ctx.Done(): - log.Infof("[pumps client] heartbeat finished") + log.Info("[pumps client] heartbeat finished") return case <-checkTick.C: // send detect binlog to pump, if this pump can return response without error @@ -559,10 +557,10 @@ func (c *PumpsClient) detect() { for _, pump := range needCheckPumps { _, err := pump.WriteBinlog(req, c.BinlogWriteTimeout) if err == nil { - log.Debugf("[pumps client] write detect binlog to unavaliable pump %s success", pump.NodeID) + log.Debug("[pumps client] write detect binlog to unavaliable pump success", zap.String("NodeID", pump.NodeID)) checkPassPumps = append(checkPassPumps, pump) } else { - log.Debugf("[pumps client] write detect binlog to pump %s error %v", pump.NodeID, err) + log.Debug("[pumps client] write detect binlog to pump failed", zap.String("NodeID", pump.NodeID), zap.Error(err)) } } @@ -575,10 +573,10 @@ func (c *PumpsClient) detect() { // Close closes the PumpsClient. func (c *PumpsClient) Close() { - log.Infof("[pumps client] is closing") + log.Info("[pumps client] is closing") c.cancel() c.wg.Wait() - log.Infof("[pumps client] is closed") + log.Info("[pumps client] is closed") } func isRetryableError(err error) bool { @@ -616,5 +614,5 @@ func copyPumps(pumps map[string]*PumpStatus) []*PumpStatus { // InitLogger initializes logger. func InitLogger(cfg *logutil.LogConfig) error { - return logutil.InitLogger(cfg) + return logutil.InitZapLogger(cfg) } diff --git a/tidb-binlog/pump_client/pump.go b/tidb-binlog/pump_client/pump.go index 960cf91cb..d0820680a 100644 --- a/tidb-binlog/pump_client/pump.go +++ b/tidb-binlog/pump_client/pump.go @@ -22,9 +22,10 @@ import ( "time" "github.com/pingcap/errors" + "github.com/pingcap/log" "github.com/pingcap/tidb-tools/tidb-binlog/node" pb "github.com/pingcap/tipb/go-binlog" - log "github.com/sirupsen/logrus" + "go.uber.org/zap" "google.golang.org/grpc" "google.golang.org/grpc/credentials" ) @@ -95,7 +96,7 @@ func (p *PumpStatus) createGrpcClient() error { return net.DialTimeout("tcp", addr, timeout) }) } - log.Debugf("[pumps client] create grpc client at %s", p.Addr) + log.Debug("[pumps client] create grpc client", zap.String("address", p.Addr)) var clientConn *grpc.ClientConn var err error if p.security != nil { diff --git a/tidb-binlog/pump_client/selector.go b/tidb-binlog/pump_client/selector.go index 1639e5792..a7ea56c64 100644 --- a/tidb-binlog/pump_client/selector.go +++ b/tidb-binlog/pump_client/selector.go @@ -18,8 +18,9 @@ import ( "strconv" "sync" + "github.com/pingcap/log" pb "github.com/pingcap/tipb/go-binlog" - log "github.com/sirupsen/logrus" + "go.uber.org/zap" ) const ( @@ -97,7 +98,7 @@ func (h *HashSelector) Select(binlog *pb.Binlog, retryTime int) *PumpStatus { } // this should never happened - log.Warnf("[pumps client] %s binlog with start ts %d don't have matched prewrite binlog", binlog.Tp, binlog.StartTs) + log.Warn("[pumps client] binlog don't have matched prewrite binlog", zap.Stringer("binlog type", binlog.Tp), zap.Int64("startTs", binlog.StartTs)) return nil } @@ -166,7 +167,7 @@ func (r *RangeSelector) Select(binlog *pb.Binlog, retryTime int) *PumpStatus { } // this should never happened - log.Warnf("[pumps client] %s binlog with start ts %d don't have matched prewrite binlog", binlog.Tp, binlog.StartTs) + log.Warn("[pumps client] binlog don't have matched prewrite binlog", zap.Stringer("binlog type", binlog.Tp), zap.Int64("startTs", binlog.StartTs)) return nil } @@ -254,10 +255,10 @@ func (s *ScoreSelector) Feedback(startTS int64, binlogType pb.BinlogType, pump * // TODO } -// NewSelector returns a PumpSelector according to the algorithm. -func NewSelector(algorithm string) PumpSelector { +// NewSelector returns a PumpSelector according to the strategy. +func NewSelector(strategy string) PumpSelector { var selector PumpSelector - switch algorithm { + switch strategy { case Range: selector = NewRangeSelector() case Hash: @@ -267,7 +268,7 @@ func NewSelector(algorithm string) PumpSelector { case LocalUnix: selector = NewLocalUnixSelector() default: - log.Warnf("[pumps client] unknown algorithm %s, use range as default", algorithm) + log.Warn("[pumps client] unknown strategy, use range as default", zap.String("strategy", strategy)) selector = NewRangeSelector() } From 9283c1423683dab21093f6db65e5faaaccec5e1c Mon Sep 17 00:00:00 2001 From: WangXiangUSTC Date: Wed, 10 Apr 2019 16:58:43 +0800 Subject: [PATCH 4/5] update go.mod and go.sum --- go.mod1 | 1 - go.sum1 | 3 --- 2 files changed, 4 deletions(-) diff --git a/go.mod1 b/go.mod1 index 46ebd6639..218c83724 100644 --- a/go.mod1 +++ b/go.mod1 @@ -35,7 +35,6 @@ require ( github.com/remyoudompheng/bigfft v0.0.0-20170806203942-52369c62f446 // indirect github.com/siddontang/go v0.0.0-20180604090527-bdc77568d726 github.com/siddontang/go-mysql v0.0.0-20190312052122-c6ab05a85eb8 - github.com/sirupsen/logrus v0.0.0-20170323161349-3bcb09397d6d github.com/soheilhy/cmux v0.1.4 // indirect github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72 // indirect github.com/tmc/grpc-websocket-proxy v0.0.0-20171017195756-830351dc03c6 // indirect diff --git a/go.sum1 b/go.sum1 index a56e10259..6277286ff 100644 --- a/go.sum1 +++ b/go.sum1 @@ -161,9 +161,6 @@ github.com/siddontang/go-log v0.0.0-20180807004314-8d05993dda07/go.mod h1:yFdBgw github.com/siddontang/go-mysql v0.0.0-20190312052122-c6ab05a85eb8 h1:8puKTg/UOIQ+ZiowY1ywmGsI08sWqrKD7HJ/j165CUM= github.com/siddontang/go-mysql v0.0.0-20190312052122-c6ab05a85eb8/go.mod h1:/b8ZcWjAShCcHp2dWpjb1vTlNyiG03UeHEQr2jteOpI= github.com/sirupsen/logrus v0.0.0-20170323161349-3bcb09397d6d/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= -github.com/sirupsen/logrus v1.1.1 h1:VzGj7lhU7KEB9e9gMpAV/v5XT2NVSvLJhJLCWbnkgXg= -github.com/sirupsen/logrus v1.1.1/go.mod h1:zrgwTnHtNr00buQ1vSptGe8m1f/BbgsPukg8qsT7A+A= -github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/spaolacci/murmur3 v0.0.0-20150829172844-0d12bf811670/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= From b86fa8c038970d0acf69392ba2fbb6619f9ac2e4 Mon Sep 17 00:00:00 2001 From: WangXiangUSTC Date: Thu, 25 Apr 2019 11:30:34 +0800 Subject: [PATCH 5/5] remove useless log pkg --- go.mod1 | 1 - 1 file changed, 1 deletion(-) diff --git a/go.mod1 b/go.mod1 index 218c83724..7e39db664 100644 --- a/go.mod1 +++ b/go.mod1 @@ -18,7 +18,6 @@ require ( github.com/konsorten/go-windows-terminal-sequences v1.0.1 // indirect github.com/kr/pretty v0.1.0 // indirect github.com/montanaflynn/stats v0.0.0-20180911141734-db72e6cae808 // indirect - github.com/ngaut/log v0.0.0-20180314031856-b8e36e7ba5ac github.com/pingcap/check v0.0.0-20190102082844-67f458068fc8 github.com/pingcap/errors v0.11.1 github.com/pingcap/goleveldb v0.0.0-20171020122428-b9ff6c35079e // indirect