From 162cb0effcf5dad670fa90a2bf32694592908908 Mon Sep 17 00:00:00 2001 From: Kevin Nisbet Date: Sun, 26 Apr 2020 22:10:22 +0000 Subject: [PATCH] bump etcd to latest 3.4.x release --- Gopkg.lock | 6 +- Gopkg.toml | 2 +- Makefile | 6 +- build.go | 2 +- vendor/go.etcd.io/etcd/clientv3/client.go | 8 --- vendor/go.etcd.io/etcd/clientv3/ctx.go | 64 +++++++++++++++++++ .../etcd/clientv3/retry_interceptor.go | 2 + .../etcd/etcdserver/api/v3rpc/rpctypes/md.go | 2 + vendor/go.etcd.io/etcd/lease/lessor.go | 9 ++- vendor/go.etcd.io/etcd/mvcc/kvstore.go | 8 +-- vendor/go.etcd.io/etcd/version/version.go | 2 +- 11 files changed, 88 insertions(+), 23 deletions(-) create mode 100644 vendor/go.etcd.io/etcd/clientv3/ctx.go diff --git a/Gopkg.lock b/Gopkg.lock index 1fcc49c57..b3398227b 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -928,7 +928,7 @@ version = "v1.3.3" [[projects]] - digest = "1:fa7ce8fc37d6376cb41865ecc2f8ecb877a5043fad7b10afcf7c56edea8c35d4" + digest = "1:92c278e11377c17aae9a3988c5e603e782c3339b6434e28c18c6ac82189f2b3b" name = "go.etcd.io/etcd" packages = [ "auth/authpb", @@ -964,8 +964,8 @@ "version", ] pruneopts = "UT" - revision = "c65a9e2dd1fd500ca4191b1f22ddfe5e019b3ca1" - version = "v3.4.4" + revision = "e694b7bb087538c146e188a29753967d189d202b" + version = "v3.4.7" [[projects]] digest = "1:19525e48b45af69339bd7c2812137dbd1e650a78fc09c7c75482e307d51aa28d" diff --git a/Gopkg.toml b/Gopkg.toml index a3807188e..89cb544ba 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -69,7 +69,7 @@ ignored = ["github.com/gravitational/planet/build/*"] [[override]] name = "github.com/etcd-io/etcd" - version = "3.4.3" + version = "=3.4.7" [[constraint]] name = "github.com/docker/docker" diff --git a/Makefile b/Makefile index a8fa2bdcf..b064ba93c 100644 --- a/Makefile +++ b/Makefile @@ -59,13 +59,15 @@ PLANET_GID ?= 980665 # v3.3.11 - 5.5.x, # v3.3.12 - 6.3.x, 6.1.x, 5.5.x # v3.3.15 - 6.3.x +# v3.3.20 - 6.3.x, 6.1.x, 5.5.x # v3.4.3 - 7.0.x -ETCD_VER := v3.3.12 v3.3.15 v3.4.3 +# v3.4.7 - 7.0.x +ETCD_VER := v3.3.12 v3.3.15 v3.4.3 v3.4.7 # This is the version of etcd we should upgrade to (from the version list) # Note: When bumping the ETCD_LATEST_VERSION, please ensure that: # - The version of etcd vendored as a library is the same (Gopkg.toml) # - Modify build.go and run the etcd upgrade integration test (go run mage.go ci:testEtcdUpgrade) -ETCD_LATEST_VER := v3.4.3 +ETCD_LATEST_VER := v3.4.7 BUILDBOX_GO_VER ?= 1.12.9 PLANET_BUILD_TAG ?= $(shell git describe --tags) diff --git a/build.go b/build.go index 29283d63c..ee92e1b8b 100644 --- a/build.go +++ b/build.go @@ -26,6 +26,6 @@ type Test mg.Namespace func (Test) TestEtcdUpgrade() error { // The "to" version should match what etcd is built with // TODO(knisbet) integrate this variable with etcd versions set in Makefile - err := upgrade.TestUpgradeBetweenVersions("v3.3.3", "v3.4.3") + err := upgrade.TestUpgradeBetweenVersions("v3.3.3", "v3.4.7") return trace.Wrap(err) } diff --git a/vendor/go.etcd.io/etcd/clientv3/client.go b/vendor/go.etcd.io/etcd/clientv3/client.go index 215e05479..a35ec679a 100644 --- a/vendor/go.etcd.io/etcd/clientv3/client.go +++ b/vendor/go.etcd.io/etcd/clientv3/client.go @@ -37,7 +37,6 @@ import ( "google.golang.org/grpc/codes" grpccredentials "google.golang.org/grpc/credentials" "google.golang.org/grpc/keepalive" - "google.golang.org/grpc/metadata" "google.golang.org/grpc/status" ) @@ -397,13 +396,6 @@ func (c *Client) dialWithBalancerCreds(ep string) grpccredentials.TransportCrede return creds } -// WithRequireLeader requires client requests to only succeed -// when the cluster has a leader. -func WithRequireLeader(ctx context.Context) context.Context { - md := metadata.Pairs(rpctypes.MetadataRequireLeaderKey, rpctypes.MetadataHasLeader) - return metadata.NewOutgoingContext(ctx, md) -} - func newClient(cfg *Config) (*Client, error) { if cfg == nil { cfg = &Config{} diff --git a/vendor/go.etcd.io/etcd/clientv3/ctx.go b/vendor/go.etcd.io/etcd/clientv3/ctx.go new file mode 100644 index 000000000..542219837 --- /dev/null +++ b/vendor/go.etcd.io/etcd/clientv3/ctx.go @@ -0,0 +1,64 @@ +// Copyright 2020 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package clientv3 + +import ( + "context" + "strings" + + "go.etcd.io/etcd/etcdserver/api/v3rpc/rpctypes" + "go.etcd.io/etcd/version" + "google.golang.org/grpc/metadata" +) + +// WithRequireLeader requires client requests to only succeed +// when the cluster has a leader. +func WithRequireLeader(ctx context.Context) context.Context { + md, ok := metadata.FromOutgoingContext(ctx) + if !ok { // no outgoing metadata ctx key, create one + md = metadata.Pairs(rpctypes.MetadataRequireLeaderKey, rpctypes.MetadataHasLeader) + return metadata.NewOutgoingContext(ctx, md) + } + copied := md.Copy() // avoid racey updates + // overwrite/add 'hasleader' key/value + metadataSet(copied, rpctypes.MetadataRequireLeaderKey, rpctypes.MetadataHasLeader) + return metadata.NewOutgoingContext(ctx, copied) +} + +// embeds client version +func withVersion(ctx context.Context) context.Context { + md, ok := metadata.FromOutgoingContext(ctx) + if !ok { // no outgoing metadata ctx key, create one + md = metadata.Pairs(rpctypes.MetadataClientAPIVersionKey, version.APIVersion) + return metadata.NewOutgoingContext(ctx, md) + } + copied := md.Copy() // avoid racey updates + // overwrite/add version key/value + metadataSet(copied, rpctypes.MetadataClientAPIVersionKey, version.APIVersion) + return metadata.NewOutgoingContext(ctx, copied) +} + +func metadataGet(md metadata.MD, k string) []string { + k = strings.ToLower(k) + return md[k] +} + +func metadataSet(md metadata.MD, k string, vals ...string) { + if len(vals) == 0 { + return + } + k = strings.ToLower(k) + md[k] = vals +} diff --git a/vendor/go.etcd.io/etcd/clientv3/retry_interceptor.go b/vendor/go.etcd.io/etcd/clientv3/retry_interceptor.go index aac679ecc..2c266e55b 100644 --- a/vendor/go.etcd.io/etcd/clientv3/retry_interceptor.go +++ b/vendor/go.etcd.io/etcd/clientv3/retry_interceptor.go @@ -38,6 +38,7 @@ import ( func (c *Client) unaryClientInterceptor(logger *zap.Logger, optFuncs ...retryOption) grpc.UnaryClientInterceptor { intOpts := reuseOrNewWithCallOptions(defaultOptions, optFuncs) return func(ctx context.Context, method string, req, reply interface{}, cc *grpc.ClientConn, invoker grpc.UnaryInvoker, opts ...grpc.CallOption) error { + ctx = withVersion(ctx) grpcOpts, retryOpts := filterCallOptions(opts) callOpts := reuseOrNewWithCallOptions(intOpts, retryOpts) // short circuit for simplicity, and avoiding allocations. @@ -103,6 +104,7 @@ func (c *Client) unaryClientInterceptor(logger *zap.Logger, optFuncs ...retryOpt func (c *Client) streamClientInterceptor(logger *zap.Logger, optFuncs ...retryOption) grpc.StreamClientInterceptor { intOpts := reuseOrNewWithCallOptions(defaultOptions, optFuncs) return func(ctx context.Context, desc *grpc.StreamDesc, cc *grpc.ClientConn, method string, streamer grpc.Streamer, opts ...grpc.CallOption) (grpc.ClientStream, error) { + ctx = withVersion(ctx) grpcOpts, retryOpts := filterCallOptions(opts) callOpts := reuseOrNewWithCallOptions(intOpts, retryOpts) // short circuit for simplicity, and avoiding allocations. diff --git a/vendor/go.etcd.io/etcd/etcdserver/api/v3rpc/rpctypes/md.go b/vendor/go.etcd.io/etcd/etcdserver/api/v3rpc/rpctypes/md.go index 5c590e1ae..90b8b835b 100644 --- a/vendor/go.etcd.io/etcd/etcdserver/api/v3rpc/rpctypes/md.go +++ b/vendor/go.etcd.io/etcd/etcdserver/api/v3rpc/rpctypes/md.go @@ -17,4 +17,6 @@ package rpctypes var ( MetadataRequireLeaderKey = "hasleader" MetadataHasLeader = "true" + + MetadataClientAPIVersionKey = "client-api-version" ) diff --git a/vendor/go.etcd.io/etcd/lease/lessor.go b/vendor/go.etcd.io/etcd/lease/lessor.go index b4437bd46..b16099fbf 100644 --- a/vendor/go.etcd.io/etcd/lease/lessor.go +++ b/vendor/go.etcd.io/etcd/lease/lessor.go @@ -291,14 +291,14 @@ func (le *lessor) Grant(id LeaseID, ttl int64) (*Lease, error) { } le.leaseMap[id] = l - item := &LeaseWithTime{id: l.ID, time: l.expiry.UnixNano()} - le.leaseExpiredNotifier.RegisterOrUpdate(item) l.persistTo(le.b) leaseTotalTTLs.Observe(float64(l.ttl)) leaseGranted.Inc() if le.isPrimary() { + item := &LeaseWithTime{id: l.ID, time: l.expiry.UnixNano()} + le.leaseExpiredNotifier.RegisterOrUpdate(item) le.scheduleCheckpointIfNeeded(l) } @@ -505,6 +505,7 @@ func (le *lessor) Demote() { } le.clearScheduledLeasesCheckpoints() + le.clearLeaseExpiredNotifier() if le.demotec != nil { close(le.demotec) @@ -648,6 +649,10 @@ func (le *lessor) clearScheduledLeasesCheckpoints() { le.leaseCheckpointHeap = make(LeaseQueue, 0) } +func (le *lessor) clearLeaseExpiredNotifier() { + le.leaseExpiredNotifier = newLeaseExpiredNotifier() +} + // expireExists returns true if expiry items exist. // It pops only when expiry item exists. // "next" is true, to indicate that it may exist in next attempt. diff --git a/vendor/go.etcd.io/etcd/mvcc/kvstore.go b/vendor/go.etcd.io/etcd/mvcc/kvstore.go index ed05bc288..997aaaf56 100644 --- a/vendor/go.etcd.io/etcd/mvcc/kvstore.go +++ b/vendor/go.etcd.io/etcd/mvcc/kvstore.go @@ -271,15 +271,15 @@ func (s *store) updateCompactRev(rev int64) (<-chan struct{}, error) { } func (s *store) compact(trace *traceutil.Trace, rev int64) (<-chan struct{}, error) { - start := time.Now() - keep := s.kvindex.Compact(rev) - trace.Step("compact in-memory index tree") ch := make(chan struct{}) var j = func(ctx context.Context) { if ctx.Err() != nil { s.compactBarrier(ctx, ch) return } + start := time.Now() + keep := s.kvindex.Compact(rev) + indexCompactionPauseMs.Observe(float64(time.Since(start) / time.Millisecond)) if !s.scheduleCompaction(rev, keep) { s.compactBarrier(nil, ch) return @@ -288,8 +288,6 @@ func (s *store) compact(trace *traceutil.Trace, rev int64) (<-chan struct{}, err } s.fifoSched.Schedule(j) - - indexCompactionPauseMs.Observe(float64(time.Since(start) / time.Millisecond)) trace.Step("schedule compaction") return ch, nil } diff --git a/vendor/go.etcd.io/etcd/version/version.go b/vendor/go.etcd.io/etcd/version/version.go index 5d6a29c6c..c2960bdd0 100644 --- a/vendor/go.etcd.io/etcd/version/version.go +++ b/vendor/go.etcd.io/etcd/version/version.go @@ -26,7 +26,7 @@ import ( var ( // MinClusterVersion is the min cluster version this etcd binary is compatible with. MinClusterVersion = "3.0.0" - Version = "3.4.4" + Version = "3.4.7" APIVersion = "unknown" // Git SHA Value will be set during build