Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[master] Add etcd-member-management controller to K3s #4001

Merged
merged 2 commits into from
Sep 14, 2021
Merged

[master] Add etcd-member-management controller to K3s #4001

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
c1ff50e
Initial leader elected etcd member management controller
Oats87 Sep 8, 2021
b7dc271
Bump etcd to v3.5.0-k3s2
Oats87 Sep 13, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,10 +29,10 @@ replace (
github.com/opencontainers/runtime-spec => github.com/opencontainers/runtime-spec v1.0.3-0.20210316141917-a8c4a9ee0f6b
github.com/rancher/k3s/pkg/data => ./pkg/data
github.com/rancher/wrangler => github.com/rancher/wrangler v0.8.5
go.etcd.io/etcd/api/v3 => github.com/k3s-io/etcd/api/v3 v3.5.0-k3s1
go.etcd.io/etcd/client/v3 => github.com/k3s-io/etcd/client/v3 v3.5.0-k3s1
go.etcd.io/etcd/etcdutl/v3 => github.com/k3s-io/etcd/etcdutl/v3 v3.5.0-k3s1
go.etcd.io/etcd/server/v3 => github.com/k3s-io/etcd/server/v3 v3.5.0-k3s1
go.etcd.io/etcd/api/v3 => github.com/k3s-io/etcd/api/v3 v3.5.0-k3s2
go.etcd.io/etcd/client/v3 => github.com/k3s-io/etcd/client/v3 v3.5.0-k3s2
go.etcd.io/etcd/etcdutl/v3 => github.com/k3s-io/etcd/etcdutl/v3 v3.5.0-k3s2
go.etcd.io/etcd/server/v3 => github.com/k3s-io/etcd/server/v3 v3.5.0-k3s2
golang.org/x/crypto => golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83
golang.org/x/net => golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4
golang.org/x/sys => golang.org/x/sys v0.0.0-20210616094352-59db8d763f22
Expand Down
16 changes: 8 additions & 8 deletions go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -562,14 +562,14 @@ github.com/k3s-io/continuity v0.0.0-20210309170710-f93269e0d5c1 h1:KEz2rd9IDbrQT
github.com/k3s-io/continuity v0.0.0-20210309170710-f93269e0d5c1/go.mod h1:EXlVlkqNba9rJe3j7w3Xa924itAMLgZH4UD/Q4PExuQ=
github.com/k3s-io/cri-tools v1.21.0-k3s1 h1:MWQtAsx4HCNXenqU/B4V9eU6HMyafkd1PnW6d4HCfos=
github.com/k3s-io/cri-tools v1.21.0-k3s1/go.mod h1:Qsz54zxINPR+WVWX9Kc3CTmuDFB1dNLCNV8jE8lUbtU=
github.com/k3s-io/etcd/api/v3 v3.5.0-k3s1 h1:hbVjpbEfKjttv+ZmYnqgR8wK7vtKIDrPxF8mcGVYWSU=
github.com/k3s-io/etcd/api/v3 v3.5.0-k3s1/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs=
github.com/k3s-io/etcd/client/v3 v3.5.0-k3s1 h1:RAXJQReqdjR6B+wL3HwWuBAdMO31fi7CGft0BdEyI5Q=
github.com/k3s-io/etcd/client/v3 v3.5.0-k3s1/go.mod h1:AIKXXVX/DQXtfTEqBryiLTUXwON+GuvO6Z7lLS/oTh0=
github.com/k3s-io/etcd/etcdutl/v3 v3.5.0-k3s1 h1:G0u+ShI11UDcP4Z8OP3CSl4/Wg63OgRBqYEiKMsUDP4=
github.com/k3s-io/etcd/etcdutl/v3 v3.5.0-k3s1/go.mod h1:o98rKMCibbFAG8QS9KmvlYDGDShmmIbmRE8vSofzYNg=
github.com/k3s-io/etcd/server/v3 v3.5.0-k3s1 h1:be2d2LS1w+OxyHE/PF6tmGVzF72+7L2FZ6BmxK1Q0ow=
github.com/k3s-io/etcd/server/v3 v3.5.0-k3s1/go.mod h1:3Ah5ruV+M+7RZr0+Y/5mNLwC+eQlni+mQmOVdCRJoS4=
github.com/k3s-io/etcd/api/v3 v3.5.0-k3s2 h1:56a/c/Mo75uU1k3erdgWk1C7dGUL5rSpNdL2B6zftKo=
github.com/k3s-io/etcd/api/v3 v3.5.0-k3s2/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs=
github.com/k3s-io/etcd/client/v3 v3.5.0-k3s2 h1:UUL/eMEf9utdaZNPwjdaupOwIgvq0sJyzvxgYWJQeJc=
github.com/k3s-io/etcd/client/v3 v3.5.0-k3s2/go.mod h1:AIKXXVX/DQXtfTEqBryiLTUXwON+GuvO6Z7lLS/oTh0=
github.com/k3s-io/etcd/etcdutl/v3 v3.5.0-k3s2 h1:Feifl9EStGdmkUnOtouh0VD9n+UbgTxk8DLOsqUdKVo=
github.com/k3s-io/etcd/etcdutl/v3 v3.5.0-k3s2/go.mod h1:o98rKMCibbFAG8QS9KmvlYDGDShmmIbmRE8vSofzYNg=
github.com/k3s-io/etcd/server/v3 v3.5.0-k3s2 h1:yw8t2/k8Gwsv462XkEVawYGn5N+FI2xG97O3lleSSMI=
github.com/k3s-io/etcd/server/v3 v3.5.0-k3s2/go.mod h1:3Ah5ruV+M+7RZr0+Y/5mNLwC+eQlni+mQmOVdCRJoS4=
github.com/k3s-io/helm-controller v0.11.3 h1:DSPAOCGHxF5pmF4vzQP5AgPT3tiGRJRZu+08hwWtbGI=
github.com/k3s-io/helm-controller v0.11.3/go.mod h1:z0ExsRRIkTO/QC//3/Esn5ItTD6AiQSluwzMaS7RI/4=
github.com/k3s-io/kine v0.8.0 h1:k6T9bI9DID7lIbktukXxg1QfeFoAQK4EIvAHoyPAe08=
Expand Down
9 changes: 5 additions & 4 deletions pkg/daemons/config/types.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -195,10 +195,11 @@ type ControlRuntimeBootstrap struct {
type ControlRuntime struct {
ControlRuntimeBootstrap

HTTPBootstrap bool
APIServerReady <-chan struct{}
ETCDReady <-chan struct{}
ClusterControllerStart func(ctx context.Context) error
HTTPBootstrap bool
APIServerReady <-chan struct{}
ETCDReady <-chan struct{}
ClusterControllerStart func(ctx context.Context) error
LeaderElectedClusterControllerStart func(ctx context.Context) error

ClientKubeAPICert string
ClientKubeAPIKey string
Expand Down
91 changes: 0 additions & 91 deletions pkg/etcd/controller.go
View file
Open in desktop

This file was deleted.

64 changes: 49 additions & 15 deletions pkg/etcd/etcd.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ import (
"github.com/rancher/k3s/pkg/daemons/control/deps"
"github.com/rancher/k3s/pkg/daemons/executor"
"github.com/rancher/k3s/pkg/version"
controllerv1 "github.com/rancher/wrangler/pkg/generated/controllers/core/v1"
"github.com/robfig/cron/v3"
"github.com/sirupsen/logrus"
"go.etcd.io/etcd/api/v3/etcdserverpb"
Expand Down Expand Up @@ -56,6 +57,10 @@ const (
defaultKeepAliveTimeout = 10 * time.Second

maxBackupRetention = 5

MasterLabel = "node-role.kubernetes.io/master"
ControlPlaneLabel = "node-role.kubernetes.io/control-plane"
EtcdRoleLabel = "node-role.kubernetes.io/etcd"
Comment on lines +61 to +63
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We already have these as consts at

k3s/pkg/server/server.go

Lines 42 to 44 in 699ea16

MasterRoleLabelKey = "node-role.kubernetes.io/master"
ControlPlaneRoleLabelKey = "node-role.kubernetes.io/control-plane"
ETCDRoleLabelKey = "node-role.kubernetes.io/etcd"

)

var (
Expand All @@ -64,8 +69,13 @@ var (
AddressKey = version.Program + "/apiaddresses"

snapshotConfigMapName = version.Program + "-etcd-snapshots"

NodeNameAnnotation = "etcd." + version.Program + ".cattle.io/node-name"
NodeAddressAnnotation = "etcd." + version.Program + ".cattle.io/node-address"
)

type NodeControllerGetter func() controllerv1.NodeController

type ETCD struct {
client *clientv3.Client
config *config.Control
Expand Down Expand Up @@ -373,21 +383,27 @@ func (e *ETCD) Register(ctx context.Context, config *config.Control, handler htt
e.config.Datastore.BackendTLSConfig.CertFile = e.runtime.ClientETCDCert
e.config.Datastore.BackendTLSConfig.KeyFile = e.runtime.ClientETCDKey

tombstoneFile := filepath.Join(etcdDBDir(e.config), "tombstone")
if _, err := os.Stat(tombstoneFile); err == nil {
logrus.Infof("tombstone file has been detected, removing data dir to rejoin the cluster")
if _, err := backupDirWithRetention(etcdDBDir(e.config), maxBackupRetention); err != nil {
return nil, err
}
}

if err := e.setName(false); err != nil {
return nil, err
}
e.config.Runtime.ClusterControllerStart = func(ctx context.Context) error {
Register(ctx, e, e.config.Runtime.Core.Core().V1().Node())
RegisterMetadataHandlers(ctx, e, e.config.Runtime.Core.Core().V1().Node())
return nil
}

tombstoneFile := filepath.Join(etcdDBDir(e.config), "tombstone")
if _, err := os.Stat(tombstoneFile); err == nil {
logrus.Infof("tombstone file has been detected, removing data dir to rejoin the cluster")
if _, err := backupDirWithRetention(etcdDBDir(e.config), maxBackupRetention); err != nil {
return nil, err
}
e.config.Runtime.LeaderElectedClusterControllerStart = func(ctx context.Context) error {
RegisterMemberHandlers(ctx, e, e.config.Runtime.Core.Core().V1().Node())
return nil
}

return e.handler(handler), err
}

Expand Down Expand Up @@ -449,7 +465,7 @@ func (e *ETCD) infoHandler() http.Handler {
})
}

// getClient returns an etcd client connected to the specified endpoints
// GetClient returns an etcd client connected to the specified endpoints
func GetClient(ctx context.Context, runtime *config.ControlRuntime, endpoints ...string) (*clientv3.Client, error) {
cfg, err := getClientConfig(ctx, runtime, endpoints...)
if err != nil {
Expand All @@ -458,7 +474,7 @@ func GetClient(ctx context.Context, runtime *config.ControlRuntime, endpoints ..
return clientv3.New(*cfg)
}

//getClientConfig generates an etcd client config connected to the specified endpoints
// getClientConfig generates an etcd client config connected to the specified endpoints
func getClientConfig(ctx context.Context, runtime *config.ControlRuntime, endpoints ...string) (*clientv3.Config, error) {
tlsConfig, err := toTLSConfig(runtime)
if err != nil {
Expand Down Expand Up @@ -623,8 +639,8 @@ func (e *ETCD) cluster(ctx context.Context, forceNew bool, options executor.Init
})
}

// removePeer removes a peer from the cluster. The peer ID and IP address must both match.
func (e *ETCD) removePeer(ctx context.Context, id, address string, removeSelf bool) error {
// RemovePeer removes a peer from the cluster. The peer name and IP address must both match.
func (e *ETCD) RemovePeer(ctx context.Context, name, address string, allowSelfRemoval bool) error {
ctx, cancel := context.WithTimeout(ctx, memberRemovalTimeout)
defer cancel()
members, err := e.client.MemberList(ctx)
Expand All @@ -633,7 +649,7 @@ func (e *ETCD) removePeer(ctx context.Context, id, address string, removeSelf bo
}

for _, member := range members.Members {
if member.Name != id {
if member.Name != name {
continue
}
for _, peerURL := range member.PeerURLs {
Expand All @@ -642,8 +658,8 @@ func (e *ETCD) removePeer(ctx context.Context, id, address string, removeSelf bo
return err
}
if u.Hostname() == address {
if e.address == address && !removeSelf {
return errors.New("node has been deleted from the cluster")
if e.address == address && !allowSelfRemoval {
return errors.New("not removing self from etcd cluster")
}
logrus.Infof("Removing name=%s id=%d address=%s from etcd", member.Name, member.ID, address)
_, err := e.client.MemberRemove(ctx, member.ID)
Expand Down Expand Up @@ -1398,9 +1414,27 @@ func (e *ETCD) GetMembersClientURLs(ctx context.Context) ([]string, error) {
return memberUrls, nil
}

// GetMembersNames will list through the member lists in etcd and return
// back a combined list of member names
func (e *ETCD) GetMembersNames(ctx context.Context) ([]string, error) {
ctx, cancel := context.WithTimeout(ctx, testTimeout)
defer cancel()

members, err := e.client.MemberList(ctx)
if err != nil {
return nil, err
}

var memberNames []string
for _, member := range members.Members {
memberNames = append(memberNames, member.Name)
}
return memberNames, nil
}

// RemoveSelf will remove the member if it exists in the cluster
func (e *ETCD) RemoveSelf(ctx context.Context) error {
if err := e.removePeer(ctx, e.name, e.address, true); err != nil {
if err := e.RemovePeer(ctx, e.name, e.address, true); err != nil {
return err
}

Expand Down
Loading