From 98c61eee1cf8ff69f9c163d87a4d5e636b9f72d6 Mon Sep 17 00:00:00 2001 From: Aylei Date: Thu, 12 Dec 2019 00:35:07 +0800 Subject: [PATCH] Sync pd and tikv configmap in controller Signed-off-by: Aylei --- manifests/crd.yaml | 10 +- .../pingcap/v1alpha1/openapi_generated.go | 17 +- pkg/apis/pingcap/v1alpha1/pd_config.go | 52 +----- pkg/apis/pingcap/v1alpha1/tidbcluster.go | 16 ++ pkg/apis/pingcap/v1alpha1/types.go | 6 + .../pingcap/v1alpha1/zz_generated.deepcopy.go | 17 -- .../tidbcluster/tidb_cluster_controller.go | 2 + pkg/manager/member/pd_member_manager.go | 88 +++++++++- pkg/manager/member/pd_member_manager_test.go | 103 +++++++++++- pkg/manager/member/template.go | 155 ++++++++++++++++++ pkg/manager/member/tidb_member_manager.go | 21 ++- pkg/manager/member/tikv_member_manager.go | 87 +++++++++- .../member/tikv_member_manager_test.go | 99 ++++++++++- pkg/manager/member/utils.go | 10 ++ pkg/pdapi/pdapi_test.go | 3 +- tests/e2e/tidbcluster/tidbcluster.go | 84 ++++++---- tests/failover.go | 5 +- 17 files changed, 643 insertions(+), 132 deletions(-) diff --git a/manifests/crd.yaml b/manifests/crd.yaml index 87ccd728177..490719ff5b9 100644 --- a/manifests/crd.yaml +++ b/manifests/crd.yaml @@ -923,7 +923,11 @@ spec: target store. format: int64 type: integer - max-store-down-time: {} + max-store-down-time: + description: MaxStoreDownTime is the max duration after + which a store will be considered to be down if it hasn't + reported heartbeats. + type: string merge-schedule-limit: description: MergeScheduleLimit is the max coexist merge schedules. @@ -994,6 +998,8 @@ spec: description: TsoSaveInterval is the interval to save timestamp. type: string type: object + configUpdateStrategy: + type: string replicas: format: int32 type: integer @@ -2192,6 +2198,8 @@ spec: type: integer type: object type: object + configUpdateStrategy: + type: string maxFailoverCount: format: int32 type: integer diff --git a/pkg/apis/pingcap/v1alpha1/openapi_generated.go b/pkg/apis/pingcap/v1alpha1/openapi_generated.go index 68501cadf9b..90fa90fe657 100644 --- a/pkg/apis/pingcap/v1alpha1/openapi_generated.go +++ b/pkg/apis/pingcap/v1alpha1/openapi_generated.go @@ -1520,7 +1520,8 @@ func schema_pkg_apis_pingcap_v1alpha1_PDScheduleConfig(ref common.ReferenceCallb "max-store-down-time": { SchemaProps: spec.SchemaProps{ Description: "MaxStoreDownTime is the max duration after which a store will be considered to be down if it hasn't reported heartbeats.", - Ref: ref("github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1.Duration"), + Type: []string{"string"}, + Format: "", }, }, "leader-schedule-limit": { @@ -1652,7 +1653,7 @@ func schema_pkg_apis_pingcap_v1alpha1_PDScheduleConfig(ref common.ReferenceCallb }, }, Dependencies: []string{ - "github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1.Duration", "github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1.PDSchedulerConfig"}, + "github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1.PDSchedulerConfig"}, } } @@ -1767,6 +1768,12 @@ func schema_pkg_apis_pingcap_v1alpha1_PDSpec(ref common.ReferenceCallback) commo Format: "", }, }, + "configUpdateStrategy": { + SchemaProps: spec.SchemaProps{ + Type: []string{"string"}, + Format: "", + }, + }, "config": { SchemaProps: spec.SchemaProps{ Description: "Config is the Configuration of pd-servers", @@ -4156,6 +4163,12 @@ func schema_pkg_apis_pingcap_v1alpha1_TiKVSpec(ref common.ReferenceCallback) com Format: "int32", }, }, + "configUpdateStrategy": { + SchemaProps: spec.SchemaProps{ + Type: []string{"string"}, + Format: "", + }, + }, "config": { SchemaProps: spec.SchemaProps{ Description: "Config is the Configuration of tikv-servers", diff --git a/pkg/apis/pingcap/v1alpha1/pd_config.go b/pkg/apis/pingcap/v1alpha1/pd_config.go index 5c81bb55b18..60e7e1f86ce 100644 --- a/pkg/apis/pingcap/v1alpha1/pd_config.go +++ b/pkg/apis/pingcap/v1alpha1/pd_config.go @@ -14,10 +14,6 @@ package v1alpha1 import ( - "fmt" - "strconv" - "time" - "github.com/pingcap/log" "github.com/pingcap/pd/pkg/metricutil" "github.com/pingcap/pd/pkg/typeutil" @@ -101,13 +97,13 @@ type PDConfig struct { Replication *PDReplicationConfig `toml:"replication,omitempty" json:"replication,omitempty"` // +optional - Namespace map[string]PDNamespaceConfig `json:"namespace,omitempty"` + Namespace map[string]PDNamespaceConfig `toml:"namespace,omitempty" json:"namespace,omitempty"` // +optional PDServerCfg *PDServerConfig `toml:"pd-server,omitempty" json:"pd-server,omitempty"` // +optional - ClusterVersion string `json:"cluster-version,omitempty"` + ClusterVersion string `toml:"cluster-version,omitempty" json:"cluster-version,omitempty"` // QuotaBackendBytes Raise alarms when backend size exceeds the given quota. 0 means use the default quota. // the default size is 2GB, the maximum is 8GB. @@ -250,7 +246,7 @@ type PDScheduleConfig struct { // MaxStoreDownTime is the max duration after which // a store will be considered to be down if it hasn't reported heartbeats. // +optional - MaxStoreDownTime Duration `toml:"max-store-down-time,omitempty" json:"max-store-down-time,omitempty"` + MaxStoreDownTime string `toml:"max-store-down-time,omitempty" json:"max-store-down-time,omitempty"` // LeaderScheduleLimit is the max coexist leader schedules. // +optional LeaderScheduleLimit *uint64 `toml:"leader-schedule-limit,omitempty" json:"leader-schedule-limit,omitempty"` @@ -369,45 +365,3 @@ type PDServerConfig struct { // +optional UseRegionStorage *bool `toml:"use-region-storage,omitempty" json:"use-region-storage,string,omitempty"` } - -// Duration is a wrapper of time.Duration for TOML and JSON. -// Copied from pingcap typeutil, add marshal to TOML support -type Duration struct { - time.Duration -} - -// NewDuration creates a Duration from time.Duration. -func NewDuration(duration time.Duration) Duration { - return Duration{Duration: duration} -} - -// MarshalJSON returns the duration as a JSON string. -func (d *Duration) MarshalJSON() ([]byte, error) { - return []byte(fmt.Sprintf(`"%s"`, d.String())), nil -} - -// UnmarshalJSON parses a JSON string into the duration. -func (d *Duration) UnmarshalJSON(text []byte) error { - s, err := strconv.Unquote(string(text)) - if err != nil { - return err - } - duration, err := time.ParseDuration(s) - if err != nil { - return err - } - d.Duration = duration - return nil -} - -// UnmarshalText parses a TOML string into the duration. -func (d *Duration) UnmarshalText(text []byte) error { - var err error - d.Duration, err = time.ParseDuration(string(text)) - return err -} - -// MarshalText marshal duration to a TOML string -func (d *Duration) MarshalText() ([]byte, error) { - return []byte(fmt.Sprintf(`"%s"`, d.String())), nil -} diff --git a/pkg/apis/pingcap/v1alpha1/tidbcluster.go b/pkg/apis/pingcap/v1alpha1/tidbcluster.go index 293c7db2c07..60df1d94fad 100644 --- a/pkg/apis/pingcap/v1alpha1/tidbcluster.go +++ b/pkg/apis/pingcap/v1alpha1/tidbcluster.go @@ -321,6 +321,22 @@ func (tc *TidbCluster) TiDBConfigUpdateStrategy() ConfigUpdateStrategy { return s } +func (tc *TidbCluster) PDConfigUpdateStrategy() ConfigUpdateStrategy { + s := tc.Spec.PD.ConfigUpdateStrategy + if string(s) == "" { + s = ConfigUpdateStrategyInPlace + } + return s +} + +func (tc *TidbCluster) TiKVConfigUpdateStrategy() ConfigUpdateStrategy { + s := tc.Spec.TiKV.ConfigUpdateStrategy + if string(s) == "" { + s = ConfigUpdateStrategyInPlace + } + return s +} + func (tc *TidbCluster) PDIsAvailable() bool { lowerLimit := tc.Spec.PD.Replicas/2 + 1 if int32(len(tc.Status.PD.Members)) < lowerLimit { diff --git a/pkg/apis/pingcap/v1alpha1/types.go b/pkg/apis/pingcap/v1alpha1/types.go index 04690c178b7..d680ae0a871 100644 --- a/pkg/apis/pingcap/v1alpha1/types.go +++ b/pkg/apis/pingcap/v1alpha1/types.go @@ -182,6 +182,9 @@ type PDSpec struct { Service *ServiceSpec `json:"service,omitempty"` StorageClassName string `json:"storageClassName,omitempty"` + // +optional + ConfigUpdateStrategy ConfigUpdateStrategy `json:"configUpdateStrategy,omitempty"` + // Config is the Configuration of pd-servers Config *PDConfig `json:"config,omitempty"` } @@ -200,6 +203,9 @@ type TiKVSpec struct { StorageClassName string `json:"storageClassName,omitempty"` MaxFailoverCount int32 `json:"maxFailoverCount,omitempty"` + // +optional + ConfigUpdateStrategy ConfigUpdateStrategy `json:"configUpdateStrategy,omitempty"` + // Config is the Configuration of tikv-servers Config *TiKVConfig `json:"config,omitempty"` } diff --git a/pkg/apis/pingcap/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/pingcap/v1alpha1/zz_generated.deepcopy.go index c461d537e51..e7fb96e8bc7 100644 --- a/pkg/apis/pingcap/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/pingcap/v1alpha1/zz_generated.deepcopy.go @@ -441,22 +441,6 @@ func (in *DataResourceList) DeepCopyObject() runtime.Object { return nil } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *Duration) DeepCopyInto(out *Duration) { - *out = *in - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Duration. -func (in *Duration) DeepCopy() *Duration { - if in == nil { - return nil - } - out := new(Duration) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *GcsStorageProvider) DeepCopyInto(out *GcsStorageProvider) { *out = *in @@ -899,7 +883,6 @@ func (in *PDScheduleConfig) DeepCopyInto(out *PDScheduleConfig) { *out = new(uint64) **out = **in } - out.MaxStoreDownTime = in.MaxStoreDownTime if in.LeaderScheduleLimit != nil { in, out := &in.LeaderScheduleLimit, &out.LeaderScheduleLimit *out = new(uint64) diff --git a/pkg/controller/tidbcluster/tidb_cluster_controller.go b/pkg/controller/tidbcluster/tidb_cluster_controller.go index 9adb0140549..1a2614dfc12 100644 --- a/pkg/controller/tidbcluster/tidb_cluster_controller.go +++ b/pkg/controller/tidbcluster/tidb_cluster_controller.go @@ -125,6 +125,7 @@ func NewController( svcControl, podControl, certControl, + typedControl, setInformer.Lister(), svcInformer.Lister(), podInformer.Lister(), @@ -140,6 +141,7 @@ func NewController( setControl, svcControl, certControl, + typedControl, setInformer.Lister(), svcInformer.Lister(), podInformer.Lister(), diff --git a/pkg/manager/member/pd_member_manager.go b/pkg/manager/member/pd_member_manager.go index 60b6401f658..173c03c4ce4 100644 --- a/pkg/manager/member/pd_member_manager.go +++ b/pkg/manager/member/pd_member_manager.go @@ -16,6 +16,7 @@ package member import ( "fmt" "strconv" + "strings" "github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1" "github.com/pingcap/tidb-operator/pkg/controller" @@ -41,6 +42,7 @@ type pdMemberManager struct { svcControl controller.ServiceControlInterface podControl controller.PodControlInterface certControl controller.CertControlInterface + typedControl controller.TypedControlInterface setLister v1.StatefulSetLister svcLister corelisters.ServiceLister podLister corelisters.PodLister @@ -58,6 +60,7 @@ func NewPDMemberManager(pdControl pdapi.PDControlInterface, svcControl controller.ServiceControlInterface, podControl controller.PodControlInterface, certControl controller.CertControlInterface, + typedControl controller.TypedControlInterface, setLister v1.StatefulSetLister, svcLister corelisters.ServiceLister, podLister corelisters.PodLister, @@ -73,6 +76,7 @@ func NewPDMemberManager(pdControl pdapi.PDControlInterface, svcControl, podControl, certControl, + typedControl, setLister, svcLister, podLister, @@ -177,16 +181,22 @@ func (pmm *pdMemberManager) syncPDStatefulSetForTidbCluster(tc *v1alpha1.TidbClu ns := tc.GetNamespace() tcName := tc.GetName() - newPDSet, err := getNewPDSetForTidbCluster(tc) - if err != nil { + oldPDSetTmp, err := pmm.setLister.StatefulSets(ns).Get(controller.PDMemberName(tcName)) + if err != nil && !errors.IsNotFound(err) { return err } + setNotExist := errors.IsNotFound(err) - oldPDSetTmp, err := pmm.setLister.StatefulSets(ns).Get(controller.PDMemberName(tcName)) - if err != nil && !errors.IsNotFound(err) { + oldPDSet := oldPDSetTmp.DeepCopy() + cm, err := pmm.syncPDConfigMap(tc, oldPDSet) + if err != nil { return err } - if errors.IsNotFound(err) { + newPDSet, err := getNewPDSetForTidbCluster(tc, cm) + if err != nil { + return err + } + if setNotExist { err = SetLastAppliedConfigAnnotation(newPDSet) if err != nil { return err @@ -208,8 +218,6 @@ func (pmm *pdMemberManager) syncPDStatefulSetForTidbCluster(tc *v1alpha1.TidbClu return controller.RequeueErrorf("TidbCluster: [%s/%s], waiting for PD cluster running", ns, tcName) } - oldPDSet := oldPDSetTmp.DeepCopy() - if err := pmm.syncTidbClusterStatus(tc, oldPDSet); err != nil { glog.Errorf("failed to sync TidbCluster: [%s/%s]'s status, error: %v", ns, tcName, err) } @@ -405,6 +413,29 @@ func (pmm *pdMemberManager) syncTidbClusterStatus(tc *v1alpha1.TidbCluster, set return nil } +// syncPDConfigMap syncs the configmap of PD +func (pmm *pdMemberManager) syncPDConfigMap(tc *v1alpha1.TidbCluster, set *apps.StatefulSet) (*corev1.ConfigMap, error) { + + // For backward compatibility, only sync tidb configmap when .pd.config is non-nil + if tc.Spec.PD.Config == nil { + return nil, nil + } + newCm, err := getPDConfigMap(tc) + if err != nil { + return nil, err + } + if set != nil && tc.PDConfigUpdateStrategy() == v1alpha1.ConfigUpdateStrategyInPlace { + inUseName := FindConfigMapVolume(&set.Spec.Template.Spec, func(name string) bool { + return strings.HasPrefix(name, controller.PDMemberName(tc.Name)) + }) + if inUseName != "" { + newCm.Name = inUseName + } + } + + return pmm.typedControl.CreateOrUpdateConfigMap(tc, newCm) +} + func (pmm *pdMemberManager) getNewPDServiceForTidbCluster(tc *v1alpha1.TidbCluster) *corev1.Service { ns := tc.Namespace tcName := tc.Name @@ -491,11 +522,14 @@ func (pmm *pdMemberManager) pdStatefulSetIsUpgrading(set *apps.StatefulSet, tc * return false, nil } -func getNewPDSetForTidbCluster(tc *v1alpha1.TidbCluster) (*apps.StatefulSet, error) { +func getNewPDSetForTidbCluster(tc *v1alpha1.TidbCluster, cm *corev1.ConfigMap) (*apps.StatefulSet, error) { ns := tc.Namespace tcName := tc.Name instanceName := tc.GetLabels()[label.InstanceLabelKey] pdConfigMap := controller.MemberConfigMapName(tc, v1alpha1.PDMemberType) + if cm != nil { + pdConfigMap = cm.Name + } annMount, annVolume := annotationsMountVolume() volMounts := []corev1.VolumeMount{ @@ -688,6 +722,44 @@ func getNewPDSetForTidbCluster(tc *v1alpha1.TidbCluster) (*apps.StatefulSet, err return pdSet, nil } +func getPDConfigMap(tc *v1alpha1.TidbCluster) (*corev1.ConfigMap, error) { + + // For backward compatibility, only sync tidb configmap when .tidb.config is non-nil + config := tc.Spec.PD.Config + if config == nil { + return nil, nil + } + confText, err := MarshalTOML(config) + if err != nil { + return nil, err + } + startScript, err := RenderPDStartScript(&PDStartScriptModel{Scheme: tc.Scheme()}) + if err != nil { + return nil, err + } + + instanceName := tc.GetLabels()[label.InstanceLabelKey] + pdLabel := label.New().Instance(instanceName).PD().Labels() + cm := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: controller.PDMemberName(tc.Name), + Namespace: tc.Namespace, + Labels: pdLabel, + OwnerReferences: []metav1.OwnerReference{controller.GetOwnerRef(tc)}, + }, + Data: map[string]string{ + "config-file": string(confText), + "startup-script": startScript, + }, + } + if tc.PDConfigUpdateStrategy() == v1alpha1.ConfigUpdateStrategyRollingUpdate { + if err := AddConfigMapDigestSuffix(cm); err != nil { + return nil, err + } + } + return cm, nil +} + type FakePDMemberManager struct { err error } diff --git a/pkg/manager/member/pd_member_manager_test.go b/pkg/manager/member/pd_member_manager_test.go index d5cb06e19fa..6460679092b 100644 --- a/pkg/manager/member/pd_member_manager_test.go +++ b/pkg/manager/member/pd_member_manager_test.go @@ -21,6 +21,7 @@ import ( "github.com/google/go-cmp/cmp" . "github.com/onsi/gomega" "github.com/pingcap/kvproto/pkg/metapb" + "github.com/pingcap/pd/pkg/typeutil" "github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1" "github.com/pingcap/tidb-operator/pkg/client/clientset/versioned/fake" informers "github.com/pingcap/tidb-operator/pkg/client/informers/externalversions" @@ -37,6 +38,7 @@ import ( kubeinformers "k8s.io/client-go/informers" kubefake "k8s.io/client-go/kubernetes/fake" "k8s.io/client-go/tools/cache" + "k8s.io/utils/pointer" ) func TestPDMemberManagerSyncCreate(t *testing.T) { @@ -790,6 +792,7 @@ func newFakePDMemberManager() (*pdMemberManager, *controller.FakeStatefulSetCont autoFailover := true pdFailover := NewFakePDFailover() pdUpgrader := NewFakePDUpgrader() + genericControll := controller.NewFakeGenericControl() return &pdMemberManager{ pdControl, @@ -797,6 +800,7 @@ func newFakePDMemberManager() (*pdMemberManager, *controller.FakeStatefulSetCont svcControl, podControl, certControl, + controller.NewTypedControl(genericControll), setInformer.Lister(), svcInformer.Lister(), podInformer.Lister(), @@ -1014,7 +1018,7 @@ func TestGetNewPDSetForTidbCluster(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - sts, err := getNewPDSetForTidbCluster(&tt.tc) + sts, err := getNewPDSetForTidbCluster(&tt.tc, nil) if (err != nil) != tt.wantErr { t.Errorf("error %v, wantErr %v", err, tt.wantErr) } @@ -1022,3 +1026,100 @@ func TestGetNewPDSetForTidbCluster(t *testing.T) { }) } } + +func TestGetPDConfigMap(t *testing.T) { + g := NewGomegaWithT(t) + testCases := []struct { + name string + tc v1alpha1.TidbCluster + expected *corev1.ConfigMap + }{ + { + name: "PD config is nil", + tc: v1alpha1.TidbCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + Namespace: "ns", + }, + }, + expected: nil, + }, + { + name: "basic", + tc: v1alpha1.TidbCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + Namespace: "ns", + }, + Spec: v1alpha1.TidbClusterSpec{ + PD: v1alpha1.PDSpec{ + ConfigUpdateStrategy: v1alpha1.ConfigUpdateStrategyInPlace, + Config: &v1alpha1.PDConfig{ + Schedule: &v1alpha1.PDScheduleConfig{ + MaxStoreDownTime: "5m", + DisableRemoveDownReplica: pointer.BoolPtr(true), + }, + Replication: &v1alpha1.PDReplicationConfig{ + MaxReplicas: func() *uint64 { i := uint64(5); return &i }(), + LocationLabels: typeutil.StringSlice{"node", "rack"}, + }, + }, + }, + }, + }, + expected: &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo-pd", + Namespace: "ns", + Labels: map[string]string{ + "app.kubernetes.io/name": "tidb-cluster", + "app.kubernetes.io/managed-by": "tidb-operator", + "app.kubernetes.io/instance": "", + "app.kubernetes.io/component": "pd", + }, + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: "pingcap.com/v1alpha1", + Kind: "TidbCluster", + Name: "foo", + UID: "", + Controller: func(b bool) *bool { + return &b + }(true), + BlockOwnerDeletion: func(b bool) *bool { + return &b + }(true), + }, + }, + }, + Data: map[string]string{ + "startup-script": "", + "config-file": `[schedule] + max-store-down-time = "5m" + disable-remove-down-replica = true + +[replication] + max-replicas = 5 + location-labels = ["node", "rack"] +`, + }, + }, + }, + } + + for _, tt := range testCases { + t.Run(tt.name, func(t *testing.T) { + cm, err := getPDConfigMap(&tt.tc) + g.Expect(err).To(Succeed()) + if tt.expected == nil { + g.Expect(cm).To(BeNil()) + return + } + // startup-script is better to be tested in e2e + cm.Data["startup-script"] = "" + if diff := cmp.Diff(*tt.expected, *cm); diff != "" { + t.Errorf("unexpected plugin configuration (-want, +got): %s", diff) + } + }) + } +} diff --git a/pkg/manager/member/template.go b/pkg/manager/member/template.go index f5c6ac9036d..255a9368eb7 100644 --- a/pkg/manager/member/template.go +++ b/pkg/manager/member/template.go @@ -18,6 +18,7 @@ import ( "text/template" ) +// TODO(aylei): it is hard to maintain script in go literal, we should figure out a better solution // tidbStartScriptTpl is the template string of tidb start script // Note: changing this will cause a rolling-update of tidb-servers var tidbStartScriptTpl = template.Must(template.New("tidb-start-script").Parse(`#!/bin/sh @@ -83,6 +84,160 @@ func RenderTiDBStartScript(model *TidbStartScriptModel) (string, error) { return renderTemplateFunc(tidbStartScriptTpl, model) } +// pdStartScriptTpl is the pd start script +// Note: changing this will cause a rolling-update of pd cluster +var pdStartScriptTpl = template.Must(template.New("pd-start-script").Parse(`#!/bin/sh + +# This script is used to start pd containers in kubernetes cluster + +# Use DownwardAPIVolumeFiles to store informations of the cluster: +# https://kubernetes.io/docs/tasks/inject-data-application/downward-api-volume-expose-pod-information/#the-downward-api +# +# runmode="normal/debug" +# + +set -uo pipefail + +ANNOTATIONS="/etc/podinfo/annotations" + +if [[ ! -f "${ANNOTATIONS}" ]] +then + echo "${ANNOTATIONS} does't exist, exiting." + exit 1 +fi +source ${ANNOTATIONS} 2>/dev/null + +runmode=${runmode:-normal} +if [[ X${runmode} == Xdebug ]] +then + echo "entering debug mode." + tail -f /dev/null +fi + +# Use HOSTNAME if POD_NAME is unset for backward compatibility. +POD_NAME=${POD_NAME:-$HOSTNAME} +# the general form of variable PEER_SERVICE_NAME is: "-pd-peer" +cluster_name=` + "`" + `echo ${PEER_SERVICE_NAME} | sed 's/-pd-peer//'` + "`" + + ` +domain="${POD_NAME}.${PEER_SERVICE_NAME}.${NAMESPACE}.svc" +discovery_url="${cluster_name}-discovery.${NAMESPACE}.svc:10261" +encoded_domain_url=` + "`" + `echo ${domain}:2380 | base64 | tr "\n" " " | sed "s/ //g"` + "`" + + ` +elapseTime=0 +period=1 +threshold=30 +while true; do +sleep ${period} +elapseTime=$(( elapseTime+period )) + +if [[ ${elapseTime} -ge ${threshold} ]] +then +echo "waiting for pd cluster ready timeout" >&2 +exit 1 +fi + +if nslookup ${domain} 2>/dev/null +then +echo "nslookup domain ${domain}.svc success" +break +else +echo "nslookup domain ${domain} failed" >&2 +fi +done + +ARGS="--data-dir=/var/lib/pd \ +--name=${POD_NAME} \ +--peer-urls={{ .Scheme }}://0.0.0.0:2380 \ +--advertise-peer-urls={{ .Scheme }}://${domain}:2380 \ +--client-urls={{ .Scheme }}://0.0.0.0:2379 \ +--advertise-client-urls={{ .Scheme }}://${domain}:2379 \ +--config=/etc/pd/pd.toml \ +" + +if [[ -f /var/lib/pd/join ]] +then +# The content of the join file is: +# demo-pd-0=http://demo-pd-0.demo-pd-peer.demo.svc:2380,demo-pd-1=http://demo-pd-1.demo-pd-peer.demo.svc:2380 +# The --join args must be: +# --join=http://demo-pd-0.demo-pd-peer.demo.svc:2380,http://demo-pd-1.demo-pd-peer.demo.svc:2380 +join=` + "`" + `cat /var/lib/pd/join | tr "," "\n" | awk -F'=' '{print $2}' | tr "\n" ","` + "`" + ` +join=${join%,} +ARGS="${ARGS} --join=${join}" +elif [[ ! -d /var/lib/pd/member/wal ]] +then +until result=$(wget -qO- -T 3 http://${discovery_url}/new/${encoded_domain_url} 2>/dev/null); do +echo "waiting for discovery service to return start args ..." +sleep $((RANDOM % 5)) +done +ARGS="${ARGS}${result}" +fi + +echo "starting pd-server ..." +sleep $((RANDOM % 10)) +echo "/pd-server ${ARGS}" +exec /pd-server ${ARGS} +`)) + +type PDStartScriptModel struct { + Scheme string +} + +func RenderPDStartScript(model *PDStartScriptModel) (string, error) { + return renderTemplateFunc(pdStartScriptTpl, model) +} + +var tikvStartScriptTpl = template.Must(template.New("tikv-start-script").Parse(`#!/bin/sh + +# This script is used to start tikv containers in kubernetes cluster + +# Use DownwardAPIVolumeFiles to store informations of the cluster: +# https://kubernetes.io/docs/tasks/inject-data-application/downward-api-volume-expose-pod-information/#the-downward-api +# +# runmode="normal/debug" +# + +set -uo pipefail + +ANNOTATIONS="/etc/podinfo/annotations" + +if [[ ! -f "${ANNOTATIONS}" ]] +then + echo "${ANNOTATIONS} does't exist, exiting." + exit 1 +fi +source ${ANNOTATIONS} 2>/dev/null + +runmode=${runmode:-normal} +if [[ X${runmode} == Xdebug ]] +then + echo "entering debug mode." + tail -f /dev/null +fi + +# Use HOSTNAME if POD_NAME is unset for backward compatibility. +POD_NAME=${POD_NAME:-$HOSTNAME} +ARGS="--pd=http://${CLUSTER_NAME}-pd:2379 \ +--advertise-addr=${POD_NAME}.${HEADLESS_SERVICE_NAME}.${NAMESPACE}.svc:20160 \ +--addr=0.0.0.0:20160 \ +--status-addr=0.0.0.0:20180 \ +--data-dir=/var/lib/tikv \ +--capacity=${CAPACITY} \ +--config=/etc/tikv/tikv.toml +" + +echo "starting tikv-server ..." +echo "/tikv-server ${ARGS}" +exec /tikv-server ${ARGS} +`)) + +type TiKVStartScriptModel struct { + Scheme string +} + +func RenderTiKVStartScript(model *TiKVStartScriptModel) (string, error) { + return renderTemplateFunc(tikvStartScriptTpl, model) +} + // pumpStartScriptTpl is the template string of pump start script // Note: changing this will cause a rolling-update of pump cluster var pumpStartScriptTpl = template.Must(template.New("pump-start-script").Parse(`set -euo pipefail diff --git a/pkg/manager/member/tidb_member_manager.go b/pkg/manager/member/tidb_member_manager.go index 684cc22b8d2..81a5161cce1 100644 --- a/pkg/manager/member/tidb_member_manager.go +++ b/pkg/manager/member/tidb_member_manager.go @@ -443,19 +443,10 @@ func getTiDBConfigMap(tc *v1alpha1.TidbCluster) (*corev1.ConfigMap, error) { "startup-script": startScript, } name := controller.TiDBMemberName(tc.Name) - if tc.TiDBConfigUpdateStrategy() == v1alpha1.ConfigUpdateStrategyRollingUpdate { - sum, err := Sha256Sum(data) - if err != nil { - return nil, err - } - suffix := fmt.Sprintf("%x", sum)[0:7] - name = fmt.Sprintf("%s-%s", name, suffix) - } - instanceName := tc.GetLabels()[label.InstanceLabelKey] tidbLabels := label.New().Instance(instanceName).TiDB().Labels() - return &corev1.ConfigMap{ + cm := &corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ Name: name, Namespace: tc.Namespace, @@ -463,7 +454,15 @@ func getTiDBConfigMap(tc *v1alpha1.TidbCluster) (*corev1.ConfigMap, error) { OwnerReferences: []metav1.OwnerReference{controller.GetOwnerRef(tc)}, }, Data: data, - }, nil + } + + if tc.TiDBConfigUpdateStrategy() == v1alpha1.ConfigUpdateStrategyRollingUpdate { + if err := AddConfigMapDigestSuffix(cm); err != nil { + return nil, err + } + } + + return cm, nil } func getNewTiDBServiceOrNil(tc *v1alpha1.TidbCluster) *corev1.Service { diff --git a/pkg/manager/member/tikv_member_manager.go b/pkg/manager/member/tikv_member_manager.go index 16194f15eff..b8d313ca95f 100644 --- a/pkg/manager/member/tikv_member_manager.go +++ b/pkg/manager/member/tikv_member_manager.go @@ -43,6 +43,7 @@ type tikvMemberManager struct { svcControl controller.ServiceControlInterface pdControl pdapi.PDControlInterface certControl controller.CertControlInterface + typedControl controller.TypedControlInterface setLister v1.StatefulSetLister svcLister corelisters.ServiceLister podLister corelisters.PodLister @@ -59,6 +60,7 @@ func NewTiKVMemberManager(pdControl pdapi.PDControlInterface, setControl controller.StatefulSetControlInterface, svcControl controller.ServiceControlInterface, certControl controller.CertControlInterface, + typedControl controller.TypedControlInterface, setLister v1.StatefulSetLister, svcLister corelisters.ServiceLister, podLister corelisters.PodLister, @@ -74,6 +76,7 @@ func NewTiKVMemberManager(pdControl pdapi.PDControlInterface, setControl: setControl, svcControl: svcControl, certControl: certControl, + typedControl: typedControl, setLister: setLister, svcLister: svcLister, autoFailover: autoFailover, @@ -163,16 +166,23 @@ func (tkmm *tikvMemberManager) syncStatefulSetForTidbCluster(tc *v1alpha1.TidbCl ns := tc.GetNamespace() tcName := tc.GetName() - newSet, err := getNewTiKVSetForTidbCluster(tc) + oldSetTmp, err := tkmm.setLister.StatefulSets(ns).Get(controller.TiKVMemberName(tcName)) + if err != nil && !errors.IsNotFound(err) { + return err + } + setNotExist := errors.IsNotFound(err) + + oldSet := oldSetTmp.DeepCopy() + cm, err := tkmm.syncTiKVConfigMap(tc, oldSet) if err != nil { return err } - oldSetTmp, err := tkmm.setLister.StatefulSets(ns).Get(controller.TiKVMemberName(tcName)) - if err != nil && !errors.IsNotFound(err) { + newSet, err := getNewTiKVSetForTidbCluster(tc, cm) + if err != nil { return err } - if errors.IsNotFound(err) { + if setNotExist { err = SetLastAppliedConfigAnnotation(newSet) if err != nil { return err @@ -191,8 +201,6 @@ func (tkmm *tikvMemberManager) syncStatefulSetForTidbCluster(tc *v1alpha1.TidbCl return nil } - oldSet := oldSetTmp.DeepCopy() - if err := tkmm.syncTidbClusterStatus(tc, oldSet); err != nil { return err } @@ -271,6 +279,27 @@ func (tkmm *tikvMemberManager) syncTiKVServerCerts(tc *v1alpha1.TidbCluster) err return tkmm.certControl.Create(controller.GetOwnerRef(tc), certOpts) } +func (tkmm *tikvMemberManager) syncTiKVConfigMap(tc *v1alpha1.TidbCluster, set *apps.StatefulSet) (*corev1.ConfigMap, error) { + // For backward compatibility, only sync tidb configmap when .tikv.config is non-nil + if tc.Spec.TiKV.Config == nil { + return nil, nil + } + newCm, err := getTikVConfigMap(tc) + if err != nil { + return nil, err + } + if set != nil && tc.TiKVConfigUpdateStrategy() == v1alpha1.ConfigUpdateStrategyInPlace { + inUseName := FindConfigMapVolume(&set.Spec.Template.Spec, func(name string) bool { + return strings.HasPrefix(name, controller.TiKVMemberName(tc.Name)) + }) + if inUseName != "" { + newCm.Name = inUseName + } + } + + return tkmm.typedControl.CreateOrUpdateConfigMap(tc, newCm) +} + func getNewServiceForTidbCluster(tc *v1alpha1.TidbCluster, svcConfig SvcConfig) *corev1.Service { ns := tc.Namespace tcName := tc.Name @@ -306,10 +335,14 @@ func getNewServiceForTidbCluster(tc *v1alpha1.TidbCluster, svcConfig SvcConfig) return &svc } -func getNewTiKVSetForTidbCluster(tc *v1alpha1.TidbCluster) (*apps.StatefulSet, error) { +func getNewTiKVSetForTidbCluster(tc *v1alpha1.TidbCluster, cm *corev1.ConfigMap) (*apps.StatefulSet, error) { ns := tc.GetNamespace() tcName := tc.GetName() tikvConfigMap := controller.MemberConfigMapName(tc, v1alpha1.TiKVMemberType) + if cm != nil { + tikvConfigMap = cm.Name + } + annMount, annVolume := annotationsMountVolume() volMounts := []corev1.VolumeMount{ annMount, @@ -529,6 +562,46 @@ func volumeClaimTemplate(q resource.Quantity, metaName string, storageClassName } } +func getTikVConfigMap(tc *v1alpha1.TidbCluster) (*corev1.ConfigMap, error) { + + config := tc.Spec.TiKV.Config + if config == nil { + return nil, nil + } + confText, err := MarshalTOML(config) + if err != nil { + return nil, err + } + startScript, err := RenderTiKVStartScript(&TiKVStartScriptModel{ + Scheme: tc.Scheme(), + }) + if err != nil { + return nil, err + } + instanceName := tc.GetLabels()[label.InstanceLabelKey] + tikvLabel := label.New().Instance(instanceName).TiKV().Labels() + cm := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: controller.TiKVMemberName(tc.Name), + Namespace: tc.Namespace, + Labels: tikvLabel, + OwnerReferences: []metav1.OwnerReference{controller.GetOwnerRef(tc)}, + }, + Data: map[string]string{ + "config-file": string(confText), + "startup-script": startScript, + }, + } + + if tc.TiKVConfigUpdateStrategy() == v1alpha1.ConfigUpdateStrategyRollingUpdate { + if err := AddConfigMapDigestSuffix(cm); err != nil { + return nil, err + } + } + + return cm, nil +} + func labelTiKV(tc *v1alpha1.TidbCluster) label.Label { instanceName := tc.GetLabels()[label.InstanceLabelKey] return label.New().Instance(instanceName).TiKV() diff --git a/pkg/manager/member/tikv_member_manager_test.go b/pkg/manager/member/tikv_member_manager_test.go index 79020c50aa9..2cbac2b74f9 100644 --- a/pkg/manager/member/tikv_member_manager_test.go +++ b/pkg/manager/member/tikv_member_manager_test.go @@ -39,6 +39,7 @@ import ( kubefake "k8s.io/client-go/kubernetes/fake" corelisters "k8s.io/client-go/listers/core/v1" "k8s.io/client-go/tools/cache" + "k8s.io/utils/pointer" ) func TestTiKVMemberManagerSyncCreate(t *testing.T) { @@ -1379,6 +1380,7 @@ func newFakeTiKVMemberManager(tc *v1alpha1.TidbCluster) ( nodeInformer := kubeinformers.NewSharedInformerFactory(kubeCli, 0).Core().V1().Nodes() tikvScaler := NewFakeTiKVScaler() tikvUpgrader := NewFakeTiKVUpgrader() + genericControl := controller.NewFakeGenericControl() tmm := &tikvMemberManager{ pdControl: pdControl, @@ -1386,6 +1388,7 @@ func newFakeTiKVMemberManager(tc *v1alpha1.TidbCluster) ( nodeLister: nodeInformer.Lister(), setControl: setControl, svcControl: svcControl, + typedControl: controller.NewTypedControl(genericControl), setLister: setInformer.Lister(), svcLister: svcInformer.Lister(), tikvScaler: tikvScaler, @@ -1548,7 +1551,7 @@ func TestGetNewTiKVSetForTidbCluster(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - sts, err := getNewTiKVSetForTidbCluster(&tt.tc) + sts, err := getNewTiKVSetForTidbCluster(&tt.tc, nil) if (err != nil) != tt.wantErr { t.Errorf("error %v, wantErr %v", err, tt.wantErr) } @@ -1804,7 +1807,7 @@ func TestTiKVInitContainers(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - sts, err := getNewTiKVSetForTidbCluster(&tt.tc) + sts, err := getNewTiKVSetForTidbCluster(&tt.tc, nil) if (err != nil) != tt.wantErr { t.Errorf("error %v, wantErr %v", err, tt.wantErr) } @@ -1823,3 +1826,95 @@ func TestTiKVInitContainers(t *testing.T) { }) } } + +func TestGetTiKVConfigMap(t *testing.T) { + g := NewGomegaWithT(t) + testCases := []struct { + name string + tc v1alpha1.TidbCluster + expected *corev1.ConfigMap + }{ + { + name: "TiKV config is nil", + tc: v1alpha1.TidbCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + Namespace: "ns", + }, + }, + expected: nil, + }, + { + name: "basic", + tc: v1alpha1.TidbCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + Namespace: "ns", + }, + Spec: v1alpha1.TidbClusterSpec{ + TiKV: v1alpha1.TiKVSpec{ + ConfigUpdateStrategy: v1alpha1.ConfigUpdateStrategyInPlace, + Config: &v1alpha1.TiKVConfig{ + GrpcKeepaliveTimeout: "30s", + Raftstore: &v1alpha1.TiKVRaftstoreConfig{ + SyncLog: pointer.BoolPtr(false), + RaftBaseTickInterval: "1s", + }, + }, + }, + }, + }, + expected: &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo-tikv", + Namespace: "ns", + Labels: map[string]string{ + "app.kubernetes.io/name": "tidb-cluster", + "app.kubernetes.io/managed-by": "tidb-operator", + "app.kubernetes.io/instance": "", + "app.kubernetes.io/component": "tikv", + }, + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: "pingcap.com/v1alpha1", + Kind: "TidbCluster", + Name: "foo", + UID: "", + Controller: func(b bool) *bool { + return &b + }(true), + BlockOwnerDeletion: func(b bool) *bool { + return &b + }(true), + }, + }, + }, + Data: map[string]string{ + "startup-script": "", + "config-file": `grpc-keepalive-timeout = "30s" + +[raftstore] + sync-log = false + raft-base-tick-interval = "1s" +`, + }, + }, + }, + } + + for _, tt := range testCases { + t.Run(tt.name, func(t *testing.T) { + cm, err := getTikVConfigMap(&tt.tc) + g.Expect(err).To(Succeed()) + if tt.expected == nil { + g.Expect(cm).To(BeNil()) + return + } + // startup-script is better to be tested in e2e + cm.Data["startup-script"] = "" + if diff := cmp.Diff(*tt.expected, *cm); diff != "" { + t.Errorf("unexpected plugin configuration (-want, +got): %s", diff) + } + }) + } +} diff --git a/pkg/manager/member/utils.go b/pkg/manager/member/utils.go index 90d7675a202..08385ca30cc 100644 --- a/pkg/manager/member/utils.go +++ b/pkg/manager/member/utils.go @@ -269,3 +269,13 @@ func Sha256Sum(v interface{}) (string, error) { sum := sha256.Sum256(data) return fmt.Sprintf("%x", sum), nil } + +func AddConfigMapDigestSuffix(cm *corev1.ConfigMap) error { + sum, err := Sha256Sum(cm.Data) + if err != nil { + return err + } + suffix := fmt.Sprintf("%x", sum)[0:7] + cm.Name = fmt.Sprintf("%s-%s", cm.Name, suffix) + return nil +} diff --git a/pkg/pdapi/pdapi_test.go b/pkg/pdapi/pdapi_test.go index 3953073d394..9b96be0115e 100644 --- a/pkg/pdapi/pdapi_test.go +++ b/pkg/pdapi/pdapi_test.go @@ -22,7 +22,6 @@ import ( "net/http" "net/http/httptest" "testing" - "time" . "github.com/onsi/gomega" "github.com/pingcap/kvproto/pkg/metapb" @@ -84,7 +83,7 @@ func TestGetConfig(t *testing.T) { g := NewGomegaWithT(t) config := &v1alpha1.PDConfig{ Schedule: &v1alpha1.PDScheduleConfig{ - MaxStoreDownTime: v1alpha1.NewDuration(10 * time.Second), + MaxStoreDownTime: "10s", }, } configBytes, err := json.Marshal(config) diff --git a/tests/e2e/tidbcluster/tidbcluster.go b/tests/e2e/tidbcluster/tidbcluster.go index 752a25563c6..f60c3098f47 100644 --- a/tests/e2e/tidbcluster/tidbcluster.go +++ b/tests/e2e/tidbcluster/tidbcluster.go @@ -481,29 +481,49 @@ var _ = ginkgo.Describe("[tidb-operator] TiDBCluster", func() { tc, err := cli.PingcapV1alpha1().TidbClusters(cluster.Namespace).Get(cluster.ClusterName, metav1.GetOptions{}) framework.ExpectNoError(err, "Expected get tidbcluster") - tidbSetName := controller.TiDBMemberName(tc.Name) - oldTiDBSet, err := c.AppsV1().StatefulSets(tc.Namespace).Get(tidbSetName, metav1.GetOptions{}) - framework.ExpectNoError(err, "Expected get TiDB statefulset") + // TODO: modify other cases to manage TiDB configmap in CRD by default + ginkgo.By("Test managing configmap in TidbCluster CRD") + setNameToRevision := map[string]string{ + controller.PDMemberName(tc.Name): "", + controller.TiKVMemberName(tc.Name): "", + controller.TiDBMemberName(tc.Name): "", + } - oldRev := oldTiDBSet.Status.CurrentRevision - framework.ExpectEqual(oldTiDBSet.Status.UpdateRevision, oldRev, "Expected tidb is not upgrading") + for setName := range setNameToRevision { + oldSet, err := c.AppsV1().StatefulSets(tc.Namespace).Get(setName, metav1.GetOptions{}) + framework.ExpectNoError(err, "Expected get statefulset %s", setName) - // TODO: modify other cases to manage TiDB configmap in CRD by default - ginkgo.By("Test managing TiDB configmap in TidbCluster CRD") + oldRev := oldSet.Status.CurrentRevision + framework.ExpectEqual(oldSet.Status.UpdateRevision, oldRev, "Expected statefulset %s is not upgrading", setName) + + setNameToRevision[setName] = oldRev + } + + tc, err = cli.PingcapV1alpha1().TidbClusters(cluster.Namespace).Get(cluster.ClusterName, metav1.GetOptions{}) + framework.ExpectNoError(err, "Expected get tidbcluster") tc.Spec.TiDB.Config = &v1alpha1.TiDBConfig{} tc.Spec.TiDB.ConfigUpdateStrategy = v1alpha1.ConfigUpdateStrategyInPlace - + tc.Spec.TiKV.Config = &v1alpha1.TiKVConfig{} + tc.Spec.TiKV.ConfigUpdateStrategy = v1alpha1.ConfigUpdateStrategyInPlace + tc.Spec.PD.Config = &v1alpha1.PDConfig{} + tc.Spec.PD.ConfigUpdateStrategy = v1alpha1.ConfigUpdateStrategyInPlace _, err = cli.PingcapV1alpha1().TidbClusters(tc.Namespace).Update(tc) framework.ExpectNoError(err, "Expected update tidbcluster") // check for 2 minutes to ensure the tidb statefulset do not get rolling-update - err = wait.PollImmediate(5*time.Second, 2*time.Minute, func() (bool, error) { - tidbSet, err := c.AppsV1().StatefulSets(tc.Namespace).Get(tidbSetName, metav1.GetOptions{}) - if err != nil { - return false, err + err = wait.PollImmediate(10*time.Second, 2*time.Minute, func() (bool, error) { + tc, err := cli.PingcapV1alpha1().TidbClusters(cluster.Namespace).Get(cluster.ClusterName, metav1.GetOptions{}) + framework.ExpectNoError(err, "Expected get tidbcluster") + framework.ExpectEqual(tc.Status.PD.Phase, v1alpha1.NormalPhase, "PD should not be updated") + framework.ExpectEqual(tc.Status.TiKV.Phase, v1alpha1.NormalPhase, "TiKV should not be updated") + framework.ExpectEqual(tc.Status.TiDB.Phase, v1alpha1.NormalPhase, "TiDB should not be updated") + + for setName, oldRev := range setNameToRevision { + newSet, err := c.AppsV1().StatefulSets(tc.Namespace).Get(setName, metav1.GetOptions{}) + framework.ExpectNoError(err, "Expected get tidb statefulset") + framework.ExpectEqual(newSet.Status.CurrentRevision, oldRev, "Expected no rolling-update of %s when manage config in-place", setName) + framework.ExpectEqual(newSet.Status.UpdateRevision, oldRev, "Expected no rolling-update of %s when manage config in-place", setName) } - framework.ExpectEqual(tidbSet.Status.CurrentRevision, oldRev, "Expected no rolling-update when manage config in-place") - framework.ExpectEqual(tidbSet.Status.UpdateRevision, oldRev, "Expected no rolling-update when manage config in-place") return false, nil }) @@ -512,23 +532,25 @@ var _ = ginkgo.Describe("[tidb-operator] TiDBCluster", func() { } err = wait.PollImmediate(5*time.Second, 3*time.Minute, func() (bool, error) { - tidbSet, err := c.AppsV1().StatefulSets(tc.Namespace).Get(tidbSetName, metav1.GetOptions{}) - if err != nil { - return false, err - } - usingName := member.FindConfigMapVolume(&tidbSet.Spec.Template.Spec, func(name string) bool { - return strings.HasPrefix(name, controller.TiDBMemberName(tc.Name)) - }) - if usingName == "" { - e2elog.Fail("cannot find configmap that used by TiDB statefulset") - } - tidbCm, err := c.CoreV1().ConfigMaps(tc.Namespace).Get(usingName, metav1.GetOptions{}) - if err != nil { - return false, err - } - if !metav1.IsControlledBy(tidbCm, tc) { - e2elog.Logf("expect tidb configmap adopted by tidbcluster, still waiting...") - return false, nil + for setName := range setNameToRevision { + newSet, err := c.AppsV1().StatefulSets(tc.Namespace).Get(setName, metav1.GetOptions{}) + if err != nil { + return false, err + } + usingName := member.FindConfigMapVolume(&newSet.Spec.Template.Spec, func(name string) bool { + return strings.HasPrefix(name, setName) + }) + if usingName == "" { + e2elog.Failf("cannot find configmap that used by %s", setName) + } + usingCm, err := c.CoreV1().ConfigMaps(tc.Namespace).Get(usingName, metav1.GetOptions{}) + if err != nil { + return false, err + } + if !metav1.IsControlledBy(usingCm, tc) { + e2elog.Logf("expect configmap of %s adopted by tidbcluster, still waiting...", setName) + return false, nil + } } return true, nil }) diff --git a/tests/failover.go b/tests/failover.go index 4507b197eaa..cf6a2978bae 100644 --- a/tests/failover.go +++ b/tests/failover.go @@ -43,7 +43,10 @@ func (oa *operatorActions) TruncateSSTFileThenCheckFailover(info *TidbClusterCon glog.Errorf("failed to get the pd config: tc=%s err=%s", info.ClusterName, err.Error()) return err } - maxStoreDownTime := pdCfg.Schedule.MaxStoreDownTime.Duration + maxStoreDownTime, err := time.ParseDuration(pdCfg.Schedule.MaxStoreDownTime) + if err != nil { + return err + } glog.Infof("truncate sst file failover config: maxStoreDownTime=%v tikvFailoverPeriod=%v", maxStoreDownTime, tikvFailoverPeriod) // find an up store