From d61af180dbe7e3ff3bf3158f47f950ced581bf08 Mon Sep 17 00:00:00 2001 From: shuijing198799 Date: Thu, 9 May 2019 11:49:07 +0800 Subject: [PATCH 01/16] add webhook and clietn --- Makefile | 5 +- cmd/webhook/main.go | 84 ++++++++++ images/tidb-operator/Dockerfile | 1 + manifests/webhook-rbac.yaml | 18 +++ manifests/webhook.yaml | 44 ++++++ pkg/label/label.go | 2 + pkg/tkctl/cmd/cmd.go | 2 + pkg/tkctl/cmd/upinfo/upinfo.go | 206 +++++++++++++++++++++++++ pkg/webhook/route/route.go | 86 +++++++++++ pkg/webhook/statefulset/statefulset.go | 81 ++++++++++ pkg/webhook/util/certs.go | 98 ++++++++++++ pkg/webhook/util/scheme.go | 39 +++++ pkg/webhook/util/util.go | 76 +++++++++ pkg/webhook/webhook.go | 126 +++++++++++++++ 14 files changed, 867 insertions(+), 1 deletion(-) create mode 100644 cmd/webhook/main.go create mode 100644 manifests/webhook-rbac.yaml create mode 100644 manifests/webhook.yaml create mode 100644 pkg/tkctl/cmd/upinfo/upinfo.go create mode 100644 pkg/webhook/route/route.go create mode 100644 pkg/webhook/statefulset/statefulset.go create mode 100644 pkg/webhook/util/certs.go create mode 100644 pkg/webhook/util/scheme.go create mode 100644 pkg/webhook/util/util.go create mode 100644 pkg/webhook/webhook.go diff --git a/Makefile b/Makefile index ff399bb8db5..21be7adfffe 100644 --- a/Makefile +++ b/Makefile @@ -29,7 +29,7 @@ docker-push: docker docker: build docker build --tag "${DOCKER_REGISTRY}/pingcap/tidb-operator:latest" images/tidb-operator -build: controller-manager scheduler discovery +build: controller-manager scheduler discovery webhook controller-manager: $(GO) -ldflags '$(LDFLAGS)' -o images/tidb-operator/bin/tidb-controller-manager cmd/controller-manager/main.go @@ -40,6 +40,9 @@ scheduler: discovery: $(GO) -ldflags '$(LDFLAGS)' -o images/tidb-operator/bin/tidb-discovery cmd/discovery/main.go +webhook: + $(GO) -ldflags '$(LDFLAGS)' -o images/tidb-operator/bin/tidb-webhook cmd/webhook/main.go + e2e-setup: # ginkgo doesn't work with retool for Go 1.11 @GO111MODULE=on CGO_ENABLED=0 go get github.com/onsi/ginkgo@v1.6.0 diff --git a/cmd/webhook/main.go b/cmd/webhook/main.go new file mode 100644 index 00000000000..69936553817 --- /dev/null +++ b/cmd/webhook/main.go @@ -0,0 +1,84 @@ +// Copyright 2018 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "os" + "os/signal" + "syscall" + + "github.com/golang/glog" + "github.com/pingcap/tidb-operator/pkg/webhook" + "github.com/pingcap/tidb-operator/pkg/webhook/util" +) + +func main() { + + cli, kubeCli, err := util.GetNewClient() + if err != nil { + glog.Fatalf("failed to get client: %v", err) + } + + ns := os.Getenv("NAMESPACE") + if ns == "" { + glog.Fatalf("fail to get namespace in environment") + } + + svc := os.Getenv("SERVICENAME") + if svc == "" { + glog.Fatalf("fail to get servicename in environment") + } + + // create cert file + cert, err := util.SetupServerCert(ns, svc) + if err != nil { + glog.Fatalf("fail to setup server cert: %v", err) + } + webhookServer := webhook.NewWebHookServer(kubeCli, cli, cert) + + // before start webhook server, create validating-webhook-configuration + err = webhookServer.RegisterWebhook(ns, svc) + if err != nil { + glog.Fatalf("fail to create validaing webhook configuration: %v", err) + } + + sigs := make(chan os.Signal, 1) + done := make(chan bool, 1) + + signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM) + + go func() { + <-sigs + + // FIXME Consider whether delete the configuration when the service is shutdown. + if err := webhookServer.UnregisterWebhook(); err != nil { + glog.Errorf("fail to delete validating configuration %v", err) + } + + // Graceful shutdown the server + if err := webhookServer.Shutdown(); err != nil { + glog.Errorf("fail to shutdown server %v", err) + } + + done <- true + }() + + if err := webhookServer.Run(); err != nil { + glog.Errorf("stop http server %v", err) + } + + <-done + + glog.Infof("webhook server terminate safely.") +} diff --git a/images/tidb-operator/Dockerfile b/images/tidb-operator/Dockerfile index 52679208071..91455a25ea9 100644 --- a/images/tidb-operator/Dockerfile +++ b/images/tidb-operator/Dockerfile @@ -4,3 +4,4 @@ RUN apk add tzdata --no-cache ADD bin/tidb-controller-manager /usr/local/bin/tidb-controller-manager ADD bin/tidb-scheduler /usr/local/bin/tidb-scheduler ADD bin/tidb-discovery /usr/local/bin/tidb-discovery +ADD bin/tidb-webhook /usr/local/bin/tidb-webhook diff --git a/manifests/webhook-rbac.yaml b/manifests/webhook-rbac.yaml new file mode 100644 index 00000000000..c4ed1acee7e --- /dev/null +++ b/manifests/webhook-rbac.yaml @@ -0,0 +1,18 @@ +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: admission-webhook-example-rbac +subjects: +- kind: ServiceAccount + namespace: pingcap + name: admission-webhook-example-sa +roleRef: + kind: ClusterRole + name: cluster-admin + apiGroup: rbac.authorization.k8s.io +--- +kind: ServiceAccount +apiVersion: v1 +metadata: + namespace: pingcap + name: admission-webhook-example-sa diff --git a/manifests/webhook.yaml b/manifests/webhook.yaml new file mode 100644 index 00000000000..026f4b3137d --- /dev/null +++ b/manifests/webhook.yaml @@ -0,0 +1,44 @@ +apiVersion: v1 +kind: Service +metadata: + name: admission-webhook-example-svc + namespace: pingcap + labels: + app: admission-webhook-example +spec: + ports: + - port: 443 + targetPort: 443 + selector: + app: admission-webhook-example +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: admission-webhook-example-deployment + namespace: pingcap + labels: + app: admission-webhook-example +spec: + replicas: 1 + selector: + matchLabels: + app: admission-webhook-example + template: + metadata: + namespace: pingcap + labels: + app: admission-webhook-example + spec: + serviceAccount: admission-webhook-example-sa + containers: + - name: admission-webhook-example + image: hub.pingcap.net/yinliang/pingcap/tidb-operator:latest + imagePullPolicy: Always + command: + - /usr/local/bin/tidb-webhook + env: + - name: NAMESPACE + value: pingcap + - name: SERVICENAME + value: admission-webhook-example-svc diff --git a/pkg/label/label.go b/pkg/label/label.go index e453027d92d..17ea6c6d951 100644 --- a/pkg/label/label.go +++ b/pkg/label/label.go @@ -50,6 +50,8 @@ const ( AnnPVCDeferDeleting = "tidb.pingcap.com/pvc-defer-deleting" // AnnPVCPodScheduling is pod scheduling annotation key, it represents whether the pod is scheduling AnnPVCPodScheduling = "tidb.pingcap.com/pod-scheduling" + // AnnTiDBPartition is pod annotation which TiDB pod chould upgrade to + AnnTiDBPartition string = "tidb.pingcap.com/tidb-partition" // PDLabelVal is PD label value PDLabelVal string = "pd" diff --git a/pkg/tkctl/cmd/cmd.go b/pkg/tkctl/cmd/cmd.go index 9db7ac5f2de..ac3f858a2cc 100644 --- a/pkg/tkctl/cmd/cmd.go +++ b/pkg/tkctl/cmd/cmd.go @@ -24,6 +24,7 @@ import ( "github.com/pingcap/tidb-operator/pkg/tkctl/cmd/get" "github.com/pingcap/tidb-operator/pkg/tkctl/cmd/info" "github.com/pingcap/tidb-operator/pkg/tkctl/cmd/list" + "github.com/pingcap/tidb-operator/pkg/tkctl/cmd/upinfo" "github.com/pingcap/tidb-operator/pkg/tkctl/cmd/use" "github.com/pingcap/tidb-operator/pkg/tkctl/config" "github.com/spf13/cobra" @@ -69,6 +70,7 @@ func NewTkcCommand(streams genericclioptions.IOStreams) *cobra.Command { info.NewCmdInfo(tkcContext, streams), use.NewCmdUse(tkcContext, streams), version.NewCmdVersion(tkcContext, streams.Out), + upinfo.NewCmdUpInfo(tkcContext, streams), }, }, { diff --git a/pkg/tkctl/cmd/upinfo/upinfo.go b/pkg/tkctl/cmd/upinfo/upinfo.go new file mode 100644 index 00000000000..912c46863e8 --- /dev/null +++ b/pkg/tkctl/cmd/upinfo/upinfo.go @@ -0,0 +1,206 @@ +// Copyright 2019. PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package upinfo + +import ( + "fmt" + "github.com/pingcap/tidb-operator/pkg/apis/pingcap.com/v1alpha1" + "github.com/pingcap/tidb-operator/pkg/client/clientset/versioned" + "github.com/pingcap/tidb-operator/pkg/controller" + "github.com/pingcap/tidb-operator/pkg/label" + "github.com/pingcap/tidb-operator/pkg/tkctl/config" + "github.com/pingcap/tidb-operator/pkg/tkctl/readable" + "github.com/pingcap/tidb-operator/pkg/util" + "github.com/spf13/cobra" + "io" + apps "k8s.io/api/apps/v1beta1" + "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/cli-runtime/pkg/genericclioptions" + "k8s.io/client-go/kubernetes" + cmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util" +) + +const ( + upinfoLongDesc = ` + Get tidb cluster component upgrade info. + + You can omit --tidbcluster= option by running 'tkc use ', +` + upinfoExample = ` + # get current tidb cluster info (set by tkc use) + tkc upinfo + + # get specified tidb cluster component upgrade info + tkc upinfo -t another-cluster +` + infoUsage = `expected 'upinfo -t CLUSTER_NAME' for the upinfo command or +using 'tkc use' to set tidb cluster first. +` + UPDATED = "updated" + UPDATING = "updating" + WAITING = "waiting" +) + +// UpInfoOptions contains the input to the list command. +type UpInfoOptions struct { + TidbClusterName string + Namespace string + + TcCli *versioned.Clientset + KubeCli *kubernetes.Clientset + + genericclioptions.IOStreams +} + +// NewUpInfoOptions returns a UpInfoOptions +func NewUpInfoOptions(streams genericclioptions.IOStreams) *UpInfoOptions { + return &UpInfoOptions{ + IOStreams: streams, + } +} + +// NewCmdUpInfo creates the upinfo command which show the tidb cluster upgrade detail information +func NewCmdUpInfo(tkcContext *config.TkcContext, streams genericclioptions.IOStreams) *cobra.Command { + o := NewUpInfoOptions(streams) + + cmd := &cobra.Command{ + Use: "upinfo", + Short: "Show tidb upgrade info.", + Example: upinfoExample, + Long: upinfoLongDesc, + Run: func(cmd *cobra.Command, args []string) { + cmdutil.CheckErr(o.Complete(tkcContext, cmd, args)) + cmdutil.CheckErr(o.Run()) + }, + SuggestFor: []string{"updateinfo", "upgradeinfo"}, + } + + return cmd +} + +func (o *UpInfoOptions) Complete(tkcContext *config.TkcContext, cmd *cobra.Command, args []string) error { + + clientConfig, err := tkcContext.ToTkcClientConfig() + if err != nil { + return err + } + + if tidbClusterName, ok := clientConfig.TidbClusterName(); ok { + o.TidbClusterName = tidbClusterName + } else { + return cmdutil.UsageErrorf(cmd, infoUsage) + } + + namespace, _, err := clientConfig.Namespace() + if err != nil { + return err + } + o.Namespace = namespace + + restConfig, err := clientConfig.RestConfig() + if err != nil { + return err + } + tcCli, err := versioned.NewForConfig(restConfig) + if err != nil { + return err + } + o.TcCli = tcCli + kubeCli, err := kubernetes.NewForConfig(restConfig) + if err != nil { + return err + } + o.KubeCli = kubeCli + + return nil +} + +func (o *UpInfoOptions) Run() error { + + tc, err := o.TcCli.PingcapV1alpha1(). + TidbClusters(o.Namespace). + Get(o.TidbClusterName, metav1.GetOptions{}) + if err != nil { + return err + } + setName := controller.TiDBMemberName(tc.Name) + set, err := o.KubeCli.AppsV1beta1().StatefulSets(o.Namespace).Get(setName, metav1.GetOptions{}) + if err != nil { + return err + } + podList, err := o.KubeCli.CoreV1().Pods(o.Namespace).List(metav1.ListOptions{ + LabelSelector: fmt.Sprintf("%s=%s,%s=%s", label.InstanceLabelKey, tc.Name, label.ComponentLabelKey, "tidb"), + }) + if err != nil { + return err + } + msg, err := renderTCUpgradeInfo(tc, set, podList) + if err != nil { + return err + } + fmt.Fprint(o.Out, msg) + return nil +} + +func renderTCUpgradeInfo(tc *v1alpha1.TidbCluster, set *apps.StatefulSet, podList *v1.PodList) (string, error) { + return readable.TabbedString(func(out io.Writer) error { + w := readable.NewPrefixWriter(out) + dbPhase := tc.Status.TiDB.Phase + w.WriteLine(readable.LEVEL_0, "Name:\t%s", tc.Name) + w.WriteLine(readable.LEVEL_0, "Namespace:\t%s", tc.Namespace) + w.WriteLine(readable.LEVEL_0, "CreationTimestamp:\t%s", tc.CreationTimestamp) + w.WriteLine(readable.LEVEL_0, "Statu:\t%s", dbPhase) + { + w.WriteLine(readable.LEVEL_1, "Name\tState\t") + w.WriteLine(readable.LEVEL_1, "----\t-----\t") + { + updateReplicas := set.Spec.UpdateStrategy.RollingUpdate.Partition + + if len(podList.Items) != 0 { + for _, pod := range podList.Items { + var state string + ordinal, err := util.GetOrdinalFromPodName(pod.Name) + if err != nil { + return err + } + if dbPhase == v1alpha1.UpgradePhase { + if (*updateReplicas) < ordinal { + state = UPDATED + } else if (*updateReplicas) == ordinal { + + state = UPDATING + + if pod.Labels[apps.ControllerRevisionHashLabelKey] == tc.Status.TiDB.StatefulSet.UpdateRevision { + if member, exist := tc.Status.TiDB.Members[pod.Name]; exist && member.Health { + state = UPDATED + } + } + + } else { + state = WAITING + } + } else { + state = UPDATED + } + w.WriteLine(readable.LEVEL_1, "%s\t%s\t", pod.Name, state) + } + } else { + w.WriteLine(readable.LEVEL_1, "no resource found") + } + } + } + return nil + }) +} diff --git a/pkg/webhook/route/route.go b/pkg/webhook/route/route.go new file mode 100644 index 00000000000..320fa11a261 --- /dev/null +++ b/pkg/webhook/route/route.go @@ -0,0 +1,86 @@ +// Copyright 2018 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package route + +import ( + "encoding/json" + "errors" + "io/ioutil" + "net/http" + + "github.com/golang/glog" + "github.com/pingcap/tidb-operator/pkg/webhook/statefulset" + "github.com/pingcap/tidb-operator/pkg/webhook/util" + "k8s.io/api/admission/v1beta1" +) + +// admitFunc is the type we use for all of our validators +type admitFunc func(v1beta1.AdmissionReview) *v1beta1.AdmissionResponse + +// serve handles the http portion of a request prior to handing to an admit +// function +func serve(w http.ResponseWriter, r *http.Request, admit admitFunc) { + + var body []byte + var contentType string + responseAdmissionReview := v1beta1.AdmissionReview{} + requestedAdmissionReview := v1beta1.AdmissionReview{} + deserializer := util.GetCodec() + + // The AdmissionReview that will be returned + if r.Body != nil { + if data, err := ioutil.ReadAll(r.Body); err == nil { + body = data + } else { + responseAdmissionReview.Response = util.ARFail(err) + goto returnData + } + } else { + err := errors.New("request body is nil!") + responseAdmissionReview.Response = util.ARFail(err) + goto returnData + } + + // verify the content type is accurate + contentType = r.Header.Get("Content-Type") + if contentType != "application/json" { + err := errors.New("expect application/json") + responseAdmissionReview.Response = util.ARFail(err) + goto returnData + } + + // The AdmissionReview that was sent to the webhook + if _, _, err := deserializer.Decode(body, nil, &requestedAdmissionReview); err != nil { + responseAdmissionReview.Response = util.ARFail(err) + } else { + // pass to admitFunc + responseAdmissionReview.Response = admit(requestedAdmissionReview) + } + + // Return the same UID + responseAdmissionReview.Response.UID = requestedAdmissionReview.Request.UID + +returnData: + respBytes, err := json.Marshal(responseAdmissionReview) + if err != nil { + glog.Errorf("%v", err) + } + if _, err := w.Write(respBytes); err != nil { + glog.Errorf("%v", err) + } +} + +func ServeStatefulSets(w http.ResponseWriter, r *http.Request) { + serve(w, r, statefulset.AdmitStatefulSets) +} diff --git a/pkg/webhook/statefulset/statefulset.go b/pkg/webhook/statefulset/statefulset.go new file mode 100644 index 00000000000..e82122c1b39 --- /dev/null +++ b/pkg/webhook/statefulset/statefulset.go @@ -0,0 +1,81 @@ +// Copyright 2018 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package statefulset + +import ( + "errors" + "fmt" + "strconv" + + "github.com/golang/glog" + "github.com/pingcap/tidb-operator/pkg/apis/pingcap.com/v1alpha1" + "github.com/pingcap/tidb-operator/pkg/label" + "github.com/pingcap/tidb-operator/pkg/webhook/util" + "k8s.io/api/admission/v1beta1" + apps "k8s.io/api/apps/v1beta1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func AdmitStatefulSets(ar v1beta1.AdmissionReview) *v1beta1.AdmissionResponse { + glog.Infof("admit statefulsets") + + setResource := metav1.GroupVersionResource{Group: "apps", Version: "v1beta1", Resource: "statefulsets"} + if ar.Request.Resource != setResource { + err := fmt.Errorf("expect resource to be %s", setResource) + glog.Errorf("%v", err) + return util.ARFail(err) + } + + cli, _, err := util.GetNewClient() + if err != nil { + glog.Errorf("failed to get kubernetes Clientset: %v", err) + return util.ARFail(err) + } + + name := ar.Request.Name + namespace := ar.Request.Namespace + + raw := ar.Request.OldObject.Raw + set := apps.StatefulSet{} + deserializer := util.GetCodec() + if _, _, err := deserializer.Decode(raw, nil, &set); err != nil { + glog.Error(err) + return util.ARFail(err) + } + + tc, err := cli.PingcapV1alpha1().TidbClusters(namespace).Get(set.Labels[label.InstanceLabelKey], metav1.GetOptions{}) + if err != nil { + glog.Errorf("fail to fetch tidbcluster info namespace %s clustername(instance) %s err %v", namespace, set.Labels[label.InstanceLabelKey], err) + return util.ARFail(err) + } + + if set.Labels[label.ComponentLabelKey] == "tidb" { + protect, ok := tc.Annotations[label.AnnTiDBPartition] + + if ok { + partition, err := strconv.ParseInt(protect, 10, 32) + if err != nil { + glog.Errorf("fail to convert protect to int namespace %s name %s err %v", namespace, name, err) + return util.ARFail(err) + } + + if (*set.Spec.UpdateStrategy.RollingUpdate.Partition) <= int32(partition) && tc.Status.TiDB.Phase == v1alpha1.UpgradePhase { + glog.Infof("set has been protect by annotations name %s namespace %s", name, namespace) + return util.ARFail(errors.New("protect by annotation")) + } + } + } + + return util.ARSuccess() +} diff --git a/pkg/webhook/util/certs.go b/pkg/webhook/util/certs.go new file mode 100644 index 00000000000..fd8028cd1f4 --- /dev/null +++ b/pkg/webhook/util/certs.go @@ -0,0 +1,98 @@ +// Copyright 2019 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package util + +import ( + "crypto/x509" + "io/ioutil" + "os" + + "github.com/golang/glog" + "k8s.io/client-go/util/cert" +) + +type CertContext struct { + Cert []byte + Key []byte + SigningCert []byte +} + +// Setup the server cert. For example, user apiservers and admission webhooks +// can use the cert to prove their identify to the kube-apiserver +func SetupServerCert(namespaceName, serviceName string) (*CertContext, error) { + certDir, err := ioutil.TempDir("", "create-server-cert") + if err != nil { + glog.Errorf("Failed to create a temp dir for cert generation %v", err) + return nil, err + } + defer os.RemoveAll(certDir) + signingKey, err := cert.NewPrivateKey() + if err != nil { + glog.Errorf("Failed to create CA private key %v", err) + return nil, err + } + signingCert, err := cert.NewSelfSignedCACert(cert.Config{CommonName: "e2e-server-cert-ca"}, signingKey) + if err != nil { + glog.Errorf("Failed to create CA cert for apiserver %v", err) + return nil, err + } + caCertFile, err := ioutil.TempFile(certDir, "ca.crt") + if err != nil { + glog.Errorf("Failed to create a temp file for ca cert generation %v", err) + return nil, err + } + if err := ioutil.WriteFile(caCertFile.Name(), cert.EncodeCertPEM(signingCert), 0644); err != nil { + glog.Errorf("Failed to write CA cert %v", err) + return nil, err + } + key, err := cert.NewPrivateKey() + if err != nil { + glog.Errorf("Failed to create private key for %v", err) + return nil, err + } + signedCert, err := cert.NewSignedCert( + cert.Config{ + CommonName: serviceName + "." + namespaceName + ".svc", + Usages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + }, + key, signingCert, signingKey, + ) + if err != nil { + glog.Errorf("Failed to create cert%v", err) + return nil, err + } + certFile, err := ioutil.TempFile(certDir, "server.crt") + if err != nil { + glog.Errorf("Failed to create a temp file for cert generation %v", err) + return nil, err + } + keyFile, err := ioutil.TempFile(certDir, "server.key") + if err != nil { + glog.Errorf("Failed to create a temp file for key generation %v", err) + return nil, err + } + if err = ioutil.WriteFile(certFile.Name(), cert.EncodeCertPEM(signedCert), 0600); err != nil { + glog.Errorf("Failed to write cert file %v", err) + return nil, err + } + if err = ioutil.WriteFile(keyFile.Name(), cert.EncodePrivateKeyPEM(key), 0644); err != nil { + glog.Errorf("Failed to write key file %v", err) + return nil, err + } + return &CertContext{ + Cert: cert.EncodeCertPEM(signedCert), + Key: cert.EncodePrivateKeyPEM(key), + SigningCert: cert.EncodeCertPEM(signingCert), + }, nil +} diff --git a/pkg/webhook/util/scheme.go b/pkg/webhook/util/scheme.go new file mode 100644 index 00000000000..d655a35940b --- /dev/null +++ b/pkg/webhook/util/scheme.go @@ -0,0 +1,39 @@ +// Copyright 2019 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package util + +import ( + admissionv1beta1 "k8s.io/api/admission/v1beta1" + admissionregistrationv1beta1 "k8s.io/api/admissionregistration/v1beta1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/serializer" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" +) + +var scheme = runtime.NewScheme() + +func init() { + addToScheme(scheme) +} + +func addToScheme(scheme *runtime.Scheme) { + utilruntime.Must(corev1.AddToScheme(scheme)) + utilruntime.Must(admissionv1beta1.AddToScheme(scheme)) + utilruntime.Must(admissionregistrationv1beta1.AddToScheme(scheme)) +} + +func GetCodec() runtime.Decoder { + return serializer.NewCodecFactory(scheme).UniversalDeserializer() +} diff --git a/pkg/webhook/util/util.go b/pkg/webhook/util/util.go new file mode 100644 index 00000000000..dcb42100561 --- /dev/null +++ b/pkg/webhook/util/util.go @@ -0,0 +1,76 @@ +// Copyright 2019 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package util + +import ( + "crypto/tls" + + "github.com/golang/glog" + "github.com/pingcap/tidb-operator/pkg/client/clientset/versioned" + "k8s.io/api/admission/v1beta1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" +) + +func GetNewClient() (versioned.Interface, kubernetes.Interface, error) { + cfg, err := rest.InClusterConfig() + if err != nil { + glog.Errorf("failed to get config: %v", err) + return nil, nil, err + } + + cli, err := versioned.NewForConfig(cfg) + if err != nil { + glog.Errorf("failed to create Clientset: %v", err) + return nil, nil, err + } + + kubeCli, err := kubernetes.NewForConfig(cfg) + if err != nil { + glog.Errorf("failed to get kubernetes Clientset: %v", err) + return nil, nil, err + } + return cli, kubeCli, nil +} + +// toAdmissionResponse is a helper function to create an AdmissionResponse +// with an embedded error +func ARFail(err error) *v1beta1.AdmissionResponse { + return &v1beta1.AdmissionResponse{ + Allowed: false, + Result: &metav1.Status{ + Message: err.Error(), + Reason: metav1.StatusReasonNotAcceptable, + }, + } +} + +// toAdmissionResponse return allow to action +func ARSuccess() *v1beta1.AdmissionResponse { + return &v1beta1.AdmissionResponse{ + Allowed: true, + } +} + +// config tls cert for server +func ConfigTLS(cert []byte, key []byte) (*tls.Config, error) { + sCert, err := tls.X509KeyPair(cert, key) + if err != nil { + return nil, err + } + return &tls.Config{ + Certificates: []tls.Certificate{sCert}, + }, nil +} diff --git a/pkg/webhook/webhook.go b/pkg/webhook/webhook.go new file mode 100644 index 00000000000..b3cb904273a --- /dev/null +++ b/pkg/webhook/webhook.go @@ -0,0 +1,126 @@ +// Copyright 2019 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package webhook + +import ( + "net/http" + "time" + + "github.com/golang/glog" + "github.com/pingcap/tidb-operator/pkg/client/clientset/versioned" + "github.com/pingcap/tidb-operator/pkg/webhook/route" + "github.com/pingcap/tidb-operator/pkg/webhook/util" + admissionV1beta1 "k8s.io/api/admissionregistration/v1beta1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" +) + +type WebhookServer struct { + // kubernetes client interface + KubeCli kubernetes.Interface + // operator client interface + Cli versioned.Interface + // cert context ca.crt key server.crt + Context *util.CertContext + // configuration name + ConfigName string + // http server + Server *http.Server +} + +func NewWebHookServer(kubecli kubernetes.Interface, cli versioned.Interface, context *util.CertContext) *WebhookServer { + + http.HandleFunc("/statefulsets", route.ServeStatefulSets) + + sCert, err := util.ConfigTLS(context.Cert, context.Key) + + if err != nil { + glog.Fatalf("failed to create scert file %v", err) + } + + server := &http.Server{ + Addr: ":443", + TLSConfig: sCert, + } + + return &WebhookServer{ + KubeCli: kubecli, + Cli: cli, + Context: context, + ConfigName: "validating-webhook-configuration", + Server: server, + } +} + +func (ws *WebhookServer) Run() error { + return ws.Server.ListenAndServeTLS("", "") +} + +func (ws *WebhookServer) Shutdown() error { + return ws.Server.Shutdown(nil) +} + +func strPtr(s string) *string { return &s } + +func (ws *WebhookServer) RegisterWebhook(namespace string, svcName string) error { + + policyFail := admissionV1beta1.Fail + + _, err := ws.KubeCli.AdmissionregistrationV1beta1().ValidatingWebhookConfigurations().Create(&admissionV1beta1.ValidatingWebhookConfiguration{ + ObjectMeta: metav1.ObjectMeta{ + Name: ws.ConfigName, + }, + Webhooks: []admissionV1beta1.Webhook{ + { + Name: "admit-statefulset-webhook.k8s.io", + Rules: []admissionV1beta1.RuleWithOperations{{ + Operations: []admissionV1beta1.OperationType{admissionV1beta1.Update}, + Rule: admissionV1beta1.Rule{ + APIGroups: []string{"apps"}, + APIVersions: []string{"v1beta1"}, + Resources: []string{"statefulsets"}, + }, + }}, + ClientConfig: admissionV1beta1.WebhookClientConfig{ + Service: &admissionV1beta1.ServiceReference{ + Namespace: namespace, + Name: svcName, + Path: strPtr("/statefulsets"), + }, + CABundle: ws.Context.SigningCert, + }, + FailurePolicy: &policyFail, + }, + }, + }) + + if err != nil { + glog.Errorf("registering webhook config %s with namespace %s error %v", ws.ConfigName, namespace, err) + return err + } + + // The webhook configuration is honored in 10s. + time.Sleep(10 * time.Second) + + return nil +} + +func (ws *WebhookServer) UnregisterWebhook() error { + err := ws.KubeCli.AdmissionregistrationV1beta1().ValidatingWebhookConfigurations().Delete(ws.ConfigName, nil) + if err != nil && !errors.IsNotFound(err) { + glog.Errorf("failed to delete webhook config %v", err) + } + return nil +} From 9a3b917850b7c287c56bb038e0f5301f3117a73f Mon Sep 17 00:00:00 2001 From: shuijing198799 Date: Fri, 10 May 2019 20:17:32 +0800 Subject: [PATCH 02/16] make change via comments --- Makefile | 6 +- cmd/{webhook => admission-controller}/main.go | 58 ++++---- images/tidb-operator/Dockerfile | 2 +- manifests/create-cert.sh | 130 ++++++++++++++++++ manifests/patch-ca.sh | 10 ++ manifests/webhook-rbac.yaml | 18 --- manifests/webhook.yaml | 102 +++++++++++--- pkg/tkctl/cmd/upinfo/upinfo.go | 11 +- pkg/webhook/route/route.go | 33 +++-- pkg/webhook/statefulset/statefulset.go | 36 +++-- pkg/webhook/util/certs.go | 98 ------------- pkg/webhook/util/util.go | 29 +--- pkg/webhook/webhook.go | 73 +--------- 13 files changed, 321 insertions(+), 285 deletions(-) rename cmd/{webhook => admission-controller}/main.go (50%) create mode 100755 manifests/create-cert.sh create mode 100755 manifests/patch-ca.sh delete mode 100644 manifests/webhook-rbac.yaml delete mode 100644 pkg/webhook/util/certs.go diff --git a/Makefile b/Makefile index 21be7adfffe..f8493fa6c17 100644 --- a/Makefile +++ b/Makefile @@ -29,7 +29,7 @@ docker-push: docker docker: build docker build --tag "${DOCKER_REGISTRY}/pingcap/tidb-operator:latest" images/tidb-operator -build: controller-manager scheduler discovery webhook +build: controller-manager scheduler discovery admission-controller controller-manager: $(GO) -ldflags '$(LDFLAGS)' -o images/tidb-operator/bin/tidb-controller-manager cmd/controller-manager/main.go @@ -40,8 +40,8 @@ scheduler: discovery: $(GO) -ldflags '$(LDFLAGS)' -o images/tidb-operator/bin/tidb-discovery cmd/discovery/main.go -webhook: - $(GO) -ldflags '$(LDFLAGS)' -o images/tidb-operator/bin/tidb-webhook cmd/webhook/main.go +admission-controller: + $(GO) -ldflags '$(LDFLAGS)' -o images/tidb-operator/bin/tidb-admission-controller cmd/admission-controller/main.go e2e-setup: # ginkgo doesn't work with retool for Go 1.11 diff --git a/cmd/webhook/main.go b/cmd/admission-controller/main.go similarity index 50% rename from cmd/webhook/main.go rename to cmd/admission-controller/main.go index 69936553817..bbe3c4ebd1d 100644 --- a/cmd/webhook/main.go +++ b/cmd/admission-controller/main.go @@ -1,4 +1,4 @@ -// Copyright 2018 PingCAP, Inc. +// Copyright 2019 PingCAP, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -14,45 +14,58 @@ package main import ( + "flag" "os" "os/signal" "syscall" "github.com/golang/glog" + "github.com/pingcap/tidb-operator/pkg/client/clientset/versioned" "github.com/pingcap/tidb-operator/pkg/webhook" - "github.com/pingcap/tidb-operator/pkg/webhook/util" + "github.com/pingcap/tidb-operator/version" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" ) -func main() { +var ( + printVersion bool + certFile string + keyFile string +) - cli, kubeCli, err := util.GetNewClient() - if err != nil { - glog.Fatalf("failed to get client: %v", err) - } +func init() { + flag.BoolVar(&printVersion, "V", false, "Show version and quit") + flag.BoolVar(&printVersion, "version", false, "Show version and quit") + flag.StringVar(&certFile, "tlsCertFile", "/etc/webhook/certs/cert.pem", "File containing the x509 Certificate for HTTPS.") + flag.StringVar(&keyFile, "tlsKeyFile", "/etc/webhook/certs/key.pem", "File containing the x509 private key to --tlsCertFile.") + flag.Parse() +} - ns := os.Getenv("NAMESPACE") - if ns == "" { - glog.Fatalf("fail to get namespace in environment") +func main() { + + if printVersion { + version.PrintVersionInfo() + os.Exit(0) } + version.LogVersionInfo() - svc := os.Getenv("SERVICENAME") - if svc == "" { - glog.Fatalf("fail to get servicename in environment") + cfg, err := rest.InClusterConfig() + if err != nil { + glog.Fatalf("failed to get config: %v", err) } - // create cert file - cert, err := util.SetupServerCert(ns, svc) + cli, err := versioned.NewForConfig(cfg) if err != nil { - glog.Fatalf("fail to setup server cert: %v", err) + glog.Fatalf("failed to create Clientset: %v", err) } - webhookServer := webhook.NewWebHookServer(kubeCli, cli, cert) - // before start webhook server, create validating-webhook-configuration - err = webhookServer.RegisterWebhook(ns, svc) + kubeCli, err := kubernetes.NewForConfig(cfg) if err != nil { - glog.Fatalf("fail to create validaing webhook configuration: %v", err) + glog.Fatalf("failed to get kubernetes Clientset: %v", err) } + webhookServer := webhook.NewWebHookServer(kubeCli, cli, certFile, keyFile) + sigs := make(chan os.Signal, 1) done := make(chan bool, 1) @@ -61,11 +74,6 @@ func main() { go func() { <-sigs - // FIXME Consider whether delete the configuration when the service is shutdown. - if err := webhookServer.UnregisterWebhook(); err != nil { - glog.Errorf("fail to delete validating configuration %v", err) - } - // Graceful shutdown the server if err := webhookServer.Shutdown(); err != nil { glog.Errorf("fail to shutdown server %v", err) diff --git a/images/tidb-operator/Dockerfile b/images/tidb-operator/Dockerfile index 91455a25ea9..a812129135b 100644 --- a/images/tidb-operator/Dockerfile +++ b/images/tidb-operator/Dockerfile @@ -4,4 +4,4 @@ RUN apk add tzdata --no-cache ADD bin/tidb-controller-manager /usr/local/bin/tidb-controller-manager ADD bin/tidb-scheduler /usr/local/bin/tidb-scheduler ADD bin/tidb-discovery /usr/local/bin/tidb-discovery -ADD bin/tidb-webhook /usr/local/bin/tidb-webhook +ADD bin/tidb-webhook /usr/local/bin/tidb-admission-controller diff --git a/manifests/create-cert.sh b/manifests/create-cert.sh new file mode 100755 index 00000000000..7540f20d0ba --- /dev/null +++ b/manifests/create-cert.sh @@ -0,0 +1,130 @@ +#!/bin/bash + +set -e + +usage() { + cat <> ${tmpdir}/csr.conf +[req] +req_extensions = v3_req +distinguished_name = req_distinguished_name +[req_distinguished_name] +[ v3_req ] +basicConstraints = CA:FALSE +keyUsage = nonRepudiation, digitalSignature, keyEncipherment +extendedKeyUsage = serverAuth +subjectAltName = @alt_names +[alt_names] +DNS.1 = ${service} +DNS.2 = ${service}.${namespace} +DNS.3 = ${service}.${namespace}.svc +EOF + +openssl genrsa -out ${tmpdir}/server-key.pem 2048 +openssl req -new -key ${tmpdir}/server-key.pem -subj "/CN=${service}.${namespace}.svc" -out ${tmpdir}/server.csr -config ${tmpdir}/csr.conf + +# clean-up any previously created CSR for our service. Ignore errors if not present. +kubectl delete csr ${csrName} 2>/dev/null || true + +# create server cert/key CSR and send to k8s API +cat <&2 + exit 1 +fi +echo ${serverCert} | openssl base64 -d -A -out ${tmpdir}/server-cert.pem + + +# create the secret with CA cert and server cert/key +kubectl create secret generic ${secret} \ + --from-file=key.pem=${tmpdir}/server-key.pem \ + --from-file=cert.pem=${tmpdir}/server-cert.pem \ + --dry-run -o yaml | + kubectl -n ${namespace} apply -f - diff --git a/manifests/patch-ca.sh b/manifests/patch-ca.sh new file mode 100755 index 00000000000..f222369e906 --- /dev/null +++ b/manifests/patch-ca.sh @@ -0,0 +1,10 @@ +#!/bin/bash +set -o errexit +set -o nounset +set -o pipefail + +CURDIR=$(cd $(dirname ${BASH_SOURCE[0]}); pwd ) + +CA_BUNDLE=$(kubectl get configmap -n kube-system extension-apiserver-authentication -o=jsonpath='{.data.client-ca-file}' | base64 | tr -d '\n') +echo $CA_BUNDLE +sed -i "s/caBundle: .*$/caBundle: ${CA_BUNDLE}/g" $CURDIR/webhook.yaml diff --git a/manifests/webhook-rbac.yaml b/manifests/webhook-rbac.yaml deleted file mode 100644 index c4ed1acee7e..00000000000 --- a/manifests/webhook-rbac.yaml +++ /dev/null @@ -1,18 +0,0 @@ -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1beta1 -metadata: - name: admission-webhook-example-rbac -subjects: -- kind: ServiceAccount - namespace: pingcap - name: admission-webhook-example-sa -roleRef: - kind: ClusterRole - name: cluster-admin - apiGroup: rbac.authorization.k8s.io ---- -kind: ServiceAccount -apiVersion: v1 -metadata: - namespace: pingcap - name: admission-webhook-example-sa diff --git a/manifests/webhook.yaml b/manifests/webhook.yaml index 026f4b3137d..da83d3bea78 100644 --- a/manifests/webhook.yaml +++ b/manifests/webhook.yaml @@ -1,44 +1,106 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: admission-webhook-cr + labels: + app: admission-webhook +rules: +- apiGroups: ["apps"] + resources: ["statefulsets"] + verbs: ["get"] +- apiGroups: ["pingcap.com"] + resources: ["tidbclusters"] + verbs: ["get"] +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: admission-webhook-sa + labels: + app: admission-webhook +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: admission-webhook-crb + labels: + app: admission-webhook +subjects: +- kind: ServiceAccount + name: admission-webhook-sa + namespace: default +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: admission-webhook-cr +--- apiVersion: v1 kind: Service metadata: - name: admission-webhook-example-svc - namespace: pingcap + name: admission-webhook-svc labels: - app: admission-webhook-example + app: admission-webhook spec: ports: - port: 443 targetPort: 443 selector: - app: admission-webhook-example + app: admission-webhook --- apiVersion: apps/v1 kind: Deployment metadata: - name: admission-webhook-example-deployment - namespace: pingcap + name: admission-webhook-deployment labels: - app: admission-webhook-example + app: admission-webhook spec: replicas: 1 selector: matchLabels: - app: admission-webhook-example + app: admission-webhook template: metadata: - namespace: pingcap labels: - app: admission-webhook-example + app: admission-webhook spec: - serviceAccount: admission-webhook-example-sa + serviceAccountName: admission-webhook-sa containers: - - name: admission-webhook-example - image: hub.pingcap.net/yinliang/pingcap/tidb-operator:latest + - name: admission-webhook-pod + image: pingcap/tidb-operator:latest imagePullPolicy: Always - command: - - /usr/local/bin/tidb-webhook - env: - - name: NAMESPACE - value: pingcap - - name: SERVICENAME - value: admission-webhook-example-svc + command: + - /usr/local/bin/tidb-admission-controller + - -tlsCertFile=/etc/webhook/certs/cert.pem + - -tlsKeyFile=/etc/webhook/certs/key.pem + - -alsologtostderr + - -v=2 + - 2>&1 + volumeMounts: + - name: webhook-certs + mountPath: /etc/webhook/certs + readOnly: true + volumes: + - name: webhook-certs + secret: + secretName: admission-webhook-certs +--- +apiVersion: admissionregistration.k8s.io/v1beta1 +kind: ValidatingWebhookConfiguration +metadata: + name: validation-webhook-cfg + labels: + app: admission-webhook +webhooks: + - name: statefulset-webhook.pingcap.net + failurePolicy: Fail + clientConfig: + service: + name: admission-webhook-svc + namespace: default + path: "/statefulsets" + caBundle: ${CA_BUNDLE} + rules: + - operations: [ "UPDATE" ] + apiGroups: [ "apps", "" ] + apiVersions: ["v1beta1"] + resources: ["statefulsets"] diff --git a/pkg/tkctl/cmd/upinfo/upinfo.go b/pkg/tkctl/cmd/upinfo/upinfo.go index 912c46863e8..f98fcd768b3 100644 --- a/pkg/tkctl/cmd/upinfo/upinfo.go +++ b/pkg/tkctl/cmd/upinfo/upinfo.go @@ -15,6 +15,8 @@ package upinfo import ( "fmt" + "io" + "github.com/pingcap/tidb-operator/pkg/apis/pingcap.com/v1alpha1" "github.com/pingcap/tidb-operator/pkg/client/clientset/versioned" "github.com/pingcap/tidb-operator/pkg/controller" @@ -23,7 +25,6 @@ import ( "github.com/pingcap/tidb-operator/pkg/tkctl/readable" "github.com/pingcap/tidb-operator/pkg/util" "github.com/spf13/cobra" - "io" apps "k8s.io/api/apps/v1beta1" "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -39,7 +40,7 @@ const ( You can omit --tidbcluster= option by running 'tkc use ', ` upinfoExample = ` - # get current tidb cluster info (set by tkc use) + # get current tidb cluster info (set by tkc user) tkc upinfo # get specified tidb cluster component upgrade info @@ -162,6 +163,12 @@ func renderTCUpgradeInfo(tc *v1alpha1.TidbCluster, set *apps.StatefulSet, podLis w.WriteLine(readable.LEVEL_0, "Namespace:\t%s", tc.Namespace) w.WriteLine(readable.LEVEL_0, "CreationTimestamp:\t%s", tc.CreationTimestamp) w.WriteLine(readable.LEVEL_0, "Statu:\t%s", dbPhase) + if dbPhase == v1alpha1.UpgradePhase { + if len(podList.Items) != 0 { + pod := podList.Items[0] + w.WriteLine(readable.LEVEL_0, "Image:\t%s ---> %s", pod.Spec.Containers[0].Image, tc.Spec.TiDB.Image) + } + } { w.WriteLine(readable.LEVEL_1, "Name\tState\t") w.WriteLine(readable.LEVEL_1, "----\t-----\t") diff --git a/pkg/webhook/route/route.go b/pkg/webhook/route/route.go index 320fa11a261..3c626ec01a8 100644 --- a/pkg/webhook/route/route.go +++ b/pkg/webhook/route/route.go @@ -28,10 +28,22 @@ import ( // admitFunc is the type we use for all of our validators type admitFunc func(v1beta1.AdmissionReview) *v1beta1.AdmissionResponse +// marshal responseAdmissionReview and send back +func marshalAndWrite(response v1beta1.AdmissionReview, w http.ResponseWriter) { + + respBytes, err := json.Marshal(response) + if err != nil { + glog.Errorf("%v", err) + } + if _, err := w.Write(respBytes); err != nil { + glog.Errorf("%v", err) + } + +} + // serve handles the http portion of a request prior to handing to an admit // function func serve(w http.ResponseWriter, r *http.Request, admit admitFunc) { - var body []byte var contentType string responseAdmissionReview := v1beta1.AdmissionReview{} @@ -44,12 +56,14 @@ func serve(w http.ResponseWriter, r *http.Request, admit admitFunc) { body = data } else { responseAdmissionReview.Response = util.ARFail(err) - goto returnData + marshalAndWrite(responseAdmissionReview, w) + return } } else { err := errors.New("request body is nil!") responseAdmissionReview.Response = util.ARFail(err) - goto returnData + marshalAndWrite(responseAdmissionReview, w) + return } // verify the content type is accurate @@ -57,7 +71,8 @@ func serve(w http.ResponseWriter, r *http.Request, admit admitFunc) { if contentType != "application/json" { err := errors.New("expect application/json") responseAdmissionReview.Response = util.ARFail(err) - goto returnData + marshalAndWrite(responseAdmissionReview, w) + return } // The AdmissionReview that was sent to the webhook @@ -71,14 +86,8 @@ func serve(w http.ResponseWriter, r *http.Request, admit admitFunc) { // Return the same UID responseAdmissionReview.Response.UID = requestedAdmissionReview.Request.UID -returnData: - respBytes, err := json.Marshal(responseAdmissionReview) - if err != nil { - glog.Errorf("%v", err) - } - if _, err := w.Write(respBytes); err != nil { - glog.Errorf("%v", err) - } + marshalAndWrite(responseAdmissionReview, w) + } func ServeStatefulSets(w http.ResponseWriter, r *http.Request) { diff --git a/pkg/webhook/statefulset/statefulset.go b/pkg/webhook/statefulset/statefulset.go index e82122c1b39..b6962598ee2 100644 --- a/pkg/webhook/statefulset/statefulset.go +++ b/pkg/webhook/statefulset/statefulset.go @@ -20,15 +20,24 @@ import ( "github.com/golang/glog" "github.com/pingcap/tidb-operator/pkg/apis/pingcap.com/v1alpha1" + "github.com/pingcap/tidb-operator/pkg/client/clientset/versioned" "github.com/pingcap/tidb-operator/pkg/label" "github.com/pingcap/tidb-operator/pkg/webhook/util" "k8s.io/api/admission/v1beta1" apps "k8s.io/api/apps/v1beta1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/rest" +) + +var ( + versionCli versioned.Interface ) func AdmitStatefulSets(ar v1beta1.AdmissionReview) *v1beta1.AdmissionResponse { - glog.Infof("admit statefulsets") + + name := ar.Request.Name + namespace := ar.Request.Namespace + glog.Infof("admit statefulsets [%s/%s]", name, namespace) setResource := metav1.GroupVersionResource{Group: "apps", Version: "v1beta1", Resource: "statefulsets"} if ar.Request.Resource != setResource { @@ -37,30 +46,35 @@ func AdmitStatefulSets(ar v1beta1.AdmissionReview) *v1beta1.AdmissionResponse { return util.ARFail(err) } - cli, _, err := util.GetNewClient() - if err != nil { - glog.Errorf("failed to get kubernetes Clientset: %v", err) - return util.ARFail(err) - } + if versionCli == nil { + cfg, err := rest.InClusterConfig() + if err != nil { + glog.Errorf("failed to get config: %v", err) + return util.ARFail(err) + } - name := ar.Request.Name - namespace := ar.Request.Namespace + versionCli, err = versioned.NewForConfig(cfg) + if err != nil { + glog.Errorf("failed to create Clientset: %v", err) + return util.ARFail(err) + } + } raw := ar.Request.OldObject.Raw set := apps.StatefulSet{} deserializer := util.GetCodec() if _, _, err := deserializer.Decode(raw, nil, &set); err != nil { - glog.Error(err) + glog.Errorf("deseriralizer fail to decode request %v", err) return util.ARFail(err) } - tc, err := cli.PingcapV1alpha1().TidbClusters(namespace).Get(set.Labels[label.InstanceLabelKey], metav1.GetOptions{}) + tc, err := versionCli.PingcapV1alpha1().TidbClusters(namespace).Get(set.Labels[label.InstanceLabelKey], metav1.GetOptions{}) if err != nil { glog.Errorf("fail to fetch tidbcluster info namespace %s clustername(instance) %s err %v", namespace, set.Labels[label.InstanceLabelKey], err) return util.ARFail(err) } - if set.Labels[label.ComponentLabelKey] == "tidb" { + if set.Labels[label.ComponentLabelKey] == label.TiDBLabelVal { protect, ok := tc.Annotations[label.AnnTiDBPartition] if ok { diff --git a/pkg/webhook/util/certs.go b/pkg/webhook/util/certs.go deleted file mode 100644 index fd8028cd1f4..00000000000 --- a/pkg/webhook/util/certs.go +++ /dev/null @@ -1,98 +0,0 @@ -// Copyright 2019 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package util - -import ( - "crypto/x509" - "io/ioutil" - "os" - - "github.com/golang/glog" - "k8s.io/client-go/util/cert" -) - -type CertContext struct { - Cert []byte - Key []byte - SigningCert []byte -} - -// Setup the server cert. For example, user apiservers and admission webhooks -// can use the cert to prove their identify to the kube-apiserver -func SetupServerCert(namespaceName, serviceName string) (*CertContext, error) { - certDir, err := ioutil.TempDir("", "create-server-cert") - if err != nil { - glog.Errorf("Failed to create a temp dir for cert generation %v", err) - return nil, err - } - defer os.RemoveAll(certDir) - signingKey, err := cert.NewPrivateKey() - if err != nil { - glog.Errorf("Failed to create CA private key %v", err) - return nil, err - } - signingCert, err := cert.NewSelfSignedCACert(cert.Config{CommonName: "e2e-server-cert-ca"}, signingKey) - if err != nil { - glog.Errorf("Failed to create CA cert for apiserver %v", err) - return nil, err - } - caCertFile, err := ioutil.TempFile(certDir, "ca.crt") - if err != nil { - glog.Errorf("Failed to create a temp file for ca cert generation %v", err) - return nil, err - } - if err := ioutil.WriteFile(caCertFile.Name(), cert.EncodeCertPEM(signingCert), 0644); err != nil { - glog.Errorf("Failed to write CA cert %v", err) - return nil, err - } - key, err := cert.NewPrivateKey() - if err != nil { - glog.Errorf("Failed to create private key for %v", err) - return nil, err - } - signedCert, err := cert.NewSignedCert( - cert.Config{ - CommonName: serviceName + "." + namespaceName + ".svc", - Usages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, - }, - key, signingCert, signingKey, - ) - if err != nil { - glog.Errorf("Failed to create cert%v", err) - return nil, err - } - certFile, err := ioutil.TempFile(certDir, "server.crt") - if err != nil { - glog.Errorf("Failed to create a temp file for cert generation %v", err) - return nil, err - } - keyFile, err := ioutil.TempFile(certDir, "server.key") - if err != nil { - glog.Errorf("Failed to create a temp file for key generation %v", err) - return nil, err - } - if err = ioutil.WriteFile(certFile.Name(), cert.EncodeCertPEM(signedCert), 0600); err != nil { - glog.Errorf("Failed to write cert file %v", err) - return nil, err - } - if err = ioutil.WriteFile(keyFile.Name(), cert.EncodePrivateKeyPEM(key), 0644); err != nil { - glog.Errorf("Failed to write key file %v", err) - return nil, err - } - return &CertContext{ - Cert: cert.EncodeCertPEM(signedCert), - Key: cert.EncodePrivateKeyPEM(key), - SigningCert: cert.EncodeCertPEM(signingCert), - }, nil -} diff --git a/pkg/webhook/util/util.go b/pkg/webhook/util/util.go index dcb42100561..a651ec0e5a8 100644 --- a/pkg/webhook/util/util.go +++ b/pkg/webhook/util/util.go @@ -16,35 +16,10 @@ package util import ( "crypto/tls" - "github.com/golang/glog" - "github.com/pingcap/tidb-operator/pkg/client/clientset/versioned" "k8s.io/api/admission/v1beta1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/kubernetes" - "k8s.io/client-go/rest" ) -func GetNewClient() (versioned.Interface, kubernetes.Interface, error) { - cfg, err := rest.InClusterConfig() - if err != nil { - glog.Errorf("failed to get config: %v", err) - return nil, nil, err - } - - cli, err := versioned.NewForConfig(cfg) - if err != nil { - glog.Errorf("failed to create Clientset: %v", err) - return nil, nil, err - } - - kubeCli, err := kubernetes.NewForConfig(cfg) - if err != nil { - glog.Errorf("failed to get kubernetes Clientset: %v", err) - return nil, nil, err - } - return cli, kubeCli, nil -} - // toAdmissionResponse is a helper function to create an AdmissionResponse // with an embedded error func ARFail(err error) *v1beta1.AdmissionResponse { @@ -65,8 +40,8 @@ func ARSuccess() *v1beta1.AdmissionResponse { } // config tls cert for server -func ConfigTLS(cert []byte, key []byte) (*tls.Config, error) { - sCert, err := tls.X509KeyPair(cert, key) +func ConfigTLS(certFile string, keyFile string) (*tls.Config, error) { + sCert, err := tls.LoadX509KeyPair(certFile, keyFile) if err != nil { return nil, err } diff --git a/pkg/webhook/webhook.go b/pkg/webhook/webhook.go index b3cb904273a..24a1d53e904 100644 --- a/pkg/webhook/webhook.go +++ b/pkg/webhook/webhook.go @@ -15,15 +15,11 @@ package webhook import ( "net/http" - "time" "github.com/golang/glog" "github.com/pingcap/tidb-operator/pkg/client/clientset/versioned" "github.com/pingcap/tidb-operator/pkg/webhook/route" "github.com/pingcap/tidb-operator/pkg/webhook/util" - admissionV1beta1 "k8s.io/api/admissionregistration/v1beta1" - "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" ) @@ -32,19 +28,15 @@ type WebhookServer struct { KubeCli kubernetes.Interface // operator client interface Cli versioned.Interface - // cert context ca.crt key server.crt - Context *util.CertContext - // configuration name - ConfigName string // http server Server *http.Server } -func NewWebHookServer(kubecli kubernetes.Interface, cli versioned.Interface, context *util.CertContext) *WebhookServer { +func NewWebHookServer(kubecli kubernetes.Interface, cli versioned.Interface, certFile string, keyFile string) *WebhookServer { http.HandleFunc("/statefulsets", route.ServeStatefulSets) - sCert, err := util.ConfigTLS(context.Cert, context.Key) + sCert, err := util.ConfigTLS(certFile, keyFile) if err != nil { glog.Fatalf("failed to create scert file %v", err) @@ -56,11 +48,9 @@ func NewWebHookServer(kubecli kubernetes.Interface, cli versioned.Interface, con } return &WebhookServer{ - KubeCli: kubecli, - Cli: cli, - Context: context, - ConfigName: "validating-webhook-configuration", - Server: server, + KubeCli: kubecli, + Cli: cli, + Server: server, } } @@ -71,56 +61,3 @@ func (ws *WebhookServer) Run() error { func (ws *WebhookServer) Shutdown() error { return ws.Server.Shutdown(nil) } - -func strPtr(s string) *string { return &s } - -func (ws *WebhookServer) RegisterWebhook(namespace string, svcName string) error { - - policyFail := admissionV1beta1.Fail - - _, err := ws.KubeCli.AdmissionregistrationV1beta1().ValidatingWebhookConfigurations().Create(&admissionV1beta1.ValidatingWebhookConfiguration{ - ObjectMeta: metav1.ObjectMeta{ - Name: ws.ConfigName, - }, - Webhooks: []admissionV1beta1.Webhook{ - { - Name: "admit-statefulset-webhook.k8s.io", - Rules: []admissionV1beta1.RuleWithOperations{{ - Operations: []admissionV1beta1.OperationType{admissionV1beta1.Update}, - Rule: admissionV1beta1.Rule{ - APIGroups: []string{"apps"}, - APIVersions: []string{"v1beta1"}, - Resources: []string{"statefulsets"}, - }, - }}, - ClientConfig: admissionV1beta1.WebhookClientConfig{ - Service: &admissionV1beta1.ServiceReference{ - Namespace: namespace, - Name: svcName, - Path: strPtr("/statefulsets"), - }, - CABundle: ws.Context.SigningCert, - }, - FailurePolicy: &policyFail, - }, - }, - }) - - if err != nil { - glog.Errorf("registering webhook config %s with namespace %s error %v", ws.ConfigName, namespace, err) - return err - } - - // The webhook configuration is honored in 10s. - time.Sleep(10 * time.Second) - - return nil -} - -func (ws *WebhookServer) UnregisterWebhook() error { - err := ws.KubeCli.AdmissionregistrationV1beta1().ValidatingWebhookConfigurations().Delete(ws.ConfigName, nil) - if err != nil && !errors.IsNotFound(err) { - glog.Errorf("failed to delete webhook config %v", err) - } - return nil -} From 841cf41bf5083a12f81288089dd9c305f8fb0239 Mon Sep 17 00:00:00 2001 From: shuijing198799 Date: Fri, 10 May 2019 21:20:12 +0800 Subject: [PATCH 03/16] fix the binary name --- images/tidb-operator/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/images/tidb-operator/Dockerfile b/images/tidb-operator/Dockerfile index a812129135b..2db8eeb21b9 100644 --- a/images/tidb-operator/Dockerfile +++ b/images/tidb-operator/Dockerfile @@ -4,4 +4,4 @@ RUN apk add tzdata --no-cache ADD bin/tidb-controller-manager /usr/local/bin/tidb-controller-manager ADD bin/tidb-scheduler /usr/local/bin/tidb-scheduler ADD bin/tidb-discovery /usr/local/bin/tidb-discovery -ADD bin/tidb-webhook /usr/local/bin/tidb-admission-controller +ADD bin/tidb-admission-controller /usr/local/bin/tidb-admission-controller From 26acae3786e8ecb697fc573eec0281c7292261b2 Mon Sep 17 00:00:00 2001 From: shuijing198799 Date: Sat, 11 May 2019 14:25:57 +0800 Subject: [PATCH 04/16] init deserialzer instead of create it everytime --- pkg/webhook/statefulset/statefulset.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/pkg/webhook/statefulset/statefulset.go b/pkg/webhook/statefulset/statefulset.go index b6962598ee2..543d3eb314b 100644 --- a/pkg/webhook/statefulset/statefulset.go +++ b/pkg/webhook/statefulset/statefulset.go @@ -18,6 +18,7 @@ import ( "fmt" "strconv" + "k8s.io/apimachinery/pkg/runtime" "github.com/golang/glog" "github.com/pingcap/tidb-operator/pkg/apis/pingcap.com/v1alpha1" "github.com/pingcap/tidb-operator/pkg/client/clientset/versioned" @@ -31,8 +32,13 @@ import ( var ( versionCli versioned.Interface + deserializer runtime.Decoder ) +func init() { + deserializer = util.GetCodec() +} + func AdmitStatefulSets(ar v1beta1.AdmissionReview) *v1beta1.AdmissionResponse { name := ar.Request.Name @@ -62,7 +68,6 @@ func AdmitStatefulSets(ar v1beta1.AdmissionReview) *v1beta1.AdmissionResponse { raw := ar.Request.OldObject.Raw set := apps.StatefulSet{} - deserializer := util.GetCodec() if _, _, err := deserializer.Decode(raw, nil, &set); err != nil { glog.Errorf("deseriralizer fail to decode request %v", err) return util.ARFail(err) From b94ef9362d3e921d0350846f58ccc17129fe86e2 Mon Sep 17 00:00:00 2001 From: shuijing198799 Date: Fri, 17 May 2019 12:01:03 +0800 Subject: [PATCH 05/16] 1. add pause test 2. use helm to manager 3. fix test case bug 4. delete useless function --- .../admission-controller-deployment.yaml | 101 ++++++++ charts/tidb-operator/values.yaml | 8 + manifests/create-cert.sh | 31 +-- manifests/patch-ca.sh | 10 - manifests/webhook.yaml | 106 -------- tests/actions.go | 238 +++++++----------- 6 files changed, 205 insertions(+), 289 deletions(-) create mode 100644 charts/tidb-operator/templates/admission-controller-deployment.yaml delete mode 100755 manifests/patch-ca.sh delete mode 100644 manifests/webhook.yaml diff --git a/charts/tidb-operator/templates/admission-controller-deployment.yaml b/charts/tidb-operator/templates/admission-controller-deployment.yaml new file mode 100644 index 00000000000..a9b9da856e0 --- /dev/null +++ b/charts/tidb-operator/templates/admission-controller-deployment.yaml @@ -0,0 +1,101 @@ +{{- if .Values.admissionController.create }} +apiVersion: v1 +kind: Secret +metadata: + name: admission-controller-certs +data: + cert.pem: {{ .Values.admissionController.cert }} + key.pem: {{ .Values.admissionController.key }} +--- +apiVersion: v1 +kind: Service +metadata: + name: admission-controller-svc + labels: + app.kubernetes.io/name: {{ template "chart.name" . }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/component: admission-controller + helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} +spec: + ports: + - port: 443 + targetPort: 443 + selector: + app.kubernetes.io/name: {{ template "chart.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/component: admission-controller +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: tidb-admission-controller-deployment + labels: + app.kubernetes.io/name: {{ template "chart.name" . }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/component: admission-controller + helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} +spec: + replicas: {{ .Values.admissionController.replicas }} + selector: + matchLabels: + app.kubernetes.io/name: {{ template "chart.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/component: admission-controller + template: + metadata: + labels: + app.kubernetes.io/name: {{ template "chart.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/component: admission-controller + spec: + serviceAccountName: admission-controller-sa + containers: + - name: tidb-admission-controller-pod + image: {{ .Values.operatorImage }} + imagePullPolicy: {{ .Values.imagePullPolicy | default "IfNotPresent" }} + command: + - /usr/local/bin/tidb-admission-controller + - -tlsCertFile=/etc/webhook/certs/cert.pem + - -tlsKeyFile=/etc/webhook/certs/key.pem + - -alsologtostderr + - -v={{ .Values.admissionController.replicas }} + - 2>&1 + volumeMounts: + - name: webhook-certs + mountPath: /etc/webhook/certs + readOnly: true + volumes: + - name: webhook-certs + secret: + secretName: admission-controller-certs + env: + - name: TZ + value: {{ .Values.timezone | default "UTC" }} +--- +apiVersion: admissionregistration.k8s.io/v1beta1 +kind: ValidatingWebhookConfiguration +metadata: + name: validation-controller-cfg + labels: + app.kubernetes.io/name: {{ template "chart.name" . }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/component: admission-controller + helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} +webhooks: + - name: statefulset-webhook.pingcap.net + failurePolicy: Fail + clientConfig: + service: + name: admission-controller-svc + path: "/statefulsets" + namespace: {{ .Release.Namespace }} + caBundle: {{ .Values.admissionController.ca }} + rules: + - operations: [ "UPDATE" ] + apiGroups: [ "apps", "" ] + apiVersions: ["v1beta1"] + resources: ["statefulsets"] +{{- end }} diff --git a/charts/tidb-operator/values.yaml b/charts/tidb-operator/values.yaml index 98a920632fc..beda73863a7 100644 --- a/charts/tidb-operator/values.yaml +++ b/charts/tidb-operator/values.yaml @@ -55,3 +55,11 @@ scheduler: kubeSchedulerImageName: gcr.io/google-containers/hyperkube # This will default to matching your kubernetes version # kubeSchedulerImageTag: + +admissionController: + create: false + replicas: 1 + logLevel: 2 + key: LS0tLS1CRUdJTiBSU0EgUFJJVkFURSBLRVktLS0tLQpNSUlFb3dJQkFBS0NBUUVBd3B5cmkvWnJJdS9TNEJlU05KZHBFN1JCeFF6MytaMTdxcXZWY2JweUY0cEZHdlMrCm1pWGF5ZDhVd3BYcDZJT3hDNDd3UWFhU1JnYm9qUTBYcjllWm1OS0NsWkxROUgwWVpTWndXSU4zL29vTEdWT3AKcDQ5ampBaDlFSmNhSTVFOUpNSlQvWFk1NHlZeTJ6a2lzaWhCUnI5b2FyMHZHZjdaeElQSGNndG5ES1ZFVm5vTgpJRGJuVTU2RE1GZk9kTWlvUURDU0ppakpPaU9pdXhFNTFVbG9aTVVVN3FCdTV5NUFtYTN2NVErd3dIR252ZUh1CmZ4TFpJN1plc3hhenlRMnU4SkFKZ1pIeWdEM1FTS1Z5REw4M1IyOHRmNDd0cm5QT1dOOXNreTNNNm9rZFZCaFQKUFVhUmRGMjkrWFFJdG56TG9Ua3RVSGtzdWVmTDF3dTNaVWdyVndJREFRQUJBb0lCQUF3bXRCOWRGM3RxOWdPLwo5UVVMbnlmM3pWTjhaODlTSWZLYUlBMmwwcEhkdEVybWxUQVN4UUp0eDJJTzhqZnVZUk9UclRTc0xkM3BlSU8yCkdaQ0ZLYlg5RHVJLzlJUVBlNEoxVC9aMjN2c3BxMSs5ZFk1Y3laNW91VVpiNk9ubm02Vzc3SHFhOHlwWHJEMDcKNkFxdnNNTzJGOUx5VlFxT2V4Sm1HQUhxN0ZoQnJWSk9QTS9lNWIydGl1SDFyUUJqVVhZWXAxVHp3R3BFM1cyQgpvRVI3bjlvNEJtNysyTnZ5bHAwNFdWWm9UNW5rRXVtcXB3ajR5NFdUYlBYNHFMUnJMVzJ4QnhnTzdUdVFQeVlFCkJUZFFZVkFhRm55akFXdXE3VDNETWJXVzQ3eDVnZ3BGR2hRNk9GYjRNUzN5RkpNZlE4eWhYblpOend6L1NvQXoKSWxub21pRUNnWUVBL2t0L0EreDhDcnZpWE1RR0duWVB6NW1sRmlmRW5FTEQwVkY0ak5pNFRWblBLYVhocEFCYgpNS3UwOXRuWXFnTHRacURBc0c4ZCtxWVlKOVZaQjMreUVnWTMwY043K1phVTlJcS81Q3R3N2p3d3JFMVEwTFN5CjhnQk9sZFVGUzVUMXFBOXhyOGx6WW1jdTl3TWpEZUNaTGxrdWdzWFlBRGF5eHRMRnpBU0kzWDBDZ1lFQXcrcTYKQnhGMjhkUFdwQk95bk5mcUIxdVc1Lyt6S3RhREh4Y1RMenRpaWdrM2NUbm1seTB0Zll4eExpeGs4aFVmV3dmQwoyejFDNmhxV1RFeS9nZTJHaWhlOFJDL1NFMGhvNEZyYUYwNVAvM0Voa0lwOFM1cWpobmdSdjJ2S1ZKU1BTLzNpCnJPRHBxNWxKU3pVc2lXWktTRFJ6RXlvRXI3Y2NodXpsNm8vYjFHTUNnWUFXMnE0bHFpRlR6YU1BZDFXZEZ6TkEKa3l4MUcwbmVTVm5UL0NCN2VCYklUT1FEZURjWlNEYnAzdVM2c3NWNE1oOG45QmFkUkw5TEFaWjJMZ1I4V2d1awo5dzRaT2hwd1NncEZHQnM2NUhTWDFLMERMMGV1R2k5aTlJZ2Jpek9Zb1dDYXBCaXhsY2RhOXE4UDRDWk1YdjBsClh6cUE0a3NSYWRrVGFXb0JkZjdlK1FLQmdRQ01PN0dZYVpuWUV5ZXFIaWRWT1A5YzhPUWRmOXVCS0RhR3hIK0kKNlZycEx4dXVRaUdaeUU2dGRSOTJTM1NlQ3h4OGpodWR6a1hkUjlhWkdyNWhnK0V2YXpucEhNbWozOHFoL0tYcQoxTFZTUkdvZ0NlQmFSWUFUT2xLSno5L3Rsc0RUUmhGQWFXVVkzUW1uZVpJOWJ5aHdjb3FFQnlsL2xQUW9aOTFTCmZsSjl2UUtCZ0dBTXRybTRFVkRTckF0QS9qeXg3NFB3SThjc2xtUEQ1d3JaWEZQVTUvc3FyRGNKbUNqSXBJRWUKRFdSaWNYL09aSUpRbDdjNkFOQjc1U1VUMHZ4dXJXeGp6WWlPVjRvUTJVUTBFc1VxcUh3RWxDVWZjclcxQnh2eQpmMnE3aGtUL0MySWwyRjhmRmxnYjJmZFVmaFM5MjZSbTF0aUUyMzJMU3MwYmZiWHArSDFmCi0tLS0tRU5EIFJTQSBQUklWQVRFIEtFWS0tLS0tCg== + cert: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURsVENDQW4yZ0F3SUJBZ0lVSmdOQXRuQ25ROVdLRWRVK29pR0lKQUxxTWFRd0RRWUpLb1pJaHZjTkFRRUwKQlFBd0ZURVRNQkVHQTFVRUF4TUthM1ZpWlhKdVpYUmxjekFlRncweE9UQTFNVGN3TXpJMU1EQmFGdzB5TURBMQpNVFl3TXpJMU1EQmFNQzh4TFRBckJnTlZCQU1USkdGa2JXbHpjMmx2YmkxamIyNTBjbTlzYkdWeUxYTjJZeTV3CmFXNW5ZMkZ3TG5OMll6Q0NBU0l3RFFZSktvWklodmNOQVFFQkJRQURnZ0VQQURDQ0FRb0NnZ0VCQU1LY3E0djIKYXlMdjB1QVhralNYYVJPMFFjVU05L21kZTZxcjFYRzZjaGVLUlJyMHZwb2wyc25mRk1LVjZlaURzUXVPOEVHbQpra1lHNkkwTkY2L1htWmpTZ3BXUzBQUjlHR1VtY0ZpRGQvNktDeGxUcWFlUFk0d0lmUkNYR2lPUlBTVENVLzEyCk9lTW1NdHM1SXJJb1FVYS9hR3E5THhuKzJjU0R4M0lMWnd5bFJGWjZEU0EyNTFPZWd6Qlh6blRJcUVBd2tpWW8KeVRvam9yc1JPZFZKYUdURkZPNmdidWN1UUptdDcrVVBzTUJ4cDczaDduOFMyU08yWHJNV3M4a05ydkNRQ1lHUgo4b0E5MEVpbGNneS9OMGR2TFgrTzdhNXp6bGpmYkpNdHpPcUpIVlFZVXoxR2tYUmR2ZmwwQ0xaOHk2RTVMVkI1CkxMbm55OWNMdDJWSUsxY0NBd0VBQWFPQndqQ0J2ekFPQmdOVkhROEJBZjhFQkFNQ0JhQXdFd1lEVlIwbEJBd3cKQ2dZSUt3WUJCUVVIQXdFd0RBWURWUjBUQVFIL0JBSXdBREFkQmdOVkhRNEVGZ1FVYlVwSk9UeGE4bkljelpnTgpTZ2hvUnRTQTcwSXdhd1lEVlIwUkJHUXdZb0lZWVdSdGFYTnphVzl1TFdOdmJuUnliMnhzWlhJdGMzWmpnaUJoClpHMXBjM05wYjI0dFkyOXVkSEp2Ykd4bGNpMXpkbU11Y0dsdVoyTmhjSUlrWVdSdGFYTnphVzl1TFdOdmJuUnkKYjJ4c1pYSXRjM1pqTG5CcGJtZGpZWEF1YzNaak1BMEdDU3FHU0liM0RRRUJDd1VBQTRJQkFRQlZPYUF5dVo1VAp3eTdaK3FYcnlJUTVDWE1DUGVNUVN5M0ZwZFVtVUEvUDJnamtsUWg3RStYdkZ5UjNzLzEzQlluYnViWlNVRlMxCjdTTEUvd3duN1N0c1hVZ3M1RkMweTRYNWpoWGZmZEQwcDVKNzQxMWwxVU9MdHVZZkFiL1VGM1c4RjZxNlhXL0kKRkFVTEhaUjdMTHNpZlZCMTFmNHZwMmw2c2V2emF2WVIyeXY5aUJwVU9oSXpZWTFTelNCOXlzOVgxZVBEUWRQVApUYVE2VHlidFIzYUE2Y3d6S2JiZ200THhSYzRWRzZzU3lFRDJBZkZWeURSMXdqb3pXQmw5OWJGWWNSSVVaZG4wCjFCQ0d5SWNSS0pYQjhoUlRabEZyMnNQRFdpL000YWEwcUd0K05jTDdPM0Zwa3JWeW9jZzhHczhlY3Fjenk4VGkKTHJueXREU1hoQW5aCi0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K + ca: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUN5RENDQWJDZ0F3SUJBZ0lCQURBTkJna3Foa2lHOXcwQkFRc0ZBREFWTVJNd0VRWURWUVFERXdwcmRXSmwKY201bGRHVnpNQjRYRFRFNU1EVXhNREEzTkRZek4xb1hEVEk1TURVd056QTNORFl6TjFvd0ZURVRNQkVHQTFVRQpBeE1LYTNWaVpYSnVaWFJsY3pDQ0FTSXdEUVlKS29aSWh2Y05BUUVCQlFBRGdnRVBBRENDQVFvQ2dnRUJBSzN4CkluTDhJREpJeC9rMHQ5QmovZGxKMEU4Q1drMkpoYzVwN29SeEhSdm8rQjFlMEthbTkzS1lpeTNGeFIvVW1iWGIKWkhGb3VEaGZkSUhjYmw4dytkQktDYUc5SlduZUlYTExoSU1wTUlTRHFGYWdDTjNZTHpmUVFsVFd1RmpXVnZFRAphWUFuOG1kd3E5bjJLaTY0a2szR2Zjb2V6cFpIVzVYVjhaU2JnVGQyNjlMRkFKLzVZNlRUd3FvMWFqM1M4czZTCnl5L2FrQ0V5Znd4UXZsZ1I1NDErWWs4VkViOURYMFk4Vm8xLzJod1NyVm5YbFRkWTNWdXlMSkwvTUxOMmpMTVkKdG5ON0VRbE5YdHNSVkF1YUI3WmFmZG04c3BpU1MyYkpQcG91YlB0azdSWmRvSmo3dGVxU0VJdTF1UUZacUNERgpmQVlBU3lXOG5ZZjFSa1B4RjVVQ0F3RUFBYU1qTUNFd0RnWURWUjBQQVFIL0JBUURBZ0trTUE4R0ExVWRFd0VCCi93UUZNQU1CQWY4d0RRWUpLb1pJaHZjTkFRRUxCUUFEZ2dFQkFDSFBPZHpXRDRmWWIyakIyeXdkaFhiT1VyamMKdHZ5bXF2eXJ4enpZazZDRGNUN3hRL2FQYTBRZUE0WkxWY3c0eTZaalZkMmR3enZxd2llNDBUWUw2eFM1KzVkMwpyNE1KRXdBaFp1bk5ZbUVybWtjZmJJL3M2Q0dBaVNjd2RhRlp5U2I3Yk5lU090NHkraDBPaDRnc2lEWU8yNUFSCkcrc3MzK2ZDbWZYVTNFKzRhdGNqcnhjSU1iRmFmejVhVEJUdlp4ajN3Z01FRzV6MjE5anplbXJZdWJsazBxd1kKbjB6d0xDTEpubkZtN0hrK0tnTjRXeUlTcjFqZzBTTDl0YmVxWi8wd0NWM01lMFB0VUgzdFR5S1dSTHV4cFk5YwpsZHlmQXlMMmM5eElYZnFrcHczMHowVEo0NHNrVTRmU3NjWlhNV0VwSFowNDdnV1FsRi9mZW9zS1c0OD0KLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo= diff --git a/manifests/create-cert.sh b/manifests/create-cert.sh index 7540f20d0ba..ee23323040b 100755 --- a/manifests/create-cert.sh +++ b/manifests/create-cert.sh @@ -18,23 +18,13 @@ usage: ${0} [OPTIONS] The following flags are required. - --service Service name of webhook. --namespace Namespace where webhook service and secret reside. - --secret Secret name for CA certificate and server certificate/key pair. EOF exit 1 } while [[ $# -gt 0 ]]; do case ${1} in - --service) - service="$2" - shift - ;; - --secret) - secret="$2" - shift - ;; --namespace) namespace="$2" shift @@ -46,9 +36,11 @@ while [[ $# -gt 0 ]]; do shift done -[ -z ${service} ] && service=admission-webhook-svc -[ -z ${secret} ] && secret=admission-webhook-certs -[ -z ${namespace} ] && namespace=default +service=admission-controller-svc +if [ -z ${namespace} ]; then + echo "need input namespace" + exit 1 +fi if [ ! -x "$(command -v openssl)" ]; then echo "openssl not found" @@ -57,7 +49,6 @@ fi csrName=${service}.${namespace} tmpdir=$(mktemp -d) -echo "creating certs in tmpdir ${tmpdir} " cat <> ${tmpdir}/csr.conf [req] @@ -119,12 +110,10 @@ if [[ ${serverCert} == '' ]]; then echo "ERROR: After approving csr ${csrName}, the signed certificate did not appear on the resource. Giving up after 10 attempts." >&2 exit 1 fi -echo ${serverCert} | openssl base64 -d -A -out ${tmpdir}/server-cert.pem +echo ${serverCert} | openssl base64 -d -A -out ${tmpdir}/server-cert.pem -# create the secret with CA cert and server cert/key -kubectl create secret generic ${secret} \ - --from-file=key.pem=${tmpdir}/server-key.pem \ - --from-file=cert.pem=${tmpdir}/server-cert.pem \ - --dry-run -o yaml | - kubectl -n ${namespace} apply -f - +CA_BUNDLE=$(kubectl get configmap -n kube-system extension-apiserver-authentication -o=jsonpath='{.data.client-ca-file}' | base64 | tr -d '\n') +echo ca: $CA_BUNDLE +echo key: $(cat ${tmpdir}/server-key.pem | base64| tr -d '\n') +echo cert: $(cat ${tmpdir}/server-cert.pem | base64| tr -d '\n') diff --git a/manifests/patch-ca.sh b/manifests/patch-ca.sh deleted file mode 100755 index f222369e906..00000000000 --- a/manifests/patch-ca.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash -set -o errexit -set -o nounset -set -o pipefail - -CURDIR=$(cd $(dirname ${BASH_SOURCE[0]}); pwd ) - -CA_BUNDLE=$(kubectl get configmap -n kube-system extension-apiserver-authentication -o=jsonpath='{.data.client-ca-file}' | base64 | tr -d '\n') -echo $CA_BUNDLE -sed -i "s/caBundle: .*$/caBundle: ${CA_BUNDLE}/g" $CURDIR/webhook.yaml diff --git a/manifests/webhook.yaml b/manifests/webhook.yaml deleted file mode 100644 index da83d3bea78..00000000000 --- a/manifests/webhook.yaml +++ /dev/null @@ -1,106 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: admission-webhook-cr - labels: - app: admission-webhook -rules: -- apiGroups: ["apps"] - resources: ["statefulsets"] - verbs: ["get"] -- apiGroups: ["pingcap.com"] - resources: ["tidbclusters"] - verbs: ["get"] ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: admission-webhook-sa - labels: - app: admission-webhook ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: admission-webhook-crb - labels: - app: admission-webhook -subjects: -- kind: ServiceAccount - name: admission-webhook-sa - namespace: default -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: admission-webhook-cr ---- -apiVersion: v1 -kind: Service -metadata: - name: admission-webhook-svc - labels: - app: admission-webhook -spec: - ports: - - port: 443 - targetPort: 443 - selector: - app: admission-webhook ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: admission-webhook-deployment - labels: - app: admission-webhook -spec: - replicas: 1 - selector: - matchLabels: - app: admission-webhook - template: - metadata: - labels: - app: admission-webhook - spec: - serviceAccountName: admission-webhook-sa - containers: - - name: admission-webhook-pod - image: pingcap/tidb-operator:latest - imagePullPolicy: Always - command: - - /usr/local/bin/tidb-admission-controller - - -tlsCertFile=/etc/webhook/certs/cert.pem - - -tlsKeyFile=/etc/webhook/certs/key.pem - - -alsologtostderr - - -v=2 - - 2>&1 - volumeMounts: - - name: webhook-certs - mountPath: /etc/webhook/certs - readOnly: true - volumes: - - name: webhook-certs - secret: - secretName: admission-webhook-certs ---- -apiVersion: admissionregistration.k8s.io/v1beta1 -kind: ValidatingWebhookConfiguration -metadata: - name: validation-webhook-cfg - labels: - app: admission-webhook -webhooks: - - name: statefulset-webhook.pingcap.net - failurePolicy: Fail - clientConfig: - service: - name: admission-webhook-svc - namespace: default - path: "/statefulsets" - caBundle: ${CA_BUNDLE} - rules: - - operations: [ "UPDATE" ] - apiGroups: [ "apps", "" ] - apiVersions: ["v1beta1"] - resources: ["statefulsets"] diff --git a/tests/actions.go b/tests/actions.go index ed2e1e5f749..148db9cbef5 100644 --- a/tests/actions.go +++ b/tests/actions.go @@ -125,7 +125,6 @@ type OperatorActions interface { CheckScaledCorrectly(info *TidbClusterConfig, podUIDsBeforeScale map[string]types.UID) error UpgradeTidbCluster(info *TidbClusterConfig) error UpgradeTidbClusterOrDie(info *TidbClusterConfig) - CheckUpgradeProgress(info *TidbClusterConfig) error DeployAdHocBackup(info *TidbClusterConfig) error CheckAdHocBackup(info *TidbClusterConfig) error DeployScheduledBackup(info *TidbClusterConfig) error @@ -302,6 +301,7 @@ func (oi *OperatorConfig) OperatorHelmSetString(m map[string]string) string { "scheduler.logLevel": "2", "controllerManager.replicas": "2", "scheduler.replicas": "2", + "admissionController.create": "true", } if oi.SchedulerTag != "" { set["scheduler.kubeSchedulerImageTag"] = oi.SchedulerTag @@ -694,6 +694,18 @@ func (oa *operatorActions) CheckScaledCorrectly(info *TidbClusterConfig, podUIDs }) } +func setPartitionAnnotation(tcName string, nameSpace string, ordinal int) error { + // add annotation to pause statefulset upgrade process + cmd := fmt.Sprintf("kubectl annotate tc %s -n %s tidb.pingcap.com/tidb-partition=%d --overwrite", + tcName, nameSpace, ordinal) + glog.Infof("%s",cmd) + _, err := exec.Command("/bin/sh", "-c", cmd).CombinedOutput() + if err != nil { + return err + } + return nil +} + func (oa *operatorActions) UpgradeTidbCluster(info *TidbClusterConfig) error { // record tikv leader count in webhook first err := webhook.GetAllKVLeaders(oa.cli, info.Namespace, info.ClusterName) @@ -702,6 +714,19 @@ func (oa *operatorActions) UpgradeTidbCluster(info *TidbClusterConfig) error { } oa.EmitEvent(info, "UpgradeTidbCluster") + // get tidbSet from apiserver + tidbSetName := controller.TiDBMemberName(info.ClusterName) + tidbSet, err := oa.kubeCli.AppsV1beta1().StatefulSets(info.Namespace).Get(tidbSetName, metav1.GetOptions{}) + if err != nil { + return pingcapErrors.Wrapf(err,"failed to get stateful set [%s/%s] setName %s",info.Namespace, info.ClusterName,tidbSetName) + } + + // add annotation to pause statefulset upgrade process + err = setPartitionAnnotation(info.ClusterName, info.Namespace, int(tidbSet.Status.Replicas - 1)) + if err != nil { + return pingcapErrors.Wrapf(err,"failed to add annotation to [%s/%s]",info.Namespace, info.ClusterName) + } + cmd := fmt.Sprintf("helm upgrade %s %s --set-string %s", info.ClusterName, oa.tidbClusterChartPath(info.OperatorTag), info.TidbClusterHelmSetString(nil)) glog.Info("[UPGRADE] " + cmd) @@ -718,157 +743,17 @@ func (oa *operatorActions) UpgradeTidbClusterOrDie(info *TidbClusterConfig) { } } -func (oa *operatorActions) CheckUpgradeProgress(info *TidbClusterConfig) error { - return wait.Poll(oa.pollInterval, DefaultPollTimeout, func() (done bool, err error) { - tc, err := oa.cli.PingcapV1alpha1().TidbClusters(info.Namespace).Get(info.ClusterName, metav1.GetOptions{}) - if err != nil { - glog.Infof("failed to get tidbcluster: [%s], error: %v", info.ClusterName, err) - return false, nil - } - - pdSetName := controller.PDMemberName(info.ClusterName) - pdSet, err := oa.kubeCli.AppsV1beta1().StatefulSets(info.Namespace).Get(pdSetName, metav1.GetOptions{}) - if err != nil { - glog.Infof("failed to get pd statefulset: [%s], error: %v", pdSetName, err) - return false, nil - } - - tikvSetName := controller.TiKVMemberName(info.ClusterName) - tikvSet, err := oa.kubeCli.AppsV1beta1().StatefulSets(info.Namespace).Get(tikvSetName, metav1.GetOptions{}) - if err != nil { - glog.Infof("failed to get tikvSet statefulset: [%s], error: %v", tikvSetName, err) - return false, nil - } - - tidbSetName := controller.TiDBMemberName(info.ClusterName) - tidbSet, err := oa.kubeCli.AppsV1beta1().StatefulSets(info.Namespace).Get(tidbSetName, metav1.GetOptions{}) - if err != nil { - glog.Infof("failed to get tidbSet statefulset: [%s], error: %v", tidbSetName, err) - return false, nil - } - - imageUpgraded := func(memberType v1alpha1.MemberType, set *v1beta1.StatefulSet) bool { - image := "" - switch memberType { - case v1alpha1.PDMemberType: - image = tc.Spec.PD.Image - case v1alpha1.TiKVMemberType: - image = tc.Spec.TiKV.Image - case v1alpha1.TiDBMemberType: - image = tc.Spec.TiDB.Image - } - memberName := string(memberType) - c, ok := getComponentContainer(set) - if !ok || c.Image != image { - glog.Infof("check %s image: getContainer(set).Image(%s) != tc.Spec.%s.Image(%s)", - memberName, c.Image, strings.ToUpper(memberName), image) - } - return ok && c.Image == image - } - setUpgraded := func(set *v1beta1.StatefulSet) bool { - return set.Generation <= *set.Status.ObservedGeneration && set.Status.CurrentRevision == set.Status.UpdateRevision - } - - // check upgrade order - if tc.Status.PD.Phase == v1alpha1.UpgradePhase { - glog.Infof("pd is upgrading") - if tc.Status.TiKV.Phase == v1alpha1.UpgradePhase { - return false, pingcapErrors.New("tikv is upgrading while pd is upgrading") - } - if tc.Status.TiDB.Phase == v1alpha1.UpgradePhase { - return false, pingcapErrors.New("tidb is upgrading while pd is upgrading") - } - if !imageUpgraded(v1alpha1.PDMemberType, pdSet) { - return false, pingcapErrors.New("pd image is not updated while pd is upgrading") - } - if !setUpgraded(pdSet) { - if imageUpgraded(v1alpha1.TiKVMemberType, tikvSet) { - return false, pingcapErrors.New("tikv image is updated while pd is upgrading") - } - if imageUpgraded(v1alpha1.TiDBMemberType, tidbSet) { - return false, pingcapErrors.New("tidb image is updated while pd is upgrading") - } - } - return false, nil - } else if tc.Status.TiKV.Phase == v1alpha1.UpgradePhase { - glog.Infof("tikv is upgrading") - if tc.Status.TiDB.Phase == v1alpha1.UpgradePhase { - return false, pingcapErrors.New("tidb is upgrading while tikv is upgrading") - } - if !imageUpgraded(v1alpha1.PDMemberType, pdSet) { - return false, pingcapErrors.New("pd image is not updated while tikv is upgrading") - } - if !setUpgraded(pdSet) { - return false, pingcapErrors.New("pd stateful set is not upgraded while tikv is upgrading") - } - if !imageUpgraded(v1alpha1.TiKVMemberType, tikvSet) { - return false, pingcapErrors.New("tikv image is not updated while tikv is upgrading") - } - if !setUpgraded(tikvSet) { - if imageUpgraded(v1alpha1.TiDBMemberType, tidbSet) { - return false, pingcapErrors.New("tidb image is updated while tikv is upgrading") - } - } - return false, nil - } else if tc.Status.TiDB.Phase == v1alpha1.UpgradePhase { - glog.Infof("tidb is upgrading") - if !imageUpgraded(v1alpha1.PDMemberType, pdSet) { - return false, pingcapErrors.New("pd image is not updated while tidb is upgrading") - } - if !setUpgraded(pdSet) { - return false, pingcapErrors.New("pd stateful set is not upgraded while tidb is upgrading") - } - if !imageUpgraded(v1alpha1.TiKVMemberType, tikvSet) { - return false, pingcapErrors.New("tikv image is not updated while tidb is upgrading") - } - if !setUpgraded(tikvSet) { - return false, pingcapErrors.New("tikv stateful set is not upgraded while tidb is upgrading") - } - if !imageUpgraded(v1alpha1.TiDBMemberType, tidbSet) { - return false, pingcapErrors.New("tidb image is not updated while tikv is upgrading") - } - return false, nil - } - - // check pd final state - if !imageUpgraded(v1alpha1.PDMemberType, pdSet) { - return false, nil - } - if !setUpgraded(pdSet) { - glog.Infof("check pd stateful set upgraded failed") - return false, nil - } - // check tikv final state - if !imageUpgraded(v1alpha1.TiKVMemberType, tikvSet) { - return false, nil - } - if !setUpgraded(tikvSet) { - glog.Infof("check tikv stateful set upgraded failed") - return false, nil - } - // check tidb final state - if !imageUpgraded(v1alpha1.TiDBMemberType, tidbSet) { - return false, nil - } - if !setUpgraded(tidbSet) { - glog.Infof("check tidb stateful set upgraded failed") - return false, nil - } - return true, nil - }) -} - func (oa *operatorActions) DeployMonitor(info *TidbClusterConfig) error { return nil } func (oa *operatorActions) CleanMonitor(info *TidbClusterConfig) error { return nil } -func getComponentContainer(set *v1beta1.StatefulSet) (corev1.Container, bool) { - name := set.Labels[label.ComponentLabelKey] - for _, c := range set.Spec.Template.Spec.Containers { - if c.Name == name { - return c, true - } +func getPodContainer (kubeCli kubernetes.Interface, namespace string, memberName string) (*corev1.Container , bool) { + name := fmt.Sprintf("%s-%d", memberName, 0) + pod, err := kubeCli.CoreV1().Pods(namespace).Get(name, metav1.GetOptions{}) + if err != nil { + glog.Errorf("fail to get pod [%s/%s]",namespace, name) + return nil,false } - return corev1.Container{}, false + return &pod.Spec.Containers[0], true } func (oa *operatorActions) pdMembersReadyFn(tc *v1alpha1.TidbCluster) (bool, error) { @@ -908,7 +793,7 @@ func (oa *operatorActions) pdMembersReadyFn(tc *v1alpha1.TidbCluster) (bool, err ns, pdSetName, pdSet.Status.ReadyReplicas, pdSet.Status.Replicas) return false, nil } - if c, ok := getComponentContainer(pdSet); !ok || tc.Spec.PD.Image != c.Image { + if c, ok := getPodContainer(oa.kubeCli, ns, pdSetName); !ok || tc.Spec.PD.Image != c.Image { glog.Infof("statefulset: %s/%s .spec.template.spec.containers[name=pd].image(%s) != %s", ns, pdSetName, c.Image, tc.Spec.PD.Image) return false, nil @@ -973,7 +858,7 @@ func (oa *operatorActions) tikvMembersReadyFn(tc *v1alpha1.TidbCluster) (bool, e ns, tikvSetName, tikvSet.Status.ReadyReplicas, tikvSet.Status.Replicas) return false, nil } - if c, ok := getComponentContainer(tikvSet); !ok || tc.Spec.TiKV.Image != c.Image { + if c, ok := getPodContainer(oa.kubeCli, ns, tikvSetName); !ok || tc.Spec.TiKV.Image != c.Image { glog.Infof("statefulset: %s/%s .spec.template.spec.containers[name=tikv].image(%s) != %s", ns, tikvSetName, c.Image, tc.Spec.TiKV.Image) return false, nil @@ -999,6 +884,39 @@ func (oa *operatorActions) tidbMembersReadyFn(tc *v1alpha1.TidbCluster) (bool, e tcName := tc.GetName() ns := tc.GetNamespace() tidbSetName := controller.TiDBMemberName(tcName) + tidbUpgradeAnnotationStr, ok := tc.Annotations[label.AnnTiDBPartition] + if !ok { + tidbUpgradeAnnotationStr = "0" + } + + tidbUpgradeAnnotation, err := strconv.ParseInt(tidbUpgradeAnnotationStr, 10, 32) + if err != nil { + return false, nil + } + + pauseCorrect := func(set *v1beta1.StatefulSet ) bool { + return (*set.Spec.UpdateStrategy.RollingUpdate.Partition) >= int32(tidbUpgradeAnnotation) + } + + upgradePaused := func() bool { + + podName := fmt.Sprintf("%s-%d", controller.TiDBMemberName(tc.Name), tidbUpgradeAnnotation) + + tidbPod, err := oa.kubeCli.CoreV1().Pods(ns).Get(podName, metav1.GetOptions{}) + if err != nil { + glog.Errorf("fail to get tidb po name %s namespace %s ", podName, ns) + return false + } + if tidbPod.Labels[v1beta1.ControllerRevisionHashLabelKey] == tc.Status.TiDB.StatefulSet.UpdateRevision && + tc.Status.TiDB.Phase == v1alpha1.UpgradePhase { + if member, ok := tc.Status.TiDB.Members[tidbPod.Name]; ok && member.Health { + return true + } + } + + return false + } + tidbSet, err := oa.kubeCli.AppsV1beta1().StatefulSets(ns).Get(tidbSetName, metav1.GetOptions{}) if err != nil { @@ -1032,8 +950,24 @@ func (oa *operatorActions) tidbMembersReadyFn(tc *v1alpha1.TidbCluster) (bool, e ns, tidbSetName, tidbSet.Status.ReadyReplicas, tidbSet.Status.Replicas) return false, nil } - if c, ok := getComponentContainer(tidbSet); !ok || tc.Spec.TiDB.Image != c.Image { - glog.Infof("statefulset: %s/%s .spec.template.spec.containers[name=tikv].image(%s) != %s", + + if !pauseCorrect(tidbSet) { + return false, fmt.Errorf("pause partition is not correct in upgrade phase [%s/%s] partition %d annotation %d", + ns, tidbSetName, (*tidbSet.Spec.UpdateStrategy.RollingUpdate.Partition), tidbUpgradeAnnotation) + } + + if upgradePaused() { + time.Sleep(30 * time.Second) + err := setPartitionAnnotation(tcName, ns, int(tidbUpgradeAnnotation - 1)) + if err != nil { + glog.Errorf("fail to set annotation for [%s/%s]", ns , tidbSetName) + return false, nil + } + return false, nil + } + + if c, ok := getPodContainer(oa.kubeCli, ns, tidbSetName); !ok || tc.Spec.TiDB.Image != c.Image { + glog.Infof("statefulset: %s/%s .spec.template.spec.containers[name=tidb].image(%s) != %s", ns, tidbSetName, c.Image, tc.Spec.TiDB.Image) return false, nil } From 2e2d8da370647e2e59524f5ac63ab080928ac557 Mon Sep 17 00:00:00 2001 From: shuijing198799 Date: Fri, 17 May 2019 12:11:25 +0800 Subject: [PATCH 06/16] make check --- .../admission-controller-deployment.yaml | 2 +- pkg/webhook/statefulset/statefulset.go | 4 ++-- tests/actions.go | 21 +++++++++---------- 3 files changed, 13 insertions(+), 14 deletions(-) diff --git a/charts/tidb-operator/templates/admission-controller-deployment.yaml b/charts/tidb-operator/templates/admission-controller-deployment.yaml index a9b9da856e0..90b29ce34f1 100644 --- a/charts/tidb-operator/templates/admission-controller-deployment.yaml +++ b/charts/tidb-operator/templates/admission-controller-deployment.yaml @@ -60,7 +60,7 @@ spec: - -tlsCertFile=/etc/webhook/certs/cert.pem - -tlsKeyFile=/etc/webhook/certs/key.pem - -alsologtostderr - - -v={{ .Values.admissionController.replicas }} + - -v={{ .Values.admissionController.logLevel }} - 2>&1 volumeMounts: - name: webhook-certs diff --git a/pkg/webhook/statefulset/statefulset.go b/pkg/webhook/statefulset/statefulset.go index 543d3eb314b..df1dad4b0dc 100644 --- a/pkg/webhook/statefulset/statefulset.go +++ b/pkg/webhook/statefulset/statefulset.go @@ -18,7 +18,6 @@ import ( "fmt" "strconv" - "k8s.io/apimachinery/pkg/runtime" "github.com/golang/glog" "github.com/pingcap/tidb-operator/pkg/apis/pingcap.com/v1alpha1" "github.com/pingcap/tidb-operator/pkg/client/clientset/versioned" @@ -27,11 +26,12 @@ import ( "k8s.io/api/admission/v1beta1" apps "k8s.io/api/apps/v1beta1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/rest" ) var ( - versionCli versioned.Interface + versionCli versioned.Interface deserializer runtime.Decoder ) diff --git a/tests/actions.go b/tests/actions.go index 148db9cbef5..781e556fdf5 100644 --- a/tests/actions.go +++ b/tests/actions.go @@ -698,7 +698,7 @@ func setPartitionAnnotation(tcName string, nameSpace string, ordinal int) error // add annotation to pause statefulset upgrade process cmd := fmt.Sprintf("kubectl annotate tc %s -n %s tidb.pingcap.com/tidb-partition=%d --overwrite", tcName, nameSpace, ordinal) - glog.Infof("%s",cmd) + glog.Infof("%s", cmd) _, err := exec.Command("/bin/sh", "-c", cmd).CombinedOutput() if err != nil { return err @@ -718,13 +718,13 @@ func (oa *operatorActions) UpgradeTidbCluster(info *TidbClusterConfig) error { tidbSetName := controller.TiDBMemberName(info.ClusterName) tidbSet, err := oa.kubeCli.AppsV1beta1().StatefulSets(info.Namespace).Get(tidbSetName, metav1.GetOptions{}) if err != nil { - return pingcapErrors.Wrapf(err,"failed to get stateful set [%s/%s] setName %s",info.Namespace, info.ClusterName,tidbSetName) + return pingcapErrors.Wrapf(err, "failed to get stateful set [%s/%s] setName %s", info.Namespace, info.ClusterName, tidbSetName) } // add annotation to pause statefulset upgrade process - err = setPartitionAnnotation(info.ClusterName, info.Namespace, int(tidbSet.Status.Replicas - 1)) + err = setPartitionAnnotation(info.ClusterName, info.Namespace, int(tidbSet.Status.Replicas-1)) if err != nil { - return pingcapErrors.Wrapf(err,"failed to add annotation to [%s/%s]",info.Namespace, info.ClusterName) + return pingcapErrors.Wrapf(err, "failed to add annotation to [%s/%s]", info.Namespace, info.ClusterName) } cmd := fmt.Sprintf("helm upgrade %s %s --set-string %s", @@ -746,12 +746,12 @@ func (oa *operatorActions) UpgradeTidbClusterOrDie(info *TidbClusterConfig) { func (oa *operatorActions) DeployMonitor(info *TidbClusterConfig) error { return nil } func (oa *operatorActions) CleanMonitor(info *TidbClusterConfig) error { return nil } -func getPodContainer (kubeCli kubernetes.Interface, namespace string, memberName string) (*corev1.Container , bool) { +func getPodContainer(kubeCli kubernetes.Interface, namespace string, memberName string) (*corev1.Container, bool) { name := fmt.Sprintf("%s-%d", memberName, 0) pod, err := kubeCli.CoreV1().Pods(namespace).Get(name, metav1.GetOptions{}) if err != nil { - glog.Errorf("fail to get pod [%s/%s]",namespace, name) - return nil,false + glog.Errorf("fail to get pod [%s/%s]", namespace, name) + return nil, false } return &pod.Spec.Containers[0], true } @@ -894,7 +894,7 @@ func (oa *operatorActions) tidbMembersReadyFn(tc *v1alpha1.TidbCluster) (bool, e return false, nil } - pauseCorrect := func(set *v1beta1.StatefulSet ) bool { + pauseCorrect := func(set *v1beta1.StatefulSet) bool { return (*set.Spec.UpdateStrategy.RollingUpdate.Partition) >= int32(tidbUpgradeAnnotation) } @@ -917,7 +917,6 @@ func (oa *operatorActions) tidbMembersReadyFn(tc *v1alpha1.TidbCluster) (bool, e return false } - tidbSet, err := oa.kubeCli.AppsV1beta1().StatefulSets(ns).Get(tidbSetName, metav1.GetOptions{}) if err != nil { glog.Errorf("failed to get statefulset: %s/%s, %v", ns, tidbSetName, err) @@ -958,9 +957,9 @@ func (oa *operatorActions) tidbMembersReadyFn(tc *v1alpha1.TidbCluster) (bool, e if upgradePaused() { time.Sleep(30 * time.Second) - err := setPartitionAnnotation(tcName, ns, int(tidbUpgradeAnnotation - 1)) + err := setPartitionAnnotation(tcName, ns, int(tidbUpgradeAnnotation-1)) if err != nil { - glog.Errorf("fail to set annotation for [%s/%s]", ns , tidbSetName) + glog.Errorf("fail to set annotation for [%s/%s]", ns, tidbSetName) return false, nil } return false, nil From 635d3b1bb1273da861e19fc95282c01ce21be0cb Mon Sep 17 00:00:00 2001 From: shuijing198799 Date: Fri, 17 May 2019 13:17:42 +0800 Subject: [PATCH 07/16] add rbac yaml --- .../templates/admission-controller-rbac.yaml | 49 +++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 charts/tidb-operator/templates/admission-controller-rbac.yaml diff --git a/charts/tidb-operator/templates/admission-controller-rbac.yaml b/charts/tidb-operator/templates/admission-controller-rbac.yaml new file mode 100644 index 00000000000..232436d1ad9 --- /dev/null +++ b/charts/tidb-operator/templates/admission-controller-rbac.yaml @@ -0,0 +1,49 @@ +{{- if .Values.admissionController.create }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: admission-controller-cr + labels: + app.kubernetes.io/name: {{ template "chart.name" . }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/component: admission-controller + helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} +rules: +- apiGroups: ["apps"] + resources: ["statefulsets"] + verbs: ["get"] +- apiGroups: ["pingcap.com"] + resources: ["tidbclusters"] + verbs: ["get"] +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: admission-controller-sa + labels: + app.kubernetes.io/name: {{ template "chart.name" . }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/component: admission-controller + helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: admission-controller-crb + labels: + app.kubernetes.io/name: {{ template "chart.name" . }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/component: admission-controller + helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} +subjects: +- kind: ServiceAccount + name: admission-controller-sa + namespace: {{ .Release.Namespace }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: admission-controller-cr +{{- end }} From 178115e222398f3f14af37ef12d481f3d1a4806b Mon Sep 17 00:00:00 2001 From: shuijing198799 Date: Mon, 20 May 2019 16:26:20 +0800 Subject: [PATCH 08/16] only pause tidb rolling upgrade once and increse the sleep time --- tests/actions.go | 30 ++++++++++++++++++++++-------- 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/tests/actions.go b/tests/actions.go index 781e556fdf5..26cf03bbfe5 100644 --- a/tests/actions.go +++ b/tests/actions.go @@ -507,7 +507,7 @@ func (oa *operatorActions) CheckTidbClusterStatus(info *TidbClusterConfig) error ns := info.Namespace tcName := info.ClusterName - if err := wait.Poll(oa.pollInterval, 30*time.Minute, func() (bool, error) { + if err := wait.Poll(oa.pollInterval, 35*time.Minute, func() (bool, error) { var tc *v1alpha1.TidbCluster var err error if tc, err = oa.cli.PingcapV1alpha1().TidbClusters(ns).Get(tcName, metav1.GetOptions{}); err != nil { @@ -753,7 +753,19 @@ func getPodContainer(kubeCli kubernetes.Interface, namespace string, memberName glog.Errorf("fail to get pod [%s/%s]", namespace, name) return nil, false } - return &pod.Spec.Containers[0], true + if len(pod.Spec.Containers) == 0 { + glog.Errorf("no container in this pod [%s/%s]", namespace, name) + return nil, false + } + + for _, container := range pod.Spec.Containers { + if container.Name == label.TiDBLabelVal || + container.Name == label.TiKVLabelVal || + container.Name == label.PDLabelVal { + return &container, true + } + } + return nil, false } func (oa *operatorActions) pdMembersReadyFn(tc *v1alpha1.TidbCluster) (bool, error) { @@ -950,14 +962,16 @@ func (oa *operatorActions) tidbMembersReadyFn(tc *v1alpha1.TidbCluster) (bool, e return false, nil } - if !pauseCorrect(tidbSet) { - return false, fmt.Errorf("pause partition is not correct in upgrade phase [%s/%s] partition %d annotation %d", + if upgradePaused() { + + time.Sleep(5 * time.Minute) + + if !pauseCorrect(tidbSet) { + return false, fmt.Errorf("pause partition is not correct in upgrade phase [%s/%s] partition %d annotation %d", ns, tidbSetName, (*tidbSet.Spec.UpdateStrategy.RollingUpdate.Partition), tidbUpgradeAnnotation) - } + } - if upgradePaused() { - time.Sleep(30 * time.Second) - err := setPartitionAnnotation(tcName, ns, int(tidbUpgradeAnnotation-1)) + err := setPartitionAnnotation(tcName, ns, 0) if err != nil { glog.Errorf("fail to set annotation for [%s/%s]", ns, tidbSetName) return false, nil From 3fa15d8469d86af9aa622525cb3acbf78ca24da9 Mon Sep 17 00:00:00 2001 From: shuijing198799 Date: Mon, 20 May 2019 16:30:58 +0800 Subject: [PATCH 09/16] make check and fix lint --- pkg/webhook/route/route.go | 2 +- pkg/webhook/statefulset/statefulset.go | 2 +- tests/actions.go | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/pkg/webhook/route/route.go b/pkg/webhook/route/route.go index 3c626ec01a8..0c343cf690c 100644 --- a/pkg/webhook/route/route.go +++ b/pkg/webhook/route/route.go @@ -1,4 +1,4 @@ -// Copyright 2018 PingCAP, Inc. +// Copyright 2019 PingCAP, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/pkg/webhook/statefulset/statefulset.go b/pkg/webhook/statefulset/statefulset.go index df1dad4b0dc..3612b513ff9 100644 --- a/pkg/webhook/statefulset/statefulset.go +++ b/pkg/webhook/statefulset/statefulset.go @@ -1,4 +1,4 @@ -// Copyright 2018 PingCAP, Inc. +// Copyright 2019 PingCAP, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/tests/actions.go b/tests/actions.go index 26cf03bbfe5..ef01a5f8ff3 100644 --- a/tests/actions.go +++ b/tests/actions.go @@ -762,8 +762,8 @@ func getPodContainer(kubeCli kubernetes.Interface, namespace string, memberName if container.Name == label.TiDBLabelVal || container.Name == label.TiKVLabelVal || container.Name == label.PDLabelVal { - return &container, true - } + return &container, true + } } return nil, false } @@ -968,7 +968,7 @@ func (oa *operatorActions) tidbMembersReadyFn(tc *v1alpha1.TidbCluster) (bool, e if !pauseCorrect(tidbSet) { return false, fmt.Errorf("pause partition is not correct in upgrade phase [%s/%s] partition %d annotation %d", - ns, tidbSetName, (*tidbSet.Spec.UpdateStrategy.RollingUpdate.Partition), tidbUpgradeAnnotation) + ns, tidbSetName, (*tidbSet.Spec.UpdateStrategy.RollingUpdate.Partition), tidbUpgradeAnnotation) } err := setPartitionAnnotation(tcName, ns, 0) From fe4387ed07146d3a79cbb2b1eca287309f2ea8ba Mon Sep 17 00:00:00 2001 From: shuijing198799 Date: Tue, 21 May 2019 10:11:32 +0800 Subject: [PATCH 10/16] add comment to introduce the usage of admission-controller --- charts/tidb-operator/values.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/charts/tidb-operator/values.yaml b/charts/tidb-operator/values.yaml index beda73863a7..d6cc8534701 100644 --- a/charts/tidb-operator/values.yaml +++ b/charts/tidb-operator/values.yaml @@ -57,9 +57,14 @@ scheduler: # kubeSchedulerImageTag: admissionController: + # switch of admission-controller create: false replicas: 1 logLevel: 2 + # The operator provides the default CA certificate and also provides a shell script to generate the certificate. + # (https://github.com/pingcap/tidb-operator/tree/master/manifests/create-cert.sh). + # Run the script and fill the corresponding entries into the values file. + # Note: create-cert.sh is forced to fill a namespace parameter, which needs to be the same as the tidb-operator's namespace. key: LS0tLS1CRUdJTiBSU0EgUFJJVkFURSBLRVktLS0tLQpNSUlFb3dJQkFBS0NBUUVBd3B5cmkvWnJJdS9TNEJlU05KZHBFN1JCeFF6MytaMTdxcXZWY2JweUY0cEZHdlMrCm1pWGF5ZDhVd3BYcDZJT3hDNDd3UWFhU1JnYm9qUTBYcjllWm1OS0NsWkxROUgwWVpTWndXSU4zL29vTEdWT3AKcDQ5ampBaDlFSmNhSTVFOUpNSlQvWFk1NHlZeTJ6a2lzaWhCUnI5b2FyMHZHZjdaeElQSGNndG5ES1ZFVm5vTgpJRGJuVTU2RE1GZk9kTWlvUURDU0ppakpPaU9pdXhFNTFVbG9aTVVVN3FCdTV5NUFtYTN2NVErd3dIR252ZUh1CmZ4TFpJN1plc3hhenlRMnU4SkFKZ1pIeWdEM1FTS1Z5REw4M1IyOHRmNDd0cm5QT1dOOXNreTNNNm9rZFZCaFQKUFVhUmRGMjkrWFFJdG56TG9Ua3RVSGtzdWVmTDF3dTNaVWdyVndJREFRQUJBb0lCQUF3bXRCOWRGM3RxOWdPLwo5UVVMbnlmM3pWTjhaODlTSWZLYUlBMmwwcEhkdEVybWxUQVN4UUp0eDJJTzhqZnVZUk9UclRTc0xkM3BlSU8yCkdaQ0ZLYlg5RHVJLzlJUVBlNEoxVC9aMjN2c3BxMSs5ZFk1Y3laNW91VVpiNk9ubm02Vzc3SHFhOHlwWHJEMDcKNkFxdnNNTzJGOUx5VlFxT2V4Sm1HQUhxN0ZoQnJWSk9QTS9lNWIydGl1SDFyUUJqVVhZWXAxVHp3R3BFM1cyQgpvRVI3bjlvNEJtNysyTnZ5bHAwNFdWWm9UNW5rRXVtcXB3ajR5NFdUYlBYNHFMUnJMVzJ4QnhnTzdUdVFQeVlFCkJUZFFZVkFhRm55akFXdXE3VDNETWJXVzQ3eDVnZ3BGR2hRNk9GYjRNUzN5RkpNZlE4eWhYblpOend6L1NvQXoKSWxub21pRUNnWUVBL2t0L0EreDhDcnZpWE1RR0duWVB6NW1sRmlmRW5FTEQwVkY0ak5pNFRWblBLYVhocEFCYgpNS3UwOXRuWXFnTHRacURBc0c4ZCtxWVlKOVZaQjMreUVnWTMwY043K1phVTlJcS81Q3R3N2p3d3JFMVEwTFN5CjhnQk9sZFVGUzVUMXFBOXhyOGx6WW1jdTl3TWpEZUNaTGxrdWdzWFlBRGF5eHRMRnpBU0kzWDBDZ1lFQXcrcTYKQnhGMjhkUFdwQk95bk5mcUIxdVc1Lyt6S3RhREh4Y1RMenRpaWdrM2NUbm1seTB0Zll4eExpeGs4aFVmV3dmQwoyejFDNmhxV1RFeS9nZTJHaWhlOFJDL1NFMGhvNEZyYUYwNVAvM0Voa0lwOFM1cWpobmdSdjJ2S1ZKU1BTLzNpCnJPRHBxNWxKU3pVc2lXWktTRFJ6RXlvRXI3Y2NodXpsNm8vYjFHTUNnWUFXMnE0bHFpRlR6YU1BZDFXZEZ6TkEKa3l4MUcwbmVTVm5UL0NCN2VCYklUT1FEZURjWlNEYnAzdVM2c3NWNE1oOG45QmFkUkw5TEFaWjJMZ1I4V2d1awo5dzRaT2hwd1NncEZHQnM2NUhTWDFLMERMMGV1R2k5aTlJZ2Jpek9Zb1dDYXBCaXhsY2RhOXE4UDRDWk1YdjBsClh6cUE0a3NSYWRrVGFXb0JkZjdlK1FLQmdRQ01PN0dZYVpuWUV5ZXFIaWRWT1A5YzhPUWRmOXVCS0RhR3hIK0kKNlZycEx4dXVRaUdaeUU2dGRSOTJTM1NlQ3h4OGpodWR6a1hkUjlhWkdyNWhnK0V2YXpucEhNbWozOHFoL0tYcQoxTFZTUkdvZ0NlQmFSWUFUT2xLSno5L3Rsc0RUUmhGQWFXVVkzUW1uZVpJOWJ5aHdjb3FFQnlsL2xQUW9aOTFTCmZsSjl2UUtCZ0dBTXRybTRFVkRTckF0QS9qeXg3NFB3SThjc2xtUEQ1d3JaWEZQVTUvc3FyRGNKbUNqSXBJRWUKRFdSaWNYL09aSUpRbDdjNkFOQjc1U1VUMHZ4dXJXeGp6WWlPVjRvUTJVUTBFc1VxcUh3RWxDVWZjclcxQnh2eQpmMnE3aGtUL0MySWwyRjhmRmxnYjJmZFVmaFM5MjZSbTF0aUUyMzJMU3MwYmZiWHArSDFmCi0tLS0tRU5EIFJTQSBQUklWQVRFIEtFWS0tLS0tCg== cert: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURsVENDQW4yZ0F3SUJBZ0lVSmdOQXRuQ25ROVdLRWRVK29pR0lKQUxxTWFRd0RRWUpLb1pJaHZjTkFRRUwKQlFBd0ZURVRNQkVHQTFVRUF4TUthM1ZpWlhKdVpYUmxjekFlRncweE9UQTFNVGN3TXpJMU1EQmFGdzB5TURBMQpNVFl3TXpJMU1EQmFNQzh4TFRBckJnTlZCQU1USkdGa2JXbHpjMmx2YmkxamIyNTBjbTlzYkdWeUxYTjJZeTV3CmFXNW5ZMkZ3TG5OMll6Q0NBU0l3RFFZSktvWklodmNOQVFFQkJRQURnZ0VQQURDQ0FRb0NnZ0VCQU1LY3E0djIKYXlMdjB1QVhralNYYVJPMFFjVU05L21kZTZxcjFYRzZjaGVLUlJyMHZwb2wyc25mRk1LVjZlaURzUXVPOEVHbQpra1lHNkkwTkY2L1htWmpTZ3BXUzBQUjlHR1VtY0ZpRGQvNktDeGxUcWFlUFk0d0lmUkNYR2lPUlBTVENVLzEyCk9lTW1NdHM1SXJJb1FVYS9hR3E5THhuKzJjU0R4M0lMWnd5bFJGWjZEU0EyNTFPZWd6Qlh6blRJcUVBd2tpWW8KeVRvam9yc1JPZFZKYUdURkZPNmdidWN1UUptdDcrVVBzTUJ4cDczaDduOFMyU08yWHJNV3M4a05ydkNRQ1lHUgo4b0E5MEVpbGNneS9OMGR2TFgrTzdhNXp6bGpmYkpNdHpPcUpIVlFZVXoxR2tYUmR2ZmwwQ0xaOHk2RTVMVkI1CkxMbm55OWNMdDJWSUsxY0NBd0VBQWFPQndqQ0J2ekFPQmdOVkhROEJBZjhFQkFNQ0JhQXdFd1lEVlIwbEJBd3cKQ2dZSUt3WUJCUVVIQXdFd0RBWURWUjBUQVFIL0JBSXdBREFkQmdOVkhRNEVGZ1FVYlVwSk9UeGE4bkljelpnTgpTZ2hvUnRTQTcwSXdhd1lEVlIwUkJHUXdZb0lZWVdSdGFYTnphVzl1TFdOdmJuUnliMnhzWlhJdGMzWmpnaUJoClpHMXBjM05wYjI0dFkyOXVkSEp2Ykd4bGNpMXpkbU11Y0dsdVoyTmhjSUlrWVdSdGFYTnphVzl1TFdOdmJuUnkKYjJ4c1pYSXRjM1pqTG5CcGJtZGpZWEF1YzNaak1BMEdDU3FHU0liM0RRRUJDd1VBQTRJQkFRQlZPYUF5dVo1VAp3eTdaK3FYcnlJUTVDWE1DUGVNUVN5M0ZwZFVtVUEvUDJnamtsUWg3RStYdkZ5UjNzLzEzQlluYnViWlNVRlMxCjdTTEUvd3duN1N0c1hVZ3M1RkMweTRYNWpoWGZmZEQwcDVKNzQxMWwxVU9MdHVZZkFiL1VGM1c4RjZxNlhXL0kKRkFVTEhaUjdMTHNpZlZCMTFmNHZwMmw2c2V2emF2WVIyeXY5aUJwVU9oSXpZWTFTelNCOXlzOVgxZVBEUWRQVApUYVE2VHlidFIzYUE2Y3d6S2JiZ200THhSYzRWRzZzU3lFRDJBZkZWeURSMXdqb3pXQmw5OWJGWWNSSVVaZG4wCjFCQ0d5SWNSS0pYQjhoUlRabEZyMnNQRFdpL000YWEwcUd0K05jTDdPM0Zwa3JWeW9jZzhHczhlY3Fjenk4VGkKTHJueXREU1hoQW5aCi0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K ca: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUN5RENDQWJDZ0F3SUJBZ0lCQURBTkJna3Foa2lHOXcwQkFRc0ZBREFWTVJNd0VRWURWUVFERXdwcmRXSmwKY201bGRHVnpNQjRYRFRFNU1EVXhNREEzTkRZek4xb1hEVEk1TURVd056QTNORFl6TjFvd0ZURVRNQkVHQTFVRQpBeE1LYTNWaVpYSnVaWFJsY3pDQ0FTSXdEUVlKS29aSWh2Y05BUUVCQlFBRGdnRVBBRENDQVFvQ2dnRUJBSzN4CkluTDhJREpJeC9rMHQ5QmovZGxKMEU4Q1drMkpoYzVwN29SeEhSdm8rQjFlMEthbTkzS1lpeTNGeFIvVW1iWGIKWkhGb3VEaGZkSUhjYmw4dytkQktDYUc5SlduZUlYTExoSU1wTUlTRHFGYWdDTjNZTHpmUVFsVFd1RmpXVnZFRAphWUFuOG1kd3E5bjJLaTY0a2szR2Zjb2V6cFpIVzVYVjhaU2JnVGQyNjlMRkFKLzVZNlRUd3FvMWFqM1M4czZTCnl5L2FrQ0V5Znd4UXZsZ1I1NDErWWs4VkViOURYMFk4Vm8xLzJod1NyVm5YbFRkWTNWdXlMSkwvTUxOMmpMTVkKdG5ON0VRbE5YdHNSVkF1YUI3WmFmZG04c3BpU1MyYkpQcG91YlB0azdSWmRvSmo3dGVxU0VJdTF1UUZacUNERgpmQVlBU3lXOG5ZZjFSa1B4RjVVQ0F3RUFBYU1qTUNFd0RnWURWUjBQQVFIL0JBUURBZ0trTUE4R0ExVWRFd0VCCi93UUZNQU1CQWY4d0RRWUpLb1pJaHZjTkFRRUxCUUFEZ2dFQkFDSFBPZHpXRDRmWWIyakIyeXdkaFhiT1VyamMKdHZ5bXF2eXJ4enpZazZDRGNUN3hRL2FQYTBRZUE0WkxWY3c0eTZaalZkMmR3enZxd2llNDBUWUw2eFM1KzVkMwpyNE1KRXdBaFp1bk5ZbUVybWtjZmJJL3M2Q0dBaVNjd2RhRlp5U2I3Yk5lU090NHkraDBPaDRnc2lEWU8yNUFSCkcrc3MzK2ZDbWZYVTNFKzRhdGNqcnhjSU1iRmFmejVhVEJUdlp4ajN3Z01FRzV6MjE5anplbXJZdWJsazBxd1kKbjB6d0xDTEpubkZtN0hrK0tnTjRXeUlTcjFqZzBTTDl0YmVxWi8wd0NWM01lMFB0VUgzdFR5S1dSTHV4cFk5YwpsZHlmQXlMMmM5eElYZnFrcHczMHowVEo0NHNrVTRmU3NjWlhNV0VwSFowNDdnV1FsRi9mZW9zS1c0OD0KLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo= From 35f3dd459b56da8120cf102ed41dfbfb90755f40 Mon Sep 17 00:00:00 2001 From: shuijing198799 Date: Wed, 22 May 2019 14:18:28 +0800 Subject: [PATCH 11/16] use yaml to manage the admission-controller and modify via comment --- .gitignore | 1 + Makefile | 2 + .../admission-controller-deployment.yaml | 101 ---------------- .../templates/admission-controller-rbac.yaml | 49 -------- charts/tidb-operator/values.yaml | 13 --- cmd/admission-controller/main.go | 4 + manifests/create-cert.sh | 31 ++--- manifests/patch-ca.sh | 10 ++ manifests/webhook.yaml | 109 ++++++++++++++++++ pkg/tkctl/cmd/upinfo/upinfo.go | 4 +- pkg/webhook/route/route.go | 19 +-- pkg/webhook/util/util.go | 4 +- tests/actions.go | 67 +++++++++-- tests/config.go | 9 ++ tests/images/e2e/Dockerfile | 3 +- 15 files changed, 216 insertions(+), 210 deletions(-) delete mode 100644 charts/tidb-operator/templates/admission-controller-deployment.yaml delete mode 100644 charts/tidb-operator/templates/admission-controller-rbac.yaml create mode 100755 manifests/patch-ca.sh create mode 100644 manifests/webhook.yaml diff --git a/.gitignore b/.gitignore index bab2459fdf6..c735f5de6da 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,7 @@ tests/images/fault-trigger/bin/ tests/images/e2e/tidb-cluster/ tests/images/e2e/tidb-backup/ tests/images/e2e/tidb-operator/ +tests/images/e2e/manifests/ *.tar tmp/ data/ diff --git a/Makefile b/Makefile index f8493fa6c17..19bd88adda6 100644 --- a/Makefile +++ b/Makefile @@ -54,9 +54,11 @@ e2e-docker: e2e-build [ -d tests/images/e2e/tidb-operator ] && rm -r tests/images/e2e/tidb-operator || true [ -d tests/images/e2e/tidb-cluster ] && rm -r tests/images/e2e/tidb-cluster || true [ -d tests/images/e2e/tidb-backup ] && rm -r tests/images/e2e/tidb-backup || true + [ -d tests/images/e2e/manifests ] && rm -r tests/images/e2e/manifests || true cp -r charts/tidb-operator tests/images/e2e cp -r charts/tidb-cluster tests/images/e2e cp -r charts/tidb-backup tests/images/e2e + cp -r manifests tests/images/e2e docker build -t "${DOCKER_REGISTRY}/pingcap/tidb-operator-e2e:latest" tests/images/e2e e2e-build: e2e-setup diff --git a/charts/tidb-operator/templates/admission-controller-deployment.yaml b/charts/tidb-operator/templates/admission-controller-deployment.yaml deleted file mode 100644 index 90b29ce34f1..00000000000 --- a/charts/tidb-operator/templates/admission-controller-deployment.yaml +++ /dev/null @@ -1,101 +0,0 @@ -{{- if .Values.admissionController.create }} -apiVersion: v1 -kind: Secret -metadata: - name: admission-controller-certs -data: - cert.pem: {{ .Values.admissionController.cert }} - key.pem: {{ .Values.admissionController.key }} ---- -apiVersion: v1 -kind: Service -metadata: - name: admission-controller-svc - labels: - app.kubernetes.io/name: {{ template "chart.name" . }} - app.kubernetes.io/managed-by: {{ .Release.Service }} - app.kubernetes.io/instance: {{ .Release.Name }} - app.kubernetes.io/component: admission-controller - helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} -spec: - ports: - - port: 443 - targetPort: 443 - selector: - app.kubernetes.io/name: {{ template "chart.name" . }} - app.kubernetes.io/instance: {{ .Release.Name }} - app.kubernetes.io/component: admission-controller ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: tidb-admission-controller-deployment - labels: - app.kubernetes.io/name: {{ template "chart.name" . }} - app.kubernetes.io/managed-by: {{ .Release.Service }} - app.kubernetes.io/instance: {{ .Release.Name }} - app.kubernetes.io/component: admission-controller - helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} -spec: - replicas: {{ .Values.admissionController.replicas }} - selector: - matchLabels: - app.kubernetes.io/name: {{ template "chart.name" . }} - app.kubernetes.io/instance: {{ .Release.Name }} - app.kubernetes.io/component: admission-controller - template: - metadata: - labels: - app.kubernetes.io/name: {{ template "chart.name" . }} - app.kubernetes.io/instance: {{ .Release.Name }} - app.kubernetes.io/component: admission-controller - spec: - serviceAccountName: admission-controller-sa - containers: - - name: tidb-admission-controller-pod - image: {{ .Values.operatorImage }} - imagePullPolicy: {{ .Values.imagePullPolicy | default "IfNotPresent" }} - command: - - /usr/local/bin/tidb-admission-controller - - -tlsCertFile=/etc/webhook/certs/cert.pem - - -tlsKeyFile=/etc/webhook/certs/key.pem - - -alsologtostderr - - -v={{ .Values.admissionController.logLevel }} - - 2>&1 - volumeMounts: - - name: webhook-certs - mountPath: /etc/webhook/certs - readOnly: true - volumes: - - name: webhook-certs - secret: - secretName: admission-controller-certs - env: - - name: TZ - value: {{ .Values.timezone | default "UTC" }} ---- -apiVersion: admissionregistration.k8s.io/v1beta1 -kind: ValidatingWebhookConfiguration -metadata: - name: validation-controller-cfg - labels: - app.kubernetes.io/name: {{ template "chart.name" . }} - app.kubernetes.io/managed-by: {{ .Release.Service }} - app.kubernetes.io/instance: {{ .Release.Name }} - app.kubernetes.io/component: admission-controller - helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} -webhooks: - - name: statefulset-webhook.pingcap.net - failurePolicy: Fail - clientConfig: - service: - name: admission-controller-svc - path: "/statefulsets" - namespace: {{ .Release.Namespace }} - caBundle: {{ .Values.admissionController.ca }} - rules: - - operations: [ "UPDATE" ] - apiGroups: [ "apps", "" ] - apiVersions: ["v1beta1"] - resources: ["statefulsets"] -{{- end }} diff --git a/charts/tidb-operator/templates/admission-controller-rbac.yaml b/charts/tidb-operator/templates/admission-controller-rbac.yaml deleted file mode 100644 index 232436d1ad9..00000000000 --- a/charts/tidb-operator/templates/admission-controller-rbac.yaml +++ /dev/null @@ -1,49 +0,0 @@ -{{- if .Values.admissionController.create }} -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: admission-controller-cr - labels: - app.kubernetes.io/name: {{ template "chart.name" . }} - app.kubernetes.io/managed-by: {{ .Release.Service }} - app.kubernetes.io/instance: {{ .Release.Name }} - app.kubernetes.io/component: admission-controller - helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} -rules: -- apiGroups: ["apps"] - resources: ["statefulsets"] - verbs: ["get"] -- apiGroups: ["pingcap.com"] - resources: ["tidbclusters"] - verbs: ["get"] ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: admission-controller-sa - labels: - app.kubernetes.io/name: {{ template "chart.name" . }} - app.kubernetes.io/managed-by: {{ .Release.Service }} - app.kubernetes.io/instance: {{ .Release.Name }} - app.kubernetes.io/component: admission-controller - helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: admission-controller-crb - labels: - app.kubernetes.io/name: {{ template "chart.name" . }} - app.kubernetes.io/managed-by: {{ .Release.Service }} - app.kubernetes.io/instance: {{ .Release.Name }} - app.kubernetes.io/component: admission-controller - helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} -subjects: -- kind: ServiceAccount - name: admission-controller-sa - namespace: {{ .Release.Namespace }} -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: admission-controller-cr -{{- end }} diff --git a/charts/tidb-operator/values.yaml b/charts/tidb-operator/values.yaml index d6cc8534701..98a920632fc 100644 --- a/charts/tidb-operator/values.yaml +++ b/charts/tidb-operator/values.yaml @@ -55,16 +55,3 @@ scheduler: kubeSchedulerImageName: gcr.io/google-containers/hyperkube # This will default to matching your kubernetes version # kubeSchedulerImageTag: - -admissionController: - # switch of admission-controller - create: false - replicas: 1 - logLevel: 2 - # The operator provides the default CA certificate and also provides a shell script to generate the certificate. - # (https://github.com/pingcap/tidb-operator/tree/master/manifests/create-cert.sh). - # Run the script and fill the corresponding entries into the values file. - # Note: create-cert.sh is forced to fill a namespace parameter, which needs to be the same as the tidb-operator's namespace. - key: LS0tLS1CRUdJTiBSU0EgUFJJVkFURSBLRVktLS0tLQpNSUlFb3dJQkFBS0NBUUVBd3B5cmkvWnJJdS9TNEJlU05KZHBFN1JCeFF6MytaMTdxcXZWY2JweUY0cEZHdlMrCm1pWGF5ZDhVd3BYcDZJT3hDNDd3UWFhU1JnYm9qUTBYcjllWm1OS0NsWkxROUgwWVpTWndXSU4zL29vTEdWT3AKcDQ5ampBaDlFSmNhSTVFOUpNSlQvWFk1NHlZeTJ6a2lzaWhCUnI5b2FyMHZHZjdaeElQSGNndG5ES1ZFVm5vTgpJRGJuVTU2RE1GZk9kTWlvUURDU0ppakpPaU9pdXhFNTFVbG9aTVVVN3FCdTV5NUFtYTN2NVErd3dIR252ZUh1CmZ4TFpJN1plc3hhenlRMnU4SkFKZ1pIeWdEM1FTS1Z5REw4M1IyOHRmNDd0cm5QT1dOOXNreTNNNm9rZFZCaFQKUFVhUmRGMjkrWFFJdG56TG9Ua3RVSGtzdWVmTDF3dTNaVWdyVndJREFRQUJBb0lCQUF3bXRCOWRGM3RxOWdPLwo5UVVMbnlmM3pWTjhaODlTSWZLYUlBMmwwcEhkdEVybWxUQVN4UUp0eDJJTzhqZnVZUk9UclRTc0xkM3BlSU8yCkdaQ0ZLYlg5RHVJLzlJUVBlNEoxVC9aMjN2c3BxMSs5ZFk1Y3laNW91VVpiNk9ubm02Vzc3SHFhOHlwWHJEMDcKNkFxdnNNTzJGOUx5VlFxT2V4Sm1HQUhxN0ZoQnJWSk9QTS9lNWIydGl1SDFyUUJqVVhZWXAxVHp3R3BFM1cyQgpvRVI3bjlvNEJtNysyTnZ5bHAwNFdWWm9UNW5rRXVtcXB3ajR5NFdUYlBYNHFMUnJMVzJ4QnhnTzdUdVFQeVlFCkJUZFFZVkFhRm55akFXdXE3VDNETWJXVzQ3eDVnZ3BGR2hRNk9GYjRNUzN5RkpNZlE4eWhYblpOend6L1NvQXoKSWxub21pRUNnWUVBL2t0L0EreDhDcnZpWE1RR0duWVB6NW1sRmlmRW5FTEQwVkY0ak5pNFRWblBLYVhocEFCYgpNS3UwOXRuWXFnTHRacURBc0c4ZCtxWVlKOVZaQjMreUVnWTMwY043K1phVTlJcS81Q3R3N2p3d3JFMVEwTFN5CjhnQk9sZFVGUzVUMXFBOXhyOGx6WW1jdTl3TWpEZUNaTGxrdWdzWFlBRGF5eHRMRnpBU0kzWDBDZ1lFQXcrcTYKQnhGMjhkUFdwQk95bk5mcUIxdVc1Lyt6S3RhREh4Y1RMenRpaWdrM2NUbm1seTB0Zll4eExpeGs4aFVmV3dmQwoyejFDNmhxV1RFeS9nZTJHaWhlOFJDL1NFMGhvNEZyYUYwNVAvM0Voa0lwOFM1cWpobmdSdjJ2S1ZKU1BTLzNpCnJPRHBxNWxKU3pVc2lXWktTRFJ6RXlvRXI3Y2NodXpsNm8vYjFHTUNnWUFXMnE0bHFpRlR6YU1BZDFXZEZ6TkEKa3l4MUcwbmVTVm5UL0NCN2VCYklUT1FEZURjWlNEYnAzdVM2c3NWNE1oOG45QmFkUkw5TEFaWjJMZ1I4V2d1awo5dzRaT2hwd1NncEZHQnM2NUhTWDFLMERMMGV1R2k5aTlJZ2Jpek9Zb1dDYXBCaXhsY2RhOXE4UDRDWk1YdjBsClh6cUE0a3NSYWRrVGFXb0JkZjdlK1FLQmdRQ01PN0dZYVpuWUV5ZXFIaWRWT1A5YzhPUWRmOXVCS0RhR3hIK0kKNlZycEx4dXVRaUdaeUU2dGRSOTJTM1NlQ3h4OGpodWR6a1hkUjlhWkdyNWhnK0V2YXpucEhNbWozOHFoL0tYcQoxTFZTUkdvZ0NlQmFSWUFUT2xLSno5L3Rsc0RUUmhGQWFXVVkzUW1uZVpJOWJ5aHdjb3FFQnlsL2xQUW9aOTFTCmZsSjl2UUtCZ0dBTXRybTRFVkRTckF0QS9qeXg3NFB3SThjc2xtUEQ1d3JaWEZQVTUvc3FyRGNKbUNqSXBJRWUKRFdSaWNYL09aSUpRbDdjNkFOQjc1U1VUMHZ4dXJXeGp6WWlPVjRvUTJVUTBFc1VxcUh3RWxDVWZjclcxQnh2eQpmMnE3aGtUL0MySWwyRjhmRmxnYjJmZFVmaFM5MjZSbTF0aUUyMzJMU3MwYmZiWHArSDFmCi0tLS0tRU5EIFJTQSBQUklWQVRFIEtFWS0tLS0tCg== - cert: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURsVENDQW4yZ0F3SUJBZ0lVSmdOQXRuQ25ROVdLRWRVK29pR0lKQUxxTWFRd0RRWUpLb1pJaHZjTkFRRUwKQlFBd0ZURVRNQkVHQTFVRUF4TUthM1ZpWlhKdVpYUmxjekFlRncweE9UQTFNVGN3TXpJMU1EQmFGdzB5TURBMQpNVFl3TXpJMU1EQmFNQzh4TFRBckJnTlZCQU1USkdGa2JXbHpjMmx2YmkxamIyNTBjbTlzYkdWeUxYTjJZeTV3CmFXNW5ZMkZ3TG5OMll6Q0NBU0l3RFFZSktvWklodmNOQVFFQkJRQURnZ0VQQURDQ0FRb0NnZ0VCQU1LY3E0djIKYXlMdjB1QVhralNYYVJPMFFjVU05L21kZTZxcjFYRzZjaGVLUlJyMHZwb2wyc25mRk1LVjZlaURzUXVPOEVHbQpra1lHNkkwTkY2L1htWmpTZ3BXUzBQUjlHR1VtY0ZpRGQvNktDeGxUcWFlUFk0d0lmUkNYR2lPUlBTVENVLzEyCk9lTW1NdHM1SXJJb1FVYS9hR3E5THhuKzJjU0R4M0lMWnd5bFJGWjZEU0EyNTFPZWd6Qlh6blRJcUVBd2tpWW8KeVRvam9yc1JPZFZKYUdURkZPNmdidWN1UUptdDcrVVBzTUJ4cDczaDduOFMyU08yWHJNV3M4a05ydkNRQ1lHUgo4b0E5MEVpbGNneS9OMGR2TFgrTzdhNXp6bGpmYkpNdHpPcUpIVlFZVXoxR2tYUmR2ZmwwQ0xaOHk2RTVMVkI1CkxMbm55OWNMdDJWSUsxY0NBd0VBQWFPQndqQ0J2ekFPQmdOVkhROEJBZjhFQkFNQ0JhQXdFd1lEVlIwbEJBd3cKQ2dZSUt3WUJCUVVIQXdFd0RBWURWUjBUQVFIL0JBSXdBREFkQmdOVkhRNEVGZ1FVYlVwSk9UeGE4bkljelpnTgpTZ2hvUnRTQTcwSXdhd1lEVlIwUkJHUXdZb0lZWVdSdGFYTnphVzl1TFdOdmJuUnliMnhzWlhJdGMzWmpnaUJoClpHMXBjM05wYjI0dFkyOXVkSEp2Ykd4bGNpMXpkbU11Y0dsdVoyTmhjSUlrWVdSdGFYTnphVzl1TFdOdmJuUnkKYjJ4c1pYSXRjM1pqTG5CcGJtZGpZWEF1YzNaak1BMEdDU3FHU0liM0RRRUJDd1VBQTRJQkFRQlZPYUF5dVo1VAp3eTdaK3FYcnlJUTVDWE1DUGVNUVN5M0ZwZFVtVUEvUDJnamtsUWg3RStYdkZ5UjNzLzEzQlluYnViWlNVRlMxCjdTTEUvd3duN1N0c1hVZ3M1RkMweTRYNWpoWGZmZEQwcDVKNzQxMWwxVU9MdHVZZkFiL1VGM1c4RjZxNlhXL0kKRkFVTEhaUjdMTHNpZlZCMTFmNHZwMmw2c2V2emF2WVIyeXY5aUJwVU9oSXpZWTFTelNCOXlzOVgxZVBEUWRQVApUYVE2VHlidFIzYUE2Y3d6S2JiZ200THhSYzRWRzZzU3lFRDJBZkZWeURSMXdqb3pXQmw5OWJGWWNSSVVaZG4wCjFCQ0d5SWNSS0pYQjhoUlRabEZyMnNQRFdpL000YWEwcUd0K05jTDdPM0Zwa3JWeW9jZzhHczhlY3Fjenk4VGkKTHJueXREU1hoQW5aCi0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K - ca: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUN5RENDQWJDZ0F3SUJBZ0lCQURBTkJna3Foa2lHOXcwQkFRc0ZBREFWTVJNd0VRWURWUVFERXdwcmRXSmwKY201bGRHVnpNQjRYRFRFNU1EVXhNREEzTkRZek4xb1hEVEk1TURVd056QTNORFl6TjFvd0ZURVRNQkVHQTFVRQpBeE1LYTNWaVpYSnVaWFJsY3pDQ0FTSXdEUVlKS29aSWh2Y05BUUVCQlFBRGdnRVBBRENDQVFvQ2dnRUJBSzN4CkluTDhJREpJeC9rMHQ5QmovZGxKMEU4Q1drMkpoYzVwN29SeEhSdm8rQjFlMEthbTkzS1lpeTNGeFIvVW1iWGIKWkhGb3VEaGZkSUhjYmw4dytkQktDYUc5SlduZUlYTExoSU1wTUlTRHFGYWdDTjNZTHpmUVFsVFd1RmpXVnZFRAphWUFuOG1kd3E5bjJLaTY0a2szR2Zjb2V6cFpIVzVYVjhaU2JnVGQyNjlMRkFKLzVZNlRUd3FvMWFqM1M4czZTCnl5L2FrQ0V5Znd4UXZsZ1I1NDErWWs4VkViOURYMFk4Vm8xLzJod1NyVm5YbFRkWTNWdXlMSkwvTUxOMmpMTVkKdG5ON0VRbE5YdHNSVkF1YUI3WmFmZG04c3BpU1MyYkpQcG91YlB0azdSWmRvSmo3dGVxU0VJdTF1UUZacUNERgpmQVlBU3lXOG5ZZjFSa1B4RjVVQ0F3RUFBYU1qTUNFd0RnWURWUjBQQVFIL0JBUURBZ0trTUE4R0ExVWRFd0VCCi93UUZNQU1CQWY4d0RRWUpLb1pJaHZjTkFRRUxCUUFEZ2dFQkFDSFBPZHpXRDRmWWIyakIyeXdkaFhiT1VyamMKdHZ5bXF2eXJ4enpZazZDRGNUN3hRL2FQYTBRZUE0WkxWY3c0eTZaalZkMmR3enZxd2llNDBUWUw2eFM1KzVkMwpyNE1KRXdBaFp1bk5ZbUVybWtjZmJJL3M2Q0dBaVNjd2RhRlp5U2I3Yk5lU090NHkraDBPaDRnc2lEWU8yNUFSCkcrc3MzK2ZDbWZYVTNFKzRhdGNqcnhjSU1iRmFmejVhVEJUdlp4ajN3Z01FRzV6MjE5anplbXJZdWJsazBxd1kKbjB6d0xDTEpubkZtN0hrK0tnTjRXeUlTcjFqZzBTTDl0YmVxWi8wd0NWM01lMFB0VUgzdFR5S1dSTHV4cFk5YwpsZHlmQXlMMmM5eElYZnFrcHczMHowVEo0NHNrVTRmU3NjWlhNV0VwSFowNDdnV1FsRi9mZW9zS1c0OD0KLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo= diff --git a/cmd/admission-controller/main.go b/cmd/admission-controller/main.go index bbe3c4ebd1d..a0ca6f18d66 100644 --- a/cmd/admission-controller/main.go +++ b/cmd/admission-controller/main.go @@ -25,6 +25,7 @@ import ( "github.com/pingcap/tidb-operator/version" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" + "k8s.io/apiserver/pkg/util/logs" ) var ( @@ -43,6 +44,9 @@ func init() { func main() { + logs.InitLogs() + defer logs.FlushLogs() + if printVersion { version.PrintVersionInfo() os.Exit(0) diff --git a/manifests/create-cert.sh b/manifests/create-cert.sh index ee23323040b..89ad82b4397 100755 --- a/manifests/create-cert.sh +++ b/manifests/create-cert.sh @@ -14,29 +14,14 @@ detailed explantion and additional instructions. The server key/cert k8s CA cert are stored in a k8s secret. -usage: ${0} [OPTIONS] - -The following flags are required. - - --namespace Namespace where webhook service and secret reside. EOF exit 1 } -while [[ $# -gt 0 ]]; do - case ${1} in - --namespace) - namespace="$2" - shift - ;; - *) - usage - ;; - esac - shift -done - +namespace=default service=admission-controller-svc +secret=admission-controller-certs + if [ -z ${namespace} ]; then echo "need input namespace" exit 1 @@ -113,7 +98,9 @@ fi echo ${serverCert} | openssl base64 -d -A -out ${tmpdir}/server-cert.pem -CA_BUNDLE=$(kubectl get configmap -n kube-system extension-apiserver-authentication -o=jsonpath='{.data.client-ca-file}' | base64 | tr -d '\n') -echo ca: $CA_BUNDLE -echo key: $(cat ${tmpdir}/server-key.pem | base64| tr -d '\n') -echo cert: $(cat ${tmpdir}/server-cert.pem | base64| tr -d '\n') +# create the secret with CA cert and server cert/key +kubectl create secret generic ${secret} \ + --from-file=key.pem=${tmpdir}/server-key.pem \ + --from-file=cert.pem=${tmpdir}/server-cert.pem \ + --dry-run -o yaml | + kubectl -n ${namespace} apply -f - diff --git a/manifests/patch-ca.sh b/manifests/patch-ca.sh new file mode 100755 index 00000000000..f222369e906 --- /dev/null +++ b/manifests/patch-ca.sh @@ -0,0 +1,10 @@ +#!/bin/bash +set -o errexit +set -o nounset +set -o pipefail + +CURDIR=$(cd $(dirname ${BASH_SOURCE[0]}); pwd ) + +CA_BUNDLE=$(kubectl get configmap -n kube-system extension-apiserver-authentication -o=jsonpath='{.data.client-ca-file}' | base64 | tr -d '\n') +echo $CA_BUNDLE +sed -i "s/caBundle: .*$/caBundle: ${CA_BUNDLE}/g" $CURDIR/webhook.yaml diff --git a/manifests/webhook.yaml b/manifests/webhook.yaml new file mode 100644 index 00000000000..2581036f019 --- /dev/null +++ b/manifests/webhook.yaml @@ -0,0 +1,109 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: admission-controller-cr + namespace: default + labels: + app: admission-webhook +rules: +- apiGroups: ["apps"] + resources: ["statefulsets"] + verbs: ["get"] +- apiGroups: ["pingcap.com"] + resources: ["tidbclusters"] + verbs: ["get"] +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: admission-controller-sa + namespace: default + labels: + app: admission-controller +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: admission-controller-crb + namespace: default + labels: + app: admission-controller +subjects: +- kind: ServiceAccount + name: admission-controller-sa + namespace: default +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: admission-controller-cr +--- +apiVersion: v1 +kind: Service +metadata: + name: admission-controller-svc + namespace: default + labels: + app: admission-controller +spec: + ports: + - port: 443 + targetPort: 443 + selector: + app: admission-controller +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: admission-controller + namespace: default + labels: + app: admission-controller +spec: + replicas: 1 + selector: + matchLabels: + app: admission-controller + template: + metadata: + labels: + app: admission-controller + spec: + serviceAccountName: admission-controller-sa + containers: + - name: admission-controller + image: hub.pingcap.net/yinliang/pingcap/tidb-operator:latest + imagePullPolicy: Always + command: + - /usr/local/bin/tidb-admission-controller + - -tlsCertFile=/etc/webhook/certs/cert.pem + - -tlsKeyFile=/etc/webhook/certs/key.pem + - -v=2 + volumeMounts: + - name: webhook-certs + mountPath: /etc/webhook/certs + readOnly: true + volumes: + - name: webhook-certs + secret: + secretName: admission-controller-certs +--- +apiVersion: admissionregistration.k8s.io/v1beta1 +kind: ValidatingWebhookConfiguration +metadata: + name: validation-admission-contorller-cfg + labels: + app: admission-controller +webhooks: + - name: statefulset-admission-controller.pingcap.net + failurePolicy: Fail + clientConfig: + service: + name: admission-controller-svc + namespace: default + path: "/statefulsets" + caBundle: ${CA_BUNDLE} + rules: + - operations: [ "UPDATE" ] + apiGroups: [ "apps", "" ] + apiVersions: ["v1beta1"] + resources: ["statefulsets"] diff --git a/pkg/tkctl/cmd/upinfo/upinfo.go b/pkg/tkctl/cmd/upinfo/upinfo.go index f98fcd768b3..b49ea965a9d 100644 --- a/pkg/tkctl/cmd/upinfo/upinfo.go +++ b/pkg/tkctl/cmd/upinfo/upinfo.go @@ -142,7 +142,7 @@ func (o *UpInfoOptions) Run() error { return err } podList, err := o.KubeCli.CoreV1().Pods(o.Namespace).List(metav1.ListOptions{ - LabelSelector: fmt.Sprintf("%s=%s,%s=%s", label.InstanceLabelKey, tc.Name, label.ComponentLabelKey, "tidb"), + LabelSelector: label.New().Instance(tc.Name).TiDB().String(), }) if err != nil { return err @@ -162,7 +162,7 @@ func renderTCUpgradeInfo(tc *v1alpha1.TidbCluster, set *apps.StatefulSet, podLis w.WriteLine(readable.LEVEL_0, "Name:\t%s", tc.Name) w.WriteLine(readable.LEVEL_0, "Namespace:\t%s", tc.Namespace) w.WriteLine(readable.LEVEL_0, "CreationTimestamp:\t%s", tc.CreationTimestamp) - w.WriteLine(readable.LEVEL_0, "Statu:\t%s", dbPhase) + w.WriteLine(readable.LEVEL_0, "Status:\t%s", dbPhase) if dbPhase == v1alpha1.UpgradePhase { if len(podList.Items) != 0 { pod := podList.Items[0] diff --git a/pkg/webhook/route/route.go b/pkg/webhook/route/route.go index 0c343cf690c..0fb336a12c3 100644 --- a/pkg/webhook/route/route.go +++ b/pkg/webhook/route/route.go @@ -51,21 +51,22 @@ func serve(w http.ResponseWriter, r *http.Request, admit admitFunc) { deserializer := util.GetCodec() // The AdmissionReview that will be returned - if r.Body != nil { - if data, err := ioutil.ReadAll(r.Body); err == nil { - body = data - } else { - responseAdmissionReview.Response = util.ARFail(err) - marshalAndWrite(responseAdmissionReview, w) - return - } - } else { + if r.Body == nil { err := errors.New("request body is nil!") responseAdmissionReview.Response = util.ARFail(err) marshalAndWrite(responseAdmissionReview, w) return } + data, err := ioutil.ReadAll(r.Body) + if err != nil { + responseAdmissionReview.Response = util.ARFail(err) + marshalAndWrite(responseAdmissionReview, w) + return + } + + body = data + // verify the content type is accurate contentType = r.Header.Get("Content-Type") if contentType != "application/json" { diff --git a/pkg/webhook/util/util.go b/pkg/webhook/util/util.go index a651ec0e5a8..7f9bd143de6 100644 --- a/pkg/webhook/util/util.go +++ b/pkg/webhook/util/util.go @@ -20,7 +20,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) -// toAdmissionResponse is a helper function to create an AdmissionResponse +// ARFail is a helper function to create an AdmissionResponse // with an embedded error func ARFail(err error) *v1beta1.AdmissionResponse { return &v1beta1.AdmissionResponse{ @@ -32,7 +32,7 @@ func ARFail(err error) *v1beta1.AdmissionResponse { } } -// toAdmissionResponse return allow to action +// ARSuccess return allow to action func ARSuccess() *v1beta1.AdmissionResponse { return &v1beta1.AdmissionResponse{ Allowed: true, diff --git a/tests/actions.go b/tests/actions.go index ef01a5f8ff3..a9c7d098007 100644 --- a/tests/actions.go +++ b/tests/actions.go @@ -301,7 +301,6 @@ func (oi *OperatorConfig) OperatorHelmSetString(m map[string]string) string { "scheduler.logLevel": "2", "controllerManager.replicas": "2", "scheduler.replicas": "2", - "admissionController.create": "true", } if oi.SchedulerTag != "" { set["scheduler.kubeSchedulerImageTag"] = oi.SchedulerTag @@ -338,6 +337,33 @@ func (oa *operatorActions) DeployOperator(info *OperatorConfig) error { return fmt.Errorf("failed to deploy operator: %v, %s", err, string(res)) } + // create cert and secret for webhook + cmd = fmt.Sprintf("%s/create-cert.sh",oa.manifestPath(info.Tag)) + glog.Info(cmd) + + res, err = exec.Command("/bin/sh", "-c", cmd).CombinedOutput() + if err != nil { + return fmt.Errorf("failed to create cert: %v, %s", err, string(res)) + } + + // patch cabundle to validating admission configuration + cmd = fmt.Sprintf("%s/patch-ca.sh",oa.manifestPath(info.Tag)) + glog.Info(cmd) + + res, err = exec.Command("/bin/sh", "-c", cmd).CombinedOutput() + if err != nil { + return fmt.Errorf("failed to patch cabundle : %v, %s", err, string(res)) + } + + // deploy statefulset webhook and configuration to hijack update statefulset opeartion + cmd = fmt.Sprintf("kubectl apply -f %s/webhook.yaml",oa.manifestPath(info.Tag)) + glog.Info(cmd) + + res, err = exec.Command("/bin/sh", "-c", cmd).CombinedOutput() + if err != nil { + return fmt.Errorf("failed to create statefulset webhook and configuration : %v, %s", err, string(res)) + } + return nil } @@ -355,7 +381,16 @@ func (oa *operatorActions) CleanOperator(info *OperatorConfig) error { return err } - res, err := exec.Command("helm", "del", "--purge", info.ReleaseName).CombinedOutput() + // delete statefulset update webhook and configuration + cmd := fmt.Sprintf("kubectl delete -f %s/webhook.yaml",oa.manifestPath(info.Tag)) + glog.Info(cmd) + + res, err := exec.Command("/bin/sh", "-c", cmd).CombinedOutput() + if err != nil && !notFound(string(res)) { + return fmt.Errorf("failed to delete statefulset webhook and configuration : %v, %s", err, string(res)) + } + + res, err = exec.Command("helm", "del", "--purge", info.ReleaseName).CombinedOutput() if err == nil || !releaseIsNotFound(err) { return nil @@ -601,6 +636,10 @@ func (oa *operatorActions) StopInsertDataTo(info *TidbClusterConfig) { info.blockWriter.Stop() } +func (oa *operatorActions) manifestPath( tag string) string { + return filepath.Join(oa.cfg.ManifestDir, tag) +} + func (oa *operatorActions) chartPath(name string, tag string) string { return filepath.Join(oa.cfg.ChartDir, tag, name) } @@ -746,7 +785,7 @@ func (oa *operatorActions) UpgradeTidbClusterOrDie(info *TidbClusterConfig) { func (oa *operatorActions) DeployMonitor(info *TidbClusterConfig) error { return nil } func (oa *operatorActions) CleanMonitor(info *TidbClusterConfig) error { return nil } -func getPodContainer(kubeCli kubernetes.Interface, namespace string, memberName string) (*corev1.Container, bool) { +func getMemberContainer(kubeCli kubernetes.Interface, namespace string, memberName string) (*corev1.Container, bool) { name := fmt.Sprintf("%s-%d", memberName, 0) pod, err := kubeCli.CoreV1().Pods(namespace).Get(name, metav1.GetOptions{}) if err != nil { @@ -759,9 +798,9 @@ func getPodContainer(kubeCli kubernetes.Interface, namespace string, memberName } for _, container := range pod.Spec.Containers { - if container.Name == label.TiDBLabelVal || - container.Name == label.TiKVLabelVal || - container.Name == label.PDLabelVal { + if container.Name == v1alpha1.PDMemberType.String() || + container.Name == v1alpha1.TiKVMemberType.String() || + container.Name == v1alpha1.TiDBMemberType.String() { return &container, true } } @@ -805,7 +844,7 @@ func (oa *operatorActions) pdMembersReadyFn(tc *v1alpha1.TidbCluster) (bool, err ns, pdSetName, pdSet.Status.ReadyReplicas, pdSet.Status.Replicas) return false, nil } - if c, ok := getPodContainer(oa.kubeCli, ns, pdSetName); !ok || tc.Spec.PD.Image != c.Image { + if c, ok := getMemberContainer(oa.kubeCli, ns, pdSetName); !ok || tc.Spec.PD.Image != c.Image { glog.Infof("statefulset: %s/%s .spec.template.spec.containers[name=pd].image(%s) != %s", ns, pdSetName, c.Image, tc.Spec.PD.Image) return false, nil @@ -870,7 +909,7 @@ func (oa *operatorActions) tikvMembersReadyFn(tc *v1alpha1.TidbCluster) (bool, e ns, tikvSetName, tikvSet.Status.ReadyReplicas, tikvSet.Status.Replicas) return false, nil } - if c, ok := getPodContainer(oa.kubeCli, ns, tikvSetName); !ok || tc.Spec.TiKV.Image != c.Image { + if c, ok := getMemberContainer(oa.kubeCli, ns, tikvSetName); !ok || tc.Spec.TiKV.Image != c.Image { glog.Infof("statefulset: %s/%s .spec.template.spec.containers[name=tikv].image(%s) != %s", ns, tikvSetName, c.Image, tc.Spec.TiKV.Image) return false, nil @@ -979,7 +1018,7 @@ func (oa *operatorActions) tidbMembersReadyFn(tc *v1alpha1.TidbCluster) (bool, e return false, nil } - if c, ok := getPodContainer(oa.kubeCli, ns, tidbSetName); !ok || tc.Spec.TiDB.Image != c.Image { + if c, ok := getMemberContainer(oa.kubeCli, ns, tidbSetName); !ok || tc.Spec.TiDB.Image != c.Image { glog.Infof("statefulset: %s/%s .spec.template.spec.containers[name=tidb].image(%s) != %s", ns, tidbSetName, c.Image, tc.Spec.TiDB.Image) return false, nil @@ -1403,6 +1442,10 @@ func releaseIsNotFound(err error) bool { return strings.Contains(err.Error(), "not found") } +func notFound(res string) bool { + return strings.Contains(res, "not found") +} + func (oa *operatorActions) cloneOperatorRepo() error { cmd := fmt.Sprintf("git clone https://github.com/pingcap/tidb-operator.git %s", oa.cfg.OperatorRepoDir) glog.Info(cmd) @@ -1417,10 +1460,12 @@ func (oa *operatorActions) cloneOperatorRepo() error { func (oa *operatorActions) checkoutTag(tagName string) error { cmd := fmt.Sprintf("cd %s && git stash -u && git checkout %s && "+ "mkdir -p %s && cp -rf charts/tidb-operator %s && "+ - "cp -rf charts/tidb-cluster %s && cp -rf charts/tidb-backup %s", + "cp -rf charts/tidb-cluster %s && cp -rf charts/tidb-backup %s &&" + + "cp -rf manifests %s", oa.cfg.OperatorRepoDir, tagName, filepath.Join(oa.cfg.ChartDir, tagName), oa.operatorChartPath(tagName), - oa.tidbClusterChartPath(tagName), oa.backupChartPath(tagName)) + oa.tidbClusterChartPath(tagName), oa.backupChartPath(tagName), + oa.manifestPath(tagName)) glog.Info(cmd) res, err := exec.Command("/bin/sh", "-c", cmd).CombinedOutput() if err != nil { diff --git a/tests/config.go b/tests/config.go index e73b6cf8419..441cd2f63e9 100644 --- a/tests/config.go +++ b/tests/config.go @@ -44,6 +44,8 @@ type Config struct { OperatorRepoDir string `yaml:"operator_repo_dir" json:"operator_repo_dir"` // chart dir ChartDir string `yaml:"chart_dir" json:"chart_dir"` + // manifest dir + ManifestDir string `yaml:"manifest_dir" json:"manifest_dir"` } // Nodes defines a series of nodes that belong to the same physical node. @@ -83,6 +85,13 @@ func NewConfig() (*Config, error) { return nil, err } cfg.ChartDir = chartDir + + manifestDir, err := ioutil.TempDir("", "manifests") + if err != nil { + return nil, err + } + cfg.ManifestDir = manifestDir + return cfg, nil } diff --git a/tests/images/e2e/Dockerfile b/tests/images/e2e/Dockerfile index 5abe477cd39..7294d5bc953 100644 --- a/tests/images/e2e/Dockerfile +++ b/tests/images/e2e/Dockerfile @@ -3,7 +3,7 @@ FROM alpine:3.5 ENV KUBECTL_VERSION=v1.12.2 ENV HELM_VERSION=v2.9.1 -RUN apk update && apk add --no-cache ca-certificates curl git +RUN apk update && apk add --no-cache ca-certificates curl git openssl bash RUN curl https://storage.googleapis.com/kubernetes-release/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl \ -o /usr/local/bin/kubectl && \ chmod +x /usr/local/bin/kubectl && \ @@ -17,5 +17,6 @@ RUN curl https://storage.googleapis.com/kubernetes-release/release/${KUBECTL_VER ADD tidb-operator /charts/e2e/tidb-operator ADD tidb-cluster /charts/e2e/tidb-cluster ADD tidb-backup /charts/e2e/tidb-backup +ADD manifests /manifests/e2e ADD bin/e2e /usr/local/bin/e2e From 2f7de796218522edb0ae5a995c9aede058576dae Mon Sep 17 00:00:00 2001 From: shuijing198799 Date: Wed, 22 May 2019 14:43:33 +0800 Subject: [PATCH 12/16] go fmt --- tests/actions.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/actions.go b/tests/actions.go index 0bf2bc32cdc..d92f61e4540 100644 --- a/tests/actions.go +++ b/tests/actions.go @@ -367,7 +367,7 @@ func (oa *operatorActions) DeployOperator(info *OperatorConfig) error { } // create cert and secret for webhook - cmd = fmt.Sprintf("%s/create-cert.sh",oa.manifestPath(info.Tag)) + cmd = fmt.Sprintf("%s/create-cert.sh", oa.manifestPath(info.Tag)) glog.Info(cmd) res, err = exec.Command("/bin/sh", "-c", cmd).CombinedOutput() @@ -376,7 +376,7 @@ func (oa *operatorActions) DeployOperator(info *OperatorConfig) error { } // patch cabundle to validating admission configuration - cmd = fmt.Sprintf("%s/patch-ca.sh",oa.manifestPath(info.Tag)) + cmd = fmt.Sprintf("%s/patch-ca.sh", oa.manifestPath(info.Tag)) glog.Info(cmd) res, err = exec.Command("/bin/sh", "-c", cmd).CombinedOutput() @@ -385,7 +385,7 @@ func (oa *operatorActions) DeployOperator(info *OperatorConfig) error { } // deploy statefulset webhook and configuration to hijack update statefulset opeartion - cmd = fmt.Sprintf("kubectl apply -f %s/webhook.yaml",oa.manifestPath(info.Tag)) + cmd = fmt.Sprintf("kubectl apply -f %s/webhook.yaml", oa.manifestPath(info.Tag)) glog.Info(cmd) res, err = exec.Command("/bin/sh", "-c", cmd).CombinedOutput() @@ -411,7 +411,7 @@ func (oa *operatorActions) CleanOperator(info *OperatorConfig) error { } // delete statefulset update webhook and configuration - cmd := fmt.Sprintf("kubectl delete -f %s/webhook.yaml",oa.manifestPath(info.Tag)) + cmd := fmt.Sprintf("kubectl delete -f %s/webhook.yaml", oa.manifestPath(info.Tag)) glog.Info(cmd) res, err := exec.Command("/bin/sh", "-c", cmd).CombinedOutput() @@ -710,7 +710,7 @@ func (oa *operatorActions) StopInsertDataTo(info *TidbClusterConfig) { info.blockWriter.Stop() } -func (oa *operatorActions) manifestPath( tag string) string { +func (oa *operatorActions) manifestPath(tag string) string { return filepath.Join(oa.cfg.ManifestDir, tag) } @@ -1596,7 +1596,7 @@ func (oa *operatorActions) cloneOperatorRepo() error { func (oa *operatorActions) checkoutTag(tagName string) error { cmd := fmt.Sprintf("cd %s && git stash -u && git checkout %s && "+ "mkdir -p %s && cp -rf charts/tidb-operator %s && "+ - "cp -rf charts/tidb-cluster %s && cp -rf charts/tidb-backup %s &&" + + "cp -rf charts/tidb-cluster %s && cp -rf charts/tidb-backup %s &&"+ "cp -rf manifests %s", oa.cfg.OperatorRepoDir, tagName, filepath.Join(oa.cfg.ChartDir, tagName), oa.operatorChartPath(tagName), From d9f7a4c49b120a01c6ae0e8cbcac158c7be2cfdc Mon Sep 17 00:00:00 2001 From: shuijing198799 Date: Wed, 22 May 2019 15:16:32 +0800 Subject: [PATCH 13/16] make check --- cmd/admission-controller/main.go | 2 +- pkg/webhook/route/route.go | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/cmd/admission-controller/main.go b/cmd/admission-controller/main.go index a0ca6f18d66..c9b16722403 100644 --- a/cmd/admission-controller/main.go +++ b/cmd/admission-controller/main.go @@ -23,9 +23,9 @@ import ( "github.com/pingcap/tidb-operator/pkg/client/clientset/versioned" "github.com/pingcap/tidb-operator/pkg/webhook" "github.com/pingcap/tidb-operator/version" + "k8s.io/apiserver/pkg/util/logs" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" - "k8s.io/apiserver/pkg/util/logs" ) var ( diff --git a/pkg/webhook/route/route.go b/pkg/webhook/route/route.go index 0fb336a12c3..b45cbece421 100644 --- a/pkg/webhook/route/route.go +++ b/pkg/webhook/route/route.go @@ -52,13 +52,14 @@ func serve(w http.ResponseWriter, r *http.Request, admit admitFunc) { // The AdmissionReview that will be returned if r.Body == nil { - err := errors.New("request body is nil!") + err := errors.New("requeset body is nil") responseAdmissionReview.Response = util.ARFail(err) marshalAndWrite(responseAdmissionReview, w) return } data, err := ioutil.ReadAll(r.Body) + if err != nil { responseAdmissionReview.Response = util.ARFail(err) marshalAndWrite(responseAdmissionReview, w) From 97c220e3101bc4bf688373c1de07e5e7d2d14988 Mon Sep 17 00:00:00 2001 From: shuijing198799 Date: Wed, 22 May 2019 15:22:30 +0800 Subject: [PATCH 14/16] no sidecar-injector --- manifests/create-cert.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/manifests/create-cert.sh b/manifests/create-cert.sh index 89ad82b4397..f2653f2938c 100755 --- a/manifests/create-cert.sh +++ b/manifests/create-cert.sh @@ -4,10 +4,10 @@ set -e usage() { cat < Date: Wed, 22 May 2019 15:40:29 +0800 Subject: [PATCH 15/16] fix manifest dir for e2e test --- manifests/create-cert.sh | 5 ----- tests/cmd/e2e/main.go | 1 + 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/manifests/create-cert.sh b/manifests/create-cert.sh index f2653f2938c..4e76ff8fa02 100755 --- a/manifests/create-cert.sh +++ b/manifests/create-cert.sh @@ -22,11 +22,6 @@ namespace=default service=admission-controller-svc secret=admission-controller-certs -if [ -z ${namespace} ]; then - echo "need input namespace" - exit 1 -fi - if [ ! -x "$(command -v openssl)" ]; then echo "openssl not found" exit 1 diff --git a/tests/cmd/e2e/main.go b/tests/cmd/e2e/main.go index a20a021f550..7aa143fb8fc 100644 --- a/tests/cmd/e2e/main.go +++ b/tests/cmd/e2e/main.go @@ -32,6 +32,7 @@ func main() { conf := tests.ParseConfigOrDie() conf.ChartDir = "/charts" + conf.ManifestDir = "/manifests" cli, kubeCli := client.NewCliOrDie() oa := tests.NewOperatorActions(cli, kubeCli, 5*time.Second, conf, nil) From d43eb93f27a580c6248adb7a4ebe5f3d2c7b4616 Mon Sep 17 00:00:00 2001 From: shuijing198799 Date: Wed, 22 May 2019 18:17:48 +0800 Subject: [PATCH 16/16] use common image --- manifests/webhook.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/manifests/webhook.yaml b/manifests/webhook.yaml index 2581036f019..489c14003bc 100644 --- a/manifests/webhook.yaml +++ b/manifests/webhook.yaml @@ -71,7 +71,7 @@ spec: serviceAccountName: admission-controller-sa containers: - name: admission-controller - image: hub.pingcap.net/yinliang/pingcap/tidb-operator:latest + image: pingcap/tidb-operator:latest imagePullPolicy: Always command: - /usr/local/bin/tidb-admission-controller