diff --git a/cmd/admission-controller/main.go b/cmd/admission-controller/main.go index 823d2542900..6e6181b6909 100644 --- a/cmd/admission-controller/main.go +++ b/cmd/admission-controller/main.go @@ -20,9 +20,12 @@ import ( "os/signal" "syscall" + "github.com/cofyc/advanced-statefulset/pkg/apis/apps/v1alpha1/helper" + asclientset "github.com/cofyc/advanced-statefulset/pkg/client/clientset/versioned" "github.com/pingcap/tidb-operator/pkg/client/clientset/versioned" informers "github.com/pingcap/tidb-operator/pkg/client/informers/externalversions" "github.com/pingcap/tidb-operator/pkg/controller" + "github.com/pingcap/tidb-operator/pkg/features" "github.com/pingcap/tidb-operator/pkg/version" "github.com/pingcap/tidb-operator/pkg/webhook" "k8s.io/apimachinery/pkg/util/wait" @@ -44,6 +47,7 @@ func init() { flag.BoolVar(&printVersion, "version", false, "Show version and quit") flag.StringVar(&certFile, "tlsCertFile", "/etc/webhook/certs/cert.pem", "File containing the x509 Certificate for HTTPS.") flag.StringVar(&keyFile, "tlsKeyFile", "/etc/webhook/certs/key.pem", "File containing the x509 private key to --tlsCertFile.") + features.DefaultFeatureGate.AddFlag(flag.CommandLine) flag.Parse() } @@ -68,13 +72,24 @@ func main() { glog.Fatalf("failed to create Clientset: %v", err) } - kubeCli, err := kubernetes.NewForConfig(cfg) + var kubeCli kubernetes.Interface + kubeCli, err = kubernetes.NewForConfig(cfg) if err != nil { glog.Fatalf("failed to get kubernetes Clientset: %v", err) } + asCli, err := asclientset.NewForConfig(cfg) + if err != nil { + glog.Fatalf("failed to get advanced-statefulset Clientset: %v", err) + } + + if features.DefaultFeatureGate.Enabled(features.AdvancedStatefulSet) { + // If AdvancedStatefulSet is enabled, we hijack the Kubernetes client to use + // AdvancedStatefulSet. + kubeCli = helper.NewHijackClient(kubeCli, asCli) + } + informerFactory := informers.NewSharedInformerFactory(cli, controller.ResyncDuration) kubeInformerFactory := kubeinformers.NewSharedInformerFactory(kubeCli, controller.ResyncDuration) - webhookServer := webhook.NewWebHookServer(kubeCli, cli, informerFactory, kubeInformerFactory, certFile, keyFile) controllerCtx, cancel := context.WithCancel(context.Background()) defer cancel() diff --git a/hack/kind-cluster-build.sh b/hack/kind-cluster-build.sh index 26e2c66e2ef..53e90509452 100755 --- a/hack/kind-cluster-build.sh +++ b/hack/kind-cluster-build.sh @@ -16,7 +16,7 @@ Options: -n,--name name of the Kubernetes cluster,default value: kind -c,--nodeNum the count of the cluster nodes,default value: 6 -k,--k8sVersion version of the Kubernetes cluster,default value: v1.12.8 - -v,--volumeNum the volumes number of each kubernetes node,default value: 9 + -v,--volumeNum the volumes number of each kubernetes node,default value: 32 Usage: $0 --name testCluster --nodeNum 4 --k8sVersion v1.12.9 EOF @@ -62,7 +62,7 @@ done clusterName=${clusterName:-kind} nodeNum=${nodeNum:-6} k8sVersion=${k8sVersion:-v1.12.8} -volumeNum=${volumeNum:-9} +volumeNum=${volumeNum:-32} echo "clusterName: ${clusterName}" echo "nodeNum: ${nodeNum}" diff --git a/manifests/webhook.yaml b/manifests/webhook.yaml index 803b3abc978..bd685bcdb1d 100644 --- a/manifests/webhook.yaml +++ b/manifests/webhook.yaml @@ -21,6 +21,12 @@ rules: - apiGroups: ["pingcap.com"] resources: ["tidbclusters"] verbs: ["*"] + - apiGroups: + - apps.pingcap.com + resources: + - statefulsets + verbs: + - '*' --- apiVersion: v1 kind: ServiceAccount @@ -86,7 +92,8 @@ spec: - /usr/local/bin/tidb-admission-controller - -tlsCertFile=/etc/webhook/certs/cert.pem - -tlsKeyFile=/etc/webhook/certs/key.pem - - -v=2 + - -v=4 + - -features=AdvancedStatefulSet=true volumeMounts: - name: webhook-certs mountPath: /etc/webhook/certs @@ -116,3 +123,7 @@ webhooks: apiGroups: [ "apps", "" ] apiVersions: ["v1beta1", "v1"] resources: ["statefulsets"] + - operations: [ "UPDATE" ] + apiGroups: [ "apps.pingcap.com"] + apiVersions: ["v1alpha1"] + resources: ["statefulsets"] diff --git a/pkg/webhook/statefulset/statefulset.go b/pkg/webhook/statefulset/statefulset.go index 09133d6bea1..d619006e548 100644 --- a/pkg/webhook/statefulset/statefulset.go +++ b/pkg/webhook/statefulset/statefulset.go @@ -18,8 +18,10 @@ import ( "fmt" "strconv" + asappsv1alpha1 "github.com/cofyc/advanced-statefulset/pkg/apis/apps/v1alpha1" "github.com/pingcap/tidb-operator/pkg/client/clientset/versioned" "github.com/pingcap/tidb-operator/pkg/controller" + "github.com/pingcap/tidb-operator/pkg/features" "github.com/pingcap/tidb-operator/pkg/label" "github.com/pingcap/tidb-operator/pkg/webhook/util" "k8s.io/api/admission/v1beta1" @@ -44,15 +46,19 @@ func AdmitStatefulSets(ar v1beta1.AdmissionReview) *v1beta1.AdmissionResponse { name := ar.Request.Name namespace := ar.Request.Namespace - glog.V(4).Infof("admit statefulsets [%s/%s]", namespace, name) + expectedGroup := "apps" + if features.DefaultFeatureGate.Enabled(features.AdvancedStatefulSet) { + expectedGroup = asappsv1alpha1.GroupName + } apiVersion := ar.Request.Resource.Version - setResource := metav1.GroupVersionResource{Group: "apps", Version: apiVersion, Resource: "statefulsets"} - if ar.Request.Resource.Group != "apps" || ar.Request.Resource.Resource != "statefulsets" { + setResource := metav1.GroupVersionResource{Group: expectedGroup, Version: apiVersion, Resource: "statefulsets"} + if ar.Request.Resource.Group != setResource.Group || ar.Request.Resource.Resource != setResource.Resource { err := fmt.Errorf("expect resource to be %s instead of %s", setResource, ar.Request.Resource) glog.Error(err) return util.ARFail(err) } + glog.V(4).Infof("admit %s [%s/%s]", setResource, namespace, name) if versionCli == nil { cfg, err := rest.InClusterConfig() diff --git a/tests/actions.go b/tests/actions.go index 2b0761e2517..eb1d7eef30f 100644 --- a/tests/actions.go +++ b/tests/actions.go @@ -231,24 +231,24 @@ type event struct { var _ = OperatorActions(&operatorActions{}) type OperatorConfig struct { - Namespace string - ReleaseName string - Image string - Tag string - SchedulerImage string - SchedulerTag string + Namespace string + ReleaseName string + Image string + Tag string + SchedulerImage string + SchedulerTag string Features []string - LogLevel string - WebhookServiceName string - WebhookSecretName string - WebhookConfigName string - Context *apimachinery.CertContext - ImagePullPolicy corev1.PullPolicy - TestMode bool - ApiServerImage string - ApiServerCert string - ApiServerKey string - ApiServerCaBundle string + LogLevel string + WebhookServiceName string + WebhookSecretName string + WebhookConfigName string + Context *apimachinery.CertContext + ImagePullPolicy corev1.PullPolicy + TestMode bool + ApiServerImage string + ApiServerCert string + ApiServerKey string + ApiServerCaBundle string AdvancedStatefulSet bool } @@ -405,11 +405,8 @@ func (oa *operatorActions) CleanCRDOrDie() { // InstallCRDOrDie install CRDs and wait for them to be established in Kubernetes. func (oa *operatorActions) InstallCRDOrDie() { oa.runKubectlOrDie("apply", "-f", oa.manifestPath("e2e/crd.yaml")) -<<<<<<< 3b3a9c36e828aedc3c7edebbe7d4ac8b288bb2c2 oa.runKubectlOrDie("apply", "-f", oa.manifestPath("e2e/data-resource-crd.yaml")) -======= oa.runKubectlOrDie("apply", "-f", oa.manifestPath("e2e/advanced-statefulset-crd.v1beta1.yaml")) ->>>>>>> helm deployment out := oa.runKubectlOrDie([]string{"get", "crds", "--no-headers", `-ojsonpath={range .items[*]}{.metadata.name}{" "}{end}`}...) waitArgs := []string{"wait", "--for=condition=Established"} for _, crd := range strings.Split(out, " ") { diff --git a/tests/cmd/e2e/main.go b/tests/cmd/e2e/main.go index e6f977e0aac..0bfacf1cb6d 100644 --- a/tests/cmd/e2e/main.go +++ b/tests/cmd/e2e/main.go @@ -54,6 +54,10 @@ func main() { } go tests.StartValidatingAdmissionWebhookServerOrDie(certCtx) + restConfig, err := client.GetConfig() + if err != nil { + panic(err) + } cli, kubeCli, asCli := client.NewCliOrDie() ocfg := newOperatorConfig() @@ -216,16 +220,19 @@ func main() { wg.Add(5) go func() { defer wg.Done() + return testAggregatedApiserver() }() go func() { defer wg.Done() + return testBasic(&wg, cluster1) testHostNetwork(&wg, cluster1) oa.CleanTidbClusterOrDie(cluster1) }() go func() { defer wg.Done() + return testBasic(&wg, cluster5) oa.CleanTidbClusterOrDie(cluster5) }() @@ -236,6 +243,7 @@ func main() { }() go func() { defer wg.Done() + return testBackupAndRestore(&wg, cluster3, cluster4) oa.CleanTidbClusterOrDie(cluster3) oa.CleanTidbClusterOrDie(cluster4) diff --git a/tests/manifests/e2e/e2e.yaml b/tests/manifests/e2e/e2e.yaml index 37892faeada..3a3a9ecfeb7 100644 --- a/tests/manifests/e2e/e2e.yaml +++ b/tests/manifests/e2e/e2e.yaml @@ -47,8 +47,8 @@ spec: command: - /usr/local/bin/e2e - --operator-tag=e2e - - --operator-image=pingcap/tidb-operator:latest - - --test-apiserver-image=pingcap/test-apiserver:latest + - --operator-image=localhost:5000/pingcap/tidb-operator:latest + - --test-apiserver-image=localhost:5000/pingcap/tidb-operator:latest - --tidb-versions=v3.0.2,v3.0.3,v3.0.4 - --chart-dir=/charts volumeMounts: