multi-cluster bootstrap in antctl

Add new subcommands to Create or Delete multi-cluster Resources.

Signed-off-by: hjiajing <hjiajing@vmware.com>

pkg/antctl/antctl.go

@@ -538,6 +538,30 @@ var CommandList = &commandList{
 			supportController: false,
 			commandGroup:      mc,
 		},
+		{
+			cobraCommand:      multicluster.AddCmd,
+			supportAgent:      false,
+			supportController: false,
+			commandGroup:      mc,
+		},
+		{
+			cobraCommand:      multicluster.CreateCmd,
+			supportAgent:      false,
+			supportController: false,
+			commandGroup:      mc,
+		},
+		{
+			cobraCommand:      multicluster.DeleteCmd,
+			supportAgent:      false,
+			supportController: false,
+			commandGroup:      mc,
+		},
+		{
+			cobraCommand:      multicluster.DeployCmd,
+			supportAgent:      false,
+			supportController: false,
+			commandGroup:      mc,
+		},
 	},
 	codec: scheme.Codecs,
 }

pkg/antctl/raw/multicluster/add/member_cluster.go

@@ -0,0 +1,115 @@
+// Copyright 2022 Antrea Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package add
+
+import (
+	"context"
+	"fmt"
+	"strings"
+
+	"github.com/spf13/cobra"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/client-go/rest"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	multiclusterv1alpha1 "antrea.io/antrea/multicluster/apis/multicluster/v1alpha1"
+	"antrea.io/antrea/pkg/antctl/raw"
+	multiclusterscheme "antrea.io/antrea/pkg/antctl/raw/multicluster/scheme"
+)
+
+type memberClusterOptions struct {
+	namespace      string
+	clusterSet     string
+	serviceAccount string
+}
+
+var memberClusterOpt *memberClusterOptions
+
+var memberClusterExamples = strings.Trim(`
+Add a new member cluster to a ClusterSet
+$ antctl mc add member-cluster <CLUSTER_ID> -n <NAMESPACE> --cluster-set <CLUSTER_SET> --service-account <SERVICE_ACCOUNT>
+`, "\n")
+
+func (o *memberClusterOptions) validateAndComplete() error {
+	if o.namespace == "" {
+		return fmt.Errorf("the Namespace cannot be empty")
+	}
+	if o.clusterSet == "" {
+		return fmt.Errorf("the cluster-set cannot be empty")
+	}
+	if o.serviceAccount == "" {
+		return fmt.Errorf("the service-account cannot be empty")
+	}
+
+	return nil
+}
+
+func NewMemberClusterCmd() *cobra.Command {
+	command := &cobra.Command{
+		Use:     "member-cluster",
+		Args:    cobra.MaximumNArgs(1),
+		Short:   "Add a new member cluster to a ClusterSet",
+		Long:    "Add a new member cluster to a ClusterSet",
+		Example: memberClusterExamples,
+		RunE:    memberClusterRunE,
+	}
+
+	o := &memberClusterOptions{}
+	memberClusterOpt = o
+	command.Flags().StringVarP(&o.namespace, "namespace", "n", "", "Namespace of member cluster")
+	command.Flags().StringVarP(&o.clusterSet, "cluster-set", "", "", "ClusterSet to add a new member cluster")
+	command.Flags().StringVarP(&o.serviceAccount, "service-account", "", "", "ServiceAccount of the member cluster")
+
+	return command
+}
+
+func memberClusterRunE(cmd *cobra.Command, args []string) error {
+	if err := memberClusterOpt.validateAndComplete(); err != nil {
+		return err
+	}
+	if len(args) != 1 {
+		return fmt.Errorf("exactly one NAME is required, got %d", len(args))
+	}
+
+	kubeconfig, err := raw.ResolveKubeconfig(cmd)
+	if err != nil {
+		return err
+	}
+	restconfigTmpl := rest.CopyConfig(kubeconfig)
+	raw.SetupKubeconfig(restconfigTmpl)
+
+	k8sClient, err := client.New(kubeconfig, client.Options{Scheme: multiclusterscheme.Scheme})
+	if err != nil {
+		return err
+	}
+
+	memberClusterID := args[0]
+	clusterSet := &multiclusterv1alpha1.ClusterSet{}
+	if err := k8sClient.Get(context.TODO(), types.NamespacedName{Name: memberClusterOpt.clusterSet, Namespace: memberClusterOpt.namespace}, clusterSet); err != nil {
+		return err
+	}
+	for _, member := range clusterSet.Spec.Members {
+		if member.ClusterID == memberClusterID {
+			return fmt.Errorf("the member cluster %s is already added to the ClusterSet %s", memberClusterID, memberClusterOpt.clusterSet)
+		}
+	}
+	clusterSet.Spec.Members = append(clusterSet.Spec.Members, multiclusterv1alpha1.MemberCluster{ClusterID: memberClusterID, ServiceAccount: memberClusterOpt.serviceAccount})
+	if err := k8sClient.Update(context.TODO(), clusterSet); err != nil {
+		return err
+	}
+
+	fmt.Fprintf(cmd.OutOrStdout(), "member cluster %s is added to ClusterSet %s successfully", memberClusterID, memberClusterOpt.clusterSet)
+	return nil
+}

pkg/antctl/raw/multicluster/commands.go

@@ -17,6 +17,10 @@ package multicluster
 import (
 	"github.com/spf13/cobra"
 
+	"antrea.io/antrea/pkg/antctl/raw/multicluster/add"
+	"antrea.io/antrea/pkg/antctl/raw/multicluster/create"
+	deleteCmd "antrea.io/antrea/pkg/antctl/raw/multicluster/delete"
+	"antrea.io/antrea/pkg/antctl/raw/multicluster/deploy"
 	"antrea.io/antrea/pkg/antctl/raw/multicluster/get"
 )
 
@@ -25,8 +29,37 @@ var GetCmd = &cobra.Command{
 	Short: "Display one or many resources in a ClusterSet",
 }
 
+var CreateCmd = &cobra.Command{
+	Use:   "create",
+	Short: "Create resources in a ClusterSet",
+}
+
+var AddCmd = &cobra.Command{
+	Use:   "add",
+	Short: "Add new Member Cluster to a ClusterSet",
+}
+
+var DeleteCmd = &cobra.Command{
+	Use:   "delete",
+	Short: "Delete Resources in a ClusterSet",
+}
+
+var DeployCmd = &cobra.Command{
+	Use:   "deploy",
+	Short: "Deploy the leader cluster or member cluster",
+}
+
 func init() {
 	GetCmd.AddCommand(get.NewClusterSetCommand())
 	GetCmd.AddCommand(get.NewResourceImportCommand())
 	GetCmd.AddCommand(get.NewResourceExportCommand())
+	CreateCmd.AddCommand(create.NewClusterClaimCmd())
+	CreateCmd.AddCommand(create.NewAccessTokenCmd())
+	CreateCmd.AddCommand(create.NewClusterSetCmd())
+	DeleteCmd.AddCommand(deleteCmd.NewMemberClusterCmd())
+	DeleteCmd.AddCommand(deleteCmd.NewClusterSetCmd())
+	DeleteCmd.AddCommand(deleteCmd.NewClusterClaimCmd())
+	AddCmd.AddCommand(add.NewMemberClusterCmd())
+	DeployCmd.AddCommand(deploy.NewLeaderClusterCmd())
+	DeployCmd.AddCommand(deploy.NewMemberClusterCmd())
 }

pkg/antctl/raw/multicluster/create/access_token.go

@@ -0,0 +1,171 @@
+// Copyright 2022 Antrea Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package create
+
+import (
+	"context"
+	"fmt"
+	"strings"
+
+	"github.com/spf13/cobra"
+	corev1 "k8s.io/api/core/v1"
+	rbacv1 "k8s.io/api/rbac/v1"
+	"k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/rest"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	"antrea.io/antrea/pkg/antctl/raw"
+	multiclusterscheme "antrea.io/antrea/pkg/antctl/raw/multicluster/scheme"
+)
+
+type accessTokenOptions struct {
+	namespace      string
+	serviceAccount string
+	roleBinding    string
+}
+
+var accessTokenOpts *accessTokenOptions
+
+var accessTokenExamples = strings.Trim(`
+Create a access token in a Kubernetes cluster for one or more member clusters, if the Service Account or RoleBinding does not exit, antctl will create one
+$ antctl create access-token -n <NAMESPACE> --service-account <SERVICE_ACCOUNT> --roleBinding <ROLE_BINDING>
+
+options: 
+-n, --namespace: The Namespace of the access token
+--service-account: The ServiceAccount used by the access token, if the Service Account does not exist, the antctl will create one
+--role-binding: The RoleBinding for the Service Account. It will bind the Service Account to the member cluster role
+`, "\n")
+
+func (o *accessTokenOptions) validateAndComplete() error {
+	if o.namespace == "" {
+		return fmt.Errorf("the namespace cannot be empty")
+	}
+	if o.serviceAccount == "" {
+		return fmt.Errorf("the service-account cannot be empty")
+	}
+	if o.roleBinding == "" {
+		return fmt.Errorf("the role-binding cannot be empty")
+	}
+	return nil
+}
+
+func NewAccessTokenCmd() *cobra.Command {
+	command := &cobra.Command{
+		Use:     "access-token",
+		Args:    cobra.MaximumNArgs(1),
+		Short:   "Create a access-token in a Kubernetes cluster",
+		Long:    "Create a access-token in a Kubernetes cluster",
+		Example: accessTokenExamples,
+		RunE:    accessTokenRunE,
+	}
+
+	o := &accessTokenOptions{}
+	accessTokenOpts = o
+	command.Flags().StringVarP(&o.namespace, "namespace", "n", "", "Namespace of the ClusterClaim")
+	command.Flags().StringVarP(&o.serviceAccount, "service-account", "", "", "Service Account of the access token")
+	command.Flags().StringVarP(&o.roleBinding, "role-binding", "", "", "RoleBinding of the Service Account")
+
+	return command
+}
+
+func accessTokenRunE(cmd *cobra.Command, args []string) error {
+	if err := accessTokenOpts.validateAndComplete(); err != nil {
+		return err
+	}
+	if len(args) != 1 {
+		return fmt.Errorf("exactly one NAME is required, got %d", len(args))
+	}
+	kubeconfig, err := raw.ResolveKubeconfig(cmd)
+	if err != nil {
+		return err
+	}
+	restconfigTmpl := rest.CopyConfig(kubeconfig)
+	raw.SetupKubeconfig(restconfigTmpl)
+	k8sClient, err := client.New(kubeconfig, client.Options{Scheme: multiclusterscheme.Scheme})
+	if err != nil {
+		return err
+	}
+
+	serviceAccount := corev1.ServiceAccount{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      accessTokenOpts.serviceAccount,
+			Namespace: accessTokenOpts.namespace,
+		},
+	}
+
+	fmt.Fprintf(cmd.OutOrStdout(), "creating ServiceAccount %s \n", accessTokenOpts.serviceAccount)
+	if err := k8sClient.Create(context.TODO(), &serviceAccount); err != nil {
+		if errors.IsAlreadyExists(err) {
+			fmt.Fprintf(cmd.OutOrStderr(), fmt.Sprintf("the ServiceAccount %s is already exists\n", accessTokenOpts.serviceAccount))
+		} else {
+			return err
+		}
+	}
+
+	fmt.Fprintf(cmd.OutOrStdout(), "creating RoleBinding %s \n", accessTokenOpts.roleBinding)
+	roleBinding := rbacv1.RoleBinding{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      accessTokenOpts.roleBinding,
+			Namespace: accessTokenOpts.namespace,
+		},
+		RoleRef: rbacv1.RoleRef{
+			APIGroup: "rbac.authorization.k8s.io",
+			Kind:     "Role",
+			Name:     "antrea-mc-member-cluster-role",
+		},
+		Subjects: []rbacv1.Subject{
+			{
+				Kind:      "ServiceAccount",
+				Name:      accessTokenOpts.serviceAccount,
+				Namespace: accessTokenOpts.namespace,
+			},
+		},
+	}
+	fmt.Fprintf(cmd.OutOrStdout(), "creating RoleBinding %s \n", accessTokenOpts.serviceAccount)
+	if err := k8sClient.Create(context.TODO(), &roleBinding); err != nil {
+		if errors.IsAlreadyExists(err) {
+			fmt.Fprintf(cmd.OutOrStderr(), fmt.Sprintf("the RoleBinding %s is already exists\n", accessTokenOpts.roleBinding))
+		} else {
+			rollback(k8sClient, accessTokenOpts.namespace, accessTokenOpts.serviceAccount, "")
+			return err
+		}
+	}
+	secretName := args[0]
+
+	fmt.Fprintf(cmd.OutOrStdout(), "creating Secret %s \n", secretName)
+	secret := corev1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      secretName,
+			Namespace: accessTokenOpts.namespace,
+			Annotations: map[string]string{
+				"kubernetes.io/service-account.name": accessTokenOpts.serviceAccount,
+			},
+		},
+		Type: "kubernetes.io/service-account-token",
+	}
+
+	if err := k8sClient.Create(context.TODO(), &secret); err != nil {
+		if errors.IsAlreadyExists(err) {
+			fmt.Fprintf(cmd.OutOrStderr(), fmt.Sprintf("the Secret %s is already exists\n", accessTokenOpts.roleBinding))
+		} else {
+			rollback(k8sClient, accessTokenOpts.namespace, accessTokenOpts.serviceAccount, accessTokenOpts.roleBinding)
+			return err
+		}
+	}
+	fmt.Fprintf(cmd.OutOrStdout(), "access-token %s is created\n", secretName)
+
+	return nil
+}