From e2e6c62c9e730c86e88fe9cab63593c60101c3e8 Mon Sep 17 00:00:00 2001 From: yuchenyao Date: Mon, 15 May 2023 17:28:19 +0800 Subject: [PATCH] install regiatration agent using multicluster controlplane Signed-off-by: yuchenyao --- pkg/cmd/join/cmd.go | 45 ++- pkg/cmd/join/exec.go | 367 ++++++++++++------ pkg/cmd/join/options.go | 9 +- pkg/cmd/join/preflight/checks.go | 52 ++- .../bootstrap_hub_kubeconfig.yaml | 9 + .../scenario/controlplane/clusterrole.yaml | 56 +++ .../clusterrolebinding-admin.yaml | 15 + .../controlplane/clusterrolebinding.yaml | 13 + .../scenario/controlplane/deployment.yaml | 48 +++ pkg/cmd/join/scenario/controlplane/role.yaml | 11 + .../scenario/controlplane/rolebinding.yaml | 14 + .../scenario/controlplane/serviceaccount.yaml | 6 + pkg/cmd/join/scenario/resources.go | 2 +- 13 files changed, 496 insertions(+), 151 deletions(-) create mode 100644 pkg/cmd/join/scenario/controlplane/bootstrap_hub_kubeconfig.yaml create mode 100644 pkg/cmd/join/scenario/controlplane/clusterrole.yaml create mode 100644 pkg/cmd/join/scenario/controlplane/clusterrolebinding-admin.yaml create mode 100644 pkg/cmd/join/scenario/controlplane/clusterrolebinding.yaml create mode 100644 pkg/cmd/join/scenario/controlplane/deployment.yaml create mode 100644 pkg/cmd/join/scenario/controlplane/role.yaml create mode 100644 pkg/cmd/join/scenario/controlplane/rolebinding.yaml create mode 100644 pkg/cmd/join/scenario/controlplane/serviceaccount.yaml diff --git a/pkg/cmd/join/cmd.go b/pkg/cmd/join/cmd.go index 1fcedb43a..0468a9676 100644 --- a/pkg/cmd/join/cmd.go +++ b/pkg/cmd/join/cmd.go @@ -5,6 +5,7 @@ import ( "fmt" "github.com/spf13/cobra" + "github.com/spf13/pflag" "k8s.io/cli-runtime/pkg/genericclioptions" genericclioptionsclusteradm "open-cluster-management.io/clusteradm/pkg/genericclioptions" "open-cluster-management.io/clusteradm/pkg/helpers" @@ -48,19 +49,39 @@ func NewCmd(clusteradmFlags *genericclioptionsclusteradm.ClusteradmFlags, stream } genericclioptionsclusteradm.SpokeMutableFeatureGate.AddFlag(cmd.Flags()) - cmd.Flags().StringVar(&o.token, "hub-token", "", "The token to access the hub") - cmd.Flags().StringVar(&o.hubAPIServer, "hub-apiserver", "", "The api server url to the hub") - cmd.Flags().StringVar(&o.caFile, "ca-file", "", "the file path to hub ca, optional") - cmd.Flags().StringVar(&o.clusterName, "cluster-name", "", "The name of the joining cluster") - cmd.Flags().StringVar(&o.outputFile, "output-file", "", "The generated resources will be copied in the specified file") - cmd.Flags().StringVar(&o.registry, "image-registry", "quay.io/open-cluster-management", "The name of the image registry serving OCM images.") - cmd.Flags().StringVar(&o.bundleVersion, "bundle-version", "default", - "The version of predefined compatible image versions (e.g. v0.6.0). Defaults to the latest released version. You can also set \"latest\" to install the latest development version.") - cmd.Flags().BoolVar(&o.forceHubInClusterEndpointLookup, "force-internal-endpoint-lookup", false, + + //joinSet contains the base flags for join command + joinSet := pflag.NewFlagSet("join", pflag.ExitOnError) + joinSet.StringVar(&o.token, "hub-token", "", "The token to access the hub") + joinSet.StringVar(&o.hubAPIServer, "hub-apiserver", "", "The api server url to the hub") + joinSet.StringVar(&o.caFile, "ca-file", "", "the file path to hub ca, optional") + joinSet.StringVar(&o.clusterName, "cluster-name", "", "The name of the joining cluster") + joinSet.BoolVar(&o.forceHubInClusterEndpointLookup, "force-internal-endpoint-lookup", false, "If true, the installed klusterlet agent will be starting the cluster registration process by "+ "looking for the internal endpoint from the public cluster-info in the hub cluster instead of from --hub-apiserver.") - cmd.Flags().BoolVar(&o.wait, "wait", false, "If true, running the cluster registration in foreground.") - cmd.Flags().StringVarP(&o.mode, "mode", "m", "default", "mode to deploy klusterlet, can be default or hosted") - cmd.Flags().StringVar(&o.managedKubeconfigFile, "managed-cluster-kubeconfig", "", "To specify the directory to external managed cluster kubeconfig in hosted mode") + joinSet.BoolVar(&o.wait, "wait", false, "If true, running the cluster registration in foreground.") + joinSet.StringVar(&o.outputFile, "output-file", "", "The generated resources will be copied in the specified file") + joinSet.StringVarP(&o.mode, "mode", "m", "default", "mode to install OCM agent, can be default, hosted or controlplane-agent") + + //klusterletSet contains the flags for deploy klusterlet + klusterletSet := pflag.NewFlagSet("default-klusterlet", pflag.ExitOnError) + klusterletSet.StringVar(&o.registry, "image-registry", "quay.io/open-cluster-management", "The name of the image registry serving OCM images.") + klusterletSet.StringVar(&o.bundleVersion, "bundle-version", "default", + "The version of predefined compatible image versions (e.g. v0.6.0). Defaults to the latest released version. You can also set \"latest\" to install the latest development version.") + klusterletSet.StringVar(&o.managedKubeconfigFile, "managed-cluster-kubeconfig", "", "To specify the directory to external managed cluster kubeconfig in hosted mode") + klusterletSet.SetAnnotation("image-registry", "mode-orinted", []string{"default", "hosted"}) + klusterletSet.SetAnnotation("bundle-version", "mode-orinted", []string{"default", "hosted"}) + klusterletSet.SetAnnotation("managed-cluster-kubeconfig", "mode-orinted", []string{"hosted"}) + + //controlplaneAgentSet contains the flags for deploy controlplane agent + controlplaneAgentSet := pflag.NewFlagSet("controlplane-agent", pflag.ExitOnError) + controlplaneAgentSet.StringVar(&o.controlplaneAgentImage, "controlplane-agent-image", "quay.io/open-cluster-management/multicluster-controlplane:latest", "The name of the controlplane agent image") + controlplaneAgentSet.SetAnnotation("controlplane-agent-image", "mode-orinted", []string{"controlplane-agent"}) + + // flagSet for join command + cmd.Flags().AddFlagSet(joinSet) + cmd.Flags().AddFlagSet(klusterletSet) + cmd.Flags().AddFlagSet(controlplaneAgentSet) + return cmd } diff --git a/pkg/cmd/join/exec.go b/pkg/cmd/join/exec.go index 11e94e1ad..ec8783ee7 100644 --- a/pkg/cmd/join/exec.go +++ b/pkg/cmd/join/exec.go @@ -12,6 +12,7 @@ import ( "github.com/ghodss/yaml" "github.com/spf13/cobra" + "github.com/spf13/pflag" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" @@ -37,9 +38,11 @@ import ( const ( OperatorNamesapce = "open-cluster-management" DefaultOperatorName = "klusterlet" - InstallModeDefault = "Default" - InstallModeHosted = "Hosted" KlusterletNamespacePrefix = "open-cluster-management-" + + InstallModeDefault = "Default" + InstallModeHosted = "Hosted" + InstallModeControlplaneAgent = "Controlplane-agent" ) func format(s string) string { @@ -50,6 +53,10 @@ func format(s string) string { } func (o *Options) complete(cmd *cobra.Command, args []string) (err error) { + if cmd.Flags() == nil { + return fmt.Errorf("no flags have been set: hub-apiserver, hub-token and cluster-name is required") + } + if o.token == "" { return fmt.Errorf("token is missing") } @@ -59,68 +66,100 @@ func (o *Options) complete(cmd *cobra.Command, args []string) (err error) { if o.clusterName == "" { return fmt.Errorf("cluster-name is missing") } - if len(o.registry) == 0 { - return fmt.Errorf("the OCM image registry should not be empty, like quay.io/open-cluster-management") - } - klog.V(1).InfoS("join options:", "dry-run", o.ClusteradmFlags.DryRun, "cluster", o.clusterName, "api-server", o.hubAPIServer, "output", o.outputFile) + if len(o.mode) == 0 { + return fmt.Errorf("the mode should not be empty, like default") + } // convert mode string to lower o.mode = format(o.mode) - // operatorNamespace is the namespace to deploy klsuterlet; - // agentNamespace is the namesapce to deploy the agents(registration agent, work agent, etc.); - // klusterletNamespace is the namespace created on the managed cluster for each klusterlet. - // - // The operatorNamespace is fixed to "open-cluster-management". - // In default mode, agentNamespace is "open-cluster-management-agent", klusterletNamespace refers to agentNamespace, all of these three namesapces are on the managed cluster; - // In hosted mode, operatorNamespace is on the management cluster, agentNamesapce is "-<6-bit random string>" on the management cluster, and the klusterletNamespace is "open-cluster-management-" on the managed cluster. - - // values for default mode - klusterletName := DefaultOperatorName - agentNamespace := KlusterletNamespacePrefix + "agent" - klusterletNamespace := agentNamespace - if o.mode == InstallModeHosted { - // add hash suffix to avoid conflict - klusterletName += "-hosted-" + helpers.RandStringRunes_az09(6) - agentNamespace = klusterletName - klusterletNamespace = KlusterletNamespacePrefix + agentNamespace - } + // ensure the flags are set correctly + cmd.Flags().Visit(func(flag *pflag.Flag) { + if flag.Changed { + if o.mode == InstallModeControlplaneAgent { + if flag.Annotations != nil && flag.Annotations["mode-orinted"] != nil && flag.Annotations["mode-orinted"][0] != "controlplane-agent" { + fmt.Fprintf(o.Streams.Out, "flag %s is not supported in mode %s\n", flag.Name, o.mode) + } + } else { + if flag.Annotations != nil && flag.Annotations["mode-orinted"] != nil { + if flag.Annotations["mode-orinted"][0] == "controlplane-agent" { + fmt.Fprintf(o.Streams.Out, "flag %s is not supported in mode %s\n", flag.Name, o.mode) + } else if flag.Annotations["mode-orinted"][0] == "hosted" && o.mode == InstallModeDefault { + fmt.Fprintf(o.Streams.Out, "flag %s is not supported in mode %s\n", flag.Name, o.mode) + } + } + } - o.values = Values{ - ClusterName: o.clusterName, - Hub: Hub{ - APIServer: o.hubAPIServer, - }, - Registry: o.registry, - Klusterlet: Klusterlet{ - Mode: o.mode, - Name: klusterletName, - AgentNamespace: agentNamespace, - KlusterletNamespace: klusterletNamespace, - }, - ManagedKubeconfig: o.managedKubeconfigFile, - RegistrationFeatures: genericclioptionsclusteradm.ConvertToFeatureGateAPI(genericclioptionsclusteradm.SpokeMutableFeatureGate, ocmfeature.DefaultSpokeRegistrationFeatureGates), - WorkFeatures: genericclioptionsclusteradm.ConvertToFeatureGateAPI(genericclioptionsclusteradm.SpokeMutableFeatureGate, ocmfeature.DefaultSpokeWorkFeatureGates), - } + } + fmt.Fprintf(o.Streams.Out, "flag %s has been set\n", flag.Name) + }) - versionBundle, err := version.GetVersionBundle(o.bundleVersion) + klog.V(1).InfoS("join options:", "dry-run", o.ClusteradmFlags.DryRun, "cluster", o.clusterName, "api-server", o.hubAPIServer, "output", o.outputFile) - if err != nil { - klog.Errorf("unable to retrieve version ", err) - return err - } + if o.mode == InstallModeControlplaneAgent { // deploy using multicluster controlplane + o.values = Values{ + ClusterName: o.clusterName, + Hub: Hub{ + APIServer: o.hubAPIServer, + }, + ControlplaneAgentImage: o.controlplaneAgentImage, + } + } else { // deploy using operator + // operatorNamespace is the namespace to deploy klsuterlet; + // agentNamespace is the namesapce to deploy the agents(registration agent, work agent, etc.); + // klusterletNamespace is the namespace created on the managed cluster for each klusterlet. + // + // The operatorNamespace is fixed to "open-cluster-management". + // In default mode, agentNamespace is "open-cluster-management-agent", klusterletNamespace refers to agentNamespace, all of these three namesapces are on the managed cluster; + // In hosted mode, operatorNamespace is on the management cluster, agentNamesapce is "-<6-bit random string>" on the management cluster, and the klusterletNamespace is "open-cluster-management-" on the managed cluster. + + // values for default mode + klusterletName := DefaultOperatorName + agentNamespace := KlusterletNamespacePrefix + "agent" + klusterletNamespace := agentNamespace + if o.mode == InstallModeHosted { + // add hash suffix to avoid conflict + klusterletName += "-hosted-" + helpers.RandStringRunes_az09(6) + agentNamespace = klusterletName + klusterletNamespace = KlusterletNamespacePrefix + agentNamespace + } - o.values.BundleVersion = BundleVersion{ - RegistrationImageVersion: versionBundle.Registration, - PlacementImageVersion: versionBundle.Placement, - WorkImageVersion: versionBundle.Work, - OperatorImageVersion: versionBundle.Operator, + o.values = Values{ + ClusterName: o.clusterName, + Hub: Hub{ + APIServer: o.hubAPIServer, + }, + Registry: o.registry, + Klusterlet: Klusterlet{ + Mode: o.mode, + Name: klusterletName, + AgentNamespace: agentNamespace, + KlusterletNamespace: klusterletNamespace, + }, + ManagedKubeconfig: o.managedKubeconfigFile, + RegistrationFeatures: genericclioptionsclusteradm.ConvertToFeatureGateAPI(genericclioptionsclusteradm.SpokeMutableFeatureGate, ocmfeature.DefaultSpokeRegistrationFeatureGates), + WorkFeatures: genericclioptionsclusteradm.ConvertToFeatureGateAPI(genericclioptionsclusteradm.SpokeMutableFeatureGate, ocmfeature.DefaultSpokeWorkFeatureGates), + } + + versionBundle, err := version.GetVersionBundle(o.bundleVersion) + + if err != nil { + klog.Errorf("unable to retrieve version ", err) + return err + } + + o.values.BundleVersion = BundleVersion{ + RegistrationImageVersion: versionBundle.Registration, + PlacementImageVersion: versionBundle.Placement, + WorkImageVersion: versionBundle.Work, + OperatorImageVersion: versionBundle.Operator, + } + klog.V(3).InfoS("Image version:", + "'registration image version'", versionBundle.Registration, + "'placement image version'", versionBundle.Placement, + "'work image version'", versionBundle.Work, + "'operator image version'", versionBundle.Operator) } - klog.V(3).InfoS("Image version:", - "'registration image version'", versionBundle.Registration, - "'placement image version'", versionBundle.Placement, - "'work image version'", versionBundle.Work, - "'operator image version'", versionBundle.Operator) // if --ca-file is set, read ca data if o.caFile != "" { @@ -177,21 +216,24 @@ func (o *Options) complete(cmd *cobra.Command, args []string) (err error) { } func (o *Options) validate() error { + checkers := []preflightinterface.Checker{ + preflight.HubKubeconfigCheck{ + Config: o.HubConfig, + }, + preflight.ClusterNameCheck{ + ClusterName: o.values.ClusterName, + }, + preflight.DeployModeCheck{ + Mode: o.mode, + InternalEndpoint: o.forceHubInClusterEndpointLookup, + ManagedKubeconfigFile: o.managedKubeconfigFile, + Registry: o.registry, + ControlplaneAgentImage: o.controlplaneAgentImage, + }, + } // preflight check if err := preflightinterface.RunChecks( - []preflightinterface.Checker{ - preflight.HubKubeconfigCheck{ - Config: o.HubConfig, - }, - preflight.DeployModeCheck{ - Mode: o.mode, - InternalEndpoint: o.forceHubInClusterEndpointLookup, - ManagedKubeconfigFile: o.managedKubeconfigFile, - }, - preflight.ClusterNameCheck{ - ClusterName: o.values.ClusterName, - }, - }, os.Stderr); err != nil { + checkers, os.Stderr); err != nil { return err } @@ -213,84 +255,129 @@ func (o *Options) validate() error { } func (o *Options) run() error { - _, apiExtensionsClient, _, err := helpers.GetClients(o.ClusteradmFlags.KubectlFactory) + kubeClient, apiExtensionsClient, _, err := helpers.GetClients(o.ClusteradmFlags.KubectlFactory) if err != nil { return err } - available, err := checkIfRegistrationOperatorAvailable(o.ClusteradmFlags.KubectlFactory) - if err != nil { - return err - } + r := reader.NewResourceReader(o.builder, o.ClusteradmFlags.DryRun, o.Streams) - files := []string{} - // If Deployment/klusterlet is not deployed, deploy it - if !available { - files = append(files, - "join/klusterlets.crd.yaml", - "join/namespace.yaml", - "join/service_account.yaml", - "join/cluster_role.yaml", - "join/cluster_role_binding.yaml", - ) - } - files = append(files, - "join/namespace_agent.yaml", - "join/bootstrap_hub_kubeconfig.yaml", - ) + if o.mode == InstallModeControlplaneAgent { + _, err = kubeClient.CoreV1().Namespaces().Get(context.TODO(), "multicluster-controlplane-agent", metav1.GetOptions{}) + if err != nil { + if errors.IsNotFound(err) { + _, err = kubeClient.CoreV1().Namespaces().Create(context.TODO(), &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: "multicluster-controlplane-agent", + }, + }, metav1.CreateOptions{}) + if err != nil { + return err + } + } else { + return err + } + } + + files := []string{} - if o.mode == InstallModeHosted { files = append(files, - "join/hosted/external_managed_kubeconfig.yaml", + "controlplane/bootstrap_hub_kubeconfig.yaml", + "controlplane/clusterrole.yaml", + "controlplane/clusterrolebinding-admin.yaml", + "controlplane/clusterrolebinding.yaml", + "controlplane/role.yaml", + "controlplane/rolebinding.yaml", + "controlplane/serviceaccount.yaml", + "controlplane/deployment.yaml", ) - } - r := reader.NewResourceReader(o.builder, o.ClusteradmFlags.DryRun, o.Streams) - err = r.Apply(scenario.Files, o.values, files...) - if err != nil { - return err - } - - if !available { - err = r.Apply(scenario.Files, o.values, "join/operator.yaml") + err = r.Apply(scenario.Files, o.values, files...) if err != nil { return err } - } - if !o.ClusteradmFlags.DryRun { - if err := wait.WaitUntilCRDReady(apiExtensionsClient, "klusterlets.operator.open-cluster-management.io", o.wait); err != nil { + if o.wait && !o.ClusteradmFlags.DryRun { + err = waitUntilControlplaneAgentConditionIsTrue(o.ClusteradmFlags.KubectlFactory, int64(o.ClusteradmFlags.Timeout), "multicluster-controlplane-agent") + if err != nil { + return err + } + } + + } else { + available, err := checkIfRegistrationOperatorAvailable(o.ClusteradmFlags.KubectlFactory) + if err != nil { return err } - } - err = r.Apply(scenario.Files, o.values, "join/klusterlets.cr.yaml") - if err != nil { - return err - } + files := []string{} + // If Deployment/klusterlet is not deployed, deploy it + if !available { + files = append(files, + "join/klusterlets.crd.yaml", + "join/namespace.yaml", + "join/service_account.yaml", + "join/cluster_role.yaml", + "join/cluster_role_binding.yaml", + ) + } + files = append(files, + "join/namespace_agent.yaml", + "join/bootstrap_hub_kubeconfig.yaml", + ) - klusterletNamespace := o.values.Klusterlet.KlusterletNamespace - agentNamespace := o.values.Klusterlet.AgentNamespace + if o.mode == InstallModeHosted { + files = append(files, + "join/hosted/external_managed_kubeconfig.yaml", + ) + } - if !available && o.wait && !o.ClusteradmFlags.DryRun { - err = waitUntilRegistrationOperatorConditionIsTrue(o.ClusteradmFlags.KubectlFactory, int64(o.ClusteradmFlags.Timeout)) + err = r.Apply(scenario.Files, o.values, files...) if err != nil { return err } - } - if o.wait && !o.ClusteradmFlags.DryRun { - if o.mode == InstallModeHosted { - err = waitUntilKlusterletConditionIsTrue(o.ClusteradmFlags.KubectlFactory, int64(o.ClusteradmFlags.Timeout), agentNamespace) + if !available { + err = r.Apply(scenario.Files, o.values, "join/operator.yaml") if err != nil { return err } - } else { - err = waitUntilKlusterletConditionIsTrue(o.ClusteradmFlags.KubectlFactory, int64(o.ClusteradmFlags.Timeout), klusterletNamespace) + } + + if !o.ClusteradmFlags.DryRun { + if err := wait.WaitUntilCRDReady(apiExtensionsClient, "klusterlets.operator.open-cluster-management.io", o.wait); err != nil { + return err + } + } + + err = r.Apply(scenario.Files, o.values, "join/klusterlets.cr.yaml") + if err != nil { + return err + } + + klusterletNamespace := o.values.Klusterlet.KlusterletNamespace + agentNamespace := o.values.Klusterlet.AgentNamespace + + if !available && o.wait && !o.ClusteradmFlags.DryRun { + err = waitUntilRegistrationOperatorConditionIsTrue(o.ClusteradmFlags.KubectlFactory, int64(o.ClusteradmFlags.Timeout)) if err != nil { return err } } + + if o.wait && !o.ClusteradmFlags.DryRun { + if o.mode == InstallModeHosted { + err = waitUntilKlusterletConditionIsTrue(o.ClusteradmFlags.KubectlFactory, int64(o.ClusteradmFlags.Timeout), agentNamespace) + if err != nil { + return err + } + } else { + err = waitUntilKlusterletConditionIsTrue(o.ClusteradmFlags.KubectlFactory, int64(o.ClusteradmFlags.Timeout), klusterletNamespace) + if err != nil { + return err + } + } + } } if len(o.outputFile) > 0 { @@ -442,6 +529,52 @@ func waitUntilKlusterletConditionIsTrue(f util.Factory, timeout int64, agentName ) } +func waitUntilControlplaneAgentConditionIsTrue(f util.Factory, timeout int64, agentNamespace string) error { + client, err := f.KubernetesClientSet() + if err != nil { + return err + } + + phase := &atomic.Value{} + phase.Store("") + agentSpinner := printer.NewSpinnerWithStatus( + "Waiting for controlplane agent to become ready...", + time.Millisecond*500, + "Controlplane agent is now available.\n", + func() string { + return phase.Load().(string) + }) + agentSpinner.Start() + defer agentSpinner.Stop() + + return helpers.WatchUntil( + func() (watch.Interface, error) { + return client.CoreV1().Pods(agentNamespace). + Watch(context.TODO(), metav1.ListOptions{ + TimeoutSeconds: &timeout, + LabelSelector: "app=multicluster-controlplane-agent", + }) + }, + func(event watch.Event) bool { + pod, ok := event.Object.(*corev1.Pod) + if !ok { + return false + } + phase.Store(printer.GetSpinnerPodStatus(pod)) + conds := make([]metav1.Condition, len(pod.Status.Conditions)) + for i := range pod.Status.Conditions { + conds[i] = metav1.Condition{ + Type: string(pod.Status.Conditions[i].Type), + Status: metav1.ConditionStatus(pod.Status.Conditions[i].Status), + Reason: pod.Status.Conditions[i].Reason, + Message: pod.Status.Conditions[i].Message, + } + } + return meta.IsStatusConditionTrue(conds, "Ready") + }, + ) +} + // Create bootstrap with token but without CA func (o *Options) createExternalBootstrapConfig() clientcmdapiv1.Config { return clientcmdapiv1.Config{ diff --git a/pkg/cmd/join/options.go b/pkg/cmd/join/options.go index e4b23e312..05ab8254b 100644 --- a/pkg/cmd/join/options.go +++ b/pkg/cmd/join/options.go @@ -23,15 +23,18 @@ type Options struct { caFile string //the name under the cluster must be imported clusterName string - // klusterlet deploy mode, supported values are "hosted" and "default" + + // OCM agent deploy mode, default to "default". mode string // managed cluster kubeconfig file, used in hosted mode managedKubeconfigFile string - //Pulling image registry of OCM registry string // version of predefined compatible image versions bundleVersion string + + controlplaneAgentImage string + //The file to output the resources will be sent to the file. outputFile string //Runs the cluster joining in foreground @@ -76,6 +79,8 @@ type Values struct { // Features is the slice of feature for work WorkFeatures []operatorv1.FeatureGate + + ControlplaneAgentImage string } // Hub: The hub values for the template diff --git a/pkg/cmd/join/preflight/checks.go b/pkg/cmd/join/preflight/checks.go index efd38f9b2..c236c3ce5 100644 --- a/pkg/cmd/join/preflight/checks.go +++ b/pkg/cmd/join/preflight/checks.go @@ -12,8 +12,9 @@ import ( ) const ( - InstallModeDefault = "Default" - InstallModeHosted = "Hosted" + InstallModeDefault = "Default" + InstallModeHosted = "Hosted" + InstallModeControlplaneAgent = "Controlplane-agent" ) type HubKubeconfigCheck struct { @@ -57,29 +58,42 @@ func (c HubKubeconfigCheck) Name() string { } type DeployModeCheck struct { - Mode string - InternalEndpoint bool - ManagedKubeconfigFile string + Mode string + InternalEndpoint bool + ManagedKubeconfigFile string + Registry string + ControlplaneAgentImage string } func (c DeployModeCheck) Check() (warningList []string, errorList []error) { - if c.Mode != InstallModeDefault && c.Mode != InstallModeHosted { - return nil, []error{errors.New("deploy mode should be default or hosted")} + if c.Mode != InstallModeDefault && c.Mode != InstallModeHosted && c.Mode != InstallModeControlplaneAgent { + return nil, []error{errors.New("deploy mode should be default, hosted or controlplane-agent")} } - if c.Mode == InstallModeDefault { - if c.ManagedKubeconfigFile != "" { - return nil, []error{errors.New("--managed-cluster-kubeconfig should not be set in default deploy mode")} + + if c.Mode == InstallModeControlplaneAgent { + if len(c.ControlplaneAgentImage) == 0 { + return nil, []error{errors.New("the controlplane agent image should not be empty, like quay.io/open-cluster-management/multicluster-controlplane:latest")} } - } else { // c.Mode == InstallModeHosted - if c.ManagedKubeconfigFile == "" { - return nil, []error{errors.New("--managed-cluster-kubeconfig should be set in hosted deploy mode")} + } else { + if len(c.Registry) == 0 { + return nil, []error{errors.New("the OCM image registry should not be empty, like quay.io/open-cluster-management")} } - // if we use kind cluster as managed cluster, the kubeconfig should be --internal, the kubeconfig can be used by klusterlet - // deployed in management cluster, but can not be used by clusteradm to validate. so we jump the validate process - if !c.InternalEndpoint { - err := helpers.ValidateKubeconfigFile(c.ManagedKubeconfigFile) - if err != nil { - return nil, []error{errors.New(fmt.Sprintf("validate managed kubeconfig file failed: %v", err))} + + if c.Mode == InstallModeDefault { + if c.ManagedKubeconfigFile != "" { + return nil, []error{errors.New("--managed-cluster-kubeconfig should not be set in default deploy mode")} + } + } else { // c.Mode == InstallModeHosted + if c.ManagedKubeconfigFile == "" { + return nil, []error{errors.New("--managed-cluster-kubeconfig should be set in hosted deploy mode")} + } + // if we use kind cluster as managed cluster, the kubeconfig should be --internal, the kubeconfig can be used by klusterlet + // deployed in management cluster, but can not be used by clusteradm to validate. so we jump the validate process + if !c.InternalEndpoint { + err := helpers.ValidateKubeconfigFile(c.ManagedKubeconfigFile) + if err != nil { + return nil, []error{errors.New(fmt.Sprintf("validate managed kubeconfig file failed: %v", err))} + } } } } diff --git a/pkg/cmd/join/scenario/controlplane/bootstrap_hub_kubeconfig.yaml b/pkg/cmd/join/scenario/controlplane/bootstrap_hub_kubeconfig.yaml new file mode 100644 index 000000000..eb6dfe783 --- /dev/null +++ b/pkg/cmd/join/scenario/controlplane/bootstrap_hub_kubeconfig.yaml @@ -0,0 +1,9 @@ +# Copyright Contributors to the Open Cluster Management project +apiVersion: v1 +kind: Secret +metadata: + name: bootstrap-kubeconfig + namespace: multicluster-controlplane-agent +type: Opaque +data: + kubeconfig: {{ .Hub.KubeConfig }} diff --git a/pkg/cmd/join/scenario/controlplane/clusterrole.yaml b/pkg/cmd/join/scenario/controlplane/clusterrole.yaml new file mode 100644 index 000000000..c814fd30e --- /dev/null +++ b/pkg/cmd/join/scenario/controlplane/clusterrole.yaml @@ -0,0 +1,56 @@ +# Copyright Contributors to the Open Cluster Management project +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: open-cluster-management:multicluster-controlplane-agent +rules: +# Allow agent to manage crds +- apiGroups: ["apiextensions.k8s.io"] + resources: ["customresourcedefinitions"] + verbs: ["create", "get", "list", "update", "watch", "patch", "delete"] +# Allow agent to get/list/watch nodes +# list nodes to calculates the capacity and allocatable resources of the managed cluster +- apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch"] +# Allow agent to list clusterclaims +- apiGroups: ["cluster.open-cluster-management.io"] + resources: ["clusterclaims"] + verbs: ["get", "list", "watch"] +# Allow agent to create/update/patch/delete namespaces, get/list/watch are contained in admin role already +- apiGroups: [""] + resources: ["namespaces"] + verbs: ["create", "update", "patch", "delete"] +# Allow agent to manage role/rolebinding/clusterrole/clusterrolebinding +- apiGroups: ["rbac.authorization.k8s.io"] + resources: ["clusterrolebindings", "rolebindings"] + verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] +- apiGroups: ["rbac.authorization.k8s.io"] + resources: ["clusterroles", "roles"] + verbs: ["get", "list", "watch", "create", "update", "patch", "delete", "escalate", "bind"] +# Allow OCM addons to setup metrics collection with Prometheus +# when it is created. +- apiGroups: ["monitoring.coreos.com"] + resources: ["servicemonitors"] + verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] +# Allow agent to manage oauth clients +- apiGroups: ["oauth.openshift.io"] + resources: ["oauthclients"] + verbs: ["get", "list", "watch", "create", "patch","update", "delete"] +# Allow agent to manage appliedmanifestworks +- apiGroups: ["work.open-cluster-management.io"] + resources: ["appliedmanifestworks"] + verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] +- apiGroups: ["work.open-cluster-management.io"] + resources: ["appliedmanifestworks/status"] + verbs: ["patch", "update"] +- apiGroups: ["work.open-cluster-management.io"] + resources: ["appliedmanifestworks/finalizers"] + verbs: ["update"] +# Allow agent to check executor permissions +- apiGroups: ["authorization.k8s.io"] + resources: ["subjectaccessreviews"] + verbs: ["create"] +- apiGroups: [""] + resources: ["serviceaccounts"] + verbs: ["impersonate"] diff --git a/pkg/cmd/join/scenario/controlplane/clusterrolebinding-admin.yaml b/pkg/cmd/join/scenario/controlplane/clusterrolebinding-admin.yaml new file mode 100644 index 000000000..a46119f13 --- /dev/null +++ b/pkg/cmd/join/scenario/controlplane/clusterrolebinding-admin.yaml @@ -0,0 +1,15 @@ +# Copyright Contributors to the Open Cluster Management project +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: open-cluster-management:multicluster-controlplane-agent:work-execution-admin +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + # We deploy a controller that could work with permission lower than cluster-admin, the tradeoff is + # responsivity because list/watch cannot be maintained over too many namespaces. + name: admin +subjects: +- kind: ServiceAccount + name: multicluster-controlplane-agent-sa + namespace: multicluster-controlplane-agent diff --git a/pkg/cmd/join/scenario/controlplane/clusterrolebinding.yaml b/pkg/cmd/join/scenario/controlplane/clusterrolebinding.yaml new file mode 100644 index 000000000..f7f6a9419 --- /dev/null +++ b/pkg/cmd/join/scenario/controlplane/clusterrolebinding.yaml @@ -0,0 +1,13 @@ +# Copyright Contributors to the Open Cluster Management project +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: open-cluster-management:multicluster-controlplane-agent +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: open-cluster-management:multicluster-controlplane-agent +subjects: +- kind: ServiceAccount + name: multicluster-controlplane-agent-sa + namespace: multicluster-controlplane-agent diff --git a/pkg/cmd/join/scenario/controlplane/deployment.yaml b/pkg/cmd/join/scenario/controlplane/deployment.yaml new file mode 100644 index 000000000..eeac0d009 --- /dev/null +++ b/pkg/cmd/join/scenario/controlplane/deployment.yaml @@ -0,0 +1,48 @@ +# Copyright Contributors to the Open Cluster Management project +kind: Deployment +apiVersion: apps/v1 +metadata: + name: multicluster-controlplane-agent + namespace: multicluster-controlplane-agent + labels: + app: multicluster-controlplane-agent +spec: + replicas: 1 + selector: + matchLabels: + app: multicluster-controlplane-agent + template: + metadata: + labels: + app: multicluster-controlplane-agent + spec: + serviceAccountName: multicluster-controlplane-agent-sa + containers: + - name: agent + image: {{.ControlplaneAgentImage}} + imagePullPolicy: IfNotPresent + args: + - "/multicluster-controlplane" + - "agent" + - "--cluster-name={{.ClusterName}}" + - "--bootstrap-kubeconfig=/spoke/bootstrap/kubeconfig" + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + privileged: false + runAsNonRoot: true + volumeMounts: + - name: bootstrap-kubeconfig + mountPath: "/spoke/bootstrap" + readOnly: true + - name: hub-kubeconfig + mountPath: "/spoke/hub-kubeconfig" + volumes: + - name: bootstrap-kubeconfig + secret: + secretName: bootstrap-kubeconfig + - name: hub-kubeconfig + emptyDir: + medium: Memory diff --git a/pkg/cmd/join/scenario/controlplane/role.yaml b/pkg/cmd/join/scenario/controlplane/role.yaml new file mode 100644 index 000000000..8efc00243 --- /dev/null +++ b/pkg/cmd/join/scenario/controlplane/role.yaml @@ -0,0 +1,11 @@ +# Copyright Contributors to the Open Cluster Management project +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: multicluster-controlplane-agent + namespace: multicluster-controlplane-agent +rules: +# create hub-kubeconfig and external-managed-registration/work secrets +- apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "list", "watch", "create", "delete", "update", "patch"] diff --git a/pkg/cmd/join/scenario/controlplane/rolebinding.yaml b/pkg/cmd/join/scenario/controlplane/rolebinding.yaml new file mode 100644 index 000000000..7f0243c78 --- /dev/null +++ b/pkg/cmd/join/scenario/controlplane/rolebinding.yaml @@ -0,0 +1,14 @@ +# Copyright Contributors to the Open Cluster Management project +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: multicluster-controlplane-agent + namespace: multicluster-controlplane-agent +roleRef: + kind: Role + name: multicluster-controlplane-agent + apiGroup: rbac.authorization.k8s.io +subjects: +- kind: ServiceAccount + name: multicluster-controlplane-agent-sa + namespace: multicluster-controlplane-agent diff --git a/pkg/cmd/join/scenario/controlplane/serviceaccount.yaml b/pkg/cmd/join/scenario/controlplane/serviceaccount.yaml new file mode 100644 index 000000000..ab9f6ec16 --- /dev/null +++ b/pkg/cmd/join/scenario/controlplane/serviceaccount.yaml @@ -0,0 +1,6 @@ +# Copyright Contributors to the Open Cluster Management project +apiVersion: v1 +kind: ServiceAccount +metadata: + name: multicluster-controlplane-agent-sa + namespace: multicluster-controlplane-agent diff --git a/pkg/cmd/join/scenario/resources.go b/pkg/cmd/join/scenario/resources.go index 1497293bf..82cfbe2ba 100644 --- a/pkg/cmd/join/scenario/resources.go +++ b/pkg/cmd/join/scenario/resources.go @@ -5,5 +5,5 @@ import ( "embed" ) -//go:embed join +//go:embed join controlplane var Files embed.FS