From bb91d4d272b341a9579c8b6a90e6e84d198ba09c Mon Sep 17 00:00:00 2001 From: adamzhoul <770822772@qq.com> Date: Mon, 11 Oct 2021 19:31:29 +0800 Subject: [PATCH] add node-servant --- cmd/yurt-node-servant/convert/convert.go | 66 +++++++ cmd/yurt-node-servant/node-servant.go | 50 ++++++ cmd/yurt-node-servant/revert/revert.go | 52 ++++++ pkg/node-servant/components/kubelet.go | 220 +++++++++++++++++++++++ pkg/node-servant/components/node.go | 117 ++++++++++++ pkg/node-servant/components/yurthub.go | 175 ++++++++++++++++++ pkg/node-servant/constant.go | 112 ++++++++++++ pkg/node-servant/convert/convert.go | 123 +++++++++++++ pkg/node-servant/convert/options.go | 113 ++++++++++++ pkg/node-servant/job.go | 71 ++++++++ pkg/node-servant/revert/options.go | 75 ++++++++ pkg/node-servant/revert/revert.go | 105 +++++++++++ 12 files changed, 1279 insertions(+) create mode 100644 cmd/yurt-node-servant/convert/convert.go create mode 100644 cmd/yurt-node-servant/node-servant.go create mode 100644 cmd/yurt-node-servant/revert/revert.go create mode 100644 pkg/node-servant/components/kubelet.go create mode 100644 pkg/node-servant/components/node.go create mode 100644 pkg/node-servant/components/yurthub.go create mode 100644 pkg/node-servant/constant.go create mode 100644 pkg/node-servant/convert/convert.go create mode 100644 pkg/node-servant/convert/options.go create mode 100644 pkg/node-servant/job.go create mode 100644 pkg/node-servant/revert/options.go create mode 100644 pkg/node-servant/revert/revert.go diff --git a/cmd/yurt-node-servant/convert/convert.go b/cmd/yurt-node-servant/convert/convert.go new file mode 100644 index 00000000000..0f8e9bff5dd --- /dev/null +++ b/cmd/yurt-node-servant/convert/convert.go @@ -0,0 +1,66 @@ +/* +Copyright 2021 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package convert + +import ( + "time" + + "github.com/spf13/cobra" + "k8s.io/klog" + + nodeconverter "github.com/openyurtio/openyurt/pkg/node-servant/convert" +) + +const ( + // defaultYurthubHealthCheckTimeout defines the default timeout for yurthub health check phase + defaultYurthubHealthCheckTimeout = 2 * time.Minute +) + +// NewConvertCmd generates a new convert command +func NewConvertCmd() *cobra.Command { + o := nodeconverter.NewConvertOptions() + cmd := &cobra.Command{ + Use: "convert --working-mode", + Short: "", + Run: func(cmd *cobra.Command, args []string) { + if err := o.Complete(cmd.Flags()); err != nil { + klog.Fatalf("fail to complete the convert option: %s", err) + } + + converter := nodeconverter.NewConverterWithOptions(o) + if err := converter.Do(); err != nil { + klog.Fatalf("fail to convert the kubernetes node to a yurt node: %s", err) + } + klog.Info("convert success") + }, + } + setFlags(cmd) + + return cmd +} + +// setFlags sets flags. +func setFlags(cmd *cobra.Command) { + cmd.Flags().String("yurthub-image", "openyurt/yurthub:latest", + "The yurthub image.") + cmd.Flags().Duration("yurthub-healthcheck-timeout", defaultYurthubHealthCheckTimeout, + "The timeout for yurthub health check.") + cmd.Flags().String("kubeadm-conf-path", "", + "The path to kubelet service conf that is used by kubelet component to join the cluster on the edge node.") + cmd.Flags().String("join-token", "", "The token used by yurthub for joining the cluster.") + cmd.Flags().String("working-mode", "edge", "The node type cloud/edge, effect yurthub workingMode.") +} diff --git a/cmd/yurt-node-servant/node-servant.go b/cmd/yurt-node-servant/node-servant.go new file mode 100644 index 00000000000..3f36d94c9be --- /dev/null +++ b/cmd/yurt-node-servant/node-servant.go @@ -0,0 +1,50 @@ +/* +Copyright 2021 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "fmt" + "math/rand" + "os" + "time" + + "github.com/openyurtio/openyurt/cmd/yurt-node-servant/convert" + "github.com/openyurtio/openyurt/cmd/yurt-node-servant/revert" + "github.com/openyurtio/openyurt/pkg/projectinfo" + "github.com/spf13/cobra" +) + +// node-servant +// running on specific node, do convert/revert job +// yurtctl convert/revert join/reset, yurtcluster operator shall start a k8s job to run this. +func main() { + rand.Seed(time.Now().UnixNano()) + + version := fmt.Sprintf("%#v", projectinfo.Get()) + rootCmd := &cobra.Command{ + Use: "node-servant", + Short: "node-servant do convert/revert specific node", + Version: version, + } + rootCmd.PersistentFlags().String("kubeconfig", "", "The path to the kubeconfig file") + rootCmd.AddCommand(convert.NewConvertCmd()) + rootCmd.AddCommand(revert.NewRevertCmd()) + + if err := rootCmd.Execute(); err != nil { // run command + os.Exit(1) + } +} diff --git a/cmd/yurt-node-servant/revert/revert.go b/cmd/yurt-node-servant/revert/revert.go new file mode 100644 index 00000000000..67d599bd21f --- /dev/null +++ b/cmd/yurt-node-servant/revert/revert.go @@ -0,0 +1,52 @@ +/* +Copyright 2021 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package revert + +import ( + "github.com/openyurtio/openyurt/pkg/node-servant/revert" + "github.com/spf13/cobra" + "k8s.io/klog" +) + +// NewRevertCmd generates a new revert command +func NewRevertCmd() *cobra.Command { + o := revert.NewRevertOptions() + cmd := &cobra.Command{ + Use: "revert", + Short: "", + Run: func(cmd *cobra.Command, args []string) { + if err := o.Complete(cmd.Flags()); err != nil { + klog.Fatalf("fail to complete the revert option: %s", err) + } + + r := revert.NewReverterWithOptions(o) + if err := r.Do(); err != nil { + klog.Fatalf("fail to revert the yurt node to a kubernetes node: %s", err) + } + klog.Info("revert success") + }, + } + setFlags(cmd) + + return cmd +} + +// setFlags sets flags. +func setFlags(cmd *cobra.Command) { + cmd.Flags().String("kubeadm-conf-path", "", + "The path to kubelet service conf that is used by kubelet component to join the cluster on the edge node.") +} diff --git a/pkg/node-servant/components/kubelet.go b/pkg/node-servant/components/kubelet.go new file mode 100644 index 00000000000..90885977a16 --- /dev/null +++ b/pkg/node-servant/components/kubelet.go @@ -0,0 +1,220 @@ +/* +Copyright 2021 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package components + +import ( + "fmt" + "io/ioutil" + "os" + "os/exec" + "path/filepath" + "regexp" + "strings" + + "k8s.io/klog" + + enutil "github.com/openyurtio/openyurt/pkg/yurtctl/util/edgenode" +) + +const ( + kubeletConfigRegularExpression = "\\-\\-kubeconfig=.*kubelet.conf" + apiserverAddrRegularExpression = "server: (http(s)?:\\/\\/)?[\\w][-\\w]{0,62}(\\.[\\w][-\\w]{0,62})*(:[\\d]{1,5})?" + + kubeAdmFlagsEnvFile = "/var/lib/kubelet/kubeadm-flags.env" + dirMode = 0755 +) + +type kubeletOperator struct { + openyurtDir string + kubeadmConfPath string +} + +func NewKubeletOperator(openyurtDir, kubeadmConfPath string) *kubeletOperator { + return &kubeletOperator{ + openyurtDir: openyurtDir, + kubeadmConfPath: kubeadmConfPath, + } +} + +// RedirectTrafficToYurtHub +func (op *kubeletOperator) RedirectTrafficToYurtHub() error { + // 1. create a working dir to store revised kubelet.conf + _, err := op.writeYurthubKubeletConfig() + if err != nil { + return err + } + + // 2. append /var/lib/kubelet/kubeadm-flags.env + if err := op.appendConfig(); err != nil { + return err + } + + // 3. restart + return restartKubeletService() +} + +// UndoRedirectTrafficToYurtHub +func (op *kubeletOperator) UndoRedirectTrafficToYurtHub() error { + if err := op.undoAppendConfig(); err != nil { + return err + } + + // to compatible the old convert way for a while here + // todo: remove this + if err := op.renameSvcBk(); err != nil { + return err + } + + if err := restartKubeletService(); err != nil { + return err + } + + if err := op.undoWriteYurthubKubeletConfig(); err != nil { + return err + } + klog.Info("revertKubelet: undoWriteYurthubKubeletConfig finished") + + return nil +} + +func (op *kubeletOperator) writeYurthubKubeletConfig() (string, error) { + err := os.MkdirAll(op.openyurtDir, dirMode) + if err != nil { + return "", err + } + fullPath := op.getYurthubKubeletConf() + err = ioutil.WriteFile(fullPath, []byte(enutil.OpenyurtKubeletConf), fileMode) + if err != nil { + return "", err + } + klog.Infof("revised kubeconfig %s is generated", fullPath) + return fullPath, nil +} + +func (op *kubeletOperator) undoWriteYurthubKubeletConfig() error { + yurtKubeletConf := op.getYurthubKubeletConf() + if _, err := enutil.FileExists(yurtKubeletConf); err != nil && os.IsNotExist(err) { + return nil + } + + return os.Remove(yurtKubeletConf) +} + +func (op *kubeletOperator) appendConfig() error { + // set env KUBELET_KUBEADM_ARGS, args set later will override before + // ExecStart: kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_CONFIG_ARGS $KUBELET_KUBEADM_ARGS $KUBELET_EXTRA_ARGS + // append setup: " --kubeconfig=$yurthubKubeletConf -bootstrap-kubeconfig= " + kubeConfigSetup := op.getAppendSetting() + + // if wrote, return + content, err := ioutil.ReadFile(kubeAdmFlagsEnvFile) + if err != nil { + return err + } + args := string(content) + if strings.Contains(args, kubeConfigSetup) { + klog.Info("kubeConfigSetup has wrote before") + return nil + } + + // append KUBELET_KUBEADM_ARGS + argsRegexp := regexp.MustCompile(`KUBELET_KUBEADM_ARGS="(.+)"`) + finding := argsRegexp.FindStringSubmatch(args) + if len(finding) != 2 { + return fmt.Errorf("error format. %s", args) + } + + r := strings.Replace(args, finding[1], fmt.Sprintf("%s %s", finding[1], kubeConfigSetup), 1) + err = ioutil.WriteFile(kubeAdmFlagsEnvFile, []byte(r), fileMode) + if err != nil { + return err + } + + return nil +} + +func (op *kubeletOperator) undoAppendConfig() error { + kubeConfigSetup := op.getAppendSetting() + contentbyte, err := ioutil.ReadFile(kubeAdmFlagsEnvFile) + if err != nil { + return err + } + + content := strings.Replace(string(contentbyte), kubeConfigSetup, "", -1) + err = ioutil.WriteFile(kubeAdmFlagsEnvFile, []byte(content), 0644) + if err != nil { + return err + } + klog.Info("revertKubelet: undoAppendConfig finished") + + return nil +} + +// we used to modify kubelet svc config to do traffic redirect +// and back up the original file +// todo: remove this +func (op *kubeletOperator) renameSvcBk() error { + kubeletSvcBk := op.getKubeletSvcBackup() + if ok, _ := enutil.FileExists(kubeletSvcBk); !ok { + return nil + } + + klog.Infof("revertKubelet: renameSvcBk %s", kubeletSvcBk) + return os.Rename(kubeletSvcBk, op.kubeadmConfPath) +} + +func (op *kubeletOperator) getAppendSetting() string { + configPath := op.getYurthubKubeletConf() + return fmt.Sprintf(" --kubeconfig=%s --bootstrap-kubeconfig= ", configPath) +} + +func (op *kubeletOperator) getYurthubKubeletConf() string { + return filepath.Join(op.openyurtDir, enutil.KubeletConfName) +} + +func (op *kubeletOperator) getKubeletSvcBackup() string { + return fmt.Sprintf(enutil.KubeletSvcBackup, op.kubeadmConfPath) +} + +func restartKubeletService() error { + klog.Info("restartKubelet: " + enutil.DaemonReload) + cmd := exec.Command("bash", "-c", enutil.DaemonReload) + if err := enutil.Exec(cmd); err != nil { + return err + } + klog.Info("restartKubelet: " + enutil.RestartKubeletSvc) + cmd = exec.Command("bash", "-c", enutil.RestartKubeletSvc) + if err := enutil.Exec(cmd); err != nil { + return err + } + klog.Infof("restartKubelet: kubelet has been restarted") + return nil +} + +func GetApiServerAddress(kubeadmConfPath string) (string, error) { + kubeletConfPath, err := enutil.GetSingleContentFromFile(kubeadmConfPath, kubeletConfigRegularExpression) + if err != nil { + return "", err + } + kubeletConfPath = strings.Split(kubeletConfPath, "=")[1] + apiserverAddr, err := enutil.GetSingleContentFromFile(kubeletConfPath, apiserverAddrRegularExpression) + if err != nil { + return "", err + } + apiserverAddr = strings.Split(apiserverAddr, " ")[1] + return apiserverAddr, nil +} diff --git a/pkg/node-servant/components/node.go b/pkg/node-servant/components/node.go new file mode 100644 index 00000000000..ed6ccb8cf79 --- /dev/null +++ b/pkg/node-servant/components/node.go @@ -0,0 +1,117 @@ +/* +Copyright 2021 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package components + +import ( + "context" + + nodeutil "github.com/openyurtio/openyurt/pkg/controller/util/node" + "github.com/openyurtio/openyurt/pkg/projectinfo" + kubeutil "github.com/openyurtio/openyurt/pkg/yurtctl/util/kubernetes" + + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/klog" +) + +const ( + AnnotationAutonomy = "node.beta.alibabacloud.com/autonomy" +) + +type nodeOperator struct { + clientSet *kubernetes.Clientset +} + +func NewNodeOperator(c *kubernetes.Clientset) *nodeOperator { + return &nodeOperator{ + clientSet: c, + } +} + +func (op *nodeOperator) LabelEdgeAndOpenAutonomous(nodeName string, isEdge bool) error { + node, err := op.clientSet.CoreV1().Nodes().Get(context.Background(), nodeName, metav1.GetOptions{}) + if err != nil { + return err + } + + nodeLabel := "true" + nodeType := "edge-node" + if !isEdge { + nodeLabel = "false" + nodeType = "cloud-node" + } + node, err = kubeutil.LabelNode(op.clientSet, node, projectinfo.GetEdgeWorkerLabelKey(), nodeLabel) + if err != nil { + return err + } + klog.Infof("label node %s as the %s", nodeName, nodeType) + + if !isEdge { + // convert edge-node -> cloud-node + if _, foundAutonomy := node.Annotations[AnnotationAutonomy]; foundAutonomy { + klog.Infof("close the %s autonomous", nodeName) + delete(node.Annotations, AnnotationAutonomy) + _, err = op.clientSet.CoreV1().Nodes().Update(context.Background(), node, metav1.UpdateOptions{}) + return err + } + + return nil // cloud-node only do label + } + + klog.Infof("open the %s autonomous", nodeName) + _, err = kubeutil.AnnotateNode(op.clientSet, node, AnnotationAutonomy, "true") + if err != nil { + return err + } + + return nil +} + +func (op *nodeOperator) UndoLabelEdgeAndCloseAutonomous(nodeName string) error { + node, err := op.clientSet.CoreV1().Nodes().Get(context.Background(), nodeName, metav1.GetOptions{}) + if err != nil { + return err + } + //var foundLabel bool + var foundLabel, foundAutonomy bool + + if _, foundLabel = node.Labels[projectinfo.GetEdgeWorkerLabelKey()]; foundLabel { + delete(node.Labels, projectinfo.GetEdgeWorkerLabelKey()) + } + + if _, foundAutonomy = node.Annotations[AnnotationAutonomy]; foundAutonomy { + delete(node.Annotations, AnnotationAutonomy) + } + + if !foundLabel && !foundAutonomy { + klog.Infof("unLabelEdgeAndOpenAutonomous: node %s is not labeled or annotationAutonomy", nodeName) + return nil + } + + if _, err = op.clientSet.CoreV1().Nodes().Update(context.Background(), node, metav1.UpdateOptions{}); err != nil { + return err + } + + klog.Infof("unLabelEdgeAndOpenAutonomous: clear %s label&annotation done", nodeName) + return nil +} + +func IsNodeReady(status *v1.NodeStatus) bool { + _, condition := nodeutil.GetNodeCondition(status, v1.NodeReady) + return condition != nil && condition.Status == v1.ConditionTrue +} diff --git a/pkg/node-servant/components/yurthub.go b/pkg/node-servant/components/yurthub.go new file mode 100644 index 00000000000..f7b8fb630df --- /dev/null +++ b/pkg/node-servant/components/yurthub.go @@ -0,0 +1,175 @@ +/* +Copyright 2021 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package components + +import ( + "fmt" + + "io/ioutil" + "net/http" + "net/url" + "os" + "path/filepath" + "strings" + "time" + + enutil "github.com/openyurtio/openyurt/pkg/yurtctl/util/edgenode" + "github.com/openyurtio/openyurt/pkg/yurthub/certificate/hubself" + "github.com/openyurtio/openyurt/pkg/yurthub/util" + + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/klog" +) + +const ( + hubHealthzCheckFrequency = 10 * time.Second + fileMode = 0666 +) + +type yurtHubOperator struct { + apiServerAddr string + yurthubImage string + joinToken string + workingMode util.WorkingMode + yurthubHealthCheckTimeout time.Duration +} + +func NewYurthubOperator(apiServerAddr string, yurthubImage string, joinToken string, + workingMode util.WorkingMode, yurthubHealthCheckTimeout time.Duration) *yurtHubOperator { + return &yurtHubOperator{ + apiServerAddr: apiServerAddr, + yurthubImage: yurthubImage, + joinToken: joinToken, + workingMode: workingMode, + yurthubHealthCheckTimeout: yurthubHealthCheckTimeout, + } +} + +// Install +func (op *yurtHubOperator) Install() error { + + // 1. put yurt-hub yaml into /etc/kubernetes/manifests + klog.Infof("setting up yurthub on node") + + // 1-1. replace variables in yaml file + klog.Infof("setting up yurthub apiServer addr") + yurthubTemplate := enutil.ReplaceRegularExpression(enutil.YurthubTemplate, + map[string]string{ + "__kubernetes_service_addr__": op.apiServerAddr, + "__yurthub_image__": op.yurthubImage, + "__join_token__": op.joinToken, + "__working_mode__": string(op.workingMode), + }) + + // 1-2. create yurthub.yaml + podManifestPath := enutil.GetPodManifestPath() + if err := enutil.EnsureDir(podManifestPath); err != nil { + return err + } + err := ioutil.WriteFile(getYurthubYaml(podManifestPath), []byte(yurthubTemplate), fileMode) + if err != nil { + return err + } + klog.Infof("create the %s/yurt-hub.yaml", podManifestPath) + + // 2. wait yurthub pod to be ready + return hubHealthcheck(op.yurthubHealthCheckTimeout) +} + +// UnInstall +func (op *yurtHubOperator) UnInstall() error { + // 1. remove the yurt-hub.yaml to delete the yurt-hub + podManifestPath := enutil.GetPodManifestPath() + yurthubYamlPath := getYurthubYaml(podManifestPath) + if _, err := enutil.FileExists(yurthubYamlPath); os.IsNotExist(err) { + klog.Infof("UnInstallYurthub: %s is not exists, skip delete", yurthubYamlPath) + } else { + err := os.Remove(yurthubYamlPath) + if err != nil { + return err + } + klog.Infof("UnInstallYurthub: %s has been removed", yurthubYamlPath) + } + + // 2. remove yurt-hub config directory and certificates in it + yurthubConf := getYurthubConf() + if _, err := enutil.FileExists(yurthubConf); os.IsNotExist(err) { + klog.Infof("UnInstallYurthub: dir %s is not exists, skip delete", yurthubConf) + return nil + } + err := os.RemoveAll(yurthubConf) + if err != nil { + return err + } + klog.Infof("UnInstallYurthub: config dir %s has been removed", yurthubConf) + return nil +} + +func getYurthubYaml(podManifestPath string) string { + return filepath.Join(podManifestPath, enutil.YurthubYamlName) +} + +func getYurthubConf() string { + return filepath.Join(hubself.HubRootDir, hubself.HubName) +} + +// hubHealthcheck will check the status of yurthub pod +func hubHealthcheck(timeout time.Duration) error { + serverHealthzURL, err := url.Parse(fmt.Sprintf("http://%s", enutil.ServerHealthzServer)) + if err != nil { + return err + } + serverHealthzURL.Path = enutil.ServerHealthzURLPath + + start := time.Now() + return wait.PollImmediate(hubHealthzCheckFrequency, timeout, func() (bool, error) { + _, err := pingClusterHealthz(http.DefaultClient, serverHealthzURL.String()) + if err != nil { + klog.Infof("yurt-hub is not ready, ping cluster healthz with result: %v", err) + return false, nil + } + klog.Infof("yurt-hub healthz is OK after %f seconds", time.Since(start).Seconds()) + return true, nil + }) +} + +func pingClusterHealthz(client *http.Client, addr string) (bool, error) { + if client == nil { + return false, fmt.Errorf("http client is invalid") + } + + resp, err := client.Get(addr) + if err != nil { + return false, err + } + + b, err := ioutil.ReadAll(resp.Body) + defer resp.Body.Close() + if err != nil { + return false, fmt.Errorf("failed to read response of cluster healthz, %v", err) + } + + if resp.StatusCode != http.StatusOK { + return false, fmt.Errorf("response status code is %d", resp.StatusCode) + } + + if strings.ToLower(string(b)) != "ok" { + return false, fmt.Errorf("cluster healthz is %s", string(b)) + } + + return true, nil +} diff --git a/pkg/node-servant/constant.go b/pkg/node-servant/constant.go new file mode 100644 index 00000000000..bbf3c8c6cf7 --- /dev/null +++ b/pkg/node-servant/constant.go @@ -0,0 +1,112 @@ +/* +Copyright 2021 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package node_servant + +const ( + + // ConvertJobNameBase is the prefix of the convert ServantJob name + ConvertJobNameBase = "yurtctl-servant-convert" + // RevertJobNameBase is the prefix of the revert ServantJob name + RevertJobNameBase = "yurtctl-servant-revert" + + // ConvertServantJobTemplate defines the yurtctl convert servant job in yaml format + ConvertServantJobTemplate = ` +apiVersion: batch/v1 +kind: Job +metadata: + name: {{.jobName}} + namespace: kube-system +spec: + template: + spec: + hostPID: true + hostNetwork: true + restartPolicy: OnFailure + nodeName: {{.nodeName}} + volumes: + - name: host-var-tmp + hostPath: + path: /var/tmp + type: Directory + containers: + - name: yurtctl-servant + image: {{.yurtctl_servant_image}} + imagePullPolicy: IfNotPresent + command: + - /bin/sh + - -c + args: + - "cp /usr/local/bin/yurtctl /tmp && nsenter -t 1 -m -u -n -i -- /var/tmp/yurtctl convert {{.sub_command}} --yurthub-image {{.yurthub_image}} {{if .yurthub_healthcheck_timeout}}--yurthub-healthcheck-timeout {{.yurthub_healthcheck_timeout}} {{end}}--join-token {{.joinToken}} && rm /tmp/yurtctl" + securityContext: + privileged: true + volumeMounts: + - mountPath: /tmp + name: host-var-tmp + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + {{if .kubeadm_conf_path }} + - name: KUBELET_SVC + value: {{.kubeadm_conf_path}} + {{end}} +` + // RevertServantJobTemplate defines the yurtctl revert servant job in yaml format + RevertServantJobTemplate = ` +apiVersion: batch/v1 +kind: Job +metadata: + name: {{.jobName}} + namespace: kube-system +spec: + template: + spec: + hostPID: true + hostNetwork: true + restartPolicy: OnFailure + nodeName: {{.nodeName}} + volumes: + - name: host-var-tmp + hostPath: + path: /var/tmp + type: Directory + containers: + - name: yurtctl-servant + image: {{.yurtctl_servant_image}} + imagePullPolicy: IfNotPresent + command: + - /bin/sh + - -c + args: + - "cp /usr/local/bin/yurtctl /tmp && nsenter -t 1 -m -u -n -i -- /var/tmp/yurtctl revert {{.sub_command}} && rm /tmp/yurtctl" + securityContext: + privileged: true + volumeMounts: + - mountPath: /tmp + name: host-var-tmp + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + {{if .kubeadm_conf_path }} + - name: KUBELET_SVC + value: {{.kubeadm_conf_path}} + {{end}} +` +) diff --git a/pkg/node-servant/convert/convert.go b/pkg/node-servant/convert/convert.go new file mode 100644 index 00000000000..d6a6d3d1f19 --- /dev/null +++ b/pkg/node-servant/convert/convert.go @@ -0,0 +1,123 @@ +/* +Copyright 2021 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package convert + +import ( + "context" + "fmt" + + "github.com/openyurtio/openyurt/pkg/node-servant/components" + enutil "github.com/openyurtio/openyurt/pkg/yurtctl/util/edgenode" + kubeutil "github.com/openyurtio/openyurt/pkg/yurtctl/util/kubernetes" + "github.com/openyurtio/openyurt/pkg/yurthub/util" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// NodeConverter +type nodeConverter struct { + Options +} + +// NewConverter +func NewConverterWithOptions(o *Options) *nodeConverter { + return &nodeConverter{ + *o, + } +} + +// Do, do the convert job. +// shall be implemented as idempotent, can execute multiple times with no side-affect. +func (n *nodeConverter) Do() error { + if err := n.validateOptions(); err != nil { + return err + } + if err := n.preflightCheck(); err != nil { + return err + } + + if err := n.installYurtHub(); err != nil { + return err + } + if err := n.convertKubelet(); err != nil { + return err + } + + if err := n.labelEdgeAndOpenAutonomous(); err != nil { + return err + } + + return nil +} + +func (n *nodeConverter) validateOptions() error { + if !util.IsSupportedWorkingMode(n.workingMode) { + return fmt.Errorf("workingMode must be pointed out as cloud or edge. got %s", n.workingMode) + } + + return nil +} + +func (n *nodeConverter) preflightCheck() error { + // 1. check the server version + if err := kubeutil.ValidateServerVersion(n.clientSet); err != nil { + return err + } + + // 2. check if critical files exist + if _, err := enutil.FileExists(n.kubeadmConfPath); err != nil { + return err + } + + // 3. check the state of node + node, err := n.clientSet.CoreV1().Nodes().Get(context.Background(), n.nodeName, metav1.GetOptions{}) + if err != nil { + return err + } + if !components.IsNodeReady(&node.Status) { + return fmt.Errorf("cannot do the convert, the status of node: %s is not 'Ready'", node.Name) + } + + // 4. check controller-manager is working.(used for yurthub cert verify) + d, _ := n.clientSet.AppsV1().Deployments("kube-system"). + Get(context.Background(), "yurt-controller-manager", metav1.GetOptions{}) + if d == nil { + return fmt.Errorf("cannot do the convert, deployment yurt-controller-manager not found") + } + + return nil +} + +func (n *nodeConverter) installYurtHub() error { + apiServerAddress, err := components.GetApiServerAddress(n.kubeadmConfPath) + if err != nil { + return err + } + op := components.NewYurthubOperator(apiServerAddress, n.yurthubImage, n.joinToken, + n.workingMode, n.yurthubHealthCheckTimeout) + return op.Install() +} + +func (n *nodeConverter) convertKubelet() error { + op := components.NewKubeletOperator(n.openyurtDir, n.kubeadmConfPath) + return op.RedirectTrafficToYurtHub() +} + +func (n *nodeConverter) labelEdgeAndOpenAutonomous() error { + op := components.NewNodeOperator(n.clientSet) + return op.LabelEdgeAndOpenAutonomous(n.nodeName, n.workingMode == util.WorkingModeEdge) +} diff --git a/pkg/node-servant/convert/options.go b/pkg/node-servant/convert/options.go new file mode 100644 index 00000000000..fb20b3af2a1 --- /dev/null +++ b/pkg/node-servant/convert/options.go @@ -0,0 +1,113 @@ +/* +Copyright 2021 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package convert + +import ( + "os" + "time" + + // todo: move util out of yurtctl + enutil "github.com/openyurtio/openyurt/pkg/yurtctl/util/edgenode" + kubeutil "github.com/openyurtio/openyurt/pkg/yurtctl/util/kubernetes" + "github.com/openyurtio/openyurt/pkg/yurthub/util" + + "github.com/spf13/pflag" + "k8s.io/client-go/kubernetes" +) + +// Options has the information that required by convert operation +type Options struct { + clientSet *kubernetes.Clientset + + yurthubImage string + yurthubHealthCheckTimeout time.Duration + workingMode util.WorkingMode + + joinToken string + kubeadmConfPath string + openyurtDir string + nodeName string +} + +// NewConvertOptions creates a new Options +func NewConvertOptions() *Options { + return &Options{} +} + +// Complete completes all the required options. +func (o *Options) Complete(flags *pflag.FlagSet) error { + yurthubImage, err := flags.GetString("yurthub-image") + if err != nil { + return err + } + o.yurthubImage = yurthubImage + + yurthubHealthCheckTimeout, err := flags.GetDuration("yurthub-healthcheck-timeout") + if err != nil { + return err + } + o.yurthubHealthCheckTimeout = yurthubHealthCheckTimeout + + kubeadmConfPath, err := flags.GetString("kubeadm-conf-path") + if err != nil { + return err + } + if kubeadmConfPath == "" { + kubeadmConfPath = os.Getenv("KUBELET_SVC") + } + if kubeadmConfPath == "" { + kubeadmConfPath = enutil.KubeletSvcPath + } + o.kubeadmConfPath = kubeadmConfPath + + nodeName, err := enutil.GetNodeName(kubeadmConfPath) + if err != nil { + return err + } + o.nodeName = nodeName + + o.clientSet, err = enutil.GenClientSet(flags) + if err != nil { + return err + } + + joinToken, err := flags.GetString("join-token") + if err != nil { + return err + } + if joinToken == "" { + joinToken, err = kubeutil.GetOrCreateJoinTokenString(o.clientSet) + if err != nil { + return err + } + } + o.joinToken = joinToken + + openyurtDir := os.Getenv("OPENYURT_DIR") + if openyurtDir == "" { + openyurtDir = enutil.OpenyurtDir + } + o.openyurtDir = openyurtDir + + workingMode, err := flags.GetString("working-mode") + if err != nil { + return err + } + o.workingMode = util.WorkingMode(workingMode) + + return nil +} diff --git a/pkg/node-servant/job.go b/pkg/node-servant/job.go new file mode 100644 index 00000000000..06d7a70c631 --- /dev/null +++ b/pkg/node-servant/job.go @@ -0,0 +1,71 @@ +/* +Copyright 2021 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package node_servant + +import ( + "fmt" + + tmplutil "github.com/openyurtio/openyurt/pkg/yurtctl/util/templates" + batchv1 "k8s.io/api/batch/v1" + k8sruntime "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/serializer" + "k8s.io/client-go/kubernetes/scheme" +) + +// RenderNodeServantJob return k8s job +// to start k8s job to run convert/revert on specific node +func RenderNodeServantJob(action string, tmplCtx map[string]string, nodeName string) (*batchv1.Job, error) { + var servantJobTemplate, jobBaseName string + switch action { + case "convert": + servantJobTemplate = ConvertServantJobTemplate + jobBaseName = ConvertJobNameBase + case "revert": + servantJobTemplate = RevertServantJobTemplate + jobBaseName = RevertJobNameBase + default: + return nil, fmt.Errorf("action invalied: %s ", action) + } + + tmplCtx["jobName"] = jobBaseName + "-" + nodeName + tmplCtx["nodeName"] = nodeName + jobYaml, err := tmplutil.SubsituteTemplate(servantJobTemplate, tmplCtx) + if err != nil { + return nil, err + } + + srvJobObj, err := YamlToObject([]byte(jobYaml)) + if err != nil { + return nil, err + } + srvJob, ok := srvJobObj.(*batchv1.Job) + if !ok { + return nil, fmt.Errorf("fail to assert yurtctl-servant job") + } + + return srvJob, nil +} + +// YamlToObject deserializes object in yaml format to a runtime.Object +func YamlToObject(yamlContent []byte) (k8sruntime.Object, error) { + decode := serializer.NewCodecFactory(scheme.Scheme).UniversalDeserializer().Decode + obj, _, err := decode(yamlContent, nil, nil) + if err != nil { + return nil, err + } + return obj, nil +} diff --git a/pkg/node-servant/revert/options.go b/pkg/node-servant/revert/options.go new file mode 100644 index 00000000000..529813dfeb2 --- /dev/null +++ b/pkg/node-servant/revert/options.go @@ -0,0 +1,75 @@ +/* +Copyright 2021 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package revert + +import ( + "os" + + enutil "github.com/openyurtio/openyurt/pkg/yurtctl/util/edgenode" + + "github.com/spf13/pflag" + "k8s.io/client-go/kubernetes" +) + +// Options has the information that required by revert operation +type Options struct { + clientSet *kubernetes.Clientset + + kubeadmConfPath string + openyurtDir string + nodeName string +} + +// NewRevertOptions creates a new Options +func NewRevertOptions() *Options { + return &Options{} +} + +// Complete completes all the required options. +func (o *Options) Complete(flags *pflag.FlagSet) error { + + kubeadmConfPath, err := flags.GetString("kubeadm-conf-path") + if err != nil { + return err + } + if kubeadmConfPath == "" { + kubeadmConfPath = os.Getenv("KUBELET_SVC") + } + if kubeadmConfPath == "" { + kubeadmConfPath = enutil.KubeletSvcPath + } + o.kubeadmConfPath = kubeadmConfPath + + nodeName, err := enutil.GetNodeName(kubeadmConfPath) + if err != nil { + return err + } + o.nodeName = nodeName + + o.clientSet, err = enutil.GenClientSet(flags) + if err != nil { + return err + } + + openyurtDir := os.Getenv("OPENYURT_DIR") + if openyurtDir == "" { + openyurtDir = enutil.OpenyurtDir + } + o.openyurtDir = openyurtDir + + return nil +} diff --git a/pkg/node-servant/revert/revert.go b/pkg/node-servant/revert/revert.go new file mode 100644 index 00000000000..6f547ca510a --- /dev/null +++ b/pkg/node-servant/revert/revert.go @@ -0,0 +1,105 @@ +/* +Copyright 2021 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package revert + +import ( + "context" + "fmt" + "time" + + "github.com/openyurtio/openyurt/pkg/node-servant/components" + "github.com/openyurtio/openyurt/pkg/yurthub/util" + + kubeutil "github.com/openyurtio/openyurt/pkg/yurtctl/util/kubernetes" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// NodeReverter +type nodeReverter struct { + Options +} + +// NewReverterWithOptions +func NewReverterWithOptions(o *Options) *nodeReverter { + return &nodeReverter{ + *o, + } +} + +// Do, do the convert job +// shall be implemented as idempotent, can execute multiple times with no side-affect. +func (n *nodeReverter) Do() error { + if err := n.validateOptions(); err != nil { + return err + } + if err := n.preflightCheck(); err != nil { + return err + } + + if err := n.revertKubelet(); err != nil { + return err + } + if err := n.unInstallYurtHub(); err != nil { + return err + } + + if err := n.unLabelEdgeAndOpenAutonomous(); err != nil { + return err + } + + // todo: delete csr create by yurthub + + return nil +} + +func (n *nodeReverter) validateOptions() error { + return nil +} + +func (n *nodeReverter) preflightCheck() error { + // 1. check the server version + if err := kubeutil.ValidateServerVersion(n.clientSet); err != nil { + return err + } + + // 2. check the state of Nodes + node, err := n.clientSet.CoreV1().Nodes().Get(context.Background(), n.nodeName, metav1.GetOptions{}) + if err != nil { + return err + } + if !components.IsNodeReady(&node.Status) { + return fmt.Errorf("cannot do the convert, the status of node: %s is not 'Ready'", node.Name) + } + + return nil +} + +func (n *nodeReverter) revertKubelet() error { + op := components.NewKubeletOperator(n.openyurtDir, n.kubeadmConfPath) + return op.UndoRedirectTrafficToYurtHub() +} + +func (n *nodeReverter) unInstallYurtHub() error { + op := components.NewYurthubOperator("", "", "", + util.WorkingModeCloud, time.Duration(1)) // params is not important here + return op.UnInstall() +} + +func (n *nodeReverter) unLabelEdgeAndOpenAutonomous() error { + op := components.NewNodeOperator(n.clientSet) + return op.UndoLabelEdgeAndCloseAutonomous(n.nodeName) +}