add node-servant

cmd/yurt-node-servant/convert/convert.go

@@ -0,0 +1,66 @@
+/*
+Copyright 2021 The OpenYurt Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package convert
+
+import (
+	"time"
+
+	"github.com/spf13/cobra"
+	"k8s.io/klog"
+
+	nodeconverter "github.com/openyurtio/openyurt/pkg/node-servant/convert"
+)
+
+const (
+	// defaultYurthubHealthCheckTimeout defines the default timeout for yurthub health check phase
+	defaultYurthubHealthCheckTimeout = 2 * time.Minute
+)
+
+// NewConvertCmd generates a new convert command
+func NewConvertCmd() *cobra.Command {
+	o := nodeconverter.NewConvertOptions()
+	cmd := &cobra.Command{
+		Use:   "convert --working-mode",
+		Short: "",
+		Run: func(cmd *cobra.Command, args []string) {
+			if err := o.Complete(cmd.Flags()); err != nil {
+				klog.Fatalf("fail to complete the convert option: %s", err)
+			}
+
+			converter := nodeconverter.NewConverterWithOptions(o)
+			if err := converter.Do(); err != nil {
+				klog.Fatalf("fail to convert the kubernetes node to a yurt node: %s", err)
+			}
+			klog.Info("convert success")
+		},
+	}
+	setFlags(cmd)
+
+	return cmd
+}
+
+// setFlags sets flags.
+func setFlags(cmd *cobra.Command) {
+	cmd.Flags().String("yurthub-image", "openyurt/yurthub:latest",
+		"The yurthub image.")
+	cmd.Flags().Duration("yurthub-healthcheck-timeout", defaultYurthubHealthCheckTimeout,
+		"The timeout for yurthub health check.")
+	cmd.Flags().String("kubeadm-conf-path", "",
+		"The path to kubelet service conf that is used by kubelet component to join the cluster on the edge node.")
+	cmd.Flags().String("join-token", "", "The token used by yurthub for joining the cluster.")
+	cmd.Flags().String("working-mode", "edge", "The node type cloud/edge, effect yurthub workingMode.")
+}

cmd/yurt-node-servant/node-servant.go

@@ -0,0 +1,50 @@
+/*
+Copyright 2021 The OpenYurt Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"fmt"
+	"math/rand"
+	"os"
+	"time"
+
+	"github.com/openyurtio/openyurt/cmd/yurt-node-servant/convert"
+	"github.com/openyurtio/openyurt/cmd/yurt-node-servant/revert"
+	"github.com/openyurtio/openyurt/pkg/projectinfo"
+	"github.com/spf13/cobra"
+)
+
+// node-servant
+// running on specific node, do convert/revert job
+// yurtctl convert/revert join/reset, yurtcluster operator shall start a k8s job to run this.
+func main() {
+	rand.Seed(time.Now().UnixNano())
+
+	version := fmt.Sprintf("%#v", projectinfo.Get())
+	rootCmd := &cobra.Command{
+		Use:     "node-servant",
+		Short:   "node-servant do convert/revert specific node",
+		Version: version,
+	}
+	rootCmd.PersistentFlags().String("kubeconfig", "", "The path to the kubeconfig file")
+	rootCmd.AddCommand(convert.NewConvertCmd())
+	rootCmd.AddCommand(revert.NewRevertCmd())
+
+	if err := rootCmd.Execute(); err != nil { // run command
+		os.Exit(1)
+	}
+}

cmd/yurt-node-servant/revert/revert.go

@@ -0,0 +1,52 @@
+/*
+Copyright 2021 The OpenYurt Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package revert
+
+import (
+	"github.com/openyurtio/openyurt/pkg/node-servant/revert"
+	"github.com/spf13/cobra"
+	"k8s.io/klog"
+)
+
+// NewRevertCmd generates a new revert command
+func NewRevertCmd() *cobra.Command {
+	o := revert.NewRevertOptions()
+	cmd := &cobra.Command{
+		Use:   "revert",
+		Short: "",
+		Run: func(cmd *cobra.Command, args []string) {
+			if err := o.Complete(cmd.Flags()); err != nil {
+				klog.Fatalf("fail to complete the revert option: %s", err)
+			}
+
+			r := revert.NewReverterWithOptions(o)
+			if err := r.Do(); err != nil {
+				klog.Fatalf("fail to revert the yurt node to a kubernetes node: %s", err)
+			}
+			klog.Info("revert success")
+		},
+	}
+	setFlags(cmd)
+
+	return cmd
+}
+
+// setFlags sets flags.
+func setFlags(cmd *cobra.Command) {
+	cmd.Flags().String("kubeadm-conf-path", "",
+		"The path to kubelet service conf that is used by kubelet component to join the cluster on the edge node.")
+}

pkg/node-servant/components/kubelet.go

@@ -0,0 +1,220 @@
+/*
+Copyright 2021 The OpenYurt Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package components
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"regexp"
+	"strings"
+
+	"k8s.io/klog"
+
+	enutil "github.com/openyurtio/openyurt/pkg/yurtctl/util/edgenode"
+)
+
+const (
+	kubeletConfigRegularExpression = "\\-\\-kubeconfig=.*kubelet.conf"
+	apiserverAddrRegularExpression = "server: (http(s)?:\\/\\/)?[\\w][-\\w]{0,62}(\\.[\\w][-\\w]{0,62})*(:[\\d]{1,5})?"
+
+	kubeAdmFlagsEnvFile = "/var/lib/kubelet/kubeadm-flags.env"
+	dirMode             = 0755
+)
+
+type kubeletOperator struct {
+	openyurtDir     string
+	kubeadmConfPath string
+}
+
+func NewKubeletOperator(openyurtDir, kubeadmConfPath string) *kubeletOperator {
+	return &kubeletOperator{
+		openyurtDir:     openyurtDir,
+		kubeadmConfPath: kubeadmConfPath,
+	}
+}
+
+// RedirectTrafficToYurtHub
+func (op *kubeletOperator) RedirectTrafficToYurtHub() error {
+	// 1. create a working dir to store revised kubelet.conf
+	_, err := op.writeYurthubKubeletConfig()
+	if err != nil {
+		return err
+	}
+
+	// 2. append /var/lib/kubelet/kubeadm-flags.env
+	if err := op.appendConfig(); err != nil {
+		return err
+	}
+
+	// 3. restart
+	return restartKubeletService()
+}
+
+// UndoRedirectTrafficToYurtHub
+func (op *kubeletOperator) UndoRedirectTrafficToYurtHub() error {
+	if err := op.undoAppendConfig(); err != nil {
+		return err
+	}
+
+	// to compatible the old convert way for a while here
+	// todo: remove this
+	if err := op.renameSvcBk(); err != nil {
+		return err
+	}
+
+	if err := restartKubeletService(); err != nil {
+		return err
+	}
+
+	if err := op.undoWriteYurthubKubeletConfig(); err != nil {
+		return err
+	}
+	klog.Info("revertKubelet: undoWriteYurthubKubeletConfig finished")
+
+	return nil
+}
+
+func (op *kubeletOperator) writeYurthubKubeletConfig() (string, error) {
+	err := os.MkdirAll(op.openyurtDir, dirMode)
+	if err != nil {
+		return "", err
+	}
+	fullPath := op.getYurthubKubeletConf()
+	err = ioutil.WriteFile(fullPath, []byte(enutil.OpenyurtKubeletConf), fileMode)
+	if err != nil {
+		return "", err
+	}
+	klog.Infof("revised kubeconfig %s is generated", fullPath)
+	return fullPath, nil
+}
+
+func (op *kubeletOperator) undoWriteYurthubKubeletConfig() error {
+	yurtKubeletConf := op.getYurthubKubeletConf()
+	if _, err := enutil.FileExists(yurtKubeletConf); err != nil && os.IsNotExist(err) {
+		return nil
+	}
+
+	return os.Remove(yurtKubeletConf)
+}
+
+func (op *kubeletOperator) appendConfig() error {
+	// set env KUBELET_KUBEADM_ARGS, args set later will override before
+	// ExecStart: kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_CONFIG_ARGS $KUBELET_KUBEADM_ARGS $KUBELET_EXTRA_ARGS
+	// append setup: " --kubeconfig=$yurthubKubeletConf -bootstrap-kubeconfig= "
+	kubeConfigSetup := op.getAppendSetting()
+
+	// if wrote, return
+	content, err := ioutil.ReadFile(kubeAdmFlagsEnvFile)
+	if err != nil {
+		return err
+	}
+	args := string(content)
+	if strings.Contains(args, kubeConfigSetup) {
+		klog.Info("kubeConfigSetup has wrote before")
+		return nil
+	}
+
+	// append KUBELET_KUBEADM_ARGS
+	argsRegexp := regexp.MustCompile(`KUBELET_KUBEADM_ARGS="(.+)"`)
+	finding := argsRegexp.FindStringSubmatch(args)
+	if len(finding) != 2 {
+		return fmt.Errorf("error format. %s", args)
+	}
+
+	r := strings.Replace(args, finding[1], fmt.Sprintf("%s %s", finding[1], kubeConfigSetup), 1)
+	err = ioutil.WriteFile(kubeAdmFlagsEnvFile, []byte(r), fileMode)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (op *kubeletOperator) undoAppendConfig() error {
+	kubeConfigSetup := op.getAppendSetting()
+	contentbyte, err := ioutil.ReadFile(kubeAdmFlagsEnvFile)
+	if err != nil {
+		return err
+	}
+
+	content := strings.Replace(string(contentbyte), kubeConfigSetup, "", -1)
+	err = ioutil.WriteFile(kubeAdmFlagsEnvFile, []byte(content), 0644)
+	if err != nil {
+		return err
+	}
+	klog.Info("revertKubelet: undoAppendConfig finished")
+
+	return nil
+}
+
+// we used to modify kubelet svc config to do traffic redirect
+// and back up the original file
+// todo: remove this
+func (op *kubeletOperator) renameSvcBk() error {
+	kubeletSvcBk := op.getKubeletSvcBackup()
+	if ok, _ := enutil.FileExists(kubeletSvcBk); !ok {
+		return nil
+	}
+
+	klog.Infof("revertKubelet: renameSvcBk %s", kubeletSvcBk)
+	return os.Rename(kubeletSvcBk, op.kubeadmConfPath)
+}
+
+func (op *kubeletOperator) getAppendSetting() string {
+	configPath := op.getYurthubKubeletConf()
+	return fmt.Sprintf(" --kubeconfig=%s --bootstrap-kubeconfig= ", configPath)
+}
+
+func (op *kubeletOperator) getYurthubKubeletConf() string {
+	return filepath.Join(op.openyurtDir, enutil.KubeletConfName)
+}
+
+func (op *kubeletOperator) getKubeletSvcBackup() string {
+	return fmt.Sprintf(enutil.KubeletSvcBackup, op.kubeadmConfPath)
+}
+
+func restartKubeletService() error {
+	klog.Info("restartKubelet: " + enutil.DaemonReload)
+	cmd := exec.Command("bash", "-c", enutil.DaemonReload)
+	if err := enutil.Exec(cmd); err != nil {
+		return err
+	}
+	klog.Info("restartKubelet: " + enutil.RestartKubeletSvc)
+	cmd = exec.Command("bash", "-c", enutil.RestartKubeletSvc)
+	if err := enutil.Exec(cmd); err != nil {
+		return err
+	}
+	klog.Infof("restartKubelet: kubelet has been restarted")
+	return nil
+}
+
+func GetApiServerAddress(kubeadmConfPath string) (string, error) {
+	kubeletConfPath, err := enutil.GetSingleContentFromFile(kubeadmConfPath, kubeletConfigRegularExpression)
+	if err != nil {
+		return "", err
+	}
+	kubeletConfPath = strings.Split(kubeletConfPath, "=")[1]
+	apiserverAddr, err := enutil.GetSingleContentFromFile(kubeletConfPath, apiserverAddrRegularExpression)
+	if err != nil {
+		return "", err
+	}
+	apiserverAddr = strings.Split(apiserverAddr, " ")[1]
+	return apiserverAddr, nil
+}