Feat/new sealer ipv6 dual (#1804)

* Support ipv6 dual stack.

Signed-off-by: huaiyou <huaiyou.cyz@alibaba-inc.com>

cmd/seautil/cmd/ipvs.go

@@ -15,7 +15,7 @@
 package cmd
 
 import (
-	"github.com/sealyun/lvscare/care"
+	"github.com/labring/lvscare/care"
 	"github.com/spf13/cobra"
 )
 

common/common.go

@@ -71,6 +71,8 @@ const (
 	DefaultRegistryDomain         = "sea.hub"
 	DefaultRegistryPort           = "5000"
 	HostIP                        = "HostIP"
+	EnvHostIPFamily               = "HostIPFamily"
+	EnvIPv6DualStack              = "IPv6DualStack"
 )
 
 // image module

go.mod

@@ -19,6 +19,7 @@ require (
 	github.com/google/uuid v1.3.0
 	github.com/hashicorp/go-multierror v1.1.1
 	github.com/imdario/mergo v0.3.13
+	github.com/labring/lvscare v1.1.1
 	github.com/lestrrat-go/file-rotatelogs v2.4.0+incompatible
 	github.com/mitchellh/go-homedir v1.1.0
 	github.com/moby/buildkit v0.9.3
@@ -33,7 +34,6 @@ require (
 	github.com/pkg/errors v0.9.1
 	github.com/pkg/sftp v1.13.0
 	github.com/rifflock/lfshook v0.0.0-20180920164130-b9218ef580f5
-	github.com/sealyun/lvscare v1.1.2-alpha.2
 	github.com/shirou/gopsutil v3.21.11+incompatible
 	github.com/sirupsen/logrus v1.9.0
 	github.com/spf13/cobra v1.5.0
@@ -179,7 +179,6 @@ require (
 	github.com/vbatts/tar-split v0.11.2 // indirect
 	github.com/vbauerster/mpb/v7 v7.4.2 // indirect
 	github.com/vishvananda/netns v0.0.0-20211101163701-50045581ed74 // indirect
-	github.com/wonderivan/logger v1.0.0 // indirect
 	github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect
 	github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
 	github.com/xeipuuv/gojsonschema v1.2.0 // indirect
@@ -214,6 +213,7 @@ require (
 
 replace (
 	github.com/containerd/containerd => github.com/containerd/containerd v1.6.6
+	github.com/labring/lvscare => github.com/fanux/lvscare v1.1.3-beta.2.0.20220525083355-d8f3f86677d1
 	github.com/opencontainers/runc => github.com/opencontainers/runc v1.1.2
 	golang.org/x/crypto => golang.org/x/crypto v0.0.0-20210817164053-32db794688a5
 	golang.org/x/net => golang.org/x/net v0.0.0-20210510120150-4163338589ed

go.sum

@@ -610,6 +610,8 @@ github.com/facebookgo/muster v0.0.0-20150708232844-fd3d7953fd52 h1:a4DFiKFJiDRGF
 github.com/facebookgo/muster v0.0.0-20150708232844-fd3d7953fd52/go.mod h1:yIquW87NGRw1FU5p5lEkpnt/QxoH5uPAOUlOVkAUuMg=
 github.com/facebookgo/stack v0.0.0-20160209184415-751773369052/go.mod h1:UbMTZqLaRiH3MsBH8va0n7s1pQYcu3uTb8G4tygF4Zg=
 github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4/go.mod h1:5tD+neXqOorC30/tWg0LCSkrqj/AR6gu8yY8/fpw1q0=
+github.com/fanux/lvscare v1.1.3-beta.2.0.20220525083355-d8f3f86677d1 h1:BNaIeOFlxyn1dOZ0wgtNxVMUqW8tbjK1+aLAAATEL1U=
+github.com/fanux/lvscare v1.1.3-beta.2.0.20220525083355-d8f3f86677d1/go.mod h1:QvpJhFlNxIVhMHRN6I3q4+85r2kKsYJWMkSKDZK+4V8=
 github.com/fatih/camelcase v1.0.0/go.mod h1:yN2Sb0lFhZJUdVvtELVWefmrXpuZESvPmqwoZc+/fpc=
 github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
 github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU=
@@ -1744,8 +1746,6 @@ github.com/sassoftware/go-rpmutils v0.0.0-20190420191620-a8f1baeba37b/go.mod h1:
 github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
 github.com/sclevine/agouti v3.0.0+incompatible/go.mod h1:b4WX9W9L1sfQKXeJf1mUTLZKJ48R1S7H23Ji7oFO5Bw=
 github.com/sclevine/spec v1.2.0/go.mod h1:W4J29eT/Kzv7/b9IWLB055Z+qvVC9vt0Arko24q7p+U=
-github.com/sealyun/lvscare v1.1.2-alpha.2 h1:SlnEAXOPn5gC6l8tgvbn9fApyJaQ0ecQHSV3cEfvQYY=
-github.com/sealyun/lvscare v1.1.2-alpha.2/go.mod h1:FtOEdsXuYtw9Jwd/Jct25K+PcpUFSDemvF4VgNygjj0=
 github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
 github.com/sebdah/goldie/v2 v2.5.3 h1:9ES/mNN+HNUbNWpVAlrzuZ7jE+Nrczbj8uFRjM7624Y=
 github.com/sebdah/goldie/v2 v2.5.3/go.mod h1:oZ9fp0+se1eapSRjfYbsV/0Hqhbuu3bJVvKI/NNtssI=
@@ -1974,8 +1974,6 @@ github.com/vmware/govmomi v0.20.3/go.mod h1:URlwyTFZX72RmxtxuaFL2Uj3fD1JTvZdx59b
 github.com/weppos/publicsuffix-go v0.15.1-0.20210807195340-dc689ff0bb59/go.mod h1:HYux0V0Zi04bHNwOHy4cXJVz/TQjYonnF6aoYhj+3QE=
 github.com/weppos/publicsuffix-go v0.15.1-0.20220329081811-9a40b608a236/go.mod h1:HYux0V0Zi04bHNwOHy4cXJVz/TQjYonnF6aoYhj+3QE=
 github.com/willf/bitset v1.1.11/go.mod h1:83CECat5yLh5zVOf4P1ErAgKA5UDvKtgyUABdr3+MjI=
-github.com/wonderivan/logger v1.0.0 h1:Z6Nz+3SNcizolx3ARH11axdD4DXjFpb2J+ziGUVlv/U=
-github.com/wonderivan/logger v1.0.0/go.mod h1:NObMfQ3WOLKfYEZuGeZQfuQfSPE5+QNgRddVMzsAT/k=
 github.com/xanzy/go-gitlab v0.31.0/go.mod h1:sPLojNBn68fMUWSxIJtdVVIP8uSBYqesTfDUseX11Ug=
 github.com/xanzy/go-gitlab v0.32.0/go.mod h1:sPLojNBn68fMUWSxIJtdVVIP8uSBYqesTfDUseX11Ug=
 github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=

pkg/clustercert/kube_certs.go

@@ -20,6 +20,7 @@ import (
 	"fmt"
 	"net"
 	"os"
+	"strings"
 
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
@@ -202,15 +203,17 @@ func GenerateAllKubernetesCerts(certPath, etcdCertPath, nodeName, serviceCIRD, D
 
 	clusterCertArgs.APIServerAltNames.IPs[nodeIP.String()] = nodeIP
 
-	_, svcSubnet, err := net.ParseCIDR(serviceCIRD)
-	if err != nil {
-		return errors.Wrapf(err, "unable to parse ServiceSubnet %v", serviceCIRD)
-	}
-	svcFirstIP, err := utilnet.GetIndexedIP(svcSubnet, 1)
-	if err != nil {
-		return err
+	for _, svcCidr := range strings.Split(serviceCIRD, ",") {
+		_, svcSubnet, err := net.ParseCIDR(svcCidr)
+		if err != nil {
+			return errors.Wrapf(err, "unable to parse ServiceSubnet %v", svcCidr)
+		}
+		svcFirstIP, err := utilnet.GetIndexedIP(svcSubnet, 1)
+		if err != nil {
+			return err
+		}
+		clusterCertArgs.APIServerAltNames.IPs[svcFirstIP.String()] = svcFirstIP
 	}
-	clusterCertArgs.APIServerAltNames.IPs[svcFirstIP.String()] = svcFirstIP
 
 	for _, altName := range altNames {
 		ip := net.ParseIP(altName)
@@ -229,7 +232,7 @@ func GenerateAllKubernetesCerts(certPath, etcdCertPath, nodeName, serviceCIRD, D
 		serviceAccount: cert.NewKeyPairFileGenerator(certPath, "sa"),
 	}
 
-	err = certService.GenerateKubeComponentCert()
+	err := certService.GenerateKubeComponentCert()
 	if err != nil {
 		return err
 	}
@@ -377,7 +380,7 @@ func getEtcdCertificateConfig(etcdCertPath, certPath, nodeName string, nodeIP ne
 		},
 		IPs: map[string]net.IP{
 			net.IPv4(127, 0, 0, 1).String(): net.IPv4(127, 0, 0, 1),
-			nodeIP.To4().String():           nodeIP.To4(),
+			nodeIP.String():                 nodeIP,
 			net.IPv6loopback.String():       net.IPv6loopback,
 		},
 	}

pkg/infradriver/ssh_infradriver.go

@@ -20,14 +20,15 @@ import (
 	"net"
 	"strings"
 
+	"github.com/containers/buildah/util"
 	"github.com/imdario/mergo"
 	"github.com/sealerio/sealer/common"
 	v1 "github.com/sealerio/sealer/types/api/v1"
 	v2 "github.com/sealerio/sealer/types/api/v2"
 	"github.com/sealerio/sealer/utils/shellcommand"
 	"github.com/sealerio/sealer/utils/ssh"
-
 	"golang.org/x/sync/errgroup"
+	k8snet "k8s.io/utils/net"
 )
 
 type SSHInfraDriver struct {
@@ -44,7 +45,7 @@ type SSHInfraDriver struct {
 
 func mergeList(hostEnv, globalEnv map[string]interface{}) map[string]interface{} {
 	if len(hostEnv) == 0 {
-		return globalEnv
+		return copyEnv(globalEnv)
 	}
 	for globalEnvKey, globalEnvValue := range globalEnv {
 		if _, ok := hostEnv[globalEnvKey]; ok {
@@ -55,6 +56,18 @@ func mergeList(hostEnv, globalEnv map[string]interface{}) map[string]interface{}
 	return hostEnv
 }
 
+func copyEnv(origin map[string]interface{}) map[string]interface{} {
+	if origin == nil {
+		return nil
+	}
+	ret := make(map[string]interface{}, len(origin))
+	for k, v := range origin {
+		ret[k] = v
+	}
+
+	return ret
+}
+
 // ConvertEnv Convert []string to map[string]interface{}
 func ConvertEnv(envList []string) (env map[string]interface{}) {
 	temp := make(map[string][]string)
@@ -91,7 +104,6 @@ func NewInfraDriver(cluster *v2.Cluster) (InfraDriver, error) {
 		roleHostsMap:      map[string][]net.IP{},
 		// todo need to separate env into app render data and sys render data
 		hostEnvMap:         map[string]map[string]interface{}{},
-		clusterEnv:         ConvertEnv(cluster.Spec.Env),
 		clusterHostAliases: cluster.Spec.HostAliases,
 	}
 
@@ -100,6 +112,21 @@ func NewInfraDriver(cluster *v2.Cluster) (InfraDriver, error) {
 		ret.hosts = append(ret.hosts, host.IPS...)
 	}
 
+	if len(ret.hosts) == 0 {
+		return nil, fmt.Errorf("no hosts specified")
+	}
+
+	if err := checkAllHostsSameFamily(ret.hosts); err != nil {
+		return nil, err
+	}
+
+	if k8snet.IsIPv6String(ret.hosts[0].String()) {
+		hostIPFamilyEnv := fmt.Sprintf("%s=%s", common.EnvHostIPFamily, k8snet.IPv6)
+		if !util.StringInSlice(hostIPFamilyEnv, cluster.Spec.Env) {
+			cluster.Spec.Env = append(cluster.Spec.Env, hostIPFamilyEnv)
+		}
+	}
+
 	// initialize sshConfigs field
 	for _, host := range cluster.Spec.Hosts {
 		if err = mergo.Merge(&host.SSH, &cluster.Spec.SSH); err != nil {
@@ -122,6 +149,8 @@ func NewInfraDriver(cluster *v2.Cluster) (InfraDriver, error) {
 		}
 	}
 
+	ret.clusterEnv = ConvertEnv(cluster.Spec.Env)
+
 	// initialize hostEnvMap field
 	// merge the host ENV and global env, the host env will overwrite cluster.Spec.Env
 	for _, host := range cluster.Spec.Hosts {
@@ -334,3 +363,21 @@ func (d *SSHInfraDriver) Execute(hosts []net.IP, f func(host net.IP) error) erro
 
 	return nil
 }
+
+func checkAllHostsSameFamily(nodeList []net.IP) error {
+	hasIPV4 := false
+	hasIPV6 := false
+	for _, ip := range nodeList {
+		if k8snet.IsIPv4(ip) {
+			hasIPV4 = true
+		} else if k8snet.IsIPv6(ip) {
+			hasIPV6 = true
+		}
+	}
+
+	if hasIPV4 && hasIPV6 {
+		return fmt.Errorf("all hosts must be in same ip family, but the node list given are mixed with ipv4 and ipv6: %v", nodeList)
+	}
+
+	return nil
+}

pkg/ipvs/ipvs.go

@@ -34,10 +34,10 @@ func LvsStaticPodYaml(vip net.IP, masters []net.IP, image string) (string, error
 		return "", fmt.Errorf("invalid args to create Lvs static pod")
 	}
 
-	args := []string{"care", "--vs", vip.String() + ":6443", "--health-path", "/healthz", "--health-schem", "https"}
+	args := []string{"care", "--vs", net.JoinHostPort(vip.String(), "6443"), "--health-path", "/healthz", "--health-schem", "https"}
 	for _, m := range masters {
 		args = append(args, "--rs")
-		args = append(args, m.String()+":6443")
+		args = append(args, net.JoinHostPort(m.String(), "6443"))
 	}
 	flag := true
 	pod := componentPod(v1.Container{

pkg/runtime/kubernetes/init.go

@@ -40,6 +40,7 @@ import (
 const (
 	RemoteCmdCopyStatic    = "mkdir -p %s && cp -f %s %s"
 	DefaultVIP             = "10.103.97.2"
+	DefaultVIPForIPv6      = "1248:4003:10bb:6a01:83b9:6360:c66d:0002"
 	DefaultAPIserverDomain = "apiserver.cluster.local"
 )
 
@@ -103,7 +104,7 @@ func (k *Runtime) createKubeConfig(master0 net.IP) error {
 		return err
 	}
 
-	controlPlaneEndpoint := fmt.Sprintf("https://%s:6443", k.getAPIServerDomain())
+	controlPlaneEndpoint := fmt.Sprintf("https://%s", net.JoinHostPort(k.getAPIServerDomain(), "6443"))
 
 	return clustercert.CreateJoinControlPlaneKubeConfigFiles(k.infra.GetClusterRootfsPath(), k.getPKIPath(),
 		"ca", hostName, controlPlaneEndpoint, "kubernetes")
@@ -131,7 +132,7 @@ func (k *Runtime) copyStaticFiles(nodes []net.IP) error {
 	return nil
 }
 
-//initMaster0 is using kubeadm init to start up the cluster master0.
+// initMaster0 is using kubeadm init to start up the cluster master0.
 func (k *Runtime) initMaster0(kubeadmConf kubeadm.KubeadmConfig, master0 net.IP) (v1beta2.BootstrapTokenDiscovery, string, error) {
 	if err := k.initKube([]net.IP{master0}); err != nil {
 		return v1beta2.BootstrapTokenDiscovery{}, "", err
@@ -174,7 +175,7 @@ func (k *Runtime) initMaster0(kubeadmConf kubeadm.KubeadmConfig, master0 net.IP)
 	return token, certKey, nil
 }
 
-//decode output to join token hash and key
+// decode output to join token hash and key
 func (k *Runtime) decodeMaster0Output(output []byte) (v1beta2.BootstrapTokenDiscovery, string) {
 	s0 := string(output)
 	logrus.Debugf("decodeOutput: %s", s0)
@@ -185,7 +186,7 @@ func (k *Runtime) decodeMaster0Output(output []byte) (v1beta2.BootstrapTokenDisc
 	return k.decodeJoinCmd(slice1[0])
 }
 
-//  192.168.0.200:6443 --token 9vr73a.a8uxyaju799qwdjv --discovery-token-ca-cert-hash sha256:7c2e69131a36ae2a042a339b33381c6d0d43887e2de83720eff5359e26aec866 --experimental-control-plane --certificate-key f8902e114ef118304e561c3ecd4d0b543adc226b7a07f675f56564185ffe0c07
+// 192.168.0.200:6443 --token 9vr73a.a8uxyaju799qwdjv --discovery-token-ca-cert-hash sha256:7c2e69131a36ae2a042a339b33381c6d0d43887e2de83720eff5359e26aec866 --experimental-control-plane --certificate-key f8902e114ef118304e561c3ecd4d0b543adc226b7a07f675f56564185ffe0c07
 func (k *Runtime) decodeJoinCmd(cmd string) (v1beta2.BootstrapTokenDiscovery, string) {
 	logrus.Debugf("[globals]decodeJoinCmd: %s", cmd)
 	stringSlice := strings.Split(cmd, " ")

pkg/runtime/kubernetes/join_nodes.go

@@ -21,16 +21,15 @@ import (
 	"path"
 	"strings"
 
-	"k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta2"
-
-	"github.com/sealerio/sealer/pkg/runtime/kubernetes/kubeadm"
-	"github.com/sealerio/sealer/utils/shellcommand"
-
 	"github.com/sirupsen/logrus"
 	"golang.org/x/sync/errgroup"
+	"k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta2"
 
 	"github.com/sealerio/sealer/pkg/ipvs"
+	"github.com/sealerio/sealer/pkg/runtime/kubernetes/kubeadm"
+	"github.com/sealerio/sealer/utils"
 	utilsnet "github.com/sealerio/sealer/utils/net"
+	"github.com/sealerio/sealer/utils/shellcommand"
 	"github.com/sealerio/sealer/utils/yaml"
 )
 
@@ -120,7 +119,9 @@ func (k *Runtime) checkMultiNetworkAddVIPRoute(node net.IP) error {
 	if result == utilsnet.RouteOK {
 		return nil
 	}
-	_, err = k.infra.Cmd(node, fmt.Sprintf(RemoteAddRoute, k.getAPIServerVIP(), node))
 
-	return err
+	cmd := fmt.Sprintf(RemoteAddRoute, k.getAPIServerVIP(), node)
+	output, err := k.infra.Cmd(node, cmd)
+
+	return utils.WrapExecResult(node, cmd, output, err)
 }

pkg/runtime/kubernetes/kubeadm/kubeadm_config.go

@@ -181,7 +181,7 @@ func NewKubeadmConfig(fromClusterFile KubeadmConfig, fromFile string,
 	//The configuration set here does not require merge
 
 	conf.InitConfiguration.LocalAPIEndpoint.AdvertiseAddress = masters[0].String()
-	conf.ControlPlaneEndpoint = fmt.Sprintf("%s:6443", apiServerDomain)
+	conf.ControlPlaneEndpoint = net.JoinHostPort(apiServerDomain, "6443")
 
 	if conf.APIServer.ExtraArgs == nil {
 		conf.APIServer.ExtraArgs = make(map[string]string)

pkg/runtime/kubernetes/runtime.go

@@ -34,6 +34,7 @@ import (
 	corev1 "k8s.io/api/core/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	k8snet "k8s.io/utils/net"
 	runtimeClient "sigs.k8s.io/controller-runtime/pkg/client"
 )
 
@@ -64,6 +65,10 @@ func NewKubeadmRuntime(clusterFileKubeConfig kubeadm.KubeadmConfig, infra infrad
 		},
 	}
 
+	if ipFamily := infra.GetClusterEnv()[common.EnvHostIPFamily]; ipFamily != nil && ipFamily.(string) == k8snet.IPv6 {
+		k.Config.VIP = DefaultVIPForIPv6
+	}
+
 	if logrus.GetLevel() == logrus.DebugLevel {
 		k.Config.Vlog = 6
 	}

pkg/runtime/kubernetes/utils.go

@@ -55,7 +55,7 @@ const (
 	RemoteAddEtcHosts       = "cat /etc/hosts |grep '%s' || echo '%s' >> /etc/hosts"
 	RemoteReplaceKubeConfig = `grep -qF "apiserver.cluster.local" %s  && sed -i 's/apiserver.cluster.local/%s/' %s && sed -i 's/apiserver.cluster.local/%s/' %s`
 	RemoveKubeConfig        = "rm -rf /usr/bin/kube* && rm -rf ~/.kube/"
-	RemoteCleanK8sOnHost    = `if which kubeadm > /dev/null 2>&1;then kubeadm reset -f %s;fi && \
+	RemoteCleanK8sOnHost    = `systemctl restart docker kubelet; if which kubeadm > /dev/null 2>&1;then kubeadm reset -f %s;fi && \
 rm -rf /etc/kubernetes/ && \
 rm -rf /etc/systemd/system/kubelet.service.d && rm -rf /etc/systemd/system/kubelet.service && \
 rm -rf /usr/bin/kubeadm && rm -rf /usr/bin/kubelet-pre-start.sh && \
@@ -132,7 +132,7 @@ func (k *Runtime) Command(version, master0IP string, name CommandType, token v1b
 	cmds := map[CommandType]string{
 		InitMaster: fmt.Sprintf("kubeadm init --config=%s/etc/kubeadm.yml --experimental-upload-certs", k.infra.GetClusterRootfsPath()),
 		JoinMaster: fmt.Sprintf("kubeadm join %s --token %s --discovery-token-ca-cert-hash %s --experimental-control-plane --certificate-key %s", net.JoinHostPort(master0IP, "6443"), token.Token, token.CACertHashes, certKey),
-		JoinNode:   fmt.Sprintf("kubeadm join %s:6443 --token %s --discovery-token-ca-cert-hash %s", net.JoinHostPort(k.getAPIServerVIP().String(), "6443"), token.Token, token.CACertHashes),
+		JoinNode:   fmt.Sprintf("kubeadm join %s --token %s --discovery-token-ca-cert-hash %s", net.JoinHostPort(k.getAPIServerVIP().String(), "6443"), token.Token, token.CACertHashes),
 	}
 
 	kv := versionUtils.Version(version)