Feat/new sealer ipv6 dual (sealerio#1804)

* Support ipv6 dual stack.

Signed-off-by: huaiyou <huaiyou.cyz@alibaba-inc.com>

bugfix: image save use ociv1 format (sealerio#1813)

* bugfix: image save use ociv1 format

* fix lint

clean /var/lib/sealer/data/linux_amd64_ before if exists (sealerio#1815)

Signed-off-by: zhy76 <958474674@qq.com>

Signed-off-by: zhy76 <958474674@qq.com>

delete cluster add prune options (sealerio#1793)

Signed-off-by: HIHIA <283304489@qq.com>

Signed-off-by: HIHIA <283304489@qq.com>

support sealer delete -f clusterfile flag

modify

cmd/sealer/cmd/cluster/delete.go

@@ -39,9 +39,10 @@ import (
 )
 
 var (
-	deleteFlags *types.Flags
-	deleteAll   bool
-	ForceDelete bool
+	deleteFlags       *types.Flags
+	deleteClusterFile string
+	deleteAll         bool
+	ForceDelete       bool
 )
 
 var longDeleteCmdDescription = `delete command is used to delete part or all of existing cluster.
@@ -69,14 +70,12 @@ func NewDeleteCmd() *cobra.Command {
 				mastersToDelete = deleteFlags.Masters
 				workersToDelete = deleteFlags.Nodes
 			)
-
-			if mastersToDelete == "" && workersToDelete == "" && !deleteAll {
+			workClusterfile := common.GetDefaultClusterfile()
+			if mastersToDelete == "" && workersToDelete == "" && !deleteAll && deleteClusterFile == "" {
 				return fmt.Errorf("you must input node ip Or set flag -a")
 			}
-
-			workClusterfile := common.GetDefaultClusterfile()
-			if deleteAll {
-				return deleteCluster(workClusterfile)
+			if deleteClusterFile != "" {
+				return deleteCluster(deleteClusterFile)
 			}
 			return scaleDownCluster(mastersToDelete, workersToDelete, workClusterfile)
 		},
@@ -85,9 +84,11 @@ func NewDeleteCmd() *cobra.Command {
 	deleteFlags = &types.Flags{}
 	deleteCmd.Flags().StringVarP(&deleteFlags.Masters, "masters", "m", "", "reduce Count or IPList to masters")
 	deleteCmd.Flags().StringVarP(&deleteFlags.Nodes, "nodes", "n", "", "reduce Count or IPList to nodes")
+	deleteCmd.Flags().StringVarP(&deleteClusterFile, "Clusterfile", "f", "Clusterfile", "delete a kubernetes cluster with Clusterfile")
 	deleteCmd.Flags().StringSliceVarP(&deleteFlags.CustomEnv, "env", "e", []string{}, "set custom environment variables")
-	deleteCmd.Flags().BoolVarP(&ForceDelete, "force", "f", false, "We also can input an --force flag to delete cluster by force")
-	deleteCmd.Flags().BoolVarP(&deleteAll, "all", "a", false, "this flags is for delete the entire cluster, default is false")
+	deleteCmd.Flags().BoolVarP(&ForceDelete, "force", "", false, "We also can input an --force flag to delete cluster by force")
+	deleteCmd.Flags().BoolVar(&deleteAll, "a", false, "this flags is for delete the entire cluster, default is false")
+	deleteCmd.Flags().BoolVarP(&imagedistributor.IsPrune, "prune", "p", true, "this flags is for delete all cluster rootfs, default is true")
 
 	return deleteCmd
 }
@@ -182,10 +183,17 @@ func deleteCluster(workClusterfile string) error {
 	}
 
 	//delete local files,including sealer workdir,cluster file under sealer,kubeconfig under home dir.
-	if err = fs.FS.RemoveAll(common.GetSealerWorkDir(), common.DefaultClusterBaseDir(infraDriver.GetClusterName()),
-		common.DefaultKubeConfigDir()); err != nil {
+	if err = fs.FS.RemoveAll(common.GetSealerWorkDir(), common.DefaultKubeConfigDir()); err != nil {
 		return err
 	}
+
+	// delete cluster file under sealer if isPrune is true
+	if imagedistributor.IsPrune {
+		if err = fs.FS.RemoveAll(common.DefaultClusterBaseDir(infraDriver.GetClusterName())); err != nil {
+			return err
+		}
+	}
+
 	return nil
 }
 

cmd/sealer/cmd/image/save.go

@@ -52,7 +52,7 @@ func NewSaveCmd() *cobra.Command {
 	}
 	saveOpts = &options.SaveOptions{}
 	flags := saveCmd.Flags()
-	flags.StringVar(&saveOpts.Format, "format", buildah.V2s2Archive, "Save image to oci-archive, oci-dir (directory with oci manifest type), docker-archive, docker-dir (directory with v2s2 manifest type)")
+	flags.StringVar(&saveOpts.Format, "format", buildah.OCIArchive, "Save image to oci-archive, oci-dir (directory with oci manifest type), docker-archive, docker-dir (directory with v2s2 manifest type)")
 	flags.StringVarP(&saveOpts.Output, "output", "o", "", "Write image to a specified file")
 	flags.BoolVarP(&saveOpts.Quiet, "quiet", "q", false, "Suppress the output")
 	flags.BoolVar(&saveOpts.Compress, "compress", false, "Compress tarball image layers when saving to a directory using the 'dir' transport. (default is same compression type as source)")

cmd/seautil/cmd/ipvs.go

@@ -15,7 +15,7 @@
 package cmd
 
 import (
-	"github.com/sealyun/lvscare/care"
+	"github.com/labring/lvscare/care"
 	"github.com/spf13/cobra"
 )
 

common/common.go

@@ -71,6 +71,8 @@ const (
 	DefaultRegistryDomain         = "sea.hub"
 	DefaultRegistryPort           = "5000"
 	HostIP                        = "HostIP"
+	EnvHostIPFamily               = "HostIPFamily"
+	EnvIPv6DualStack              = "IPv6DualStack"
 )
 
 // image module

go.mod

@@ -19,6 +19,7 @@ require (
 	github.com/google/uuid v1.3.0
 	github.com/hashicorp/go-multierror v1.1.1
 	github.com/imdario/mergo v0.3.13
+	github.com/labring/lvscare v1.1.1
 	github.com/lestrrat-go/file-rotatelogs v2.4.0+incompatible
 	github.com/mitchellh/go-homedir v1.1.0
 	github.com/moby/buildkit v0.9.3
@@ -33,7 +34,6 @@ require (
 	github.com/pkg/errors v0.9.1
 	github.com/pkg/sftp v1.13.0
 	github.com/rifflock/lfshook v0.0.0-20180920164130-b9218ef580f5
-	github.com/sealyun/lvscare v1.1.2-alpha.2
 	github.com/shirou/gopsutil v3.21.11+incompatible
 	github.com/sirupsen/logrus v1.9.0
 	github.com/spf13/cobra v1.5.0
@@ -179,7 +179,6 @@ require (
 	github.com/vbatts/tar-split v0.11.2 // indirect
 	github.com/vbauerster/mpb/v7 v7.4.2 // indirect
 	github.com/vishvananda/netns v0.0.0-20211101163701-50045581ed74 // indirect
-	github.com/wonderivan/logger v1.0.0 // indirect
 	github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect
 	github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
 	github.com/xeipuuv/gojsonschema v1.2.0 // indirect
@@ -214,6 +213,7 @@ require (
 
 replace (
 	github.com/containerd/containerd => github.com/containerd/containerd v1.6.6
+	github.com/labring/lvscare => github.com/fanux/lvscare v1.1.3-beta.2.0.20220525083355-d8f3f86677d1
 	github.com/opencontainers/runc => github.com/opencontainers/runc v1.1.2
 	golang.org/x/crypto => golang.org/x/crypto v0.0.0-20210817164053-32db794688a5
 	golang.org/x/net => golang.org/x/net v0.0.0-20210510120150-4163338589ed

go.sum

@@ -610,6 +610,8 @@ github.com/facebookgo/muster v0.0.0-20150708232844-fd3d7953fd52 h1:a4DFiKFJiDRGF
 github.com/facebookgo/muster v0.0.0-20150708232844-fd3d7953fd52/go.mod h1:yIquW87NGRw1FU5p5lEkpnt/QxoH5uPAOUlOVkAUuMg=
 github.com/facebookgo/stack v0.0.0-20160209184415-751773369052/go.mod h1:UbMTZqLaRiH3MsBH8va0n7s1pQYcu3uTb8G4tygF4Zg=
 github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4/go.mod h1:5tD+neXqOorC30/tWg0LCSkrqj/AR6gu8yY8/fpw1q0=
+github.com/fanux/lvscare v1.1.3-beta.2.0.20220525083355-d8f3f86677d1 h1:BNaIeOFlxyn1dOZ0wgtNxVMUqW8tbjK1+aLAAATEL1U=
+github.com/fanux/lvscare v1.1.3-beta.2.0.20220525083355-d8f3f86677d1/go.mod h1:QvpJhFlNxIVhMHRN6I3q4+85r2kKsYJWMkSKDZK+4V8=
 github.com/fatih/camelcase v1.0.0/go.mod h1:yN2Sb0lFhZJUdVvtELVWefmrXpuZESvPmqwoZc+/fpc=
 github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
 github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU=
@@ -1744,8 +1746,6 @@ github.com/sassoftware/go-rpmutils v0.0.0-20190420191620-a8f1baeba37b/go.mod h1:
 github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
 github.com/sclevine/agouti v3.0.0+incompatible/go.mod h1:b4WX9W9L1sfQKXeJf1mUTLZKJ48R1S7H23Ji7oFO5Bw=
 github.com/sclevine/spec v1.2.0/go.mod h1:W4J29eT/Kzv7/b9IWLB055Z+qvVC9vt0Arko24q7p+U=
-github.com/sealyun/lvscare v1.1.2-alpha.2 h1:SlnEAXOPn5gC6l8tgvbn9fApyJaQ0ecQHSV3cEfvQYY=
-github.com/sealyun/lvscare v1.1.2-alpha.2/go.mod h1:FtOEdsXuYtw9Jwd/Jct25K+PcpUFSDemvF4VgNygjj0=
 github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
 github.com/sebdah/goldie/v2 v2.5.3 h1:9ES/mNN+HNUbNWpVAlrzuZ7jE+Nrczbj8uFRjM7624Y=
 github.com/sebdah/goldie/v2 v2.5.3/go.mod h1:oZ9fp0+se1eapSRjfYbsV/0Hqhbuu3bJVvKI/NNtssI=
@@ -1974,8 +1974,6 @@ github.com/vmware/govmomi v0.20.3/go.mod h1:URlwyTFZX72RmxtxuaFL2Uj3fD1JTvZdx59b
 github.com/weppos/publicsuffix-go v0.15.1-0.20210807195340-dc689ff0bb59/go.mod h1:HYux0V0Zi04bHNwOHy4cXJVz/TQjYonnF6aoYhj+3QE=
 github.com/weppos/publicsuffix-go v0.15.1-0.20220329081811-9a40b608a236/go.mod h1:HYux0V0Zi04bHNwOHy4cXJVz/TQjYonnF6aoYhj+3QE=
 github.com/willf/bitset v1.1.11/go.mod h1:83CECat5yLh5zVOf4P1ErAgKA5UDvKtgyUABdr3+MjI=
-github.com/wonderivan/logger v1.0.0 h1:Z6Nz+3SNcizolx3ARH11axdD4DXjFpb2J+ziGUVlv/U=
-github.com/wonderivan/logger v1.0.0/go.mod h1:NObMfQ3WOLKfYEZuGeZQfuQfSPE5+QNgRddVMzsAT/k=
 github.com/xanzy/go-gitlab v0.31.0/go.mod h1:sPLojNBn68fMUWSxIJtdVVIP8uSBYqesTfDUseX11Ug=
 github.com/xanzy/go-gitlab v0.32.0/go.mod h1:sPLojNBn68fMUWSxIJtdVVIP8uSBYqesTfDUseX11Ug=
 github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=

pkg/clustercert/kube_certs.go

@@ -20,6 +20,7 @@ import (
 	"fmt"
 	"net"
 	"os"
+	"strings"
 
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
@@ -202,15 +203,17 @@ func GenerateAllKubernetesCerts(certPath, etcdCertPath, nodeName, serviceCIRD, D
 
 	clusterCertArgs.APIServerAltNames.IPs[nodeIP.String()] = nodeIP
 
-	_, svcSubnet, err := net.ParseCIDR(serviceCIRD)
-	if err != nil {
-		return errors.Wrapf(err, "unable to parse ServiceSubnet %v", serviceCIRD)
-	}
-	svcFirstIP, err := utilnet.GetIndexedIP(svcSubnet, 1)
-	if err != nil {
-		return err
+	for _, svcCidr := range strings.Split(serviceCIRD, ",") {
+		_, svcSubnet, err := net.ParseCIDR(svcCidr)
+		if err != nil {
+			return errors.Wrapf(err, "unable to parse ServiceSubnet %v", svcCidr)
+		}
+		svcFirstIP, err := utilnet.GetIndexedIP(svcSubnet, 1)
+		if err != nil {
+			return err
+		}
+		clusterCertArgs.APIServerAltNames.IPs[svcFirstIP.String()] = svcFirstIP
 	}
-	clusterCertArgs.APIServerAltNames.IPs[svcFirstIP.String()] = svcFirstIP
 
 	for _, altName := range altNames {
 		ip := net.ParseIP(altName)
@@ -229,7 +232,7 @@ func GenerateAllKubernetesCerts(certPath, etcdCertPath, nodeName, serviceCIRD, D
 		serviceAccount: cert.NewKeyPairFileGenerator(certPath, "sa"),
 	}
 
-	err = certService.GenerateKubeComponentCert()
+	err := certService.GenerateKubeComponentCert()
 	if err != nil {
 		return err
 	}
@@ -377,7 +380,7 @@ func getEtcdCertificateConfig(etcdCertPath, certPath, nodeName string, nodeIP ne
 		},
 		IPs: map[string]net.IP{
 			net.IPv4(127, 0, 0, 1).String(): net.IPv4(127, 0, 0, 1),
-			nodeIP.To4().String():           nodeIP.To4(),
+			nodeIP.String():                 nodeIP,
 			net.IPv6loopback.String():       net.IPv6loopback,
 		},
 	}

pkg/imagedistributor/mount.go

@@ -17,12 +17,14 @@ package imagedistributor
 import (
 	"fmt"
 	"net"
+	"os"
 	"path/filepath"
 
 	"github.com/sealerio/sealer/common"
 	imagecommon "github.com/sealerio/sealer/pkg/define/options"
 	"github.com/sealerio/sealer/pkg/imageengine"
 	v1 "github.com/sealerio/sealer/types/api/v1"
+	osi "github.com/sealerio/sealer/utils/os"
 	"github.com/sealerio/sealer/utils/os/fs"
 )
 
@@ -33,6 +35,12 @@ type buildAhMounter struct {
 func (b buildAhMounter) Mount(imageName string, platform v1.Platform) (string, error) {
 	path := platform.OS + "_" + platform.Architecture + "_" + platform.Variant
 	mountDir := filepath.Join(common.DefaultSealerDataDir, path)
+	if osi.IsFileExist(mountDir) {
+		err := os.RemoveAll(mountDir)
+		if err != nil {
+			return "", err
+		}
+	}
 	if err := b.imageEngine.Pull(&imagecommon.PullOptions{
 		Quiet:      false,
 		PullPolicy: "missing",

pkg/imagedistributor/scp_distributor.go

@@ -37,6 +37,8 @@ const (
 	RegistryDirName = "registry"
 )
 
+var IsPrune bool
+
 type scpDistributor struct {
 	configs        []v1.Config
 	infraDriver    infradriver.InfraDriver
@@ -146,6 +148,10 @@ func (s *scpDistributor) renderRootfs(mountDir string) error {
 }
 
 func (s *scpDistributor) Restore(targetDir string, hosts []net.IP) error {
+	if !IsPrune {
+		return nil
+	}
+
 	rmRootfsCMD := fmt.Sprintf("rm -rf %s", targetDir)
 
 	eg, _ := errgroup.WithContext(context.Background())

pkg/infradriver/ssh_infradriver.go

@@ -20,14 +20,15 @@ import (
 	"net"
 	"strings"
 
+	"github.com/containers/buildah/util"
 	"github.com/imdario/mergo"
 	"github.com/sealerio/sealer/common"
 	v1 "github.com/sealerio/sealer/types/api/v1"
 	v2 "github.com/sealerio/sealer/types/api/v2"
 	"github.com/sealerio/sealer/utils/shellcommand"
 	"github.com/sealerio/sealer/utils/ssh"
-
 	"golang.org/x/sync/errgroup"
+	k8snet "k8s.io/utils/net"
 )
 
 type SSHInfraDriver struct {
@@ -44,7 +45,7 @@ type SSHInfraDriver struct {
 
 func mergeList(hostEnv, globalEnv map[string]interface{}) map[string]interface{} {
 	if len(hostEnv) == 0 {
-		return globalEnv
+		return copyEnv(globalEnv)
 	}
 	for globalEnvKey, globalEnvValue := range globalEnv {
 		if _, ok := hostEnv[globalEnvKey]; ok {
@@ -55,6 +56,18 @@ func mergeList(hostEnv, globalEnv map[string]interface{}) map[string]interface{}
 	return hostEnv
 }
 
+func copyEnv(origin map[string]interface{}) map[string]interface{} {
+	if origin == nil {
+		return nil
+	}
+	ret := make(map[string]interface{}, len(origin))
+	for k, v := range origin {
+		ret[k] = v
+	}
+
+	return ret
+}
+
 // ConvertEnv Convert []string to map[string]interface{}
 func ConvertEnv(envList []string) (env map[string]interface{}) {
 	temp := make(map[string][]string)
@@ -91,7 +104,6 @@ func NewInfraDriver(cluster *v2.Cluster) (InfraDriver, error) {
 		roleHostsMap:      map[string][]net.IP{},
 		// todo need to separate env into app render data and sys render data
 		hostEnvMap:         map[string]map[string]interface{}{},
-		clusterEnv:         ConvertEnv(cluster.Spec.Env),
 		clusterHostAliases: cluster.Spec.HostAliases,
 	}
 
@@ -100,6 +112,21 @@ func NewInfraDriver(cluster *v2.Cluster) (InfraDriver, error) {
 		ret.hosts = append(ret.hosts, host.IPS...)
 	}
 
+	if len(ret.hosts) == 0 {
+		return nil, fmt.Errorf("no hosts specified")
+	}
+
+	if err := checkAllHostsSameFamily(ret.hosts); err != nil {
+		return nil, err
+	}
+
+	if k8snet.IsIPv6String(ret.hosts[0].String()) {
+		hostIPFamilyEnv := fmt.Sprintf("%s=%s", common.EnvHostIPFamily, k8snet.IPv6)
+		if !util.StringInSlice(hostIPFamilyEnv, cluster.Spec.Env) {
+			cluster.Spec.Env = append(cluster.Spec.Env, hostIPFamilyEnv)
+		}
+	}
+
 	// initialize sshConfigs field
 	for _, host := range cluster.Spec.Hosts {
 		if err = mergo.Merge(&host.SSH, &cluster.Spec.SSH); err != nil {
@@ -122,6 +149,8 @@ func NewInfraDriver(cluster *v2.Cluster) (InfraDriver, error) {
 		}
 	}
 
+	ret.clusterEnv = ConvertEnv(cluster.Spec.Env)
+
 	// initialize hostEnvMap field
 	// merge the host ENV and global env, the host env will overwrite cluster.Spec.Env
 	for _, host := range cluster.Spec.Hosts {
@@ -334,3 +363,17 @@ func (d *SSHInfraDriver) Execute(hosts []net.IP, f func(host net.IP) error) erro
 
 	return nil
 }
+
+func checkAllHostsSameFamily(nodeList []net.IP) error {
+	var netFamily bool
+	for i, ip := range nodeList {
+		if i == 0 {
+			netFamily = k8snet.IsIPv4(ip)
+		}
+
+		if netFamily != k8snet.IsIPv4(ip) {
+			return fmt.Errorf("all hosts must be in same ip family, but the node list given are mixed with ipv4 and ipv6: %v", nodeList)
+		}
+	}
+	return nil
+}