Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Optimize k8s runtime #1600

Merged
merged 4 commits into from
Jul 27, 2022
Merged
Show file tree
Hide file tree
Changes from 3 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 4 additions & 2 deletions apply/processor/create.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,8 @@ package processor
import (
"fmt"

"github.com/sealerio/sealer/pkg/registry"

"github.com/sealerio/sealer/pkg/clusterfile"
"github.com/sealerio/sealer/pkg/config"
"github.com/sealerio/sealer/pkg/filesystem"
Expand Down Expand Up @@ -105,7 +107,7 @@ func (c *CreateProcessor) RunConfig(cluster *v2.Cluster) error {

func (c *CreateProcessor) MountRootfs(cluster *v2.Cluster) error {
hosts := cluster.GetAllIPList()
regConfig := kubernetes.GetRegistryConfig(platform.DefaultMountClusterImageDir(cluster.Name), cluster.GetMaster0IP())
regConfig := registry.GetConfig(platform.DefaultMountClusterImageDir(cluster.Name), cluster.GetMaster0IP())
if net.NotInIPList(regConfig.IP, hosts) {
hosts = append(hosts, regConfig.IP)
}
Expand All @@ -119,7 +121,7 @@ func (c *CreateProcessor) MountRootfs(cluster *v2.Cluster) error {
}

func (c *CreateProcessor) Init(cluster *v2.Cluster) error {
return c.Runtime.Init(cluster)
return c.Runtime.Init()
}

func (c *CreateProcessor) Join(cluster *v2.Cluster) error {
Expand Down
4 changes: 3 additions & 1 deletion apply/processor/delete.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,8 @@ package processor
import (
"fmt"

"github.com/sealerio/sealer/pkg/registry"

"github.com/sealerio/sealer/common"
"github.com/sealerio/sealer/pkg/clusterfile"
"github.com/sealerio/sealer/pkg/filesystem"
Expand Down Expand Up @@ -65,7 +67,7 @@ func (d *DeleteProcessor) GetPhasePluginFunc(phase plugin.Phase) func(cluster *v

func (d *DeleteProcessor) UnMountRootfs(cluster *v2.Cluster) error {
hosts := cluster.GetAllIPList()
config := kubernetes.GetRegistryConfig(common.DefaultTheClusterRootfsDir(cluster.Name), cluster.GetMaster0IP())
config := registry.GetConfig(common.DefaultTheClusterRootfsDir(cluster.Name), cluster.GetMaster0IP())
if utilsnet.NotInIPList(config.IP, hosts) {
hosts = append(hosts, config.IP)
}
Expand Down
10 changes: 6 additions & 4 deletions apply/processor/gen.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,8 @@ import (
"net"
"strconv"

"github.com/sealerio/sealer/pkg/registry"

"github.com/sealerio/sealer/common"
"github.com/sealerio/sealer/pkg/client/k8s"
"github.com/sealerio/sealer/pkg/clusterfile"
Expand Down Expand Up @@ -51,7 +53,7 @@ type ParserArg struct {
}

type GenerateProcessor struct {
Runtime *kubernetes.KubeadmRuntime
Runtime *kubernetes.Runtime
ImageManager image.Service
ImageMounter clusterimage.Interface
}
Expand Down Expand Up @@ -146,7 +148,7 @@ func (g *GenerateProcessor) MountRootfs(cluster *v2.Cluster) error {
return err
}
hosts := cluster.GetAllIPList()
regConfig := kubernetes.GetRegistryConfig(common.DefaultTheClusterRootfsDir(cluster.Name), cluster.GetMaster0IP())
regConfig := registry.GetConfig(common.DefaultTheClusterRootfsDir(cluster.Name), cluster.GetMaster0IP())
if utilsnet.NotInIPList(regConfig.IP, hosts) {
hosts = append(hosts, regConfig.IP)
}
Expand Down Expand Up @@ -174,7 +176,7 @@ func (g *GenerateProcessor) MountImage(cluster *v2.Cluster) error {
if err != nil {
return err
}
g.Runtime = runt.(*kubernetes.KubeadmRuntime)
g.Runtime = runt.(*kubernetes.Runtime)
return nil
}

Expand All @@ -187,7 +189,7 @@ func (g *GenerateProcessor) ApplyRegistry(cluster *v2.Cluster) error {
if err != nil {
return err
}
rt, ok := runt.(*kubernetes.KubeadmRuntime)
rt, ok := runt.(*kubernetes.Runtime)
if !ok {
return fmt.Errorf("invalid type")
}
Expand Down
6 changes: 4 additions & 2 deletions apply/processor/scale.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,8 @@ import (
"fmt"
"net"

"github.com/sealerio/sealer/pkg/runtime/kubernetes/kubeadm"

"github.com/sealerio/sealer/common"
"github.com/sealerio/sealer/pkg/clusterfile"
"github.com/sealerio/sealer/pkg/config"
Expand All @@ -33,7 +35,7 @@ type ScaleProcessor struct {
fileSystem cloudfilesystem.Interface
ClusterFile clusterfile.Interface
Runtime runtime.Interface
KubeadmConfig *kubernetes.KubeadmConfig
KubeadmConfig *kubeadm.KubeadmConfig
Config config.Interface
Plugins plugin.Plugins
MastersToJoin []net.IP
Expand Down Expand Up @@ -125,7 +127,7 @@ func (s *ScaleProcessor) Delete(cluster *v2.Cluster) error {
return s.Runtime.DeleteNodes(s.NodesToDelete)
}

func NewScaleProcessor(kubeadmConfig *kubernetes.KubeadmConfig, clusterFile clusterfile.Interface, masterToJoin, masterToDelete, nodeToJoin, nodeToDelete []net.IP) (Processor, error) {
func NewScaleProcessor(kubeadmConfig *kubeadm.KubeadmConfig, clusterFile clusterfile.Interface, masterToJoin, masterToDelete, nodeToJoin, nodeToDelete []net.IP) (Processor, error) {
fs, err := filesystem.NewFilesystem(common.DefaultTheClusterRootfsDir(clusterFile.GetCluster().Name))
if err != nil {
return nil, err
Expand Down
4 changes: 2 additions & 2 deletions apply/processor/upgrade.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,8 @@ import (
"github.com/sealerio/sealer/common"
"github.com/sealerio/sealer/pkg/filesystem"
"github.com/sealerio/sealer/pkg/filesystem/cloudfilesystem"
"github.com/sealerio/sealer/pkg/registry"
"github.com/sealerio/sealer/pkg/runtime"
"github.com/sealerio/sealer/pkg/runtime/kubernetes"
v2 "github.com/sealerio/sealer/types/api/v2"
"github.com/sealerio/sealer/utils/net"
)
Expand All @@ -46,7 +46,7 @@ func (u UpgradeProcessor) Execute(cluster *v2.Cluster) error {
func (u UpgradeProcessor) MountRootfs(cluster *v2.Cluster) error {
//some hosts already mounted when scaled cluster.
hosts := cluster.GetAllIPList()
regConfig := kubernetes.GetRegistryConfig(common.DefaultTheClusterRootfsDir(cluster.Name), cluster.GetMaster0IP())
regConfig := registry.GetConfig(common.DefaultTheClusterRootfsDir(cluster.Name), cluster.GetMaster0IP())
if net.NotInIPList(regConfig.IP, hosts) {
hosts = append(hosts, regConfig.IP)
}
Expand Down
9 changes: 5 additions & 4 deletions pkg/clusterfile/clusterfile.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,8 @@ import (
"errors"
"sync"

"github.com/sealerio/sealer/pkg/runtime/kubernetes"
"github.com/sealerio/sealer/pkg/runtime/kubernetes/kubeadm"

v1 "github.com/sealerio/sealer/types/api/v1"
v2 "github.com/sealerio/sealer/types/api/v2"
)
Expand All @@ -29,7 +30,7 @@ type ClusterFile struct {
path string
Cluster v2.Cluster
Configs []v1.Config
KubeConfig *kubernetes.KubeadmConfig
KubeConfig *kubeadm.KubeadmConfig
Plugins []v1.Plugin
}

Expand All @@ -43,7 +44,7 @@ type Interface interface {
GetCluster() v2.Cluster
GetConfigs() []v1.Config
GetPlugins() []v1.Plugin
GetKubeadmConfig() *kubernetes.KubeadmConfig
GetKubeadmConfig() *kubeadm.KubeadmConfig
}

func (c *ClusterFile) GetCluster() v2.Cluster {
Expand All @@ -58,7 +59,7 @@ func (c *ClusterFile) GetPlugins() []v1.Plugin {
return c.Plugins
}

func (c *ClusterFile) GetKubeadmConfig() *kubernetes.KubeadmConfig {
func (c *ClusterFile) GetKubeadmConfig() *kubeadm.KubeadmConfig {
return c.KubeConfig
}

Expand Down
5 changes: 3 additions & 2 deletions pkg/clusterfile/pre_process.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,10 +20,11 @@ import (
"io"
"os"

"github.com/sealerio/sealer/pkg/runtime/kubernetes/kubeadm"

"github.com/sealerio/sealer/common"
"github.com/sealerio/sealer/pkg/config"
"github.com/sealerio/sealer/pkg/env"
"github.com/sealerio/sealer/pkg/runtime/kubernetes"
v1 "github.com/sealerio/sealer/types/api/v1"
v2 "github.com/sealerio/sealer/types/api/v2"
yaml2 "github.com/sealerio/sealer/utils"
Expand Down Expand Up @@ -162,7 +163,7 @@ func (c *ClusterFile) DecodePlugins(data []byte) error {
}

func (c *ClusterFile) DecodeKubeadmConfig(data []byte) error {
kubeadmConfig, err := kubernetes.LoadKubeadmConfigs(string(data), yaml2.DecodeCRDFromString)
kubeadmConfig, err := kubeadm.LoadKubeadmConfigs(string(data), yaml2.DecodeCRDFromString)
if err != nil {
return err
}
Expand Down
5 changes: 3 additions & 2 deletions pkg/filesystem/cloudfilesystem/nydus.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,9 +20,10 @@ import (
"net"
"path/filepath"

"github.com/sealerio/sealer/pkg/registry"

"github.com/sealerio/sealer/common"
"github.com/sealerio/sealer/pkg/env"
"github.com/sealerio/sealer/pkg/runtime/kubernetes"
v2 "github.com/sealerio/sealer/types/api/v2"
"github.com/sealerio/sealer/utils/exec"
utilsnet "github.com/sealerio/sealer/utils/net"
Expand Down Expand Up @@ -93,7 +94,7 @@ func mountNydusRootfs(ipList []net.IP, target string, cluster *v2.Cluster, initF
nydusdCleanCmd = fmt.Sprintf(RemoteNydusdStop, filepath.Join(nydusdDir, "clean.sh"), nydusdDir)
cleanCmd = fmt.Sprintf("echo '%s' >> "+common.DefaultClusterClearBashFile, nydusdCleanCmd, cluster.Name)
envProcessor = env.NewEnvProcessor(cluster)
config = kubernetes.GetRegistryConfig(platform.DefaultMountClusterImageDir(cluster.Name), cluster.GetMaster0IP())
config = registry.GetConfig(platform.DefaultMountClusterImageDir(cluster.Name), cluster.GetMaster0IP())
initCmd = fmt.Sprintf(RemoteChmod, target, config.Domain, config.Port)
)
_, err = exec.RunSimpleCmd(nydusdfileCpCmd)
Expand Down
5 changes: 3 additions & 2 deletions pkg/filesystem/cloudfilesystem/overlay2.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,9 +21,10 @@ import (
"strings"
"sync"

"github.com/sealerio/sealer/pkg/registry"

"github.com/sealerio/sealer/common"
"github.com/sealerio/sealer/pkg/env"
"github.com/sealerio/sealer/pkg/runtime/kubernetes"
v2 "github.com/sealerio/sealer/types/api/v2"
utilsnet "github.com/sealerio/sealer/utils/net"
"github.com/sealerio/sealer/utils/platform"
Expand Down Expand Up @@ -65,7 +66,7 @@ func mountRootfs(ipList []net.IP, target string, cluster *v2.Cluster, initFlag b
*sync.RWMutex
mountDirs map[string]bool
}{&sync.RWMutex{}, make(map[string]bool)}
config := kubernetes.GetRegistryConfig(platform.DefaultMountClusterImageDir(cluster.Name), cluster.GetMaster0IP())
config := registry.GetConfig(platform.DefaultMountClusterImageDir(cluster.Name), cluster.GetMaster0IP())
eg, _ := errgroup.WithContext(context.Background())
for _, IP := range ipList {
ip := IP
Expand Down
86 changes: 86 additions & 0 deletions pkg/registry/config.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,86 @@
// Copyright © 2021 Alibaba Group Holding Ltd.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package registry

import (
"fmt"
"net"
"path/filepath"

"github.com/sealerio/sealer/common"
osi "github.com/sealerio/sealer/utils/os"
"github.com/sealerio/sealer/utils/yaml"

"github.com/sirupsen/logrus"
"golang.org/x/crypto/bcrypt"
)

const (
ConfigFile = "registry.yml"
SeaHub = "sea.hub"
)

type Config struct {
IP net.IP `yaml:"ip,omitempty"`
Domain string `yaml:"domain,omitempty"`
Port string `yaml:"port,omitempty"`
Username string `json:"username,omitempty"`
Password string `json:"password,omitempty"`
}

func (c *Config) GenerateHtPasswd() (string, error) {
if c.Username == "" || c.Password == "" {
return "", fmt.Errorf("failed to generate htpasswd: registry username or passwodr is empty")
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

passwodr => password

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ht?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

i think this func means to gen HTTP basic authentication with username and password. so it could be rename to "GenerateHttpBasicAuth"

}
pwdHash, err := bcrypt.GenerateFromPassword([]byte(c.Password), bcrypt.DefaultCost)
if err != nil {
return "", fmt.Errorf("failed to generate registry password: %v", err)
}
return c.Username + ":" + string(pwdHash), nil
}

func (c *Config) Repo() string {
return fmt.Sprintf("%s:%s", c.Domain, c.Port)
}

func GetConfig(rootfs string, defaultRegistryIP net.IP) *Config {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

defaultRegistryIP => registryIP

var config Config
var DefaultConfig = &Config{
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

DefaultConfig => defaultConfig

IP: defaultRegistryIP,
Domain: SeaHub,
Port: "5000",
}
registryConfigPath := filepath.Join(rootfs, common.EtcDir, ConfigFile)
if !osi.IsFileExist(registryConfigPath) {
logrus.Debug("use default registry config")
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

default registry configuration is used: \n %+v, DefaultConfig

return DefaultConfig
}
err := yaml.UnmarshalFile(registryConfigPath, &config)
if err != nil {
logrus.Errorf("failed to read registry config: %v", err)
return DefaultConfig
}
if config.IP == nil {
config.IP = DefaultConfig.IP
}
if config.Port == "" {
config.Port = DefaultConfig.Port
}
if config.Domain == "" {
config.Domain = DefaultConfig.Domain
}
logrus.Debugf("show registry info, IP: %s, Domain: %s", config.IP, config.Domain)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The ultimate registry configration is:

return &config
}
6 changes: 2 additions & 4 deletions pkg/runtime/interface.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,13 +16,11 @@ package runtime

import (
"net"

v2 "github.com/sealerio/sealer/types/api/v2"
)

type Interface interface {
// Init exec init phase for cluster, v2.Cluster deliver cluster's information.
Init(cluster *v2.Cluster) error
// Init exec init phase for cluster. TODO: make the annotation more comprehensive
Init() error
// Upgrade exec upgrading phase for cluster.TODO: make the annotation more comprehensive
Upgrade() error
// Reset exec reset phase for cluster.TODO: make the annotation more comprehensive
Expand Down
32 changes: 26 additions & 6 deletions pkg/runtime/kubernetes/common.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,10 +15,30 @@
package kubernetes

const (
Cluster = "Cluster"
InitConfiguration = "InitConfiguration"
JoinConfiguration = "JoinConfiguration"
ClusterConfiguration = "ClusterConfiguration"
KubeProxyConfiguration = "KubeProxyConfiguration"
KubeletConfiguration = "KubeletConfiguration"
AuditPolicyYml = "audit-policy.yml"
)

var (
ContainerdShell = `if grep "SystemdCgroup = true" /etc/containerd/config.toml &> /dev/null; then
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is that possible we try not to write shell in go later?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

yes, We have too many operations done by the shell,it is hard to read and maintain. this pr only move out some code section nothing to do with the logic. I plan to optimize them in the next steps.

driver=systemd
else
driver=cgroupfs
fi
echo ${driver}`
DockerShell = `driver=$(docker info -f "{{.CgroupDriver}}")
echo "${driver}"`
)

// StaticFile :static file should not be template, will never be changed while initialization.
type StaticFile struct {
DestinationDir string
Name string
}

//MasterStaticFiles Put static files here, can be moved to all master nodes before kubeadm execution
var MasterStaticFiles = []*StaticFile{
{
DestinationDir: "/etc/kubernetes",
Name: AuditPolicyYml,
},
}
Loading