diff --git a/go.mod b/go.mod index 119695bddd7..bd005d9ce29 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,6 @@ go 1.16 require ( github.com/BurntSushi/toml v0.4.1 // indirect - github.com/Microsoft/go-winio v0.4.15 github.com/aliyun/alibaba-cloud-sdk-go v1.61.579 github.com/davecgh/go-spew v1.1.1 github.com/daviddengcn/go-colortext v1.0.0 @@ -14,7 +13,6 @@ require ( github.com/fsnotify/fsnotify v1.4.10-0.20200417215612-7f4cf4dd2b52 // indirect github.com/google/uuid v1.1.2 github.com/gorilla/mux v1.7.4 - github.com/lithammer/dedent v1.1.0 github.com/mattn/go-isatty v0.0.14 // indirect github.com/onsi/ginkgo/v2 v2.1.4 github.com/onsi/gomega v1.19.0 @@ -40,8 +38,6 @@ require ( k8s.io/controller-manager v0.22.3 k8s.io/klog/v2 v2.9.0 k8s.io/kube-controller-manager v0.22.3 - k8s.io/kubelet v0.22.3 - k8s.io/system-validators v1.6.0 k8s.io/utils v0.0.0-20210930125809-cb0fa318a74b sigs.k8s.io/apiserver-network-proxy v0.0.15 sigs.k8s.io/yaml v1.3.0 // indirect diff --git a/go.sum b/go.sum index c3a97accca2..6bc985375d9 100644 --- a/go.sum +++ b/go.sum @@ -65,8 +65,6 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03 github.com/BurntSushi/toml v0.4.1 h1:GaI7EiDXDRfa8VshkTj7Fym7ha+y8/XxIgD2okUIjLw= github.com/BurntSushi/toml v0.4.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/Microsoft/go-winio v0.4.15 h1:qkLXKzb1QoVatRyd/YlXZ/Kg0m5K3SPuoD82jjSOaBc= -github.com/Microsoft/go-winio v0.4.15/go.mod h1:tTuCMEN+UleMWgg9dVx4Hu52b1bJo+59jBh3ajtinzw= github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= github.com/NYTimes/gziphandler v1.1.1 h1:ZUDjpQae29j0ryrS0u/B8HZfJBtBQHjqw2rQ2cqUQ3I= github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c= @@ -441,8 +439,6 @@ github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/lithammer/dedent v1.1.0 h1:VNzHMVCBNG1j0fh3OrsFRkVUwStdDArbgBWoPAffktY= -github.com/lithammer/dedent v1.1.0/go.mod h1:jrXYCQtgg0nJiN+StA2KgR7w6CiQNv9Fd/Z9BP0jIOc= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= @@ -573,7 +569,6 @@ github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= -github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= @@ -870,7 +865,6 @@ golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191022100944-742c48ecaeb7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1226,10 +1220,6 @@ k8s.io/kube-openapi v0.0.0-20200410145947-61e04a5be9a6/go.mod h1:GRQhZsXIAJ1xR0C k8s.io/kube-openapi v0.0.0-20200805222855-6aeccd4b50c6/go.mod h1:UuqjUnNftUyPE5H64/qeyjQoUZhGpeFDVdxjTeEVN2o= k8s.io/kube-openapi v0.0.0-20210421082810-95288971da7e h1:KLHHjkdQFomZy8+06csTWZ0m1343QqxZhR2LJ1OxCYM= k8s.io/kube-openapi v0.0.0-20210421082810-95288971da7e/go.mod h1:vHXdDvt9+2spS2Rx9ql3I8tycm3H9FDfdUoIuKCefvw= -k8s.io/kubelet v0.22.3 h1:C21Kg66Zzvc21uJITEPg4stGMcSZsR1JB+7+6Uwm8zs= -k8s.io/kubelet v0.22.3/go.mod h1:9nUZNGUigU2uAIm7kgf8BsvYDI9KjIE5nt9+yI1+p7w= -k8s.io/system-validators v1.6.0 h1:21qaPNdZ+mQrm4qc5shU0T5Eh49t/miFqZsn4sW8Hr0= -k8s.io/system-validators v1.6.0/go.mod h1:bPldcLgkIUK22ALflnsXk8pvkTEndYdNuaHH6gRrl0Q= k8s.io/utils v0.0.0-20200324210504-a9aa75ae1b89/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew= k8s.io/utils v0.0.0-20200729134348-d5654de09c73/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20200912215256-4140de9c8800/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= diff --git a/pkg/util/kubernetes/kubeadm/app/apis/kubeadm/types.go b/pkg/util/kubernetes/kubeadm/app/apis/kubeadm/types.go deleted file mode 100644 index af4a448aa5b..00000000000 --- a/pkg/util/kubernetes/kubeadm/app/apis/kubeadm/types.go +++ /dev/null @@ -1,464 +0,0 @@ -/* -Copyright 2016 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package kubeadm - -import ( - "crypto/x509" - - v1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime/schema" - - bootstraptokenv1 "github.com/openyurtio/openyurt/pkg/util/kubernetes/kubeadm/app/apis/bootstraptoken/v1" - "github.com/openyurtio/openyurt/pkg/util/kubernetes/kubeadm/app/features" -) - -// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object - -// InitConfiguration contains a list of fields that are specifically "kubeadm init"-only runtime -// information. The cluster-wide config is stored in ClusterConfiguration. The InitConfiguration -// object IS NOT uploaded to the kubeadm-config ConfigMap in the cluster, only the -// ClusterConfiguration is. -type InitConfiguration struct { - metav1.TypeMeta - - // ClusterConfiguration holds the cluster-wide information, and embeds that struct (which can be (un)marshalled separately as well) - // When InitConfiguration is marshalled to bytes in the external version, this information IS NOT preserved (which can be seen from - // the `json:"-"` tag in the external variant of these API types. - ClusterConfiguration `json:"-"` - - // BootstrapTokens is respected at `kubeadm init` time and describes a set of Bootstrap Tokens to create. - BootstrapTokens []bootstraptokenv1.BootstrapToken - - // NodeRegistration holds fields that relate to registering the new control-plane node to the cluster - NodeRegistration NodeRegistrationOptions - - // LocalAPIEndpoint represents the endpoint of the API server instance that's deployed on this control plane node - // In HA setups, this differs from ClusterConfiguration.ControlPlaneEndpoint in the sense that ControlPlaneEndpoint - // is the global endpoint for the cluster, which then loadbalances the requests to each individual API server. This - // configuration object lets you customize what IP/DNS name and port the local API server advertises it's accessible - // on. By default, kubeadm tries to auto-detect the IP of the default interface and use that, but in case that process - // fails you may set the desired value here. - LocalAPIEndpoint APIEndpoint - - // CertificateKey sets the key with which certificates and keys are encrypted prior to being uploaded in - // a secret in the cluster during the uploadcerts init phase. - CertificateKey string - - // SkipPhases is a list of phases to skip during command execution. - // The list of phases can be obtained with the "kubeadm init --help" command. - // The flag "--skip-phases" takes precedence over this field. - SkipPhases []string - - // Patches contains options related to applying patches to components deployed by kubeadm during - // "kubeadm init". - Patches *Patches -} - -// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object - -// ClusterConfiguration contains cluster-wide configuration for a kubeadm cluster -type ClusterConfiguration struct { - metav1.TypeMeta - - // ComponentConfigs holds component configs known to kubeadm, should long-term only exist in the internal kubeadm API - // +k8s:conversion-gen=false - ComponentConfigs ComponentConfigMap - - // Etcd holds configuration for etcd. - Etcd Etcd - - // Networking holds configuration for the networking topology of the cluster. - Networking Networking - // KubernetesVersion is the target version of the control plane. - KubernetesVersion string - - // ControlPlaneEndpoint sets a stable IP address or DNS name for the control plane; it - // can be a valid IP address or a RFC-1123 DNS subdomain, both with optional TCP port. - // In case the ControlPlaneEndpoint is not specified, the AdvertiseAddress + BindPort - // are used; in case the ControlPlaneEndpoint is specified but without a TCP port, - // the BindPort is used. - // Possible usages are: - // e.g. In a cluster with more than one control plane instances, this field should be - // assigned the address of the external load balancer in front of the - // control plane instances. - // e.g. in environments with enforced node recycling, the ControlPlaneEndpoint - // could be used for assigning a stable DNS to the control plane. - ControlPlaneEndpoint string - - // APIServer contains extra settings for the API server control plane component - APIServer APIServer - - // ControllerManager contains extra settings for the controller manager control plane component - ControllerManager ControlPlaneComponent - - // Scheduler contains extra settings for the scheduler control plane component - Scheduler ControlPlaneComponent - - // DNS defines the options for the DNS add-on installed in the cluster. - DNS DNS - - // CertificatesDir specifies where to store or look for all required certificates. - CertificatesDir string - - // ImageRepository sets the container registry to pull images from. - // If empty, `k8s.gcr.io` will be used by default; in case of kubernetes version is a CI build (kubernetes version starts with `ci/` or `ci-cross/`) - // `gcr.io/k8s-staging-ci-images` will be used as a default for control plane components and for kube-proxy, while `k8s.gcr.io` - // will be used for all the other images. - ImageRepository string - - // CIImageRepository is the container registry for core images generated by CI. - // Useful for running kubeadm with images from CI builds. - // +k8s:conversion-gen=false - CIImageRepository string - - // FeatureGates enabled by the user. - FeatureGates map[string]bool - - // The cluster name - ClusterName string -} - -// ControlPlaneComponent holds settings common to control plane component of the cluster -type ControlPlaneComponent struct { - // ExtraArgs is an extra set of flags to pass to the control plane component. - // A key in this map is the flag name as it appears on the - // command line except without leading dash(es). - // TODO: This is temporary and ideally we would like to switch all components to - // use ComponentConfig + ConfigMaps. - ExtraArgs map[string]string - - // ExtraVolumes is an extra set of host volumes, mounted to the control plane component. - ExtraVolumes []HostPathMount -} - -// APIServer holds settings necessary for API server deployments in the cluster -type APIServer struct { - ControlPlaneComponent - - // CertSANs sets extra Subject Alternative Names for the API Server signing cert. - CertSANs []string - - // TimeoutForControlPlane controls the timeout that we use for API server to appear - TimeoutForControlPlane *metav1.Duration -} - -// DNSAddOnType defines string identifying DNS add-on types -// TODO: Remove with v1beta2 https://github.com/kubernetes/kubeadm/issues/2459 -type DNSAddOnType string - -const ( - // CoreDNS add-on type - // TODO: Remove with v1beta2 https://github.com/kubernetes/kubeadm/issues/2459 - CoreDNS DNSAddOnType = "CoreDNS" -) - -// DNS defines the DNS addon that should be used in the cluster -type DNS struct { - // Type defines the DNS add-on to be used - // TODO: Used only in validation over the internal type. Remove with v1beta2 https://github.com/kubernetes/kubeadm/issues/2459 - Type DNSAddOnType - - // ImageMeta allows to customize the image used for the DNS component - ImageMeta `json:",inline"` -} - -// ImageMeta allows to customize the image used for components that are not -// originated from the Kubernetes/Kubernetes release process -type ImageMeta struct { - // ImageRepository sets the container registry to pull images from. - // if not set, the ImageRepository defined in ClusterConfiguration will be used instead. - ImageRepository string - - // ImageTag allows to specify a tag for the image. - // In case this value is set, kubeadm does not change automatically the version of the above components during upgrades. - ImageTag string - - //TODO: evaluate if we need also a ImageName based on user feedbacks -} - -// APIEndpoint struct contains elements of API server instance deployed on a node. -type APIEndpoint struct { - // AdvertiseAddress sets the IP address for the API server to advertise. - AdvertiseAddress string - - // BindPort sets the secure port for the API Server to bind to. - // Defaults to 6443. - BindPort int32 -} - -// NodeRegistrationOptions holds fields that relate to registering a new control-plane or node to the cluster, either via "kubeadm init" or "kubeadm join" -type NodeRegistrationOptions struct { - - // Name is the `.Metadata.Name` field of the Node API object that will be created in this `kubeadm init` or `kubeadm join` operation. - // This field is also used in the CommonName field of the kubelet's client certificate to the API server. - // Defaults to the hostname of the node if not provided. - Name string - - // CRISocket is used to retrieve container runtime info. This information will be annotated to the Node API object, for later re-use - CRISocket string - - // Taints specifies the taints the Node API object should be registered with. If this field is unset, i.e. nil, in the `kubeadm init` process - // it will be defaulted to []v1.Taint{'node-role.kubernetes.io/master=""'}. If you don't want to taint your control-plane node, set this field to an - // empty slice, i.e. `taints: []` in the YAML file. This field is solely used for Node registration. - Taints []v1.Taint - - // KubeletExtraArgs passes through extra arguments to the kubelet. The arguments here are passed to the kubelet command line via the environment file - // kubeadm writes at runtime for the kubelet to source. This overrides the generic base-level configuration in the kubelet-config-1.X ConfigMap - // Flags have higher priority when parsing. These values are local and specific to the node kubeadm is executing on. - // A key in this map is the flag name as it appears on the - // command line except without leading dash(es). - KubeletExtraArgs map[string]string - - // IgnorePreflightErrors provides a slice of pre-flight errors to be ignored when the current node is registered. - IgnorePreflightErrors []string - - // ImagePullPolicy specifies the policy for image pulling during kubeadm "init" and "join" operations. - // The value of this field must be one of "Always", "IfNotPresent" or "Never". - // If this field is unset kubeadm will default it to "IfNotPresent", or pull the required images if not present on the host. - ImagePullPolicy v1.PullPolicy `json:"imagePullPolicy,omitempty"` -} - -// Networking contains elements describing cluster's networking configuration. -type Networking struct { - // ServiceSubnet is the subnet used by k8s services. Defaults to "10.96.0.0/12". - ServiceSubnet string - // PodSubnet is the subnet used by pods. - PodSubnet string - // DNSDomain is the dns domain used by k8s services. Defaults to "cluster.local". - DNSDomain string -} - -// Etcd contains elements describing Etcd configuration. -type Etcd struct { - - // Local provides configuration knobs for configuring the local etcd instance - // Local and External are mutually exclusive - Local *LocalEtcd - - // External describes how to connect to an external etcd cluster - // Local and External are mutually exclusive - External *ExternalEtcd -} - -// LocalEtcd describes that kubeadm should run an etcd cluster locally -type LocalEtcd struct { - // ImageMeta allows to customize the container used for etcd - ImageMeta `json:",inline"` - - // DataDir is the directory etcd will place its data. - // Defaults to "/var/lib/etcd". - DataDir string - - // ExtraArgs are extra arguments provided to the etcd binary - // when run inside a static pod. - // A key in this map is the flag name as it appears on the - // command line except without leading dash(es). - ExtraArgs map[string]string - - // ServerCertSANs sets extra Subject Alternative Names for the etcd server signing cert. - ServerCertSANs []string - // PeerCertSANs sets extra Subject Alternative Names for the etcd peer signing cert. - PeerCertSANs []string -} - -// ExternalEtcd describes an external etcd cluster -type ExternalEtcd struct { - - // Endpoints of etcd members. Useful for using external etcd. - // If not provided, kubeadm will run etcd in a static pod. - Endpoints []string - // CAFile is an SSL Certificate Authority file used to secure etcd communication. - CAFile string - // CertFile is an SSL certification file used to secure etcd communication. - CertFile string - // KeyFile is an SSL key file used to secure etcd communication. - KeyFile string -} - -// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object - -// JoinConfiguration contains elements describing a particular node. -type JoinConfiguration struct { - metav1.TypeMeta - - // NodeRegistration holds fields that relate to registering the new control-plane node to the cluster - NodeRegistration NodeRegistrationOptions - - // CACertPath is the path to the SSL certificate authority used to - // secure comunications between node and control-plane. - // Defaults to "/etc/kubernetes/pki/ca.crt". - CACertPath string - - // Discovery specifies the options for the kubelet to use during the TLS Bootstrap process - Discovery Discovery - - // ControlPlane defines the additional control plane instance to be deployed on the joining node. - // If nil, no additional control plane instance will be deployed. - ControlPlane *JoinControlPlane - - // SkipPhases is a list of phases to skip during command execution. - // The list of phases can be obtained with the "kubeadm join --help" command. - // The flag "--skip-phases" takes precedence over this field. - SkipPhases []string - - // Patches contains options related to applying patches to components deployed by kubeadm during - // "kubeadm join". - Patches *Patches -} - -// JoinControlPlane contains elements describing an additional control plane instance to be deployed on the joining node. -type JoinControlPlane struct { - // LocalAPIEndpoint represents the endpoint of the API server instance to be deployed on this node. - LocalAPIEndpoint APIEndpoint - - // CertificateKey is the key that is used for decryption of certificates after they are downloaded from the secret - // upon joining a new control plane node. The corresponding encryption key is in the InitConfiguration. - CertificateKey string -} - -// Discovery specifies the options for the kubelet to use during the TLS Bootstrap process -type Discovery struct { - // BootstrapToken is used to set the options for bootstrap token based discovery - // BootstrapToken and File are mutually exclusive - BootstrapToken *BootstrapTokenDiscovery - - // File is used to specify a file or URL to a kubeconfig file from which to load cluster information - // BootstrapToken and File are mutually exclusive - File *FileDiscovery - - // TLSBootstrapToken is a token used for TLS bootstrapping. - // If .BootstrapToken is set, this field is defaulted to .BootstrapToken.Token, but can be overridden. - // If .File is set, this field **must be set** in case the KubeConfigFile does not contain any other authentication information - TLSBootstrapToken string - - // Timeout modifies the discovery timeout - Timeout *metav1.Duration -} - -// BootstrapTokenDiscovery is used to set the options for bootstrap token based discovery -type BootstrapTokenDiscovery struct { - // Token is a token used to validate cluster information - // fetched from the control-plane. - Token string - - // APIServerEndpoint is an IP or domain name to the API server from which info will be fetched. - APIServerEndpoint string - - // CACertHashes specifies a set of public key pins to verify - // when token-based discovery is used. The root CA found during discovery - // must match one of these values. Specifying an empty set disables root CA - // pinning, which can be unsafe. Each hash is specified as ":", - // where the only currently supported type is "sha256". This is a hex-encoded - // SHA-256 hash of the Subject Public Key Info (SPKI) object in DER-encoded - // ASN.1. These hashes can be calculated using, for example, OpenSSL. - CACertHashes []string - - // UnsafeSkipCAVerification allows token-based discovery - // without CA verification via CACertHashes. This can weaken - // the security of kubeadm since other nodes can impersonate the control-plane. - UnsafeSkipCAVerification bool -} - -// FileDiscovery is used to specify a file or URL to a kubeconfig file from which to load cluster information -type FileDiscovery struct { - // KubeConfigPath is used to specify the actual file path or URL to the kubeconfig file from which to load cluster information - KubeConfigPath string -} - -// GetControlPlaneImageRepository returns name of image repository -// for control plane images (API,Controller Manager,Scheduler and Proxy) -// It will override location with CI registry name in case user requests special -// Kubernetes version from CI build area. -// (See: kubeadmconstants.DefaultCIImageRepository) -func (cfg *ClusterConfiguration) GetControlPlaneImageRepository() string { - if cfg.CIImageRepository != "" { - return cfg.CIImageRepository - } - return cfg.ImageRepository -} - -// PublicKeyAlgorithm returns the type of encryption keys used in the cluster. -func (cfg *ClusterConfiguration) PublicKeyAlgorithm() x509.PublicKeyAlgorithm { - if features.Enabled(cfg.FeatureGates, features.PublicKeysECDSA) { - return x509.ECDSA - } - - return x509.RSA -} - -// HostPathMount contains elements describing volumes that are mounted from the -// host. -type HostPathMount struct { - // Name of the volume inside the pod template. - Name string - // HostPath is the path in the host that will be mounted inside - // the pod. - HostPath string - // MountPath is the path inside the pod where hostPath will be mounted. - MountPath string - // ReadOnly controls write access to the volume - ReadOnly bool - // PathType is the type of the HostPath. - PathType v1.HostPathType -} - -// Patches contains options related to applying patches to components deployed by kubeadm. -type Patches struct { - // Directory is a path to a directory that contains files named "target[suffix][+patchtype].extension". - // For example, "kube-apiserver0+merge.yaml" or just "etcd.json". "target" can be one of - // "kube-apiserver", "kube-controller-manager", "kube-scheduler", "etcd". "patchtype" can be one - // of "strategic" "merge" or "json" and they match the patch formats supported by kubectl. - // The default "patchtype" is "strategic". "extension" must be either "json" or "yaml". - // "suffix" is an optional string that can be used to determine which patches are applied - // first alpha-numerically. - Directory string -} - -// DocumentMap is a convenient way to describe a map between a YAML document and its GVK type -// +k8s:deepcopy-gen=false -type DocumentMap map[schema.GroupVersionKind][]byte - -// ComponentConfig holds a known component config -type ComponentConfig interface { - // DeepCopy should create a new deep copy of the component config in place - DeepCopy() ComponentConfig - - // Marshal is marshalling the config into a YAML document returned as a byte slice - Marshal() ([]byte, error) - - // Unmarshal loads the config from a document map. No config in the document map is no error. - Unmarshal(docmap DocumentMap) error - - // Default patches the component config with kubeadm preferred defaults - Default(cfg *ClusterConfiguration, localAPIEndpoint *APIEndpoint, nodeRegOpts *NodeRegistrationOptions) - - // IsUserSupplied indicates if the component config was supplied or modified by a user or was kubeadm generated - IsUserSupplied() bool - - // SetUserSupplied sets the state of the component config "user supplied" flag to, either true, or false. - SetUserSupplied(userSupplied bool) - - // Set can be used to set the internal configuration in the ComponentConfig - Set(interface{}) - - // Get can be used to get the internal configuration in the ComponentConfig - Get() interface{} -} - -// ComponentConfigMap is a map between a group name (as in GVK group) and a ComponentConfig -type ComponentConfigMap map[string]ComponentConfig diff --git a/pkg/util/kubernetes/kubeadm/app/cmd/options/constant.go b/pkg/util/kubernetes/kubeadm/app/cmd/options/constant.go deleted file mode 100644 index dba685a7c62..00000000000 --- a/pkg/util/kubernetes/kubeadm/app/cmd/options/constant.go +++ /dev/null @@ -1,152 +0,0 @@ -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package options - -const ( - // APIServerAdvertiseAddress flag sets the IP address the API Server will advertise it's listening on. Specify '0.0.0.0' to use the address of the default network interface. - APIServerAdvertiseAddress = "apiserver-advertise-address" - - // APIServerBindPort flag sets the port for the API Server to bind to. - APIServerBindPort = "apiserver-bind-port" - - // APIServerCertSANs flag sets extra Subject Alternative Names (SANs) to use for the API Server serving certificate. Can be both IP addresses and DNS names. - APIServerCertSANs = "apiserver-cert-extra-sans" - - // APIServerExtraArgs flag sets a extra flags to pass to the API Server or override default ones in form of =. - APIServerExtraArgs = "apiserver-extra-args" - - // CertificatesDir flag sets the path where to save and read the certificates. - CertificatesDir = "cert-dir" - - // CfgPath flag sets the path to kubeadm config file. - CfgPath = "config" - - // ControllerManagerExtraArgs flag sets extra flags to pass to the Controller Manager or override default ones in form of =. - ControllerManagerExtraArgs = "controller-manager-extra-args" - - // ControlPlaneEndpoint flag sets a stable IP address or DNS name for the control plane. - ControlPlaneEndpoint = "control-plane-endpoint" - - // DryRun flag instruct kubeadm to don't apply any changes; just output what would be done. - DryRun = "dry-run" - - // FeatureGatesString flag sets key=value pairs that describe feature gates for various features. - FeatureGatesString = "feature-gates" - - // IgnorePreflightErrors sets the path a list of checks whose errors will be shown as warnings. Example: 'IsPrivilegedUser,Swap'. Value 'all' ignores errors from all checks. - IgnorePreflightErrors = "ignore-preflight-errors" - - // ImageRepository sets the container registry to pull control plane images from. - ImageRepository = "image-repository" - - // KubeconfigDir flag sets the path where to save the kubeconfig file. - KubeconfigDir = "kubeconfig-dir" - - // KubeconfigPath flag sets the kubeconfig file to use when talking to the cluster. If the flag is not set, a set of standard locations are searched for an existing KubeConfig file. - KubeconfigPath = "kubeconfig" - - // KubernetesVersion flag sets the Kubernetes version for the control plane. - KubernetesVersion = "kubernetes-version" - - // KubeletVersion flag sets the version for the kubelet config. - KubeletVersion = "kubelet-version" - - // NetworkingDNSDomain flag sets the domain for services, e.g. "myorg.internal". - NetworkingDNSDomain = "service-dns-domain" - - // NetworkingServiceSubnet flag sets the range of IP address for service VIPs. - NetworkingServiceSubnet = "service-cidr" - - // NetworkingPodSubnet flag sets the range of IP addresses for the pod network. If set, the control plane will automatically allocate CIDRs for every node. - NetworkingPodSubnet = "pod-network-cidr" - - // NodeCRISocket flag sets the CRI socket to connect to. - NodeCRISocket = "cri-socket" - - // NodeName flag sets the node name. - NodeName = "node-name" - - // SchedulerExtraArgs flag sets extra flags to pass to the Scheduler or override default ones in form of =". - SchedulerExtraArgs = "scheduler-extra-args" - - // SkipTokenPrint flag instruct kubeadm to skip printing of the default bootstrap token generated by 'kubeadm init'. - SkipTokenPrint = "skip-token-print" - - // CSROnly flag instructs kubeadm to create CSRs instead of automatically creating or renewing certs - CSROnly = "csr-only" - - // CSRDir flag sets the location for CSRs and flags to be output - CSRDir = "csr-dir" - - // TokenStr flags sets both the discovery-token and the tls-bootstrap-token when those values are not provided - TokenStr = "token" - - // TokenTTL flag sets the time to live for token - TokenTTL = "token-ttl" - - // TokenUsages flag sets the usages of the token - TokenUsages = "usages" - - // TokenGroups flag sets the authentication groups of the token - TokenGroups = "groups" - - // TokenDescription flag sets the description of the token - TokenDescription = "description" - - // TLSBootstrapToken flag sets the token used to temporarily authenticate with the Kubernetes Control Plane to submit a certificate signing request (CSR) for a locally created key pair - TLSBootstrapToken = "tls-bootstrap-token" - - // TokenDiscovery flag sets the token used to validate cluster information fetched from the API server (for token-based discovery) - TokenDiscovery = "discovery-token" - - // TokenDiscoveryCAHash flag instruct kubeadm to validate that the root CA public key matches this hash (for token-based discovery) - TokenDiscoveryCAHash = "discovery-token-ca-cert-hash" - - // TokenDiscoverySkipCAHash flag instruct kubeadm to skip CA hash verification (for token-based discovery) - TokenDiscoverySkipCAHash = "discovery-token-unsafe-skip-ca-verification" - - // FileDiscovery flag sets the file or URL from which to load cluster information (for file-based discovery) - FileDiscovery = "discovery-file" - - // ControlPlane flag instruct kubeadm to create a new control plane instance on this node - ControlPlane = "control-plane" - - // UploadCerts flag instruct kubeadm to upload certificates - UploadCerts = "upload-certs" - - // CertificateKey flag sets the key used to encrypt and decrypt certificate secrets - CertificateKey = "certificate-key" - - // SkipCertificateKeyPrint flag instruct kubeadm to skip printing certificate key used to encrypt certs by 'kubeadm init'. - SkipCertificateKeyPrint = "skip-certificate-key-print" - - // ForceReset flag instruct kubeadm to reset the node without prompting for confirmation - ForceReset = "force" - - // CertificateRenewal flag instruct kubeadm to execute certificate renewal during upgrades - CertificateRenewal = "certificate-renewal" - - // EtcdUpgrade flag instruct kubeadm to execute etcd upgrade during upgrades - EtcdUpgrade = "etcd-upgrade" - - // Patches flag sets the folder where kubeadm component patches are stored - Patches = "patches" - - // ExperimentalPatches (DEPRECATED) is the same as Patches - // TODO: https://github.com/kubernetes/kubeadm/issues/2046 remove in 1.23 - ExperimentalPatches = "experimental-patches" -) diff --git a/pkg/util/kubernetes/kubeadm/app/cmd/phases/workflow/phase.go b/pkg/util/kubernetes/kubeadm/app/cmd/phases/workflow/phase.go deleted file mode 100644 index 554635ac56c..00000000000 --- a/pkg/util/kubernetes/kubeadm/app/cmd/phases/workflow/phase.go +++ /dev/null @@ -1,86 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package workflow - -import ( - "github.com/spf13/cobra" - "github.com/spf13/pflag" -) - -// Phase provides an implementation of a workflow phase that allows -// creation of new phases by simply instantiating a variable of this type. -type Phase struct { - // name of the phase. - // Phase name should be unique among peer phases (phases belonging to - // the same workflow or phases belonging to the same parent phase). - Name string - - // Aliases returns the aliases for the phase. - Aliases []string - - // Short description of the phase. - Short string - - // Long returns the long description of the phase. - Long string - - // Example returns the example for the phase. - Example string - - // Hidden define if the phase should be hidden in the workflow help. - // e.g. PrintFilesIfDryRunning phase in the kubeadm init workflow is candidate for being hidden to the users - Hidden bool - - // Phases defines a nested, ordered sequence of phases. - Phases []Phase - - // RunAllSiblings allows to assign to a phase the responsibility to - // run all the sibling phases - // Nb. phase marked as RunAllSiblings can not have Run functions - RunAllSiblings bool - - // Run defines a function implementing the phase action. - // It is recommended to implent type assertion, e.g. using golang type switch, - // for validating the RunData type. - Run func(data RunData) error - - // RunIf define a function that implements a condition that should be checked - // before executing the phase action. - // If this function return nil, the phase action is always executed. - RunIf func(data RunData) (bool, error) - - // InheritFlags defines the list of flags that the cobra command generated for this phase should Inherit - // from local flags defined in the parent command / or additional flags defined in the phase runner. - // If the values is not set or empty, no flags will be assigned to the command - // Nb. global flags are automatically inherited by nested cobra command - InheritFlags []string - - // LocalFlags defines the list of flags that should be assigned to the cobra command generated - // for this phase. - // Nb. if two or phases have the same local flags, please consider using local flags in the parent command - // or additional flags defined in the phase runner. - LocalFlags *pflag.FlagSet - - // ArgsValidator defines the positional arg function to be used for validating args for this phase - // If not set a phase will adopt the args of the top level command. - ArgsValidator cobra.PositionalArgs -} - -// AppendPhase adds the given phase to the nested, ordered sequence of phases. -func (t *Phase) AppendPhase(phase Phase) { - t.Phases = append(t.Phases, phase) -} diff --git a/pkg/util/kubernetes/kubeadm/app/cmd/phases/workflow/runner.go b/pkg/util/kubernetes/kubeadm/app/cmd/phases/workflow/runner.go deleted file mode 100644 index 32e44e5f1d6..00000000000 --- a/pkg/util/kubernetes/kubeadm/app/cmd/phases/workflow/runner.go +++ /dev/null @@ -1,484 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package workflow - -import ( - "fmt" - "strings" - - "github.com/pkg/errors" - "github.com/spf13/cobra" - "github.com/spf13/pflag" -) - -// phaseSeparator defines the separator to be used when concatenating nested -// phase names -const phaseSeparator = "/" - -// RunnerOptions defines the options supported during the execution of a -// kubeadm composable workflows -type RunnerOptions struct { - // FilterPhases defines the list of phases to be executed (if empty, all). - FilterPhases []string - - // SkipPhases defines the list of phases to be excluded by execution (if empty, none). - SkipPhases []string -} - -// RunData defines the data shared among all the phases included in the workflow, that is any type. -type RunData = interface{} - -// Runner implements management of composable kubeadm workflows. -type Runner struct { - // Options that regulate the runner behavior. - Options RunnerOptions - - // Phases composing the workflow to be managed by the runner. - Phases []Phase - - // runDataInitializer defines a function that creates the runtime data shared - // among all the phases included in the workflow - runDataInitializer func(*cobra.Command, []string) (RunData, error) - - // runData is part of the internal state of the runner and it is used for implementing - // a singleton in the InitData methods (thus avoiding to initialize data - // more than one time) - runData RunData - - // runCmd is part of the internal state of the runner and it is used to track the - // command that will trigger the runner (only if the runner is BindToCommand). - runCmd *cobra.Command - - // cmdAdditionalFlags holds additional, shared flags that could be added to the subcommands generated - // for phases. Flags could be inherited from the parent command too or added directly to each phase - cmdAdditionalFlags *pflag.FlagSet - - // phaseRunners is part of the internal state of the runner and provides - // a list of wrappers to phases composing the workflow with contextual - // information supporting phase execution. - phaseRunners []*phaseRunner -} - -// phaseRunner provides a wrapper to a Phase with the addition of a set -// of contextual information derived by the workflow managed by the Runner. -// TODO: If we ever decide to get more sophisticated we can swap this type with a well defined dag or tree library. -type phaseRunner struct { - // Phase provide access to the phase implementation - Phase - - // provide access to the parent phase in the workflow managed by the Runner. - parent *phaseRunner - - // level define the level of nesting of this phase into the workflow managed by - // the Runner. - level int - - // selfPath contains all the elements of the path that identify the phase into - // the workflow managed by the Runner. - selfPath []string - - // generatedName is the full name of the phase, that corresponds to the absolute - // path of the phase in the workflow managed by the Runner. - generatedName string - - // use is the phase usage string that will be printed in the workflow help. - // It corresponds to the relative path of the phase in the workflow managed by the Runner. - use string -} - -// NewRunner return a new runner for composable kubeadm workflows. -func NewRunner() *Runner { - return &Runner{ - Phases: []Phase{}, - } -} - -// AppendPhase adds the given phase to the ordered sequence of phases managed by the runner. -func (e *Runner) AppendPhase(t Phase) { - e.Phases = append(e.Phases, t) -} - -// computePhaseRunFlags return a map defining which phase should be run and which not. -// PhaseRunFlags are computed according to RunnerOptions. -func (e *Runner) computePhaseRunFlags() (map[string]bool, error) { - // Initialize support data structure - phaseRunFlags := map[string]bool{} - phaseHierarchy := map[string][]string{} - e.visitAll(func(p *phaseRunner) error { - // Initialize phaseRunFlags assuming that all the phases should be run. - phaseRunFlags[p.generatedName] = true - - // Initialize phaseHierarchy for the current phase (the list of phases - // depending on the current phase - phaseHierarchy[p.generatedName] = []string{} - - // Register current phase as part of its own parent hierarchy - parent := p.parent - for parent != nil { - phaseHierarchy[parent.generatedName] = append(phaseHierarchy[parent.generatedName], p.generatedName) - parent = parent.parent - } - return nil - }) - - // If a filter option is specified, set all phaseRunFlags to false except for - // the phases included in the filter and their hierarchy of nested phases. - if len(e.Options.FilterPhases) > 0 { - for i := range phaseRunFlags { - phaseRunFlags[i] = false - } - for _, f := range e.Options.FilterPhases { - if _, ok := phaseRunFlags[f]; !ok { - return phaseRunFlags, errors.Errorf("invalid phase name: %s", f) - } - phaseRunFlags[f] = true - for _, c := range phaseHierarchy[f] { - phaseRunFlags[c] = true - } - } - } - - // If a phase skip option is specified, set the corresponding phaseRunFlags - // to false and apply the same change to the underlying hierarchy - for _, f := range e.Options.SkipPhases { - if _, ok := phaseRunFlags[f]; !ok { - return phaseRunFlags, errors.Errorf("invalid phase name: %s", f) - } - phaseRunFlags[f] = false - for _, c := range phaseHierarchy[f] { - phaseRunFlags[c] = false - } - } - - return phaseRunFlags, nil -} - -// SetDataInitializer allows to setup a function that initialize the runtime data shared -// among all the phases included in the workflow. -// The method will receive in input the cmd that triggers the Runner (only if the runner is BindToCommand) -func (e *Runner) SetDataInitializer(builder func(cmd *cobra.Command, args []string) (RunData, error)) { - e.runDataInitializer = builder -} - -// InitData triggers the creation of runtime data shared among all the phases included in the workflow. -// This action can be executed explicitly out, when it is necessary to get the RunData -// before actually executing Run, or implicitly when invoking Run. -func (e *Runner) InitData(args []string) (RunData, error) { - if e.runData == nil && e.runDataInitializer != nil { - var err error - if e.runData, err = e.runDataInitializer(e.runCmd, args); err != nil { - return nil, err - } - } - - return e.runData, nil -} - -// Run the kubeadm composable kubeadm workflows. -func (e *Runner) Run(args []string) error { - e.prepareForExecution() - - // determine which phase should be run according to RunnerOptions - phaseRunFlags, err := e.computePhaseRunFlags() - if err != nil { - return err - } - - // builds the runner data - var data RunData - if data, err = e.InitData(args); err != nil { - return err - } - - err = e.visitAll(func(p *phaseRunner) error { - // if the phase should not be run, skip the phase. - if run, ok := phaseRunFlags[p.generatedName]; !run || !ok { - return nil - } - - // Errors if phases that are meant to create special subcommands only - // are wrongly assigned Run Methods - if p.RunAllSiblings && (p.RunIf != nil || p.Run != nil) { - return errors.Errorf("phase marked as RunAllSiblings can not have Run functions %s", p.generatedName) - } - - // If the phase defines a condition to be checked before executing the phase action. - if p.RunIf != nil { - // Check the condition and returns if the condition isn't satisfied (or fails) - ok, err := p.RunIf(data) - if err != nil { - return errors.Wrapf(err, "error execution run condition for phase %s", p.generatedName) - } - - if !ok { - return nil - } - } - - // Runs the phase action (if defined) - if p.Run != nil { - if err := p.Run(data); err != nil { - return errors.Wrapf(err, "error execution phase %s", p.generatedName) - } - } - - return nil - }) - - return err -} - -// Help returns text with the list of phases included in the workflow. -func (e *Runner) Help(cmdUse string) string { - e.prepareForExecution() - - // computes the max length of for each phase use line - maxLength := 0 - e.visitAll(func(p *phaseRunner) error { - if !p.Hidden && !p.RunAllSiblings { - length := len(p.use) - if maxLength < length { - maxLength = length - } - } - return nil - }) - - // prints the list of phases indented by level and formatted using the maxlength - // the list is enclosed in a mardown code block for ensuring better readability in the public web site - line := fmt.Sprintf("The %q command executes the following phases:\n", cmdUse) - line += "```\n" - offset := 2 - e.visitAll(func(p *phaseRunner) error { - if !p.Hidden && !p.RunAllSiblings { - padding := maxLength - len(p.use) + offset - line += strings.Repeat(" ", offset*p.level) // indentation - line += p.use // name + aliases - line += strings.Repeat(" ", padding) // padding right up to max length (+ offset for spacing) - line += p.Short // phase short description - line += "\n" - } - - return nil - }) - line += "```" - return line -} - -// SetAdditionalFlags allows to define flags to be added -// to the subcommands generated for each phase (but not existing in the parent command). -// Please note that this command needs to be done before BindToCommand. -// Nb. if a flag is used only by one phase, please consider using phase LocalFlags. -func (e *Runner) SetAdditionalFlags(fn func(*pflag.FlagSet)) { - // creates a new NewFlagSet - e.cmdAdditionalFlags = pflag.NewFlagSet("phaseAdditionalFlags", pflag.ContinueOnError) - // invokes the function that sets additional flags - fn(e.cmdAdditionalFlags) -} - -// BindToCommand bind the Runner to a cobra command by altering -// command help, adding phase related flags and by adding phases subcommands -// Please note that this command needs to be done once all the phases are added to the Runner. -func (e *Runner) BindToCommand(cmd *cobra.Command) { - // keep track of the command triggering the runner - e.runCmd = cmd - - // return early if no phases were added - if len(e.Phases) == 0 { - return - } - - e.prepareForExecution() - - // adds the phases subcommand - phaseCommand := &cobra.Command{ - Use: "phase", - Short: fmt.Sprintf("Use this command to invoke single phase of the %s workflow", cmd.Name()), - } - - cmd.AddCommand(phaseCommand) - - // generate all the nested subcommands for invoking single phases - subcommands := map[string]*cobra.Command{} - e.visitAll(func(p *phaseRunner) error { - // skip hidden phases - if p.Hidden { - return nil - } - - // initialize phase selector - phaseSelector := p.generatedName - - // if requested, set the phase to run all the sibling phases - if p.RunAllSiblings { - phaseSelector = p.parent.generatedName - } - - // creates phase subcommand - phaseCmd := &cobra.Command{ - Use: strings.ToLower(p.Name), - Short: p.Short, - Long: p.Long, - Example: p.Example, - Aliases: p.Aliases, - RunE: func(cmd *cobra.Command, args []string) error { - // if the phase has subphases, print the help and exits - if len(p.Phases) > 0 { - return cmd.Help() - } - - // overrides the command triggering the Runner using the phaseCmd - e.runCmd = cmd - e.Options.FilterPhases = []string{phaseSelector} - return e.Run(args) - }, - } - - // makes the new command inherits local flags from the parent command - // Nb. global flags will be inherited automatically - inheritsFlags(cmd.Flags(), phaseCmd.Flags(), p.InheritFlags) - - // makes the new command inherits additional flags for phases - if e.cmdAdditionalFlags != nil { - inheritsFlags(e.cmdAdditionalFlags, phaseCmd.Flags(), p.InheritFlags) - } - - // If defined, added phase local flags - if p.LocalFlags != nil { - p.LocalFlags.VisitAll(func(f *pflag.Flag) { - phaseCmd.Flags().AddFlag(f) - }) - } - - // if this phase has children (not a leaf) it doesn't accept any args - if len(p.Phases) > 0 { - phaseCmd.Args = cobra.NoArgs - } else { - if p.ArgsValidator == nil { - phaseCmd.Args = cmd.Args - } else { - phaseCmd.Args = p.ArgsValidator - } - } - - // adds the command to parent - if p.level == 0 { - phaseCommand.AddCommand(phaseCmd) - } else { - subcommands[p.parent.generatedName].AddCommand(phaseCmd) - } - - subcommands[p.generatedName] = phaseCmd - return nil - }) - - // alters the command description to show available phases - if cmd.Long != "" { - cmd.Long = fmt.Sprintf("%s\n\n%s\n", cmd.Long, e.Help(cmd.Use)) - } else { - cmd.Long = fmt.Sprintf("%s\n\n%s\n", cmd.Short, e.Help(cmd.Use)) - } - - // adds phase related flags to the main command - cmd.Flags().StringSliceVar(&e.Options.SkipPhases, "skip-phases", nil, "List of phases to be skipped") -} - -func inheritsFlags(sourceFlags, targetFlags *pflag.FlagSet, cmdFlags []string) { - // If the list of flag to be inherited from the parent command is not defined, no flag is added - if cmdFlags == nil { - return - } - - // add all the flags to be inherited to the target flagSet - sourceFlags.VisitAll(func(f *pflag.Flag) { - for _, c := range cmdFlags { - if f.Name == c { - targetFlags.AddFlag(f) - } - } - }) -} - -// visitAll provides a utility method for visiting all the phases in the workflow -// in the execution order and executing a func on each phase. -// Nested phase are visited immediately after their parent phase. -func (e *Runner) visitAll(fn func(*phaseRunner) error) error { - for _, currentRunner := range e.phaseRunners { - if err := fn(currentRunner); err != nil { - return err - } - } - return nil -} - -// prepareForExecution initialize the internal state of the Runner (the list of phaseRunner). -func (e *Runner) prepareForExecution() { - e.phaseRunners = []*phaseRunner{} - var parentRunner *phaseRunner - for _, phase := range e.Phases { - // skips phases that are meant to create special subcommands only - if phase.RunAllSiblings { - continue - } - - // add phases to the execution list - addPhaseRunner(e, parentRunner, phase) - } -} - -// addPhaseRunner adds the phaseRunner for a given phase to the phaseRunners list -func addPhaseRunner(e *Runner, parentRunner *phaseRunner, phase Phase) { - // computes contextual information derived by the workflow managed by the Runner. - use := cleanName(phase.Name) - generatedName := use - selfPath := []string{generatedName} - - if parentRunner != nil { - generatedName = strings.Join([]string{parentRunner.generatedName, generatedName}, phaseSeparator) - use = fmt.Sprintf("%s%s", phaseSeparator, use) - selfPath = append(parentRunner.selfPath, selfPath...) - } - - // creates the phaseRunner - currentRunner := &phaseRunner{ - Phase: phase, - parent: parentRunner, - level: len(selfPath) - 1, - selfPath: selfPath, - generatedName: generatedName, - use: use, - } - - // adds to the phaseRunners list - e.phaseRunners = append(e.phaseRunners, currentRunner) - - // iterate for the nested, ordered list of phases, thus storing - // phases in the expected executing order (child phase are stored immediately after their parent phase). - for _, childPhase := range phase.Phases { - addPhaseRunner(e, currentRunner, childPhase) - } -} - -// cleanName makes phase name suitable for the runner help, by lowercasing the name -// and removing args descriptors, if any -func cleanName(name string) string { - ret := strings.ToLower(name) - if pos := strings.Index(ret, " "); pos != -1 { - ret = ret[:pos] - } - return ret -} diff --git a/pkg/util/kubernetes/kubeadm/app/cmd/phases/workflow/runner_test.go b/pkg/util/kubernetes/kubeadm/app/cmd/phases/workflow/runner_test.go deleted file mode 100644 index ef546fdd367..00000000000 --- a/pkg/util/kubernetes/kubeadm/app/cmd/phases/workflow/runner_test.go +++ /dev/null @@ -1,625 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package workflow - -import ( - "fmt" - "reflect" - "strings" - "testing" - - "github.com/pkg/errors" - "github.com/spf13/cobra" - "github.com/spf13/pflag" -) - -func phaseBuilder(name string, phases ...Phase) Phase { - return Phase{ - Name: name, - Short: fmt.Sprintf("long description for %s ...", name), - Phases: phases, - } -} - -func TestComputePhaseRunFlags(t *testing.T) { - - var usecases = []struct { - name string - options RunnerOptions - expected map[string]bool - expectedError bool - }{ - { - name: "no options > all phases", - options: RunnerOptions{}, - expected: map[string]bool{"foo": true, "foo/bar": true, "foo/baz": true, "qux": true}, - }, - { - name: "options can filter phases", - options: RunnerOptions{FilterPhases: []string{"foo/baz", "qux"}}, - expected: map[string]bool{"foo": false, "foo/bar": false, "foo/baz": true, "qux": true}, - }, - { - name: "options can filter phases - hierarchy is considered", - options: RunnerOptions{FilterPhases: []string{"foo"}}, - expected: map[string]bool{"foo": true, "foo/bar": true, "foo/baz": true, "qux": false}, - }, - { - name: "options can skip phases", - options: RunnerOptions{SkipPhases: []string{"foo/bar", "qux"}}, - expected: map[string]bool{"foo": true, "foo/bar": false, "foo/baz": true, "qux": false}, - }, - { - name: "options can skip phases - hierarchy is considered", - options: RunnerOptions{SkipPhases: []string{"foo"}}, - expected: map[string]bool{"foo": false, "foo/bar": false, "foo/baz": false, "qux": true}, - }, - { - name: "skip options have higher precedence than filter options", - options: RunnerOptions{ - FilterPhases: []string{"foo"}, // "foo", "foo/bar", "foo/baz" true - SkipPhases: []string{"foo/bar"}, // "foo/bar" false - }, - expected: map[string]bool{"foo": true, "foo/bar": false, "foo/baz": true, "qux": false}, - }, - { - name: "invalid filter option", - options: RunnerOptions{FilterPhases: []string{"invalid"}}, - expectedError: true, - }, - { - name: "invalid skip option", - options: RunnerOptions{SkipPhases: []string{"invalid"}}, - expectedError: true, - }, - } - for _, u := range usecases { - t.Run(u.name, func(t *testing.T) { - var w = Runner{ - Phases: []Phase{ - phaseBuilder("foo", - phaseBuilder("bar"), - phaseBuilder("baz"), - ), - phaseBuilder("qux"), - }, - } - - w.prepareForExecution() - w.Options = u.options - actual, err := w.computePhaseRunFlags() - if (err != nil) != u.expectedError { - t.Errorf("Unexpected error: %v", err) - } - if err != nil { - return - } - if !reflect.DeepEqual(actual, u.expected) { - t.Errorf("\nactual:\n\t%v\nexpected:\n\t%v\n", actual, u.expected) - } - }) - } -} - -func phaseBuilder1(name string, runIf func(data RunData) (bool, error), phases ...Phase) Phase { - return Phase{ - Name: name, - Short: fmt.Sprintf("long description for %s ...", name), - Phases: phases, - Run: runBuilder(name), - RunIf: runIf, - } -} - -var callstack []string - -func runBuilder(name string) func(data RunData) error { - return func(data RunData) error { - callstack = append(callstack, name) - return nil - } -} - -func runConditionTrue(data RunData) (bool, error) { - return true, nil -} - -func runConditionFalse(data RunData) (bool, error) { - return false, nil -} - -func TestRunOrderAndConditions(t *testing.T) { - var w = Runner{ - Phases: []Phase{ - phaseBuilder1("foo", nil, - phaseBuilder1("bar", runConditionTrue), - phaseBuilder1("baz", runConditionFalse), - ), - phaseBuilder1("qux", runConditionTrue), - }, - } - - var usecases = []struct { - name string - options RunnerOptions - expectedOrder []string - }{ - { - name: "Run respect runCondition", - expectedOrder: []string{"foo", "bar", "qux"}, - }, - { - name: "Run takes options into account", - options: RunnerOptions{FilterPhases: []string{"foo"}, SkipPhases: []string{"foo/baz"}}, - expectedOrder: []string{"foo", "bar"}, - }, - } - for _, u := range usecases { - t.Run(u.name, func(t *testing.T) { - callstack = []string{} - w.Options = u.options - err := w.Run([]string{}) - if err != nil { - t.Errorf("Unexpected error: %v", err) - } - if !reflect.DeepEqual(callstack, u.expectedOrder) { - t.Errorf("\ncallstack:\n\t%v\nexpected:\n\t%v\n", callstack, u.expectedOrder) - } - }) - } -} - -func phaseBuilder2(name string, runIf func(data RunData) (bool, error), run func(data RunData) error, phases ...Phase) Phase { - return Phase{ - Name: name, - Short: fmt.Sprintf("long description for %s ...", name), - Phases: phases, - Run: run, - RunIf: runIf, - } -} - -func runPass(data RunData) error { - return nil -} - -func runFails(data RunData) error { - return errors.New("run fails") -} - -func runConditionPass(data RunData) (bool, error) { - return true, nil -} - -func runConditionFails(data RunData) (bool, error) { - return false, errors.New("run condition fails") -} - -func TestRunHandleErrors(t *testing.T) { - var w = Runner{ - Phases: []Phase{ - phaseBuilder2("foo", runConditionPass, runPass), - phaseBuilder2("bar", runConditionPass, runFails), - phaseBuilder2("baz", runConditionFails, runPass), - }, - } - - var usecases = []struct { - name string - options RunnerOptions - expectedError bool - }{ - { - name: "no errors", - options: RunnerOptions{FilterPhases: []string{"foo"}}, - }, - { - name: "run fails", - options: RunnerOptions{FilterPhases: []string{"bar"}}, - expectedError: true, - }, - { - name: "run condition fails", - options: RunnerOptions{FilterPhases: []string{"baz"}}, - expectedError: true, - }, - } - for _, u := range usecases { - t.Run(u.name, func(t *testing.T) { - w.Options = u.options - err := w.Run([]string{}) - if (err != nil) != u.expectedError { - t.Errorf("Unexpected error: %v", err) - } - }) - } -} - -func phaseBuilder3(name string, hidden bool, phases ...Phase) Phase { - return Phase{ - Name: name, - Short: fmt.Sprintf("long description for %s ...", name), - Phases: phases, - Hidden: hidden, - } -} - -func TestHelp(t *testing.T) { - var w = Runner{ - Phases: []Phase{ - phaseBuilder3("foo", false, - phaseBuilder3("bar [arg]", false), - phaseBuilder3("baz", true), - ), - phaseBuilder3("qux", false), - }, - } - - expected := "The \"myCommand\" command executes the following phases:\n" + - "```\n" + - "foo long description for foo ...\n" + - " /bar long description for bar [arg] ...\n" + - "qux long description for qux ...\n" + - "```" - - actual := w.Help("myCommand") - if !reflect.DeepEqual(actual, expected) { - t.Errorf("\nactual:\n\t%v\nexpected:\n\t%v\n", actual, expected) - } -} - -func phaseBuilder4(name string, cmdFlags []string, phases ...Phase) Phase { - return Phase{ - Name: name, - Phases: phases, - InheritFlags: cmdFlags, - } -} - -func phaseBuilder5(name string, flags *pflag.FlagSet) Phase { - return Phase{ - Name: name, - LocalFlags: flags, - } -} - -type argTest struct { - args cobra.PositionalArgs - pass []string - fail []string -} - -func phaseBuilder6(name string, args cobra.PositionalArgs, phases ...Phase) Phase { - return Phase{ - Name: name, - Short: fmt.Sprintf("long description for %s ...", name), - Phases: phases, - ArgsValidator: args, - } -} - -// customArgs is a custom cobra.PositionArgs function -func customArgs(cmd *cobra.Command, args []string) error { - for _, a := range args { - if a != "qux" { - return fmt.Errorf("arg %s does not equal qux", a) - } - } - return nil -} - -func TestBindToCommandArgRequirements(t *testing.T) { - - // because cobra.ExactArgs(1) == cobra.ExactArgs(3), it is needed - // to run test argument sets that both pass and fail to ensure the correct function was set. - var usecases = []struct { - name string - runner Runner - testCases map[string]argTest - cmd *cobra.Command - }{ - { - name: "leaf command, no defined args, follow parent", - runner: Runner{ - Phases: []Phase{phaseBuilder("foo")}, - }, - testCases: map[string]argTest{ - "phase foo": { - pass: []string{"one", "two", "three"}, - fail: []string{"one", "two"}, - args: cobra.ExactArgs(3), - }, - }, - cmd: &cobra.Command{ - Use: "init", - Args: cobra.ExactArgs(3), - }, - }, - { - name: "container cmd expect none, custom arg check for leaf", - runner: Runner{ - Phases: []Phase{phaseBuilder6("foo", cobra.NoArgs, - phaseBuilder6("bar", cobra.ExactArgs(1)), - phaseBuilder6("baz", customArgs), - )}, - }, - testCases: map[string]argTest{ - "phase foo": { - pass: []string{}, - fail: []string{"one"}, - args: cobra.NoArgs, - }, - "phase foo bar": { - pass: []string{"one"}, - fail: []string{"one", "two"}, - args: cobra.ExactArgs(1), - }, - "phase foo baz": { - pass: []string{"qux"}, - fail: []string{"one"}, - args: customArgs, - }, - }, - cmd: &cobra.Command{ - Use: "init", - Args: cobra.NoArgs, - }, - }, - } - - for _, rt := range usecases { - t.Run(rt.name, func(t *testing.T) { - - rt.runner.BindToCommand(rt.cmd) - - // Checks that cmd gets a new phase subcommand - phaseCmd := getCmd(rt.cmd, "phase") - if phaseCmd == nil { - t.Error("cmd didn't have phase subcommand\n") - return - } - - for c, args := range rt.testCases { - - cCmd := getCmd(rt.cmd, c) - if cCmd == nil { - t.Errorf("cmd didn't have %s subcommand\n", c) - continue - } - - // Ensure it is the expected function - if reflect.ValueOf(cCmd.Args).Pointer() != reflect.ValueOf(args.args).Pointer() { - t.Error("The function poiners where not equal.") - } - - // Test passing argument set - err := cCmd.Args(cCmd, args.pass) - - if err != nil { - t.Errorf("command %s should validate the args: %v\n %v", cCmd.Name(), args.pass, err) - } - - // Test failing argument set - err = cCmd.Args(cCmd, args.fail) - - if err == nil { - t.Errorf("command %s should fail to validate the args: %v\n %v", cCmd.Name(), args.pass, err) - } - } - - }) - } -} - -func TestBindToCommand(t *testing.T) { - - var dummy string - localFlags := pflag.NewFlagSet("dummy", pflag.ContinueOnError) - localFlags.StringVarP(&dummy, "flag4", "d", "d", "d") - - var usecases = []struct { - name string - runner Runner - expectedCmdAndFlags map[string][]string - setAdditionalFlags func(*pflag.FlagSet) - }{ - { - name: "when there are no phases, cmd should be left untouched", - runner: Runner{}, - }, - { - name: "phases should not inherits any parent flags by default", - runner: Runner{ - Phases: []Phase{phaseBuilder4("foo", nil)}, - }, - expectedCmdAndFlags: map[string][]string{ - "phase foo": {}, - }, - }, - { - name: "phases should be allowed to select parent flags to inherits", - runner: Runner{ - Phases: []Phase{phaseBuilder4("foo", []string{"flag1"})}, - }, - expectedCmdAndFlags: map[string][]string{ - "phase foo": {"flag1"}, //not "flag2" - }, - }, - { - name: "it should be possible to apply additional flags to all phases", - runner: Runner{ - Phases: []Phase{ - phaseBuilder4("foo", []string{"flag3"}), - phaseBuilder4("bar", []string{"flag1", "flag2", "flag3"}), - phaseBuilder4("baz", []string{"flag1"}), //test if additional flags are filtered too - }, - }, - setAdditionalFlags: func(flags *pflag.FlagSet) { - var dummy3 string - flags.StringVarP(&dummy3, "flag3", "c", "c", "c") - }, - expectedCmdAndFlags: map[string][]string{ - "phase foo": {"flag3"}, - "phase bar": {"flag1", "flag2", "flag3"}, - "phase baz": {"flag1"}, - }, - }, - { - name: "it should be possible to apply custom flags to single phases", - runner: Runner{ - Phases: []Phase{phaseBuilder5("foo", localFlags)}, - }, - expectedCmdAndFlags: map[string][]string{ - "phase foo": {"flag4"}, - }, - }, - { - name: "all the above applies to nested phases too", - runner: Runner{ - Phases: []Phase{ - phaseBuilder4("foo", []string{"flag3"}, - phaseBuilder4("bar", []string{"flag1", "flag2", "flag3"}), - phaseBuilder4("baz", []string{"flag1"}), //test if additional flags are filtered too - phaseBuilder5("qux", localFlags), - ), - }, - }, - setAdditionalFlags: func(flags *pflag.FlagSet) { - var dummy3 string - flags.StringVarP(&dummy3, "flag3", "c", "c", "c") - }, - expectedCmdAndFlags: map[string][]string{ - "phase foo": {"flag3"}, - "phase foo bar": {"flag1", "flag2", "flag3"}, - "phase foo baz": {"flag1"}, - "phase foo qux": {"flag4"}, - }, - }, - } - for _, rt := range usecases { - t.Run(rt.name, func(t *testing.T) { - - var dummy1, dummy2 string - cmd := &cobra.Command{ - Use: "init", - } - - cmd.Flags().StringVarP(&dummy1, "flag1", "a", "a", "a") - cmd.Flags().StringVarP(&dummy2, "flag2", "b", "b", "b") - - if rt.setAdditionalFlags != nil { - rt.runner.SetAdditionalFlags(rt.setAdditionalFlags) - } - - rt.runner.BindToCommand(cmd) - - // in case of no phases, checks that cmd is untouched - if len(rt.runner.Phases) == 0 { - if cmd.Long != "" { - t.Error("cmd.Long is set while it should be leaved untouched\n") - } - - if cmd.Flags().Lookup("skip-phases") != nil { - t.Error("cmd has skip-phases flag while it should not\n") - } - - if getCmd(cmd, "phase") != nil { - t.Error("cmd has phase subcommand while it should not\n") - } - - return - } - - // Otherwise, if there are phases - - // Checks that cmd get the description set and the skip-phases flags - if cmd.Long == "" { - t.Error("cmd.Long not set\n") - } - - if cmd.Flags().Lookup("skip-phases") == nil { - t.Error("cmd didn't have skip-phases flag\n") - } - - // Checks that cmd gets a new phase subcommand (without local flags) - phaseCmd := getCmd(cmd, "phase") - if phaseCmd == nil { - t.Error("cmd didn't have phase subcommand\n") - return - } - if err := cmdHasFlags(phaseCmd); err != nil { - t.Errorf("command phase didn't have expected flags: %v\n", err) - } - - // Checks that cmd subcommand gets subcommand for phases (without flags properly sets) - for c, flags := range rt.expectedCmdAndFlags { - - cCmd := getCmd(cmd, c) - if cCmd == nil { - t.Errorf("cmd didn't have %s subcommand\n", c) - continue - } - - if err := cmdHasFlags(cCmd, flags...); err != nil { - t.Errorf("command %s didn't have expected flags: %v\n", c, err) - } - } - - }) - } -} - -func getCmd(parent *cobra.Command, nestedName string) *cobra.Command { - names := strings.Split(nestedName, " ") - for i, n := range names { - for _, c := range parent.Commands() { - if c.Name() == n { - if i == len(names)-1 { - return c - } - parent = c - } - } - } - - return nil -} - -func cmdHasFlags(cmd *cobra.Command, expectedFlags ...string) error { - flags := []string{} - cmd.Flags().VisitAll(func(f *pflag.Flag) { - flags = append(flags, f.Name) - }) - - for _, e := range expectedFlags { - found := false - for _, f := range flags { - if f == e { - found = true - } - } - if !found { - return errors.Errorf("flag %q does not exists in %s", e, flags) - } - } - - if len(flags) != len(expectedFlags) { - return errors.Errorf("expected flags %s, got %s", expectedFlags, flags) - } - - return nil -} diff --git a/pkg/util/kubernetes/kubeadm/app/constants/constants.go b/pkg/util/kubernetes/kubeadm/app/constants/constants.go index f2853a8eaaa..319b912af66 100644 --- a/pkg/util/kubernetes/kubeadm/app/constants/constants.go +++ b/pkg/util/kubernetes/kubeadm/app/constants/constants.go @@ -17,634 +17,26 @@ limitations under the License. package constants import ( - "fmt" - "io/ioutil" - "net" - "os" - "path" - "path/filepath" - "strings" "time" - "github.com/pkg/errors" - v1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/util/version" - "k8s.io/apimachinery/pkg/util/wait" bootstrapapi "k8s.io/cluster-bootstrap/token/api" - utilnet "k8s.io/utils/net" ) const ( - // KubernetesDir is the directory Kubernetes owns for storing various configuration files - KubernetesDir = "/etc/kubernetes" - // ManifestsSubDirName defines directory name to store manifests - ManifestsSubDirName = "manifests" - // TempDirForKubeadm defines temporary directory for kubeadm - // should be joined with KubernetesDir. - TempDirForKubeadm = "tmp" - - // CertificateValidity defines the validity for all the signed certificates generated by kubeadm - CertificateValidity = time.Hour * 24 * 365 - - // CACertAndKeyBaseName defines certificate authority base name - CACertAndKeyBaseName = "ca" - // CACertName defines certificate name - CACertName = "ca.crt" - // CAKeyName defines certificate name - CAKeyName = "ca.key" - - // APIServerCertAndKeyBaseName defines API's server certificate and key base name - APIServerCertAndKeyBaseName = "apiserver" - // APIServerCertName defines API's server certificate name - APIServerCertName = "apiserver.crt" - // APIServerKeyName defines API's server key name - APIServerKeyName = "apiserver.key" - // APIServerCertCommonName defines API's server certificate common name (CN) - APIServerCertCommonName = "kube-apiserver" - - // APIServerKubeletClientCertAndKeyBaseName defines kubelet client certificate and key base name - APIServerKubeletClientCertAndKeyBaseName = "apiserver-kubelet-client" - // APIServerKubeletClientCertName defines kubelet client certificate name - APIServerKubeletClientCertName = "apiserver-kubelet-client.crt" - // APIServerKubeletClientKeyName defines kubelet client key name - APIServerKubeletClientKeyName = "apiserver-kubelet-client.key" - // APIServerKubeletClientCertCommonName defines kubelet client certificate common name (CN) - APIServerKubeletClientCertCommonName = "kube-apiserver-kubelet-client" - - // EtcdCACertAndKeyBaseName defines etcd's CA certificate and key base name - EtcdCACertAndKeyBaseName = "etcd/ca" - // EtcdCACertName defines etcd's CA certificate name - EtcdCACertName = "etcd/ca.crt" - // EtcdCAKeyName defines etcd's CA key name - EtcdCAKeyName = "etcd/ca.key" - - // EtcdServerCertAndKeyBaseName defines etcd's server certificate and key base name - EtcdServerCertAndKeyBaseName = "etcd/server" - // EtcdServerCertName defines etcd's server certificate name - EtcdServerCertName = "etcd/server.crt" - // EtcdServerKeyName defines etcd's server key name - EtcdServerKeyName = "etcd/server.key" - - // EtcdListenClientPort defines the port etcd listen on for client traffic - EtcdListenClientPort = 2379 - // EtcdMetricsPort is the port at which to obtain etcd metrics and health status - EtcdMetricsPort = 2381 - - // EtcdPeerCertAndKeyBaseName defines etcd's peer certificate and key base name - EtcdPeerCertAndKeyBaseName = "etcd/peer" - // EtcdPeerCertName defines etcd's peer certificate name - EtcdPeerCertName = "etcd/peer.crt" - // EtcdPeerKeyName defines etcd's peer key name - EtcdPeerKeyName = "etcd/peer.key" - - // EtcdListenPeerPort defines the port etcd listen on for peer traffic - EtcdListenPeerPort = 2380 - - // EtcdHealthcheckClientCertAndKeyBaseName defines etcd's healthcheck client certificate and key base name - EtcdHealthcheckClientCertAndKeyBaseName = "etcd/healthcheck-client" - // EtcdHealthcheckClientCertName defines etcd's healthcheck client certificate name - EtcdHealthcheckClientCertName = "etcd/healthcheck-client.crt" - // EtcdHealthcheckClientKeyName defines etcd's healthcheck client key name - EtcdHealthcheckClientKeyName = "etcd/healthcheck-client.key" - // EtcdHealthcheckClientCertCommonName defines etcd's healthcheck client certificate common name (CN) - EtcdHealthcheckClientCertCommonName = "kube-etcd-healthcheck-client" - - // APIServerEtcdClientCertAndKeyBaseName defines apiserver's etcd client certificate and key base name - APIServerEtcdClientCertAndKeyBaseName = "apiserver-etcd-client" - // APIServerEtcdClientCertName defines apiserver's etcd client certificate name - APIServerEtcdClientCertName = "apiserver-etcd-client.crt" - // APIServerEtcdClientKeyName defines apiserver's etcd client key name - APIServerEtcdClientKeyName = "apiserver-etcd-client.key" - // APIServerEtcdClientCertCommonName defines apiserver's etcd client certificate common name (CN) - APIServerEtcdClientCertCommonName = "kube-apiserver-etcd-client" - - // ServiceAccountKeyBaseName defines SA key base name - ServiceAccountKeyBaseName = "sa" - // ServiceAccountPublicKeyName defines SA public key base name - ServiceAccountPublicKeyName = "sa.pub" - // ServiceAccountPrivateKeyName defines SA private key base name - ServiceAccountPrivateKeyName = "sa.key" - - // FrontProxyCACertAndKeyBaseName defines front proxy CA certificate and key base name - FrontProxyCACertAndKeyBaseName = "front-proxy-ca" - // FrontProxyCACertName defines front proxy CA certificate name - FrontProxyCACertName = "front-proxy-ca.crt" - // FrontProxyCAKeyName defines front proxy CA key name - FrontProxyCAKeyName = "front-proxy-ca.key" - - // FrontProxyClientCertAndKeyBaseName defines front proxy certificate and key base name - FrontProxyClientCertAndKeyBaseName = "front-proxy-client" - // FrontProxyClientCertName defines front proxy certificate name - FrontProxyClientCertName = "front-proxy-client.crt" - // FrontProxyClientKeyName defines front proxy key name - FrontProxyClientKeyName = "front-proxy-client.key" - // FrontProxyClientCertCommonName defines front proxy certificate common name - FrontProxyClientCertCommonName = "front-proxy-client" //used as subject.commonname attribute (CN) - - // AdminKubeConfigFileName defines name for the kubeconfig aimed to be used by the superuser/admin of the cluster - AdminKubeConfigFileName = "admin.conf" - // KubeletBootstrapKubeConfigFileName defines the file name for the kubeconfig that the kubelet will use to do - // the TLS bootstrap to get itself an unique credential - KubeletBootstrapKubeConfigFileName = "bootstrap-kubelet.conf" - - // KubeletKubeConfigFileName defines the file name for the kubeconfig that the control-plane kubelet will use for talking - // to the API server - KubeletKubeConfigFileName = "kubelet.conf" - // ControllerManagerKubeConfigFileName defines the file name for the controller manager's kubeconfig file - ControllerManagerKubeConfigFileName = "controller-manager.conf" - // SchedulerKubeConfigFileName defines the file name for the scheduler's kubeconfig file - SchedulerKubeConfigFileName = "scheduler.conf" - - // Some well-known users and groups in the core Kubernetes authorization system - - // ControllerManagerUser defines the well-known user the controller-manager should be authenticated as - ControllerManagerUser = "system:kube-controller-manager" - // SchedulerUser defines the well-known user the scheduler should be authenticated as - SchedulerUser = "system:kube-scheduler" - // SystemPrivilegedGroup defines the well-known group for the apiservers. This group is also superuser by default - // (i.e. bound to the cluster-admin ClusterRole) - SystemPrivilegedGroup = "system:masters" - // NodesGroup defines the well-known group for all nodes. - NodesGroup = "system:nodes" - // NodesUserPrefix defines the user name prefix as requested by the Node authorizer. - NodesUserPrefix = "system:node:" - // NodesClusterRoleBinding defines the well-known ClusterRoleBinding which binds the too permissive system:node - // ClusterRole to the system:nodes group. Since kubeadm is using the Node Authorizer, this ClusterRoleBinding's - // system:nodes group subject is removed if present. - NodesClusterRoleBinding = "system:node" - // APICallRetryInterval defines how long kubeadm should wait before retrying a failed API operation APICallRetryInterval = 500 * time.Millisecond - // DiscoveryRetryInterval specifies how long kubeadm should wait before retrying to connect to the control-plane when doing discovery - DiscoveryRetryInterval = 5 * time.Second - // PatchNodeTimeout specifies how long kubeadm should wait for applying the label and taint on the control-plane before timing out - PatchNodeTimeout = 2 * time.Minute - // TLSBootstrapTimeout specifies how long kubeadm should wait for the kubelet to perform the TLS Bootstrap - TLSBootstrapTimeout = 5 * time.Minute - // TLSBootstrapRetryInterval specifies how long kubeadm should wait before retrying the TLS Bootstrap check - TLSBootstrapRetryInterval = 5 * time.Second // APICallWithWriteTimeout specifies how long kubeadm should wait for api calls with at least one write APICallWithWriteTimeout = 40 * time.Second - // APICallWithReadTimeout specifies how long kubeadm should wait for api calls with only reads - APICallWithReadTimeout = 15 * time.Second - // PullImageRetry specifies how many times ContainerRuntime retries when pulling image failed - PullImageRetry = 5 - - // DefaultControlPlaneTimeout specifies the default control plane (actually API Server) timeout for use by kubeadm - DefaultControlPlaneTimeout = 4 * time.Minute - - // MinimumAddressesInServiceSubnet defines minimum amount of nodes the Service subnet should allow. - // We need at least ten, because the DNS service is always at the tenth cluster clusterIP - MinimumAddressesInServiceSubnet = 10 - - // MaximumBitsForServiceSubnet defines maximum possible size of the service subnet in terms of bits. - // For example, if the value is 20, then the largest supported service subnet is /12 for IPv4 and /108 for IPv6. - // Note however that anything in between /108 and /112 will be clamped to /112 due to the limitations of the underlying allocation logic. - // TODO: https://github.com/kubernetes/enhancements/pull/1881 - MaximumBitsForServiceSubnet = 20 - - // MinimumAddressesInPodSubnet defines minimum amount of pods in the cluster. - // We need at least more than services, an IPv4 /28 or IPv6 /128 subnet means 14 util addresses - MinimumAddressesInPodSubnet = 14 - - // PodSubnetNodeMaskMaxDiff is limited to 16 due to an issue with uncompressed IP bitmap in core: - // xref: #44918 - // The node subnet mask size must be no more than the pod subnet mask size + 16 - PodSubnetNodeMaskMaxDiff = 16 - - // DefaultTokenDuration specifies the default amount of time that a bootstrap token will be valid - // Default behaviour is 24 hours - DefaultTokenDuration = 24 * time.Hour - - // DefaultCertTokenDuration specifies the default amount of time that the token used by upload certs will be valid - // Default behaviour is 2 hours - DefaultCertTokenDuration = 2 * time.Hour - - // CertificateKeySize specifies the size of the key used to encrypt certificates on uploadcerts phase - CertificateKeySize = 32 - - // LabelNodeRoleOldControlPlane specifies that a node hosts control-plane components - // DEPRECATED: https://github.com/kubernetes/kubeadm/issues/2200 - LabelNodeRoleOldControlPlane = "node-role.kubernetes.io/master" - - // LabelNodeRoleControlPlane specifies that a node hosts control-plane components - LabelNodeRoleControlPlane = "node-role.kubernetes.io/control-plane" - - // LabelExcludeFromExternalLB can be set on a node to exclude it from external load balancers. - // This is added to control plane nodes to preserve backwards compatibility with a legacy behavior. - LabelExcludeFromExternalLB = "node.kubernetes.io/exclude-from-external-load-balancers" - - // AnnotationKubeadmCRISocket specifies the annotation kubeadm uses to preserve the crisocket information given to kubeadm at - // init/join time for use later. kubeadm annotates the node object with this information - AnnotationKubeadmCRISocket = "kubeadm.alpha.kubernetes.io/cri-socket" - - // UnknownCRISocket defines the undetected or unknown CRI socket - UnknownCRISocket = "/var/run/unknown.sock" - - // KubeadmConfigConfigMap specifies in what ConfigMap in the kube-system namespace the `kubeadm init` configuration should be stored - KubeadmConfigConfigMap = "kubeadm-config" - - // ClusterConfigurationConfigMapKey specifies in what ConfigMap key the cluster configuration should be stored - ClusterConfigurationConfigMapKey = "ClusterConfiguration" - - // KubeProxyConfigMap specifies in what ConfigMap in the kube-system namespace the kube-proxy configuration should be stored - KubeProxyConfigMap = "kube-proxy" - - // KubeProxyConfigMapKey specifies in what ConfigMap key the component config of kube-proxy should be stored - KubeProxyConfigMapKey = "config.conf" - - // KubeletBaseConfigurationConfigMapPrefix specifies in what ConfigMap in the kube-system namespace the initial remote configuration of kubelet should be stored - KubeletBaseConfigurationConfigMapPrefix = "kubelet-config-" - - // KubeletBaseConfigurationConfigMapKey specifies in what ConfigMap key the initial remote configuration of kubelet should be stored - KubeletBaseConfigurationConfigMapKey = "kubelet" - - // KubeletBaseConfigMapRolePrefix defines the base kubelet configuration ConfigMap. - KubeletBaseConfigMapRolePrefix = "kubeadm:kubelet-config-" - - // KubeletRunDirectory specifies the directory where the kubelet runtime information is stored. - KubeletRunDirectory = "/var/lib/kubelet" - - // KubeletConfigurationFileName specifies the file name on the node which stores initial remote configuration of kubelet - // This file should exist under KubeletRunDirectory - KubeletConfigurationFileName = "config.yaml" - - // KubeletEnvFileName is a file "kubeadm init" writes at runtime. Using that interface, kubeadm can customize certain - // kubelet flags conditionally based on the environment at runtime. Also, parameters given to the configuration file - // might be passed through this file. "kubeadm init" writes one variable, with the name ${KubeletEnvFileVariableName}. - // This file should exist under KubeletRunDirectory - KubeletEnvFileName = "kubeadm-flags.env" - - // KubeletEnvFileVariableName specifies the shell script variable name "kubeadm init" should write a value to in KubeletEnvFile - KubeletEnvFileVariableName = "KUBELET_KUBEADM_ARGS" - - // KubeletHealthzPort is the port of the kubelet healthz endpoint - KubeletHealthzPort = 10248 - - // MinExternalEtcdVersion indicates minimum external etcd version which kubeadm supports - MinExternalEtcdVersion = "3.2.18" - - // DefaultEtcdVersion indicates the default etcd version that kubeadm uses - DefaultEtcdVersion = "3.5.0-0" - - // Etcd defines variable used internally when referring to etcd component - Etcd = "etcd" - // KubeAPIServer defines variable used internally when referring to kube-apiserver component - KubeAPIServer = "kube-apiserver" - // KubeControllerManager defines variable used internally when referring to kube-controller-manager component - KubeControllerManager = "kube-controller-manager" - // KubeScheduler defines variable used internally when referring to kube-scheduler component - KubeScheduler = "kube-scheduler" - // KubeProxy defines variable used internally when referring to kube-proxy component - KubeProxy = "kube-proxy" - // CoreDNS defines variable used internally when referring to the CoreDNS component - CoreDNS = "CoreDNS" - // Kubelet defines variable used internally when referring to the Kubelet - Kubelet = "kubelet" - - // KubeCertificatesVolumeName specifies the name for the Volume that is used for injecting certificates to control plane components (can be both a hostPath volume or a projected, all-in-one volume) - KubeCertificatesVolumeName = "k8s-certs" - - // KubeConfigVolumeName specifies the name for the Volume that is used for injecting the kubeconfig to talk securely to the api server for a control plane component if applicable - KubeConfigVolumeName = "kubeconfig" // NodeBootstrapTokenAuthGroup specifies which group a Node Bootstrap Token should be authenticated in NodeBootstrapTokenAuthGroup = "system:bootstrappers:kubeadm:default-node-token" - - // DefaultCIImageRepository points to image registry where CI uploads images from ci-cross build job - DefaultCIImageRepository = "gcr.io/k8s-staging-ci-images" - - // CoreDNSConfigMap specifies in what ConfigMap in the kube-system namespace the CoreDNS config should be stored - CoreDNSConfigMap = "coredns" - - // CoreDNSDeploymentName specifies the name of the Deployment for CoreDNS add-on - CoreDNSDeploymentName = "coredns" - - // CoreDNSImageName specifies the name of the image for CoreDNS add-on - CoreDNSImageName = "coredns" - - // CoreDNSVersion is the version of CoreDNS to be deployed if it is used - CoreDNSVersion = "v1.8.4" - - // ClusterConfigurationKind is the string kind value for the ClusterConfiguration struct - ClusterConfigurationKind = "ClusterConfiguration" - - // InitConfigurationKind is the string kind value for the InitConfiguration struct - InitConfigurationKind = "InitConfiguration" - - // JoinConfigurationKind is the string kind value for the JoinConfiguration struct - JoinConfigurationKind = "JoinConfiguration" - - // YAMLDocumentSeparator is the separator for YAML documents - // TODO: Find a better place for this constant - YAMLDocumentSeparator = "---\n" - - // DefaultAPIServerBindAddress is the default bind address for the API Server - DefaultAPIServerBindAddress = "0.0.0.0" - - // ControlPlaneNumCPU is the number of CPUs required on control-plane - ControlPlaneNumCPU = 2 - - // ControlPlaneMem is the number of megabytes of memory required on the control-plane - // Below that amount of RAM running a stable control plane would be difficult. - ControlPlaneMem = 1700 - - // KubeadmCertsSecret specifies in what Secret in the kube-system namespace the certificates should be stored - KubeadmCertsSecret = "kubeadm-certs" - - // KubeletPort is the default port for the kubelet server on each host machine. - // May be overridden by a flag at startup. - KubeletPort = 10250 - // KubeSchedulerPort is the default port for the scheduler status server. - // May be overridden by a flag at startup. - KubeSchedulerPort = 10259 - // KubeControllerManagerPort is the default port for the controller manager status server. - // May be overridden by a flag at startup. - KubeControllerManagerPort = 10257 - - // EtcdAdvertiseClientUrlsAnnotationKey is the annotation key on every etcd pod, describing the - // advertise client URLs - EtcdAdvertiseClientUrlsAnnotationKey = "kubeadm.kubernetes.io/etcd.advertise-client-urls" - // KubeAPIServerAdvertiseAddressEndpointAnnotationKey is the annotation key on every apiserver pod, - // describing the API endpoint (advertise address and bind port of the api server) - KubeAPIServerAdvertiseAddressEndpointAnnotationKey = "kubeadm.kubernetes.io/kube-apiserver.advertise-address.endpoint" - // ComponentConfigHashAnnotationKey holds the config map annotation key that kubeadm uses to store - // a SHA256 sum to check for user changes - ComponentConfigHashAnnotationKey = "kubeadm.kubernetes.io/component-config.hash" - - // ControlPlaneTier is the value used in the tier label to identify control plane components - ControlPlaneTier = "control-plane" - - // Mode* constants were copied from pkg/kubeapiserver/authorizer/modes - // to avoid kubeadm dependency on the internal module - // TODO: share Mode* constants in component config - - // ModeAlwaysAllow is the mode to set all requests as authorized - ModeAlwaysAllow string = "AlwaysAllow" - // ModeAlwaysDeny is the mode to set no requests as authorized - ModeAlwaysDeny string = "AlwaysDeny" - // ModeABAC is the mode to use Attribute Based Access Control to authorize - ModeABAC string = "ABAC" - // ModeWebhook is the mode to make an external webhook call to authorize - ModeWebhook string = "Webhook" - // ModeRBAC is the mode to use Role Based Access Control to authorize - ModeRBAC string = "RBAC" - // ModeNode is an authorization mode that authorizes API requests made by kubelets. - ModeNode string = "Node" - - // PauseVersion indicates the default pause image version for kubeadm - PauseVersion = "3.5" - - // CgroupDriverSystemd holds the systemd driver type - CgroupDriverSystemd = "systemd" - - // The username of the user that kube-controller-manager runs as. - KubeControllerManagerUserName string = "kubeadm-kcm" - // The username of the user that kube-apiserver runs as. - KubeAPIServerUserName string = "kubeadm-kas" - // The username of the user that kube-scheduler runs as. - KubeSchedulerUserName string = "kubeadm-ks" - // The username of the user that etcd runs as. - EtcdUserName string = "kubeadm-etcd" - // The group of users that are allowed to read the service account private key. - ServiceAccountKeyReadersGroupName string = "kubeadm-sa-key-readers" ) var ( - // OldControlPlaneTaint is the taint to apply on the PodSpec for being able to run that Pod on the control-plane - // DEPRECATED: https://github.com/kubernetes/kubeadm/issues/2200 - OldControlPlaneTaint = v1.Taint{ - Key: LabelNodeRoleOldControlPlane, - Effect: v1.TaintEffectNoSchedule, - } - - // OldControlPlaneToleration is the toleration to apply on the PodSpec for being able to run that Pod on the control-plane - // DEPRECATED: https://github.com/kubernetes/kubeadm/issues/2200 - OldControlPlaneToleration = v1.Toleration{ - Key: LabelNodeRoleOldControlPlane, - Effect: v1.TaintEffectNoSchedule, - } - - // ControlPlaneTaint is the taint to apply on the PodSpec for being able to run that Pod on the control-plane - ControlPlaneTaint = v1.Taint{ - Key: LabelNodeRoleControlPlane, - Effect: v1.TaintEffectNoSchedule, - } - - // ControlPlaneToleration is the toleration to apply on the PodSpec for being able to run that Pod on the control-plane - ControlPlaneToleration = v1.Toleration{ - Key: LabelNodeRoleControlPlane, - Effect: v1.TaintEffectNoSchedule, - } // DefaultTokenUsages specifies the default functions a token will get DefaultTokenUsages = bootstrapapi.KnownTokenUsages // DefaultTokenGroups specifies the default groups that this token will authenticate as when used for authentication DefaultTokenGroups = []string{NodeBootstrapTokenAuthGroup} - - // ControlPlaneComponents defines the control-plane component names - ControlPlaneComponents = []string{KubeAPIServer, KubeControllerManager, KubeScheduler} - - // MinimumControlPlaneVersion specifies the minimum control plane version kubeadm can deploy - MinimumControlPlaneVersion = version.MustParseSemantic("v1.18.0") - - // MinimumKubeletVersion specifies the minimum version of kubelet which kubeadm supports - MinimumKubeletVersion = version.MustParseSemantic("v1.18.0") - - // CurrentKubernetesVersion specifies current Kubernetes version supported by kubeadm - CurrentKubernetesVersion = version.MustParseSemantic("v1.18.0") - - // SupportedEtcdVersion lists officially supported etcd versions with corresponding Kubernetes releases - SupportedEtcdVersion = map[uint8]string{ - 13: "3.2.24", - 14: "3.3.10", - 15: "3.3.10", - 16: "3.3.17-0", - 17: "3.4.3-0", - 18: "3.4.3-0", - 19: "3.4.13-0", - 20: "3.4.13-0", - 21: "3.4.13-0", - 22: "3.5.0-0", - 23: "3.5.0-0", - } - - // KubeadmCertsClusterRoleName sets the name for the ClusterRole that allows - // the bootstrap tokens to access the kubeadm-certs Secret during the join of a new control-plane - KubeadmCertsClusterRoleName = fmt.Sprintf("kubeadm:%s", KubeadmCertsSecret) - - // StaticPodMirroringDefaultRetry is used a backoff strategy for - // waiting for static pods to be mirrored to the apiserver. - StaticPodMirroringDefaultRetry = wait.Backoff{ - Steps: 30, - Duration: 1 * time.Second, - Factor: 1.0, - Jitter: 0.1, - } ) - -// EtcdSupportedVersion returns officially supported version of etcd for a specific Kubernetes release -// If passed version is not in the given list, the function returns the nearest version with a warning -func EtcdSupportedVersion(supportedEtcdVersion map[uint8]string, versionString string) (etcdVersion *version.Version, warning, err error) { - kubernetesVersion, err := version.ParseSemantic(versionString) - if err != nil { - return nil, nil, err - } - desiredVersion, etcdStringVersion := uint8(kubernetesVersion.Minor()), "" - - min, max := ^uint8(0), uint8(0) - for k, v := range supportedEtcdVersion { - if desiredVersion == k { - etcdStringVersion = v - break - } - if k < min { - min = k - } - if k > max { - max = k - } - } - - if len(etcdStringVersion) == 0 { - if desiredVersion < min { - etcdStringVersion = supportedEtcdVersion[min] - } - if desiredVersion > max { - etcdStringVersion = supportedEtcdVersion[max] - } - warning = fmt.Errorf("could not find officially supported version of etcd for Kubernetes %s, falling back to the nearest etcd version (%s)", - versionString, etcdStringVersion) - } - - etcdVersion, err = version.ParseSemantic(etcdStringVersion) - if err != nil { - return nil, nil, err - } - - return etcdVersion, warning, nil -} - -// GetStaticPodDirectory returns the location on the disk where the Static Pod should be present -func GetStaticPodDirectory() string { - return filepath.Join(KubernetesDir, ManifestsSubDirName) -} - -// GetStaticPodFilepath returns the location on the disk where the Static Pod should be present -func GetStaticPodFilepath(componentName, manifestsDir string) string { - return filepath.Join(manifestsDir, componentName+".yaml") -} - -// GetAdminKubeConfigPath returns the location on the disk where admin kubeconfig is located by default -func GetAdminKubeConfigPath() string { - return filepath.Join(KubernetesDir, AdminKubeConfigFileName) -} - -// GetBootstrapKubeletKubeConfigPath returns the location on the disk where bootstrap kubelet kubeconfig is located by default -func GetBootstrapKubeletKubeConfigPath() string { - return filepath.Join(KubernetesDir, KubeletBootstrapKubeConfigFileName) -} - -// GetKubeletKubeConfigPath returns the location on the disk where kubelet kubeconfig is located by default -func GetKubeletKubeConfigPath() string { - return filepath.Join(KubernetesDir, KubeletKubeConfigFileName) -} - -// CreateTempDirForKubeadm is a function that creates a temporary directory under /etc/kubernetes/tmp (not using /tmp as that would potentially be dangerous) -func CreateTempDirForKubeadm(kubernetesDir, dirName string) (string, error) { - tempDir := path.Join(KubernetesDir, TempDirForKubeadm) - if len(kubernetesDir) != 0 { - tempDir = path.Join(kubernetesDir, TempDirForKubeadm) - } - - // creates target folder if not already exists - if err := os.MkdirAll(tempDir, 0700); err != nil { - return "", errors.Wrapf(err, "failed to create directory %q", tempDir) - } - - tempDir, err := ioutil.TempDir(tempDir, dirName) - if err != nil { - return "", errors.Wrap(err, "couldn't create a temporary directory") - } - return tempDir, nil -} - -// CreateTimestampDirForKubeadm is a function that creates a temporary directory under /etc/kubernetes/tmp formatted with the current date -func CreateTimestampDirForKubeadm(kubernetesDir, dirName string) (string, error) { - tempDir := path.Join(KubernetesDir, TempDirForKubeadm) - if len(kubernetesDir) != 0 { - tempDir = path.Join(kubernetesDir, TempDirForKubeadm) - } - - // creates target folder if not already exists - if err := os.MkdirAll(tempDir, 0700); err != nil { - return "", errors.Wrapf(err, "failed to create directory %q", tempDir) - } - - timestampDirName := fmt.Sprintf("%s-%s", dirName, time.Now().Format("2006-01-02-15-04-05")) - timestampDir := path.Join(tempDir, timestampDirName) - if err := os.Mkdir(timestampDir, 0700); err != nil { - return "", errors.Wrap(err, "could not create timestamp directory") - } - - return timestampDir, nil -} - -// GetDNSIP returns a dnsIP, which is 10th IP in svcSubnet CIDR range -func GetDNSIP(svcSubnetList string, isDualStack bool) (net.IP, error) { - // Get the service subnet CIDR - svcSubnetCIDR, err := GetKubernetesServiceCIDR(svcSubnetList, isDualStack) - if err != nil { - return nil, errors.Wrapf(err, "unable to get internal Kubernetes Service IP from the given service CIDR (%s)", svcSubnetList) - } - - // Selects the 10th IP in service subnet CIDR range as dnsIP - dnsIP, err := utilnet.GetIndexedIP(svcSubnetCIDR, 10) - if err != nil { - return nil, errors.Wrap(err, "unable to get internal Kubernetes Service IP from the given service CIDR") - } - - return dnsIP, nil -} - -// GetKubernetesServiceCIDR returns the default Service CIDR for the Kubernetes internal service -func GetKubernetesServiceCIDR(svcSubnetList string, isDualStack bool) (*net.IPNet, error) { - if isDualStack { - // The default service address family for the cluster is the address family of the first - // service cluster IP range configured via the `--service-cluster-ip-range` flag - // of the kube-controller-manager and kube-apiserver. - svcSubnets, err := utilnet.ParseCIDRs(strings.Split(svcSubnetList, ",")) - if err != nil { - return nil, errors.Wrapf(err, "unable to parse ServiceSubnet %v", svcSubnetList) - } - if len(svcSubnets) == 0 { - return nil, errors.New("received empty ServiceSubnet for dual-stack") - } - return svcSubnets[0], nil - } - // internal IP address for the API server - _, svcSubnet, err := net.ParseCIDR(svcSubnetList) - if err != nil { - return nil, errors.Wrapf(err, "unable to parse ServiceSubnet %v", svcSubnetList) - } - return svcSubnet, nil -} - -// GetAPIServerVirtualIP returns the IP of the internal Kubernetes API service -func GetAPIServerVirtualIP(svcSubnetList string, isDualStack bool) (net.IP, error) { - svcSubnet, err := GetKubernetesServiceCIDR(svcSubnetList, isDualStack) - if err != nil { - return nil, errors.Wrap(err, "unable to get internal Kubernetes Service IP from the given service CIDR") - } - internalAPIServerVirtualIP, err := utilnet.GetIndexedIP(svcSubnet, 1) - if err != nil { - return nil, errors.Wrapf(err, "unable to get the first IP address from the given CIDR: %s", svcSubnet.String()) - } - return internalAPIServerVirtualIP, nil -} - -// GetKubeletConfigMapName returns the right ConfigMap name for the right branch of k8s -func GetKubeletConfigMapName(k8sVersion *version.Version) string { - return fmt.Sprintf("%s%d.%d", KubeletBaseConfigurationConfigMapPrefix, k8sVersion.Major(), k8sVersion.Minor()) -} diff --git a/pkg/util/kubernetes/kubeadm/app/constants/constants_unix.go b/pkg/util/kubernetes/kubeadm/app/constants/constants_unix.go deleted file mode 100644 index 812faf064b5..00000000000 --- a/pkg/util/kubernetes/kubeadm/app/constants/constants_unix.go +++ /dev/null @@ -1,25 +0,0 @@ -//go:build !windows -// +build !windows - -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package constants - -const ( - // DefaultDockerCRISocket defines the default Docker CRI socket - DefaultDockerCRISocket = "/var/run/dockershim.sock" -) diff --git a/pkg/util/kubernetes/kubeadm/app/constants/constants_windows.go b/pkg/util/kubernetes/kubeadm/app/constants/constants_windows.go deleted file mode 100644 index 6daae0a1fff..00000000000 --- a/pkg/util/kubernetes/kubeadm/app/constants/constants_windows.go +++ /dev/null @@ -1,24 +0,0 @@ -// +build windows - -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package constants - -const ( - // DefaultDockerCRISocket defines the default Docker CRI socket - DefaultDockerCRISocket = "npipe:////./pipe/docker_engine" -) diff --git a/pkg/util/kubernetes/kubeadm/app/features/features.go b/pkg/util/kubernetes/kubeadm/app/features/features.go deleted file mode 100644 index 1288cb5e5d8..00000000000 --- a/pkg/util/kubernetes/kubeadm/app/features/features.go +++ /dev/null @@ -1,181 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package features - -import ( - "fmt" - "sort" - "strconv" - "strings" - - "github.com/pkg/errors" - "k8s.io/apimachinery/pkg/util/version" - "k8s.io/component-base/featuregate" -) - -const ( - // IPv6DualStack is expected to be beta in v1.21 - IPv6DualStack = "IPv6DualStack" - // PublicKeysECDSA is expected to be alpha in v1.19 - PublicKeysECDSA = "PublicKeysECDSA" - // RootlessControlPlane is expected to be in alpha in v1.22 - RootlessControlPlane = "RootlessControlPlane" -) - -// InitFeatureGates are the default feature gates for the init command -var InitFeatureGates = FeatureList{ - IPv6DualStack: {FeatureSpec: featuregate.FeatureSpec{Default: true, PreRelease: featuregate.Beta}}, - PublicKeysECDSA: {FeatureSpec: featuregate.FeatureSpec{Default: false, PreRelease: featuregate.Alpha}}, - RootlessControlPlane: {FeatureSpec: featuregate.FeatureSpec{Default: false, PreRelease: featuregate.Alpha}}, -} - -// Feature represents a feature being gated -type Feature struct { - featuregate.FeatureSpec - MinimumVersion *version.Version - HiddenInHelpText bool - DeprecationMessage string -} - -// FeatureList represents a list of feature gates -type FeatureList map[string]Feature - -// ValidateVersion ensures that a feature gate list is compatible with the chosen Kubernetes version -func ValidateVersion(allFeatures FeatureList, requestedFeatures map[string]bool, requestedVersion string) error { - if requestedVersion == "" { - return nil - } - parsedExpVersion, err := version.ParseSemantic(requestedVersion) - if err != nil { - return errors.Wrapf(err, "error parsing version %s", requestedVersion) - } - for k := range requestedFeatures { - if minVersion := allFeatures[k].MinimumVersion; minVersion != nil { - if !parsedExpVersion.AtLeast(minVersion) { - return errors.Errorf( - "the requested Kubernetes version (%s) is incompatible with the %s feature gate, which needs %s as a minimum", - requestedVersion, k, minVersion) - } - } - } - return nil -} - -// Enabled indicates whether a feature name has been enabled -func Enabled(featureList map[string]bool, featureName string) bool { - if enabled, ok := featureList[string(featureName)]; ok { - return enabled - } - return InitFeatureGates[string(featureName)].Default -} - -// Supports indicates whether a feature name is supported on the given -// feature set -func Supports(featureList FeatureList, featureName string) bool { - for k, v := range featureList { - if featureName == string(k) { - return v.PreRelease != featuregate.Deprecated - } - } - return false -} - -// Keys returns a slice of feature names for a given feature set -func Keys(featureList FeatureList) []string { - var list []string - for k := range featureList { - list = append(list, string(k)) - } - return list -} - -// KnownFeatures returns a slice of strings describing the FeatureList features. -func KnownFeatures(f *FeatureList) []string { - var known []string - for k, v := range *f { - if v.HiddenInHelpText { - continue - } - - pre := "" - if v.PreRelease != featuregate.GA { - pre = fmt.Sprintf("%s - ", v.PreRelease) - } - known = append(known, fmt.Sprintf("%s=true|false (%sdefault=%t)", k, pre, v.Default)) - } - sort.Strings(known) - return known -} - -// NewFeatureGate parses a string of the form "key1=value1,key2=value2,..." into a -// map[string]bool of known keys or returns an error. -func NewFeatureGate(f *FeatureList, value string) (map[string]bool, error) { - featureGate := map[string]bool{} - for _, s := range strings.Split(value, ",") { - if len(s) == 0 { - continue - } - - arr := strings.SplitN(s, "=", 2) - if len(arr) != 2 { - return nil, errors.Errorf("missing bool value for feature-gate key:%s", s) - } - - k := strings.TrimSpace(arr[0]) - v := strings.TrimSpace(arr[1]) - - featureSpec, ok := (*f)[k] - if !ok { - return nil, errors.Errorf("unrecognized feature-gate key: %s", k) - } - - if featureSpec.PreRelease == featuregate.Deprecated { - return nil, errors.Errorf("feature-gate key is deprecated: %s", k) - } - - boolValue, err := strconv.ParseBool(v) - if err != nil { - return nil, errors.Errorf("invalid value %v for feature-gate key: %s, use true|false instead", v, k) - } - featureGate[k] = boolValue - } - - return featureGate, nil -} - -// CheckDeprecatedFlags takes a list of existing feature gate flags and validates against the current feature flag set. -// It used during upgrades for ensuring consistency of feature gates used in an existing cluster, that might -// be created with a previous version of kubeadm, with the set of features currently supported by kubeadm -func CheckDeprecatedFlags(f *FeatureList, features map[string]bool) map[string]string { - deprecatedMsg := map[string]string{} - for k := range features { - featureSpec, ok := (*f)[k] - if !ok { - // This case should never happen, it is implemented only as a sentinel - // for removal of flags executed when flags are still in use (always before deprecate, then after one cycle remove) - deprecatedMsg[k] = fmt.Sprintf("Unknown feature gate flag: %s", k) - } - - if featureSpec.PreRelease == featuregate.Deprecated { - if _, ok := deprecatedMsg[k]; !ok { - deprecatedMsg[k] = featureSpec.DeprecationMessage - } - } - } - - return deprecatedMsg -} diff --git a/pkg/util/kubernetes/kubeadm/app/features/features_test.go b/pkg/util/kubernetes/kubeadm/app/features/features_test.go deleted file mode 100644 index 81d29f3e6a8..00000000000 --- a/pkg/util/kubernetes/kubeadm/app/features/features_test.go +++ /dev/null @@ -1,219 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package features - -import ( - "reflect" - "testing" - - "k8s.io/apimachinery/pkg/util/version" - "k8s.io/component-base/featuregate" -) - -func TestKnownFeatures(t *testing.T) { - var someFeatures = FeatureList{ - "feature2": {FeatureSpec: featuregate.FeatureSpec{Default: true, PreRelease: featuregate.Alpha}}, - "feature1": {FeatureSpec: featuregate.FeatureSpec{Default: false, PreRelease: featuregate.Beta}}, - "feature3": {FeatureSpec: featuregate.FeatureSpec{Default: false, PreRelease: featuregate.GA}}, - "hidden": {FeatureSpec: featuregate.FeatureSpec{Default: false, PreRelease: featuregate.GA}, HiddenInHelpText: true}, - } - - r := KnownFeatures(&someFeatures) - - if len(r) != 3 { - t.Errorf("KnownFeatures returned %d values, expected 3", len(r)) - } - - // check the first value is feature1 (the list should be sorted); prerelease and default should be present - f1 := "feature1=true|false (BETA - default=false)" - if r[0] != f1 { - t.Errorf("KnownFeatures returned %s values, expected %s", r[0], f1) - } - // check the second value is feature2; prerelease and default should be present - f2 := "feature2=true|false (ALPHA - default=true)" - if r[1] != f2 { - t.Errorf("KnownFeatures returned %s values, expected %s", r[1], f2) - } - // check the second value is feature3; prerelease should not be shown for GA features; default should be present - f3 := "feature3=true|false (default=false)" - if r[2] != f3 { - t.Errorf("KnownFeatures returned %s values, expected %s", r[2], f3) - } -} - -func TestNewFeatureGate(t *testing.T) { - var someFeatures = FeatureList{ - "feature1": {FeatureSpec: featuregate.FeatureSpec{Default: false, PreRelease: featuregate.Beta}}, - "feature2": {FeatureSpec: featuregate.FeatureSpec{Default: true, PreRelease: featuregate.Alpha}}, - "deprecated": {FeatureSpec: featuregate.FeatureSpec{Default: true, PreRelease: featuregate.Deprecated}}, - } - - var tests = []struct { - value string - expectedError bool - expectedFeaturesGate map[string]bool - }{ - { //invalid value (missing =) - value: "invalidValue", - expectedError: true, - }, - { //invalid value (missing =) - value: "feature1=true,invalidValue", - expectedError: true, - }, - { //invalid value (not a boolean) - value: "feature1=notABoolean", - expectedError: true, - }, - { //invalid value (not a boolean) - value: "feature1=true,feature2=notABoolean", - expectedError: true, - }, - { //unrecognized feature-gate key - value: "unknownFeature=false", - expectedError: true, - }, - { //unrecognized feature-gate key - value: "feature1=true,unknownFeature=false", - expectedError: true, - }, - { //deprecated feature-gate key - value: "deprecated=true", - expectedError: true, - }, - { //one feature - value: "feature1=true", - expectedError: false, - expectedFeaturesGate: map[string]bool{"feature1": true}, - }, - { //two features - value: "feature1=true,feature2=false", - expectedError: false, - expectedFeaturesGate: map[string]bool{"feature1": true, "feature2": false}, - }, - } - - for _, test := range tests { - t.Run(test.value, func(t *testing.T) { - r, err := NewFeatureGate(&someFeatures, test.value) - - if !test.expectedError && err != nil { - t.Errorf("NewFeatureGate failed when not expected: %v", err) - return - } else if test.expectedError && err == nil { - t.Error("NewFeatureGate didn't failed when expected") - return - } - - if !reflect.DeepEqual(r, test.expectedFeaturesGate) { - t.Errorf("NewFeatureGate returned a unexpected value") - } - }) - } -} - -func TestValidateVersion(t *testing.T) { - var someFeatures = FeatureList{ - "feature1": {FeatureSpec: featuregate.FeatureSpec{Default: false, PreRelease: featuregate.Beta}}, - "feature2": {FeatureSpec: featuregate.FeatureSpec{Default: true, PreRelease: featuregate.Alpha}, MinimumVersion: version.MustParseSemantic("v1.17.0").WithPreRelease("alpha.1")}, - } - - var tests = []struct { - name string - requestedVersion string - requestedFeatures map[string]bool - expectedError bool - }{ - { - name: "no min version", - requestedFeatures: map[string]bool{"feature1": true}, - expectedError: false, - }, - { - name: "min version but correct value given", - requestedFeatures: map[string]bool{"feature2": true}, - requestedVersion: "v1.17.0", - expectedError: false, - }, - { - name: "min version and incorrect value given", - requestedFeatures: map[string]bool{"feature2": true}, - requestedVersion: "v1.11.2", - expectedError: true, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - err := ValidateVersion(someFeatures, test.requestedFeatures, test.requestedVersion) - if !test.expectedError && err != nil { - t.Errorf("ValidateVersion failed when not expected: %v", err) - return - } else if test.expectedError && err == nil { - t.Error("ValidateVersion didn't failed when expected") - return - } - }) - } -} - -// TestEnabledDefaults tests that Enabled returns the default values for -// each feature gate when no feature gates are specified. -func TestEnabledDefaults(t *testing.T) { - for featureName, feature := range InitFeatureGates { - featureList := make(map[string]bool) - - enabled := Enabled(featureList, featureName) - if enabled != feature.Default { - t.Errorf("Enabled returned %v instead of default value %v for feature %s", enabled, feature.Default, featureName) - } - } -} - -func TestCheckDeprecatedFlags(t *testing.T) { - dummyMessage := "dummy message" - var someFeatures = FeatureList{ - "feature1": {FeatureSpec: featuregate.FeatureSpec{Default: false, PreRelease: featuregate.Beta}}, - "deprecated": {FeatureSpec: featuregate.FeatureSpec{Default: true, PreRelease: featuregate.Deprecated}, DeprecationMessage: dummyMessage}, - } - - var tests = []struct { - name string - features map[string]bool - expectedMsg map[string]string - }{ - { - name: "deprecated feature", - features: map[string]bool{"deprecated": true}, - expectedMsg: map[string]string{"deprecated": dummyMessage}, - }, - { - name: "valid feature", - features: map[string]bool{"feature1": true}, - expectedMsg: map[string]string{}, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - msg := CheckDeprecatedFlags(&someFeatures, test.features) - if !reflect.DeepEqual(test.expectedMsg, msg) { - t.Error("CheckDeprecatedFlags didn't returned expected message") - } - }) - } -} diff --git a/pkg/util/kubernetes/kubeadm/app/phases/kubelet/kubelet.go b/pkg/util/kubernetes/kubeadm/app/phases/kubelet/kubelet.go deleted file mode 100644 index 9a511158358..00000000000 --- a/pkg/util/kubernetes/kubeadm/app/phases/kubelet/kubelet.go +++ /dev/null @@ -1,42 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package kubelet - -import ( - "fmt" - - "github.com/openyurtio/openyurt/pkg/util/kubernetes/kubeadm/app/util/initsystem" -) - -// TryStopKubelet attempts to bring down the kubelet service momentarily -func TryStopKubelet() { - // If we notice that the kubelet service is inactive, try to start it - initSystem, err := initsystem.GetInitSystem() - if err != nil { - fmt.Println("[kubelet-start] no supported init system detected, won't make sure the kubelet not running for a short period of time while setting up configuration for it.") - return - } - - if !initSystem.ServiceExists("kubelet") { - fmt.Println("[kubelet-start] couldn't detect a kubelet service, can't make sure the kubelet not running for a short period of time while setting up configuration for it.") - } - - // This runs "systemctl daemon-reload && systemctl stop kubelet" - if err := initSystem.ServiceStop("kubelet"); err != nil { - fmt.Printf("[kubelet-start] WARNING: unable to stop the kubelet service momentarily: [%v]\n", err) - } -} diff --git a/pkg/util/kubernetes/kubeadm/app/preflight/checks.go b/pkg/util/kubernetes/kubeadm/app/preflight/checks.go deleted file mode 100644 index b0bf8c39cdc..00000000000 --- a/pkg/util/kubernetes/kubeadm/app/preflight/checks.go +++ /dev/null @@ -1,754 +0,0 @@ -/* -Copyright 2016 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package preflight - -import ( - "bufio" - "bytes" - "fmt" - "io" - "net" - "net/http" - "net/url" - "os" - "runtime" - "strings" - - "github.com/pkg/errors" - v1 "k8s.io/api/core/v1" - netutil "k8s.io/apimachinery/pkg/util/net" - "k8s.io/apimachinery/pkg/util/sets" - "k8s.io/apimachinery/pkg/util/validation" - versionutil "k8s.io/apimachinery/pkg/util/version" - "k8s.io/klog/v2" - system "k8s.io/system-validators/validators" - utilsexec "k8s.io/utils/exec" - utilsnet "k8s.io/utils/net" - - kubeadmconstants "github.com/openyurtio/openyurt/pkg/util/kubernetes/kubeadm/app/constants" - "github.com/openyurtio/openyurt/pkg/util/kubernetes/kubeadm/app/util/initsystem" - utilruntime "github.com/openyurtio/openyurt/pkg/util/kubernetes/kubeadm/app/util/runtime" -) - -// Error defines struct for communicating error messages generated by preflight checks -type Error struct { - Msg string -} - -// Error implements the standard error interface -func (e *Error) Error() string { - return fmt.Sprintf("[preflight] Some fatal errors occurred:\n%s%s", e.Msg, "[preflight] If you know what you are doing, you can make a check non-fatal with `--ignore-preflight-errors=...`") -} - -// Preflight identifies this error as a preflight error -func (e *Error) Preflight() bool { - return true -} - -// Checker validates the state of the system to ensure kubeadm will be -// successful as often as possible. -type Checker interface { - Check() (warnings, errorList []error) - Name() string -} - -// ContainerRuntimeCheck verifies the container runtime. -type ContainerRuntimeCheck struct { - runtime utilruntime.ContainerRuntime -} - -// Name returns label for RuntimeCheck. -func (ContainerRuntimeCheck) Name() string { - return "CRI" -} - -// Check validates the container runtime -func (crc ContainerRuntimeCheck) Check() (warnings, errorList []error) { - klog.V(1).Infoln("validating the container runtime") - if err := crc.runtime.IsRunning(); err != nil { - errorList = append(errorList, err) - } - return warnings, errorList -} - -// ServiceCheck verifies that the given service is enabled and active. If we do not -// detect a supported init system however, all checks are skipped and a warning is -// returned. -type ServiceCheck struct { - Service string - CheckIfActive bool - Label string -} - -// Name returns label for ServiceCheck. If not provided, will return based on the service parameter -func (sc ServiceCheck) Name() string { - if sc.Label != "" { - return sc.Label - } - return fmt.Sprintf("Service-%s", strings.Title(sc.Service)) -} - -// Check validates if the service is enabled and active. -func (sc ServiceCheck) Check() (warnings, errorList []error) { - klog.V(1).Infof("validating if the %q service is enabled and active", sc.Service) - initSystem, err := initsystem.GetInitSystem() - if err != nil { - return []error{err}, nil - } - - if !initSystem.ServiceExists(sc.Service) { - return []error{errors.Errorf("%s service does not exist", sc.Service)}, nil - } - - if !initSystem.ServiceIsEnabled(sc.Service) { - warnings = append(warnings, - errors.Errorf("%s service is not enabled, please run '%s'", - sc.Service, initSystem.EnableCommand(sc.Service))) - } - - if sc.CheckIfActive && !initSystem.ServiceIsActive(sc.Service) { - errorList = append(errorList, - errors.Errorf("%s service is not active, please run 'systemctl start %s.service'", - sc.Service, sc.Service)) - } - - return warnings, errorList -} - -// FirewalldCheck checks if firewalld is enabled or active. If it is, warn the user that there may be problems -// if no actions are taken. -type FirewalldCheck struct { - ports []int -} - -// Name returns label for FirewalldCheck. -func (FirewalldCheck) Name() string { - return "Firewalld" -} - -// Check validates if the firewall is enabled and active. -func (fc FirewalldCheck) Check() (warnings, errorList []error) { - klog.V(1).Infoln("validating if the firewall is enabled and active") - initSystem, err := initsystem.GetInitSystem() - if err != nil { - return []error{err}, nil - } - - if !initSystem.ServiceExists("firewalld") { - return nil, nil - } - - if initSystem.ServiceIsActive("firewalld") { - err := errors.Errorf("firewalld is active, please ensure ports %v are open or your cluster may not function correctly", - fc.ports) - return []error{err}, nil - } - - return nil, nil -} - -// PortOpenCheck ensures the given port is available for use. -type PortOpenCheck struct { - port int - label string -} - -// Name returns name for PortOpenCheck. If not known, will return "PortXXXX" based on port number -func (poc PortOpenCheck) Name() string { - if poc.label != "" { - return poc.label - } - return fmt.Sprintf("Port-%d", poc.port) -} - -// Check validates if the particular port is available. -func (poc PortOpenCheck) Check() (warnings, errorList []error) { - klog.V(1).Infof("validating availability of port %d", poc.port) - - ln, err := net.Listen("tcp", fmt.Sprintf(":%d", poc.port)) - if err != nil { - errorList = []error{errors.Errorf("Port %d is in use", poc.port)} - } - if ln != nil { - if err = ln.Close(); err != nil { - warnings = append(warnings, - errors.Errorf("when closing port %d, encountered %v", poc.port, err)) - } - } - - return warnings, errorList -} - -// IsPrivilegedUserCheck verifies user is privileged (linux - root, windows - Administrator) -type IsPrivilegedUserCheck struct{} - -// Name returns name for IsPrivilegedUserCheck -func (IsPrivilegedUserCheck) Name() string { - return "IsPrivilegedUser" -} - -// DirAvailableCheck checks if the given directory either does not exist, or is empty. -type DirAvailableCheck struct { - Path string - Label string -} - -// Name returns label for individual DirAvailableChecks. If not known, will return based on path. -func (dac DirAvailableCheck) Name() string { - if dac.Label != "" { - return dac.Label - } - return fmt.Sprintf("DirAvailable-%s", strings.Replace(dac.Path, "/", "-", -1)) -} - -// Check validates if a directory does not exist or empty. -func (dac DirAvailableCheck) Check() (warnings, errorList []error) { - klog.V(1).Infof("validating the existence and emptiness of directory %s", dac.Path) - - // If it doesn't exist we are good: - if _, err := os.Stat(dac.Path); os.IsNotExist(err) { - return nil, nil - } - - f, err := os.Open(dac.Path) - if err != nil { - return nil, []error{errors.Wrapf(err, "unable to check if %s is empty", dac.Path)} - } - defer f.Close() - - _, err = f.Readdirnames(1) - if err != io.EOF { - return nil, []error{errors.Errorf("%s is not empty", dac.Path)} - } - - return nil, nil -} - -// FileAvailableCheck checks that the given file does not already exist. -type FileAvailableCheck struct { - Path string - Label string -} - -// Name returns label for individual FileAvailableChecks. If not known, will return based on path. -func (fac FileAvailableCheck) Name() string { - if fac.Label != "" { - return fac.Label - } - return fmt.Sprintf("FileAvailable-%s", strings.Replace(fac.Path, "/", "-", -1)) -} - -// Check validates if the given file does not already exist. -func (fac FileAvailableCheck) Check() (warnings, errorList []error) { - klog.V(1).Infof("validating the existence of file %s", fac.Path) - - if _, err := os.Stat(fac.Path); err == nil { - return nil, []error{errors.Errorf("%s already exists", fac.Path)} - } - return nil, nil -} - -// FileExistingCheck checks that the given file does not already exist. -type FileExistingCheck struct { - Path string - Label string -} - -// Name returns label for individual FileExistingChecks. If not known, will return based on path. -func (fac FileExistingCheck) Name() string { - if fac.Label != "" { - return fac.Label - } - return fmt.Sprintf("FileExisting-%s", strings.Replace(fac.Path, "/", "-", -1)) -} - -// Check validates if the given file already exists. -func (fac FileExistingCheck) Check() (warnings, errorList []error) { - klog.V(1).Infof("validating the existence of file %s", fac.Path) - - if _, err := os.Stat(fac.Path); err != nil { - return nil, []error{errors.Errorf("%s doesn't exist", fac.Path)} - } - return nil, nil -} - -// FileContentCheck checks that the given file contains the string Content. -type FileContentCheck struct { - Path string - Content []byte - Label string -} - -// Name returns label for individual FileContentChecks. If not known, will return based on path. -func (fcc FileContentCheck) Name() string { - if fcc.Label != "" { - return fcc.Label - } - return fmt.Sprintf("FileContent-%s", strings.Replace(fcc.Path, "/", "-", -1)) -} - -// Check validates if the given file contains the given content. -func (fcc FileContentCheck) Check() (warnings, errorList []error) { - klog.V(1).Infof("validating the contents of file %s", fcc.Path) - f, err := os.Open(fcc.Path) - if err != nil { - return nil, []error{errors.Errorf("%s does not exist", fcc.Path)} - } - - lr := io.LimitReader(f, int64(len(fcc.Content))) - defer f.Close() - - buf := &bytes.Buffer{} - _, err = io.Copy(buf, lr) - if err != nil { - return nil, []error{errors.Errorf("%s could not be read", fcc.Path)} - } - - if !bytes.Equal(buf.Bytes(), fcc.Content) { - return nil, []error{errors.Errorf("%s contents are not set to %s", fcc.Path, fcc.Content)} - } - return nil, []error{} - -} - -// InPathCheck checks if the given executable is present in $PATH -type InPathCheck struct { - executable string - mandatory bool - exec utilsexec.Interface - label string - suggestion string -} - -// Name returns label for individual InPathCheck. If not known, will return based on path. -func (ipc InPathCheck) Name() string { - if ipc.label != "" { - return ipc.label - } - return fmt.Sprintf("FileExisting-%s", strings.Replace(ipc.executable, "/", "-", -1)) -} - -// Check validates if the given executable is present in the path. -func (ipc InPathCheck) Check() (warnings, errs []error) { - klog.V(1).Infof("validating the presence of executable %s", ipc.executable) - _, err := ipc.exec.LookPath(ipc.executable) - if err != nil { - if ipc.mandatory { - // Return as an error: - return nil, []error{errors.Errorf("%s not found in system path", ipc.executable)} - } - // Return as a warning: - warningMessage := fmt.Sprintf("%s not found in system path", ipc.executable) - if ipc.suggestion != "" { - warningMessage += fmt.Sprintf("\nSuggestion: %s", ipc.suggestion) - } - return []error{errors.New(warningMessage)}, nil - } - return nil, nil -} - -// HostnameCheck checks if hostname match dns sub domain regex. -// If hostname doesn't match this regex, kubelet will not launch static pods like kube-apiserver/kube-controller-manager and so on. -type HostnameCheck struct { - nodeName string -} - -// Name will return Hostname as name for HostnameCheck -func (HostnameCheck) Name() string { - return "Hostname" -} - -// Check validates if hostname match dns sub domain regex. -// Check hostname length and format -func (hc HostnameCheck) Check() (warnings, errorList []error) { - klog.V(1).Infoln("checking whether the given node name is valid and reachable using net.LookupHost") - for _, msg := range validation.IsQualifiedName(hc.nodeName) { - warnings = append(warnings, errors.Errorf("invalid node name format %q: %s", hc.nodeName, msg)) - } - - addr, err := net.LookupHost(hc.nodeName) - if addr == nil { - warnings = append(warnings, errors.Errorf("hostname \"%s\" could not be reached", hc.nodeName)) - } - if err != nil { - warnings = append(warnings, errors.Wrapf(err, "hostname \"%s\"", hc.nodeName)) - } - return warnings, errorList -} - -// HTTPProxyCheck checks if https connection to specific host is going -// to be done directly or over proxy. If proxy detected, it will return warning. -type HTTPProxyCheck struct { - Proto string - Host string -} - -// Name returns HTTPProxy as name for HTTPProxyCheck -func (hst HTTPProxyCheck) Name() string { - return "HTTPProxy" -} - -// Check validates http connectivity type, direct or via proxy. -func (hst HTTPProxyCheck) Check() (warnings, errorList []error) { - klog.V(1).Infoln("validating if the connectivity type is via proxy or direct") - u := &url.URL{Scheme: hst.Proto, Host: hst.Host} - if utilsnet.IsIPv6String(hst.Host) { - u.Host = net.JoinHostPort(hst.Host, "1234") - } - - req, err := http.NewRequest("GET", u.String(), nil) - if err != nil { - return nil, []error{err} - } - - proxy, err := netutil.SetOldTransportDefaults(&http.Transport{}).Proxy(req) - if err != nil { - return nil, []error{err} - } - if proxy != nil { - return []error{errors.Errorf("Connection to %q uses proxy %q. If that is not intended, adjust your proxy settings", u, proxy)}, nil - } - return nil, nil -} - -// HTTPProxyCIDRCheck checks if https connection to specific subnet is going -// to be done directly or over proxy. If proxy detected, it will return warning. -// Similar to HTTPProxyCheck above, but operates with subnets and uses API -// machinery transport defaults to simulate kube-apiserver accessing cluster -// services and pods. -type HTTPProxyCIDRCheck struct { - Proto string - CIDR string -} - -// Name will return HTTPProxyCIDR as name for HTTPProxyCIDRCheck -func (HTTPProxyCIDRCheck) Name() string { - return "HTTPProxyCIDR" -} - -// Check validates http connectivity to first IP address in the CIDR. -// If it is not directly connected and goes via proxy it will produce warning. -func (subnet HTTPProxyCIDRCheck) Check() (warnings, errorList []error) { - klog.V(1).Infoln("validating http connectivity to first IP address in the CIDR") - if len(subnet.CIDR) == 0 { - return nil, nil - } - - _, cidr, err := net.ParseCIDR(subnet.CIDR) - if err != nil { - return nil, []error{errors.Wrapf(err, "error parsing CIDR %q", subnet.CIDR)} - } - - testIP, err := utilsnet.GetIndexedIP(cidr, 1) - if err != nil { - return nil, []error{errors.Wrapf(err, "unable to get first IP address from the given CIDR (%s)", cidr.String())} - } - - testIPstring := testIP.String() - if len(testIP) == net.IPv6len { - testIPstring = fmt.Sprintf("[%s]:1234", testIP) - } - url := fmt.Sprintf("%s://%s/", subnet.Proto, testIPstring) - - req, err := http.NewRequest("GET", url, nil) - if err != nil { - return nil, []error{err} - } - - // Utilize same transport defaults as it will be used by API server - proxy, err := netutil.SetOldTransportDefaults(&http.Transport{}).Proxy(req) - if err != nil { - return nil, []error{err} - } - if proxy != nil { - return []error{errors.Errorf("connection to %q uses proxy %q. This may lead to malfunctional cluster setup. Make sure that Pod and Services IP ranges specified correctly as exceptions in proxy configuration", subnet.CIDR, proxy)}, nil - } - return nil, nil -} - -// SystemVerificationCheck defines struct used for running the system verification node check in test/e2e_node/system -type SystemVerificationCheck struct { - IsDocker bool -} - -// Name will return SystemVerification as name for SystemVerificationCheck -func (SystemVerificationCheck) Name() string { - return "SystemVerification" -} - -// Check runs all individual checks -func (sysver SystemVerificationCheck) Check() (warnings, errorList []error) { - klog.V(1).Infoln("running all checks") - // Create a buffered writer and choose a quite large value (1M) and suppose the output from the system verification test won't exceed the limit - // Run the system verification check, but write to out buffered writer instead of stdout - bufw := bufio.NewWriterSize(os.Stdout, 1*1024*1024) - reporter := &system.StreamReporter{WriteStream: bufw} - - var errs []error - var warns []error - // All the common validators we'd like to run: - var validators = []system.Validator{ - &system.KernelValidator{Reporter: reporter}} - - // run the docker validator only with docker runtime - if sysver.IsDocker { - validators = append(validators, &system.DockerValidator{Reporter: reporter}) - } - - if runtime.GOOS == "linux" { - //add linux validators - validators = append(validators, - &system.OSValidator{Reporter: reporter}, - &system.CgroupsValidator{Reporter: reporter}) - } - - // Run all validators - for _, v := range validators { - warn, err := v.Validate(system.DefaultSysSpec) - if err != nil { - errs = append(errs, err...) - } - if warn != nil { - warns = append(warns, warn...) - } - } - - if len(errs) != 0 { - // Only print the output from the system verification check if the check failed - fmt.Println("[preflight] The system verification failed. Printing the output from the verification:") - bufw.Flush() - return warns, errs - } - return warns, nil -} - -// KubernetesVersionCheck validates Kubernetes and kubeadm versions -type KubernetesVersionCheck struct { - KubeadmVersion string - KubernetesVersion string -} - -// Name will return KubernetesVersion as name for KubernetesVersionCheck -func (KubernetesVersionCheck) Name() string { - return "KubernetesVersion" -} - -// Check validates Kubernetes and kubeadm versions -func (kubever KubernetesVersionCheck) Check() (warnings, errorList []error) { - klog.V(1).Infoln("validating Kubernetes and kubeadm version") - // Skip this check for "super-custom builds", where apimachinery/the overall codebase version is not set. - if strings.HasPrefix(kubever.KubeadmVersion, "v0.0.0") { - return nil, nil - } - - kadmVersion, err := versionutil.ParseSemantic(kubever.KubeadmVersion) - if err != nil { - return nil, []error{errors.Wrapf(err, "couldn't parse kubeadm version %q", kubever.KubeadmVersion)} - } - - k8sVersion, err := versionutil.ParseSemantic(kubever.KubernetesVersion) - if err != nil { - return nil, []error{errors.Wrapf(err, "couldn't parse Kubernetes version %q", kubever.KubernetesVersion)} - } - - // Checks if k8sVersion greater or equal than the first unsupported versions by current version of kubeadm, - // that is major.minor+1 (all patch and pre-releases versions included) - // NB. in semver patches number is a numeric, while prerelease is a string where numeric identifiers always have lower precedence than non-numeric identifiers. - // thus setting the value to x.y.0-0 we are defining the very first patch - prereleases within x.y minor release. - firstUnsupportedVersion := versionutil.MustParseSemantic(fmt.Sprintf("%d.%d.%s", kadmVersion.Major(), kadmVersion.Minor()+1, "0-0")) - if k8sVersion.AtLeast(firstUnsupportedVersion) { - return []error{errors.Errorf("Kubernetes version is greater than kubeadm version. Please consider to upgrade kubeadm. Kubernetes version: %s. Kubeadm version: %d.%d.x", k8sVersion, kadmVersion.Components()[0], kadmVersion.Components()[1])}, nil - } - - return nil, nil -} - -// KubeletVersionCheck validates installed kubelet version -type KubeletVersionCheck struct { - KubernetesVersion string - exec utilsexec.Interface -} - -// Name will return KubeletVersion as name for KubeletVersionCheck -func (KubeletVersionCheck) Name() string { - return "KubeletVersion" -} - -// Check validates kubelet version. It should be not less than minimal supported version -func (kubever KubeletVersionCheck) Check() (warnings, errorList []error) { - klog.V(1).Infoln("validating kubelet version") - kubeletVersion, err := GetKubeletVersion(kubever.exec) - if err != nil { - return nil, []error{errors.Wrap(err, "couldn't get kubelet version")} - } - if kubeletVersion.LessThan(kubeadmconstants.MinimumKubeletVersion) { - return nil, []error{errors.Errorf("Kubelet version %q is lower than kubeadm can support. Please upgrade kubelet", kubeletVersion)} - } - - if kubever.KubernetesVersion != "" { - k8sVersion, err := versionutil.ParseSemantic(kubever.KubernetesVersion) - if err != nil { - return nil, []error{errors.Wrapf(err, "couldn't parse Kubernetes version %q", kubever.KubernetesVersion)} - } - if kubeletVersion.Major() > k8sVersion.Major() || kubeletVersion.Minor() > k8sVersion.Minor() { - return nil, []error{errors.Errorf("the kubelet version is higher than the control plane version. This is not a supported version skew and may lead to a malfunctional cluster. Kubelet version: %q Control plane version: %q", kubeletVersion, k8sVersion)} - } - } - return nil, nil -} - -// SwapCheck warns if swap is enabled -type SwapCheck struct{} - -// Name will return Swap as name for SwapCheck -func (SwapCheck) Name() string { - return "Swap" -} - -// Check validates whether swap is enabled or not -func (swc SwapCheck) Check() (warnings, errorList []error) { - klog.V(1).Infoln("validating whether swap is enabled or not") - f, err := os.Open("/proc/swaps") - if err != nil { - // /proc/swaps not available, thus no reasons to warn - return nil, nil - } - defer f.Close() - var buf []string - scanner := bufio.NewScanner(f) - for scanner.Scan() { - buf = append(buf, scanner.Text()) - } - if err := scanner.Err(); err != nil { - return nil, []error{errors.Wrap(err, "error parsing /proc/swaps")} - } - - if len(buf) > 1 { - return nil, []error{errors.New("running with swap on is not supported. Please disable swap")} - } - - return nil, nil -} - -// ImagePullCheck will pull container images used by kubeadm -type ImagePullCheck struct { - runtime utilruntime.ContainerRuntime - imageList []string - imagePullPolicy v1.PullPolicy -} - -// Name returns the label for ImagePullCheck -func (ImagePullCheck) Name() string { - return "ImagePull" -} - -// Check pulls images required by kubeadm. This is a mutating check -func (ipc ImagePullCheck) Check() (warnings, errorList []error) { - policy := ipc.imagePullPolicy - klog.V(1).Infof("using image pull policy: %s", policy) - for _, image := range ipc.imageList { - switch policy { - case v1.PullNever: - klog.V(1).Infof("skipping pull of image: %s", image) - continue - case v1.PullIfNotPresent: - ret, err := ipc.runtime.ImageExists(image) - if ret && err == nil { - klog.V(1).Infof("image exists: %s", image) - continue - } - if err != nil { - errorList = append(errorList, errors.Wrapf(err, "failed to check if image %s exists", image)) - } - fallthrough // Proceed with pulling the image if it does not exist - case v1.PullAlways: - klog.V(1).Infof("pulling: %s", image) - if err := ipc.runtime.PullImage(image); err != nil { - errorList = append(errorList, errors.Wrapf(err, "failed to pull image %s", image)) - } - default: - // If the policy is unknown return early with an error - errorList = append(errorList, errors.Errorf("unsupported pull policy %q", policy)) - return warnings, errorList - } - } - return warnings, errorList -} - -// NumCPUCheck checks if current number of CPUs is not less than required -type NumCPUCheck struct { - NumCPU int -} - -// Name returns the label for NumCPUCheck -func (NumCPUCheck) Name() string { - return "NumCPU" -} - -// Check number of CPUs required by kubeadm -func (ncc NumCPUCheck) Check() (warnings, errorList []error) { - numCPU := runtime.NumCPU() - if numCPU < ncc.NumCPU { - errorList = append(errorList, errors.Errorf("the number of available CPUs %d is less than the required %d", numCPU, ncc.NumCPU)) - } - return warnings, errorList -} - -// RunRootCheckOnly initializes checks slice of structs and call RunChecks -func RunRootCheckOnly(ignorePreflightErrors sets.String) error { - checks := []Checker{ - IsPrivilegedUserCheck{}, - } - - return RunChecks(checks, os.Stderr, ignorePreflightErrors) -} - -// RunChecks runs each check, displays it's warnings/errors, and once all -// are processed will exit if any errors occurred. -func RunChecks(checks []Checker, ww io.Writer, ignorePreflightErrors sets.String) error { - var errsBuffer bytes.Buffer - - for _, c := range checks { - name := c.Name() - warnings, errs := c.Check() - - if setHasItemOrAll(ignorePreflightErrors, name) { - // Decrease severity of errors to warnings for this check - warnings = append(warnings, errs...) - errs = []error{} - } - - for _, w := range warnings { - io.WriteString(ww, fmt.Sprintf("\t[WARNING %s]: %v\n", name, w)) - } - for _, i := range errs { - errsBuffer.WriteString(fmt.Sprintf("\t[ERROR %s]: %v\n", name, i.Error())) - } - } - if errsBuffer.Len() > 0 { - return &Error{Msg: errsBuffer.String()} - } - return nil -} - -// setHasItemOrAll is helper function that return true if item is present in the set (case insensitive) or special key 'all' is present -func setHasItemOrAll(s sets.String, item string) bool { - if s.Has("all") || s.Has(strings.ToLower(item)) { - return true - } - return false -} diff --git a/pkg/util/kubernetes/kubeadm/app/preflight/checks_unix.go b/pkg/util/kubernetes/kubeadm/app/preflight/checks_unix.go deleted file mode 100644 index 5a9da3b7c44..00000000000 --- a/pkg/util/kubernetes/kubeadm/app/preflight/checks_unix.go +++ /dev/null @@ -1,35 +0,0 @@ -//go:build !windows -// +build !windows - -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package preflight - -import ( - "os" - - "github.com/pkg/errors" -) - -// Check validates if an user has elevated (root) privileges. -func (ipuc IsPrivilegedUserCheck) Check() (warnings, errorList []error) { - if os.Getuid() != 0 { - return nil, []error{errors.New("user is not running as root")} - } - - return nil, nil -} diff --git a/pkg/util/kubernetes/kubeadm/app/preflight/checks_windows.go b/pkg/util/kubernetes/kubeadm/app/preflight/checks_windows.go deleted file mode 100644 index 5828fb96ff8..00000000000 --- a/pkg/util/kubernetes/kubeadm/app/preflight/checks_windows.go +++ /dev/null @@ -1,51 +0,0 @@ -//go:build windows -// +build windows - -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package preflight - -import ( - "os/user" - - "github.com/pkg/errors" -) - -// The "Well-known SID" of Administrator group -// https://support.microsoft.com/en-us/help/243330/well-known-security-identifiers-in-windows-operating-systems -const administratorSID = "S-1-5-32-544" - -// Check validates if a user has elevated (administrator) privileges. -func (ipuc IsPrivilegedUserCheck) Check() (warnings, errorList []error) { - currUser, err := user.Current() - if err != nil { - return nil, []error{errors.Wrap(err, "cannot get current user")} - } - - groupIds, err := currUser.GroupIds() - if err != nil { - return nil, []error{errors.Wrap(err, "cannot get group IDs for current user")} - } - - for _, sid := range groupIds { - if sid == administratorSID { - return nil, nil - } - } - - return nil, []error{errors.New("user is not running as administrator")} -} diff --git a/pkg/util/kubernetes/kubeadm/app/preflight/utils.go b/pkg/util/kubernetes/kubeadm/app/preflight/utils.go deleted file mode 100644 index c5a2c335fca..00000000000 --- a/pkg/util/kubernetes/kubeadm/app/preflight/utils.go +++ /dev/null @@ -1,44 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package preflight - -import ( - "regexp" - "strings" - - "github.com/pkg/errors" - "k8s.io/apimachinery/pkg/util/version" - utilsexec "k8s.io/utils/exec" -) - -// GetKubeletVersion is helper function that returns version of kubelet available in $PATH -func GetKubeletVersion(execer utilsexec.Interface) (*version.Version, error) { - kubeletVersionRegex := regexp.MustCompile(`^\s*Kubernetes v((0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)([-0-9a-zA-Z_\.+]*)?)\s*$`) - - command := execer.Command("kubelet", "--version") - out, err := command.Output() - if err != nil { - return nil, errors.Wrap(err, "cannot execute 'kubelet --version'") - } - - cleanOutput := strings.TrimSpace(string(out)) - subs := kubeletVersionRegex.FindAllStringSubmatch(cleanOutput, -1) - if len(subs) != 1 || len(subs[0]) < 2 { - return nil, errors.Errorf("Unable to parse output from Kubelet: %q", cleanOutput) - } - return version.ParseSemantic(subs[0][1]) -} diff --git a/pkg/util/kubernetes/kubeadm/app/preflight/utils_test.go b/pkg/util/kubernetes/kubeadm/app/preflight/utils_test.go deleted file mode 100644 index ad7944e3085..00000000000 --- a/pkg/util/kubernetes/kubeadm/app/preflight/utils_test.go +++ /dev/null @@ -1,64 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package preflight - -import ( - "testing" - - "github.com/pkg/errors" - utilsexec "k8s.io/utils/exec" - fakeexec "k8s.io/utils/exec/testing" -) - -func TestGetKubeletVersion(t *testing.T) { - cases := []struct { - output string - expected string - err error - valid bool - }{ - {"Kubernetes v1.7.0", "1.7.0", nil, true}, - {"Kubernetes v1.8.0-alpha.2.1231+afabd012389d53a", "1.8.0-alpha.2.1231+afabd012389d53a", nil, true}, - {"something-invalid", "", nil, false}, - {"command not found", "", errors.New("kubelet not found"), false}, - {"", "", nil, false}, - } - - for _, tc := range cases { - t.Run(tc.output, func(t *testing.T) { - fcmd := fakeexec.FakeCmd{ - OutputScript: []fakeexec.FakeAction{ - func() ([]byte, []byte, error) { return []byte(tc.output), nil, tc.err }, - }, - } - fexec := &fakeexec.FakeExec{ - CommandScript: []fakeexec.FakeCommandAction{ - func(cmd string, args ...string) utilsexec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, - }, - } - ver, err := GetKubeletVersion(fexec) - switch { - case err != nil && tc.valid: - t.Errorf("GetKubeletVersion: unexpected error for %q. Error: %v", tc.output, err) - case err == nil && !tc.valid: - t.Errorf("GetKubeletVersion: error expected for key %q, but result is %q", tc.output, ver) - case ver != nil && ver.String() != tc.expected: - t.Errorf("GetKubeletVersion: unexpected version result for key %q. Expected: %q Actual: %q", tc.output, tc.expected, ver) - } - }) - } -} diff --git a/pkg/util/kubernetes/kubeadm/app/util/apiclient/idempotency.go b/pkg/util/kubernetes/kubeadm/app/util/apiclient/idempotency.go index 98a7f0e7e0e..cf576c708b0 100644 --- a/pkg/util/kubernetes/kubeadm/app/util/apiclient/idempotency.go +++ b/pkg/util/kubernetes/kubeadm/app/util/apiclient/idempotency.go @@ -18,16 +18,12 @@ package apiclient import ( "context" - "encoding/json" - "fmt" "github.com/pkg/errors" v1 "k8s.io/api/core/v1" rbac "k8s.io/api/rbac/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" - "k8s.io/apimachinery/pkg/util/strategicpatch" "k8s.io/apimachinery/pkg/util/wait" clientset "k8s.io/client-go/kubernetes" clientsetretry "k8s.io/client-go/util/retry" @@ -54,54 +50,6 @@ func CreateOrUpdateConfigMap(client clientset.Interface, cm *v1.ConfigMap) error return nil } -// CreateOrMutateConfigMap tries to create the ConfigMap provided as cm. If the resource exists already, the latest version will be fetched from -// the cluster and mutator callback will be called on it, then an Update of the mutated ConfigMap will be performed. This function is resilient -// to conflicts, and a retry will be issued if the ConfigMap was modified on the server between the refresh and the update (while the mutation was -// taking place) -func CreateOrMutateConfigMap(client clientset.Interface, cm *v1.ConfigMap, mutator ConfigMapMutator) error { - var lastError error - err := wait.PollImmediate(constants.APICallRetryInterval, constants.APICallWithWriteTimeout, func() (bool, error) { - if _, err := client.CoreV1().ConfigMaps(cm.ObjectMeta.Namespace).Create(context.TODO(), cm, metav1.CreateOptions{}); err != nil { - lastError = err - if apierrors.IsAlreadyExists(err) { - lastError = MutateConfigMap(client, metav1.ObjectMeta{Namespace: cm.ObjectMeta.Namespace, Name: cm.ObjectMeta.Name}, mutator) - return lastError == nil, nil - } - return false, nil - } - return true, nil - }) - if err == nil { - return nil - } - return lastError -} - -// MutateConfigMap takes a ConfigMap Object Meta (namespace and name), retrieves the resource from the server and tries to mutate it -// by calling to the mutator callback, then an Update of the mutated ConfigMap will be performed. This function is resilient -// to conflicts, and a retry will be issued if the ConfigMap was modified on the server between the refresh and the update (while the mutation was -// taking place). -func MutateConfigMap(client clientset.Interface, meta metav1.ObjectMeta, mutator ConfigMapMutator) error { - var lastError error - err := wait.PollImmediate(constants.APICallRetryInterval, constants.APICallWithWriteTimeout, func() (bool, error) { - configMap, err := client.CoreV1().ConfigMaps(meta.Namespace).Get(context.TODO(), meta.Name, metav1.GetOptions{}) - if err != nil { - lastError = err - return false, nil - } - if err = mutator(configMap); err != nil { - lastError = errors.Wrap(err, "unable to mutate ConfigMap") - return false, nil - } - _, lastError = client.CoreV1().ConfigMaps(configMap.ObjectMeta.Namespace).Update(context.TODO(), configMap, metav1.UpdateOptions{}) - return lastError == nil, nil - }) - if err == nil { - return nil - } - return lastError -} - // CreateOrUpdateSecret creates a Secret if the target resource doesn't exist. If the resource exists already, this function will update the resource instead. func CreateOrUpdateSecret(client clientset.Interface, secret *v1.Secret) error { if _, err := client.CoreV1().Secrets(secret.ObjectMeta.Namespace).Create(context.TODO(), secret, metav1.CreateOptions{}); err != nil { @@ -162,56 +110,6 @@ func CreateOrUpdateRoleBinding(client clientset.Interface, roleBinding *rbac.Rol return lastError } -// PatchNodeOnce executes patchFn on the node object found by the node name. -// This is a condition function meant to be used with wait.Poll. false, nil -// implies it is safe to try again, an error indicates no more tries should be -// made and true indicates success. -func PatchNodeOnce(client clientset.Interface, nodeName string, patchFn func(*v1.Node)) func() (bool, error) { - return func() (bool, error) { - // First get the node object - n, err := client.CoreV1().Nodes().Get(context.TODO(), nodeName, metav1.GetOptions{}) - if err != nil { - // TODO this should only be for timeouts - return false, nil - } - - // The node may appear to have no labels at first, - // so we wait for it to get hostname label. - if _, found := n.ObjectMeta.Labels[v1.LabelHostname]; !found { - return false, nil - } - - oldData, err := json.Marshal(n) - if err != nil { - return false, errors.Wrapf(err, "failed to marshal unmodified node %q into JSON", n.Name) - } - - // Execute the mutating function - patchFn(n) - - newData, err := json.Marshal(n) - if err != nil { - return false, errors.Wrapf(err, "failed to marshal modified node %q into JSON", n.Name) - } - - patchBytes, err := strategicpatch.CreateTwoWayMergePatch(oldData, newData, v1.Node{}) - if err != nil { - return false, errors.Wrap(err, "failed to create two way merge patch") - } - - if _, err := client.CoreV1().Nodes().Patch(context.TODO(), n.Name, types.StrategicMergePatchType, patchBytes, metav1.PatchOptions{}); err != nil { - // TODO also check for timeouts - if apierrors.IsConflict(err) { - fmt.Println("Temporarily unable to update node metadata due to conflict (will retry)") - return false, nil - } - return false, errors.Wrapf(err, "error patching node %q through apiserver", n.Name) - } - - return true, nil - } -} - // GetConfigMapWithRetry tries to retrieve a ConfigMap using the given client, // retrying if we get an unexpected error. // diff --git a/pkg/util/kubernetes/kubeadm/app/util/apiclient/idempotency_test.go b/pkg/util/kubernetes/kubeadm/app/util/apiclient/idempotency_test.go deleted file mode 100644 index 13879cf0edd..00000000000 --- a/pkg/util/kubernetes/kubeadm/app/util/apiclient/idempotency_test.go +++ /dev/null @@ -1,182 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package apiclient - -import ( - "context" - "testing" - - "github.com/pkg/errors" - v1 "k8s.io/api/core/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/client-go/kubernetes/fake" - core "k8s.io/client-go/testing" -) - -const configMapName = "configmap" - -func TestPatchNodeNonErrorCases(t *testing.T) { - testcases := []struct { - name string - lookupName string - node v1.Node - success bool - }{ - { - name: "simple update", - lookupName: "testnode", - node: v1.Node{ - ObjectMeta: metav1.ObjectMeta{ - Name: "testnode", - Labels: map[string]string{v1.LabelHostname: ""}, - }, - }, - success: true, - }, - { - name: "node does not exist", - lookupName: "whale", - success: false, - }, - { - name: "node not labelled yet", - lookupName: "robin", - node: v1.Node{ - ObjectMeta: metav1.ObjectMeta{ - Name: "robin", - }, - }, - success: false, - }, - } - - for _, tc := range testcases { - t.Run(tc.name, func(t *testing.T) { - client := fake.NewSimpleClientset() - _, err := client.CoreV1().Nodes().Create(context.TODO(), &tc.node, metav1.CreateOptions{}) - if err != nil { - t.Fatalf("failed to create node to fake client: %v", err) - } - conditionFunction := PatchNodeOnce(client, tc.lookupName, func(node *v1.Node) { - node.Annotations = map[string]string{ - "updatedBy": "test", - } - }) - success, err := conditionFunction() - if err != nil { - t.Fatalf("did not expect error: %v", err) - } - if success != tc.success { - t.Fatalf("expected %v got %v", tc.success, success) - } - }) - } -} - -func TestCreateOrMutateConfigMap(t *testing.T) { - client := fake.NewSimpleClientset() - err := CreateOrMutateConfigMap(client, &v1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Name: configMapName, - Namespace: metav1.NamespaceSystem, - }, - Data: map[string]string{ - "key": "some-value", - }, - }, func(cm *v1.ConfigMap) error { - t.Fatal("mutate should not have been called, since the ConfigMap should have been created instead of mutated") - return nil - }) - if err != nil { - t.Fatalf("error creating ConfigMap: %v", err) - } - _, err = client.CoreV1().ConfigMaps(metav1.NamespaceSystem).Get(context.TODO(), configMapName, metav1.GetOptions{}) - if err != nil { - t.Fatalf("error retrieving ConfigMap: %v", err) - } -} - -func createClientAndConfigMap(t *testing.T) *fake.Clientset { - client := fake.NewSimpleClientset() - _, err := client.CoreV1().ConfigMaps(metav1.NamespaceSystem).Create(context.TODO(), &v1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Name: configMapName, - Namespace: metav1.NamespaceSystem, - }, - Data: map[string]string{ - "key": "some-value", - }, - }, metav1.CreateOptions{}) - if err != nil { - t.Fatalf("error creating ConfigMap: %v", err) - } - return client -} - -func TestMutateConfigMap(t *testing.T) { - client := createClientAndConfigMap(t) - - err := MutateConfigMap(client, metav1.ObjectMeta{ - Name: configMapName, - Namespace: metav1.NamespaceSystem, - }, func(cm *v1.ConfigMap) error { - cm.Data["key"] = "some-other-value" - return nil - }) - if err != nil { - t.Fatalf("error mutating regular ConfigMap: %v", err) - } - - cm, _ := client.CoreV1().ConfigMaps(metav1.NamespaceSystem).Get(context.TODO(), configMapName, metav1.GetOptions{}) - if cm.Data["key"] != "some-other-value" { - t.Fatalf("ConfigMap mutation was invalid, has: %q", cm.Data["key"]) - } -} - -func TestMutateConfigMapWithConflict(t *testing.T) { - client := createClientAndConfigMap(t) - - // Mimic that the first 5 updates of the ConfigMap returns a conflict, whereas the sixth update - // succeeds - conflict := 5 - client.PrependReactor("update", "configmaps", func(action core.Action) (bool, runtime.Object, error) { - update := action.(core.UpdateAction) - if conflict > 0 { - conflict-- - return true, update.GetObject(), apierrors.NewConflict(action.GetResource().GroupResource(), configMapName, errors.New("conflict")) - } - return false, update.GetObject(), nil - }) - - err := MutateConfigMap(client, metav1.ObjectMeta{ - Name: configMapName, - Namespace: metav1.NamespaceSystem, - }, func(cm *v1.ConfigMap) error { - cm.Data["key"] = "some-other-value" - return nil - }) - if err != nil { - t.Fatalf("error mutating conflicting ConfigMap: %v", err) - } - - cm, _ := client.CoreV1().ConfigMaps(metav1.NamespaceSystem).Get(context.TODO(), configMapName, metav1.GetOptions{}) - if cm.Data["key"] != "some-other-value" { - t.Fatalf("ConfigMap mutation with conflict was invalid, has: %q", cm.Data["key"]) - } -} diff --git a/pkg/util/kubernetes/kubeadm/app/util/apiclient/wait.go b/pkg/util/kubernetes/kubeadm/app/util/apiclient/wait.go index 5b27cf2236e..c222d03b7eb 100644 --- a/pkg/util/kubernetes/kubeadm/app/util/apiclient/wait.go +++ b/pkg/util/kubernetes/kubeadm/app/util/apiclient/wait.go @@ -17,240 +17,11 @@ limitations under the License. package apiclient import ( - "context" - "fmt" - "io" - "net/http" "time" - "github.com/pkg/errors" - v1 "k8s.io/api/core/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - netutil "k8s.io/apimachinery/pkg/util/net" "k8s.io/apimachinery/pkg/util/wait" - clientset "k8s.io/client-go/kubernetes" - - kubeadmconstants "github.com/openyurtio/openyurt/pkg/util/kubernetes/kubeadm/app/constants" ) -// Waiter is an interface for waiting for criteria in Kubernetes to happen -type Waiter interface { - // WaitForAPI waits for the API Server's /healthz endpoint to become "ok" - WaitForAPI() error - // WaitForPodsWithLabel waits for Pods in the kube-system namespace to become Ready - WaitForPodsWithLabel(kvLabel string) error - // WaitForPodToDisappear waits for the given Pod in the kube-system namespace to be deleted - WaitForPodToDisappear(staticPodName string) error - // WaitForStaticPodSingleHash fetches sha256 hash for the control plane static pod - WaitForStaticPodSingleHash(nodeName string, component string) (string, error) - // WaitForStaticPodHashChange waits for the given static pod component's static pod hash to get updated. - // By doing that we can be sure that the kubelet has restarted the given Static Pod - WaitForStaticPodHashChange(nodeName, component, previousHash string) error - // WaitForStaticPodControlPlaneHashes fetches sha256 hashes for the control plane static pods - WaitForStaticPodControlPlaneHashes(nodeName string) (map[string]string, error) - // WaitForHealthyKubelet blocks until the kubelet /healthz endpoint returns 'ok' - WaitForHealthyKubelet(initialTimeout time.Duration, healthzEndpoint string) error - // WaitForKubeletAndFunc is a wrapper for WaitForHealthyKubelet that also blocks for a function - WaitForKubeletAndFunc(f func() error) error - // SetTimeout adjusts the timeout to the specified duration - SetTimeout(timeout time.Duration) -} - -// KubeWaiter is an implementation of Waiter that is backed by a Kubernetes client -type KubeWaiter struct { - client clientset.Interface - timeout time.Duration - writer io.Writer -} - -// NewKubeWaiter returns a new Waiter object that talks to the given Kubernetes cluster -func NewKubeWaiter(client clientset.Interface, timeout time.Duration, writer io.Writer) Waiter { - return &KubeWaiter{ - client: client, - timeout: timeout, - writer: writer, - } -} - -// WaitForAPI waits for the API Server's /healthz endpoint to report "ok" -func (w *KubeWaiter) WaitForAPI() error { - start := time.Now() - return wait.PollImmediate(kubeadmconstants.APICallRetryInterval, w.timeout, func() (bool, error) { - healthStatus := 0 - w.client.Discovery().RESTClient().Get().AbsPath("/healthz").Do(context.TODO()).StatusCode(&healthStatus) - if healthStatus != http.StatusOK { - return false, nil - } - - fmt.Printf("[apiclient] All control plane components are healthy after %f seconds\n", time.Since(start).Seconds()) - return true, nil - }) -} - -// WaitForPodsWithLabel will lookup pods with the given label and wait until they are all -// reporting status as running. -func (w *KubeWaiter) WaitForPodsWithLabel(kvLabel string) error { - - lastKnownPodNumber := -1 - return wait.PollImmediate(kubeadmconstants.APICallRetryInterval, w.timeout, func() (bool, error) { - listOpts := metav1.ListOptions{LabelSelector: kvLabel} - pods, err := w.client.CoreV1().Pods(metav1.NamespaceSystem).List(context.TODO(), listOpts) - if err != nil { - fmt.Fprintf(w.writer, "[apiclient] Error getting Pods with label selector %q [%v]\n", kvLabel, err) - return false, nil - } - - if lastKnownPodNumber != len(pods.Items) { - fmt.Fprintf(w.writer, "[apiclient] Found %d Pods for label selector %s\n", len(pods.Items), kvLabel) - lastKnownPodNumber = len(pods.Items) - } - - if len(pods.Items) == 0 { - return false, nil - } - - for _, pod := range pods.Items { - if pod.Status.Phase != v1.PodRunning { - return false, nil - } - } - - return true, nil - }) -} - -// WaitForPodToDisappear blocks until it timeouts or gets a "NotFound" response from the API Server when getting the Static Pod in question -func (w *KubeWaiter) WaitForPodToDisappear(podName string) error { - return wait.PollImmediate(kubeadmconstants.APICallRetryInterval, w.timeout, func() (bool, error) { - _, err := w.client.CoreV1().Pods(metav1.NamespaceSystem).Get(context.TODO(), podName, metav1.GetOptions{}) - if apierrors.IsNotFound(err) { - fmt.Printf("[apiclient] The old Pod %q is now removed (which is desired)\n", podName) - return true, nil - } - return false, nil - }) -} - -// WaitForHealthyKubelet blocks until the kubelet /healthz endpoint returns 'ok' -func (w *KubeWaiter) WaitForHealthyKubelet(initialTimeout time.Duration, healthzEndpoint string) error { - time.Sleep(initialTimeout) - fmt.Printf("[kubelet-check] Initial timeout of %v passed.\n", initialTimeout) - return TryRunCommand(func() error { - client := &http.Client{Transport: netutil.SetOldTransportDefaults(&http.Transport{})} - resp, err := client.Get(healthzEndpoint) - if err != nil { - fmt.Println("[kubelet-check] It seems like the kubelet isn't running or healthy.") - fmt.Printf("[kubelet-check] The HTTP call equal to 'curl -sSL %s' failed with error: %v.\n", healthzEndpoint, err) - return err - } - defer resp.Body.Close() - if resp.StatusCode != http.StatusOK { - fmt.Println("[kubelet-check] It seems like the kubelet isn't running or healthy.") - fmt.Printf("[kubelet-check] The HTTP call equal to 'curl -sSL %s' returned HTTP code %d\n", healthzEndpoint, resp.StatusCode) - return errors.New("the kubelet healthz endpoint is unhealthy") - } - return nil - }, 5) // a failureThreshold of five means waiting for a total of 155 seconds -} - -// WaitForKubeletAndFunc waits primarily for the function f to execute, even though it might take some time. If that takes a long time, and the kubelet -// /healthz continuously are unhealthy, kubeadm will error out after a period of exponential backoff -func (w *KubeWaiter) WaitForKubeletAndFunc(f func() error) error { - errorChan := make(chan error, 1) - - go func(errC chan error, waiter Waiter) { - if err := waiter.WaitForHealthyKubelet(40*time.Second, fmt.Sprintf("http://localhost:%d/healthz", kubeadmconstants.KubeletHealthzPort)); err != nil { - errC <- err - } - }(errorChan, w) - - go func(errC chan error, waiter Waiter) { - // This main goroutine sends whatever the f function returns (error or not) to the channel - // This in order to continue on success (nil error), or just fail if the function returns an error - errC <- f() - }(errorChan, w) - - // This call is blocking until one of the goroutines sends to errorChan - return <-errorChan -} - -// SetTimeout adjusts the timeout to the specified duration -func (w *KubeWaiter) SetTimeout(timeout time.Duration) { - w.timeout = timeout -} - -// WaitForStaticPodControlPlaneHashes blocks until it timeouts or gets a hash map for all components and their Static Pods -func (w *KubeWaiter) WaitForStaticPodControlPlaneHashes(nodeName string) (map[string]string, error) { - - componentHash := "" - var err error - mirrorPodHashes := map[string]string{} - for _, component := range kubeadmconstants.ControlPlaneComponents { - err = wait.PollImmediate(kubeadmconstants.APICallRetryInterval, w.timeout, func() (bool, error) { - componentHash, err = getStaticPodSingleHash(w.client, nodeName, component) - if err != nil { - return false, nil - } - return true, nil - }) - if err != nil { - return nil, err - } - mirrorPodHashes[component] = componentHash - } - - return mirrorPodHashes, nil -} - -// WaitForStaticPodSingleHash blocks until it timeouts or gets a hash for a single component and its Static Pod -func (w *KubeWaiter) WaitForStaticPodSingleHash(nodeName string, component string) (string, error) { - - componentPodHash := "" - var err error - err = wait.PollImmediate(kubeadmconstants.APICallRetryInterval, w.timeout, func() (bool, error) { - componentPodHash, err = getStaticPodSingleHash(w.client, nodeName, component) - if err != nil { - return false, nil - } - return true, nil - }) - - return componentPodHash, err -} - -// WaitForStaticPodHashChange blocks until it timeouts or notices that the Mirror Pod (for the Static Pod, respectively) has changed -// This implicitly means this function blocks until the kubelet has restarted the Static Pod in question -func (w *KubeWaiter) WaitForStaticPodHashChange(nodeName, component, previousHash string) error { - return wait.PollImmediate(kubeadmconstants.APICallRetryInterval, w.timeout, func() (bool, error) { - - hash, err := getStaticPodSingleHash(w.client, nodeName, component) - if err != nil { - return false, nil - } - // We should continue polling until the UID changes - if hash == previousHash { - return false, nil - } - - return true, nil - }) -} - -// getStaticPodSingleHash computes hashes for a single Static Pod resource -func getStaticPodSingleHash(client clientset.Interface, nodeName string, component string) (string, error) { - - staticPodName := fmt.Sprintf("%s-%s", component, nodeName) - staticPod, err := client.CoreV1().Pods(metav1.NamespaceSystem).Get(context.TODO(), staticPodName, metav1.GetOptions{}) - if err != nil { - return "", err - } - - staticPodHash := staticPod.Annotations["kubernetes.io/config.hash"] - fmt.Printf("Static pod: %s hash: %s\n", staticPodName, staticPodHash) - return staticPodHash, nil -} - // TryRunCommand runs a function a maximum of failureThreshold times, and retries on error. If failureThreshold is hit; the last error is returned func TryRunCommand(f func() error, failureThreshold int) error { backoff := wait.Backoff{ diff --git a/pkg/util/kubernetes/kubeadm/app/util/arguments.go b/pkg/util/kubernetes/kubeadm/app/util/arguments.go deleted file mode 100644 index 6b0fa5d7c7b..00000000000 --- a/pkg/util/kubernetes/kubeadm/app/util/arguments.go +++ /dev/null @@ -1,98 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package util - -import ( - "fmt" - "sort" - "strings" - - "github.com/pkg/errors" -) - -// BuildArgumentListFromMap takes two string-string maps, one with the base arguments and one -// with optional override arguments. In the return list override arguments will precede base -// arguments -func BuildArgumentListFromMap(baseArguments map[string]string, overrideArguments map[string]string) []string { - var command []string - var keys []string - - argsMap := make(map[string]string) - - for k, v := range baseArguments { - argsMap[k] = v - } - - for k, v := range overrideArguments { - argsMap[k] = v - } - - for k := range argsMap { - keys = append(keys, k) - } - - sort.Strings(keys) - for _, k := range keys { - command = append(command, fmt.Sprintf("--%s=%s", k, argsMap[k])) - } - - return command -} - -// ParseArgumentListToMap parses a CLI argument list in the form "--foo=bar" to a string-string map -func ParseArgumentListToMap(arguments []string) map[string]string { - resultingMap := map[string]string{} - for i, arg := range arguments { - key, val, err := parseArgument(arg) - - // Ignore if the first argument doesn't satisfy the criteria, it's most often the binary name - // Warn in all other cases, but don't error out. This can happen only if the user has edited the argument list by hand, so they might know what they are doing - if err != nil { - if i != 0 { - fmt.Printf("[kubeadm] WARNING: The component argument %q could not be parsed correctly. The argument must be of the form %q. Skipping...\n", arg, "--") - } - continue - } - - resultingMap[key] = val - } - return resultingMap -} - -// parseArgument parses the argument "--foo=bar" to "foo" and "bar" -func parseArgument(arg string) (string, string, error) { - if !strings.HasPrefix(arg, "--") { - return "", "", errors.New("the argument should start with '--'") - } - if !strings.Contains(arg, "=") { - return "", "", errors.New("the argument should have a '=' between the flag and the value") - } - // Remove the starting -- - arg = strings.TrimPrefix(arg, "--") - // Split the string on =. Return only two substrings, since we want only key/value, but the value can include '=' as well - keyvalSlice := strings.SplitN(arg, "=", 2) - - // Make sure both a key and value is present - if len(keyvalSlice) != 2 { - return "", "", errors.New("the argument must have both a key and a value") - } - if len(keyvalSlice[0]) == 0 { - return "", "", errors.New("the argument must have a key") - } - - return keyvalSlice[0], keyvalSlice[1], nil -} diff --git a/pkg/util/kubernetes/kubeadm/app/util/initsystem/initsystem.go b/pkg/util/kubernetes/kubeadm/app/util/initsystem/initsystem.go deleted file mode 100644 index 48e152f0a9b..00000000000 --- a/pkg/util/kubernetes/kubeadm/app/util/initsystem/initsystem.go +++ /dev/null @@ -1,41 +0,0 @@ -/* -Copyright 2016 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package initsystem - -// InitSystem is the interface that describe behaviors of an init system -type InitSystem interface { - // return a string describing how to enable a service - EnableCommand(service string) string - - // ServiceStart tries to start a specific service - ServiceStart(service string) error - - // ServiceStop tries to stop a specific service - ServiceStop(service string) error - - // ServiceRestart tries to reload the environment and restart the specific service - ServiceRestart(service string) error - - // ServiceExists ensures the service is defined for this init system. - ServiceExists(service string) bool - - // ServiceIsEnabled ensures the service is enabled to start on each boot. - ServiceIsEnabled(service string) bool - - // ServiceIsActive ensures the service is running, or attempting to run. (crash looping in the case of kubelet) - ServiceIsActive(service string) bool -} diff --git a/pkg/util/kubernetes/kubeadm/app/util/initsystem/initsystem_unix.go b/pkg/util/kubernetes/kubeadm/app/util/initsystem/initsystem_unix.go deleted file mode 100644 index 7bd4c4f2f65..00000000000 --- a/pkg/util/kubernetes/kubeadm/app/util/initsystem/initsystem_unix.go +++ /dev/null @@ -1,165 +0,0 @@ -//go:build !windows -// +build !windows - -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package initsystem - -import ( - "fmt" - "os/exec" - "strings" -) - -// OpenRCInitSystem defines openrc -type OpenRCInitSystem struct{} - -// ServiceStart tries to start a specific service -func (openrc OpenRCInitSystem) ServiceStart(service string) error { - args := []string{service, "start"} - return exec.Command("rc-service", args...).Run() -} - -// ServiceStop tries to stop a specific service -func (openrc OpenRCInitSystem) ServiceStop(service string) error { - args := []string{service, "stop"} - return exec.Command("rc-service", args...).Run() -} - -// ServiceRestart tries to reload the environment and restart the specific service -func (openrc OpenRCInitSystem) ServiceRestart(service string) error { - args := []string{service, "restart"} - return exec.Command("rc-service", args...).Run() -} - -// ServiceExists ensures the service is defined for this init system. -// openrc writes to stderr if a service is not found or not enabled -// this is in contrast to systemd which only writes to stdout. -// Hence, we use the Combinedoutput, and ignore the error. -func (openrc OpenRCInitSystem) ServiceExists(service string) bool { - args := []string{service, "status"} - outBytes, _ := exec.Command("rc-service", args...).CombinedOutput() - return !strings.Contains(string(outBytes), "does not exist") -} - -// ServiceIsEnabled ensures the service is enabled to start on each boot. -func (openrc OpenRCInitSystem) ServiceIsEnabled(service string) bool { - args := []string{"show", "default"} - outBytes, _ := exec.Command("rc-update", args...).Output() - return strings.Contains(string(outBytes), service) -} - -// ServiceIsActive ensures the service is running, or attempting to run. (crash looping in the case of kubelet) -func (openrc OpenRCInitSystem) ServiceIsActive(service string) bool { - args := []string{service, "status"} - outBytes, _ := exec.Command("rc-service", args...).CombinedOutput() - outStr := string(outBytes) - return !strings.Contains(outStr, "stopped") && !strings.Contains(outStr, "does not exist") -} - -// EnableCommand return a string describing how to enable a service -func (openrc OpenRCInitSystem) EnableCommand(service string) string { - return fmt.Sprintf("rc-update add %s default", service) -} - -// SystemdInitSystem defines systemd -type SystemdInitSystem struct{} - -// EnableCommand return a string describing how to enable a service -func (sysd SystemdInitSystem) EnableCommand(service string) string { - return fmt.Sprintf("systemctl enable %s.service", service) -} - -// reloadSystemd reloads the systemd daemon -func (sysd SystemdInitSystem) reloadSystemd() error { - if err := exec.Command("systemctl", "daemon-reload").Run(); err != nil { - return fmt.Errorf("failed to reload systemd: %v", err) - } - return nil -} - -// ServiceStart tries to start a specific service -func (sysd SystemdInitSystem) ServiceStart(service string) error { - // Before we try to start any service, make sure that systemd is ready - if err := sysd.reloadSystemd(); err != nil { - return err - } - args := []string{"start", service} - return exec.Command("systemctl", args...).Run() -} - -// ServiceRestart tries to reload the environment and restart the specific service -func (sysd SystemdInitSystem) ServiceRestart(service string) error { - // Before we try to restart any service, make sure that systemd is ready - if err := sysd.reloadSystemd(); err != nil { - return err - } - args := []string{"restart", service} - return exec.Command("systemctl", args...).Run() -} - -// ServiceStop tries to stop a specific service -func (sysd SystemdInitSystem) ServiceStop(service string) error { - args := []string{"stop", service} - return exec.Command("systemctl", args...).Run() -} - -// ServiceExists ensures the service is defined for this init system. -func (sysd SystemdInitSystem) ServiceExists(service string) bool { - args := []string{"status", service} - outBytes, _ := exec.Command("systemctl", args...).Output() - output := string(outBytes) - return !strings.Contains(output, "Loaded: not-found") -} - -// ServiceIsEnabled ensures the service is enabled to start on each boot. -func (sysd SystemdInitSystem) ServiceIsEnabled(service string) bool { - args := []string{"is-enabled", service} - err := exec.Command("systemctl", args...).Run() - return err == nil -} - -// ServiceIsActive will check is the service is "active". In the case of -// crash looping services (kubelet in our case) status will return as -// "activating", so we will consider this active as well. -func (sysd SystemdInitSystem) ServiceIsActive(service string) bool { - args := []string{"is-active", service} - // Ignoring error here, command returns non-0 if in "activating" status: - outBytes, _ := exec.Command("systemctl", args...).Output() - output := strings.TrimSpace(string(outBytes)) - if output == "active" || output == "activating" { - return true - } - return false -} - -// GetInitSystem returns an InitSystem for the current system, or nil -// if we cannot detect a supported init system. -// This indicates we will skip init system checks, not an error. -func GetInitSystem() (InitSystem, error) { - // Assume existence of systemctl in path implies this is a systemd system: - _, err := exec.LookPath("systemctl") - if err == nil { - return &SystemdInitSystem{}, nil - } - _, err = exec.LookPath("openrc") - if err == nil { - return &OpenRCInitSystem{}, nil - } - - return nil, fmt.Errorf("no supported init system detected, skipping checking for services") -} diff --git a/pkg/util/kubernetes/kubeadm/app/util/initsystem/initsystem_windows.go b/pkg/util/kubernetes/kubeadm/app/util/initsystem/initsystem_windows.go deleted file mode 100644 index 394272ddcb3..00000000000 --- a/pkg/util/kubernetes/kubeadm/app/util/initsystem/initsystem_windows.go +++ /dev/null @@ -1,245 +0,0 @@ -// +build windows - -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package initsystem - -import ( - "fmt" - "time" - - "golang.org/x/sys/windows/svc" - "golang.org/x/sys/windows/svc/mgr" -) - -// WindowsInitSystem is the windows implementation of InitSystem -type WindowsInitSystem struct{} - -// EnableCommand return a string describing how to enable a service -func (sysd WindowsInitSystem) EnableCommand(service string) string { - return fmt.Sprintf("Set-Service '%s' -StartupType Automatic", service) -} - -// ServiceStart tries to start a specific service -// Following Windows documentation: https://docs.microsoft.com/en-us/windows/desktop/Services/starting-a-service -func (sysd WindowsInitSystem) ServiceStart(service string) error { - m, err := mgr.Connect() - if err != nil { - return err - } - defer m.Disconnect() - - s, err := m.OpenService(service) - if err != nil { - return fmt.Errorf("could not access service %s: %v", service, err) - } - defer s.Close() - - // Check if service is already started - status, err := s.Query() - if err != nil { - return fmt.Errorf("could not query service %s: %v", service, err) - } - - if status.State != svc.Stopped && status.State != svc.StopPending { - return nil - } - - timeout := time.Now().Add(10 * time.Second) - for status.State != svc.Stopped { - if timeout.Before(time.Now()) { - return fmt.Errorf("timeout waiting for %s service to stop", service) - } - time.Sleep(300 * time.Millisecond) - status, err = s.Query() - if err != nil { - return fmt.Errorf("could not retrieve %s service status: %v", service, err) - } - } - - // Start the service - err = s.Start("is", "manual-started") - if err != nil { - return fmt.Errorf("could not start service %s: %v", service, err) - } - - // Check that the start was successful - status, err = s.Query() - if err != nil { - return fmt.Errorf("could not query service %s: %v", service, err) - } - timeout = time.Now().Add(10 * time.Second) - for status.State != svc.Running { - if timeout.Before(time.Now()) { - return fmt.Errorf("timeout waiting for %s service to start", service) - } - time.Sleep(300 * time.Millisecond) - status, err = s.Query() - if err != nil { - return fmt.Errorf("could not retrieve %s service status: %v", service, err) - } - } - return nil -} - -// ServiceRestart tries to reload the environment and restart the specific service -func (sysd WindowsInitSystem) ServiceRestart(service string) error { - if err := sysd.ServiceStop(service); err != nil { - return fmt.Errorf("couldn't stop service %s: %v", service, err) - } - if err := sysd.ServiceStart(service); err != nil { - return fmt.Errorf("couldn't start service %s: %v", service, err) - } - - return nil -} - -// ServiceStop tries to stop a specific service -// Following Windows documentation: https://docs.microsoft.com/en-us/windows/desktop/Services/stopping-a-service -func (sysd WindowsInitSystem) ServiceStop(service string) error { - m, err := mgr.Connect() - if err != nil { - return err - } - defer m.Disconnect() - - s, err := m.OpenService(service) - if err != nil { - return fmt.Errorf("could not access service %s: %v", service, err) - } - defer s.Close() - - // Check if service is already stopped - status, err := s.Query() - if err != nil { - return fmt.Errorf("could not query service %s: %v", service, err) - } - - if status.State == svc.Stopped { - return nil - } - - // If StopPending, check that service eventually stops - if status.State == svc.StopPending { - timeout := time.Now().Add(10 * time.Second) - for status.State != svc.Stopped { - if timeout.Before(time.Now()) { - return fmt.Errorf("timeout waiting for %s service to stop", service) - } - time.Sleep(300 * time.Millisecond) - status, err = s.Query() - if err != nil { - return fmt.Errorf("could not retrieve %s service status: %v", service, err) - } - } - return nil - } - - // Stop the service - status, err = s.Control(svc.Stop) - if err != nil { - return fmt.Errorf("could not stop service %s: %v", service, err) - } - - // Check that the stop was successful - status, err = s.Query() - if err != nil { - return fmt.Errorf("could not query service %s: %v", service, err) - } - timeout := time.Now().Add(10 * time.Second) - for status.State != svc.Stopped { - if timeout.Before(time.Now()) { - return fmt.Errorf("timeout waiting for %s service to stop", service) - } - time.Sleep(300 * time.Millisecond) - status, err = s.Query() - if err != nil { - return fmt.Errorf("could not retrieve %s service status: %v", service, err) - } - } - return nil -} - -// ServiceExists ensures the service is defined for this init system. -func (sysd WindowsInitSystem) ServiceExists(service string) bool { - m, err := mgr.Connect() - if err != nil { - return false - } - defer m.Disconnect() - s, err := m.OpenService(service) - if err != nil { - return false - } - defer s.Close() - - return true -} - -// ServiceIsEnabled ensures the service is enabled to start on each boot. -func (sysd WindowsInitSystem) ServiceIsEnabled(service string) bool { - m, err := mgr.Connect() - if err != nil { - return false - } - defer m.Disconnect() - - s, err := m.OpenService(service) - if err != nil { - return false - } - defer s.Close() - - c, err := s.Config() - if err != nil { - return false - } - - return c.StartType != mgr.StartDisabled -} - -// ServiceIsActive ensures the service is running, or attempting to run. (crash looping in the case of kubelet) -func (sysd WindowsInitSystem) ServiceIsActive(service string) bool { - m, err := mgr.Connect() - if err != nil { - return false - } - defer m.Disconnect() - s, err := m.OpenService(service) - if err != nil { - return false - } - defer s.Close() - - status, err := s.Query() - if err != nil { - return false - } - return status.State == svc.Running -} - -// GetInitSystem returns an InitSystem for the current system, or nil -// if we cannot detect a supported init system. -// This indicates we will skip init system checks, not an error. -func GetInitSystem() (InitSystem, error) { - m, err := mgr.Connect() - if err != nil { - return nil, fmt.Errorf("no supported init system detected: %v", err) - } - defer m.Disconnect() - return &WindowsInitSystem{}, nil -} diff --git a/pkg/util/kubernetes/kubeadm/app/util/runtime/runtime.go b/pkg/util/kubernetes/kubeadm/app/util/runtime/runtime.go deleted file mode 100644 index b4f99377aa7..00000000000 --- a/pkg/util/kubernetes/kubeadm/app/util/runtime/runtime.go +++ /dev/null @@ -1,234 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package util - -import ( - "path/filepath" - goruntime "runtime" - "strings" - - "github.com/pkg/errors" - errorsutil "k8s.io/apimachinery/pkg/util/errors" - utilsexec "k8s.io/utils/exec" - - "github.com/openyurtio/openyurt/pkg/util/kubernetes/kubeadm/app/constants" -) - -// ContainerRuntime is an interface for working with container runtimes -type ContainerRuntime interface { - IsDocker() bool - IsRunning() error - ListKubeContainers() ([]string, error) - RemoveContainers(containers []string) error - PullImage(image string) error - ImageExists(image string) (bool, error) -} - -// CRIRuntime is a struct that interfaces with the CRI -type CRIRuntime struct { - exec utilsexec.Interface - criSocket string -} - -// DockerRuntime is a struct that interfaces with the Docker daemon -type DockerRuntime struct { - exec utilsexec.Interface -} - -// NewContainerRuntime sets up and returns a ContainerRuntime struct -func NewContainerRuntime(execer utilsexec.Interface, criSocket string) (ContainerRuntime, error) { - var toolName string - var runtime ContainerRuntime - - if criSocket != constants.DefaultDockerCRISocket { - toolName = "crictl" - // !!! temporary work around crictl warning: - // Using "/var/run/crio/crio.sock" as endpoint is deprecated, - // please consider using full url format "unix:///var/run/crio/crio.sock" - if filepath.IsAbs(criSocket) && goruntime.GOOS != "windows" { - criSocket = "unix://" + criSocket - } - runtime = &CRIRuntime{execer, criSocket} - } else { - toolName = "docker" - runtime = &DockerRuntime{execer} - } - - if _, err := execer.LookPath(toolName); err != nil { - return nil, errors.Wrapf(err, "%s is required for container runtime", toolName) - } - - return runtime, nil -} - -// IsDocker returns true if the runtime is docker -func (runtime *CRIRuntime) IsDocker() bool { - return false -} - -// IsDocker returns true if the runtime is docker -func (runtime *DockerRuntime) IsDocker() bool { - return true -} - -// IsRunning checks if runtime is running -func (runtime *CRIRuntime) IsRunning() error { - if out, err := runtime.exec.Command("crictl", "-r", runtime.criSocket, "info").CombinedOutput(); err != nil { - return errors.Wrapf(err, "container runtime is not running: output: %s, error", string(out)) - } - return nil -} - -// IsRunning checks if runtime is running -func (runtime *DockerRuntime) IsRunning() error { - if out, err := runtime.exec.Command("docker", "info").CombinedOutput(); err != nil { - return errors.Wrapf(err, "container runtime is not running: output: %s, error", string(out)) - } - return nil -} - -// ListKubeContainers lists running k8s CRI pods -func (runtime *CRIRuntime) ListKubeContainers() ([]string, error) { - out, err := runtime.exec.Command("crictl", "-r", runtime.criSocket, "pods", "-q").CombinedOutput() - if err != nil { - return nil, errors.Wrapf(err, "output: %s, error", string(out)) - } - pods := []string{} - pods = append(pods, strings.Fields(string(out))...) - return pods, nil -} - -// ListKubeContainers lists running k8s containers -func (runtime *DockerRuntime) ListKubeContainers() ([]string, error) { - output, err := runtime.exec.Command("docker", "ps", "-a", "--filter", "name=k8s_", "-q").CombinedOutput() - return strings.Fields(string(output)), err -} - -// RemoveContainers removes running k8s pods -func (runtime *CRIRuntime) RemoveContainers(containers []string) error { - errs := []error{} - for _, container := range containers { - out, err := runtime.exec.Command("crictl", "-r", runtime.criSocket, "stopp", container).CombinedOutput() - if err != nil { - // don't stop on errors, try to remove as many containers as possible - errs = append(errs, errors.Wrapf(err, "failed to stop running pod %s: output: %s, error", container, string(out))) - } else { - out, err = runtime.exec.Command("crictl", "-r", runtime.criSocket, "rmp", container).CombinedOutput() - if err != nil { - errs = append(errs, errors.Wrapf(err, "failed to remove running container %s: output: %s, error", container, string(out))) - } - } - } - return errorsutil.NewAggregate(errs) -} - -// RemoveContainers removes running containers -func (runtime *DockerRuntime) RemoveContainers(containers []string) error { - errs := []error{} - for _, container := range containers { - out, err := runtime.exec.Command("docker", "stop", container).CombinedOutput() - if err != nil { - // don't stop on errors, try to remove as many containers as possible - errs = append(errs, errors.Wrapf(err, "failed to stop running container %s: output: %s, error", container, string(out))) - } else { - out, err = runtime.exec.Command("docker", "rm", "--volumes", container).CombinedOutput() - if err != nil { - errs = append(errs, errors.Wrapf(err, "failed to remove running container %s: output: %s, error", container, string(out))) - } - } - } - return errorsutil.NewAggregate(errs) -} - -// PullImage pulls the image -func (runtime *CRIRuntime) PullImage(image string) error { - var err error - var out []byte - for i := 0; i < constants.PullImageRetry; i++ { - out, err = runtime.exec.Command("crictl", "-r", runtime.criSocket, "pull", image).CombinedOutput() - if err == nil { - return nil - } - } - return errors.Wrapf(err, "output: %s, error", out) -} - -// PullImage pulls the image -func (runtime *DockerRuntime) PullImage(image string) error { - var err error - var out []byte - for i := 0; i < constants.PullImageRetry; i++ { - out, err = runtime.exec.Command("docker", "pull", image).CombinedOutput() - if err == nil { - return nil - } - } - return errors.Wrapf(err, "output: %s, error", out) -} - -// ImageExists checks to see if the image exists on the system -func (runtime *CRIRuntime) ImageExists(image string) (bool, error) { - err := runtime.exec.Command("crictl", "-r", runtime.criSocket, "inspecti", image).Run() - return err == nil, nil -} - -// ImageExists checks to see if the image exists on the system -func (runtime *DockerRuntime) ImageExists(image string) (bool, error) { - err := runtime.exec.Command("docker", "inspect", image).Run() - return err == nil, nil -} - -// detectCRISocketImpl is separated out only for test purposes, DON'T call it directly, use DetectCRISocket instead -func detectCRISocketImpl(isSocket func(string) bool) (string, error) { - foundCRISockets := []string{} - knownCRISockets := []string{ - // Docker and containerd sockets are special cased below, hence not to be included here - "/var/run/crio/crio.sock", - } - - if isSocket(dockerSocket) { - // the path in dockerSocket is not CRI compatible, hence we should replace it with a CRI compatible socket - foundCRISockets = append(foundCRISockets, constants.DefaultDockerCRISocket) - } else if isSocket(containerdSocket) { - // Docker 18.09 gets bundled together with containerd, thus having both dockerSocket and containerdSocket present. - // For compatibility reasons, we use the containerd socket only if Docker is not detected. - foundCRISockets = append(foundCRISockets, containerdSocket) - } - - for _, socket := range knownCRISockets { - if isSocket(socket) { - foundCRISockets = append(foundCRISockets, socket) - } - } - - switch len(foundCRISockets) { - case 0: - // Fall back to Docker if no CRI is detected, we can error out later on if we need it - return constants.DefaultDockerCRISocket, nil - case 1: - // Precisely one CRI found, use that - return foundCRISockets[0], nil - default: - // Multiple CRIs installed? - return "", errors.Errorf("Found multiple CRI sockets, please use --cri-socket to select one: %s", strings.Join(foundCRISockets, ", ")) - } -} - -// DetectCRISocket uses a list of known CRI sockets to detect one. If more than one or none is discovered, an error is returned. -func DetectCRISocket() (string, error) { - return detectCRISocketImpl(isExistingSocket) -} diff --git a/pkg/util/kubernetes/kubeadm/app/util/runtime/runtime_test.go b/pkg/util/kubernetes/kubeadm/app/util/runtime/runtime_test.go deleted file mode 100644 index d10d05c0c6c..00000000000 --- a/pkg/util/kubernetes/kubeadm/app/util/runtime/runtime_test.go +++ /dev/null @@ -1,462 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package util - -import ( - "io/ioutil" - "net" - "os" - "reflect" - "runtime" - "testing" - - "github.com/pkg/errors" - "k8s.io/utils/exec" - fakeexec "k8s.io/utils/exec/testing" - - "github.com/openyurtio/openyurt/pkg/util/kubernetes/kubeadm/app/constants" -) - -func TestNewContainerRuntime(t *testing.T) { - execLookPathOK := fakeexec.FakeExec{ - LookPathFunc: func(cmd string) (string, error) { return "/usr/bin/crictl", nil }, - } - execLookPathErr := fakeexec.FakeExec{ - LookPathFunc: func(cmd string) (string, error) { return "", errors.Errorf("%s not found", cmd) }, - } - cases := []struct { - name string - execer fakeexec.FakeExec - criSocket string - isDocker bool - isError bool - }{ - {"valid: default cri socket", execLookPathOK, constants.DefaultDockerCRISocket, true, false}, - {"valid: cri-o socket url", execLookPathOK, "unix:///var/run/crio/crio.sock", false, false}, - {"valid: cri-o socket path", execLookPathOK, "/var/run/crio/crio.sock", false, false}, - {"invalid: no crictl", execLookPathErr, "unix:///var/run/crio/crio.sock", false, true}, - } - - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - runtime, err := NewContainerRuntime(&tc.execer, tc.criSocket) - if err != nil { - if !tc.isError { - t.Fatalf("unexpected NewContainerRuntime error. criSocket: %s, error: %v", tc.criSocket, err) - } - return // expected error occurs, impossible to test runtime further - } - if tc.isError && err == nil { - t.Fatalf("unexpected NewContainerRuntime success. criSocket: %s", tc.criSocket) - } - isDocker := runtime.IsDocker() - if tc.isDocker != isDocker { - t.Fatalf("unexpected isDocker() result %v for the criSocket %s", isDocker, tc.criSocket) - } - }) - } -} - -func genFakeActions(fcmd *fakeexec.FakeCmd, num int) []fakeexec.FakeCommandAction { - var actions []fakeexec.FakeCommandAction - for i := 0; i < num; i++ { - actions = append(actions, func(cmd string, args ...string) exec.Cmd { - return fakeexec.InitFakeCmd(fcmd, cmd, args...) - }) - } - return actions -} - -func TestIsRunning(t *testing.T) { - fcmd := fakeexec.FakeCmd{ - CombinedOutputScript: []fakeexec.FakeAction{ - func() ([]byte, []byte, error) { return nil, nil, nil }, - func() ([]byte, []byte, error) { return []byte("error"), nil, &fakeexec.FakeExitError{Status: 1} }, - func() ([]byte, []byte, error) { return nil, nil, nil }, - func() ([]byte, []byte, error) { return []byte("error"), nil, &fakeexec.FakeExitError{Status: 1} }, - }, - } - - criExecer := fakeexec.FakeExec{ - CommandScript: genFakeActions(&fcmd, len(fcmd.CombinedOutputScript)), - LookPathFunc: func(cmd string) (string, error) { return "/usr/bin/crictl", nil }, - } - - dockerExecer := fakeexec.FakeExec{ - CommandScript: genFakeActions(&fcmd, len(fcmd.CombinedOutputScript)), - LookPathFunc: func(cmd string) (string, error) { return "/usr/bin/docker", nil }, - } - - cases := []struct { - name string - criSocket string - execer fakeexec.FakeExec - isError bool - }{ - {"valid: CRI-O is running", "unix:///var/run/crio/crio.sock", criExecer, false}, - {"invalid: CRI-O is not running", "unix:///var/run/crio/crio.sock", criExecer, true}, - {"valid: docker is running", constants.DefaultDockerCRISocket, dockerExecer, false}, - {"invalid: docker is not running", constants.DefaultDockerCRISocket, dockerExecer, true}, - } - - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - runtime, err := NewContainerRuntime(&tc.execer, tc.criSocket) - if err != nil { - t.Fatalf("unexpected NewContainerRuntime error: %v", err) - } - isRunning := runtime.IsRunning() - if tc.isError && isRunning == nil { - t.Error("unexpected IsRunning() success") - } - if !tc.isError && isRunning != nil { - t.Error("unexpected IsRunning() error") - } - }) - } -} - -func TestListKubeContainers(t *testing.T) { - fcmd := fakeexec.FakeCmd{ - CombinedOutputScript: []fakeexec.FakeAction{ - func() ([]byte, []byte, error) { return []byte("k8s_p1\nk8s_p2"), nil, nil }, - func() ([]byte, []byte, error) { return nil, nil, &fakeexec.FakeExitError{Status: 1} }, - func() ([]byte, []byte, error) { return []byte("k8s_p1\nk8s_p2"), nil, nil }, - }, - } - execer := fakeexec.FakeExec{ - CommandScript: genFakeActions(&fcmd, len(fcmd.CombinedOutputScript)), - LookPathFunc: func(cmd string) (string, error) { return "/usr/bin/crictl", nil }, - } - - cases := []struct { - name string - criSocket string - isError bool - }{ - {"valid: list containers using CRI socket url", "unix:///var/run/crio/crio.sock", false}, - {"invalid: list containers using CRI socket url", "unix:///var/run/crio/crio.sock", true}, - {"valid: list containers using docker", constants.DefaultDockerCRISocket, false}, - } - - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - runtime, err := NewContainerRuntime(&execer, tc.criSocket) - if err != nil { - t.Fatalf("unexpected NewContainerRuntime error: %v", err) - } - - containers, err := runtime.ListKubeContainers() - if tc.isError { - if err == nil { - t.Errorf("unexpected ListKubeContainers success") - } - return - } else if err != nil { - t.Errorf("unexpected ListKubeContainers error: %v", err) - } - - if !reflect.DeepEqual(containers, []string{"k8s_p1", "k8s_p2"}) { - t.Errorf("unexpected ListKubeContainers output: %v", containers) - } - }) - } -} - -func TestRemoveContainers(t *testing.T) { - fakeOK := func() ([]byte, []byte, error) { return nil, nil, nil } - fakeErr := func() ([]byte, []byte, error) { return []byte("error"), nil, &fakeexec.FakeExitError{Status: 1} } - fcmd := fakeexec.FakeCmd{ - CombinedOutputScript: []fakeexec.FakeAction{ - fakeOK, fakeOK, fakeOK, fakeOK, fakeOK, fakeOK, // Test case 1 - fakeOK, fakeOK, fakeOK, fakeErr, fakeOK, fakeOK, - fakeErr, fakeOK, fakeOK, fakeErr, fakeOK, - fakeOK, fakeOK, fakeOK, fakeOK, fakeOK, fakeOK, - fakeOK, fakeOK, fakeOK, fakeErr, fakeOK, fakeOK, - fakeErr, fakeOK, fakeOK, fakeErr, fakeOK, - }, - } - execer := fakeexec.FakeExec{ - CommandScript: genFakeActions(&fcmd, len(fcmd.CombinedOutputScript)), - LookPathFunc: func(cmd string) (string, error) { return "/usr/bin/crictl", nil }, - } - - cases := []struct { - name string - criSocket string - containers []string - isError bool - }{ - {"valid: remove containers using CRI", "unix:///var/run/crio/crio.sock", []string{"k8s_p1", "k8s_p2", "k8s_p3"}, false}, // Test case 1 - {"invalid: CRI rmp failure", "unix:///var/run/crio/crio.sock", []string{"k8s_p1", "k8s_p2", "k8s_p3"}, true}, - {"invalid: CRI stopp failure", "unix:///var/run/crio/crio.sock", []string{"k8s_p1", "k8s_p2", "k8s_p3"}, true}, - {"valid: remove containers using docker", constants.DefaultDockerCRISocket, []string{"k8s_c1", "k8s_c2", "k8s_c3"}, false}, - {"invalid: docker rm failure", constants.DefaultDockerCRISocket, []string{"k8s_c1", "k8s_c2", "k8s_c3"}, true}, - {"invalid: docker stop failure", constants.DefaultDockerCRISocket, []string{"k8s_c1", "k8s_c2", "k8s_c3"}, true}, - } - - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - runtime, err := NewContainerRuntime(&execer, tc.criSocket) - if err != nil { - t.Fatalf("unexpected NewContainerRuntime error: %v, criSocket: %s", err, tc.criSocket) - } - - err = runtime.RemoveContainers(tc.containers) - if !tc.isError && err != nil { - t.Errorf("unexpected RemoveContainers errors: %v, criSocket: %s, containers: %v", err, tc.criSocket, tc.containers) - } - if tc.isError && err == nil { - t.Errorf("unexpected RemoveContainers success, criSocket: %s, containers: %v", tc.criSocket, tc.containers) - } - }) - } -} - -func TestPullImage(t *testing.T) { - fcmd := fakeexec.FakeCmd{ - CombinedOutputScript: []fakeexec.FakeAction{ - func() ([]byte, []byte, error) { return nil, nil, nil }, - // If the pull fails, it will be retried 5 times (see PullImageRetry in constants/constants.go) - func() ([]byte, []byte, error) { return []byte("error"), nil, &fakeexec.FakeExitError{Status: 1} }, - func() ([]byte, []byte, error) { return []byte("error"), nil, &fakeexec.FakeExitError{Status: 1} }, - func() ([]byte, []byte, error) { return []byte("error"), nil, &fakeexec.FakeExitError{Status: 1} }, - func() ([]byte, []byte, error) { return []byte("error"), nil, &fakeexec.FakeExitError{Status: 1} }, - func() ([]byte, []byte, error) { return []byte("error"), nil, &fakeexec.FakeExitError{Status: 1} }, - func() ([]byte, []byte, error) { return nil, nil, nil }, - // If the pull fails, it will be retried 5 times (see PullImageRetry in constants/constants.go) - func() ([]byte, []byte, error) { return []byte("error"), nil, &fakeexec.FakeExitError{Status: 1} }, - func() ([]byte, []byte, error) { return []byte("error"), nil, &fakeexec.FakeExitError{Status: 1} }, - func() ([]byte, []byte, error) { return []byte("error"), nil, &fakeexec.FakeExitError{Status: 1} }, - func() ([]byte, []byte, error) { return []byte("error"), nil, &fakeexec.FakeExitError{Status: 1} }, - func() ([]byte, []byte, error) { return []byte("error"), nil, &fakeexec.FakeExitError{Status: 1} }, - }, - } - execer := fakeexec.FakeExec{ - CommandScript: genFakeActions(&fcmd, len(fcmd.CombinedOutputScript)), - LookPathFunc: func(cmd string) (string, error) { return "/usr/bin/crictl", nil }, - } - - cases := []struct { - name string - criSocket string - image string - isError bool - }{ - {"valid: pull image using CRI", "unix:///var/run/crio/crio.sock", "image1", false}, - {"invalid: CRI pull error", "unix:///var/run/crio/crio.sock", "image2", true}, - {"valid: pull image using docker", constants.DefaultDockerCRISocket, "image1", false}, - {"invalid: docker pull error", constants.DefaultDockerCRISocket, "image2", true}, - } - - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - runtime, err := NewContainerRuntime(&execer, tc.criSocket) - if err != nil { - t.Fatalf("unexpected NewContainerRuntime error: %v, criSocket: %s", err, tc.criSocket) - } - - err = runtime.PullImage(tc.image) - if !tc.isError && err != nil { - t.Errorf("unexpected PullImage error: %v, criSocket: %s, image: %s", err, tc.criSocket, tc.image) - } - if tc.isError && err == nil { - t.Errorf("unexpected PullImage success, criSocket: %s, image: %s", tc.criSocket, tc.image) - } - }) - } -} - -func TestImageExists(t *testing.T) { - fcmd := fakeexec.FakeCmd{ - RunScript: []fakeexec.FakeAction{ - func() ([]byte, []byte, error) { return nil, nil, nil }, - func() ([]byte, []byte, error) { return nil, nil, &fakeexec.FakeExitError{Status: 1} }, - func() ([]byte, []byte, error) { return nil, nil, nil }, - func() ([]byte, []byte, error) { return nil, nil, &fakeexec.FakeExitError{Status: 1} }, - }, - } - execer := fakeexec.FakeExec{ - CommandScript: genFakeActions(&fcmd, len(fcmd.RunScript)), - LookPathFunc: func(cmd string) (string, error) { return "/usr/bin/crictl", nil }, - } - - cases := []struct { - name string - criSocket string - image string - result bool - }{ - {"valid: test if image exists using CRI", "unix:///var/run/crio/crio.sock", "image1", false}, - {"invalid: CRI inspecti failure", "unix:///var/run/crio/crio.sock", "image2", true}, - {"valid: test if image exists using docker", constants.DefaultDockerCRISocket, "image1", false}, - {"invalid: docker inspect failure", constants.DefaultDockerCRISocket, "image2", true}, - } - - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - runtime, err := NewContainerRuntime(&execer, tc.criSocket) - if err != nil { - t.Fatalf("unexpected NewContainerRuntime error: %v, criSocket: %s", err, tc.criSocket) - } - - result, err := runtime.ImageExists(tc.image) - if !tc.result != result { - t.Errorf("unexpected ImageExists result: %t, criSocket: %s, image: %s, expected result: %t", err, tc.criSocket, tc.image, tc.result) - } - }) - } -} - -func TestIsExistingSocket(t *testing.T) { - // this test is not expected to work on Windows - if runtime.GOOS == "windows" { - return - } - - const tempPrefix = "test.kubeadm.runtime.isExistingSocket." - tests := []struct { - name string - proc func(*testing.T) - }{ - { - name: "Valid domain socket is detected as such", - proc: func(t *testing.T) { - tmpFile, err := ioutil.TempFile("", tempPrefix) - if err != nil { - t.Fatalf("unexpected error by TempFile: %v", err) - } - theSocket := tmpFile.Name() - os.Remove(theSocket) - tmpFile.Close() - - con, err := net.Listen("unix", theSocket) - if err != nil { - t.Fatalf("unexpected error while dialing a socket: %v", err) - } - defer con.Close() - - if !isExistingSocket(theSocket) { - t.Fatalf("isExistingSocket(%q) gave unexpected result. Should have been true, instead of false", theSocket) - } - }, - }, - { - name: "Regular file is not a domain socket", - proc: func(t *testing.T) { - tmpFile, err := ioutil.TempFile("", tempPrefix) - if err != nil { - t.Fatalf("unexpected error by TempFile: %v", err) - } - theSocket := tmpFile.Name() - defer os.Remove(theSocket) - tmpFile.Close() - - if isExistingSocket(theSocket) { - t.Fatalf("isExistingSocket(%q) gave unexpected result. Should have been false, instead of true", theSocket) - } - }, - }, - { - name: "Non existent socket is not a domain socket", - proc: func(t *testing.T) { - const theSocket = "/non/existent/socket" - if isExistingSocket(theSocket) { - t.Fatalf("isExistingSocket(%q) gave unexpected result. Should have been false, instead of true", theSocket) - } - }, - }, - } - - for _, test := range tests { - t.Run(test.name, test.proc) - } -} - -func TestDetectCRISocketImpl(t *testing.T) { - tests := []struct { - name string - existingSockets []string - expectedError bool - expectedSocket string - }{ - { - name: "No existing sockets, use Docker", - existingSockets: []string{}, - expectedError: false, - expectedSocket: constants.DefaultDockerCRISocket, - }, - { - name: "One valid CRI socket leads to success", - existingSockets: []string{"/var/run/crio/crio.sock"}, - expectedError: false, - expectedSocket: "/var/run/crio/crio.sock", - }, - { - name: "Correct Docker CRI socket is returned", - existingSockets: []string{"/var/run/docker.sock"}, - expectedError: false, - expectedSocket: constants.DefaultDockerCRISocket, - }, - { - name: "CRI and Docker sockets lead to an error", - existingSockets: []string{ - "/var/run/docker.sock", - "/var/run/crio/crio.sock", - }, - expectedError: true, - }, - { - name: "Docker and containerd lead to Docker being used", - existingSockets: []string{ - "/var/run/docker.sock", - "/run/containerd/containerd.sock", - }, - expectedError: false, - expectedSocket: constants.DefaultDockerCRISocket, - }, - { - name: "A couple of CRI sockets lead to an error", - existingSockets: []string{ - "/var/run/crio/crio.sock", - "/run/containerd/containerd.sock", - }, - expectedError: true, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - socket, err := detectCRISocketImpl(func(path string) bool { - for _, existing := range test.existingSockets { - if path == existing { - return true - } - } - - return false - }) - if (err != nil) != test.expectedError { - t.Fatalf("detectCRISocketImpl returned unexpected result\n\tExpected error: %t\n\tGot error: %t", test.expectedError, err != nil) - } - if !test.expectedError && socket != test.expectedSocket { - t.Fatalf("detectCRISocketImpl returned unexpected CRI socket\n\tExpected socket: %s\n\tReturned socket: %s", - test.expectedSocket, socket) - } - }) - } -} diff --git a/pkg/util/kubernetes/kubeadm/app/util/runtime/runtime_unix.go b/pkg/util/kubernetes/kubeadm/app/util/runtime/runtime_unix.go deleted file mode 100644 index 11bc059dea7..00000000000 --- a/pkg/util/kubernetes/kubeadm/app/util/runtime/runtime_unix.go +++ /dev/null @@ -1,39 +0,0 @@ -//go:build !windows -// +build !windows - -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package util - -import ( - "os" -) - -const ( - dockerSocket = "/var/run/docker.sock" // The Docker socket is not CRI compatible - containerdSocket = "/run/containerd/containerd.sock" -) - -// isExistingSocket checks if path exists and is domain socket -func isExistingSocket(path string) bool { - fileInfo, err := os.Stat(path) - if err != nil { - return false - } - - return fileInfo.Mode()&os.ModeSocket != 0 -} diff --git a/pkg/util/kubernetes/kubeadm/app/util/runtime/runtime_windows.go b/pkg/util/kubernetes/kubeadm/app/util/runtime/runtime_windows.go deleted file mode 100644 index 0c6a7b496dc..00000000000 --- a/pkg/util/kubernetes/kubeadm/app/util/runtime/runtime_windows.go +++ /dev/null @@ -1,38 +0,0 @@ -// +build windows - -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package util - -import ( - winio "github.com/Microsoft/go-winio" -) - -const ( - dockerSocket = "//./pipe/docker_engine" // The Docker socket is not CRI compatible - containerdSocket = "//./pipe/containerd-containerd" // Proposed containerd named pipe for Windows -) - -// isExistingSocket checks if path exists and is domain socket -func isExistingSocket(path string) bool { - _, err := winio.DialPipe(path, nil) - if err != nil { - return false - } - - return true -} diff --git a/pkg/util/kubernetes/kubelet/apis/config/register.go b/pkg/util/kubernetes/kubelet/apis/config/register.go deleted file mode 100644 index cbebd990ab9..00000000000 --- a/pkg/util/kubernetes/kubelet/apis/config/register.go +++ /dev/null @@ -1,44 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package config - -import ( - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/runtime/schema" -) - -// GroupName is the group name used in this package -const GroupName = "kubelet.config.k8s.io" - -// SchemeGroupVersion is group version used to register these objects -var SchemeGroupVersion = schema.GroupVersion{Group: GroupName, Version: runtime.APIVersionInternal} - -var ( - // SchemeBuilder is the scheme builder with scheme init functions to run for this API package - SchemeBuilder = runtime.NewSchemeBuilder(addKnownTypes) - // AddToScheme is a global function that registers this API group & version to a scheme - AddToScheme = SchemeBuilder.AddToScheme -) - -// addKnownTypes registers known types to the given scheme -func addKnownTypes(scheme *runtime.Scheme) error { - scheme.AddKnownTypes(SchemeGroupVersion, - &KubeletConfiguration{}, - &SerializedNodeConfigSource{}, - ) - return nil -} diff --git a/pkg/util/kubernetes/kubelet/apis/config/scheme/scheme.go b/pkg/util/kubernetes/kubelet/apis/config/scheme/scheme.go deleted file mode 100644 index e47dc5fac24..00000000000 --- a/pkg/util/kubernetes/kubelet/apis/config/scheme/scheme.go +++ /dev/null @@ -1,43 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. -Copyright 2021 The OpenYurt Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package scheme - -import ( - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/runtime/serializer" - - kubeletconfig "github.com/openyurtio/openyurt/pkg/util/kubernetes/kubelet/apis/config" - kubeletconfigv1beta1 "github.com/openyurtio/openyurt/pkg/util/kubernetes/kubelet/apis/config/v1beta1" -) - -// Utility functions for the Kubelet's kubeletconfig API group - -// NewSchemeAndCodecs is a utility function that returns a Scheme and CodecFactory -// that understand the types in the kubeletconfig API group. Passing mutators allows -// for adjusting the behavior of the CodecFactory, for example enable strict decoding. -func NewSchemeAndCodecs(mutators ...serializer.CodecFactoryOptionsMutator) (*runtime.Scheme, *serializer.CodecFactory, error) { - scheme := runtime.NewScheme() - if err := kubeletconfig.AddToScheme(scheme); err != nil { - return nil, nil, err - } - if err := kubeletconfigv1beta1.AddToScheme(scheme); err != nil { - return nil, nil, err - } - codecs := serializer.NewCodecFactory(scheme, mutators...) - return scheme, &codecs, nil -} diff --git a/pkg/util/kubernetes/kubelet/apis/config/types.go b/pkg/util/kubernetes/kubelet/apis/config/types.go deleted file mode 100644 index 818a089465d..00000000000 --- a/pkg/util/kubernetes/kubelet/apis/config/types.go +++ /dev/null @@ -1,420 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package config - -import ( - v1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -) - -// HairpinMode denotes how the kubelet should configure networking to handle -// hairpin packets. -type HairpinMode string - -// Enum settings for different ways to handle hairpin packets. -const ( - // Set the hairpin flag on the veth of containers in the respective - // container runtime. - HairpinVeth = "hairpin-veth" - // Make the container bridge promiscuous. This will force it to accept - // hairpin packets, even if the flag isn't set on ports of the bridge. - PromiscuousBridge = "promiscuous-bridge" - // Neither of the above. If the kubelet is started in this hairpin mode - // and kube-proxy is running in iptables mode, hairpin packets will be - // dropped by the container bridge. - HairpinNone = "none" -) - -// ResourceChangeDetectionStrategy denotes a mode in which internal -// managers (secret, configmap) are discovering object changes. -type ResourceChangeDetectionStrategy string - -// Enum settings for different strategies of kubelet managers. -const ( - // GetChangeDetectionStrategy is a mode in which kubelet fetches - // necessary objects directly from apiserver. - GetChangeDetectionStrategy ResourceChangeDetectionStrategy = "Get" - // TTLCacheChangeDetectionStrategy is a mode in which kubelet uses - // ttl cache for object directly fetched from apiserver. - TTLCacheChangeDetectionStrategy ResourceChangeDetectionStrategy = "Cache" - // WatchChangeDetectionStrategy is a mode in which kubelet uses - // watches to observe changes to objects that are in its interest. - WatchChangeDetectionStrategy ResourceChangeDetectionStrategy = "Watch" - // RestrictedTopologyManagerPolicy is a mode in which kubelet only allows - // pods with optimal NUMA node alignment for requested resources - RestrictedTopologyManagerPolicy = "restricted" - // BestEffortTopologyManagerPolicy is a mode in which kubelet will favour - // pods with NUMA alignment of CPU and device resources. - BestEffortTopologyManagerPolicy = "best-effort" - // NoneTopologyManager Policy is a mode in which kubelet has no knowledge - // of NUMA alignment of a pod's CPU and device resources. - NoneTopologyManagerPolicy = "none" - // SingleNumaNodeTopologyManager Policy iis a mode in which kubelet only allows - // pods with a single NUMA alignment of CPU and device resources. - SingleNumaNodeTopologyManager = "single-numa-node" -) - -// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object - -// KubeletConfiguration contains the configuration for the Kubelet -type KubeletConfiguration struct { - metav1.TypeMeta - - // staticPodPath is the path to the directory containing local (static) pods to - // run, or the path to a single static pod file. - StaticPodPath string - // syncFrequency is the max period between synchronizing running - // containers and config - SyncFrequency metav1.Duration - // fileCheckFrequency is the duration between checking config files for - // new data - FileCheckFrequency metav1.Duration - // httpCheckFrequency is the duration between checking http for new data - HTTPCheckFrequency metav1.Duration - // staticPodURL is the URL for accessing static pods to run - StaticPodURL string - // staticPodURLHeader is a map of slices with HTTP headers to use when accessing the podURL - StaticPodURLHeader map[string][]string - // address is the IP address for the Kubelet to serve on (set to 0.0.0.0 - // for all interfaces) - Address string - // port is the port for the Kubelet to serve on. - Port int32 - // readOnlyPort is the read-only port for the Kubelet to serve on with - // no authentication/authorization (set to 0 to disable) - ReadOnlyPort int32 - // tlsCertFile is the file containing x509 Certificate for HTTPS. (CA cert, - // if any, concatenated after server cert). If tlsCertFile and - // tlsPrivateKeyFile are not provided, a self-signed certificate - // and key are generated for the public address and saved to the directory - // passed to the Kubelet's --cert-dir flag. - TLSCertFile string - // tlsPrivateKeyFile is the file containing x509 private key matching tlsCertFile - TLSPrivateKeyFile string - // TLSCipherSuites is the list of allowed cipher suites for the server. - // Values are from tls package constants (https://golang.org/pkg/crypto/tls/#pkg-constants). - TLSCipherSuites []string - // TLSMinVersion is the minimum TLS version supported. - // Values are from tls package constants (https://golang.org/pkg/crypto/tls/#pkg-constants). - TLSMinVersion string - // rotateCertificates enables client certificate rotation. The Kubelet will request a - // new certificate from the certificates.k8s.io API. This requires an approver to approve the - // certificate signing requests. The RotateKubeletClientCertificate feature - // must be enabled. - RotateCertificates bool - // serverTLSBootstrap enables server certificate bootstrap. Instead of self - // signing a serving certificate, the Kubelet will request a certificate from - // the certificates.k8s.io API. This requires an approver to approve the - // certificate signing requests. The RotateKubeletServerCertificate feature - // must be enabled. - ServerTLSBootstrap bool - // authentication specifies how requests to the Kubelet's server are authenticated - Authentication KubeletAuthentication - // authorization specifies how requests to the Kubelet's server are authorized - Authorization KubeletAuthorization - // registryPullQPS is the limit of registry pulls per second. - // Set to 0 for no limit. - RegistryPullQPS int32 - // registryBurst is the maximum size of bursty pulls, temporarily allows - // pulls to burst to this number, while still not exceeding registryPullQPS. - // Only used if registryPullQPS > 0. - RegistryBurst int32 - // eventRecordQPS is the maximum event creations per second. If 0, there - // is no limit enforced. - EventRecordQPS int32 - // eventBurst is the maximum size of a burst of event creations, temporarily - // allows event creations to burst to this number, while still not exceeding - // eventRecordQPS. Only used if eventRecordQPS > 0. - EventBurst int32 - // enableDebuggingHandlers enables server endpoints for log collection - // and local running of containers and commands - EnableDebuggingHandlers bool - // enableContentionProfiling enables lock contention profiling, if enableDebuggingHandlers is true. - EnableContentionProfiling bool - // healthzPort is the port of the localhost healthz endpoint (set to 0 to disable) - HealthzPort int32 - // healthzBindAddress is the IP address for the healthz server to serve on - HealthzBindAddress string - // oomScoreAdj is The oom-score-adj value for kubelet process. Values - // must be within the range [-1000, 1000]. - OOMScoreAdj int32 - // clusterDomain is the DNS domain for this cluster. If set, kubelet will - // configure all containers to search this domain in addition to the - // host's search domains. - ClusterDomain string - // clusterDNS is a list of IP addresses for a cluster DNS server. If set, - // kubelet will configure all containers to use this for DNS resolution - // instead of the host's DNS servers. - ClusterDNS []string - // streamingConnectionIdleTimeout is the maximum time a streaming connection - // can be idle before the connection is automatically closed. - StreamingConnectionIdleTimeout metav1.Duration - // nodeStatusUpdateFrequency is the frequency that kubelet computes node - // status. If node lease feature is not enabled, it is also the frequency that - // kubelet posts node status to master. In that case, be cautious when - // changing the constant, it must work with nodeMonitorGracePeriod in nodecontroller. - NodeStatusUpdateFrequency metav1.Duration - // nodeStatusReportFrequency is the frequency that kubelet posts node - // status to master if node status does not change. Kubelet will ignore this - // frequency and post node status immediately if any change is detected. It is - // only used when node lease feature is enabled. - NodeStatusReportFrequency metav1.Duration - // nodeLeaseDurationSeconds is the duration the Kubelet will set on its corresponding Lease. - NodeLeaseDurationSeconds int32 - // imageMinimumGCAge is the minimum age for an unused image before it is - // garbage collected. - ImageMinimumGCAge metav1.Duration - // imageGCHighThresholdPercent is the percent of disk usage after which - // image garbage collection is always run. The percent is calculated as - // this field value out of 100. - ImageGCHighThresholdPercent int32 - // imageGCLowThresholdPercent is the percent of disk usage before which - // image garbage collection is never run. Lowest disk usage to garbage - // collect to. The percent is calculated as this field value out of 100. - ImageGCLowThresholdPercent int32 - // How frequently to calculate and cache volume disk usage for all pods - VolumeStatsAggPeriod metav1.Duration - // KubeletCgroups is the absolute name of cgroups to isolate the kubelet in - KubeletCgroups string - // SystemCgroups is absolute name of cgroups in which to place - // all non-kernel processes that are not already in a container. Empty - // for no container. Rolling back the flag requires a reboot. - SystemCgroups string - // CgroupRoot is the root cgroup to use for pods. - // If CgroupsPerQOS is enabled, this is the root of the QoS cgroup hierarchy. - CgroupRoot string - // Enable QoS based Cgroup hierarchy: top level cgroups for QoS Classes - // And all Burstable and BestEffort pods are brought up under their - // specific top level QoS cgroup. - CgroupsPerQOS bool - // driver that the kubelet uses to manipulate cgroups on the host (cgroupfs or systemd) - CgroupDriver string - // CPUManagerPolicy is the name of the policy to use. - // Requires the CPUManager feature gate to be enabled. - CPUManagerPolicy string - // CPU Manager reconciliation period. - // Requires the CPUManager feature gate to be enabled. - CPUManagerReconcilePeriod metav1.Duration - // TopologyManagerPolicy is the name of the policy to use. - // Policies other than "none" require the TopologyManager feature gate to be enabled. - TopologyManagerPolicy string - // Map of QoS resource reservation percentages (memory only for now). - // Requires the QOSReserved feature gate to be enabled. - QOSReserved map[string]string - // runtimeRequestTimeout is the timeout for all runtime requests except long running - // requests - pull, logs, exec and attach. - RuntimeRequestTimeout metav1.Duration - // hairpinMode specifies how the Kubelet should configure the container - // bridge for hairpin packets. - // Setting this flag allows endpoints in a Service to loadbalance back to - // themselves if they should try to access their own Service. Values: - // "promiscuous-bridge": make the container bridge promiscuous. - // "hairpin-veth": set the hairpin flag on container veth interfaces. - // "none": do nothing. - // Generally, one must set --hairpin-mode=hairpin-veth to achieve hairpin NAT, - // because promiscuous-bridge assumes the existence of a container bridge named cbr0. - HairpinMode string - // maxPods is the number of pods that can run on this Kubelet. - MaxPods int32 - // The CIDR to use for pod IP addresses, only used in standalone mode. - // In cluster mode, this is obtained from the master. - PodCIDR string - // The maximum number of processes per pod. If -1, the kubelet defaults to the node allocatable pid capacity. - PodPidsLimit int64 - // ResolverConfig is the resolver configuration file used as the basis - // for the container DNS resolution configuration. - ResolverConfig string - // cpuCFSQuota enables CPU CFS quota enforcement for containers that - // specify CPU limits - CPUCFSQuota bool - // CPUCFSQuotaPeriod sets the CPU CFS quota period value, cpu.cfs_period_us, defaults to 100ms - CPUCFSQuotaPeriod metav1.Duration - // maxOpenFiles is Number of files that can be opened by Kubelet process. - MaxOpenFiles int64 - // contentType is contentType of requests sent to apiserver. - ContentType string - // kubeAPIQPS is the QPS to use while talking with kubernetes apiserver - KubeAPIQPS int32 - // kubeAPIBurst is the burst to allow while talking with kubernetes - // apiserver - KubeAPIBurst int32 - // serializeImagePulls when enabled, tells the Kubelet to pull images one at a time. - SerializeImagePulls bool - // Map of signal names to quantities that defines hard eviction thresholds. For example: {"memory.available": "300Mi"}. - EvictionHard map[string]string - // Map of signal names to quantities that defines soft eviction thresholds. For example: {"memory.available": "300Mi"}. - EvictionSoft map[string]string - // Map of signal names to quantities that defines grace periods for each soft eviction signal. For example: {"memory.available": "30s"}. - EvictionSoftGracePeriod map[string]string - // Duration for which the kubelet has to wait before transitioning out of an eviction pressure condition. - EvictionPressureTransitionPeriod metav1.Duration - // Maximum allowed grace period (in seconds) to use when terminating pods in response to a soft eviction threshold being met. - EvictionMaxPodGracePeriod int32 - // Map of signal names to quantities that defines minimum reclaims, which describe the minimum - // amount of a given resource the kubelet will reclaim when performing a pod eviction while - // that resource is under pressure. For example: {"imagefs.available": "2Gi"} - EvictionMinimumReclaim map[string]string - // podsPerCore is the maximum number of pods per core. Cannot exceed MaxPods. - // If 0, this field is ignored. - PodsPerCore int32 - // enableControllerAttachDetach enables the Attach/Detach controller to - // manage attachment/detachment of volumes scheduled to this node, and - // disables kubelet from executing any attach/detach operations - EnableControllerAttachDetach bool - // protectKernelDefaults, if true, causes the Kubelet to error if kernel - // flags are not as it expects. Otherwise the Kubelet will attempt to modify - // kernel flags to match its expectation. - ProtectKernelDefaults bool - // If true, Kubelet ensures a set of iptables rules are present on host. - // These rules will serve as utility for various components, e.g. kube-proxy. - // The rules will be created based on IPTablesMasqueradeBit and IPTablesDropBit. - MakeIPTablesUtilChains bool - // iptablesMasqueradeBit is the bit of the iptables fwmark space to mark for SNAT - // Values must be within the range [0, 31]. Must be different from other mark bits. - // Warning: Please match the value of the corresponding parameter in kube-proxy. - // TODO: clean up IPTablesMasqueradeBit in kube-proxy - IPTablesMasqueradeBit int32 - // iptablesDropBit is the bit of the iptables fwmark space to mark for dropping packets. - // Values must be within the range [0, 31]. Must be different from other mark bits. - IPTablesDropBit int32 - // featureGates is a map of feature names to bools that enable or disable alpha/experimental - // features. This field modifies piecemeal the built-in default values from - // "k8s.io/kubernetes/pkg/features/kube_features.go". - FeatureGates map[string]bool - // Tells the Kubelet to fail to start if swap is enabled on the node. - FailSwapOn bool - // A quantity defines the maximum size of the container log file before it is rotated. For example: "5Mi" or "256Ki". - ContainerLogMaxSize string - // Maximum number of container log files that can be present for a container. - ContainerLogMaxFiles int32 - // ConfigMapAndSecretChangeDetectionStrategy is a mode in which config map and secret managers are running. - ConfigMapAndSecretChangeDetectionStrategy ResourceChangeDetectionStrategy - // A comma separated whitelist of unsafe sysctls or sysctl patterns (ending in *). - // Unsafe sysctl groups are kernel.shm*, kernel.msg*, kernel.sem, fs.mqueue.*, and net.*. - // These sysctls are namespaced but not allowed by default. For example: "kernel.msg*,net.ipv4.route.min_pmtu" - // +optional - AllowedUnsafeSysctls []string - - /* the following fields are meant for Node Allocatable */ - - // A set of ResourceName=ResourceQuantity (e.g. cpu=200m,memory=150G,pid=100) pairs - // that describe resources reserved for non-kubernetes components. - // Currently only cpu and memory are supported. - // See http://kubernetes.io/docs/user-guide/compute-resources for more detail. - SystemReserved map[string]string - // A set of ResourceName=ResourceQuantity (e.g. cpu=200m,memory=150G,pid=100) pairs - // that describe resources reserved for kubernetes system components. - // Currently cpu, memory and local ephemeral storage for root file system are supported. - // See http://kubernetes.io/docs/user-guide/compute-resources for more detail. - KubeReserved map[string]string - // This flag helps kubelet identify absolute name of top level cgroup used to enforce `SystemReserved` compute resource reservation for OS system daemons. - // Refer to [Node Allocatable](https://git.k8s.io/community/contributors/design-proposals/node/node-allocatable.md) doc for more information. - SystemReservedCgroup string - // This flag helps kubelet identify absolute name of top level cgroup used to enforce `KubeReserved` compute resource reservation for Kubernetes node system daemons. - // Refer to [Node Allocatable](https://git.k8s.io/community/contributors/design-proposals/node/node-allocatable.md) doc for more information. - KubeReservedCgroup string - // This flag specifies the various Node Allocatable enforcements that Kubelet needs to perform. - // This flag accepts a list of options. Acceptable options are `pods`, `system-reserved` & `kube-reserved`. - // Refer to [Node Allocatable](https://git.k8s.io/community/contributors/design-proposals/node/node-allocatable.md) doc for more information. - EnforceNodeAllocatable []string - // This option specifies the cpu list reserved for the host level system threads and kubernetes related threads. - // This provide a "static" CPU list rather than the "dynamic" list by system-reserved and kube-reserved. - // This option overwrites CPUs provided by system-reserved and kube-reserved. - ReservedSystemCPUs string - // The previous version for which you want to show hidden metrics. - // Only the previous minor version is meaningful, other values will not be allowed. - // The format is ., e.g.: '1.16'. - // The purpose of this format is make sure you have the opportunity to notice if the next release hides additional metrics, - // rather than being surprised when they are permanently removed in the release after that. - ShowHiddenMetricsForVersion string -} - -// KubeletAuthorizationMode denotes the authorization mode for the kubelet -type KubeletAuthorizationMode string - -const ( - // KubeletAuthorizationModeAlwaysAllow authorizes all authenticated requests - KubeletAuthorizationModeAlwaysAllow KubeletAuthorizationMode = "AlwaysAllow" - // KubeletAuthorizationModeWebhook uses the SubjectAccessReview API to determine authorization - KubeletAuthorizationModeWebhook KubeletAuthorizationMode = "Webhook" -) - -// KubeletAuthorization holds the state related to the authorization in the kublet. -type KubeletAuthorization struct { - // mode is the authorization mode to apply to requests to the kubelet server. - // Valid values are AlwaysAllow and Webhook. - // Webhook mode uses the SubjectAccessReview API to determine authorization. - Mode KubeletAuthorizationMode - - // webhook contains settings related to Webhook authorization. - Webhook KubeletWebhookAuthorization -} - -// KubeletWebhookAuthorization holds the state related to the Webhook -// Authorization in the Kubelet. -type KubeletWebhookAuthorization struct { - // cacheAuthorizedTTL is the duration to cache 'authorized' responses from the webhook authorizer. - CacheAuthorizedTTL metav1.Duration - // cacheUnauthorizedTTL is the duration to cache 'unauthorized' responses from the webhook authorizer. - CacheUnauthorizedTTL metav1.Duration -} - -// KubeletAuthentication holds the Kubetlet Authentication setttings. -type KubeletAuthentication struct { - // x509 contains settings related to x509 client certificate authentication - X509 KubeletX509Authentication - // webhook contains settings related to webhook bearer token authentication - Webhook KubeletWebhookAuthentication - // anonymous contains settings related to anonymous authentication - Anonymous KubeletAnonymousAuthentication -} - -// KubeletX509Authentication contains settings related to x509 client certificate authentication -type KubeletX509Authentication struct { - // clientCAFile is the path to a PEM-encoded certificate bundle. If set, any request presenting a client certificate - // signed by one of the authorities in the bundle is authenticated with a username corresponding to the CommonName, - // and groups corresponding to the Organization in the client certificate. - ClientCAFile string -} - -// KubeletWebhookAuthentication contains settings related to webhook authentication -type KubeletWebhookAuthentication struct { - // enabled allows bearer token authentication backed by the tokenreviews.authentication.k8s.io API - Enabled bool - // cacheTTL enables caching of authentication results - CacheTTL metav1.Duration -} - -// KubeletAnonymousAuthentication enables anonymous requests to the kubetlet server. -type KubeletAnonymousAuthentication struct { - // enabled allows anonymous requests to the kubelet server. - // Requests that are not rejected by another authentication method are treated as anonymous requests. - // Anonymous requests have a username of system:anonymous, and a group name of system:unauthenticated. - Enabled bool -} - -// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object - -// SerializedNodeConfigSource allows us to serialize NodeConfigSource -// This type is used internally by the Kubelet for tracking checkpointed dynamic configs. -// It exists in the kubeletconfig API group because it is classified as a versioned input to the Kubelet. -type SerializedNodeConfigSource struct { - metav1.TypeMeta - // Source is the source that we are serializing - // +optional - Source v1.NodeConfigSource -} diff --git a/pkg/util/kubernetes/kubelet/apis/config/v1beta1/defaults.go b/pkg/util/kubernetes/kubelet/apis/config/v1beta1/defaults.go deleted file mode 100644 index 60b0e588cc9..00000000000 --- a/pkg/util/kubernetes/kubelet/apis/config/v1beta1/defaults.go +++ /dev/null @@ -1,232 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. -Copyright 2021 The OpenYurt Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package v1beta1 - -import ( - "time" - - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - kruntime "k8s.io/apimachinery/pkg/runtime" - kubeletconfigv1beta1 "k8s.io/kubelet/config/v1beta1" - utilpointer "k8s.io/utils/pointer" -) - -const ( - // TODO: Move these constants to k8s.io/kubelet/config/v1beta1 instead? - DefaultIPTablesMasqueradeBit = 14 - DefaultIPTablesDropBit = 15 - - KubeletOOMScoreAdj int = -999 - - // system default DNS resolver configuration - ResolvConfDefault = "/etc/resolv.conf" - - // KubeletPort is the default port for the kubelet server on each host machine. - // May be overridden by a flag at startup. - KubeletPort = 10250 -) - -var ( - zeroDuration = metav1.Duration{} - // TODO: Move these constants to k8s.io/kubelet/config/v1beta1 instead? - // Refer to [Node Allocatable](https://git.k8s.io/community/contributors/design-proposals/node/node-allocatable.md) doc for more information. - DefaultNodeAllocatableEnforcement = []string{"pods"} -) - -func addDefaultingFuncs(scheme *kruntime.Scheme) error { - return RegisterDefaults(scheme) -} - -func SetDefaults_KubeletConfiguration(obj *kubeletconfigv1beta1.KubeletConfiguration) { - if obj.SyncFrequency == zeroDuration { - obj.SyncFrequency = metav1.Duration{Duration: 1 * time.Minute} - } - if obj.FileCheckFrequency == zeroDuration { - obj.FileCheckFrequency = metav1.Duration{Duration: 20 * time.Second} - } - if obj.HTTPCheckFrequency == zeroDuration { - obj.HTTPCheckFrequency = metav1.Duration{Duration: 20 * time.Second} - } - if obj.Address == "" { - obj.Address = "0.0.0.0" - } - if obj.Port == 0 { - obj.Port = KubeletPort - } - if obj.Authentication.Anonymous.Enabled == nil { - obj.Authentication.Anonymous.Enabled = utilpointer.BoolPtr(false) - } - if obj.Authentication.Webhook.Enabled == nil { - obj.Authentication.Webhook.Enabled = utilpointer.BoolPtr(true) - } - if obj.Authentication.Webhook.CacheTTL == zeroDuration { - obj.Authentication.Webhook.CacheTTL = metav1.Duration{Duration: 2 * time.Minute} - } - if obj.Authorization.Mode == "" { - obj.Authorization.Mode = kubeletconfigv1beta1.KubeletAuthorizationModeWebhook - } - if obj.Authorization.Webhook.CacheAuthorizedTTL == zeroDuration { - obj.Authorization.Webhook.CacheAuthorizedTTL = metav1.Duration{Duration: 5 * time.Minute} - } - if obj.Authorization.Webhook.CacheUnauthorizedTTL == zeroDuration { - obj.Authorization.Webhook.CacheUnauthorizedTTL = metav1.Duration{Duration: 30 * time.Second} - } - if obj.RegistryPullQPS == nil { - obj.RegistryPullQPS = utilpointer.Int32Ptr(5) - } - if obj.RegistryBurst == 0 { - obj.RegistryBurst = 10 - } - if obj.EventRecordQPS == nil { - obj.EventRecordQPS = utilpointer.Int32Ptr(5) - } - if obj.EventBurst == 0 { - obj.EventBurst = 10 - } - if obj.EnableDebuggingHandlers == nil { - obj.EnableDebuggingHandlers = utilpointer.BoolPtr(true) - } - if obj.HealthzPort == nil { - obj.HealthzPort = utilpointer.Int32Ptr(10248) - } - if obj.HealthzBindAddress == "" { - obj.HealthzBindAddress = "127.0.0.1" - } - if obj.OOMScoreAdj == nil { - obj.OOMScoreAdj = utilpointer.Int32Ptr(int32(KubeletOOMScoreAdj)) - } - if obj.StreamingConnectionIdleTimeout == zeroDuration { - obj.StreamingConnectionIdleTimeout = metav1.Duration{Duration: 4 * time.Hour} - } - if obj.NodeStatusReportFrequency == zeroDuration { - // For backward compatibility, NodeStatusReportFrequency's default value is - // set to NodeStatusUpdateFrequency if NodeStatusUpdateFrequency is set - // explicitly. - if obj.NodeStatusUpdateFrequency == zeroDuration { - obj.NodeStatusReportFrequency = metav1.Duration{Duration: 5 * time.Minute} - } else { - obj.NodeStatusReportFrequency = obj.NodeStatusUpdateFrequency - } - } - if obj.NodeStatusUpdateFrequency == zeroDuration { - obj.NodeStatusUpdateFrequency = metav1.Duration{Duration: 10 * time.Second} - } - if obj.NodeLeaseDurationSeconds == 0 { - obj.NodeLeaseDurationSeconds = 40 - } - if obj.ImageMinimumGCAge == zeroDuration { - obj.ImageMinimumGCAge = metav1.Duration{Duration: 2 * time.Minute} - } - if obj.ImageGCHighThresholdPercent == nil { - // default is below docker's default dm.min_free_space of 90% - obj.ImageGCHighThresholdPercent = utilpointer.Int32Ptr(85) - } - if obj.ImageGCLowThresholdPercent == nil { - obj.ImageGCLowThresholdPercent = utilpointer.Int32Ptr(80) - } - if obj.VolumeStatsAggPeriod == zeroDuration { - obj.VolumeStatsAggPeriod = metav1.Duration{Duration: time.Minute} - } - if obj.CgroupsPerQOS == nil { - obj.CgroupsPerQOS = utilpointer.BoolPtr(true) - } - if obj.CgroupDriver == "" { - obj.CgroupDriver = "cgroupfs" - } - if obj.CPUManagerPolicy == "" { - obj.CPUManagerPolicy = "none" - } - if obj.CPUManagerReconcilePeriod == zeroDuration { - // Keep the same as default NodeStatusUpdateFrequency - obj.CPUManagerReconcilePeriod = metav1.Duration{Duration: 10 * time.Second} - } - if obj.TopologyManagerPolicy == "" { - obj.TopologyManagerPolicy = kubeletconfigv1beta1.NoneTopologyManagerPolicy - } - if obj.RuntimeRequestTimeout == zeroDuration { - obj.RuntimeRequestTimeout = metav1.Duration{Duration: 2 * time.Minute} - } - if obj.HairpinMode == "" { - obj.HairpinMode = kubeletconfigv1beta1.PromiscuousBridge - } - if obj.MaxPods == 0 { - obj.MaxPods = 110 - } - // default nil or negative value to -1 (implies node allocatable pid limit) - if obj.PodPidsLimit == nil || *obj.PodPidsLimit < int64(0) { - temp := int64(-1) - obj.PodPidsLimit = &temp - } - if obj.ResolverConfig == "" { - obj.ResolverConfig = ResolvConfDefault - } - if obj.CPUCFSQuota == nil { - obj.CPUCFSQuota = utilpointer.BoolPtr(true) - } - if obj.CPUCFSQuotaPeriod == nil { - obj.CPUCFSQuotaPeriod = &metav1.Duration{Duration: 100 * time.Millisecond} - } - if obj.MaxOpenFiles == 0 { - obj.MaxOpenFiles = 1000000 - } - if obj.ContentType == "" { - obj.ContentType = "application/vnd.kubernetes.protobuf" - } - if obj.KubeAPIQPS == nil { - obj.KubeAPIQPS = utilpointer.Int32Ptr(5) - } - if obj.KubeAPIBurst == 0 { - obj.KubeAPIBurst = 10 - } - if obj.SerializeImagePulls == nil { - obj.SerializeImagePulls = utilpointer.BoolPtr(true) - } - if obj.EvictionHard == nil { - obj.EvictionHard = DefaultEvictionHard - } - if obj.EvictionPressureTransitionPeriod == zeroDuration { - obj.EvictionPressureTransitionPeriod = metav1.Duration{Duration: 5 * time.Minute} - } - if obj.EnableControllerAttachDetach == nil { - obj.EnableControllerAttachDetach = utilpointer.BoolPtr(true) - } - if obj.MakeIPTablesUtilChains == nil { - obj.MakeIPTablesUtilChains = utilpointer.BoolPtr(true) - } - if obj.IPTablesMasqueradeBit == nil { - obj.IPTablesMasqueradeBit = utilpointer.Int32Ptr(DefaultIPTablesMasqueradeBit) - } - if obj.IPTablesDropBit == nil { - obj.IPTablesDropBit = utilpointer.Int32Ptr(DefaultIPTablesDropBit) - } - if obj.FailSwapOn == nil { - obj.FailSwapOn = utilpointer.BoolPtr(true) - } - if obj.ContainerLogMaxSize == "" { - obj.ContainerLogMaxSize = "10Mi" - } - if obj.ContainerLogMaxFiles == nil { - obj.ContainerLogMaxFiles = utilpointer.Int32Ptr(5) - } - if obj.ConfigMapAndSecretChangeDetectionStrategy == "" { - obj.ConfigMapAndSecretChangeDetectionStrategy = kubeletconfigv1beta1.WatchChangeDetectionStrategy - } - if obj.EnforceNodeAllocatable == nil { - obj.EnforceNodeAllocatable = DefaultNodeAllocatableEnforcement - } -} diff --git a/pkg/util/kubernetes/kubelet/apis/config/v1beta1/defaults_others.go b/pkg/util/kubernetes/kubelet/apis/config/v1beta1/defaults_others.go deleted file mode 100644 index 2b8ac3d4306..00000000000 --- a/pkg/util/kubernetes/kubelet/apis/config/v1beta1/defaults_others.go +++ /dev/null @@ -1,27 +0,0 @@ -//go:build !linux -// +build !linux - -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package v1beta1 - -// DefaultEvictionHard includes default options for hard eviction. -var DefaultEvictionHard = map[string]string{ - "memory.available": "100Mi", - "nodefs.available": "10%", - "imagefs.available": "15%", -} diff --git a/pkg/util/kubernetes/kubelet/apis/config/v1beta1/register.go b/pkg/util/kubernetes/kubelet/apis/config/v1beta1/register.go deleted file mode 100644 index 1b981b5791d..00000000000 --- a/pkg/util/kubernetes/kubelet/apis/config/v1beta1/register.go +++ /dev/null @@ -1,43 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package v1beta1 - -import ( - "k8s.io/apimachinery/pkg/runtime/schema" - kubeletconfigv1beta1 "k8s.io/kubelet/config/v1beta1" -) - -// GroupName is the group name used in this package -const GroupName = "kubelet.config.k8s.io" - -// SchemeGroupVersion is group version used to register these objects -var SchemeGroupVersion = schema.GroupVersion{Group: GroupName, Version: "v1beta1"} - -var ( - // localSchemeBuilder extends the SchemeBuilder instance with the external types. In this package, - // defaulting and conversion init funcs are registered as well. - localSchemeBuilder = &kubeletconfigv1beta1.SchemeBuilder - // AddToScheme is a global function that registers this API group & version to a scheme - AddToScheme = localSchemeBuilder.AddToScheme -) - -func init() { - // We only register manually written functions here. The registration of the - // generated functions takes place in the generated files. The separation - // makes the code compile even when the generated files are missing. - localSchemeBuilder.Register(addDefaultingFuncs) -} diff --git a/pkg/util/kubernetes/kubelet/apis/config/v1beta1/zz_generated.conversion.go b/pkg/util/kubernetes/kubelet/apis/config/v1beta1/zz_generated.conversion.go deleted file mode 100644 index 630425c9e34..00000000000 --- a/pkg/util/kubernetes/kubelet/apis/config/v1beta1/zz_generated.conversion.go +++ /dev/null @@ -1,565 +0,0 @@ -//go:build !ignore_autogenerated -// +build !ignore_autogenerated - -/* -Copyright The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Code generated by conversion-gen. DO NOT EDIT. - -package v1beta1 - -import ( - unsafe "unsafe" - - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" - conversion "k8s.io/apimachinery/pkg/conversion" - runtime "k8s.io/apimachinery/pkg/runtime" - v1beta1 "k8s.io/kubelet/config/v1beta1" - - config "github.com/openyurtio/openyurt/pkg/util/kubernetes/kubelet/apis/config" -) - -func init() { - localSchemeBuilder.Register(RegisterConversions) -} - -// RegisterConversions adds conversion functions to the given scheme. -// Public to allow building arbitrary schemes. -func RegisterConversions(s *runtime.Scheme) error { - if err := s.AddGeneratedConversionFunc((*v1beta1.KubeletAnonymousAuthentication)(nil), (*config.KubeletAnonymousAuthentication)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_v1beta1_KubeletAnonymousAuthentication_To_config_KubeletAnonymousAuthentication(a.(*v1beta1.KubeletAnonymousAuthentication), b.(*config.KubeletAnonymousAuthentication), scope) - }); err != nil { - return err - } - if err := s.AddGeneratedConversionFunc((*config.KubeletAnonymousAuthentication)(nil), (*v1beta1.KubeletAnonymousAuthentication)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_config_KubeletAnonymousAuthentication_To_v1beta1_KubeletAnonymousAuthentication(a.(*config.KubeletAnonymousAuthentication), b.(*v1beta1.KubeletAnonymousAuthentication), scope) - }); err != nil { - return err - } - if err := s.AddGeneratedConversionFunc((*v1beta1.KubeletAuthentication)(nil), (*config.KubeletAuthentication)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_v1beta1_KubeletAuthentication_To_config_KubeletAuthentication(a.(*v1beta1.KubeletAuthentication), b.(*config.KubeletAuthentication), scope) - }); err != nil { - return err - } - if err := s.AddGeneratedConversionFunc((*config.KubeletAuthentication)(nil), (*v1beta1.KubeletAuthentication)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_config_KubeletAuthentication_To_v1beta1_KubeletAuthentication(a.(*config.KubeletAuthentication), b.(*v1beta1.KubeletAuthentication), scope) - }); err != nil { - return err - } - if err := s.AddGeneratedConversionFunc((*v1beta1.KubeletAuthorization)(nil), (*config.KubeletAuthorization)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_v1beta1_KubeletAuthorization_To_config_KubeletAuthorization(a.(*v1beta1.KubeletAuthorization), b.(*config.KubeletAuthorization), scope) - }); err != nil { - return err - } - if err := s.AddGeneratedConversionFunc((*config.KubeletAuthorization)(nil), (*v1beta1.KubeletAuthorization)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_config_KubeletAuthorization_To_v1beta1_KubeletAuthorization(a.(*config.KubeletAuthorization), b.(*v1beta1.KubeletAuthorization), scope) - }); err != nil { - return err - } - if err := s.AddGeneratedConversionFunc((*v1beta1.KubeletConfiguration)(nil), (*config.KubeletConfiguration)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_v1beta1_KubeletConfiguration_To_config_KubeletConfiguration(a.(*v1beta1.KubeletConfiguration), b.(*config.KubeletConfiguration), scope) - }); err != nil { - return err - } - if err := s.AddGeneratedConversionFunc((*config.KubeletConfiguration)(nil), (*v1beta1.KubeletConfiguration)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_config_KubeletConfiguration_To_v1beta1_KubeletConfiguration(a.(*config.KubeletConfiguration), b.(*v1beta1.KubeletConfiguration), scope) - }); err != nil { - return err - } - if err := s.AddGeneratedConversionFunc((*v1beta1.KubeletWebhookAuthentication)(nil), (*config.KubeletWebhookAuthentication)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_v1beta1_KubeletWebhookAuthentication_To_config_KubeletWebhookAuthentication(a.(*v1beta1.KubeletWebhookAuthentication), b.(*config.KubeletWebhookAuthentication), scope) - }); err != nil { - return err - } - if err := s.AddGeneratedConversionFunc((*config.KubeletWebhookAuthentication)(nil), (*v1beta1.KubeletWebhookAuthentication)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_config_KubeletWebhookAuthentication_To_v1beta1_KubeletWebhookAuthentication(a.(*config.KubeletWebhookAuthentication), b.(*v1beta1.KubeletWebhookAuthentication), scope) - }); err != nil { - return err - } - if err := s.AddGeneratedConversionFunc((*v1beta1.KubeletWebhookAuthorization)(nil), (*config.KubeletWebhookAuthorization)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_v1beta1_KubeletWebhookAuthorization_To_config_KubeletWebhookAuthorization(a.(*v1beta1.KubeletWebhookAuthorization), b.(*config.KubeletWebhookAuthorization), scope) - }); err != nil { - return err - } - if err := s.AddGeneratedConversionFunc((*config.KubeletWebhookAuthorization)(nil), (*v1beta1.KubeletWebhookAuthorization)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_config_KubeletWebhookAuthorization_To_v1beta1_KubeletWebhookAuthorization(a.(*config.KubeletWebhookAuthorization), b.(*v1beta1.KubeletWebhookAuthorization), scope) - }); err != nil { - return err - } - if err := s.AddGeneratedConversionFunc((*v1beta1.KubeletX509Authentication)(nil), (*config.KubeletX509Authentication)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_v1beta1_KubeletX509Authentication_To_config_KubeletX509Authentication(a.(*v1beta1.KubeletX509Authentication), b.(*config.KubeletX509Authentication), scope) - }); err != nil { - return err - } - if err := s.AddGeneratedConversionFunc((*config.KubeletX509Authentication)(nil), (*v1beta1.KubeletX509Authentication)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_config_KubeletX509Authentication_To_v1beta1_KubeletX509Authentication(a.(*config.KubeletX509Authentication), b.(*v1beta1.KubeletX509Authentication), scope) - }); err != nil { - return err - } - if err := s.AddGeneratedConversionFunc((*v1beta1.SerializedNodeConfigSource)(nil), (*config.SerializedNodeConfigSource)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_v1beta1_SerializedNodeConfigSource_To_config_SerializedNodeConfigSource(a.(*v1beta1.SerializedNodeConfigSource), b.(*config.SerializedNodeConfigSource), scope) - }); err != nil { - return err - } - if err := s.AddGeneratedConversionFunc((*config.SerializedNodeConfigSource)(nil), (*v1beta1.SerializedNodeConfigSource)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_config_SerializedNodeConfigSource_To_v1beta1_SerializedNodeConfigSource(a.(*config.SerializedNodeConfigSource), b.(*v1beta1.SerializedNodeConfigSource), scope) - }); err != nil { - return err - } - return nil -} - -func autoConvert_v1beta1_KubeletAnonymousAuthentication_To_config_KubeletAnonymousAuthentication(in *v1beta1.KubeletAnonymousAuthentication, out *config.KubeletAnonymousAuthentication, s conversion.Scope) error { - if err := v1.Convert_Pointer_bool_To_bool(&in.Enabled, &out.Enabled, s); err != nil { - return err - } - return nil -} - -// Convert_v1beta1_KubeletAnonymousAuthentication_To_config_KubeletAnonymousAuthentication is an autogenerated conversion function. -func Convert_v1beta1_KubeletAnonymousAuthentication_To_config_KubeletAnonymousAuthentication(in *v1beta1.KubeletAnonymousAuthentication, out *config.KubeletAnonymousAuthentication, s conversion.Scope) error { - return autoConvert_v1beta1_KubeletAnonymousAuthentication_To_config_KubeletAnonymousAuthentication(in, out, s) -} - -func autoConvert_config_KubeletAnonymousAuthentication_To_v1beta1_KubeletAnonymousAuthentication(in *config.KubeletAnonymousAuthentication, out *v1beta1.KubeletAnonymousAuthentication, s conversion.Scope) error { - if err := v1.Convert_bool_To_Pointer_bool(&in.Enabled, &out.Enabled, s); err != nil { - return err - } - return nil -} - -// Convert_config_KubeletAnonymousAuthentication_To_v1beta1_KubeletAnonymousAuthentication is an autogenerated conversion function. -func Convert_config_KubeletAnonymousAuthentication_To_v1beta1_KubeletAnonymousAuthentication(in *config.KubeletAnonymousAuthentication, out *v1beta1.KubeletAnonymousAuthentication, s conversion.Scope) error { - return autoConvert_config_KubeletAnonymousAuthentication_To_v1beta1_KubeletAnonymousAuthentication(in, out, s) -} - -func autoConvert_v1beta1_KubeletAuthentication_To_config_KubeletAuthentication(in *v1beta1.KubeletAuthentication, out *config.KubeletAuthentication, s conversion.Scope) error { - if err := Convert_v1beta1_KubeletX509Authentication_To_config_KubeletX509Authentication(&in.X509, &out.X509, s); err != nil { - return err - } - if err := Convert_v1beta1_KubeletWebhookAuthentication_To_config_KubeletWebhookAuthentication(&in.Webhook, &out.Webhook, s); err != nil { - return err - } - if err := Convert_v1beta1_KubeletAnonymousAuthentication_To_config_KubeletAnonymousAuthentication(&in.Anonymous, &out.Anonymous, s); err != nil { - return err - } - return nil -} - -// Convert_v1beta1_KubeletAuthentication_To_config_KubeletAuthentication is an autogenerated conversion function. -func Convert_v1beta1_KubeletAuthentication_To_config_KubeletAuthentication(in *v1beta1.KubeletAuthentication, out *config.KubeletAuthentication, s conversion.Scope) error { - return autoConvert_v1beta1_KubeletAuthentication_To_config_KubeletAuthentication(in, out, s) -} - -func autoConvert_config_KubeletAuthentication_To_v1beta1_KubeletAuthentication(in *config.KubeletAuthentication, out *v1beta1.KubeletAuthentication, s conversion.Scope) error { - if err := Convert_config_KubeletX509Authentication_To_v1beta1_KubeletX509Authentication(&in.X509, &out.X509, s); err != nil { - return err - } - if err := Convert_config_KubeletWebhookAuthentication_To_v1beta1_KubeletWebhookAuthentication(&in.Webhook, &out.Webhook, s); err != nil { - return err - } - if err := Convert_config_KubeletAnonymousAuthentication_To_v1beta1_KubeletAnonymousAuthentication(&in.Anonymous, &out.Anonymous, s); err != nil { - return err - } - return nil -} - -// Convert_config_KubeletAuthentication_To_v1beta1_KubeletAuthentication is an autogenerated conversion function. -func Convert_config_KubeletAuthentication_To_v1beta1_KubeletAuthentication(in *config.KubeletAuthentication, out *v1beta1.KubeletAuthentication, s conversion.Scope) error { - return autoConvert_config_KubeletAuthentication_To_v1beta1_KubeletAuthentication(in, out, s) -} - -func autoConvert_v1beta1_KubeletAuthorization_To_config_KubeletAuthorization(in *v1beta1.KubeletAuthorization, out *config.KubeletAuthorization, s conversion.Scope) error { - out.Mode = config.KubeletAuthorizationMode(in.Mode) - if err := Convert_v1beta1_KubeletWebhookAuthorization_To_config_KubeletWebhookAuthorization(&in.Webhook, &out.Webhook, s); err != nil { - return err - } - return nil -} - -// Convert_v1beta1_KubeletAuthorization_To_config_KubeletAuthorization is an autogenerated conversion function. -func Convert_v1beta1_KubeletAuthorization_To_config_KubeletAuthorization(in *v1beta1.KubeletAuthorization, out *config.KubeletAuthorization, s conversion.Scope) error { - return autoConvert_v1beta1_KubeletAuthorization_To_config_KubeletAuthorization(in, out, s) -} - -func autoConvert_config_KubeletAuthorization_To_v1beta1_KubeletAuthorization(in *config.KubeletAuthorization, out *v1beta1.KubeletAuthorization, s conversion.Scope) error { - out.Mode = v1beta1.KubeletAuthorizationMode(in.Mode) - if err := Convert_config_KubeletWebhookAuthorization_To_v1beta1_KubeletWebhookAuthorization(&in.Webhook, &out.Webhook, s); err != nil { - return err - } - return nil -} - -// Convert_config_KubeletAuthorization_To_v1beta1_KubeletAuthorization is an autogenerated conversion function. -func Convert_config_KubeletAuthorization_To_v1beta1_KubeletAuthorization(in *config.KubeletAuthorization, out *v1beta1.KubeletAuthorization, s conversion.Scope) error { - return autoConvert_config_KubeletAuthorization_To_v1beta1_KubeletAuthorization(in, out, s) -} - -func autoConvert_v1beta1_KubeletConfiguration_To_config_KubeletConfiguration(in *v1beta1.KubeletConfiguration, out *config.KubeletConfiguration, s conversion.Scope) error { - out.StaticPodPath = in.StaticPodPath - out.SyncFrequency = in.SyncFrequency - out.FileCheckFrequency = in.FileCheckFrequency - out.HTTPCheckFrequency = in.HTTPCheckFrequency - out.StaticPodURL = in.StaticPodURL - out.StaticPodURLHeader = *(*map[string][]string)(unsafe.Pointer(&in.StaticPodURLHeader)) - out.Address = in.Address - out.Port = in.Port - out.ReadOnlyPort = in.ReadOnlyPort - out.TLSCertFile = in.TLSCertFile - out.TLSPrivateKeyFile = in.TLSPrivateKeyFile - out.TLSCipherSuites = *(*[]string)(unsafe.Pointer(&in.TLSCipherSuites)) - out.TLSMinVersion = in.TLSMinVersion - out.RotateCertificates = in.RotateCertificates - out.ServerTLSBootstrap = in.ServerTLSBootstrap - if err := Convert_v1beta1_KubeletAuthentication_To_config_KubeletAuthentication(&in.Authentication, &out.Authentication, s); err != nil { - return err - } - if err := Convert_v1beta1_KubeletAuthorization_To_config_KubeletAuthorization(&in.Authorization, &out.Authorization, s); err != nil { - return err - } - if err := v1.Convert_Pointer_int32_To_int32(&in.RegistryPullQPS, &out.RegistryPullQPS, s); err != nil { - return err - } - out.RegistryBurst = in.RegistryBurst - if err := v1.Convert_Pointer_int32_To_int32(&in.EventRecordQPS, &out.EventRecordQPS, s); err != nil { - return err - } - out.EventBurst = in.EventBurst - if err := v1.Convert_Pointer_bool_To_bool(&in.EnableDebuggingHandlers, &out.EnableDebuggingHandlers, s); err != nil { - return err - } - out.EnableContentionProfiling = in.EnableContentionProfiling - if err := v1.Convert_Pointer_int32_To_int32(&in.HealthzPort, &out.HealthzPort, s); err != nil { - return err - } - out.HealthzBindAddress = in.HealthzBindAddress - if err := v1.Convert_Pointer_int32_To_int32(&in.OOMScoreAdj, &out.OOMScoreAdj, s); err != nil { - return err - } - out.ClusterDomain = in.ClusterDomain - out.ClusterDNS = *(*[]string)(unsafe.Pointer(&in.ClusterDNS)) - out.StreamingConnectionIdleTimeout = in.StreamingConnectionIdleTimeout - out.NodeStatusUpdateFrequency = in.NodeStatusUpdateFrequency - out.NodeStatusReportFrequency = in.NodeStatusReportFrequency - out.NodeLeaseDurationSeconds = in.NodeLeaseDurationSeconds - out.ImageMinimumGCAge = in.ImageMinimumGCAge - if err := v1.Convert_Pointer_int32_To_int32(&in.ImageGCHighThresholdPercent, &out.ImageGCHighThresholdPercent, s); err != nil { - return err - } - if err := v1.Convert_Pointer_int32_To_int32(&in.ImageGCLowThresholdPercent, &out.ImageGCLowThresholdPercent, s); err != nil { - return err - } - out.VolumeStatsAggPeriod = in.VolumeStatsAggPeriod - out.KubeletCgroups = in.KubeletCgroups - out.SystemCgroups = in.SystemCgroups - out.CgroupRoot = in.CgroupRoot - if err := v1.Convert_Pointer_bool_To_bool(&in.CgroupsPerQOS, &out.CgroupsPerQOS, s); err != nil { - return err - } - out.CgroupDriver = in.CgroupDriver - out.CPUManagerPolicy = in.CPUManagerPolicy - out.CPUManagerReconcilePeriod = in.CPUManagerReconcilePeriod - out.TopologyManagerPolicy = in.TopologyManagerPolicy - out.QOSReserved = *(*map[string]string)(unsafe.Pointer(&in.QOSReserved)) - out.RuntimeRequestTimeout = in.RuntimeRequestTimeout - out.HairpinMode = in.HairpinMode - out.MaxPods = in.MaxPods - out.PodCIDR = in.PodCIDR - if err := v1.Convert_Pointer_int64_To_int64(&in.PodPidsLimit, &out.PodPidsLimit, s); err != nil { - return err - } - out.ResolverConfig = in.ResolverConfig - if err := v1.Convert_Pointer_bool_To_bool(&in.CPUCFSQuota, &out.CPUCFSQuota, s); err != nil { - return err - } - if err := v1.Convert_Pointer_v1_Duration_To_v1_Duration(&in.CPUCFSQuotaPeriod, &out.CPUCFSQuotaPeriod, s); err != nil { - return err - } - out.MaxOpenFiles = in.MaxOpenFiles - out.ContentType = in.ContentType - if err := v1.Convert_Pointer_int32_To_int32(&in.KubeAPIQPS, &out.KubeAPIQPS, s); err != nil { - return err - } - out.KubeAPIBurst = in.KubeAPIBurst - if err := v1.Convert_Pointer_bool_To_bool(&in.SerializeImagePulls, &out.SerializeImagePulls, s); err != nil { - return err - } - out.EvictionHard = *(*map[string]string)(unsafe.Pointer(&in.EvictionHard)) - out.EvictionSoft = *(*map[string]string)(unsafe.Pointer(&in.EvictionSoft)) - out.EvictionSoftGracePeriod = *(*map[string]string)(unsafe.Pointer(&in.EvictionSoftGracePeriod)) - out.EvictionPressureTransitionPeriod = in.EvictionPressureTransitionPeriod - out.EvictionMaxPodGracePeriod = in.EvictionMaxPodGracePeriod - out.EvictionMinimumReclaim = *(*map[string]string)(unsafe.Pointer(&in.EvictionMinimumReclaim)) - out.PodsPerCore = in.PodsPerCore - if err := v1.Convert_Pointer_bool_To_bool(&in.EnableControllerAttachDetach, &out.EnableControllerAttachDetach, s); err != nil { - return err - } - out.ProtectKernelDefaults = in.ProtectKernelDefaults - if err := v1.Convert_Pointer_bool_To_bool(&in.MakeIPTablesUtilChains, &out.MakeIPTablesUtilChains, s); err != nil { - return err - } - if err := v1.Convert_Pointer_int32_To_int32(&in.IPTablesMasqueradeBit, &out.IPTablesMasqueradeBit, s); err != nil { - return err - } - if err := v1.Convert_Pointer_int32_To_int32(&in.IPTablesDropBit, &out.IPTablesDropBit, s); err != nil { - return err - } - out.FeatureGates = *(*map[string]bool)(unsafe.Pointer(&in.FeatureGates)) - if err := v1.Convert_Pointer_bool_To_bool(&in.FailSwapOn, &out.FailSwapOn, s); err != nil { - return err - } - out.ContainerLogMaxSize = in.ContainerLogMaxSize - if err := v1.Convert_Pointer_int32_To_int32(&in.ContainerLogMaxFiles, &out.ContainerLogMaxFiles, s); err != nil { - return err - } - out.ConfigMapAndSecretChangeDetectionStrategy = config.ResourceChangeDetectionStrategy(in.ConfigMapAndSecretChangeDetectionStrategy) - out.SystemReserved = *(*map[string]string)(unsafe.Pointer(&in.SystemReserved)) - out.KubeReserved = *(*map[string]string)(unsafe.Pointer(&in.KubeReserved)) - out.ReservedSystemCPUs = in.ReservedSystemCPUs - out.ShowHiddenMetricsForVersion = in.ShowHiddenMetricsForVersion - out.SystemReservedCgroup = in.SystemReservedCgroup - out.KubeReservedCgroup = in.KubeReservedCgroup - out.EnforceNodeAllocatable = *(*[]string)(unsafe.Pointer(&in.EnforceNodeAllocatable)) - out.AllowedUnsafeSysctls = *(*[]string)(unsafe.Pointer(&in.AllowedUnsafeSysctls)) - return nil -} - -// Convert_v1beta1_KubeletConfiguration_To_config_KubeletConfiguration is an autogenerated conversion function. -func Convert_v1beta1_KubeletConfiguration_To_config_KubeletConfiguration(in *v1beta1.KubeletConfiguration, out *config.KubeletConfiguration, s conversion.Scope) error { - return autoConvert_v1beta1_KubeletConfiguration_To_config_KubeletConfiguration(in, out, s) -} - -func autoConvert_config_KubeletConfiguration_To_v1beta1_KubeletConfiguration(in *config.KubeletConfiguration, out *v1beta1.KubeletConfiguration, s conversion.Scope) error { - out.StaticPodPath = in.StaticPodPath - out.SyncFrequency = in.SyncFrequency - out.FileCheckFrequency = in.FileCheckFrequency - out.HTTPCheckFrequency = in.HTTPCheckFrequency - out.StaticPodURL = in.StaticPodURL - out.StaticPodURLHeader = *(*map[string][]string)(unsafe.Pointer(&in.StaticPodURLHeader)) - out.Address = in.Address - out.Port = in.Port - out.ReadOnlyPort = in.ReadOnlyPort - out.TLSCertFile = in.TLSCertFile - out.TLSPrivateKeyFile = in.TLSPrivateKeyFile - out.TLSCipherSuites = *(*[]string)(unsafe.Pointer(&in.TLSCipherSuites)) - out.TLSMinVersion = in.TLSMinVersion - out.RotateCertificates = in.RotateCertificates - out.ServerTLSBootstrap = in.ServerTLSBootstrap - if err := Convert_config_KubeletAuthentication_To_v1beta1_KubeletAuthentication(&in.Authentication, &out.Authentication, s); err != nil { - return err - } - if err := Convert_config_KubeletAuthorization_To_v1beta1_KubeletAuthorization(&in.Authorization, &out.Authorization, s); err != nil { - return err - } - if err := v1.Convert_int32_To_Pointer_int32(&in.RegistryPullQPS, &out.RegistryPullQPS, s); err != nil { - return err - } - out.RegistryBurst = in.RegistryBurst - if err := v1.Convert_int32_To_Pointer_int32(&in.EventRecordQPS, &out.EventRecordQPS, s); err != nil { - return err - } - out.EventBurst = in.EventBurst - if err := v1.Convert_bool_To_Pointer_bool(&in.EnableDebuggingHandlers, &out.EnableDebuggingHandlers, s); err != nil { - return err - } - out.EnableContentionProfiling = in.EnableContentionProfiling - if err := v1.Convert_int32_To_Pointer_int32(&in.HealthzPort, &out.HealthzPort, s); err != nil { - return err - } - out.HealthzBindAddress = in.HealthzBindAddress - if err := v1.Convert_int32_To_Pointer_int32(&in.OOMScoreAdj, &out.OOMScoreAdj, s); err != nil { - return err - } - out.ClusterDomain = in.ClusterDomain - out.ClusterDNS = *(*[]string)(unsafe.Pointer(&in.ClusterDNS)) - out.StreamingConnectionIdleTimeout = in.StreamingConnectionIdleTimeout - out.NodeStatusUpdateFrequency = in.NodeStatusUpdateFrequency - out.NodeStatusReportFrequency = in.NodeStatusReportFrequency - out.NodeLeaseDurationSeconds = in.NodeLeaseDurationSeconds - out.ImageMinimumGCAge = in.ImageMinimumGCAge - if err := v1.Convert_int32_To_Pointer_int32(&in.ImageGCHighThresholdPercent, &out.ImageGCHighThresholdPercent, s); err != nil { - return err - } - if err := v1.Convert_int32_To_Pointer_int32(&in.ImageGCLowThresholdPercent, &out.ImageGCLowThresholdPercent, s); err != nil { - return err - } - out.VolumeStatsAggPeriod = in.VolumeStatsAggPeriod - out.KubeletCgroups = in.KubeletCgroups - out.SystemCgroups = in.SystemCgroups - out.CgroupRoot = in.CgroupRoot - if err := v1.Convert_bool_To_Pointer_bool(&in.CgroupsPerQOS, &out.CgroupsPerQOS, s); err != nil { - return err - } - out.CgroupDriver = in.CgroupDriver - out.CPUManagerPolicy = in.CPUManagerPolicy - out.CPUManagerReconcilePeriod = in.CPUManagerReconcilePeriod - out.TopologyManagerPolicy = in.TopologyManagerPolicy - out.QOSReserved = *(*map[string]string)(unsafe.Pointer(&in.QOSReserved)) - out.RuntimeRequestTimeout = in.RuntimeRequestTimeout - out.HairpinMode = in.HairpinMode - out.MaxPods = in.MaxPods - out.PodCIDR = in.PodCIDR - if err := v1.Convert_int64_To_Pointer_int64(&in.PodPidsLimit, &out.PodPidsLimit, s); err != nil { - return err - } - out.ResolverConfig = in.ResolverConfig - if err := v1.Convert_bool_To_Pointer_bool(&in.CPUCFSQuota, &out.CPUCFSQuota, s); err != nil { - return err - } - if err := v1.Convert_v1_Duration_To_Pointer_v1_Duration(&in.CPUCFSQuotaPeriod, &out.CPUCFSQuotaPeriod, s); err != nil { - return err - } - out.MaxOpenFiles = in.MaxOpenFiles - out.ContentType = in.ContentType - if err := v1.Convert_int32_To_Pointer_int32(&in.KubeAPIQPS, &out.KubeAPIQPS, s); err != nil { - return err - } - out.KubeAPIBurst = in.KubeAPIBurst - if err := v1.Convert_bool_To_Pointer_bool(&in.SerializeImagePulls, &out.SerializeImagePulls, s); err != nil { - return err - } - out.EvictionHard = *(*map[string]string)(unsafe.Pointer(&in.EvictionHard)) - out.EvictionSoft = *(*map[string]string)(unsafe.Pointer(&in.EvictionSoft)) - out.EvictionSoftGracePeriod = *(*map[string]string)(unsafe.Pointer(&in.EvictionSoftGracePeriod)) - out.EvictionPressureTransitionPeriod = in.EvictionPressureTransitionPeriod - out.EvictionMaxPodGracePeriod = in.EvictionMaxPodGracePeriod - out.EvictionMinimumReclaim = *(*map[string]string)(unsafe.Pointer(&in.EvictionMinimumReclaim)) - out.PodsPerCore = in.PodsPerCore - if err := v1.Convert_bool_To_Pointer_bool(&in.EnableControllerAttachDetach, &out.EnableControllerAttachDetach, s); err != nil { - return err - } - out.ProtectKernelDefaults = in.ProtectKernelDefaults - if err := v1.Convert_bool_To_Pointer_bool(&in.MakeIPTablesUtilChains, &out.MakeIPTablesUtilChains, s); err != nil { - return err - } - if err := v1.Convert_int32_To_Pointer_int32(&in.IPTablesMasqueradeBit, &out.IPTablesMasqueradeBit, s); err != nil { - return err - } - if err := v1.Convert_int32_To_Pointer_int32(&in.IPTablesDropBit, &out.IPTablesDropBit, s); err != nil { - return err - } - out.FeatureGates = *(*map[string]bool)(unsafe.Pointer(&in.FeatureGates)) - if err := v1.Convert_bool_To_Pointer_bool(&in.FailSwapOn, &out.FailSwapOn, s); err != nil { - return err - } - out.ContainerLogMaxSize = in.ContainerLogMaxSize - if err := v1.Convert_int32_To_Pointer_int32(&in.ContainerLogMaxFiles, &out.ContainerLogMaxFiles, s); err != nil { - return err - } - out.ConfigMapAndSecretChangeDetectionStrategy = v1beta1.ResourceChangeDetectionStrategy(in.ConfigMapAndSecretChangeDetectionStrategy) - out.AllowedUnsafeSysctls = *(*[]string)(unsafe.Pointer(&in.AllowedUnsafeSysctls)) - out.SystemReserved = *(*map[string]string)(unsafe.Pointer(&in.SystemReserved)) - out.KubeReserved = *(*map[string]string)(unsafe.Pointer(&in.KubeReserved)) - out.SystemReservedCgroup = in.SystemReservedCgroup - out.KubeReservedCgroup = in.KubeReservedCgroup - out.EnforceNodeAllocatable = *(*[]string)(unsafe.Pointer(&in.EnforceNodeAllocatable)) - out.ReservedSystemCPUs = in.ReservedSystemCPUs - out.ShowHiddenMetricsForVersion = in.ShowHiddenMetricsForVersion - return nil -} - -// Convert_config_KubeletConfiguration_To_v1beta1_KubeletConfiguration is an autogenerated conversion function. -func Convert_config_KubeletConfiguration_To_v1beta1_KubeletConfiguration(in *config.KubeletConfiguration, out *v1beta1.KubeletConfiguration, s conversion.Scope) error { - return autoConvert_config_KubeletConfiguration_To_v1beta1_KubeletConfiguration(in, out, s) -} - -func autoConvert_v1beta1_KubeletWebhookAuthentication_To_config_KubeletWebhookAuthentication(in *v1beta1.KubeletWebhookAuthentication, out *config.KubeletWebhookAuthentication, s conversion.Scope) error { - if err := v1.Convert_Pointer_bool_To_bool(&in.Enabled, &out.Enabled, s); err != nil { - return err - } - out.CacheTTL = in.CacheTTL - return nil -} - -// Convert_v1beta1_KubeletWebhookAuthentication_To_config_KubeletWebhookAuthentication is an autogenerated conversion function. -func Convert_v1beta1_KubeletWebhookAuthentication_To_config_KubeletWebhookAuthentication(in *v1beta1.KubeletWebhookAuthentication, out *config.KubeletWebhookAuthentication, s conversion.Scope) error { - return autoConvert_v1beta1_KubeletWebhookAuthentication_To_config_KubeletWebhookAuthentication(in, out, s) -} - -func autoConvert_config_KubeletWebhookAuthentication_To_v1beta1_KubeletWebhookAuthentication(in *config.KubeletWebhookAuthentication, out *v1beta1.KubeletWebhookAuthentication, s conversion.Scope) error { - if err := v1.Convert_bool_To_Pointer_bool(&in.Enabled, &out.Enabled, s); err != nil { - return err - } - out.CacheTTL = in.CacheTTL - return nil -} - -// Convert_config_KubeletWebhookAuthentication_To_v1beta1_KubeletWebhookAuthentication is an autogenerated conversion function. -func Convert_config_KubeletWebhookAuthentication_To_v1beta1_KubeletWebhookAuthentication(in *config.KubeletWebhookAuthentication, out *v1beta1.KubeletWebhookAuthentication, s conversion.Scope) error { - return autoConvert_config_KubeletWebhookAuthentication_To_v1beta1_KubeletWebhookAuthentication(in, out, s) -} - -func autoConvert_v1beta1_KubeletWebhookAuthorization_To_config_KubeletWebhookAuthorization(in *v1beta1.KubeletWebhookAuthorization, out *config.KubeletWebhookAuthorization, s conversion.Scope) error { - out.CacheAuthorizedTTL = in.CacheAuthorizedTTL - out.CacheUnauthorizedTTL = in.CacheUnauthorizedTTL - return nil -} - -// Convert_v1beta1_KubeletWebhookAuthorization_To_config_KubeletWebhookAuthorization is an autogenerated conversion function. -func Convert_v1beta1_KubeletWebhookAuthorization_To_config_KubeletWebhookAuthorization(in *v1beta1.KubeletWebhookAuthorization, out *config.KubeletWebhookAuthorization, s conversion.Scope) error { - return autoConvert_v1beta1_KubeletWebhookAuthorization_To_config_KubeletWebhookAuthorization(in, out, s) -} - -func autoConvert_config_KubeletWebhookAuthorization_To_v1beta1_KubeletWebhookAuthorization(in *config.KubeletWebhookAuthorization, out *v1beta1.KubeletWebhookAuthorization, s conversion.Scope) error { - out.CacheAuthorizedTTL = in.CacheAuthorizedTTL - out.CacheUnauthorizedTTL = in.CacheUnauthorizedTTL - return nil -} - -// Convert_config_KubeletWebhookAuthorization_To_v1beta1_KubeletWebhookAuthorization is an autogenerated conversion function. -func Convert_config_KubeletWebhookAuthorization_To_v1beta1_KubeletWebhookAuthorization(in *config.KubeletWebhookAuthorization, out *v1beta1.KubeletWebhookAuthorization, s conversion.Scope) error { - return autoConvert_config_KubeletWebhookAuthorization_To_v1beta1_KubeletWebhookAuthorization(in, out, s) -} - -func autoConvert_v1beta1_KubeletX509Authentication_To_config_KubeletX509Authentication(in *v1beta1.KubeletX509Authentication, out *config.KubeletX509Authentication, s conversion.Scope) error { - out.ClientCAFile = in.ClientCAFile - return nil -} - -// Convert_v1beta1_KubeletX509Authentication_To_config_KubeletX509Authentication is an autogenerated conversion function. -func Convert_v1beta1_KubeletX509Authentication_To_config_KubeletX509Authentication(in *v1beta1.KubeletX509Authentication, out *config.KubeletX509Authentication, s conversion.Scope) error { - return autoConvert_v1beta1_KubeletX509Authentication_To_config_KubeletX509Authentication(in, out, s) -} - -func autoConvert_config_KubeletX509Authentication_To_v1beta1_KubeletX509Authentication(in *config.KubeletX509Authentication, out *v1beta1.KubeletX509Authentication, s conversion.Scope) error { - out.ClientCAFile = in.ClientCAFile - return nil -} - -// Convert_config_KubeletX509Authentication_To_v1beta1_KubeletX509Authentication is an autogenerated conversion function. -func Convert_config_KubeletX509Authentication_To_v1beta1_KubeletX509Authentication(in *config.KubeletX509Authentication, out *v1beta1.KubeletX509Authentication, s conversion.Scope) error { - return autoConvert_config_KubeletX509Authentication_To_v1beta1_KubeletX509Authentication(in, out, s) -} - -func autoConvert_v1beta1_SerializedNodeConfigSource_To_config_SerializedNodeConfigSource(in *v1beta1.SerializedNodeConfigSource, out *config.SerializedNodeConfigSource, s conversion.Scope) error { - out.Source = in.Source - return nil -} - -// Convert_v1beta1_SerializedNodeConfigSource_To_config_SerializedNodeConfigSource is an autogenerated conversion function. -func Convert_v1beta1_SerializedNodeConfigSource_To_config_SerializedNodeConfigSource(in *v1beta1.SerializedNodeConfigSource, out *config.SerializedNodeConfigSource, s conversion.Scope) error { - return autoConvert_v1beta1_SerializedNodeConfigSource_To_config_SerializedNodeConfigSource(in, out, s) -} - -func autoConvert_config_SerializedNodeConfigSource_To_v1beta1_SerializedNodeConfigSource(in *config.SerializedNodeConfigSource, out *v1beta1.SerializedNodeConfigSource, s conversion.Scope) error { - out.Source = in.Source - return nil -} - -// Convert_config_SerializedNodeConfigSource_To_v1beta1_SerializedNodeConfigSource is an autogenerated conversion function. -func Convert_config_SerializedNodeConfigSource_To_v1beta1_SerializedNodeConfigSource(in *config.SerializedNodeConfigSource, out *v1beta1.SerializedNodeConfigSource, s conversion.Scope) error { - return autoConvert_config_SerializedNodeConfigSource_To_v1beta1_SerializedNodeConfigSource(in, out, s) -} diff --git a/pkg/util/kubernetes/kubelet/apis/config/v1beta1/zz_generated.defaults.go b/pkg/util/kubernetes/kubelet/apis/config/v1beta1/zz_generated.defaults.go deleted file mode 100644 index eb800358626..00000000000 --- a/pkg/util/kubernetes/kubelet/apis/config/v1beta1/zz_generated.defaults.go +++ /dev/null @@ -1,39 +0,0 @@ -//go:build !ignore_autogenerated -// +build !ignore_autogenerated - -/* -Copyright The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Code generated by defaulter-gen. DO NOT EDIT. - -package v1beta1 - -import ( - runtime "k8s.io/apimachinery/pkg/runtime" - v1beta1 "k8s.io/kubelet/config/v1beta1" -) - -// RegisterDefaults adds defaulters functions to the given scheme. -// Public to allow building arbitrary schemes. -// All generated defaulters are covering - they call all nested defaulters. -func RegisterDefaults(scheme *runtime.Scheme) error { - scheme.AddTypeDefaultingFunc(&v1beta1.KubeletConfiguration{}, func(obj interface{}) { SetObjectDefaults_KubeletConfiguration(obj.(*v1beta1.KubeletConfiguration)) }) - return nil -} - -func SetObjectDefaults_KubeletConfiguration(in *v1beta1.KubeletConfiguration) { - SetDefaults_KubeletConfiguration(in) -} diff --git a/pkg/util/kubernetes/kubelet/apis/config/zz_generated.deepcopy.go b/pkg/util/kubernetes/kubelet/apis/config/zz_generated.deepcopy.go deleted file mode 100644 index 5584478a392..00000000000 --- a/pkg/util/kubernetes/kubelet/apis/config/zz_generated.deepcopy.go +++ /dev/null @@ -1,285 +0,0 @@ -//go:build !ignore_autogenerated -// +build !ignore_autogenerated - -/* -Copyright The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Code generated by deepcopy-gen. DO NOT EDIT. - -package config - -import ( - runtime "k8s.io/apimachinery/pkg/runtime" -) - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *KubeletAnonymousAuthentication) DeepCopyInto(out *KubeletAnonymousAuthentication) { - *out = *in - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KubeletAnonymousAuthentication. -func (in *KubeletAnonymousAuthentication) DeepCopy() *KubeletAnonymousAuthentication { - if in == nil { - return nil - } - out := new(KubeletAnonymousAuthentication) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *KubeletAuthentication) DeepCopyInto(out *KubeletAuthentication) { - *out = *in - out.X509 = in.X509 - out.Webhook = in.Webhook - out.Anonymous = in.Anonymous - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KubeletAuthentication. -func (in *KubeletAuthentication) DeepCopy() *KubeletAuthentication { - if in == nil { - return nil - } - out := new(KubeletAuthentication) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *KubeletAuthorization) DeepCopyInto(out *KubeletAuthorization) { - *out = *in - out.Webhook = in.Webhook - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KubeletAuthorization. -func (in *KubeletAuthorization) DeepCopy() *KubeletAuthorization { - if in == nil { - return nil - } - out := new(KubeletAuthorization) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *KubeletConfiguration) DeepCopyInto(out *KubeletConfiguration) { - *out = *in - out.TypeMeta = in.TypeMeta - out.SyncFrequency = in.SyncFrequency - out.FileCheckFrequency = in.FileCheckFrequency - out.HTTPCheckFrequency = in.HTTPCheckFrequency - if in.StaticPodURLHeader != nil { - in, out := &in.StaticPodURLHeader, &out.StaticPodURLHeader - *out = make(map[string][]string, len(*in)) - for key, val := range *in { - var outVal []string - if val == nil { - (*out)[key] = nil - } else { - in, out := &val, &outVal - *out = make([]string, len(*in)) - copy(*out, *in) - } - (*out)[key] = outVal - } - } - if in.TLSCipherSuites != nil { - in, out := &in.TLSCipherSuites, &out.TLSCipherSuites - *out = make([]string, len(*in)) - copy(*out, *in) - } - out.Authentication = in.Authentication - out.Authorization = in.Authorization - if in.ClusterDNS != nil { - in, out := &in.ClusterDNS, &out.ClusterDNS - *out = make([]string, len(*in)) - copy(*out, *in) - } - out.StreamingConnectionIdleTimeout = in.StreamingConnectionIdleTimeout - out.NodeStatusUpdateFrequency = in.NodeStatusUpdateFrequency - out.NodeStatusReportFrequency = in.NodeStatusReportFrequency - out.ImageMinimumGCAge = in.ImageMinimumGCAge - out.VolumeStatsAggPeriod = in.VolumeStatsAggPeriod - out.CPUManagerReconcilePeriod = in.CPUManagerReconcilePeriod - if in.QOSReserved != nil { - in, out := &in.QOSReserved, &out.QOSReserved - *out = make(map[string]string, len(*in)) - for key, val := range *in { - (*out)[key] = val - } - } - out.RuntimeRequestTimeout = in.RuntimeRequestTimeout - out.CPUCFSQuotaPeriod = in.CPUCFSQuotaPeriod - if in.EvictionHard != nil { - in, out := &in.EvictionHard, &out.EvictionHard - *out = make(map[string]string, len(*in)) - for key, val := range *in { - (*out)[key] = val - } - } - if in.EvictionSoft != nil { - in, out := &in.EvictionSoft, &out.EvictionSoft - *out = make(map[string]string, len(*in)) - for key, val := range *in { - (*out)[key] = val - } - } - if in.EvictionSoftGracePeriod != nil { - in, out := &in.EvictionSoftGracePeriod, &out.EvictionSoftGracePeriod - *out = make(map[string]string, len(*in)) - for key, val := range *in { - (*out)[key] = val - } - } - out.EvictionPressureTransitionPeriod = in.EvictionPressureTransitionPeriod - if in.EvictionMinimumReclaim != nil { - in, out := &in.EvictionMinimumReclaim, &out.EvictionMinimumReclaim - *out = make(map[string]string, len(*in)) - for key, val := range *in { - (*out)[key] = val - } - } - if in.FeatureGates != nil { - in, out := &in.FeatureGates, &out.FeatureGates - *out = make(map[string]bool, len(*in)) - for key, val := range *in { - (*out)[key] = val - } - } - if in.AllowedUnsafeSysctls != nil { - in, out := &in.AllowedUnsafeSysctls, &out.AllowedUnsafeSysctls - *out = make([]string, len(*in)) - copy(*out, *in) - } - if in.SystemReserved != nil { - in, out := &in.SystemReserved, &out.SystemReserved - *out = make(map[string]string, len(*in)) - for key, val := range *in { - (*out)[key] = val - } - } - if in.KubeReserved != nil { - in, out := &in.KubeReserved, &out.KubeReserved - *out = make(map[string]string, len(*in)) - for key, val := range *in { - (*out)[key] = val - } - } - if in.EnforceNodeAllocatable != nil { - in, out := &in.EnforceNodeAllocatable, &out.EnforceNodeAllocatable - *out = make([]string, len(*in)) - copy(*out, *in) - } - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KubeletConfiguration. -func (in *KubeletConfiguration) DeepCopy() *KubeletConfiguration { - if in == nil { - return nil - } - out := new(KubeletConfiguration) - in.DeepCopyInto(out) - return out -} - -// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. -func (in *KubeletConfiguration) DeepCopyObject() runtime.Object { - if c := in.DeepCopy(); c != nil { - return c - } - return nil -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *KubeletWebhookAuthentication) DeepCopyInto(out *KubeletWebhookAuthentication) { - *out = *in - out.CacheTTL = in.CacheTTL - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KubeletWebhookAuthentication. -func (in *KubeletWebhookAuthentication) DeepCopy() *KubeletWebhookAuthentication { - if in == nil { - return nil - } - out := new(KubeletWebhookAuthentication) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *KubeletWebhookAuthorization) DeepCopyInto(out *KubeletWebhookAuthorization) { - *out = *in - out.CacheAuthorizedTTL = in.CacheAuthorizedTTL - out.CacheUnauthorizedTTL = in.CacheUnauthorizedTTL - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KubeletWebhookAuthorization. -func (in *KubeletWebhookAuthorization) DeepCopy() *KubeletWebhookAuthorization { - if in == nil { - return nil - } - out := new(KubeletWebhookAuthorization) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *KubeletX509Authentication) DeepCopyInto(out *KubeletX509Authentication) { - *out = *in - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KubeletX509Authentication. -func (in *KubeletX509Authentication) DeepCopy() *KubeletX509Authentication { - if in == nil { - return nil - } - out := new(KubeletX509Authentication) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *SerializedNodeConfigSource) DeepCopyInto(out *SerializedNodeConfigSource) { - *out = *in - out.TypeMeta = in.TypeMeta - in.Source.DeepCopyInto(&out.Source) - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SerializedNodeConfigSource. -func (in *SerializedNodeConfigSource) DeepCopy() *SerializedNodeConfigSource { - if in == nil { - return nil - } - out := new(SerializedNodeConfigSource) - in.DeepCopyInto(out) - return out -} - -// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. -func (in *SerializedNodeConfigSource) DeepCopyObject() runtime.Object { - if c := in.DeepCopy(); c != nil { - return c - } - return nil -} diff --git a/pkg/util/kubernetes/kubelet/kubeletconfig/util/codec/codec.go b/pkg/util/kubernetes/kubelet/kubeletconfig/util/codec/codec.go deleted file mode 100644 index ce4c74c0b5a..00000000000 --- a/pkg/util/kubernetes/kubelet/kubeletconfig/util/codec/codec.go +++ /dev/null @@ -1,106 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. -Copyright 2021 The OpenYurt Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package codec - -import ( - "fmt" - - "github.com/pkg/errors" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/runtime/schema" - "k8s.io/apimachinery/pkg/runtime/serializer" - "k8s.io/component-base/codec" - "k8s.io/klog/v2" - kubeletconfigv1beta1 "k8s.io/kubelet/config/v1beta1" - - kubeletconfig "github.com/openyurtio/openyurt/pkg/util/kubernetes/kubelet/apis/config" - "github.com/openyurtio/openyurt/pkg/util/kubernetes/kubelet/apis/config/scheme" -) - -// EncodeKubeletConfig encodes an internal KubeletConfiguration to an external YAML representation. -func EncodeKubeletConfig(internal *kubeletconfig.KubeletConfiguration, targetVersion schema.GroupVersion) ([]byte, error) { - encoder, err := NewKubeletconfigYAMLEncoder(targetVersion) - if err != nil { - return nil, err - } - // encoder will convert to external version - data, err := runtime.Encode(encoder, internal) - if err != nil { - return nil, err - } - return data, nil -} - -// NewKubeletconfigYAMLEncoder returns an encoder that can write objects in the kubeletconfig API group to YAML. -func NewKubeletconfigYAMLEncoder(targetVersion schema.GroupVersion) (runtime.Encoder, error) { - _, codecs, err := scheme.NewSchemeAndCodecs() - if err != nil { - return nil, err - } - mediaType := "application/yaml" - info, ok := runtime.SerializerInfoForMediaType(codecs.SupportedMediaTypes(), mediaType) - if !ok { - return nil, fmt.Errorf("unsupported media type %q", mediaType) - } - return codecs.EncoderForVersion(info.Serializer, targetVersion), nil -} - -// DecodeKubeletConfiguration decodes a serialized KubeletConfiguration to the internal type. -func DecodeKubeletConfiguration(kubeletCodecs *serializer.CodecFactory, data []byte) (*kubeletconfig.KubeletConfiguration, error) { - var ( - obj runtime.Object - gvk *schema.GroupVersionKind - ) - - // The UniversalDecoder runs defaulting and returns the internal type by default. - obj, gvk, err := kubeletCodecs.UniversalDecoder().Decode(data, nil, nil) - if err != nil { - // Try strict decoding first. If that fails decode with a lenient - // decoder, which has only v1beta1 registered, and log a warning. - // The lenient path is to be dropped when support for v1beta1 is dropped. - if !runtime.IsStrictDecodingError(err) { - return nil, errors.Wrap(err, "failed to decode") - } - - var lenientErr error - _, lenientCodecs, lenientErr := codec.NewLenientSchemeAndCodecs( - kubeletconfig.AddToScheme, - kubeletconfigv1beta1.AddToScheme, - ) - - if lenientErr != nil { - return nil, lenientErr - } - - obj, gvk, lenientErr = lenientCodecs.UniversalDecoder().Decode(data, nil, nil) - if lenientErr != nil { - // Lenient decoding failed with the current version, return the - // original strict error. - return nil, fmt.Errorf("failed lenient decoding: %w", err) - } - // Continue with the v1beta1 object that was decoded leniently, but emit a warning. - klog.Warningf("using lenient decoding as strict decoding failed: %v", err) - } - - internalKC, ok := obj.(*kubeletconfig.KubeletConfiguration) - if !ok { - return nil, fmt.Errorf("failed to cast object to KubeletConfiguration, unexpected type: %v", gvk) - } - - return internalKC, nil -} diff --git a/pkg/yurtadm/cmd/cmd.go b/pkg/yurtadm/cmd/cmd.go index 3f5b3d3e3fd..01a7daa8abc 100644 --- a/pkg/yurtadm/cmd/cmd.go +++ b/pkg/yurtadm/cmd/cmd.go @@ -46,7 +46,7 @@ func NewYurtadmCommand() *cobra.Command { cmds.PersistentFlags().String("kubeconfig", "", "The path to the kubeconfig file") cmds.AddCommand(yurtinit.NewCmdInit()) cmds.AddCommand(join.NewCmdJoin(os.Stdin, os.Stdout, os.Stderr)) - cmds.AddCommand(reset.NewCmdReset(os.Stdin, os.Stdout, nil)) + cmds.AddCommand(reset.NewCmdReset(os.Stdin, os.Stdout, os.Stderr)) cmds.AddCommand(token.NewCmdToken(os.Stdin, os.Stdout, os.Stderr)) klog.InitFlags(nil) diff --git a/pkg/yurtadm/cmd/join/join.go b/pkg/yurtadm/cmd/join/join.go index f0da6dd09b9..16a2b3c9e33 100644 --- a/pkg/yurtadm/cmd/join/join.go +++ b/pkg/yurtadm/cmd/join/join.go @@ -30,7 +30,6 @@ import ( clientcmdapi "k8s.io/client-go/tools/clientcmd/api" "k8s.io/klog/v2" - "github.com/openyurtio/openyurt/pkg/util/kubernetes/kubeadm/app/cmd/options" "github.com/openyurtio/openyurt/pkg/yurtadm/cmd/join/joindata" yurtphases "github.com/openyurtio/openyurt/pkg/yurtadm/cmd/join/phases" yurtconstants "github.com/openyurtio/openyurt/pkg/yurtadm/constants" @@ -106,7 +105,7 @@ func NewCmdJoin(in io.Reader, out io.Writer, outErr io.Writer) *cobra.Command { // addJoinConfigFlags adds join flags bound to the config to the specified flagset func addJoinConfigFlags(flagSet *flag.FlagSet, joinOptions *joinOptions) { flagSet.StringVar( - &joinOptions.token, options.TokenStr, "", + &joinOptions.token, yurtconstants.TokenStr, "", "Use this token for both discovery-token and tls-bootstrap-token when those values are not provided.", ) flagSet.StringVar( @@ -114,11 +113,11 @@ func addJoinConfigFlags(flagSet *flag.FlagSet, joinOptions *joinOptions) { "Sets the node is edge or cloud", ) flagSet.StringVar( - &joinOptions.nodeName, options.NodeName, joinOptions.nodeName, + &joinOptions.nodeName, yurtconstants.NodeName, joinOptions.nodeName, `Specify the node name. if not specified, hostname will be used.`, ) flagSet.StringVar( - &joinOptions.criSocket, options.NodeCRISocket, joinOptions.criSocket, + &joinOptions.criSocket, yurtconstants.NodeCRISocket, joinOptions.criSocket, "Path to the CRI socket to connect", ) flagSet.StringVar( @@ -134,15 +133,15 @@ func addJoinConfigFlags(flagSet *flag.FlagSet, joinOptions *joinOptions) { "Sets the image version of yurthub component", ) flagSet.StringSliceVar( - &joinOptions.caCertHashes, options.TokenDiscoveryCAHash, joinOptions.caCertHashes, + &joinOptions.caCertHashes, yurtconstants.TokenDiscoveryCAHash, joinOptions.caCertHashes, "For token-based discovery, validate that the root CA public key matches this hash (format: \":\").", ) flagSet.BoolVar( - &joinOptions.unsafeSkipCAVerification, options.TokenDiscoverySkipCAHash, false, + &joinOptions.unsafeSkipCAVerification, yurtconstants.TokenDiscoverySkipCAHash, false, "For token-based discovery, allow joining without --discovery-token-ca-cert-hash pinning.", ) flagSet.StringSliceVar( - &joinOptions.ignorePreflightErrors, options.IgnorePreflightErrors, joinOptions.ignorePreflightErrors, + &joinOptions.ignorePreflightErrors, yurtconstants.IgnorePreflightErrors, joinOptions.ignorePreflightErrors, "A list of checks whose errors will be shown as warnings. Example: 'IsPrivilegedUser,Swap'. Value 'all' ignores errors from all checks.", ) flagSet.StringVar( diff --git a/pkg/yurtadm/cmd/join/phases/postcheck.go b/pkg/yurtadm/cmd/join/phases/postcheck.go index 9dc522a8199..aaa77e17dcc 100644 --- a/pkg/yurtadm/cmd/join/phases/postcheck.go +++ b/pkg/yurtadm/cmd/join/phases/postcheck.go @@ -25,9 +25,9 @@ import ( "k8s.io/apimachinery/pkg/util/wait" "k8s.io/klog/v2" - "github.com/openyurtio/openyurt/pkg/util/kubernetes/kubeadm/app/util/initsystem" "github.com/openyurtio/openyurt/pkg/yurtadm/cmd/join/joindata" "github.com/openyurtio/openyurt/pkg/yurtadm/constants" + "github.com/openyurtio/openyurt/pkg/yurtadm/util/initsystem" ) // RunPostCheck executes the node health check process. diff --git a/pkg/yurtadm/cmd/reset/phases/cleanupnode.go b/pkg/yurtadm/cmd/reset/phases/cleanupnode.go deleted file mode 100644 index 4bfd88f6200..00000000000 --- a/pkg/yurtadm/cmd/reset/phases/cleanupnode.go +++ /dev/null @@ -1,154 +0,0 @@ -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package phases - -import ( - "errors" - "os" - "path/filepath" - - "k8s.io/klog/v2" - utilsexec "k8s.io/utils/exec" - - "github.com/openyurtio/openyurt/pkg/util/kubernetes/kubeadm/app/cmd/options" - "github.com/openyurtio/openyurt/pkg/util/kubernetes/kubeadm/app/cmd/phases/workflow" - kubeadmconstants "github.com/openyurtio/openyurt/pkg/util/kubernetes/kubeadm/app/constants" - kubeutil "github.com/openyurtio/openyurt/pkg/util/kubernetes/kubeadm/app/phases/kubelet" - utilruntime "github.com/openyurtio/openyurt/pkg/util/kubernetes/kubeadm/app/util/runtime" -) - -// NewCleanupNodePhase creates a kubeadm workflow phase that cleanup the node -func NewCleanupNodePhase() workflow.Phase { - return workflow.Phase{ - Name: "cleanup-node", - Aliases: []string{"cleanupnode"}, - Short: "Run cleanup node.", - Run: runCleanupNode, - InheritFlags: []string{ - options.NodeCRISocket, - }, - } -} - -func runCleanupNode(c workflow.RunData) error { - r, ok := c.(resetData) - if !ok { - return errors.New("cleanup-node phase invoked with an invalid data struct") - } - - // Try to stop the kubelet service - klog.Infoln("[reset] Stopping the kubelet service") - kubeutil.TryStopKubelet() - - // Try to unmount mounted directories under kubeadmconstants.KubeletRunDirectory in order to be able to remove the kubeadmconstants.KubeletRunDirectory directory later - klog.Infof("[reset] Unmounting mounted directories in %q", kubeadmconstants.KubeletRunDirectory) - // In case KubeletRunDirectory holds a symbolic link, evaluate it - kubeletRunDir, err := absoluteKubeletRunDirectory() - if err == nil { - // Only clean absoluteKubeletRunDirectory if umountDirsCmd passed without error - r.AddDirsToClean(kubeletRunDir) - } - - klog.V(1).Info("[reset] Removing Kubernetes-managed containers") - if err := removeContainers(utilsexec.New(), r.CRISocketPath()); err != nil { - klog.Warningf("[reset] Failed to remove containers: %v", err) - } - - r.AddDirsToClean("/var/lib/dockershim", "/var/run/kubernetes", "/var/lib/cni") - - // Remove contents from the config and pki directories - klog.V(1).Infoln("[reset] Removing contents from the config and pki directories") - certsDir := filepath.Join(kubeadmconstants.KubernetesDir, "pki") - resetConfigDir(kubeadmconstants.KubernetesDir, certsDir) - - return nil -} - -func absoluteKubeletRunDirectory() (string, error) { - absoluteKubeletRunDirectory, err := filepath.EvalSymlinks(kubeadmconstants.KubeletRunDirectory) - if err != nil { - klog.Warningf("[reset] Failed to evaluate the %q directory. Skipping its unmount and cleanup: %v", kubeadmconstants.KubeletRunDirectory, err) - return "", err - } - err = unmountKubeletDirectory(absoluteKubeletRunDirectory) - if err != nil { - klog.Warningf("[reset] Failed to unmount mounted directories in %s", kubeadmconstants.KubeletRunDirectory) - return "", err - } - return absoluteKubeletRunDirectory, nil -} - -func removeContainers(execer utilsexec.Interface, criSocketPath string) error { - containerRuntime, err := utilruntime.NewContainerRuntime(execer, criSocketPath) - if err != nil { - return err - } - containers, err := containerRuntime.ListKubeContainers() - if err != nil { - return err - } - return containerRuntime.RemoveContainers(containers) -} - -// resetConfigDir is used to cleanup the files kubeadm writes in /etc/kubernetes/. -func resetConfigDir(configPathDir, pkiPathDir string) { - dirsToClean := []string{ - filepath.Join(configPathDir, kubeadmconstants.ManifestsSubDirName), - pkiPathDir, - } - klog.Infof("[reset] Deleting contents of config directories: %v", dirsToClean) - for _, dir := range dirsToClean { - if err := CleanDir(dir); err != nil { - klog.Warningf("[reset] Failed to delete contents of %q directory: %v", dir, err) - } - } - - filesToClean := []string{ - filepath.Join(configPathDir, kubeadmconstants.KubeletKubeConfigFileName), - } - klog.Infof("[reset] Deleting files: %v", filesToClean) - for _, path := range filesToClean { - if err := os.RemoveAll(path); err != nil { - klog.Warningf("[reset] Failed to remove file: %q [%v]", path, err) - } - } -} - -// CleanDir removes everything in a directory, but not the directory itself -func CleanDir(filePath string) error { - // If the directory doesn't even exist there's nothing to do, and we do - // not consider this an error - if _, err := os.Stat(filePath); os.IsNotExist(err) { - return nil - } - - d, err := os.Open(filePath) - if err != nil { - return err - } - defer d.Close() - names, err := d.Readdirnames(-1) - if err != nil { - return err - } - for _, name := range names { - if err = os.RemoveAll(filepath.Join(filePath, name)); err != nil { - return err - } - } - return nil -} diff --git a/pkg/yurtadm/cmd/reset/phases/cleanyurtfile.go b/pkg/yurtadm/cmd/reset/phases/cleanyurtfile.go index 2b63a3436b7..d0898ff1629 100644 --- a/pkg/yurtadm/cmd/reset/phases/cleanyurtfile.go +++ b/pkg/yurtadm/cmd/reset/phases/cleanyurtfile.go @@ -22,19 +22,10 @@ import ( "k8s.io/klog/v2" - "github.com/openyurtio/openyurt/pkg/util/kubernetes/kubeadm/app/cmd/phases/workflow" "github.com/openyurtio/openyurt/pkg/yurtadm/constants" ) -func NewCleanYurtFilePhase() workflow.Phase { - return workflow.Phase{ - Name: "Clean up the directories and files related to openyurt.", - Short: "Clean up the directories and files related to openyurt.", - Run: runCleanfile, - } -} - -func runCleanfile(c workflow.RunData) error { +func RunCleanYurtFile() error { for _, comp := range []string{"kubectl", "kubeadm", "kubelet"} { target := fmt.Sprintf("/usr/bin/%s", comp) if err := os.RemoveAll(target); err != nil { diff --git a/pkg/yurtadm/cmd/reset/phases/data.go b/pkg/yurtadm/cmd/reset/phases/data.go deleted file mode 100644 index 47a6e19df6f..00000000000 --- a/pkg/yurtadm/cmd/reset/phases/data.go +++ /dev/null @@ -1,33 +0,0 @@ -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package phases - -import ( - "io" - - "k8s.io/apimachinery/pkg/util/sets" -) - -// resetData is the interface to use for reset phases. -// The "resetData" type from "cmd/reset.go" must satisfy this interface. -type resetData interface { - ForceReset() bool - InputReader() io.Reader - IgnorePreflightErrors() sets.String - AddDirsToClean(dirs ...string) - CRISocketPath() string -} diff --git a/pkg/yurtadm/cmd/reset/phases/preflight.go b/pkg/yurtadm/cmd/reset/phases/preflight.go deleted file mode 100644 index e88d8d33faf..00000000000 --- a/pkg/yurtadm/cmd/reset/phases/preflight.go +++ /dev/null @@ -1,68 +0,0 @@ -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package phases - -import ( - "bufio" - "errors" - "strings" - - "k8s.io/klog/v2" - - "github.com/openyurtio/openyurt/pkg/util/kubernetes/kubeadm/app/cmd/options" - "github.com/openyurtio/openyurt/pkg/util/kubernetes/kubeadm/app/cmd/phases/workflow" - "github.com/openyurtio/openyurt/pkg/util/kubernetes/kubeadm/app/preflight" -) - -// NewPreflightPhase creates a kubeadm workflow phase implements preflight checks for reset -func NewPreflightPhase() workflow.Phase { - return workflow.Phase{ - Name: "preflight", - Aliases: []string{"pre-flight"}, - Short: "Run reset pre-flight checks", - Long: "Run pre-flight checks for kubeadm reset.", - Run: runPreflight, - InheritFlags: []string{ - options.IgnorePreflightErrors, - options.ForceReset, - }, - } -} - -// runPreflight executes preflight checks logic. -func runPreflight(c workflow.RunData) error { - r, ok := c.(resetData) - if !ok { - return errors.New("preflight phase invoked with an invalid data struct") - } - - if !r.ForceReset() { - klog.Infoln("[reset] WARNING: Changes made to this host by 'kubeadm init' or 'kubeadm join' will be reverted.") - klog.Info("[reset] Are you sure you want to proceed? [y/N]: ") - s := bufio.NewScanner(r.InputReader()) - s.Scan() - if err := s.Err(); err != nil { - return err - } - if strings.ToLower(s.Text()) != "y" { - return errors.New("aborted reset operation") - } - } - - klog.Infoln("[preflight] Running pre-flight checks") - return preflight.RunRootCheckOnly(r.IgnorePreflightErrors()) -} diff --git a/pkg/yurtadm/cmd/reset/phases/resetnode.go b/pkg/yurtadm/cmd/reset/phases/resetnode.go new file mode 100644 index 00000000000..2391e93d9a4 --- /dev/null +++ b/pkg/yurtadm/cmd/reset/phases/resetnode.go @@ -0,0 +1,52 @@ +/* +Copyright 2021 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package phases + +import ( + "io" + "os/exec" + "strconv" + "strings" + + "k8s.io/klog/v2" + + "github.com/openyurtio/openyurt/pkg/yurtadm/cmd/reset/resetdata" + "github.com/openyurtio/openyurt/pkg/yurtadm/constants" +) + +func RunResetNode(data resetdata.YurtResetData, in io.Reader, out io.Writer, outErr io.Writer) error { + if _, err := exec.LookPath("kubeadm"); err != nil { + klog.Fatalf("kubeadm is not installed, you can refer to this link for installation: %s.", constants.KubeadmInstallUrl) + return err + } + + kubeadmCmd := exec.Command("kubeadm", "reset", + "--cert-dir="+data.CertificatesDir(), + "--cri-socket="+data.CRISocketPath(), + "--force="+strconv.FormatBool(data.ForceReset()), + "--ignore-preflight-errors="+strings.Join(data.IgnorePreflightErrors(), ","), + ) + kubeadmCmd.Stdin = in + kubeadmCmd.Stdout = out + kubeadmCmd.Stderr = outErr + + if err := kubeadmCmd.Run(); err != nil { + return err + } + + return nil +} diff --git a/pkg/yurtadm/cmd/reset/phases/unmount.go b/pkg/yurtadm/cmd/reset/phases/unmount.go deleted file mode 100644 index 1796da898b9..00000000000 --- a/pkg/yurtadm/cmd/reset/phases/unmount.go +++ /dev/null @@ -1,30 +0,0 @@ -//go:build !linux -// +build !linux - -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package phases - -import ( - "k8s.io/klog/v2" -) - -// unmountKubeletDirectory is a NOOP on all but linux. -func unmountKubeletDirectory(absoluteKubeletRunDirectory string) error { - klog.Warning("Cannot unmount filesystems on current OS, all mounted file systems will need to be manually unmounted") - return nil -} diff --git a/pkg/yurtadm/cmd/reset/phases/unmount_linux.go b/pkg/yurtadm/cmd/reset/phases/unmount_linux.go deleted file mode 100644 index d885fec6258..00000000000 --- a/pkg/yurtadm/cmd/reset/phases/unmount_linux.go +++ /dev/null @@ -1,47 +0,0 @@ -//go:build linux -// +build linux - -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package phases - -import ( - "os" - "strings" - "syscall" - - "k8s.io/klog/v2" -) - -// unmountKubeletDirectory unmounts all paths that contain KubeletRunDirectory -func unmountKubeletDirectory(absoluteKubeletRunDirectory string) error { - raw, err := os.ReadFile("/proc/mounts") - if err != nil { - return err - } - mounts := strings.Split(string(raw), "\n") - for _, mount := range mounts { - m := strings.Split(mount, " ") - if len(m) < 2 || !strings.HasPrefix(m[1], absoluteKubeletRunDirectory) { - continue - } - if err := syscall.Unmount(m[1], 0); err != nil { - klog.Warningf("[reset] Failed to unmount mounted directory in %s: %s", absoluteKubeletRunDirectory, m[1]) - } - } - return nil -} diff --git a/pkg/yurtadm/cmd/reset/reset.go b/pkg/yurtadm/cmd/reset/reset.go index 9df8a108d6c..d450738c12d 100644 --- a/pkg/yurtadm/cmd/reset/reset.go +++ b/pkg/yurtadm/cmd/reset/reset.go @@ -19,39 +19,17 @@ package reset import ( "io" - "strings" - "github.com/lithammer/dedent" "github.com/spf13/cobra" flag "github.com/spf13/pflag" - "k8s.io/apimachinery/pkg/util/sets" - "k8s.io/klog/v2" - "github.com/openyurtio/openyurt/pkg/util/kubernetes/kubeadm/app/cmd/options" - "github.com/openyurtio/openyurt/pkg/util/kubernetes/kubeadm/app/cmd/phases/workflow" - utilruntime "github.com/openyurtio/openyurt/pkg/util/kubernetes/kubeadm/app/util/runtime" yurtphases "github.com/openyurtio/openyurt/pkg/yurtadm/cmd/reset/phases" -) - -var ( - iptablesCleanupInstructions = dedent.Dedent(` - The reset process does not reset or clean up iptables rules or IPVS tables. - If you wish to reset iptables, you must do so manually by using the "iptables" command. - - If your cluster was setup to utilize IPVS, run ipvsadm --clear (or similar) - to reset your system's IPVS tables. - - The reset process does not clean your kubeconfig files and you must remove them manually. - Please, check the contents of the $HOME/.kube/config file. - `) - - cniCleanupInstructions = dedent.Dedent(` - The reset process does not clean CNI configuration. To do so, you must remove /etc/cni/net.d - `) + "github.com/openyurtio/openyurt/pkg/yurtadm/constants" ) // resetOptions defines all the options exposed via flags by kubeadm reset. type resetOptions struct { + certificatesDir string criSocketPath string forceReset bool ignorePreflightErrors []string @@ -60,127 +38,111 @@ type resetOptions struct { // resetData defines all the runtime information used when running the kubeadm reset workflow; // this data is shared across all the phases that are included in the workflow. type resetData struct { + certificatesDir string criSocketPath string forceReset bool - ignorePreflightErrors sets.String - inputReader io.Reader - outputWriter io.Writer - dirsToClean []string + ignorePreflightErrors []string } // newResetOptions returns a struct ready for being used for creating cmd join flags. func newResetOptions() *resetOptions { return &resetOptions{ - forceReset: false, + certificatesDir: constants.DefaultCertificatesDir, + forceReset: false, } } // newResetData returns a new resetData struct to be used for the execution of the kubeadm reset workflow. -func newResetData(cmd *cobra.Command, options *resetOptions, in io.Reader, out io.Writer) (*resetData, error) { - var criSocketPath string - var err error - if options.criSocketPath == "" { - criSocketPath, err = utilruntime.DetectCRISocket() - if err != nil { - return nil, err - } - klog.V(1).Infof("[reset] Detected and using CRI socket: %s", criSocketPath) - } else { - criSocketPath = options.criSocketPath - klog.V(1).Infof("[reset] Using specified CRI socket: %s", criSocketPath) - } - - var ignoreErrors sets.String - for _, item := range options.ignorePreflightErrors { - ignoreErrors.Insert(strings.ToLower(item)) - } - +func newResetData(options *resetOptions) (*resetData, error) { return &resetData{ - criSocketPath: criSocketPath, + certificatesDir: options.certificatesDir, + criSocketPath: options.criSocketPath, forceReset: options.forceReset, - ignorePreflightErrors: ignoreErrors, - inputReader: in, - outputWriter: out, + ignorePreflightErrors: options.ignorePreflightErrors, }, nil } // AddResetFlags adds reset flags func AddResetFlags(flagSet *flag.FlagSet, resetOptions *resetOptions) { + flagSet.StringVar( + &resetOptions.certificatesDir, constants.CertificatesDir, resetOptions.certificatesDir, + "The path to the directory where the certificates are stored. If specified, clean this directory. (default \"/etc/kubernetes/pki\")", + ) flagSet.BoolVarP( - &resetOptions.forceReset, options.ForceReset, "f", false, + &resetOptions.forceReset, constants.ForceReset, "f", false, "Reset the node without prompting for confirmation.", ) flagSet.StringVar( - &resetOptions.criSocketPath, options.NodeCRISocket, resetOptions.criSocketPath, + &resetOptions.criSocketPath, constants.NodeCRISocket, resetOptions.criSocketPath, "Path to the CRI socket to connect", ) flagSet.StringSliceVar( - &resetOptions.ignorePreflightErrors, options.IgnorePreflightErrors, resetOptions.ignorePreflightErrors, + &resetOptions.ignorePreflightErrors, constants.IgnorePreflightErrors, resetOptions.ignorePreflightErrors, "A list of checks whose errors will be shown as warnings. Example: 'IsPrivilegedUser,Swap'. Value 'all' ignores errors from all checks.", ) } -// NewCmdReset returns the "yurtadm reset" command -func NewCmdReset(in io.Reader, out io.Writer, resetOptions *resetOptions) *cobra.Command { - if resetOptions == nil { - resetOptions = newResetOptions() +type nodeReseter struct { + *resetData + inReader io.Reader + outWriter io.Writer + outErrWriter io.Writer +} + +func newReseterWithResetData(o *resetData, in io.Reader, out io.Writer, outErr io.Writer) *nodeReseter { + return &nodeReseter{ + o, + in, + out, + outErr, } - resetRunner := workflow.NewRunner() +} + +// Run use kubeadm to reset the node. +func (nodeReseter *nodeReseter) Run() error { + resetData := nodeReseter.resetData + + if err := yurtphases.RunResetNode(resetData, nodeReseter.inReader, nodeReseter.outWriter, nodeReseter.outErrWriter); err != nil { + return err + } + + if err := yurtphases.RunCleanYurtFile(); err != nil { + return err + } + + return nil +} + +// NewCmdReset returns the "yurtadm reset" command +func NewCmdReset(in io.Reader, out io.Writer, outErr io.Writer) *cobra.Command { + resetOptions := newResetOptions() cmd := &cobra.Command{ Use: "reset", Short: "Performs a best effort revert of changes made to this host by 'yurtadm join'", RunE: func(cmd *cobra.Command, args []string) error { - c, err := resetRunner.InitData(args) + o, err := newResetData(resetOptions) if err != nil { return err } - err = resetRunner.Run(args) - if err != nil { + reseter := newReseterWithResetData(o, in, out, outErr) + if err := reseter.Run(); err != nil { return err } - // Then clean contents from the stateful kubelet, etcd and cni directories - data := c.(*resetData) - cleanDirs(data) - - // output help text instructing user how to remove cni folders - klog.Info(cniCleanupInstructions) - // Output help text instructing user how to remove iptables rules - klog.Info(iptablesCleanupInstructions) return nil }, } AddResetFlags(cmd.Flags(), resetOptions) - // initialize the workflow runner with the list of phases - resetRunner.AppendPhase(yurtphases.NewPreflightPhase()) - resetRunner.AppendPhase(yurtphases.NewCleanupNodePhase()) - resetRunner.AppendPhase(yurtphases.NewCleanYurtFilePhase()) - - // sets the data builder function, that will be used by the runner - // both when running the entire workflow or single phases - resetRunner.SetDataInitializer(func(cmd *cobra.Command, args []string) (workflow.RunData, error) { - return newResetData(cmd, resetOptions, in, out) - }) - - // binds the Runner to kubeadm init command by altering - // command help, adding --skip-phases flag and by adding phases subcommands - resetRunner.BindToCommand(cmd) - return cmd } -func cleanDirs(data *resetData) { - klog.Infof("[reset] Deleting contents of stateful directories: %v\n", data.dirsToClean) - for _, dir := range data.dirsToClean { - klog.V(1).Infof("[reset] Deleting contents of %s", dir) - if err := yurtphases.CleanDir(dir); err != nil { - klog.Warningf("[reset] Failed to delete contents of %q directory: %v", dir, err) - } - } +// CertificatesDir returns the certificatesDir flag. +func (r *resetData) CertificatesDir() string { + return r.certificatesDir } // ForceReset returns the forceReset flag. @@ -188,21 +150,11 @@ func (r *resetData) ForceReset() bool { return r.forceReset } -// InputReader returns the io.reader used to read messages. -func (r *resetData) InputReader() io.Reader { - return r.inputReader -} - // IgnorePreflightErrors returns the list of preflight errors to ignore. -func (r *resetData) IgnorePreflightErrors() sets.String { +func (r *resetData) IgnorePreflightErrors() []string { return r.ignorePreflightErrors } -// AddDirsToClean add a list of dirs to the list of dirs that will be removed. -func (r *resetData) AddDirsToClean(dirs ...string) { - r.dirsToClean = append(r.dirsToClean, dirs...) -} - // CRISocketPath returns the criSocketPath. func (r *resetData) CRISocketPath() string { return r.criSocketPath diff --git a/pkg/util/kubernetes/kubelet/apis/config/v1beta1/zz_generated.deepcopy.go b/pkg/yurtadm/cmd/reset/resetdata/data.go similarity index 73% rename from pkg/util/kubernetes/kubelet/apis/config/v1beta1/zz_generated.deepcopy.go rename to pkg/yurtadm/cmd/reset/resetdata/data.go index a47bc95e8a7..dbeecd38809 100644 --- a/pkg/util/kubernetes/kubelet/apis/config/v1beta1/zz_generated.deepcopy.go +++ b/pkg/yurtadm/cmd/reset/resetdata/data.go @@ -1,8 +1,5 @@ -//go:build !ignore_autogenerated -// +build !ignore_autogenerated - /* -Copyright The Kubernetes Authors. +Copyright 2022 The OpenYurt Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -17,6 +14,11 @@ See the License for the specific language governing permissions and limitations under the License. */ -// Code generated by deepcopy-gen. DO NOT EDIT. +package resetdata -package v1beta1 +type YurtResetData interface { + CertificatesDir() string + ForceReset() bool + IgnorePreflightErrors() []string + CRISocketPath() string +} diff --git a/pkg/yurtadm/constants/constants.go b/pkg/yurtadm/constants/constants.go index 5ec8536ab78..e3b3c5b9cdc 100644 --- a/pkg/yurtadm/constants/constants.go +++ b/pkg/yurtadm/constants/constants.go @@ -41,6 +41,7 @@ const ( KubeletSvcPath = "/usr/lib/systemd/system/kubelet.service.d/10-kubeadm.conf" YurthubStaticPodFileName = "yurthub.yaml" PauseImagePath = "registry.cn-hangzhou.aliyuncs.com/google_containers/pause:3.2" + DefaultCertificatesDir = "/etc/kubernetes/pki" DefaultDockerCRISocket = "/var/run/dockershim.sock" YurthubYamlName = "yurt-hub.yaml" // ManifestsSubDirName defines directory name to store manifests @@ -74,6 +75,41 @@ const ( EdgeNode = "edge" CloudNode = "cloud" + // AdminKubeConfigPath + AdminKubeConfigPath = "/etc/kubernetes/admin.conf" + // CertificatesDir + CertificatesDir = "cert-dir" + // ForceReset + ForceReset = "force" + // IgnorePreflightErrors sets the path a list of checks whose errors will be shown as warnings. Example: 'IsPrivilegedUser,Swap'. Value 'all' ignores errors from all checks. + IgnorePreflightErrors = "ignore-preflight-errors" + // kubeconfigPath + KubeconfigPath = "kubeconfig" + // KubernetesResourceServer flag sets the address for download k8s node resources. + KubernetesResourceServer = "kubernetes-resource-server" + // NodeCRISocket flag sets the CRI socket to connect to. + NodeCRISocket = "cri-socket" + // NodeLabels flag sets the labels for worker node. + NodeLabels = "node-labels" + // NodeName flag sets the node name. + NodeName = "node-name" + // NodeType flag sets the type of worker node to edge or cloud. + NodeType = "node-type" + // Organizations flag sets the extra organizations of hub agent client certificate. + Organizations = "organizations" + // PauseImage flag sets the pause image for worker node. + PauseImage = "pause-image" + // TokenStr flags sets both the discovery-token and the tls-bootstrap-token when those values are not provided + TokenStr = "token" + // TokenDiscoveryCAHash flag instruct kubeadm to validate that the root CA public key matches this hash (for token-based discovery) + TokenDiscoveryCAHash = "discovery-token-ca-cert-hash" + // TokenDiscoverySkipCAHash flag instruct kubeadm to skip CA hash verification (for token-based discovery) + TokenDiscoverySkipCAHash = "discovery-token-unsafe-skip-ca-verification" + // YurtHubImage flag sets the yurthub image for worker node. + YurtHubImage = "yurthub-image" + // YurtHubServerAddr flag set the address of yurthub server (not proxy server!) + YurtHubServerAddr = "yurthub-server-addr" + ServerHealthzServer = "127.0.0.1:10267" ServerHealthzURLPath = "/v1/healthz" DefaultOpenYurtImageRegistry = "registry.cn-hangzhou.aliyuncs.com/openyurt" diff --git a/pkg/yurtadm/constants/join_options.go b/pkg/yurtadm/constants/join_options.go deleted file mode 100644 index 7781723df51..00000000000 --- a/pkg/yurtadm/constants/join_options.go +++ /dev/null @@ -1,41 +0,0 @@ -/* -Copyright 2022 The OpenYurt Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package constants - -const ( - - // NodeType flag sets the type of worker node to edge or cloud. - NodeType = "node-type" - - // Organizations flag sets the extra organizations of hub agent client certificate. - Organizations = "organizations" - - // NodeLabels flag sets the labels for worker node. - NodeLabels = "node-labels" - - // PauseImage flag sets the pause image for worker node. - PauseImage = "pause-image" - - // YurtHubImage flag sets the yurthub image for worker node. - YurtHubImage = "yurthub-image" - - // KubernetesResourceServer flag sets the address for download k8s node resources. - KubernetesResourceServer = "kubernetes-resource-server" - - // YurtHubServerAddr flag set the address of yurthub server (not proxy server!) - YurtHubServerAddr = "yurthub-server-addr" -) diff --git a/pkg/util/kubernetes/kubelet/apis/config/v1beta1/defaults_linux.go b/pkg/yurtadm/util/initsystem/initsystem.go similarity index 60% rename from pkg/util/kubernetes/kubelet/apis/config/v1beta1/defaults_linux.go rename to pkg/yurtadm/util/initsystem/initsystem.go index 6d3ec77dc7c..b59c399418c 100644 --- a/pkg/util/kubernetes/kubelet/apis/config/v1beta1/defaults_linux.go +++ b/pkg/yurtadm/util/initsystem/initsystem.go @@ -1,8 +1,6 @@ -//go:build linux -// +build linux - /* -Copyright 2018 The Kubernetes Authors. +Copyright 2022 The OpenYurt Authors. +Copyright 2016 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -17,12 +15,10 @@ See the License for the specific language governing permissions and limitations under the License. */ -package v1beta1 +package initsystem -// DefaultEvictionHard includes default options for hard eviction. -var DefaultEvictionHard = map[string]string{ - "memory.available": "100Mi", - "nodefs.available": "10%", - "nodefs.inodesFree": "5%", - "imagefs.available": "15%", +// InitSystem is the interface that describe behaviors of an init system +type InitSystem interface { + // ServiceIsActive ensures the service is running, or attempting to run. (crash looping in the case of kubelet) + ServiceIsActive(service string) bool } diff --git a/pkg/yurtadm/util/initsystem/initsystem_unix.go b/pkg/yurtadm/util/initsystem/initsystem_unix.go new file mode 100644 index 00000000000..40f06510be4 --- /dev/null +++ b/pkg/yurtadm/util/initsystem/initsystem_unix.go @@ -0,0 +1,72 @@ +//go:build !windows +// +build !windows + +/* +Copyright 2022 The OpenYurt Authors. +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package initsystem + +import ( + "fmt" + "os/exec" + "strings" +) + +// OpenRCInitSystem defines openrc +type OpenRCInitSystem struct{} + +// ServiceIsActive ensures the service is running, or attempting to run. (crash looping in the case of kubelet) +func (openrc OpenRCInitSystem) ServiceIsActive(service string) bool { + args := []string{service, "status"} + outBytes, _ := exec.Command("rc-service", args...).CombinedOutput() + outStr := string(outBytes) + return !strings.Contains(outStr, "stopped") && !strings.Contains(outStr, "does not exist") +} + +// SystemdInitSystem defines systemd +type SystemdInitSystem struct{} + +// ServiceIsActive will check is the service is "active". In the case of +// crash looping services (kubelet in our case) status will return as +// "activating", so we will consider this active as well. +func (sysd SystemdInitSystem) ServiceIsActive(service string) bool { + args := []string{"is-active", service} + // Ignoring error here, command returns non-0 if in "activating" status: + outBytes, _ := exec.Command("systemctl", args...).Output() + output := strings.TrimSpace(string(outBytes)) + if output == "active" || output == "activating" { + return true + } + return false +} + +// GetInitSystem returns an InitSystem for the current system, or nil +// if we cannot detect a supported init system. +// This indicates we will skip init system checks, not an error. +func GetInitSystem() (InitSystem, error) { + // Assume existence of systemctl in path implies this is a systemd system: + _, err := exec.LookPath("systemctl") + if err == nil { + return &SystemdInitSystem{}, nil + } + _, err = exec.LookPath("openrc") + if err == nil { + return &OpenRCInitSystem{}, nil + } + + return nil, fmt.Errorf("no supported init system detected, skipping checking for services") +} diff --git a/pkg/yurtadm/util/initsystem/initsystem_windows.go b/pkg/yurtadm/util/initsystem/initsystem_windows.go new file mode 100644 index 00000000000..bbc5aab965b --- /dev/null +++ b/pkg/yurtadm/util/initsystem/initsystem_windows.go @@ -0,0 +1,62 @@ +//go:build windows +// +build windows + +/* +Copyright 2022 The OpenYurt Authors. +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package initsystem + +import ( + "fmt" + "golang.org/x/sys/windows/svc" + "golang.org/x/sys/windows/svc/mgr" +) + +// WindowsInitSystem is the windows implementation of InitSystem +type WindowsInitSystem struct{} + +// ServiceIsActive ensures the service is running, or attempting to run. (crash looping in the case of kubelet) +func (sysd WindowsInitSystem) ServiceIsActive(service string) bool { + m, err := mgr.Connect() + if err != nil { + return false + } + defer m.Disconnect() + s, err := m.OpenService(service) + if err != nil { + return false + } + defer s.Close() + + status, err := s.Query() + if err != nil { + return false + } + return status.State == svc.Running +} + +// GetInitSystem returns an InitSystem for the current system, or nil +// if we cannot detect a supported init system. +// This indicates we will skip init system checks, not an error. +func GetInitSystem() (InitSystem, error) { + m, err := mgr.Connect() + if err != nil { + return nil, fmt.Errorf("no supported init system detected: %v", err) + } + defer m.Disconnect() + return &WindowsInitSystem{}, nil +}