diff --git a/charts/openyurt/templates/yurt-manager-auto-generated.yaml b/charts/openyurt/templates/yurt-manager-auto-generated.yaml index 909cd399e64..a8de5c4eec0 100644 --- a/charts/openyurt/templates/yurt-manager-auto-generated.yaml +++ b/charts/openyurt/templates/yurt-manager-auto-generated.yaml @@ -497,6 +497,130 @@ status: conditions: [] storedVersions: [] --- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.7.0 + creationTimestamp: null + name: staticpods.apps.openyurt.io +spec: + group: apps.openyurt.io + names: + kind: StaticPod + listKind: StaticPodList + plural: staticpods + shortNames: + - sp + singular: staticpod + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: CreationTimestamp is a timestamp representing the server time when + this object was created. It is not guaranteed to be set in happens-before + order across separate operations. Clients may not set this value. It is represented + in RFC3339 form and is in UTC. + jsonPath: .metadata.creationTimestamp + name: AGE + type: date + - description: The total number of static pods + jsonPath: .status.totalNumber + name: TotalNumber + type: integer + - description: The number of ready static pods + jsonPath: .status.readyNumber + name: ReadyNumber + type: integer + - description: The number of static pods that have been upgraded + jsonPath: .status.upgradedNumber + name: UpgradedNumber + type: integer + name: v1alpha1 + schema: + openAPIV3Schema: + description: StaticPod is the Schema for the staticpods API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: StaticPodSpec defines the desired state of StaticPod + properties: + revisionHistoryLimit: + description: The number of old history to retain to allow rollback. + Defaults to 10. + format: int32 + type: integer + staticPodManifest: + description: StaticPodManifest indicates the Static Pod desired to + be upgraded. The corresponding manifest file name is `StaticPodManifest.yaml`. + type: string + template: + description: An object that describes the desired upgrade static pod. + x-kubernetes-preserve-unknown-fields: true + upgradeStrategy: + description: An upgrade strategy to replace existing static pods with + new ones. + properties: + maxUnavailable: + anyOf: + - type: integer + - type: string + description: Auto upgrade config params. Present only if type + = "auto". + x-kubernetes-int-or-string: true + type: + description: Type of Static Pod upgrade. Can be "auto" or "ota". + type: string + type: object + type: object + status: + description: StaticPodStatus defines the observed state of StaticPod + properties: + observedGeneration: + description: The most recent generation observed by the static pod + controller. + format: int64 + type: integer + readyNumber: + description: The number of ready static pods. + format: int32 + type: integer + totalNumber: + description: The total number of nodes that are running the static + pod. + format: int32 + type: integer + upgradedNumber: + description: The number of nodes that are running updated static pod. + format: int32 + type: integer + required: + - readyNumber + - totalNumber + - upgradedNumber + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: @@ -580,6 +704,32 @@ rules: - get - patch - update +- apiGroups: + - apps.openyurt.io + resources: + - staticpods + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - apps.openyurt.io + resources: + - staticpods/finalizers + verbs: + - update +- apiGroups: + - apps.openyurt.io + resources: + - staticpods/status + verbs: + - get + - patch + - update - apiGroups: - certificates.k8s.io resources: @@ -616,6 +766,18 @@ rules: - patch - update - watch +- apiGroups: + - "" + resources: + - configmaps + verbs: + - create + - delete + - get + - list + - patch + - update + - watch - apiGroups: - "" resources: @@ -764,6 +926,27 @@ webhooks: resources: - nodepools sideEffects: None +- admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + service: + name: webhook-service + namespace: kube-system + path: /mutate-apps-openyurt-io-v1alpha1-staticpod + failurePolicy: Fail + name: mutate.apps.v1alpha1.staticpod.openyurt.io + rules: + - apiGroups: + - apps.openyurt.io + apiVersions: + - v1alpha1 + operations: + - CREATE + - UPDATE + resources: + - staticpods + sideEffects: None --- apiVersion: admissionregistration.k8s.io/v1 kind: ValidatingWebhookConfiguration @@ -815,6 +998,28 @@ webhooks: sideEffects: None - admissionReviewVersions: - v1 + - v1beta1 + clientConfig: + service: + name: webhook-service + namespace: kube-system + path: /validate-apps-openyurt-io-v1alpha1-staticpod + failurePolicy: Fail + name: validate.apps.v1alpha1.staticpod.openyurt.io + rules: + - apiGroups: + - apps.openyurt.io + apiVersions: + - v1alpha1 + operations: + - CREATE + - UPDATE + resources: + - staticpods + sideEffects: None +- admissionReviewVersions: + - v1 + - v1beta1 clientConfig: service: name: webhook-service diff --git a/cmd/yurt-manager/app/options/options.go b/cmd/yurt-manager/app/options/options.go index 3c343ed6645..58acbba2b03 100644 --- a/cmd/yurt-manager/app/options/options.go +++ b/cmd/yurt-manager/app/options/options.go @@ -24,18 +24,20 @@ import ( // YurtManagerOptions is the main context object for the yurt-manager. type YurtManagerOptions struct { - Generic *GenericOptions - NodePoolController *NodePoolControllerOptions - GatewayController *GatewayControllerOptions + Generic *GenericOptions + NodePoolController *NodePoolControllerOptions + GatewayController *GatewayControllerOptions + StaticPodController *StaticPodControllerOptions } // NewYurtManagerOptions creates a new YurtManagerOptions with a default config. func NewYurtManagerOptions() (*YurtManagerOptions, error) { s := YurtManagerOptions{ - Generic: NewGenericOptions(), - NodePoolController: NewNodePoolControllerOptions(), - GatewayController: NewGatewayControllerOptions(), + Generic: NewGenericOptions(), + NodePoolController: NewNodePoolControllerOptions(), + GatewayController: NewGatewayControllerOptions(), + StaticPodController: NewStaticPodControllerOptions(), } return &s, nil @@ -46,7 +48,7 @@ func (y *YurtManagerOptions) Flags() cliflag.NamedFlagSets { y.Generic.AddFlags(fss.FlagSet("generic")) y.NodePoolController.AddFlags(fss.FlagSet("nodepool controller")) y.GatewayController.AddFlags(fss.FlagSet("gateway controller")) - + y.StaticPodController.AddFlags(fss.FlagSet("staticpod controller")) // Please Add Other controller flags @kadisi return fss @@ -58,6 +60,7 @@ func (y *YurtManagerOptions) Validate() error { errs = append(errs, y.Generic.Validate()...) errs = append(errs, y.NodePoolController.Validate()...) errs = append(errs, y.GatewayController.Validate()...) + errs = append(errs, y.StaticPodController.Validate()...) return utilerrors.NewAggregate(errs) } @@ -69,6 +72,9 @@ func (y *YurtManagerOptions) ApplyTo(c *config.Config) error { if err := y.NodePoolController.ApplyTo(&c.ComponentConfig.NodePoolController); err != nil { return err } + if err := y.StaticPodController.ApplyTo(&c.ComponentConfig.StaticPodController); err != nil { + return err + } return nil } diff --git a/cmd/yurt-manager/app/options/staticpodcontroller.go b/cmd/yurt-manager/app/options/staticpodcontroller.go new file mode 100644 index 00000000000..cebf176b4c9 --- /dev/null +++ b/cmd/yurt-manager/app/options/staticpodcontroller.go @@ -0,0 +1,64 @@ +/* +Copyright 2023 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the License); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an AS IS BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package options + +import ( + "github.com/spf13/pflag" + + "github.com/openyurtio/openyurt/pkg/controller/staticpod/config" +) + +const DefaultUpgradeWorkerImage = "openyurt/node-servant:latest" + +type StaticPodControllerOptions struct { + *config.StaticPodControllerConfiguration +} + +func NewStaticPodControllerOptions() *StaticPodControllerOptions { + return &StaticPodControllerOptions{ + &config.StaticPodControllerConfiguration{ + UpgradeWorkerImage: DefaultUpgradeWorkerImage, + }, + } +} + +// AddFlags adds flags related to staticpod for yurt-manager to the specified FlagSet. +func (o *StaticPodControllerOptions) AddFlags(fs *pflag.FlagSet) { + if o == nil { + return + } + + fs.StringVar(&o.UpgradeWorkerImage, "upgrade-worker-image", o.UpgradeWorkerImage, "Specify the worker pod image used for static pod upgrade.") +} + +// ApplyTo fills up staticpod config with options. +func (o *StaticPodControllerOptions) ApplyTo(cfg *config.StaticPodControllerConfiguration) error { + if o == nil { + return nil + } + cfg.UpgradeWorkerImage = o.UpgradeWorkerImage + return nil +} + +// Validate checks validation of StaticPodControllerOptions. +func (o *StaticPodControllerOptions) Validate() []error { + if o == nil { + return nil + } + errs := []error{} + return errs +} diff --git a/go.mod b/go.mod index 296b5a0000c..729e3211190 100644 --- a/go.mod +++ b/go.mod @@ -32,6 +32,7 @@ require ( k8s.io/api v0.22.3 k8s.io/apimachinery v0.22.3 k8s.io/apiserver v0.22.3 + k8s.io/cli-runtime v0.22.3 k8s.io/client-go v0.22.3 k8s.io/cluster-bootstrap v0.22.3 k8s.io/component-base v0.22.3 @@ -58,7 +59,6 @@ require ( github.com/russross/blackfriday v1.5.2 // indirect github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca // indirect go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5 // indirect - k8s.io/cli-runtime v0.22.3 sigs.k8s.io/kustomize/api v0.8.11 // indirect sigs.k8s.io/kustomize/kyaml v0.11.0 // indirect ) diff --git a/pkg/apis/apps/v1alpha1/default.go b/pkg/apis/apps/v1alpha1/default.go index be1a486b028..0fdbbe67fc9 100644 --- a/pkg/apis/apps/v1alpha1/default.go +++ b/pkg/apis/apps/v1alpha1/default.go @@ -16,6 +16,8 @@ limitations under the License. package v1alpha1 +import "k8s.io/apimachinery/pkg/util/intstr" + // SetDefaultsNodePool set default values for NodePool. func SetDefaultsNodePool(obj *NodePool) { // example for set default value for NodePool @@ -24,3 +26,22 @@ func SetDefaultsNodePool(obj *NodePool) { } } + +// SetDefaultsStaticPod set default values for StaticPod. +func SetDefaultsStaticPod(obj *StaticPod) { + // Set default upgrade strategy to "auto" with max-unavailable to "10%" + strategy := &obj.Spec.UpgradeStrategy + if strategy.Type == "" { + strategy.Type = AutoStaticPodUpgradeStrategyType + } + if strategy.Type == AutoStaticPodUpgradeStrategyType && strategy.MaxUnavailable == nil { + v := intstr.FromString("10%") + strategy.MaxUnavailable = &v + } + + // Set default RevisionHistoryLimit to 10 + if obj.Spec.RevisionHistoryLimit == nil { + obj.Spec.RevisionHistoryLimit = new(int32) + *obj.Spec.RevisionHistoryLimit = 10 + } +} diff --git a/pkg/apis/apps/v1alpha1/staticpod_types.go b/pkg/apis/apps/v1alpha1/staticpod_types.go new file mode 100644 index 00000000000..425df7b852b --- /dev/null +++ b/pkg/apis/apps/v1alpha1/staticpod_types.go @@ -0,0 +1,110 @@ +/* +Copyright 2023 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the License); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an AS IS BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" +) + +// StaticPodUpgradeStrategy defines a strategy to upgrade a static pod. +type StaticPodUpgradeStrategy struct { + // Type of Static Pod upgrade. Can be "auto" or "ota". + Type StaticPodUpgradeStrategyType `json:"type,omitempty"` + + // Auto upgrade config params. Present only if type = "auto". + //+optional + MaxUnavailable *intstr.IntOrString `json:"maxUnavailable,omitempty"` +} + +// StaticPodUpgradeStrategyType is a strategy according to which a static pod gets upgraded. +type StaticPodUpgradeStrategyType string + +const ( + AutoStaticPodUpgradeStrategyType StaticPodUpgradeStrategyType = "auto" + OTAStaticPodUpgradeStrategyType StaticPodUpgradeStrategyType = "ota" +) + +// StaticPodSpec defines the desired state of StaticPod +type StaticPodSpec struct { + // StaticPodManifest indicates the Static Pod desired to be upgraded. The corresponding + // manifest file name is `StaticPodManifest.yaml`. + StaticPodManifest string `json:"staticPodManifest,omitempty"` + + // An upgrade strategy to replace existing static pods with new ones. + UpgradeStrategy StaticPodUpgradeStrategy `json:"upgradeStrategy,omitempty"` + + // The number of old history to retain to allow rollback. + // Defaults to 10. + // +optional + RevisionHistoryLimit *int32 `json:"revisionHistoryLimit,omitempty"` + + // An object that describes the desired upgrade static pod. + // +optional + // +kubebuilder:pruning:PreserveUnknownFields + // +kubebuilder:validation:Schemaless + Template corev1.PodTemplateSpec `json:"template,omitempty"` +} + +// StaticPodStatus defines the observed state of StaticPod +type StaticPodStatus struct { + // The total number of nodes that are running the static pod. + TotalNumber int32 `json:"totalNumber"` + + // The number of ready static pods. + ReadyNumber int32 `json:"readyNumber"` + + // The number of nodes that are running updated static pod. + UpgradedNumber int32 `json:"upgradedNumber"` + + // The most recent generation observed by the static pod controller. + // +optional + ObservedGeneration int64 `json:"observedGeneration"` +} + +// +genclient +// +k8s:openapi-gen=true +// +kubebuilder:object:root=true +// +kubebuilder:subresource:status +// +kubebuilder:resource:shortName=sp +// +kubebuilder:printcolumn:name="AGE",type="date",JSONPath=".metadata.creationTimestamp",description="CreationTimestamp is a timestamp representing the server time when this object was created. It is not guaranteed to be set in happens-before order across separate operations. Clients may not set this value. It is represented in RFC3339 form and is in UTC." +//+kubebuilder:printcolumn:name="TotalNumber",type="integer",JSONPath=".status.totalNumber",description="The total number of static pods" +//+kubebuilder:printcolumn:name="ReadyNumber",type="integer",JSONPath=".status.readyNumber",description="The number of ready static pods" +//+kubebuilder:printcolumn:name="UpgradedNumber",type="integer",JSONPath=".status.upgradedNumber",description="The number of static pods that have been upgraded" + +// StaticPod is the Schema for the staticpods API +type StaticPod struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec StaticPodSpec `json:"spec,omitempty"` + Status StaticPodStatus `json:"status,omitempty"` +} + +//+kubebuilder:object:root=true + +// StaticPodList contains a list of StaticPod +type StaticPodList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []StaticPod `json:"items"` +} + +func init() { + SchemeBuilder.Register(&StaticPod{}, &StaticPodList{}) +} diff --git a/pkg/apis/apps/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/apps/v1alpha1/zz_generated.deepcopy.go index 00b18209ced..c574b252792 100644 --- a/pkg/apis/apps/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/apps/v1alpha1/zz_generated.deepcopy.go @@ -25,6 +25,7 @@ import ( corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/intstr" ) // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. @@ -146,3 +147,119 @@ func (in *NodePoolStatus) DeepCopy() *NodePoolStatus { in.DeepCopyInto(out) return out } + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *StaticPod) DeepCopyInto(out *StaticPod) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + out.Status = in.Status +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new StaticPod. +func (in *StaticPod) DeepCopy() *StaticPod { + if in == nil { + return nil + } + out := new(StaticPod) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *StaticPod) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *StaticPodList) DeepCopyInto(out *StaticPodList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]StaticPod, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new StaticPodList. +func (in *StaticPodList) DeepCopy() *StaticPodList { + if in == nil { + return nil + } + out := new(StaticPodList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *StaticPodList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *StaticPodSpec) DeepCopyInto(out *StaticPodSpec) { + *out = *in + in.UpgradeStrategy.DeepCopyInto(&out.UpgradeStrategy) + if in.RevisionHistoryLimit != nil { + in, out := &in.RevisionHistoryLimit, &out.RevisionHistoryLimit + *out = new(int32) + **out = **in + } + in.Template.DeepCopyInto(&out.Template) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new StaticPodSpec. +func (in *StaticPodSpec) DeepCopy() *StaticPodSpec { + if in == nil { + return nil + } + out := new(StaticPodSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *StaticPodStatus) DeepCopyInto(out *StaticPodStatus) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new StaticPodStatus. +func (in *StaticPodStatus) DeepCopy() *StaticPodStatus { + if in == nil { + return nil + } + out := new(StaticPodStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *StaticPodUpgradeStrategy) DeepCopyInto(out *StaticPodUpgradeStrategy) { + *out = *in + if in.MaxUnavailable != nil { + in, out := &in.MaxUnavailable, &out.MaxUnavailable + *out = new(intstr.IntOrString) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new StaticPodUpgradeStrategy. +func (in *StaticPodUpgradeStrategy) DeepCopy() *StaticPodUpgradeStrategy { + if in == nil { + return nil + } + out := new(StaticPodUpgradeStrategy) + in.DeepCopyInto(out) + return out +} diff --git a/pkg/controller/add_staticpod.go b/pkg/controller/add_staticpod.go new file mode 100644 index 00000000000..eae1f74183e --- /dev/null +++ b/pkg/controller/add_staticpod.go @@ -0,0 +1,30 @@ +/* +Copyright 2023 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the License); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an AS IS BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "github.com/openyurtio/openyurt/pkg/controller/staticpod" +) + +// Note !!! @kadisi +// Do not change the name of the file @kadisi +// Auto generate by make addcontroller command !!! +// Note !!! + +func init() { + controllerAddFuncs = append(controllerAddFuncs, staticpod.Add) +} diff --git a/pkg/controller/apis/config/types.go b/pkg/controller/apis/config/types.go index 3d226ffeead..0ecfbd9ff71 100644 --- a/pkg/controller/apis/config/types.go +++ b/pkg/controller/apis/config/types.go @@ -23,6 +23,7 @@ import ( gatewayconfig "github.com/openyurtio/openyurt/pkg/controller/gateway/config" nodepoolconfig "github.com/openyurtio/openyurt/pkg/controller/nodepool/config" + staticpodconfig "github.com/openyurtio/openyurt/pkg/controller/staticpod/config" ) // YurtControllerManagerConfiguration contains elements describing yurt-controller manager. @@ -46,6 +47,9 @@ type YurtManagerConfiguration struct { // GatewayControllerConfiguration holds configuration for GatewayController related features. GatewayController gatewayconfig.GatewayControllerConfiguration + + // StaticPodControllerConfiguration holds configuration for StaticPodController related features. + StaticPodController staticpodconfig.StaticPodControllerConfiguration } type GenericConfiguration struct { diff --git a/pkg/controller/staticpod/config/types.go b/pkg/controller/staticpod/config/types.go new file mode 100644 index 00000000000..2a055143da5 --- /dev/null +++ b/pkg/controller/staticpod/config/types.go @@ -0,0 +1,23 @@ +/* +Copyright 2023 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the License); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an AS IS BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package config + +// StaticPodControllerConfiguration contains elements describing GatewayController. +type StaticPodControllerConfiguration struct { + // UpgradeWorkerImage specify the image used to execute the upgrade task + UpgradeWorkerImage string +} diff --git a/pkg/controller/staticpod/staticpod_controller.go b/pkg/controller/staticpod/staticpod_controller.go new file mode 100644 index 00000000000..66b3d7d0e74 --- /dev/null +++ b/pkg/controller/staticpod/staticpod_controller.go @@ -0,0 +1,561 @@ +/* +Copyright 2023 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the License); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an AS IS BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package staticpod + +import ( + "context" + "flag" + "fmt" + + corev1 "k8s.io/api/core/v1" + kerr "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/tools/record" + "k8s.io/klog/v2" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/predicate" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + "sigs.k8s.io/controller-runtime/pkg/source" + + appconfig "github.com/openyurtio/openyurt/cmd/yurt-manager/app/config" + appsv1alpha1 "github.com/openyurtio/openyurt/pkg/apis/apps/v1alpha1" + "github.com/openyurtio/openyurt/pkg/controller/staticpod/config" + "github.com/openyurtio/openyurt/pkg/controller/staticpod/upgradeinfo" + "github.com/openyurtio/openyurt/pkg/controller/staticpod/util" + utilclient "github.com/openyurtio/openyurt/pkg/util/client" + utildiscovery "github.com/openyurtio/openyurt/pkg/util/discovery" +) + +func init() { + flag.IntVar(&concurrentReconciles, "staticpod-workers", concurrentReconciles, "Max concurrent workers for StaticPod controller.") +} + +var ( + concurrentReconciles = 3 + controllerKind = appsv1alpha1.SchemeGroupVersion.WithKind("StaticPod") + True = true +) + +const ( + controllerName = "StaticPod-controller" + + StaticPodHashAnnotation = "openyurt.io/static-pod-hash" + OTALatestManifestAnnotation = "openyurt.io/ota-latest-version" + + hostPathVolumeName = "hostpath" + hostPathVolumeMountPath = "/etc/kubernetes/manifests/" + configMapVolumeName = "configmap" + configMapVolumeMountPath = "/data" + hostPathVolumeSourcePath = hostPathVolumeMountPath + + // UpgradeWorkerPodPrefix is the name prefix of worker pod which used for static pod upgrade + UpgradeWorkerPodPrefix = "yurt-static-pod-upgrade-worker-" + UpgradeWorkerContainerName = "upgrade-worker" + + ArgTmpl = "/usr/local/bin/node-servant static-pod-upgrade --name=%s --namespace=%s --manifest=%s --hash=%s --mode=%s" +) + +// upgradeWorker is the pod template used for static pod upgrade +// Fields need be set +// 1. name of worker pod: `yurt-static-pod-upgrade-worker-node-hash` +// 2. node of worker pod +// 3. image of worker pod, default is "openyurt/node-servant:latest" +// 4. annotation `openyurt.io/static-pod-hash` +// 5. the corresponding configmap +var ( + upgradeWorker = &corev1.Pod{ + Spec: corev1.PodSpec{ + HostPID: true, + HostNetwork: true, + RestartPolicy: corev1.RestartPolicyNever, + Containers: []corev1.Container{{ + Name: UpgradeWorkerContainerName, + Command: []string{"/bin/sh", "-c"}, + VolumeMounts: []corev1.VolumeMount{ + { + Name: hostPathVolumeName, + MountPath: hostPathVolumeMountPath, + }, + { + Name: configMapVolumeName, + MountPath: configMapVolumeMountPath, + }, + }, + ImagePullPolicy: corev1.PullIfNotPresent, + SecurityContext: &corev1.SecurityContext{ + Privileged: &True, + }, + }}, + Volumes: []corev1.Volume{{ + Name: hostPathVolumeName, + VolumeSource: corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: hostPathVolumeSourcePath, + }, + }}, + }, + }, + } +) + +func Format(format string, args ...interface{}) string { + s := fmt.Sprintf(format, args...) + return fmt.Sprintf("%s: %s", controllerName, s) +} + +// Add creates a new StaticPod Controller and adds it to the Manager with default RBAC. The Manager will set fields on the Controller +// and Start it when the Manager is Started. +func Add(c *appconfig.CompletedConfig, mgr manager.Manager) error { + if !utildiscovery.DiscoverGVK(controllerKind) { + return nil + } + return add(mgr, newReconciler(c, mgr)) +} + +var _ reconcile.Reconciler = &ReconcileStaticPod{} + +// ReconcileStaticPod reconciles a StaticPod object +type ReconcileStaticPod struct { + client.Client + scheme *runtime.Scheme + recorder record.EventRecorder + Configuration config.StaticPodControllerConfiguration +} + +// newReconciler returns a new reconcile.Reconciler +func newReconciler(c *appconfig.CompletedConfig, mgr manager.Manager) reconcile.Reconciler { + return &ReconcileStaticPod{ + Client: utilclient.NewClientFromManager(mgr, controllerName), + scheme: mgr.GetScheme(), + recorder: mgr.GetEventRecorderFor(controllerName), + Configuration: c.ComponentConfig.StaticPodController, + } +} + +// add adds a new Controller to mgr with r as the reconcile.Reconciler +func add(mgr manager.Manager, r reconcile.Reconciler) error { + // Create a new controller + c, err := controller.New(controllerName, mgr, controller.Options{Reconciler: r, MaxConcurrentReconciles: concurrentReconciles}) + if err != nil { + return err + } + + // 1. Watch for changes to StaticPod + if err := c.Watch(&source.Kind{Type: &appsv1alpha1.StaticPod{}}, &handler.EnqueueRequestForObject{}); err != nil { + return err + } + + // 2. Watch for changes to node + // When node turn ready, reconcile all StaticPod instances + // nodeReadyPredicate filter events which are node turn ready + nodeReadyPredicate := predicate.Funcs{ + CreateFunc: func(evt event.CreateEvent) bool { + return false + }, + DeleteFunc: func(evt event.DeleteEvent) bool { + return false + }, + UpdateFunc: func(evt event.UpdateEvent) bool { + return nodeTurnReady(evt) + }, + GenericFunc: func(evt event.GenericEvent) bool { + return false + }, + } + + reconcileAllStaticPods := func(c client.Client) []reconcile.Request { + staticPodList := &appsv1alpha1.StaticPodList{} + if err := c.List(context.TODO(), staticPodList); err != nil { + return nil + } + var requests []reconcile.Request + for _, staticPod := range staticPodList.Items { + requests = append(requests, reconcile.Request{NamespacedName: types.NamespacedName{ + Namespace: staticPod.Namespace, + Name: staticPod.Name, + }}) + } + return requests + } + + if err := c.Watch(&source.Kind{Type: &corev1.Node{}}, + handler.EnqueueRequestsFromMapFunc( + func(client.Object) []reconcile.Request { + return reconcileAllStaticPods(mgr.GetClient()) + }), nodeReadyPredicate); err != nil { + return err + } + + // 3. Watch for changes to upgrade worker pods which are created by static-pod-controller + if err := c.Watch(&source.Kind{Type: &corev1.Pod{}}, &handler.EnqueueRequestForOwner{IsController: true, OwnerType: &appsv1alpha1.StaticPod{}}); err != nil { + return err + } + + return nil +} + +// nodeTurnReady filter events: old node is not-ready or unknown, new node is ready +func nodeTurnReady(evt event.UpdateEvent) bool { + if _, ok := evt.ObjectOld.(*corev1.Node); !ok { + return false + } + + oldNode := evt.ObjectOld.(*corev1.Node) + newNode := evt.ObjectNew.(*corev1.Node) + + _, onc := util.GetNodeCondition(&oldNode.Status, corev1.NodeReady) + _, nnc := util.GetNodeCondition(&newNode.Status, corev1.NodeReady) + + oldReady := (onc != nil) && ((onc.Status == corev1.ConditionFalse) || (onc.Status == corev1.ConditionUnknown)) + newReady := (nnc != nil) && (nnc.Status == corev1.ConditionTrue) + + return oldReady && newReady +} + +//+kubebuilder:rbac:groups=apps.openyurt.io,resources=staticpods,verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups=apps.openyurt.io,resources=staticpods/status,verbs=get;update;patch +//+kubebuilder:rbac:groups=apps.openyurt.io,resources=staticpods/finalizers,verbs=update +//+kubebuilder:rbac:groups=core,resources=pods,verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups=core,resources=nodes,verbs=get;list;watch;update;patch +//+kubebuilder:rbac:groups=core,resources=configmaps,verbs=get;list;watch;create;update;patch;delete + +// Reconcile reads that state of the cluster for a StaticPod object and makes changes based on the state read +// and what is in the StaticPod.Spec +func (r *ReconcileStaticPod) Reconcile(_ context.Context, request reconcile.Request) (reconcile.Result, error) { + + // Note !!!!!!!!!! + // We strongly recommend use Format() to encapsulation because Format() can print logs by module + // @kadisi + klog.V(4).Infof(Format("Reconcile StaticPod %s", request.Name)) + + // Fetch the StaticPod instance + instance := &appsv1alpha1.StaticPod{} + if err := r.Get(context.TODO(), request.NamespacedName, instance); err != nil { + klog.Errorf("Fail to get StaticPod %v, %v", request.NamespacedName, err) + return ctrl.Result{}, client.IgnoreNotFound(err) + } + + if instance.DeletionTimestamp != nil { + return reconcile.Result{}, nil + } + + var ( + // totalNumber represents the total number of nodes running the target static pod + totalNumber int32 + + // readyNumber represents the number of ready static pods + readyNumber int32 + + // upgradedNumber represents the number of nodes that have been upgraded + upgradedNumber int32 + ) + + // The later upgrade operation is conducted based on upgradeInfos + upgradeInfos, err := upgradeinfo.New(r.Client, instance, UpgradeWorkerPodPrefix) + if err != nil { + klog.Errorf(Format("Fail to get static pod and worker pod upgrade info for nodes of StaticPod %v, %v", + request.NamespacedName, err)) + return ctrl.Result{}, err + } + totalNumber = int32(len(upgradeInfos)) + // There are no nodes running target static pods in the cluster + if totalNumber == 0 { + klog.Infof(Format("No static pods need to be upgraded of StaticPod %v", request.NamespacedName)) + return r.updateStaticPodStatus(instance, totalNumber, totalNumber, totalNumber) + } + + // The latest hash value for static pod spec + // This hash value is used in three places + // 1. Automatically added to the annotation of static pods to facilitate checking if the running static pods are up-to-date + // 2. Automatically added to the annotation of worker pods to facilitate checking if the worker pods are up-to-date + // 3. Added to static pods' corresponding configmap to facilitate checking if the configmap is up-to-date + latestHash := util.ComputeHash(&instance.Spec.Template) + + // The latest static pod manifest generated from user-specified template + // The above hash value will be added to the annotation + latestManifest, err := util.GenStaticPodManifest(&instance.Spec.Template, latestHash) + if err != nil { + klog.Errorf(Format("Fail to generate static pod manifest of StaticPod %v, %v", request.NamespacedName, err)) + return ctrl.Result{}, err + } + + // Sync the corresponding configmap to the latest state + if err := r.syncConfigMap(instance, latestHash, latestManifest); err != nil { + klog.Errorf(Format("Fail to sync the corresponding configmap of StaticPod %v, %v", request.NamespacedName, err)) + return ctrl.Result{}, err + } + + // Complete upgrade info + { + // Count the number of upgraded nodes + upgradedNumber = upgradeinfo.SetUpgradeNeededInfos(upgradeInfos, latestHash) + + readyNumber = upgradeinfo.ReadyStaticPodsNumber(upgradeInfos) + + // Set node ready info + if err := checkReadyNodes(r.Client, upgradeInfos); err != nil { + klog.Errorf(Format("Fail to check node ready status of StaticPod %v,%v", request.NamespacedName, err)) + return ctrl.Result{}, err + } + } + + // Sync worker pods + allSucceeded := true + deletePods := make([]*corev1.Pod, 0) + { + for node, info := range upgradeInfos { + if info.WorkerPod == nil { + continue + } + + hash := info.WorkerPod.Annotations[StaticPodHashAnnotation] + // If the worker pod is not up-to-date, then it can be recreated directly + if hash != latestHash { + deletePods = append(deletePods, info.WorkerPod) + continue + } + // If the worker pod is up-to-date, there are three possible situations + // 1. The worker pod is failed, then some irreparable failure has occurred. Just stop reconcile and update status + // 2. The worker pod is succeeded, then this node must be up-to-date. Just delete this worker pod + // 3. The worker pod is running, pending or unknown, then just wait + switch info.WorkerPod.Status.Phase { + case corev1.PodFailed: + r.recorder.Eventf(instance, corev1.EventTypeWarning, "StaticPod Upgrade Failed", "Fail to upgrade node: %v", node) + klog.Errorf("Fail to continue upgrade, cause worker pod %s of StaticPod %v in node %s failed", + info.WorkerPod.Name, request.NamespacedName, node) + return reconcile.Result{}, + fmt.Errorf("fail to continue upgrade, cause worker pod %s of StaticPod %v in node %s failed", + info.WorkerPod.Name, request.NamespacedName, node) + case corev1.PodSucceeded: + deletePods = append(deletePods, info.WorkerPod) + default: + // In this node, the latest worker pod is still running, and we don't need to create new worker for it. + info.WorkerPodRunning = true + allSucceeded = false + } + } + } + + // Clean up unused pods + if err := r.removeUnusedPods(deletePods); err != nil { + klog.Errorf(Format("Fail to remove unused pods of StaticPod %v, %v", request.NamespacedName, err)) + return reconcile.Result{}, err + } + + // If all nodes have been upgraded, just return + // Put this here because we need to clean up the worker pods first + if totalNumber == upgradedNumber { + klog.Infof(Format("All static pods have been upgraded of StaticPod %v", request.NamespacedName)) + return r.updateStaticPodStatus(instance, totalNumber, readyNumber, upgradedNumber) + } + + switch instance.Spec.UpgradeStrategy.Type { + // Auto Upgrade is to automate the upgrade process for the target static pods on ready nodes + // It supports rolling update and the max-unavailable number can be specified by users + case appsv1alpha1.AutoStaticPodUpgradeStrategyType: + if !allSucceeded { + klog.V(5).Infof(Format("Wait last round auto upgrade to finish of StaticPod %v", request.NamespacedName)) + return r.updateStaticPodStatus(instance, totalNumber, readyNumber, upgradedNumber) + } + + if err := r.autoUpgrade(instance, upgradeInfos, latestHash); err != nil { + klog.Errorf(Format("Fail to auto upgrade of StaticPod %v, %v", request.NamespacedName, err)) + return ctrl.Result{}, err + } + return r.updateStaticPodStatus(instance, totalNumber, readyNumber, upgradedNumber) + + // OTA Upgrade can help users control the timing of static pods upgrade + // It will set PodNeedUpgrade condition and work with YurtHub component + case appsv1alpha1.OTAStaticPodUpgradeStrategyType: + if err := r.otaUpgrade(instance, upgradeInfos, latestHash); err != nil { + klog.Errorf(Format("Fail to ota upgrade of StaticPod %v, %v", request.NamespacedName, err)) + return ctrl.Result{}, err + } + return r.updateStaticPodStatus(instance, totalNumber, readyNumber, upgradedNumber) + } + + return ctrl.Result{}, nil +} + +// syncConfigMap moves the target static pod's corresponding configmap to the latest state +func (r *ReconcileStaticPod) syncConfigMap(instance *appsv1alpha1.StaticPod, hash, data string) error { + cmName := util.WithConfigMapPrefix(util.Hyphen(instance.Namespace, instance.Name)) + cm := &corev1.ConfigMap{} + if err := r.Get(context.TODO(), types.NamespacedName{Name: cmName, Namespace: instance.Namespace}, cm); err != nil { + // if the configmap does not exist, then create a new one + if kerr.IsNotFound(err) { + cm = &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: cmName, + Namespace: instance.Namespace, + Annotations: map[string]string{ + StaticPodHashAnnotation: hash, + }, + }, + + Data: map[string]string{ + instance.Spec.StaticPodManifest: data, + }, + } + if err := r.Create(context.TODO(), cm, &client.CreateOptions{}); err != nil { + return err + } + return nil + } + return err + } + + // if the hash value in the annotation of the cm does not match the latest hash, then update the data in the cm + if cm.Annotations[StaticPodHashAnnotation] != hash { + cm.Annotations[StaticPodHashAnnotation] = hash + cm.Data[instance.Spec.StaticPodManifest] = data + + if err := r.Update(context.TODO(), cm, &client.UpdateOptions{}); err != nil { + return err + } + } + + return nil +} + +// autoUpgrade automatically rolling upgrade the target static pods in cluster +func (r *ReconcileStaticPod) autoUpgrade(instance *appsv1alpha1.StaticPod, infos map[string]*upgradeinfo.UpgradeInfo, hash string) error { + // readyUpgradeWaitingNodes represents nodes that need to create worker pods + readyUpgradeWaitingNodes := upgradeinfo.ReadyUpgradeWaitingNodes(infos) + + waitingNumber := len(readyUpgradeWaitingNodes) + if waitingNumber == 0 { + return nil + } + + // max is the maximum number of nodes can be upgraded in current round in auto upgrade mode + max, err := util.UnavailableCount(&instance.Spec.UpgradeStrategy, len(infos)) + if err != nil { + return err + } + + if waitingNumber < max { + max = waitingNumber + } + + readyUpgradeWaitingNodes = readyUpgradeWaitingNodes[:max] + if err := createUpgradeWorker(r.Client, instance, readyUpgradeWaitingNodes, hash, + string(appsv1alpha1.AutoStaticPodUpgradeStrategyType), r.Configuration.UpgradeWorkerImage); err != nil { + return err + } + return nil +} + +// otaUpgrade adds condition PodNeedUpgrade to the target static pods and issue the latest manifest to corresponding nodes +func (r *ReconcileStaticPod) otaUpgrade(instance *appsv1alpha1.StaticPod, infos map[string]*upgradeinfo.UpgradeInfo, hash string) error { + upgradeNeededNodes := upgradeinfo.UpgradeNeededNodes(infos) + upgradedNodes := upgradeinfo.UpgradedNodes(infos) + + // Set condition for upgrade needed static pods + for _, n := range upgradeNeededNodes { + if err := util.SetPodUpgradeCondition(r.Client, corev1.ConditionTrue, infos[n].StaticPod); err != nil { + return err + } + } + + // Set condition for upgraded static pods + for _, n := range upgradedNodes { + if err := util.SetPodUpgradeCondition(r.Client, corev1.ConditionFalse, infos[n].StaticPod); err != nil { + return err + } + } + + return nil +} + +// removeUnusedPods delete pods, include two situations: out-of-date worker pods and succeeded worker pods +func (r *ReconcileStaticPod) removeUnusedPods(pods []*corev1.Pod) error { + for _, pod := range pods { + if err := r.Delete(context.TODO(), pod, &client.DeleteOptions{}); err != nil { + return err + } + klog.V(4).Infof(Format("Delete upgrade worker pod %v", pod.Name)) + } + return nil +} + +// createUpgradeWorker creates static pod upgrade worker to the given nodes +func createUpgradeWorker(c client.Client, instance *appsv1alpha1.StaticPod, nodes []string, hash, mode, img string) error { + for _, node := range nodes { + pod := upgradeWorker.DeepCopy() + pod.Name = UpgradeWorkerPodPrefix + util.Hyphen(node, hash) + pod.Namespace = instance.Namespace + pod.Spec.NodeName = node + metav1.SetMetaDataAnnotation(&pod.ObjectMeta, StaticPodHashAnnotation, hash) + pod.Spec.Volumes = append(pod.Spec.Volumes, corev1.Volume{ + Name: configMapVolumeName, + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: util.WithConfigMapPrefix(util.Hyphen(instance.Namespace, instance.Name)), + }, + }, + }, + }) + pod.Spec.Containers[0].Args = []string{fmt.Sprintf(ArgTmpl, util.Hyphen(instance.Name, node), instance.Namespace, + instance.Spec.StaticPodManifest, hash, mode)} + pod.Spec.Containers[0].Image = img + if err := controllerutil.SetControllerReference(instance, pod, c.Scheme()); err != nil { + return err + } + + if err := c.Create(context.TODO(), pod, &client.CreateOptions{}); err != nil { + return err + } + klog.Infof(Format("Create static pod upgrade worker %s of StaticPod %s", pod.Name, instance.Name)) + } + + return nil +} + +// checkReadyNodes checks and sets the ready status for every node which has the target static pod +func checkReadyNodes(client client.Client, infos map[string]*upgradeinfo.UpgradeInfo) error { + for node, info := range infos { + ready, err := util.NodeReadyByName(client, node) + if err != nil { + return err + } + info.Ready = ready + } + return nil +} + +// updateStatus set the status of instance to the given values +func (r *ReconcileStaticPod) updateStaticPodStatus(instance *appsv1alpha1.StaticPod, totalNum, readyNum, upgradedNum int32) (reconcile.Result, error) { + instance.Status.TotalNumber = totalNum + instance.Status.ReadyNumber = readyNum + instance.Status.UpgradedNumber = upgradedNum + + if err := r.Client.Status().Update(context.TODO(), instance); err != nil { + return reconcile.Result{Requeue: true}, err + } + + return reconcile.Result{}, nil +} diff --git a/pkg/controller/staticpod/staticpod_controller_test.go b/pkg/controller/staticpod/staticpod_controller_test.go new file mode 100644 index 00000000000..d411a15ac14 --- /dev/null +++ b/pkg/controller/staticpod/staticpod_controller_test.go @@ -0,0 +1,159 @@ +/* +Copyright 2023 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package staticpod + +import ( + "context" + "testing" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + "sigs.k8s.io/controller-runtime/pkg/client" + fakeclint "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + appsv1alpha1 "github.com/openyurtio/openyurt/pkg/apis/apps/v1alpha1" + "github.com/openyurtio/openyurt/pkg/controller/staticpod/util" +) + +const ( + TestStaticPodName = "nginx" + TestStaticPodImage = "nginx:1.19.1" +) + +var ( + DefaultMaxUnavailable = intstr.FromString("10%") + TestNodes = []string{"node1", "node2", "node3", "node4"} +) + +func prepareStaticPods() []client.Object { + var pods []client.Object + for _, node := range TestNodes { + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: util.Hyphen(TestStaticPodName, node), + OwnerReferences: []metav1.OwnerReference{{Kind: "Node"}}, + Namespace: metav1.NamespaceDefault, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: TestStaticPodName, + Image: TestStaticPodImage, + }, + }, + NodeName: node, + }, + } + + pods = append(pods, client.Object(pod)) + } + return pods +} + +func prepareNodes() []client.Object { + var nodes []client.Object + for _, node := range TestNodes { + node := &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{Name: node}, + Status: corev1.NodeStatus{ + Conditions: []corev1.NodeCondition{{ + Type: corev1.NodeReady, + Status: corev1.ConditionTrue}}}, + } + nodes = append(nodes, node) + } + return nodes +} + +func TestReconcile(t *testing.T) { + var strategy = []appsv1alpha1.StaticPodUpgradeStrategy{ + {Type: appsv1alpha1.OTAStaticPodUpgradeStrategyType}, + {Type: appsv1alpha1.AutoStaticPodUpgradeStrategyType, MaxUnavailable: &DefaultMaxUnavailable}, + } + staticPods := prepareStaticPods() + nodes := prepareNodes() + instance := &appsv1alpha1.StaticPod{ + ObjectMeta: metav1.ObjectMeta{ + Name: TestStaticPodName, + Namespace: metav1.NamespaceDefault, + }, + Spec: appsv1alpha1.StaticPodSpec{ + StaticPodManifest: "nginx", + Template: corev1.PodTemplateSpec{}, + }, + } + + scheme := runtime.NewScheme() + if err := appsv1alpha1.AddToScheme(scheme); err != nil { + t.Fatal("Fail to add yurt custom resource") + } + if err := clientgoscheme.AddToScheme(scheme); err != nil { + t.Fatal("Fail to add kubernetes clint-go custom resource") + } + + for _, s := range strategy { + instance.Spec.UpgradeStrategy = s + c := fakeclint.NewClientBuilder().WithScheme(scheme).WithRuntimeObjects(instance).WithObjects(staticPods...).WithObjects(nodes...).Build() + + var req = reconcile.Request{NamespacedName: types.NamespacedName{Namespace: metav1.NamespaceDefault, Name: TestStaticPodName}} + rsp := ReconcileStaticPod{ + Client: c, + scheme: scheme, + } + + _, err := rsp.Reconcile(context.TODO(), req) + if err != nil { + t.Fatalf("failed to control static-pod controller") + } + } +} + +func Test_nodeTurnReady(t *testing.T) { + evt := event.UpdateEvent{ + ObjectNew: &corev1.Node{ + Status: corev1.NodeStatus{ + Conditions: []corev1.NodeCondition{ + { + Type: corev1.NodeReady, + Status: corev1.ConditionTrue, + }, + }, + }, + }, + ObjectOld: &corev1.Node{ + Status: corev1.NodeStatus{ + Conditions: []corev1.NodeCondition{ + { + Type: corev1.NodeReady, + Status: corev1.ConditionFalse, + }, + }, + }, + }, + } + t.Run("Test_nodeTurnReady", func(t *testing.T) { + if got := nodeTurnReady(evt); got != true { + t.Errorf("nodeTurnReady() = %v, want true", got) + } + }) +} diff --git a/pkg/controller/staticpod/upgradeinfo/upgrade_info.go b/pkg/controller/staticpod/upgradeinfo/upgrade_info.go new file mode 100644 index 00000000000..cd79c6e9487 --- /dev/null +++ b/pkg/controller/staticpod/upgradeinfo/upgrade_info.go @@ -0,0 +1,191 @@ +/* +Copyright 2023 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package upgradeinfo + +import ( + "context" + "strings" + + corev1 "k8s.io/api/core/v1" + "k8s.io/kubectl/pkg/util/podutils" + "sigs.k8s.io/controller-runtime/pkg/client" + + appsv1alpha1 "github.com/openyurtio/openyurt/pkg/apis/apps/v1alpha1" + "github.com/openyurtio/openyurt/pkg/controller/staticpod/util" +) + +const ( + StaticPodHashAnnotation = "openyurt.io/static-pod-hash" + OTALatestManifestAnnotation = "openyurt.io/ota-latest-version" +) + +// UpgradeInfo is a structure that stores some information used by static pods to upgrade. +type UpgradeInfo struct { + // Static pod running on the node + StaticPod *corev1.Pod + + // Upgrade worker pod running on the node + WorkerPod *corev1.Pod + + // Indicate whether the static pod on the node needs to be upgraded. + // If true, the static pod is not up-to-date and needs to be upgraded. + UpgradeNeeded bool + + // Indicate whether the worker pod on the node is running. + // If true, then the upgrade operation is in progress and does not + // need to create a new worker pod. + WorkerPodRunning bool + + // Indicate whether the node is ready. It's used in Auto mode. + Ready bool +} + +// New constructs the upgrade information for nodes which have the target static pod +func New(c client.Client, instance *appsv1alpha1.StaticPod, workerPodName string) (map[string]*UpgradeInfo, error) { + infos := make(map[string]*UpgradeInfo) + + var podList, upgradeWorkerPodList corev1.PodList + if err := c.List(context.TODO(), &podList, &client.ListOptions{Namespace: instance.Namespace}); err != nil { + return nil, err + } + + if err := c.List(context.TODO(), &upgradeWorkerPodList, &client.ListOptions{Namespace: instance.Namespace}); err != nil { + return nil, err + } + + for i, pod := range podList.Items { + nodeName := pod.Spec.NodeName + if nodeName == "" || pod.DeletionTimestamp != nil { + continue + } + + // The name format of mirror static pod is `StaticPodName-NodeName` + if util.Hyphen(instance.Name, nodeName) == pod.Name && isStaticPod(&pod) { + if info := infos[nodeName]; info == nil { + infos[nodeName] = &UpgradeInfo{} + } + infos[nodeName].StaticPod = &podList.Items[i] + } + } + + for i, pod := range upgradeWorkerPodList.Items { + nodeName := pod.Spec.NodeName + if nodeName == "" || pod.DeletionTimestamp != nil { + continue + } + // The name format of worker pods are `WorkerPodName-NodeName-Hash` Todo: may lead to mismatch + if strings.Contains(pod.Name, workerPodName) { + if info := infos[nodeName]; info == nil { + infos[nodeName] = &UpgradeInfo{} + } + infos[nodeName].WorkerPod = &upgradeWorkerPodList.Items[i] + } + } + + return infos, nil +} + +// isStaticPod judges whether a pod is static by its OwnerReference +func isStaticPod(pod *corev1.Pod) bool { + for _, ownerRef := range pod.GetOwnerReferences() { + if ownerRef.Kind == "Node" { + return true + } + } + return false +} + +// ReadyUpgradeWaitingNodes gets those nodes that satisfied +// 1. node is ready +// 2. node needs to be upgraded +// 3. no latest worker pod running on the node +// On these nodes, new worker pods need to be created for auto mode +func ReadyUpgradeWaitingNodes(infos map[string]*UpgradeInfo) []string { + var nodes []string + for node, info := range infos { + if info.UpgradeNeeded && !info.WorkerPodRunning && info.Ready { + nodes = append(nodes, node) + } + } + return nodes +} + +// ReadyNodes gets nodes that are ready +func ReadyNodes(infos map[string]*UpgradeInfo) []string { + var nodes []string + for node, info := range infos { + if info.Ready { + nodes = append(nodes, node) + } + } + return nodes +} + +// UpgradeNeededNodes gets nodes that are not running the latest static pods +func UpgradeNeededNodes(infos map[string]*UpgradeInfo) []string { + var nodes []string + for node, info := range infos { + if info.UpgradeNeeded { + nodes = append(nodes, node) + } + } + return nodes +} + +// UpgradedNodes gets nodes that are running the latest static pods +func UpgradedNodes(infos map[string]*UpgradeInfo) []string { + var nodes []string + for node, info := range infos { + if !info.UpgradeNeeded { + nodes = append(nodes, node) + } + } + return nodes +} + +// SetUpgradeNeededInfos sets `UpgradeNeeded` flag and counts the number of upgraded nodes +func SetUpgradeNeededInfos(infos map[string]*UpgradeInfo, latestHash string) int32 { + var upgradedNumber int32 + + for _, info := range infos { + if info.StaticPod != nil { + if info.StaticPod.Annotations[StaticPodHashAnnotation] != latestHash { + // Indicate the static pod in this node needs to be upgraded + info.UpgradeNeeded = true + continue + } + upgradedNumber++ + } + } + + return upgradedNumber +} + +// ReadyStaticPodsNumber counts the number of ready static pods +func ReadyStaticPodsNumber(infos map[string]*UpgradeInfo) int32 { + var readyNumber int32 + + for _, info := range infos { + if info.StaticPod != nil { + if podutils.IsPodReady(info.StaticPod) { + readyNumber++ + } + } + } + + return readyNumber +} diff --git a/pkg/controller/staticpod/upgradeinfo/upgrade_info_test.go b/pkg/controller/staticpod/upgradeinfo/upgrade_info_test.go new file mode 100644 index 00000000000..fde5588b5ad --- /dev/null +++ b/pkg/controller/staticpod/upgradeinfo/upgrade_info_test.go @@ -0,0 +1,188 @@ +/* +Copyright 2023 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package upgradeinfo + +import ( + "reflect" + "testing" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/rand" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + appsv1alpha1 "github.com/openyurtio/openyurt/pkg/apis/apps/v1alpha1" +) + +const ( + UpgradeWorkerPodPrefix = "static-pod-upgrade-worker-" +) + +var ( + fakeStaticPodNodes = []string{"node1", "node2", "node3", "node4"} + fakeWorkerPodNodes = []string{"node1", "node2"} + fakeStaticPodName = "nginx" +) + +func newPod(podName string, nodeName string, namespace string, isStaticPod bool) *corev1.Pod { + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: UpgradeWorkerPodPrefix + rand.String(10), + Namespace: namespace, + }, + Spec: corev1.PodSpec{NodeName: nodeName}, + } + + if isStaticPod { + pod.Name = podName + "-" + nodeName + pod.ObjectMeta.OwnerReferences = []metav1.OwnerReference{{Kind: "Node"}} + } + + return pod +} + +func newPods(nodes []string, namespace string, isStaticPod bool) []client.Object { + var pods []client.Object + for _, n := range nodes { + pods = append(pods, client.Object(newPod(fakeStaticPodName, n, namespace, isStaticPod))) + } + return pods +} + +func newStaticPod() *appsv1alpha1.StaticPod { + return &appsv1alpha1.StaticPod{ + TypeMeta: metav1.TypeMeta{}, + ObjectMeta: metav1.ObjectMeta{ + Name: fakeStaticPodName, + Namespace: metav1.NamespaceDefault}, + Spec: appsv1alpha1.StaticPodSpec{}, + Status: appsv1alpha1.StaticPodStatus{}, + } +} + +func Test_ConstructStaticPodsUpgradeInfoList(t *testing.T) { + staticPods := newPods(fakeStaticPodNodes, metav1.NamespaceDefault, true) + workerPods := newPods(fakeWorkerPodNodes, metav1.NamespaceDefault, false) + expect := map[string]*UpgradeInfo{ + "node1": { + StaticPod: staticPods[0].(*corev1.Pod), + WorkerPod: workerPods[0].(*corev1.Pod), + }, + "node2": { + StaticPod: staticPods[1].(*corev1.Pod), + WorkerPod: workerPods[1].(*corev1.Pod), + }, + "node3": { + StaticPod: staticPods[2].(*corev1.Pod), + }, + "node4": { + StaticPod: staticPods[3].(*corev1.Pod), + }, + } + + pods := append(staticPods, workerPods...) + c := fake.NewClientBuilder().WithObjects(pods...).Build() + + t.Run("test", func(t *testing.T) { + spi, _ := New(c, newStaticPod(), UpgradeWorkerPodPrefix) + + if !reflect.DeepEqual(spi, expect) { + t.Fatalf("Fail to test ConstructStaticPodsUpgradeInfoList, got %v, want %v", spi, expect) + } + }) +} + +func TestNodes(t *testing.T) { + tHash := "tHash" + fHash := "fHash" + spi := map[string]*UpgradeInfo{ + "node1": { + WorkerPodRunning: true, + UpgradeNeeded: true, + Ready: true, + }, + "node2": { + StaticPod: &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + OTALatestManifestAnnotation: tHash, + }, + }, + }, + UpgradeNeeded: true, + Ready: true, + }, + "node3": { + StaticPod: &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + OTALatestManifestAnnotation: fHash, + }, + }, + }, + UpgradeNeeded: true, + Ready: true, + }, + "node4": { + Ready: true, + }, + "node5": {}, + } + + expectReadyUpgradeWaitingNodes := map[string]struct{}{"node2": {}, "node3": {}} + expectReadyNodes := map[string]struct{}{"node1": {}, "node2": {}, "node3": {}, "node4": {}} + expectUpgradeNeededNodes := map[string]struct{}{"node1": {}, "node2": {}, "node3": {}} + expectUpgradedNodes := map[string]struct{}{"node4": {}, "node5": {}} + + t.Run("TestReadyUpgradeWaitingNodes", func(t *testing.T) { + if got := ReadyUpgradeWaitingNodes(spi); !hasCommonElement(got, expectReadyUpgradeWaitingNodes) { + t.Fatalf("ReadyUpgradeWaitingNodes = %v, want %v", got, expectReadyUpgradeWaitingNodes) + } + }) + + t.Run("ReadyNodes", func(t *testing.T) { + if got := ReadyNodes(spi); !hasCommonElement(got, expectReadyNodes) { + t.Fatalf("ReadyNodes got %v, want %v", got, expectReadyNodes) + } + }) + + t.Run("UpgradeNeededNodes", func(t *testing.T) { + if got := UpgradeNeededNodes(spi); !hasCommonElement(got, expectUpgradeNeededNodes) { + t.Fatalf("UpgradeNeededNodes got %v, want %v", got, expectUpgradeNeededNodes) + } + }) + + t.Run("UpgradedNodes", func(t *testing.T) { + if got := UpgradedNodes(spi); !hasCommonElement(got, expectUpgradedNodes) { + t.Fatalf("UpgradedNodes got %v, want %v", got, expectUpgradedNodes) + } + }) +} + +func hasCommonElement(a []string, b map[string]struct{}) bool { + if len(a) != len(b) { + return false + } + + for _, i := range a { + if b[i] != struct{}{} { + return false + } + } + return true +} diff --git a/pkg/controller/staticpod/util/util.go b/pkg/controller/staticpod/util/util.go new file mode 100644 index 00000000000..84a0567c9e3 --- /dev/null +++ b/pkg/controller/staticpod/util/util.go @@ -0,0 +1,200 @@ +/* +Copyright 2023 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package util + +import ( + "bytes" + "context" + "fmt" + "hash" + "hash/fnv" + + "github.com/davecgh/go-spew/spew" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/apimachinery/pkg/util/rand" + "k8s.io/cli-runtime/pkg/printers" + "sigs.k8s.io/controller-runtime/pkg/client" + + appsv1alpha1 "github.com/openyurtio/openyurt/pkg/apis/apps/v1alpha1" +) + +const ( + ConfigMapPrefix = "static-pod-cm-" + + // PodNeedUpgrade indicates whether the pod is able to upgrade. + PodNeedUpgrade corev1.PodConditionType = "PodNeedUpgrade" + + StaticPodHashAnnotation = "openyurt.io/static-pod-hash" +) + +var ( + PodGVK = corev1.SchemeGroupVersion.WithKind("Pod") +) + +func Hyphen(str1, str2 string) string { + return str1 + "-" + str2 +} + +// WithConfigMapPrefix add prefix `static-pod-cm-` to the given string +func WithConfigMapPrefix(str string) string { + return ConfigMapPrefix + str +} + +// UnavailableCount returns 0 if unavailability is not requested, the expected +// unavailability number to allow out of numberToUpgrade if requested, or an error if +// the unavailability percentage requested is invalid. +func UnavailableCount(us *appsv1alpha1.StaticPodUpgradeStrategy, numberToUpgrade int) (int, error) { + if us == nil || us.Type != appsv1alpha1.AutoStaticPodUpgradeStrategyType { + return 0, nil + } + return intstr.GetScaledValueFromIntOrPercent(us.MaxUnavailable, numberToUpgrade, true) +} + +// ComputeHash returns a hash value calculated from pod template +func ComputeHash(template *corev1.PodTemplateSpec) string { + podSpecHasher := fnv.New32a() + DeepHashObject(podSpecHasher, *template) + + return rand.SafeEncodeString(fmt.Sprint(podSpecHasher.Sum32())) +} + +// DeepHashObject writes specified object to hash using the spew library +// which follows pointers and prints actual values of the nested objects +// ensuring the hash does not change when a pointer changes. +func DeepHashObject(hasher hash.Hash, objectToWrite interface{}) { + hasher.Reset() + printer := spew.ConfigState{ + Indent: " ", + SortKeys: true, + DisableMethods: true, + SpewKeys: true, + } + printer.Fprintf(hasher, "%#v", objectToWrite) +} + +// GenStaticPodManifest generates manifest from use-specified template +func GenStaticPodManifest(tmplSpec *corev1.PodTemplateSpec, hash string) (string, error) { + pod := &corev1.Pod{ObjectMeta: *tmplSpec.ObjectMeta.DeepCopy(), Spec: *tmplSpec.Spec.DeepCopy()} + // latest hash value will be added to the annotation to facilitate checking if the running static pods are latest + metav1.SetMetaDataAnnotation(&pod.ObjectMeta, StaticPodHashAnnotation, hash) + + pod.GetObjectKind().SetGroupVersionKind(PodGVK) + + var buf bytes.Buffer + y := printers.YAMLPrinter{} + if err := y.PrintObj(pod, &buf); err != nil { + return "", err + } + + return buf.String(), nil +} + +// NodeReadyByName check if the given node is ready +func NodeReadyByName(c client.Client, nodeName string) (bool, error) { + node := &corev1.Node{} + if err := c.Get(context.TODO(), types.NamespacedName{Name: nodeName}, node); err != nil { + return false, err + } + + _, nc := GetNodeCondition(&node.Status, corev1.NodeReady) + + return nc != nil && nc.Status == corev1.ConditionTrue, nil +} + +// GetNodeCondition extracts the provided condition from the given status and returns that. +// Returns nil and -1 if the condition is not present, and the index of the located condition. +func GetNodeCondition(status *corev1.NodeStatus, conditionType corev1.NodeConditionType) (int, *corev1.NodeCondition) { + if status == nil { + return -1, nil + } + for i := range status.Conditions { + if status.Conditions[i].Type == conditionType { + return i, &status.Conditions[i] + } + } + return -1, nil +} + +// SetPodUpgradeCondition set pod condition `PodNeedUpgrade` to the specified value +func SetPodUpgradeCondition(c client.Client, status corev1.ConditionStatus, pod *corev1.Pod) error { + cond := &corev1.PodCondition{ + Type: PodNeedUpgrade, + Status: status, + } + if change := UpdatePodCondition(&pod.Status, cond); change { + if err := c.Status().Update(context.TODO(), pod, &client.UpdateOptions{}); err != nil { + return err + } + } + + return nil +} + +// UpdatePodCondition updates existing pod condition or creates a new one. Sets LastTransitionTime to now if the +// status has changed. +// Returns true if pod condition has changed or has been added. +func UpdatePodCondition(status *corev1.PodStatus, condition *corev1.PodCondition) bool { + condition.LastTransitionTime = metav1.Now() + // Try to find this pod condition. + conditionIndex, oldCondition := GetPodCondition(status, condition.Type) + + if oldCondition == nil { + // We are adding new pod condition. + status.Conditions = append(status.Conditions, *condition) + return true + } + // We are updating an existing condition, so we need to check if it has changed. + if condition.Status == oldCondition.Status { + condition.LastTransitionTime = oldCondition.LastTransitionTime + } + + isEqual := condition.Status == oldCondition.Status && + condition.Reason == oldCondition.Reason && + condition.Message == oldCondition.Message && + condition.LastProbeTime.Equal(&oldCondition.LastProbeTime) && + condition.LastTransitionTime.Equal(&oldCondition.LastTransitionTime) + + status.Conditions[conditionIndex] = *condition + // Return true if one of the fields have changed. + return !isEqual +} + +// GetPodCondition extracts the provided condition from the given status and returns that. +// Returns nil and -1 if the condition is not present, and the index of the located condition. +func GetPodCondition(status *corev1.PodStatus, conditionType corev1.PodConditionType) (int, *corev1.PodCondition) { + if status == nil { + return -1, nil + } + return GetPodConditionFromList(status.Conditions, conditionType) +} + +// GetPodConditionFromList extracts the provided condition from the given list of condition and +// returns the index of the condition and the condition. Returns -1 and nil if the condition is not present. +func GetPodConditionFromList(conditions []corev1.PodCondition, conditionType corev1.PodConditionType) (int, *corev1.PodCondition) { + if conditions == nil { + return -1, nil + } + for i := range conditions { + if conditions[i].Type == conditionType { + return i, &conditions[i] + } + } + return -1, nil +} diff --git a/pkg/webhook/add_v1alpha1_staticpod.go b/pkg/webhook/add_v1alpha1_staticpod.go new file mode 100644 index 00000000000..1b707150a9c --- /dev/null +++ b/pkg/webhook/add_v1alpha1_staticpod.go @@ -0,0 +1,25 @@ +/* +Copyright 2023 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the License); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an AS IS BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package webhook + +import ( + "github.com/openyurtio/openyurt/pkg/webhook/staticpod/v1alpha1" +) + +func init() { + addWebhook(&v1alpha1.StaticPodHandler{}) +} diff --git a/pkg/webhook/pod/v1/pod_handler.go b/pkg/webhook/pod/v1/pod_handler.go index f15c095d63a..55dd14b39da 100644 --- a/pkg/webhook/pod/v1/pod_handler.go +++ b/pkg/webhook/pod/v1/pod_handler.go @@ -43,7 +43,7 @@ func (webhook *PodHandler) SetupWebhookWithManager(mgr ctrl.Manager) (string, st Complete() } -// +kubebuilder:webhook:path=/validate-core-openyurt-io-v1-pod,mutating=false,failurePolicy=fail,sideEffects=None,admissionReviewVersions=v1;v1beta1,groups=core.openyurt.io,resources=pods,verbs=delete,versions=v1,name=validate.core.v1.pod.openyurt.io +// +kubebuilder:webhook:path=/validate-core-openyurt-io-v1-pod,mutating=false,failurePolicy=fail,sideEffects=None,admissionReviewVersions=v1;v1beta1,groups="",resources=pods,verbs=delete,versions=v1,name=validate.core.v1.pod.openyurt.io // Cluster implements a validating and defaulting webhook for PodHandler. type PodHandler struct { diff --git a/pkg/webhook/staticpod/v1alpha1/staticpod_default.go b/pkg/webhook/staticpod/v1alpha1/staticpod_default.go new file mode 100644 index 00000000000..78af8464059 --- /dev/null +++ b/pkg/webhook/staticpod/v1alpha1/staticpod_default.go @@ -0,0 +1,39 @@ +/* +Copyright 2023 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the License); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an AS IS BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + "context" + "fmt" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/runtime" + + "github.com/openyurtio/openyurt/pkg/apis/apps/v1alpha1" +) + +// Default satisfies the defaulting webhook interface. +func (webhook *StaticPodHandler) Default(ctx context.Context, obj runtime.Object) error { + sp, ok := obj.(*v1alpha1.StaticPod) + if !ok { + return apierrors.NewBadRequest(fmt.Sprintf("expected a StaticPod but got a %T", obj)) + } + + v1alpha1.SetDefaultsStaticPod(sp) + + return nil +} diff --git a/pkg/webhook/staticpod/v1alpha1/staticpod_handler.go b/pkg/webhook/staticpod/v1alpha1/staticpod_handler.go new file mode 100644 index 00000000000..0efeabb1483 --- /dev/null +++ b/pkg/webhook/staticpod/v1alpha1/staticpod_handler.go @@ -0,0 +1,51 @@ +/* +Copyright 2023 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the License); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an AS IS BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client/apiutil" + "sigs.k8s.io/controller-runtime/pkg/webhook" + + "github.com/openyurtio/openyurt/pkg/apis/apps/v1alpha1" + "github.com/openyurtio/openyurt/pkg/webhook/util" +) + +// SetupWebhookWithManager sets up Cluster webhooks. mutate path, validatepath, error +func (webhook *StaticPodHandler) SetupWebhookWithManager(mgr ctrl.Manager) (string, string, error) { + gvk, err := apiutil.GVKForObject(&v1alpha1.StaticPod{}, mgr.GetScheme()) + if err != nil { + return "", "", err + } + return util.GenerateMutatePath(gvk), + util.GenerateValidatePath(gvk), + ctrl.NewWebhookManagedBy(mgr). + For(&v1alpha1.StaticPod{}). + WithDefaulter(webhook). + WithValidator(webhook). + Complete() +} + +// +kubebuilder:webhook:path=/validate-apps-openyurt-io-v1alpha1-staticpod,mutating=false,failurePolicy=fail,sideEffects=None,admissionReviewVersions=v1;v1beta1,groups=apps.openyurt.io,resources=staticpods,verbs=create;update,versions=v1alpha1,name=validate.apps.v1alpha1.staticpod.openyurt.io +// +kubebuilder:webhook:path=/mutate-apps-openyurt-io-v1alpha1-staticpod,mutating=true,failurePolicy=fail,sideEffects=None,admissionReviewVersions=v1;v1beta1,groups=apps.openyurt.io,resources=staticpods,verbs=create;update,versions=v1alpha1,name=mutate.apps.v1alpha1.staticpod.openyurt.io + +// Cluster implements a validating and defaulting webhook for Cluster. +type StaticPodHandler struct { +} + +var _ webhook.CustomDefaulter = &StaticPodHandler{} +var _ webhook.CustomValidator = &StaticPodHandler{} diff --git a/pkg/webhook/staticpod/v1alpha1/staticpod_validation.go b/pkg/webhook/staticpod/v1alpha1/staticpod_validation.go new file mode 100644 index 00000000000..849d1170ca5 --- /dev/null +++ b/pkg/webhook/staticpod/v1alpha1/staticpod_validation.go @@ -0,0 +1,104 @@ +/* +Copyright 2023 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the License); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an AS IS BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + "context" + "fmt" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/validation/field" + "k8s.io/klog/v2" + + "github.com/openyurtio/openyurt/pkg/apis/apps/v1alpha1" +) + +// ValidateCreate implements webhook.CustomValidator so a webhook will be registered for the type. +func (webhook *StaticPodHandler) ValidateCreate(ctx context.Context, obj runtime.Object) error { + sp, ok := obj.(*v1alpha1.StaticPod) + if !ok { + return apierrors.NewBadRequest(fmt.Sprintf("expected a StaticPod but got a %T", obj)) + } + + return validate(sp) +} + +// ValidateUpdate implements webhook.CustomValidator so a webhook will be registered for the type. +func (webhook *StaticPodHandler) ValidateUpdate(ctx context.Context, oldObj, newObj runtime.Object) error { + newSP, ok := newObj.(*v1alpha1.StaticPod) + if !ok { + return apierrors.NewBadRequest(fmt.Sprintf("expected a StaticPod but got a %T", newObj)) + } + oldSP, ok := oldObj.(*v1alpha1.StaticPod) + if !ok { + return apierrors.NewBadRequest(fmt.Sprintf("expected a StaticPod but got a %T", oldObj)) + } + + if err := validate(newSP); err != nil { + return err + } + + if err := validate(oldSP); err != nil { + return err + } + + return nil +} + +// ValidateDelete implements webhook.CustomValidator so a webhook will be registered for the type. +func (webhook *StaticPodHandler) ValidateDelete(_ context.Context, obj runtime.Object) error { + return nil +} + +func validate(obj *v1alpha1.StaticPod) error { + if allErrs := validateStaticPodSpec(&obj.Spec); len(allErrs) > 0 { + return apierrors.NewInvalid(v1alpha1.GroupVersion.WithKind("StaticPod").GroupKind(), obj.Name, allErrs) + } + + klog.Infof("Validate StaticPod %s successfully ...", klog.KObj(obj)) + + return nil +} + +// validateStaticPodSpec validates the staticpod spec. +func validateStaticPodSpec(spec *v1alpha1.StaticPodSpec) field.ErrorList { + var allErrs field.ErrorList + + if spec.StaticPodManifest == "" { + allErrs = append(allErrs, field.Required(field.NewPath("spec").Child("StaticPodManifest"), + "StaticPodManifest is required")) + } + + strategy := &spec.UpgradeStrategy + + if strategy.Type != v1alpha1.AutoStaticPodUpgradeStrategyType && strategy.Type != v1alpha1.OTAStaticPodUpgradeStrategyType { + allErrs = append(allErrs, field.NotSupported(field.NewPath("spec").Child("upgradeStrategy"), + strategy, []string{"auto", "ota"})) + } + + if strategy.Type == v1alpha1.AutoStaticPodUpgradeStrategyType && strategy.MaxUnavailable == nil { + allErrs = append(allErrs, field.Required(field.NewPath("spec").Child("upgradeStrategy"), + "max-unavailable is required in auto mode")) + } + + if allErrs != nil { + return allErrs + } + + return nil +}