From 1b32748ac66c145f5f54ea2fd89e58ca19972703 Mon Sep 17 00:00:00 2001 From: kobzonega <122476665+kobzonega@users.noreply.github.com> Date: Thu, 14 Mar 2024 13:10:46 +0300 Subject: [PATCH] Remote Resources sync (#180) --- .github/workflows/run-tests.yml | 1 + Makefile | 10 +- api/v1alpha1/common_types.go | 16 +- api/v1alpha1/remotedatabasenodeset_types.go | 11 +- api/v1alpha1/remotestoragenodeset_types.go | 11 +- api/v1alpha1/zz_generated.deepcopy.go | 80 +++ deploy/ydb-operator/Chart.yaml | 4 +- .../crds/remotedatabasenodeset.yaml | 95 ++++ .../crds/remotestoragenodeset.yaml | 97 +++- internal/controllers/constants/constants.go | 16 +- internal/controllers/database/controller.go | 131 +++-- internal/controllers/database/init.go | 15 +- internal/controllers/database/sync.go | 54 +- .../controllers/databasenodeset/controller.go | 32 +- internal/controllers/databasenodeset/sync.go | 82 ++- .../remotedatabasenodeset/controller.go | 138 +++-- .../remotedatabasenodeset/controller_test.go | 423 ++++++++++++--- .../remotedatabasenodeset/remote_objects.go | 511 ++++++++++++++++++ .../controllers/remotedatabasenodeset/sync.go | 98 ++-- .../remotestoragenodeset/controller.go | 140 +++-- .../remotestoragenodeset/controller_test.go | 429 ++++++++++++--- .../remotestoragenodeset/remote_objects.go | 508 +++++++++++++++++ .../controllers/remotestoragenodeset/sync.go | 95 ++-- internal/controllers/storage/controller.go | 120 ++-- internal/controllers/storage/init.go | 16 +- internal/controllers/storage/sync.go | 51 +- .../controllers/storagenodeset/controller.go | 32 +- internal/controllers/storagenodeset/sync.go | 82 ++- internal/resources/database.go | 18 +- internal/resources/database_statefulset.go | 58 +- internal/resources/remotedatabasenodeset.go | 92 ++++ internal/resources/remotestoragenodeset.go | 87 ++- internal/resources/resource.go | 158 +++++- internal/resources/servicemonitor.go | 4 +- internal/resources/storage.go | 35 +- internal/resources/storage_statefulset.go | 44 +- samples/remote-rbac.yml | 92 ++++ 37 files changed, 3178 insertions(+), 708 deletions(-) create mode 100644 internal/controllers/remotedatabasenodeset/remote_objects.go create mode 100644 internal/controllers/remotestoragenodeset/remote_objects.go create mode 100644 samples/remote-rbac.yml diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index 65383a39..887ee000 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -63,6 +63,7 @@ jobs: uses: golangci/golangci-lint-action@v2 with: version: v1.48.0 + args: --out-format=colored-line-number code-format-check: concurrency: group: lint-autoformat-${{ github.ref }} diff --git a/Makefile b/Makefile index 9b56c442..4854fa61 100644 --- a/Makefile +++ b/Makefile @@ -77,14 +77,16 @@ kind-load: docker tag cr.yandex/yc/ydb-operator:latest kind/ydb-operator:current kind load docker-image kind/ydb-operator:current --name kind-ydb-operator +.PHONY: unit-test unit-test: manifests generate fmt vet envtest ## Run unit tests - KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) -p path)" go test -v -timeout 1800s -p 1 ./internal/controllers/... -ginkgo.vv -coverprofile cover.out + KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) -p path)" go test -v -timeout 1800s -p 1 ./internal/controllers/... -ginkgo.v -coverprofile cover.out -e2e-test: docker-build kind-init kind-load ## Run e2e tests - go test -v -timeout 1800s -p 1 ./e2e/... -args -ginkgo.vv +.PHONY: e2e-test +e2e-test: manifests generate fmt vet docker-build kind-init kind-load ## Run e2e tests + go test -v -timeout 1800s -p 1 ./e2e/... -args -ginkgo.v .PHONY: test -test: unit-test test ## Run all tests +test: unit-test e2e-test ## Run all tests .PHONY: clean clean: diff --git a/api/v1alpha1/common_types.go b/api/v1alpha1/common_types.go index ae05d917..6ce115ee 100644 --- a/api/v1alpha1/common_types.go +++ b/api/v1alpha1/common_types.go @@ -1,6 +1,11 @@ package v1alpha1 -import corev1 "k8s.io/api/core/v1" +import ( + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/ydb-platform/ydb-kubernetes-operator/internal/controllers/constants" +) // NamespacedRef TODO: replace StorageRef type NamespacedRef struct { @@ -40,3 +45,12 @@ type RemoteSpec struct { // +required Cluster string `json:"cluster"` } + +type RemoteResource struct { + Group string `json:"group"` + Version string `json:"version"` + Kind string `json:"kind"` + Name string `json:"name"` + State constants.RemoteResourceState `json:"state"` + Conditions []metav1.Condition `json:"conditions,omitempty"` +} diff --git a/api/v1alpha1/remotedatabasenodeset_types.go b/api/v1alpha1/remotedatabasenodeset_types.go index 1dedba84..a6ee9e34 100644 --- a/api/v1alpha1/remotedatabasenodeset_types.go +++ b/api/v1alpha1/remotedatabasenodeset_types.go @@ -2,6 +2,8 @@ package v1alpha1 import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/ydb-platform/ydb-kubernetes-operator/internal/controllers/constants" ) //+kubebuilder:object:root=true @@ -18,7 +20,14 @@ type RemoteDatabaseNodeSet struct { Spec DatabaseNodeSetSpec `json:"spec,omitempty"` // +optional // +kubebuilder:default:={state: "Pending"} - Status DatabaseNodeSetStatus `json:"status,omitempty"` + Status RemoteDatabaseNodeSetStatus `json:"status,omitempty"` +} + +// DatabaseNodeSetStatus defines the observed state +type RemoteDatabaseNodeSetStatus struct { + State constants.ClusterState `json:"state"` + Conditions []metav1.Condition `json:"conditions,omitempty"` + RemoteResources []RemoteResource `json:"remoteResources,omitempty"` } //+kubebuilder:object:root=true diff --git a/api/v1alpha1/remotestoragenodeset_types.go b/api/v1alpha1/remotestoragenodeset_types.go index dde72221..c31395eb 100644 --- a/api/v1alpha1/remotestoragenodeset_types.go +++ b/api/v1alpha1/remotestoragenodeset_types.go @@ -2,6 +2,8 @@ package v1alpha1 import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/ydb-platform/ydb-kubernetes-operator/internal/controllers/constants" ) //+kubebuilder:object:root=true @@ -18,7 +20,14 @@ type RemoteStorageNodeSet struct { Spec StorageNodeSetSpec `json:"spec,omitempty"` // +optional // +kubebuilder:default:={state: "Pending"} - Status StorageNodeSetStatus `json:"status,omitempty"` + Status RemoteStorageNodeSetStatus `json:"status,omitempty"` +} + +// DatabaseNodeSetStatus defines the observed state +type RemoteStorageNodeSetStatus struct { + State constants.ClusterState `json:"state"` + Conditions []metav1.Condition `json:"conditions,omitempty"` + RemoteResources []RemoteResource `json:"remoteResources,omitempty"` } //+kubebuilder:object:root=true diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index be0a6a33..49875190 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -824,6 +824,57 @@ func (in *RemoteDatabaseNodeSetList) DeepCopyObject() runtime.Object { return nil } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *RemoteDatabaseNodeSetStatus) DeepCopyInto(out *RemoteDatabaseNodeSetStatus) { + *out = *in + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make([]metav1.Condition, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.RemoteResources != nil { + in, out := &in.RemoteResources, &out.RemoteResources + *out = make([]RemoteResource, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RemoteDatabaseNodeSetStatus. +func (in *RemoteDatabaseNodeSetStatus) DeepCopy() *RemoteDatabaseNodeSetStatus { + if in == nil { + return nil + } + out := new(RemoteDatabaseNodeSetStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *RemoteResource) DeepCopyInto(out *RemoteResource) { + *out = *in + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make([]metav1.Condition, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RemoteResource. +func (in *RemoteResource) DeepCopy() *RemoteResource { + if in == nil { + return nil + } + out := new(RemoteResource) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *RemoteSpec) DeepCopyInto(out *RemoteSpec) { *out = *in @@ -898,6 +949,35 @@ func (in *RemoteStorageNodeSetList) DeepCopyObject() runtime.Object { return nil } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *RemoteStorageNodeSetStatus) DeepCopyInto(out *RemoteStorageNodeSetStatus) { + *out = *in + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make([]metav1.Condition, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.RemoteResources != nil { + in, out := &in.RemoteResources, &out.RemoteResources + *out = make([]RemoteResource, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RemoteStorageNodeSetStatus. +func (in *RemoteStorageNodeSetStatus) DeepCopy() *RemoteStorageNodeSetStatus { + if in == nil { + return nil + } + out := new(RemoteStorageNodeSetStatus) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ServerlessDatabaseResources) DeepCopyInto(out *ServerlessDatabaseResources) { *out = *in diff --git a/deploy/ydb-operator/Chart.yaml b/deploy/ydb-operator/Chart.yaml index bf33aed6..0f601df6 100644 --- a/deploy/ydb-operator/Chart.yaml +++ b/deploy/ydb-operator/Chart.yaml @@ -15,10 +15,10 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.4.42 +version: 0.5.0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: "0.4.42" +appVersion: "0.5.0" diff --git a/deploy/ydb-operator/crds/remotedatabasenodeset.yaml b/deploy/ydb-operator/crds/remotedatabasenodeset.yaml index cdc58359..8c193a63 100644 --- a/deploy/ydb-operator/crds/remotedatabasenodeset.yaml +++ b/deploy/ydb-operator/crds/remotedatabasenodeset.yaml @@ -4636,6 +4636,101 @@ spec: - type type: object type: array + remoteResources: + items: + properties: + conditions: + items: + description: "Condition contains details for one aspect of + the current state of this API Resource. --- This struct + is intended for direct use as an array at the field path + .status.conditions. For example, \n \ttype FooStatus struct{ + \t // Represents the observations of a foo's current + state. \t // Known .status.conditions.type are: \"Available\", + \"Progressing\", and \"Degraded\" \t // +patchMergeKey=type + \t // +patchStrategy=merge \t // +listType=map \t + \ // +listMapKey=type \t Conditions []metav1.Condition + `json:\"conditions,omitempty\" patchStrategy:\"merge\" patchMergeKey:\"type\" + protobuf:\"bytes,1,rep,name=conditions\"` \n \t // other + fields \t}" + properties: + lastTransitionTime: + description: lastTransitionTime is the last time the condition + transitioned from one status to another. This should + be when the underlying condition changed. If that is + not known, then using the time when the API field changed + is acceptable. + format: date-time + type: string + message: + description: message is a human readable message indicating + details about the transition. This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: observedGeneration represents the .metadata.generation + that the condition was set based upon. For instance, + if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration + is 9, the condition is out of date with respect to the + current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: reason contains a programmatic identifier + indicating the reason for the condition's last transition. + Producers of specific condition types may define expected + values and meanings for this field, and whether the + values are considered a guaranteed API. The value should + be a CamelCase string. This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, + Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + --- Many .condition.type values are consistent across + resources like Available, but because arbitrary conditions + can be useful (see .node.status.conditions), the ability + to deconflict is important. The regex it matches is + (dns1123SubdomainFmt/)?(qualifiedNameFmt) + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + group: + type: string + kind: + type: string + name: + type: string + state: + type: string + version: + type: string + required: + - group + - kind + - name + - state + - version + type: object + type: array state: type: string required: diff --git a/deploy/ydb-operator/crds/remotestoragenodeset.yaml b/deploy/ydb-operator/crds/remotestoragenodeset.yaml index 1ce18d18..78bcae7d 100644 --- a/deploy/ydb-operator/crds/remotestoragenodeset.yaml +++ b/deploy/ydb-operator/crds/remotestoragenodeset.yaml @@ -4498,7 +4498,7 @@ spec: status: default: state: Pending - description: StorageNodeSetStatus defines the observed state + description: DatabaseNodeSetStatus defines the observed state properties: conditions: items: @@ -4570,6 +4570,101 @@ spec: - type type: object type: array + remoteResources: + items: + properties: + conditions: + items: + description: "Condition contains details for one aspect of + the current state of this API Resource. --- This struct + is intended for direct use as an array at the field path + .status.conditions. For example, \n \ttype FooStatus struct{ + \t // Represents the observations of a foo's current + state. \t // Known .status.conditions.type are: \"Available\", + \"Progressing\", and \"Degraded\" \t // +patchMergeKey=type + \t // +patchStrategy=merge \t // +listType=map \t + \ // +listMapKey=type \t Conditions []metav1.Condition + `json:\"conditions,omitempty\" patchStrategy:\"merge\" patchMergeKey:\"type\" + protobuf:\"bytes,1,rep,name=conditions\"` \n \t // other + fields \t}" + properties: + lastTransitionTime: + description: lastTransitionTime is the last time the condition + transitioned from one status to another. This should + be when the underlying condition changed. If that is + not known, then using the time when the API field changed + is acceptable. + format: date-time + type: string + message: + description: message is a human readable message indicating + details about the transition. This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: observedGeneration represents the .metadata.generation + that the condition was set based upon. For instance, + if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration + is 9, the condition is out of date with respect to the + current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: reason contains a programmatic identifier + indicating the reason for the condition's last transition. + Producers of specific condition types may define expected + values and meanings for this field, and whether the + values are considered a guaranteed API. The value should + be a CamelCase string. This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, + Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + --- Many .condition.type values are consistent across + resources like Available, but because arbitrary conditions + can be useful (see .node.status.conditions), the ability + to deconflict is important. The regex it matches is + (dns1123SubdomainFmt/)?(qualifiedNameFmt) + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + group: + type: string + kind: + type: string + name: + type: string + state: + type: string + version: + type: string + required: + - group + - kind + - name + - state + - version + type: object + type: array state: type: string required: diff --git a/internal/controllers/constants/constants.go b/internal/controllers/constants/constants.go index 20597e3e..48a73002 100644 --- a/internal/controllers/constants/constants.go +++ b/internal/controllers/constants/constants.go @@ -2,7 +2,10 @@ package constants import "time" -type ClusterState string +type ( + ClusterState string + RemoteResourceState string +) const ( StorageKind = "Storage" @@ -18,6 +21,7 @@ const ( DatabasePausedCondition = "DatabasePaused" DatabaseTenantInitializedCondition = "TenantInitialized" DatabaseNodeSetReadyCondition = "DatabaseNodeSetReady" + RemoteResourceSyncedCondition = "ResourceSynced" Stop = true Continue = false @@ -55,11 +59,15 @@ const ( StorageNodeSetReady ClusterState = "Ready" StorageNodeSetPaused ClusterState = "Paused" + ResourceSyncPending RemoteResourceState = "Pending" + ResourceSyncSuccess RemoteResourceState = "Synced" + TenantCreationRequeueDelay = 30 * time.Second StorageAwaitRequeueDelay = 30 * time.Second SharedDatabaseAwaitRequeueDelay = 30 * time.Second - OwnerControllerKey = ".metadata.controller" - DatabaseRefField = ".spec.databaseRef.name" - StorageRefField = ".spec.storageRef.name" + OwnerControllerField = ".metadata.controller" + DatabaseRefField = ".spec.databaseRef.name" + StorageRefField = ".spec.storageRef.name" + SecretField = ".spec.secrets" ) diff --git a/internal/controllers/database/controller.go b/internal/controllers/database/controller.go index 7c5a7d8c..d49088ea 100644 --- a/internal/controllers/database/controller.go +++ b/internal/controllers/database/controller.go @@ -6,20 +6,23 @@ import ( "github.com/go-logr/logr" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/errors" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/rest" "k8s.io/client-go/tools/record" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/handler" "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/predicate" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + "sigs.k8s.io/controller-runtime/pkg/source" - ydbv1alpha1 "github.com/ydb-platform/ydb-kubernetes-operator/api/v1alpha1" - "github.com/ydb-platform/ydb-kubernetes-operator/internal/annotations" + "github.com/ydb-platform/ydb-kubernetes-operator/api/v1alpha1" . "github.com/ydb-platform/ydb-kubernetes-operator/internal/controllers/constants" //nolint:revive,stylecheck + "github.com/ydb-platform/ydb-kubernetes-operator/internal/resources" ) // Reconciler reconciles a Database object @@ -34,6 +37,9 @@ type Reconciler struct { //+kubebuilder:rbac:groups=ydb.tech,resources=databases,verbs=get;list;watch;create;update;patch;delete //+kubebuilder:rbac:groups=ydb.tech,resources=databases/status,verbs=get;update;patch //+kubebuilder:rbac:groups=ydb.tech,resources=databases/finalizers,verbs=update +//+kubebuilder:rbac:groups=ydb.tech,resources=remotedatabasenodesets,verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups=ydb.tech,resources=remotedatabasenodesets/status,verbs=get;update;patch +//+kubebuilder:rbac:groups=ydb.tech,resources=remotedatabasenodesets/finalizers,verbs=update //+kubebuilder:rbac:groups=ydb.tech,resources=databasenodesets,verbs=get;list;watch;create;update;patch;delete //+kubebuilder:rbac:groups=ydb.tech,resources=databasenodesets/status,verbs=get;update;patch //+kubebuilder:rbac:groups=ydb.tech,resources=databasenodesets/finalizers,verbs=update @@ -53,10 +59,10 @@ type Reconciler struct { func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { r.Log = log.FromContext(ctx) - resource := &ydbv1alpha1.Database{} + resource := &v1alpha1.Database{} err := r.Get(ctx, req.NamespacedName, resource) if err != nil { - if errors.IsNotFound(err) { + if apierrors.IsNotFound(err) { r.Log.Info("Database resource not found") return ctrl.Result{Requeue: false}, nil } @@ -71,40 +77,21 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu return result, err } -func ignoreDeletionPredicate() predicate.Predicate { - return predicate.Funcs{ - UpdateFunc: func(e event.UpdateEvent) bool { - generationChanged := e.ObjectOld.GetGeneration() != e.ObjectNew.GetGeneration() - annotationsChanged := !annotations.CompareYdbTechAnnotations(e.ObjectOld.GetAnnotations(), e.ObjectNew.GetAnnotations()) - _, isService := e.ObjectOld.(*corev1.Service) - - return generationChanged || annotationsChanged || isService - }, - DeleteFunc: func(e event.DeleteEvent) bool { - // Evaluates to false if the object has been confirmed deleted. - return !e.DeleteStateUnknown - }, - } -} - -// SetupWithManager sets up the controller with the Manager. -func (r *Reconciler) SetupWithManager(mgr ctrl.Manager) error { - r.Recorder = mgr.GetEventRecorderFor(DatabaseKind) - - controller := ctrl.NewControllerManagedBy(mgr) +// Create FieldIndexer to usage for List requests in Reconcile +func createFieldIndexers(mgr ctrl.Manager) error { if err := mgr.GetFieldIndexer().IndexField( context.Background(), - &ydbv1alpha1.RemoteDatabaseNodeSet{}, - OwnerControllerKey, + &v1alpha1.RemoteDatabaseNodeSet{}, + OwnerControllerField, func(obj client.Object) []string { // grab the RemoteDatabaseNodeSet object, extract the owner... - remoteDatabaseNodeSet := obj.(*ydbv1alpha1.RemoteDatabaseNodeSet) + remoteDatabaseNodeSet := obj.(*v1alpha1.RemoteDatabaseNodeSet) owner := metav1.GetControllerOf(remoteDatabaseNodeSet) if owner == nil { return nil } // ...make sure it's a Database... - if owner.APIVersion != ydbv1alpha1.GroupVersion.String() || owner.Kind != DatabaseKind { + if owner.APIVersion != v1alpha1.GroupVersion.String() || owner.Kind != DatabaseKind { return nil } @@ -113,19 +100,20 @@ func (r *Reconciler) SetupWithManager(mgr ctrl.Manager) error { }); err != nil { return err } + if err := mgr.GetFieldIndexer().IndexField( context.Background(), - &ydbv1alpha1.DatabaseNodeSet{}, - OwnerControllerKey, + &v1alpha1.DatabaseNodeSet{}, + OwnerControllerField, func(obj client.Object) []string { // grab the DatabaseNodeSet object, extract the owner... - databaseNodeSet := obj.(*ydbv1alpha1.DatabaseNodeSet) + databaseNodeSet := obj.(*v1alpha1.DatabaseNodeSet) owner := metav1.GetControllerOf(databaseNodeSet) if owner == nil { return nil } // ...make sure it's a Database... - if owner.APIVersion != ydbv1alpha1.GroupVersion.String() || owner.Kind != DatabaseKind { + if owner.APIVersion != v1alpha1.GroupVersion.String() || owner.Kind != DatabaseKind { return nil } @@ -135,13 +123,76 @@ func (r *Reconciler) SetupWithManager(mgr ctrl.Manager) error { return err } + return mgr.GetFieldIndexer().IndexField( + context.Background(), + &v1alpha1.Database{}, + SecretField, + func(obj client.Object) []string { + secrets := []string{} + // grab the Database object, extract secrets from spec... + database := obj.(*v1alpha1.Database) + + // ...append declared Secret to index... + for _, secret := range database.Spec.Secrets { + secrets = append(secrets, secret.Name) + } + + // ...and if so, return it + return secrets + }) +} + +// SetupWithManager sets up the controller with the Manager. +func (r *Reconciler) SetupWithManager(mgr ctrl.Manager) error { + r.Recorder = mgr.GetEventRecorderFor(DatabaseKind) + controller := ctrl.NewControllerManagedBy(mgr) + if err := createFieldIndexers(mgr); err != nil { + r.Log.Error(err, "unexpected FieldIndexer error") + return err + } + return controller. - For(&ydbv1alpha1.Database{}). - Owns(&ydbv1alpha1.RemoteDatabaseNodeSet{}). - Owns(&ydbv1alpha1.DatabaseNodeSet{}). - Owns(&corev1.Service{}). + For(&v1alpha1.Database{}). + Owns(&v1alpha1.RemoteDatabaseNodeSet{}). + Owns(&v1alpha1.DatabaseNodeSet{}). Owns(&appsv1.StatefulSet{}). Owns(&corev1.ConfigMap{}). - WithEventFilter(ignoreDeletionPredicate()). + Owns(&corev1.Service{}). + Watches( + &source.Kind{Type: &corev1.Secret{}}, + handler.EnqueueRequestsFromMapFunc(r.findDatabasesForSecret), + ). + WithEventFilter(predicate.Or( + predicate.GenerationChangedPredicate{}, + resources.IgnoreDeletetionPredicate(), + resources.LastAppliedAnnotationPredicate(), + resources.IsServicePredicate(), + resources.IsSecretPredicate(), + )). Complete(r) } + +// Find all Databases which using Secret and make request for Reconcile +func (r *Reconciler) findDatabasesForSecret(secret client.Object) []reconcile.Request { + attachedDatabases := &v1alpha1.DatabaseList{} + err := r.List( + context.Background(), + attachedDatabases, + client.InNamespace(secret.GetNamespace()), + client.MatchingFields{SecretField: secret.GetName()}, + ) + if err != nil { + return []reconcile.Request{} + } + + requests := make([]reconcile.Request, len(attachedDatabases.Items)) + for i, item := range attachedDatabases.Items { + requests[i] = reconcile.Request{ + NamespacedName: types.NamespacedName{ + Name: item.GetName(), + Namespace: item.GetNamespace(), + }, + } + } + return requests +} diff --git a/internal/controllers/database/init.go b/internal/controllers/database/init.go index cc973581..012c8797 100644 --- a/internal/controllers/database/init.go +++ b/internal/controllers/database/init.go @@ -52,23 +52,18 @@ func (r *Reconciler) setInitialStatus( return Stop, ctrl.Result{RequeueAfter: DefaultRequeueDelay}, nil } - changed := false - if meta.FindStatusCondition(database.Status.Conditions, DatabaseTenantInitializedCondition) == nil { + if database.Status.State == DatabasePending || + meta.FindStatusCondition(database.Status.Conditions, DatabaseTenantInitializedCondition) == nil { meta.SetStatusCondition(&database.Status.Conditions, metav1.Condition{ Type: DatabaseTenantInitializedCondition, Status: "False", Reason: ReasonInProgress, Message: "Tenant creation in progress", }) - changed = true - } - if database.Status.State == DatabasePending { database.Status.State = DatabasePreparing - changed = true - } - if changed { - return r.setState(ctx, database) + return r.updateStatus(ctx, database) } + return Continue, ctrl.Result{Requeue: false}, nil } @@ -85,7 +80,7 @@ func (r *Reconciler) setInitDatabaseCompleted( }) database.Status.State = DatabaseReady - return r.setState(ctx, database) + return r.updateStatus(ctx, database) } func (r *Reconciler) handleTenantCreation( diff --git a/internal/controllers/database/sync.go b/internal/controllers/database/sync.go index 7d97f766..60cb1ef7 100644 --- a/internal/controllers/database/sync.go +++ b/internal/controllers/database/sync.go @@ -138,7 +138,7 @@ func (r *Reconciler) waitForDatabaseNodeSetsToReady( fmt.Sprintf("Starting to track readiness of running nodeSets objects, expected: %d", len(database.Spec.NodeSets)), ) database.Status.State = DatabaseProvisioning - return r.setState(ctx, database) + return r.updateStatus(ctx, database) } for _, nodeSetSpec := range database.Spec.NodeSets { @@ -166,7 +166,6 @@ func (r *Reconciler) waitForDatabaseNodeSetsToReady( "ProvisioningFailed", fmt.Sprintf("%s with name %s was not found: %s", nodeSetKind, nodeSetName, err), ) - return Stop, ctrl.Result{RequeueAfter: DefaultRequeueDelay}, nil } r.Recorder.Event( database, @@ -217,7 +216,7 @@ func (r *Reconciler) waitForStatefulSetToScale( fmt.Sprintf("Starting to track number of running database pods, expected: %d", database.Spec.Nodes), ) database.Status.State = DatabaseProvisioning - return r.setState(ctx, database) + return r.updateStatus(ctx, database) } if database.Spec.ServerlessResources != nil { @@ -367,22 +366,23 @@ func (r *Reconciler) handleResourcesSync( } } - r.Log.Info("resource sync complete") return Continue, ctrl.Result{Requeue: false}, nil } -func (r *Reconciler) setState( +func (r *Reconciler) updateStatus( ctx context.Context, database *resources.DatabaseBuilder, ) (bool, ctrl.Result, error) { + r.Log.Info("running step updateStatus") + databaseCr := &v1alpha1.Database{} - err := r.Get(ctx, client.ObjectKey{ + err := r.Get(ctx, types.NamespacedName{ Namespace: database.Namespace, Name: database.Name, }, databaseCr) if err != nil { r.Recorder.Event( - databaseCr, + database, corev1.EventTypeWarning, "ControllerError", "Failed fetching CR before status update", @@ -391,21 +391,21 @@ func (r *Reconciler) setState( } oldStatus := databaseCr.Status.State - databaseCr.Status.State = database.Status.State - databaseCr.Status.Conditions = database.Status.Conditions - - err = r.Status().Update(ctx, databaseCr) - if err != nil { - r.Recorder.Event( - databaseCr, - corev1.EventTypeWarning, - "ControllerError", - fmt.Sprintf("failed setting status: %s", err), - ) - return Stop, ctrl.Result{RequeueAfter: DefaultRequeueDelay}, err - } else if oldStatus != databaseCr.Status.State { + if oldStatus != database.Status.State { + databaseCr.Status.State = database.Status.State + databaseCr.Status.Conditions = database.Status.Conditions + err = r.Status().Update(ctx, databaseCr) + if err != nil { + r.Recorder.Event( + database, + corev1.EventTypeWarning, + "ControllerError", + fmt.Sprintf("failed setting status: %s", err), + ) + return Stop, ctrl.Result{RequeueAfter: DefaultRequeueDelay}, err + } r.Recorder.Event( - databaseCr, + database, corev1.EventTypeNormal, "StatusChanged", fmt.Sprintf("Database moved from %s to %s", oldStatus, databaseCr.Status.State), @@ -425,7 +425,7 @@ func (r *Reconciler) syncNodeSetSpecInline( if err := r.List(ctx, databaseNodeSets, client.InNamespace(database.Namespace), client.MatchingFields{ - OwnerControllerKey: database.Name, + OwnerControllerField: database.Name, }, ); err != nil { r.Recorder.Event( @@ -475,7 +475,7 @@ func (r *Reconciler) syncNodeSetSpecInline( if err := r.List(ctx, remoteDatabaseNodeSets, client.InNamespace(database.Namespace), client.MatchingFields{ - OwnerControllerKey: database.Name, + OwnerControllerField: database.Name, }, ); err != nil { r.Recorder.Event( @@ -522,7 +522,6 @@ func (r *Reconciler) syncNodeSetSpecInline( } } - r.Log.Info("syncNodeSetSpecInline complete") return Continue, ctrl.Result{Requeue: false}, nil } @@ -540,14 +539,14 @@ func (r *Reconciler) handlePauseResume( Message: "State Database set to Paused", }) database.Status.State = DatabasePaused - return r.setState(ctx, database) + return r.updateStatus(ctx, database) } if database.Status.State == DatabasePaused && !database.Spec.Pause { r.Log.Info("`pause: false` was noticed, moving Database to state `Ready`") meta.RemoveStatusCondition(&database.Status.Conditions, DatabasePausedCondition) database.Status.State = DatabaseReady - return r.setState(ctx, database) + return r.updateStatus(ctx, database) } return Continue, ctrl.Result{}, nil @@ -585,7 +584,8 @@ func (r *Reconciler) handleFirstStart( func (r *Reconciler) checkDatabaseFrozen( database *resources.DatabaseBuilder, ) (bool, ctrl.Result) { - r.Log.Info("running step checkStorageFrozen") + r.Log.Info("running step checkDatabaseFrozen") + if !database.Spec.OperatorSync { r.Log.Info("`operatorSync: false` is set, no further steps will be run") return Stop, ctrl.Result{} diff --git a/internal/controllers/databasenodeset/controller.go b/internal/controllers/databasenodeset/controller.go index fdd508e6..545e7277 100644 --- a/internal/controllers/databasenodeset/controller.go +++ b/internal/controllers/databasenodeset/controller.go @@ -11,13 +11,12 @@ import ( "k8s.io/client-go/tools/record" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/event" "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/predicate" - api "github.com/ydb-platform/ydb-kubernetes-operator/api/v1alpha1" - "github.com/ydb-platform/ydb-kubernetes-operator/internal/annotations" + "github.com/ydb-platform/ydb-kubernetes-operator/api/v1alpha1" . "github.com/ydb-platform/ydb-kubernetes-operator/internal/controllers/constants" //nolint:revive,stylecheck + "github.com/ydb-platform/ydb-kubernetes-operator/internal/resources" ) // Reconciler reconciles a DatabaseNodeSet object @@ -42,7 +41,7 @@ type Reconciler struct { func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { logger := log.FromContext(ctx) - crDatabaseNodeSet := &api.DatabaseNodeSet{} + crDatabaseNodeSet := &v1alpha1.DatabaseNodeSet{} err := r.Get(ctx, req.NamespacedName, crDatabaseNodeSet) if err != nil { if apierrors.IsNotFound(err) { @@ -61,29 +60,18 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu return result, err } -func ignoreDeletionPredicate() predicate.Predicate { - return predicate.Funcs{ - UpdateFunc: func(e event.UpdateEvent) bool { - generationChanged := e.ObjectOld.GetGeneration() != e.ObjectNew.GetGeneration() - annotationsChanged := !annotations.CompareYdbTechAnnotations(e.ObjectOld.GetAnnotations(), e.ObjectNew.GetAnnotations()) - - return generationChanged || annotationsChanged - }, - DeleteFunc: func(e event.DeleteEvent) bool { - // Evaluates to false if the object has been confirmed deleted. - return !e.DeleteStateUnknown - }, - } -} - // SetupWithManager sets up the controller with the Manager. func (r *Reconciler) SetupWithManager(mgr ctrl.Manager) error { - r.Recorder = mgr.GetEventRecorderFor("DatabaseNodeSet") + r.Recorder = mgr.GetEventRecorderFor(DatabaseNodeSetKind) controller := ctrl.NewControllerManagedBy(mgr) return controller. - For(&api.DatabaseNodeSet{}). + For(&v1alpha1.DatabaseNodeSet{}). Owns(&appsv1.StatefulSet{}). - WithEventFilter(ignoreDeletionPredicate()). + WithEventFilter(predicate.Or( + predicate.GenerationChangedPredicate{}, + resources.IgnoreDeletetionPredicate(), + resources.LastAppliedAnnotationPredicate()), + ). Complete(r) } diff --git a/internal/controllers/databasenodeset/sync.go b/internal/controllers/databasenodeset/sync.go index 0a357a9f..ceeb1d8a 100644 --- a/internal/controllers/databasenodeset/sync.go +++ b/internal/controllers/databasenodeset/sync.go @@ -7,7 +7,7 @@ import ( appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/errors" + apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -16,12 +16,12 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" - ydbv1alpha1 "github.com/ydb-platform/ydb-kubernetes-operator/api/v1alpha1" + "github.com/ydb-platform/ydb-kubernetes-operator/api/v1alpha1" . "github.com/ydb-platform/ydb-kubernetes-operator/internal/controllers/constants" //nolint:revive,stylecheck "github.com/ydb-platform/ydb-kubernetes-operator/internal/resources" ) -func (r *Reconciler) Sync(ctx context.Context, crDatabaseNodeSet *ydbv1alpha1.DatabaseNodeSet) (ctrl.Result, error) { +func (r *Reconciler) Sync(ctx context.Context, crDatabaseNodeSet *v1alpha1.DatabaseNodeSet) (ctrl.Result, error) { var stop bool var result ctrl.Result var err error @@ -32,7 +32,7 @@ func (r *Reconciler) Sync(ctx context.Context, crDatabaseNodeSet *ydbv1alpha1.Da return result, err } - stop, result = r.checkDatabaseNodeSetFrozen(&databaseNodeSet) + stop, result = r.checkDatabaseFrozen(&databaseNodeSet) if stop { return result, nil } @@ -132,7 +132,7 @@ func (r *Reconciler) waitForStatefulSetToScale( fmt.Sprintf("Starting to track number of running databaseNodeSet pods, expected: %d", databaseNodeSet.Spec.Nodes), ) databaseNodeSet.Status.State = DatabaseNodeSetProvisioning - return r.setState(ctx, databaseNodeSet) + return r.updateStatus(ctx, databaseNodeSet) } found := &appsv1.StatefulSet{} @@ -141,7 +141,7 @@ func (r *Reconciler) waitForStatefulSetToScale( Namespace: databaseNodeSet.Namespace, }, found) if err != nil { - if errors.IsNotFound(err) { + if apierrors.IsNotFound(err) { r.Recorder.Event( databaseNodeSet, corev1.EventTypeWarning, @@ -195,15 +195,7 @@ func (r *Reconciler) waitForStatefulSetToScale( return Stop, ctrl.Result{RequeueAfter: DefaultRequeueDelay}, nil } - if databaseNodeSet.Spec.Pause { - meta.SetStatusCondition(&databaseNodeSet.Status.Conditions, metav1.Condition{ - Type: DatabasePausedCondition, - Status: "True", - Reason: ReasonCompleted, - Message: "Scaled DatabaseNodeSet to 0 successfully", - }) - databaseNodeSet.Status.State = DatabaseNodeSetPaused - } else { + if databaseNodeSet.Status.State == DatabaseNodeSetProvisioning { meta.SetStatusCondition(&databaseNodeSet.Status.Conditions, metav1.Condition{ Type: DatabaseNodeSetReadyCondition, Status: "True", @@ -211,23 +203,26 @@ func (r *Reconciler) waitForStatefulSetToScale( Message: fmt.Sprintf("Scaled DatabaseNodeSet to %d successfully", databaseNodeSet.Spec.Nodes), }) databaseNodeSet.Status.State = DatabaseNodeSetReady + return r.updateStatus(ctx, databaseNodeSet) } - return r.setState(ctx, databaseNodeSet) + return Continue, ctrl.Result{Requeue: false}, nil } -func (r *Reconciler) setState( +func (r *Reconciler) updateStatus( ctx context.Context, databaseNodeSet *resources.DatabaseNodeSetResource, ) (bool, ctrl.Result, error) { - crdatabaseNodeSet := &ydbv1alpha1.DatabaseNodeSet{} - err := r.Get(ctx, client.ObjectKey{ + r.Log.Info("running step updateStatus") + + crDatabaseNodeSet := &v1alpha1.DatabaseNodeSet{} + err := r.Get(ctx, types.NamespacedName{ Namespace: databaseNodeSet.Namespace, Name: databaseNodeSet.Name, - }, crdatabaseNodeSet) + }, crDatabaseNodeSet) if err != nil { r.Recorder.Event( - crdatabaseNodeSet, + databaseNodeSet, corev1.EventTypeWarning, "ControllerError", "Failed fetching CR before status update", @@ -235,22 +230,21 @@ func (r *Reconciler) setState( return Stop, ctrl.Result{RequeueAfter: DefaultRequeueDelay}, err } - oldStatus := crdatabaseNodeSet.Status.State - crdatabaseNodeSet.Status.State = databaseNodeSet.Status.State - crdatabaseNodeSet.Status.Conditions = databaseNodeSet.Status.Conditions - - err = r.Status().Update(ctx, crdatabaseNodeSet) - if err != nil { - r.Recorder.Event( - crdatabaseNodeSet, - corev1.EventTypeWarning, - "ControllerError", - fmt.Sprintf("Failed setting status: %s", err), - ) - return Stop, ctrl.Result{RequeueAfter: DefaultRequeueDelay}, err - } else if oldStatus != databaseNodeSet.Status.State { + oldStatus := crDatabaseNodeSet.Status.State + if oldStatus != databaseNodeSet.Status.State { + crDatabaseNodeSet.Status.State = databaseNodeSet.Status.State + crDatabaseNodeSet.Status.Conditions = databaseNodeSet.Status.Conditions + if err = r.Status().Update(ctx, crDatabaseNodeSet); err != nil { + r.Recorder.Event( + databaseNodeSet, + corev1.EventTypeWarning, + "ControllerError", + fmt.Sprintf("Failed setting status: %s", err), + ) + return Stop, ctrl.Result{RequeueAfter: DefaultRequeueDelay}, err + } r.Recorder.Event( - crdatabaseNodeSet, + databaseNodeSet, corev1.EventTypeNormal, "StatusChanged", fmt.Sprintf("DatabaseNodeSet moved from %s to %s", oldStatus, databaseNodeSet.Status.State), @@ -278,7 +272,6 @@ func (r *Reconciler) handlePauseResume( r.Log.Info("running step handlePauseResume") if databaseNodeSet.Status.State == DatabaseReady && databaseNodeSet.Spec.Pause { r.Log.Info("`pause: true` was noticed, moving DatabaseNodeSet to state `Paused`") - meta.RemoveStatusCondition(&databaseNodeSet.Status.Conditions, DatabaseNodeSetReadyCondition) meta.SetStatusCondition(&databaseNodeSet.Status.Conditions, metav1.Condition{ Type: DatabasePausedCondition, Status: "False", @@ -286,29 +279,24 @@ func (r *Reconciler) handlePauseResume( Message: "Transitioning DatabaseNodeSet to Paused state", }) databaseNodeSet.Status.State = DatabaseNodeSetPaused - return r.setState(ctx, databaseNodeSet) + return r.updateStatus(ctx, databaseNodeSet) } if databaseNodeSet.Status.State == DatabaseNodeSetPaused && !databaseNodeSet.Spec.Pause { r.Log.Info("`pause: false` was noticed, moving DatabaseNodeSet to state `Ready`") meta.RemoveStatusCondition(&databaseNodeSet.Status.Conditions, DatabasePausedCondition) - meta.SetStatusCondition(&databaseNodeSet.Status.Conditions, metav1.Condition{ - Type: DatabaseNodeSetReadyCondition, - Status: "False", - Reason: ReasonInProgress, - Message: "Recovering DatabaseNodeSet from Paused state", - }) databaseNodeSet.Status.State = DatabaseNodeSetReady - return r.setState(ctx, databaseNodeSet) + return r.updateStatus(ctx, databaseNodeSet) } return Continue, ctrl.Result{}, nil } -func (r *Reconciler) checkDatabaseNodeSetFrozen( +func (r *Reconciler) checkDatabaseFrozen( databaseNodeSet *resources.DatabaseNodeSetResource, ) (bool, ctrl.Result) { - r.Log.Info("running step checkStorageFrozen") + r.Log.Info("running step checkDatabaseFrozen") + if !databaseNodeSet.Spec.OperatorSync { r.Log.Info("`operatorSync: false` is set, no further steps will be run") return Stop, ctrl.Result{} diff --git a/internal/controllers/remotedatabasenodeset/controller.go b/internal/controllers/remotedatabasenodeset/controller.go index ee1ca8f3..c6cf3e9c 100644 --- a/internal/controllers/remotedatabasenodeset/controller.go +++ b/internal/controllers/remotedatabasenodeset/controller.go @@ -4,6 +4,7 @@ import ( "context" "github.com/go-logr/logr" + corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" @@ -15,10 +16,10 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/cluster" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" - "sigs.k8s.io/controller-runtime/pkg/event" "sigs.k8s.io/controller-runtime/pkg/handler" "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/predicate" + "sigs.k8s.io/controller-runtime/pkg/reconcile" "sigs.k8s.io/controller-runtime/pkg/source" "github.com/ydb-platform/ydb-kubernetes-operator/api/v1alpha1" @@ -56,9 +57,6 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu logger := log.FromContext(ctx) remoteDatabaseNodeSet := &v1alpha1.RemoteDatabaseNodeSet{} - // we'll ignore not-found errors, since they can't be fixed by an immediate - // requeue (we'll need to wait for a new notification), and we can get them - // on deleted requests. if err := r.RemoteClient.Get(ctx, req.NamespacedName, remoteDatabaseNodeSet); err != nil { if apierrors.IsNotFound(err) { logger.Info("RemoteDatabaseNodeSet resource not found on remote cluster") @@ -84,7 +82,7 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu // The object is being deleted if controllerutil.ContainsFinalizer(remoteDatabaseNodeSet, ydbannotations.RemoteFinalizerKey) { // our finalizer is present, so lets handle any external dependency - if err := r.deleteExternalResources(ctx, req.NamespacedName); err != nil { + if err := r.deleteExternalResources(ctx, remoteDatabaseNodeSet); err != nil { // if fail to delete the external dependency here, return with error // so that it can be retried. return ctrl.Result{RequeueAfter: DefaultRequeueDelay}, err @@ -109,62 +107,68 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu return result, err } -func (r *Reconciler) deleteExternalResources(ctx context.Context, key types.NamespacedName) error { - logger := log.FromContext(ctx) - - databaseNodeSet := &v1alpha1.DatabaseNodeSet{} - if err := r.Client.Get(ctx, key, databaseNodeSet); err != nil { - if apierrors.IsNotFound(err) { - logger.Info("DatabaseNodeSet not found") - return nil - } - logger.Error(err, "unable to get DatabaseNodeSet") - return err - } - - if err := r.Client.Delete(ctx, databaseNodeSet); err != nil { - logger.Error(err, "unable to delete DatabaseNodeSet") - return err - } - - return nil -} - -func ignoreDeletionPredicate() predicate.Predicate { - return predicate.Funcs{ - UpdateFunc: func(e event.UpdateEvent) bool { - generationChanged := e.ObjectOld.GetGeneration() != e.ObjectNew.GetGeneration() - annotationsChanged := !ydbannotations.CompareYdbTechAnnotations(e.ObjectOld.GetAnnotations(), e.ObjectNew.GetAnnotations()) - - return generationChanged || annotationsChanged - }, - DeleteFunc: func(e event.DeleteEvent) bool { - // Evaluates to false if the object has been confirmed deleted. - return !e.DeleteStateUnknown - }, - } -} - // SetupWithManager sets up the controller with the Manager. func (r *Reconciler) SetupWithManager(mgr ctrl.Manager, remoteCluster *cluster.Cluster) error { cluster := *remoteCluster - remoteDatabaseNodeSet := &v1alpha1.RemoteDatabaseNodeSet{} r.Recorder = mgr.GetEventRecorderFor(RemoteDatabaseNodeSetKind) r.RemoteRecorder = cluster.GetEventRecorderFor(RemoteDatabaseNodeSetKind) r.RemoteClient = cluster.GetClient() + annotationFilter := func(mapObj client.Object) []reconcile.Request { + requests := make([]reconcile.Request, 0) + + annotations := mapObj.GetAnnotations() + primaryResourceName, exist := annotations[ydbannotations.PrimaryResourceDatabaseAnnotation] + if exist { + databaseNodeSets := &v1alpha1.DatabaseNodeSetList{} + if err := r.Client.List( + context.Background(), + databaseNodeSets, + client.InNamespace(mapObj.GetNamespace()), + client.MatchingFields{ + DatabaseRefField: primaryResourceName, + }, + ); err != nil { + return requests + } + for _, databaseNodeSet := range databaseNodeSets.Items { + requests = append(requests, reconcile.Request{ + NamespacedName: types.NamespacedName{ + Namespace: databaseNodeSet.GetNamespace(), + Name: databaseNodeSet.GetName(), + }, + }) + } + } + return requests + } + isNodeSetFromMgmt, err := buildLocalSelector() if err != nil { return err } + if err := mgr.GetFieldIndexer().IndexField( + context.Background(), + &v1alpha1.DatabaseNodeSet{}, + DatabaseRefField, + func(obj client.Object) []string { + databaseNodeSet := obj.(*v1alpha1.DatabaseNodeSet) + return []string{databaseNodeSet.Spec.DatabaseRef.Name} + }); err != nil { + return err + } + return ctrl.NewControllerManagedBy(mgr). Named(RemoteDatabaseNodeSetKind). Watches( - source.NewKindWithCache(remoteDatabaseNodeSet, cluster.GetCache()), + source.NewKindWithCache(&v1alpha1.RemoteDatabaseNodeSet{}, cluster.GetCache()), &handler.EnqueueRequestForObject{}, - builder.WithPredicates(ignoreDeletionPredicate()), + builder.WithPredicates(predicate.Or( + predicate.GenerationChangedPredicate{}, + resources.LastAppliedAnnotationPredicate(), + )), ). Watches( &source.Kind{Type: &v1alpha1.DatabaseNodeSet{}}, @@ -173,6 +177,19 @@ func (r *Reconciler) SetupWithManager(mgr ctrl.Manager, remoteCluster *cluster.C resources.LabelExistsPredicate(isNodeSetFromMgmt), ), ). + Watches( + &source.Kind{Type: &corev1.Service{}}, + handler.EnqueueRequestsFromMapFunc(annotationFilter), + ). + Watches( + &source.Kind{Type: &corev1.Secret{}}, + handler.EnqueueRequestsFromMapFunc(annotationFilter), + ). + Watches( + &source.Kind{Type: &corev1.ConfigMap{}}, + handler.EnqueueRequestsFromMapFunc(annotationFilter), + ). + WithEventFilter(resources.IgnoreDeletetionPredicate()). Complete(r) } @@ -203,3 +220,36 @@ func BuildRemoteSelector(remoteCluster string) (labels.Selector, error) { labelRequirements = append(labelRequirements, *remoteClusterRequirement) return labels.NewSelector().Add(labelRequirements...), nil } + +func (r *Reconciler) deleteExternalResources( + ctx context.Context, + crRemoteDatabaseNodeSet *v1alpha1.RemoteDatabaseNodeSet, +) error { + logger := log.FromContext(ctx) + + databaseNodeSet := &v1alpha1.DatabaseNodeSet{} + if err := r.Client.Get(ctx, types.NamespacedName{ + Name: crRemoteDatabaseNodeSet.Name, + Namespace: crRemoteDatabaseNodeSet.Namespace, + }, databaseNodeSet); err != nil { + if apierrors.IsNotFound(err) { + logger.Info("DatabaseNodeSet not found") + } else { + logger.Error(err, "unable to get DatabaseNodeSet") + return err + } + } else { + if err := r.Client.Delete(ctx, databaseNodeSet); err != nil { + logger.Error(err, "unable to delete DatabaseNodeSet") + return err + } + } + + remoteDatabaseNodeSet := resources.NewRemoteDatabaseNodeSet(crRemoteDatabaseNodeSet) + if _, _, err := r.removeUnusedRemoteObjects(ctx, &remoteDatabaseNodeSet, []client.Object{}); err != nil { + logger.Error(err, "unable to delete unused remote resources") + return err + } + + return nil +} diff --git a/internal/controllers/remotedatabasenodeset/controller_test.go b/internal/controllers/remotedatabasenodeset/controller_test.go index d08c4f1c..4d4e97d9 100644 --- a/internal/controllers/remotedatabasenodeset/controller_test.go +++ b/internal/controllers/remotedatabasenodeset/controller_test.go @@ -22,24 +22,28 @@ import ( ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/cache" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/apiutil" "sigs.k8s.io/controller-runtime/pkg/cluster" "sigs.k8s.io/controller-runtime/pkg/envtest" logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/log/zap" - api "github.com/ydb-platform/ydb-kubernetes-operator/api/v1alpha1" + "github.com/ydb-platform/ydb-kubernetes-operator/api/v1alpha1" testobjects "github.com/ydb-platform/ydb-kubernetes-operator/e2e/tests/test-objects" + ydbannotations "github.com/ydb-platform/ydb-kubernetes-operator/internal/annotations" . "github.com/ydb-platform/ydb-kubernetes-operator/internal/controllers/constants" "github.com/ydb-platform/ydb-kubernetes-operator/internal/controllers/database" "github.com/ydb-platform/ydb-kubernetes-operator/internal/controllers/databasenodeset" "github.com/ydb-platform/ydb-kubernetes-operator/internal/controllers/remotedatabasenodeset" "github.com/ydb-platform/ydb-kubernetes-operator/internal/controllers/storage" + "github.com/ydb-platform/ydb-kubernetes-operator/internal/resources" "github.com/ydb-platform/ydb-kubernetes-operator/internal/test" ) const ( testRemoteCluster = "remote-cluster" testNodeSetName = "nodeset" + testSecretName = "remote-secret" ) var ( @@ -77,7 +81,7 @@ var _ = BeforeSuite(func() { ErrorIfCRDPathMissing: true, } - err := api.AddToScheme(scheme.Scheme) + err := v1alpha1.AddToScheme(scheme.Scheme) Expect(err).ShouldNot(HaveOccurred()) localCfg, err := localEnv.Start() @@ -111,7 +115,7 @@ var _ = BeforeSuite(func() { o.Scheme = scheme.Scheme o.NewCache = cache.BuilderWithOptions(cache.Options{ SelectorsByObject: cache.SelectorsByObject{ - &api.RemoteDatabaseNodeSet{}: {Label: databaseSelector}, + &v1alpha1.RemoteDatabaseNodeSet{}: {Label: databaseSelector}, }, }) }) @@ -124,6 +128,7 @@ var _ = BeforeSuite(func() { Client: localManager.GetClient(), Scheme: localManager.GetScheme(), Config: localManager.GetConfig(), + Log: logf.Log, }).SetupWithManager(localManager) Expect(err).ShouldNot(HaveOccurred()) @@ -131,6 +136,7 @@ var _ = BeforeSuite(func() { Client: localManager.GetClient(), Scheme: localManager.GetScheme(), Config: localManager.GetConfig(), + Log: logf.Log, }).SetupWithManager(localManager) Expect(err).ShouldNot(HaveOccurred()) @@ -138,6 +144,7 @@ var _ = BeforeSuite(func() { Client: localManager.GetClient(), Scheme: localManager.GetScheme(), Config: localManager.GetConfig(), + Log: logf.Log, }).SetupWithManager(localManager) Expect(err).ShouldNot(HaveOccurred()) @@ -145,12 +152,14 @@ var _ = BeforeSuite(func() { Client: remoteManager.GetClient(), Scheme: remoteManager.GetScheme(), Config: remoteManager.GetConfig(), + Log: logf.Log, }).SetupWithManager(remoteManager) Expect(err).ShouldNot(HaveOccurred()) err = (&remotedatabasenodeset.Reconciler{ Client: remoteManager.GetClient(), Scheme: remoteManager.GetScheme(), + Log: logf.Log, }).SetupWithManager(remoteManager, &remoteCluster) Expect(err).ShouldNot(HaveOccurred()) @@ -184,24 +193,33 @@ var _ = AfterSuite(func() { var _ = Describe("RemoteDatabaseNodeSet controller tests", func() { var localNamespace corev1.Namespace var remoteNamespace corev1.Namespace - var storageSample *api.Storage - var databaseSample *api.Database + var storageSample *v1alpha1.Storage + var databaseSample *v1alpha1.Database BeforeEach(func() { storageSample = testobjects.DefaultStorage(filepath.Join("..", "..", "..", "e2e", "tests", "data", "storage-block-4-2-config.yaml")) databaseSample = testobjects.DefaultDatabase() - databaseSample.Spec.NodeSets = append(databaseSample.Spec.NodeSets, api.DatabaseNodeSetSpecInline{ + databaseSample.Spec.NodeSets = append(databaseSample.Spec.NodeSets, v1alpha1.DatabaseNodeSetSpecInline{ Name: testNodeSetName + "-local", - DatabaseNodeSpec: api.DatabaseNodeSpec{ + DatabaseNodeSpec: v1alpha1.DatabaseNodeSpec{ Nodes: 4, }, }) - databaseSample.Spec.NodeSets = append(databaseSample.Spec.NodeSets, api.DatabaseNodeSetSpecInline{ + databaseSample.Spec.NodeSets = append(databaseSample.Spec.NodeSets, v1alpha1.DatabaseNodeSetSpecInline{ Name: testNodeSetName + "-remote", - Remote: &api.RemoteSpec{ + Remote: &v1alpha1.RemoteSpec{ Cluster: testRemoteCluster, }, - DatabaseNodeSpec: api.DatabaseNodeSpec{ + DatabaseNodeSpec: v1alpha1.DatabaseNodeSpec{ + Nodes: 4, + }, + }) + databaseSample.Spec.NodeSets = append(databaseSample.Spec.NodeSets, v1alpha1.DatabaseNodeSetSpecInline{ + Name: testNodeSetName + "-remote-dedicated", + Remote: &v1alpha1.RemoteSpec{ + Cluster: testRemoteCluster, + }, + DatabaseNodeSpec: v1alpha1.DatabaseNodeSpec{ Nodes: 4, }, }) @@ -223,7 +241,7 @@ var _ = Describe("RemoteDatabaseNodeSet controller tests", func() { By("issuing create Storage commands...") Expect(localClient.Create(ctx, storageSample)).Should(Succeed()) By("checking that Storage created on local cluster...") - foundStorage := api.Storage{} + foundStorage := v1alpha1.Storage{} Eventually(func() bool { Expect(localClient.Get(ctx, types.NamespacedName{ Name: storageSample.Name, @@ -235,10 +253,20 @@ var _ = Describe("RemoteDatabaseNodeSet controller tests", func() { foundStorage.Status.State = StorageReady Expect(localClient.Status().Update(ctx, &foundStorage)).Should(Succeed()) + By("set status Ready to Storage...") + Eventually(func() error { + foundStorage := v1alpha1.Storage{} + Expect(localClient.Get(ctx, types.NamespacedName{ + Name: storageSample.Name, + Namespace: testobjects.YdbNamespace, + }, &foundStorage)) + return localClient.Status().Update(ctx, &foundStorage) + }, test.Timeout, test.Interval).ShouldNot(HaveOccurred()) + By("issuing create Database commands...") Expect(localClient.Create(ctx, databaseSample)).Should(Succeed()) By("checking that Database created on local cluster...") - foundDatabase := api.Database{} + foundDatabase := v1alpha1.Database{} Eventually(func() bool { Expect(localClient.Get(ctx, types.NamespacedName{ Name: databaseSample.Name, @@ -246,65 +274,96 @@ var _ = Describe("RemoteDatabaseNodeSet controller tests", func() { }, &foundDatabase)) return foundDatabase.Status.State == DatabaseProvisioning }, test.Timeout, test.Interval).Should(BeTrue()) - }) - AfterEach(func() { - deleteAll(localEnv, localClient, &localNamespace) - deleteAll(remoteEnv, remoteClient, &localNamespace) - }) + By("checking that DatabaseNodeSet created on local cluster...") + Eventually(func() error { + foundDatabaseNodeSet := &v1alpha1.DatabaseNodeSet{} + return localClient.Get(ctx, types.NamespacedName{ + Name: databaseSample.Name + "-" + testNodeSetName + "-local", + Namespace: testobjects.YdbNamespace, + }, foundDatabaseNodeSet) + }, test.Timeout, test.Interval).ShouldNot(HaveOccurred()) - When("Create Database with RemoteDatabaseNodeSet in k8s-mgmt-cluster", func() { - It("Should create DatabaseNodeSet and sync resources in k8s-data-cluster", func() { - By("checking that DatabaseNodeSet created on local cluster...") - Eventually(func() bool { - foundDatabaseNodeSet := api.DatabaseNodeSetList{} + By("checking that RemoteDatabaseNodeSet created on local cluster...") + Eventually(func() error { + foundRemoteDatabaseNodeSet := &v1alpha1.RemoteDatabaseNodeSet{} + return localClient.Get(ctx, types.NamespacedName{ + Name: databaseSample.Name + "-" + testNodeSetName + "-remote", + Namespace: testobjects.YdbNamespace, + }, foundRemoteDatabaseNodeSet) + }, test.Timeout, test.Interval).ShouldNot(HaveOccurred()) + + By("checking that dedicated RemoteDatabaseNodeSet created on local cluster...") + Eventually(func() error { + foundRemoteDatabaseNodeSet := &v1alpha1.RemoteDatabaseNodeSet{} + return localClient.Get(ctx, types.NamespacedName{ + Name: databaseSample.Name + "-" + testNodeSetName + "-remote-dedicated", + Namespace: testobjects.YdbNamespace, + }, foundRemoteDatabaseNodeSet) + }, test.Timeout, test.Interval).ShouldNot(HaveOccurred()) - Expect(localClient.List(ctx, &foundDatabaseNodeSet, client.InNamespace( - testobjects.YdbNamespace, - ))).Should(Succeed()) + By("checking that DatabaseNodeSet created on remote cluster...") + Eventually(func() bool { + foundDatabaseNodeSetOnRemote := v1alpha1.DatabaseNodeSetList{} - for _, nodeset := range foundDatabaseNodeSet.Items { - if nodeset.Name == databaseSample.Name+"-"+testNodeSetName+"-local" { - return true - } + Expect(remoteClient.List(ctx, &foundDatabaseNodeSetOnRemote, client.InNamespace( + testobjects.YdbNamespace, + ))).Should(Succeed()) + + for _, nodeset := range foundDatabaseNodeSetOnRemote.Items { + if nodeset.Name == databaseSample.Name+"-"+testNodeSetName+"-remote" { + return true } - return false - }, test.Timeout, test.Interval).Should(BeTrue()) + } + return false + }, test.Timeout, test.Interval).Should(BeTrue()) - By("checking that RemoteDatabaseNodeSet created on local cluster...") - Eventually(func() bool { - foundRemoteDatabaseNodeSet := api.RemoteDatabaseNodeSetList{} + By("checking that dedicated DatabaseNodeSet created on remote cluster...") + Eventually(func() bool { + foundDatabaseNodeSetOnRemote := v1alpha1.DatabaseNodeSetList{} - Expect(localClient.List(ctx, &foundRemoteDatabaseNodeSet, client.InNamespace( - testobjects.YdbNamespace, - ))).Should(Succeed()) + Expect(remoteClient.List(ctx, &foundDatabaseNodeSetOnRemote, client.InNamespace( + testobjects.YdbNamespace, + ))).Should(Succeed()) - for _, nodeset := range foundRemoteDatabaseNodeSet.Items { - if nodeset.Name == databaseSample.Name+"-"+testNodeSetName+"-remote" { - return true - } + for _, nodeset := range foundDatabaseNodeSetOnRemote.Items { + if nodeset.Name == databaseSample.Name+"-"+testNodeSetName+"-remote-dedicated" { + return true } - return false - }, test.Timeout, test.Interval).Should(BeTrue()) + } + return false + }, test.Timeout, test.Interval).Should(BeTrue()) + }) - By("checking that DatabaseNodeSet created on remote cluster...") - Eventually(func() bool { - foundDatabaseNodeSetOnRemote := api.DatabaseNodeSetList{} + AfterEach(func() { + deleteAll(localEnv, localClient, &localNamespace) + deleteAll(remoteEnv, remoteClient, &localNamespace) + }) - Expect(remoteClient.List(ctx, &foundDatabaseNodeSetOnRemote, client.InNamespace( - testobjects.YdbNamespace, - ))).Should(Succeed()) + When("Created RemoteDatabaseNodeSet in k8s-mgmt-cluster", func() { + It("Should receive status from k8s-data-cluster", func() { + By("set dedicated DatabaseNodeSet status to Ready on remote cluster...") + foundDedicatedDatabaseNodeSetOnRemote := v1alpha1.DatabaseNodeSet{} + Expect(remoteClient.Get(ctx, types.NamespacedName{ + Name: databaseSample.Name + "-" + testNodeSetName + "-remote-dedicated", + Namespace: testobjects.YdbNamespace, + }, &foundDedicatedDatabaseNodeSetOnRemote)).Should(Succeed()) - for _, nodeset := range foundDatabaseNodeSetOnRemote.Items { - if nodeset.Name == databaseSample.Name+"-"+testNodeSetName+"-remote" { - return true - } - } - return false - }, test.Timeout, test.Interval).Should(BeTrue()) + foundDedicatedDatabaseNodeSetOnRemote.Status.State = DatabaseNodeSetReady + foundDedicatedDatabaseNodeSetOnRemote.Status.Conditions = append( + foundDedicatedDatabaseNodeSetOnRemote.Status.Conditions, + metav1.Condition{ + Type: DatabaseNodeSetReadyCondition, + Status: "True", + Reason: ReasonCompleted, + LastTransitionTime: metav1.NewTime(time.Now()), + Message: fmt.Sprintf("Scaled databaseNodeSet to %d successfully", foundDedicatedDatabaseNodeSetOnRemote.Spec.Nodes), + }, + ) + Expect(remoteClient.Status().Update(ctx, &foundDedicatedDatabaseNodeSetOnRemote)).Should(Succeed()) By("set DatabaseNodeSet status to Ready on remote cluster...") - foundDatabaseNodeSetOnRemote := api.DatabaseNodeSet{} + foundDatabaseNodeSetOnRemote := v1alpha1.DatabaseNodeSet{} Expect(remoteClient.Get(ctx, types.NamespacedName{ Name: databaseSample.Name + "-" + testNodeSetName + "-remote", Namespace: testobjects.YdbNamespace, @@ -323,9 +382,24 @@ var _ = Describe("RemoteDatabaseNodeSet controller tests", func() { ) Expect(remoteClient.Status().Update(ctx, &foundDatabaseNodeSetOnRemote)).Should(Succeed()) + By("checking that dedicated RemoteDatabaseNodeSet status updated on local cluster...") + Eventually(func() bool { + foundRemoteDatabaseNodeSetOnRemote := v1alpha1.RemoteDatabaseNodeSet{} + Expect(localClient.Get(ctx, types.NamespacedName{ + Name: databaseSample.Name + "-" + testNodeSetName + "-remote-dedicated", + Namespace: testobjects.YdbNamespace, + }, &foundRemoteDatabaseNodeSetOnRemote)).Should(Succeed()) + + return meta.IsStatusConditionPresentAndEqual( + foundRemoteDatabaseNodeSetOnRemote.Status.Conditions, + DatabaseNodeSetReadyCondition, + metav1.ConditionTrue, + ) && foundRemoteDatabaseNodeSetOnRemote.Status.State == DatabaseNodeSetReady + }, test.Timeout, test.Interval).Should(BeTrue()) + By("checking that RemoteDatabaseNodeSet status updated on local cluster...") Eventually(func() bool { - foundRemoteDatabaseNodeSetOnRemote := api.RemoteDatabaseNodeSet{} + foundRemoteDatabaseNodeSetOnRemote := v1alpha1.RemoteDatabaseNodeSet{} Expect(localClient.Get(ctx, types.NamespacedName{ Name: databaseSample.Name + "-" + testNodeSetName + "-remote", Namespace: testobjects.YdbNamespace, @@ -339,51 +413,213 @@ var _ = Describe("RemoteDatabaseNodeSet controller tests", func() { }, test.Timeout, test.Interval).Should(BeTrue()) }) }) - When("Delete database with RemoteDatabaseNodeSet in k8s-mgmt-cluster", func() { - It("Should delete all resources in k8s-data-cluster", func() { - By("checking that RemoteDatabaseNodeSet created on local cluster...") - Eventually(func() bool { - foundRemoteDatabaseNodeSet := api.RemoteDatabaseNodeSetList{} - Expect(localClient.List(ctx, &foundRemoteDatabaseNodeSet, client.InNamespace( + When("Created RemoteDatabaseNodeSet with Secrets in k8s-mgmt-cluster", func() { + It("Should sync Secrets into k8s-data-cluster", func() { + By("create simple Secret in Database namespace") + simpleSecret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: testSecretName, + Namespace: testobjects.YdbNamespace, + }, + StringData: map[string]string{ + "message": "Hello from k8s-mgmt-cluster", + }, + } + Expect(localClient.Create(ctx, simpleSecret)) + + By("checking that Database updated on local cluster...") + Eventually(func() error { + foundDatabase := &v1alpha1.Database{} + Expect(localClient.Get(ctx, types.NamespacedName{ + Name: databaseSample.Name, + Namespace: testobjects.YdbNamespace, + }, foundDatabase)) + + foundDatabase.Spec.Secrets = append( + foundDatabase.Spec.Secrets, + &corev1.LocalObjectReference{ + Name: testSecretName, + }, + ) + return localClient.Update(ctx, foundDatabase) + }, test.Timeout, test.Interval).ShouldNot(HaveOccurred()) + + By("checking that Secrets are synced...") + Eventually(func() error { + foundRemoteDatabaseNodeSet := &v1alpha1.RemoteDatabaseNodeSet{} + Expect(localClient.Get(ctx, types.NamespacedName{ + Name: databaseSample.Name + "-" + testNodeSetName + "-remote", + Namespace: testobjects.YdbNamespace, + }, foundRemoteDatabaseNodeSet)).Should(Succeed()) + + localSecret := &corev1.Secret{} + err := localClient.Get(ctx, types.NamespacedName{ + Name: testSecretName, + Namespace: testobjects.YdbNamespace, + }, localSecret) + if err != nil { + return err + } + + remoteSecret := &corev1.Secret{} + err = remoteClient.Get(ctx, types.NamespacedName{ + Name: testSecretName, + Namespace: testobjects.YdbNamespace, + }, remoteSecret) + if err != nil { + return err + } + + primaryResourceName, exist := remoteSecret.Annotations[ydbannotations.PrimaryResourceDatabaseAnnotation] + if !exist { + return fmt.Errorf("annotation %s does not exist on remoteSecret %s", ydbannotations.PrimaryResourceDatabaseAnnotation, remoteSecret.Name) + } + if primaryResourceName != foundRemoteDatabaseNodeSet.Spec.DatabaseRef.Name { + return fmt.Errorf("primaryResourceName %s does not equal databaseRef name %s", primaryResourceName, foundRemoteDatabaseNodeSet.Spec.DatabaseRef.Name) + } + + remoteRV, exist := remoteSecret.Annotations[ydbannotations.RemoteResourceVersionAnnotation] + if !exist { + return fmt.Errorf("annotation %s does not exist on remoteSecret %s", ydbannotations.RemoteResourceVersionAnnotation, remoteSecret.Name) + } + if localSecret.GetResourceVersion() != remoteRV { + return fmt.Errorf("localRV %s does not equal remoteRV %s", localSecret.GetResourceVersion(), remoteRV) + } + + return nil + }, test.Timeout, test.Interval).ShouldNot(HaveOccurred()) + }) + }) + + When("Created RemoteDatabaseNodeSet with Services in k8s-mgmt-cluster", func() { + It("Should sync Services into k8s-data-cluster", func() { + By("checking that Services are synced...") + Eventually(func() error { + foundRemoteDatabaseNodeSet := &v1alpha1.RemoteDatabaseNodeSet{} + Expect(localClient.Get(ctx, types.NamespacedName{ + Name: databaseSample.Name + "-" + testNodeSetName + "-remote", + Namespace: testobjects.YdbNamespace, + }, foundRemoteDatabaseNodeSet)).Should(Succeed()) + + foundServices := corev1.ServiceList{} + Expect(localClient.List(ctx, &foundServices, client.InNamespace( testobjects.YdbNamespace, ))).Should(Succeed()) + for _, localService := range foundServices.Items { + if !strings.HasPrefix(localService.Name, databaseSample.Name) { + continue + } + remoteService := &corev1.Service{} + err := remoteClient.Get(ctx, types.NamespacedName{ + Name: localService.Name, + Namespace: localService.Namespace, + }, remoteService) + if err != nil { + return err + } + } + return nil + }, test.Timeout, test.Interval).ShouldNot(HaveOccurred()) + + By("checking that dedicated RemoteDatabaseNodeSet RemoteStatus are updated...") + Eventually(func() bool { + foundRemoteDatabaseNodeSet := &v1alpha1.RemoteDatabaseNodeSet{} + Expect(localClient.Get(ctx, types.NamespacedName{ + Name: databaseSample.Name + "-" + testNodeSetName + "-remote-dedicated", + Namespace: testobjects.YdbNamespace, + }, foundRemoteDatabaseNodeSet)).Should(Succeed()) + + foundConfigMap := corev1.ConfigMap{} + Expect(remoteClient.Get(ctx, types.NamespacedName{ + Name: databaseSample.Name, + Namespace: testobjects.YdbNamespace, + }, &foundConfigMap)).Should(Succeed()) - for _, nodeset := range foundRemoteDatabaseNodeSet.Items { - if nodeset.Name == databaseSample.Name+"-"+testNodeSetName+"-remote" { - return true + gvk, err := apiutil.GVKForObject(foundConfigMap.DeepCopy(), scheme.Scheme) + Expect(err).ShouldNot(HaveOccurred()) + + for idx := range foundRemoteDatabaseNodeSet.Status.RemoteResources { + remoteResource := foundRemoteDatabaseNodeSet.Status.RemoteResources[idx] + if resources.EqualRemoteResourceWithObject( + &remoteResource, + testobjects.YdbNamespace, + foundConfigMap.DeepCopy(), + gvk, + ) { + if meta.IsStatusConditionPresentAndEqual( + remoteResource.Conditions, + RemoteResourceSyncedCondition, + metav1.ConditionTrue, + ) { + return true + } } } return false }, test.Timeout, test.Interval).Should(BeTrue()) - By("checking that DatabaseNodeSet created on remote cluster...") + By("checking that RemoteDatabaseNodeSet RemoteStatus are updated...") Eventually(func() bool { - foundDatabaseNodeSet := api.DatabaseNodeSetList{} + foundRemoteDatabaseNodeSet := &v1alpha1.RemoteDatabaseNodeSet{} + Expect(localClient.Get(ctx, types.NamespacedName{ + Name: databaseSample.Name + "-" + testNodeSetName + "-remote", + Namespace: testobjects.YdbNamespace, + }, foundRemoteDatabaseNodeSet)).Should(Succeed()) - Expect(remoteClient.List(ctx, &foundDatabaseNodeSet, client.InNamespace( - testobjects.YdbNamespace, - ))).Should(Succeed()) + foundConfigMap := corev1.ConfigMap{} + Expect(remoteClient.Get(ctx, types.NamespacedName{ + Name: databaseSample.Name, + Namespace: testobjects.YdbNamespace, + }, &foundConfigMap)).Should(Succeed()) - for _, nodeset := range foundDatabaseNodeSet.Items { - if nodeset.Name == databaseSample.Name+"-"+testNodeSetName+"-remote" { - return true + gvk, err := apiutil.GVKForObject(foundConfigMap.DeepCopy(), scheme.Scheme) + Expect(err).ShouldNot(HaveOccurred()) + + for idx := range foundRemoteDatabaseNodeSet.Status.RemoteResources { + remoteResource := foundRemoteDatabaseNodeSet.Status.RemoteResources[idx] + if resources.EqualRemoteResourceWithObject( + &remoteResource, + testobjects.YdbNamespace, + foundConfigMap.DeepCopy(), + gvk, + ) { + if meta.IsStatusConditionPresentAndEqual( + remoteResource.Conditions, + RemoteResourceSyncedCondition, + metav1.ConditionTrue, + ) { + return true + } } } return false }, test.Timeout, test.Interval).Should(BeTrue()) + }) + }) + When("Delete RemoteDatabaseNodeSet in k8s-mgmt-cluster", func() { + It("Should delete resources in k8s-data-cluster", func() { By("delete RemoteDatabaseNodeSet on local cluster...") Eventually(func() error { - foundDatabase := api.Database{} + foundDatabase := v1alpha1.Database{} Expect(localClient.Get(ctx, types.NamespacedName{ Name: databaseSample.Name, Namespace: testobjects.YdbNamespace, }, &foundDatabase)).Should(Succeed()) - foundDatabase.Spec.NodeSets = []api.DatabaseNodeSetSpecInline{ + foundDatabase.Spec.NodeSets = []v1alpha1.DatabaseNodeSetSpecInline{ { Name: testNodeSetName + "-local", - DatabaseNodeSpec: api.DatabaseNodeSpec{ + DatabaseNodeSpec: v1alpha1.DatabaseNodeSpec{ + Nodes: 4, + }, + }, + { + Name: testNodeSetName + "-remote-dedicated", + Remote: &v1alpha1.RemoteSpec{ + Cluster: testRemoteCluster, + }, + DatabaseNodeSpec: v1alpha1.DatabaseNodeSpec{ Nodes: 4, }, }, @@ -393,7 +629,7 @@ var _ = Describe("RemoteDatabaseNodeSet controller tests", func() { By("checking that DatabaseNodeSet deleted from remote cluster...") Eventually(func() bool { - foundDatabaseNodeSetOnRemote := api.DatabaseNodeSet{} + foundDatabaseNodeSetOnRemote := v1alpha1.DatabaseNodeSet{} err := remoteClient.Get(ctx, types.NamespacedName{ Name: databaseSample.Name + "-" + testNodeSetName + "-remote", @@ -405,7 +641,7 @@ var _ = Describe("RemoteDatabaseNodeSet controller tests", func() { By("checking that RemoteDatabaseNodeSet deleted from local cluster...") Eventually(func() bool { - foundRemoteDatabaseNodeSet := api.RemoteDatabaseNodeSet{} + foundRemoteDatabaseNodeSet := v1alpha1.RemoteDatabaseNodeSet{} err := localClient.Get(ctx, types.NamespacedName{ Name: databaseSample.Name + "-" + testNodeSetName + "-remote", @@ -414,6 +650,33 @@ var _ = Describe("RemoteDatabaseNodeSet controller tests", func() { return apierrors.IsNotFound(err) }, test.Timeout, test.Interval).Should(BeTrue()) + + By("checking that Services for dedicated DatabaseNodeSet exisiting in remote cluster...") + Eventually(func() error { + foundDedicatedDatabaseNodeSetOnRemote := &v1alpha1.DatabaseNodeSet{} + Expect(remoteClient.Get(ctx, types.NamespacedName{ + Name: databaseSample.Name + "-" + testNodeSetName + "-remote-dedicated", + Namespace: testobjects.YdbNamespace, + }, foundDedicatedDatabaseNodeSetOnRemote)).Should(Succeed()) + + databaseServices := corev1.ServiceList{} + Expect(localClient.List(ctx, &databaseServices, + client.InNamespace(testobjects.YdbNamespace), + )).Should(Succeed()) + for _, databaseService := range databaseServices.Items { + remoteService := &corev1.Service{} + if !strings.HasPrefix(databaseService.GetName(), databaseSample.Name) { + continue + } + if err := remoteClient.Get(ctx, types.NamespacedName{ + Name: databaseService.GetName(), + Namespace: databaseService.GetNamespace(), + }, remoteService); err != nil { + return err + } + } + return nil + }, test.Timeout, test.Interval).ShouldNot(HaveOccurred()) }) }) }) diff --git a/internal/controllers/remotedatabasenodeset/remote_objects.go b/internal/controllers/remotedatabasenodeset/remote_objects.go new file mode 100644 index 00000000..bcecef09 --- /dev/null +++ b/internal/controllers/remotedatabasenodeset/remote_objects.go @@ -0,0 +1,511 @@ +package remotedatabasenodeset + +import ( + "context" + "fmt" + "reflect" + + "github.com/banzaicloud/k8s-objectmatcher/patch" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/selection" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/apiutil" + + "github.com/ydb-platform/ydb-kubernetes-operator/api/v1alpha1" + ydbannotations "github.com/ydb-platform/ydb-kubernetes-operator/internal/annotations" + . "github.com/ydb-platform/ydb-kubernetes-operator/internal/controllers/constants" //nolint:revive,stylecheck + ydblabels "github.com/ydb-platform/ydb-kubernetes-operator/internal/labels" + "github.com/ydb-platform/ydb-kubernetes-operator/internal/resources" +) + +var ( + annotator = patch.NewAnnotator(ydbannotations.LastAppliedAnnotation) + patchMaker = patch.NewPatchMaker(annotator) +) + +func (r *Reconciler) initRemoteResourcesStatus( + ctx context.Context, + remoteDatabaseNodeSet *resources.RemoteDatabaseNodeSetResource, + remoteObjects []client.Object, +) (bool, ctrl.Result, error) { + r.Log.Info("running step initRemoteResourcesStatus") + + syncedResources := []v1alpha1.RemoteResource{} + // copy actual slice to local variable + if remoteDatabaseNodeSet.Status.RemoteResources != nil { + syncedResources = append(syncedResources, remoteDatabaseNodeSet.Status.RemoteResources...) + } + + for idx := range remoteObjects { + remoteObj := remoteObjects[idx] + remoteObjGVK, err := apiutil.GVKForObject(remoteObj, r.Scheme) + if err != nil { + r.Recorder.Event( + remoteDatabaseNodeSet, + corev1.EventTypeWarning, + "ControllerError", + fmt.Sprintf("Failed to recognize GVK for remote object %s with name %s: %s", remoteObjGVK.Kind, remoteObj.GetName(), err), + ) + return Stop, ctrl.Result{RequeueAfter: DefaultRequeueDelay}, err + } + + existInStatus := false + for i := range syncedResources { + syncedResource := syncedResources[i] + if resources.EqualRemoteResourceWithObject( + &syncedResource, + remoteDatabaseNodeSet.Namespace, + remoteObj, + remoteObjGVK, + ) { + existInStatus = true + break + } + } + + if !existInStatus { + remoteDatabaseNodeSet.Status.RemoteResources = append( + remoteDatabaseNodeSet.Status.RemoteResources, + v1alpha1.RemoteResource{ + Group: remoteObjGVK.Group, + Version: remoteObjGVK.Version, + Kind: remoteObjGVK.Kind, + Name: remoteObj.GetName(), + State: ResourceSyncPending, + Conditions: []metav1.Condition{}, + }, + ) + } + } + + return r.updateRemoteResourcesStatus(ctx, remoteDatabaseNodeSet) +} + +func (r *Reconciler) syncRemoteObjects( + ctx context.Context, + remoteDatabaseNodeSet *resources.RemoteDatabaseNodeSetResource, + remoteObjects []client.Object, +) (bool, ctrl.Result, error) { + r.Log.Info("running step syncRemoteObjects") + + for _, remoteObj := range remoteObjects { + // Determine actual GVK for generic client.Object + remoteObjGVK, err := apiutil.GVKForObject(remoteObj, r.Scheme) + if err != nil { + r.Recorder.Event( + remoteDatabaseNodeSet, + corev1.EventTypeWarning, + "ControllerError", + fmt.Sprintf("Failed to recognize GVK for remote object %s with name %s: %s", remoteObjGVK.Kind, remoteObj.GetName(), err), + ) + return Stop, ctrl.Result{RequeueAfter: DefaultRequeueDelay}, err + } + + // Get object to sync from remote cluster + err = r.RemoteClient.Get(ctx, types.NamespacedName{ + Name: remoteObj.GetName(), + Namespace: remoteObj.GetNamespace(), + }, remoteObj) + if err != nil { + // Resource not found on remote cluster but we should retry + if apierrors.IsNotFound(err) { + r.Recorder.Event( + remoteDatabaseNodeSet, + corev1.EventTypeWarning, + "ProvisioningFailed", + fmt.Sprintf("Resource %s with name %s was not found on remote cluster: %s", remoteObjGVK.Kind, remoteObj.GetName(), err), + ) + r.RemoteRecorder.Event( + remoteDatabaseNodeSet, + corev1.EventTypeWarning, + "ProvisioningFailed", + fmt.Sprintf("Resource %s with name %s was not found: %s", remoteObjGVK.Kind, remoteObj.GetName(), err), + ) + } else { + r.Recorder.Event( + remoteDatabaseNodeSet, + corev1.EventTypeWarning, + "ControllerError", + fmt.Sprintf("Failed to get resource %s with name %s on remote cluster: %s", remoteObjGVK.Kind, remoteObj.GetName(), err), + ) + r.RemoteRecorder.Event( + remoteDatabaseNodeSet, + corev1.EventTypeWarning, + "ControllerError", + fmt.Sprintf("Failed to get resource %s with name %s: %s", remoteObjGVK.Kind, remoteObj.GetName(), err), + ) + } + return Stop, ctrl.Result{RequeueAfter: DefaultRequeueDelay}, err + } + + // Create client.Object from api.RemoteResource struct + localObj := resources.CreateResource(remoteObj) + remoteDatabaseNodeSet.SetPrimaryResourceAnnotations(localObj) + // Check object existence in local cluster + err = r.Client.Get(ctx, types.NamespacedName{ + Name: remoteObj.GetName(), + Namespace: remoteObj.GetNamespace(), + }, localObj) + //nolint:nestif + if err != nil { + if !apierrors.IsNotFound(err) { + r.Recorder.Event( + remoteDatabaseNodeSet, + corev1.EventTypeWarning, + "ControllerError", + fmt.Sprintf("Failed to get resource %s with name %s: %s", remoteObjGVK.Kind, remoteObj.GetName(), err), + ) + return Stop, ctrl.Result{RequeueAfter: DefaultRequeueDelay}, err + } + // Object does not exist in local cluster + // Try to create resource in remote cluster + if err := r.Client.Create(ctx, localObj); err != nil { + r.Recorder.Event( + remoteDatabaseNodeSet, + corev1.EventTypeWarning, + "ControllerError", + fmt.Sprintf("Failed to create resource %s with name %s: %s", remoteObjGVK.Kind, remoteObj.GetName(), err), + ) + return Stop, ctrl.Result{RequeueAfter: DefaultRequeueDelay}, nil + } + r.Recorder.Event( + remoteDatabaseNodeSet, + corev1.EventTypeNormal, + "Provisioning", + fmt.Sprintf("RemoteSync CREATE resource %s with name %s", remoteObjGVK.Kind, remoteObj.GetName()), + ) + } else { + // Update client.Object for local object with spec from remote object + updatedObj := resources.UpdateResource(localObj, remoteObj) + remoteDatabaseNodeSet.SetPrimaryResourceAnnotations(updatedObj) + // Remote object existing in local cluster, сheck the need for an update + // Get diff resources and compare bytes by k8s-objectmatcher PatchMaker + patchResult, err := patchMaker.Calculate(localObj, updatedObj, + []patch.CalculateOption{ + patch.IgnoreStatusFields(), + }..., + ) + if err != nil { + r.Recorder.Event( + remoteDatabaseNodeSet, + corev1.EventTypeWarning, + "ControllerError", + fmt.Sprintf("Failed to get diff for remote resource %s with name %s: %s", remoteObjGVK.Kind, remoteObj.GetName(), err), + ) + return Stop, ctrl.Result{RequeueAfter: DefaultRequeueDelay}, nil + } + // We need to check patchResult by k8s-objectmatcher and resourceVersion from annotation + // And update if localObj does not match updatedObj from remote cluster + if !patchResult.IsEmpty() || + remoteObj.GetResourceVersion() != localObj.GetAnnotations()[ydbannotations.RemoteResourceVersionAnnotation] { + r.Recorder.Event( + remoteDatabaseNodeSet, + corev1.EventTypeNormal, + "Provisioning", + fmt.Sprintf("Patch for resource %s with name %s: %s", remoteObjGVK.Kind, remoteObj.GetName(), string(patchResult.Patch)), + ) + // Try to update resource in local cluster + if err := r.Client.Update(ctx, updatedObj); err != nil { + r.Recorder.Event( + remoteDatabaseNodeSet, + corev1.EventTypeWarning, + "ControllerError", + fmt.Sprintf("Failed to update resource %s with name %s: %s", remoteObjGVK.Kind, remoteObj.GetName(), err), + ) + return Stop, ctrl.Result{RequeueAfter: DefaultRequeueDelay}, nil + } + r.Recorder.Event( + remoteDatabaseNodeSet, + corev1.EventTypeNormal, + "Provisioning", + fmt.Sprintf("RemoteSync UPDATE resource %s with name %s resourceVersion %s", remoteObjGVK.Kind, remoteObj.GetName(), remoteObj.GetResourceVersion()), + ) + } + } + // Set status for remote resource in RemoteDatabaseNodeSet object + remoteDatabaseNodeSet.SetRemoteResourceStatus(localObj, remoteObjGVK) + } + + return r.updateRemoteResourcesStatus(ctx, remoteDatabaseNodeSet) +} + +func (r *Reconciler) removeUnusedRemoteObjects( + ctx context.Context, + remoteDatabaseNodeSet *resources.RemoteDatabaseNodeSetResource, + remoteObjects []client.Object, +) (bool, ctrl.Result, error) { + r.Log.Info("running step removeUnusedRemoteObjects") + // We should check every remote resource to need existence in cluster + // Get processed remote resources from object Status + candidatesToDelete := []v1alpha1.RemoteResource{} + + // Remove remote resource from candidates to delete if it declared + // to using in current RemoteDatabaseNodeSet spec + for idx := range remoteDatabaseNodeSet.Status.RemoteResources { + remoteResource := remoteDatabaseNodeSet.Status.RemoteResources[idx] + existInSpec := false + for i := range remoteObjects { + declaredObj := remoteObjects[i] + declaredObjGVK, err := apiutil.GVKForObject(declaredObj, r.Scheme) + if err != nil { + return Stop, ctrl.Result{RequeueAfter: DefaultRequeueDelay}, err + } + if resources.EqualRemoteResourceWithObject( + &remoteResource, + remoteDatabaseNodeSet.Namespace, + declaredObj, + declaredObjGVK, + ) { + existInSpec = true + break + } + } + if !existInSpec { + candidatesToDelete = append(candidatesToDelete, remoteResource) + } + } + + // Check resources usage in another DatabaseNodeSet and make List request + // only if we have candidates to Delete + resourcesToDelete := []v1alpha1.RemoteResource{} + if len(candidatesToDelete) > 0 { + resourcesUsedInAnotherObject, err := r.getRemoteObjectsUsedInNamespace(ctx, remoteDatabaseNodeSet, remoteObjects) + if err != nil { + r.Recorder.Event( + remoteDatabaseNodeSet, + corev1.EventTypeWarning, + "ProvisioningFailed", + fmt.Sprintf("Failed to get resources used in another object: %s", err), + ) + return Stop, ctrl.Result{RequeueAfter: DefaultRequeueDelay}, err + } + for idx := range candidatesToDelete { + remoteResource := candidatesToDelete[idx] + isCandidateExistInANotherObject := false + // Remove resource from cadidates to Delete if another object using it now + for i := range resourcesUsedInAnotherObject { + usedObj := resourcesUsedInAnotherObject[i] + usedObjGVK, err := apiutil.GVKForObject(usedObj, r.Scheme) + if err != nil { + return Stop, ctrl.Result{RequeueAfter: DefaultRequeueDelay}, err + } + if resources.EqualRemoteResourceWithObject( + &remoteResource, + remoteDatabaseNodeSet.Namespace, + usedObj, + usedObjGVK, + ) { + isCandidateExistInANotherObject = true + break + } + } + if !isCandidateExistInANotherObject { + resourcesToDelete = append(resourcesToDelete, remoteResource) + } + } + } + + // Remove unused remote resource from cluster and make API call DELETE + // for every candidate to Delete + for _, recourceToDelete := range resourcesToDelete { + // Convert RemoteResource struct from Status to client.Object + remoteObj, err := resources.ConvertRemoteResourceToObject( + recourceToDelete, + remoteDatabaseNodeSet.Namespace, + ) + if err != nil { + return Stop, ctrl.Result{RequeueAfter: DefaultRequeueDelay}, err + } + + // Determine actual GVK for generic client.Object + remoteResourceGVK, err := apiutil.GVKForObject(remoteObj, r.Scheme) + if err != nil { + r.Recorder.Event( + remoteDatabaseNodeSet, + corev1.EventTypeWarning, + "ControllerError", + fmt.Sprintf("Failed to recognize GVK for remote object %v: %s", remoteObj, err), + ) + return Stop, ctrl.Result{RequeueAfter: DefaultRequeueDelay}, err + } + + // Try to get resource in local cluster + if err := r.Client.Get(ctx, types.NamespacedName{ + Name: remoteObj.GetName(), + Namespace: remoteObj.GetNamespace(), + }, remoteObj); err != nil { + if !apierrors.IsNotFound(err) { + r.Recorder.Event( + remoteDatabaseNodeSet, + corev1.EventTypeWarning, + "ControllerError", + fmt.Sprintf("Failed to get resource %s with name %s: %s", remoteResourceGVK.Kind, remoteObj.GetName(), err), + ) + return Stop, ctrl.Result{RequeueAfter: DefaultRequeueDelay}, err + } + } + + // Skip resource deletion because it using in some Storage + // check by existence of annotation `ydb.tech/primary-resource-storage` + if _, exist := remoteObj.GetAnnotations()[ydbannotations.PrimaryResourceStorageAnnotation]; exist { + continue + } + + // Try to delete unused resource from local cluster + if err := r.Client.Delete(ctx, remoteObj); err != nil { + if !apierrors.IsNotFound(err) { + r.Recorder.Event( + remoteDatabaseNodeSet, + corev1.EventTypeWarning, + "ControllerError", + fmt.Sprintf("Failed to delete resource %s with name %s: %s", remoteResourceGVK.Kind, remoteObj.GetName(), err), + ) + return Stop, ctrl.Result{RequeueAfter: DefaultRequeueDelay}, err + } + } + + r.Recorder.Event( + remoteDatabaseNodeSet, + corev1.EventTypeNormal, + "Provisioning", + fmt.Sprintf("RemoteSync DELETE resource %s with name %s", remoteResourceGVK.Kind, remoteObj.GetName()), + ) + // Remove status for remote resource from RemoteDatabaseNodeSet object + remoteDatabaseNodeSet.RemoveRemoteResourceStatus(remoteObj, remoteResourceGVK) + } + + return r.updateRemoteResourcesStatus(ctx, remoteDatabaseNodeSet) +} + +func (r *Reconciler) getRemoteObjectsUsedInNamespace( + ctx context.Context, + remoteDatabaseNodeSet *resources.RemoteDatabaseNodeSetResource, + remoteObjs []client.Object, +) ([]client.Object, error) { + remoteObjectsUsedInNamespace := []client.Object{} + + // Create label requirement that label `ydb.tech/database-nodeset` which not equal + // to current DatabaseNodeSet object for exclude current nodeSet from List result + labelRequirement, err := labels.NewRequirement( + ydblabels.DatabaseNodeSetComponent, + selection.NotEquals, + []string{remoteDatabaseNodeSet.Labels[ydblabels.DatabaseNodeSetComponent]}, + ) + if err != nil { + return nil, err + } + + // Search another DatabaseNodeSets in current namespace with the same DatabaseRef + // but exclude current nodeSet from result + databaseNodeSets := &v1alpha1.DatabaseNodeSetList{} + if err := r.Client.List( + ctx, + databaseNodeSets, + client.InNamespace(remoteDatabaseNodeSet.Namespace), + client.MatchingLabelsSelector{ + Selector: labels.NewSelector().Add(*labelRequirement), + }, + client.MatchingFields{ + DatabaseRefField: remoteDatabaseNodeSet.Spec.DatabaseRef.Name, + }, + ); err != nil { + return nil, err + } + + // We found some DatabaseNodeSet and should check objects usage + if len(databaseNodeSets.Items) > 0 { + for _, remoteObj := range remoteObjs { + switch obj := remoteObj.(type) { + // If client.Object typed by Secret search existence + // in another DatabaseNodeSet spec.secrets + case *corev1.Secret: + for _, databaseNodeSet := range databaseNodeSets.Items { + for _, secret := range databaseNodeSet.Spec.Secrets { + if obj.GetName() == secret.Name { + remoteObjectsUsedInNamespace = append( + remoteObjectsUsedInNamespace, + obj, + ) + } + } + } + // Else client.Object typed by ConfigMap or Service + // which always used in another DatabaseNodeSet + default: + remoteObjectsUsedInNamespace = append( + remoteObjectsUsedInNamespace, + obj, + ) + } + } + } + + return remoteObjectsUsedInNamespace, nil +} + +func (r *Reconciler) updateRemoteResourcesStatus( + ctx context.Context, + remoteDatabaseNodeSet *resources.RemoteDatabaseNodeSetResource, +) (bool, ctrl.Result, error) { + crRemoteDatabaseNodeSet := &v1alpha1.RemoteDatabaseNodeSet{} + err := r.RemoteClient.Get(ctx, types.NamespacedName{ + Name: remoteDatabaseNodeSet.Name, + Namespace: remoteDatabaseNodeSet.Namespace, + }, crRemoteDatabaseNodeSet) + if err != nil { + r.Recorder.Event( + remoteDatabaseNodeSet, + corev1.EventTypeWarning, + "ControllerError", + "Failed fetching RemoteDatabaseNodeSet on remote cluster before remote status update", + ) + r.RemoteRecorder.Event( + remoteDatabaseNodeSet, + corev1.EventTypeWarning, + "ControllerError", + "Failed fetching RemoteDatabaseNodeSet before remote status update", + ) + return Stop, ctrl.Result{RequeueAfter: DefaultRequeueDelay}, err + } + + oldSyncResources := append([]v1alpha1.RemoteResource{}, crRemoteDatabaseNodeSet.Status.RemoteResources...) + crRemoteDatabaseNodeSet.Status.RemoteResources = append([]v1alpha1.RemoteResource{}, remoteDatabaseNodeSet.Status.RemoteResources...) + + if !reflect.DeepEqual(oldSyncResources, remoteDatabaseNodeSet.Status.RemoteResources) { + if err = r.RemoteClient.Status().Update(ctx, crRemoteDatabaseNodeSet); err != nil { + r.Recorder.Event( + remoteDatabaseNodeSet, + corev1.EventTypeWarning, + "ControllerError", + fmt.Sprintf("Failed to update status for remote resources on remote cluster: %s", err), + ) + r.RemoteRecorder.Event( + remoteDatabaseNodeSet, + corev1.EventTypeWarning, + "ControllerError", + fmt.Sprintf("Failed to update status for remote resources: %s", err), + ) + return Stop, ctrl.Result{RequeueAfter: DefaultRequeueDelay}, err + } + r.Recorder.Event( + remoteDatabaseNodeSet, + corev1.EventTypeNormal, + "StatusChanged", + "Status updated for remote resources on remote cluster", + ) + r.RemoteRecorder.Event( + remoteDatabaseNodeSet, + corev1.EventTypeNormal, + "StatusChanged", + "Status updated for remote resources", + ) + return Stop, ctrl.Result{RequeueAfter: StatusUpdateRequeueDelay}, nil + } + + return Continue, ctrl.Result{Requeue: false}, nil +} diff --git a/internal/controllers/remotedatabasenodeset/sync.go b/internal/controllers/remotedatabasenodeset/sync.go index 24336ab1..05741319 100644 --- a/internal/controllers/remotedatabasenodeset/sync.go +++ b/internal/controllers/remotedatabasenodeset/sync.go @@ -11,28 +11,40 @@ import ( ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" - ydbv1alpha1 "github.com/ydb-platform/ydb-kubernetes-operator/api/v1alpha1" + "github.com/ydb-platform/ydb-kubernetes-operator/api/v1alpha1" . "github.com/ydb-platform/ydb-kubernetes-operator/internal/controllers/constants" //nolint:revive,stylecheck "github.com/ydb-platform/ydb-kubernetes-operator/internal/resources" ) -func (r *Reconciler) Sync(ctx context.Context, crRemoteDatabaseNodeSet *ydbv1alpha1.RemoteDatabaseNodeSet) (ctrl.Result, error) { +func (r *Reconciler) Sync(ctx context.Context, crRemoteDatabaseNodeSet *v1alpha1.RemoteDatabaseNodeSet) (ctrl.Result, error) { var stop bool var result ctrl.Result var err error remoteDatabaseNodeSet := resources.NewRemoteDatabaseNodeSet(crRemoteDatabaseNodeSet) + remoteObjects := remoteDatabaseNodeSet.GetRemoteObjects() + + stop, result, err = r.initRemoteResourcesStatus(ctx, &remoteDatabaseNodeSet, remoteObjects) + if stop { + return result, err + } + + stop, result, err = r.syncRemoteObjects(ctx, &remoteDatabaseNodeSet, remoteObjects) + if stop { + return result, err + } + stop, result, err = r.handleResourcesSync(ctx, &remoteDatabaseNodeSet) if stop { return result, err } - stop, result, err = r.updateStatus(ctx, crRemoteDatabaseNodeSet) + stop, result, err = r.removeUnusedRemoteObjects(ctx, &remoteDatabaseNodeSet, remoteObjects) if stop { return result, err } - return result, err + return ctrl.Result{}, nil } func (r *Reconciler) handleResourcesSync( @@ -85,24 +97,23 @@ func (r *Reconciler) handleResourcesSync( ) } } - r.Log.Info("resource sync complete") - return Continue, ctrl.Result{Requeue: false}, nil + + return r.updateRemoteStatus(ctx, remoteDatabaseNodeSet) } -func (r *Reconciler) updateStatus( +func (r *Reconciler) updateRemoteStatus( ctx context.Context, - crRemoteDatabaseNodeSet *ydbv1alpha1.RemoteDatabaseNodeSet, + remoteDatabaseNodeSet *resources.RemoteDatabaseNodeSetResource, ) (bool, ctrl.Result, error) { - r.Log.Info("running step updateStatus") - - databaseNodeSet := &ydbv1alpha1.DatabaseNodeSet{} - err := r.Client.Get(ctx, types.NamespacedName{ - Name: crRemoteDatabaseNodeSet.Name, - Namespace: crRemoteDatabaseNodeSet.Namespace, - }, databaseNodeSet) - if err != nil { + r.Log.Info("running step updateRemoteStatus") + + crDatabaseNodeSet := &v1alpha1.DatabaseNodeSet{} + if err := r.Client.Get(ctx, types.NamespacedName{ + Name: remoteDatabaseNodeSet.Name, + Namespace: remoteDatabaseNodeSet.Namespace, + }, crDatabaseNodeSet); err != nil { r.Recorder.Event( - crRemoteDatabaseNodeSet, + remoteDatabaseNodeSet, corev1.EventTypeWarning, "ControllerError", "Failed fetching DatabaseNodeSet before status update", @@ -110,39 +121,62 @@ func (r *Reconciler) updateStatus( return Stop, ctrl.Result{RequeueAfter: DefaultRequeueDelay}, err } - oldStatus := crRemoteDatabaseNodeSet.Status.State - crRemoteDatabaseNodeSet.Status.State = databaseNodeSet.Status.State - crRemoteDatabaseNodeSet.Status.Conditions = databaseNodeSet.Status.Conditions - - err = r.RemoteClient.Status().Update(ctx, crRemoteDatabaseNodeSet) - if err != nil { + crRemoteDatabaseNodeSet := &v1alpha1.RemoteDatabaseNodeSet{} + if err := r.RemoteClient.Get(ctx, types.NamespacedName{ + Name: remoteDatabaseNodeSet.Name, + Namespace: remoteDatabaseNodeSet.Namespace, + }, crRemoteDatabaseNodeSet); err != nil { r.Recorder.Event( - crRemoteDatabaseNodeSet, + remoteDatabaseNodeSet, corev1.EventTypeWarning, "ControllerError", - fmt.Sprintf("Failed setting status on remote cluster: %s", err), + "Failed fetching RemoteDatabaseNodeSet on remote cluster before status update", ) r.RemoteRecorder.Event( - crRemoteDatabaseNodeSet, + remoteDatabaseNodeSet, corev1.EventTypeWarning, "ControllerError", - fmt.Sprintf("Failed setting status: %s", err), + "Failed fetching RemoteDatabaseNodeSet before status update", ) return Stop, ctrl.Result{RequeueAfter: DefaultRequeueDelay}, err - } else if oldStatus != crRemoteDatabaseNodeSet.Status.State { + } + + oldStatus := crRemoteDatabaseNodeSet.Status.State + if oldStatus != crDatabaseNodeSet.Status.State { + crRemoteDatabaseNodeSet.Status.State = crDatabaseNodeSet.Status.State + crRemoteDatabaseNodeSet.Status.Conditions = crDatabaseNodeSet.Status.Conditions + if err := r.RemoteClient.Status().Update(ctx, crRemoteDatabaseNodeSet); err != nil { + r.Recorder.Event( + remoteDatabaseNodeSet, + corev1.EventTypeWarning, + "ControllerError", + fmt.Sprintf("Failed to update status on remote cluster: %s", err), + ) + r.RemoteRecorder.Event( + remoteDatabaseNodeSet, + corev1.EventTypeWarning, + "ControllerError", + fmt.Sprintf("Failed to update status: %s", err), + ) + return Stop, ctrl.Result{RequeueAfter: DefaultRequeueDelay}, err + } r.Recorder.Event( - crRemoteDatabaseNodeSet, + remoteDatabaseNodeSet, corev1.EventTypeNormal, "StatusChanged", - fmt.Sprintf("DatabaseNodeSet moved from %s to %s on remote cluster", oldStatus, crRemoteDatabaseNodeSet.Status.State), + "DatabaseNodeSet status updated on remote cluster", ) r.RemoteRecorder.Event( - crRemoteDatabaseNodeSet, + remoteDatabaseNodeSet, corev1.EventTypeNormal, "StatusChanged", - fmt.Sprintf("DatabaseNodeSet moved from %s to %s", oldStatus, crRemoteDatabaseNodeSet.Status.State), + "RemoteDatabaseNodeSet status updated", ) + + r.Log.Info("step updateRemoteStatus requeue reconcile") + return Stop, ctrl.Result{RequeueAfter: StatusUpdateRequeueDelay}, nil } + r.Log.Info("step updateRemoteStatus completed") return Continue, ctrl.Result{Requeue: false}, nil } diff --git a/internal/controllers/remotestoragenodeset/controller.go b/internal/controllers/remotestoragenodeset/controller.go index 53820d21..cb2664b8 100644 --- a/internal/controllers/remotestoragenodeset/controller.go +++ b/internal/controllers/remotestoragenodeset/controller.go @@ -4,6 +4,7 @@ import ( "context" "github.com/go-logr/logr" + corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" @@ -15,13 +16,13 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/cluster" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" - "sigs.k8s.io/controller-runtime/pkg/event" "sigs.k8s.io/controller-runtime/pkg/handler" "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/predicate" + "sigs.k8s.io/controller-runtime/pkg/reconcile" "sigs.k8s.io/controller-runtime/pkg/source" - v1alpha1 "github.com/ydb-platform/ydb-kubernetes-operator/api/v1alpha1" + "github.com/ydb-platform/ydb-kubernetes-operator/api/v1alpha1" ydbannotations "github.com/ydb-platform/ydb-kubernetes-operator/internal/annotations" . "github.com/ydb-platform/ydb-kubernetes-operator/internal/controllers/constants" //nolint:revive,stylecheck ydblabels "github.com/ydb-platform/ydb-kubernetes-operator/internal/labels" @@ -58,7 +59,7 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu remoteStorageNodeSet := &v1alpha1.RemoteStorageNodeSet{} if err := r.RemoteClient.Get(ctx, req.NamespacedName, remoteStorageNodeSet); err != nil { if apierrors.IsNotFound(err) { - logger.Info("RemoteStorageNodeSet resource not found") + logger.Info("RemoteStorageNodeSet resource not found on remote cluster") return ctrl.Result{Requeue: false}, nil } logger.Error(err, "unable to get RemoteStorageNodeSet on remote cluster") @@ -106,65 +107,68 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu return result, err } -func (r *Reconciler) deleteExternalResources(ctx context.Context, remoteStorageNodeSet *v1alpha1.RemoteStorageNodeSet) error { - logger := log.FromContext(ctx) - - storageNodeSet := &v1alpha1.StorageNodeSet{} - if err := r.Client.Get(ctx, types.NamespacedName{ - Name: remoteStorageNodeSet.Name, - Namespace: remoteStorageNodeSet.Namespace, - }, storageNodeSet); err != nil { - if apierrors.IsNotFound(err) { - logger.Info("StorageNodeSet not found") - return nil - } - logger.Error(err, "unable to get StorageNodeSet") - return err - } - - if err := r.Client.Delete(ctx, storageNodeSet); err != nil { - logger.Error(err, "unable to delete StorageNodeSet") - return err - } - - return nil -} - -func ignoreDeletionPredicate() predicate.Predicate { - return predicate.Funcs{ - UpdateFunc: func(e event.UpdateEvent) bool { - generationChanged := e.ObjectOld.GetGeneration() != e.ObjectNew.GetGeneration() - annotationsChanged := !ydbannotations.CompareYdbTechAnnotations(e.ObjectOld.GetAnnotations(), e.ObjectNew.GetAnnotations()) - - return generationChanged || annotationsChanged - }, - DeleteFunc: func(e event.DeleteEvent) bool { - // Evaluates to false if the object has been confirmed deleted. - return !e.DeleteStateUnknown - }, - } -} - // SetupWithManager sets up the controller with the Manager. func (r *Reconciler) SetupWithManager(mgr ctrl.Manager, remoteCluster *cluster.Cluster) error { cluster := *remoteCluster - remoteStorageNodeSet := &v1alpha1.RemoteStorageNodeSet{} r.Recorder = mgr.GetEventRecorderFor(RemoteStorageNodeSetKind) r.RemoteRecorder = cluster.GetEventRecorderFor(RemoteStorageNodeSetKind) r.RemoteClient = cluster.GetClient() + annotationFilter := func(mapObj client.Object) []reconcile.Request { + requests := make([]reconcile.Request, 0) + + annotations := mapObj.GetAnnotations() + primaryResourceName, exist := annotations[ydbannotations.PrimaryResourceStorageAnnotation] + if exist { + storageNodeSets := &v1alpha1.StorageNodeSetList{} + if err := r.Client.List( + context.Background(), + storageNodeSets, + client.InNamespace(mapObj.GetNamespace()), + client.MatchingFields{ + StorageRefField: primaryResourceName, + }, + ); err != nil { + return requests + } + for _, storageNodeSet := range storageNodeSets.Items { + requests = append(requests, reconcile.Request{ + NamespacedName: types.NamespacedName{ + Namespace: storageNodeSet.GetNamespace(), + Name: storageNodeSet.GetName(), + }, + }) + } + } + return requests + } + isNodeSetFromMgmt, err := buildLocalSelector() if err != nil { return err } + if err := mgr.GetFieldIndexer().IndexField( + context.Background(), + &v1alpha1.StorageNodeSet{}, + StorageRefField, + func(obj client.Object) []string { + storageNodeSet := obj.(*v1alpha1.StorageNodeSet) + return []string{storageNodeSet.Spec.StorageRef.Name} + }); err != nil { + return err + } + return ctrl.NewControllerManagedBy(mgr). Named(RemoteStorageNodeSetKind). Watches( - source.NewKindWithCache(remoteStorageNodeSet, cluster.GetCache()), + source.NewKindWithCache(&v1alpha1.RemoteStorageNodeSet{}, cluster.GetCache()), &handler.EnqueueRequestForObject{}, - builder.WithPredicates(ignoreDeletionPredicate()), + builder.WithPredicates(predicate.Or( + predicate.GenerationChangedPredicate{}, + resources.LastAppliedAnnotationPredicate(), + )), ). Watches( &source.Kind{Type: &v1alpha1.StorageNodeSet{}}, @@ -173,6 +177,19 @@ func (r *Reconciler) SetupWithManager(mgr ctrl.Manager, remoteCluster *cluster.C resources.LabelExistsPredicate(isNodeSetFromMgmt), ), ). + Watches( + &source.Kind{Type: &corev1.Service{}}, + handler.EnqueueRequestsFromMapFunc(annotationFilter), + ). + Watches( + &source.Kind{Type: &corev1.Secret{}}, + handler.EnqueueRequestsFromMapFunc(annotationFilter), + ). + Watches( + &source.Kind{Type: &corev1.ConfigMap{}}, + handler.EnqueueRequestsFromMapFunc(annotationFilter), + ). + WithEventFilter(resources.IgnoreDeletetionPredicate()). Complete(r) } @@ -203,3 +220,36 @@ func BuildRemoteSelector(remoteCluster string) (labels.Selector, error) { labelRequirements = append(labelRequirements, *remoteClusterRequirement) return labels.NewSelector().Add(labelRequirements...), nil } + +func (r *Reconciler) deleteExternalResources( + ctx context.Context, + crRemoteStorageNodeSet *v1alpha1.RemoteStorageNodeSet, +) error { + logger := log.FromContext(ctx) + + storageNodeSet := &v1alpha1.StorageNodeSet{} + if err := r.Client.Get(ctx, types.NamespacedName{ + Name: crRemoteStorageNodeSet.Name, + Namespace: crRemoteStorageNodeSet.Namespace, + }, storageNodeSet); err != nil { + if apierrors.IsNotFound(err) { + logger.Info("StorageNodeSet not found") + } else { + logger.Error(err, "unable to get StorageNodeSet") + return err + } + } else { + if err := r.Client.Delete(ctx, storageNodeSet); err != nil { + logger.Error(err, "unable to delete StorageNodeSet") + return err + } + } + + remoteStorageNodeSet := resources.NewRemoteStorageNodeSet(crRemoteStorageNodeSet) + if _, _, err := r.removeUnusedRemoteObjects(ctx, &remoteStorageNodeSet, []client.Object{}); err != nil { + logger.Error(err, "unable to delete unused remote resources") + return err + } + + return nil +} diff --git a/internal/controllers/remotestoragenodeset/controller_test.go b/internal/controllers/remotestoragenodeset/controller_test.go index 3b52397d..87d94a83 100644 --- a/internal/controllers/remotestoragenodeset/controller_test.go +++ b/internal/controllers/remotestoragenodeset/controller_test.go @@ -22,22 +22,26 @@ import ( ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/cache" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/apiutil" "sigs.k8s.io/controller-runtime/pkg/cluster" "sigs.k8s.io/controller-runtime/pkg/envtest" logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/log/zap" - api "github.com/ydb-platform/ydb-kubernetes-operator/api/v1alpha1" + "github.com/ydb-platform/ydb-kubernetes-operator/api/v1alpha1" testobjects "github.com/ydb-platform/ydb-kubernetes-operator/e2e/tests/test-objects" + ydbannotations "github.com/ydb-platform/ydb-kubernetes-operator/internal/annotations" . "github.com/ydb-platform/ydb-kubernetes-operator/internal/controllers/constants" "github.com/ydb-platform/ydb-kubernetes-operator/internal/controllers/remotestoragenodeset" "github.com/ydb-platform/ydb-kubernetes-operator/internal/controllers/storage" "github.com/ydb-platform/ydb-kubernetes-operator/internal/controllers/storagenodeset" + "github.com/ydb-platform/ydb-kubernetes-operator/internal/resources" "github.com/ydb-platform/ydb-kubernetes-operator/internal/test" ) const ( testRemoteCluster = "remote-cluster" + testSecretName = "remote-secret" testNodeSetName = "nodeset" ) @@ -76,7 +80,7 @@ var _ = BeforeSuite(func() { ErrorIfCRDPathMissing: true, } - err := api.AddToScheme(scheme.Scheme) + err := v1alpha1.AddToScheme(scheme.Scheme) Expect(err).ToNot(HaveOccurred()) // +kubebuilder:scaffold:scheme @@ -107,7 +111,7 @@ var _ = BeforeSuite(func() { o.Scheme = scheme.Scheme o.NewCache = cache.BuilderWithOptions(cache.Options{ SelectorsByObject: cache.SelectorsByObject{ - &api.RemoteStorageNodeSet{}: {Label: storageSelector}, + &v1alpha1.RemoteStorageNodeSet{}: {Label: storageSelector}, }, }) }) @@ -120,6 +124,7 @@ var _ = BeforeSuite(func() { Client: localManager.GetClient(), Scheme: localManager.GetScheme(), Config: localManager.GetConfig(), + Log: logf.Log, }).SetupWithManager(localManager) Expect(err).ShouldNot(HaveOccurred()) @@ -127,6 +132,7 @@ var _ = BeforeSuite(func() { Client: localManager.GetClient(), Scheme: localManager.GetScheme(), Config: localManager.GetConfig(), + Log: logf.Log, }).SetupWithManager(localManager) Expect(err).ShouldNot(HaveOccurred()) @@ -134,12 +140,14 @@ var _ = BeforeSuite(func() { Client: remoteManager.GetClient(), Scheme: remoteManager.GetScheme(), Config: remoteManager.GetConfig(), + Log: logf.Log, }).SetupWithManager(remoteManager) Expect(err).ShouldNot(HaveOccurred()) err = (&remotestoragenodeset.Reconciler{ Client: remoteManager.GetClient(), Scheme: remoteManager.GetScheme(), + Log: logf.Log, }).SetupWithManager(remoteManager, &remoteCluster) Expect(err).ShouldNot(HaveOccurred()) @@ -173,22 +181,31 @@ var _ = AfterSuite(func() { var _ = Describe("RemoteStorageNodeSet controller tests", func() { var localNamespace corev1.Namespace var remoteNamespace corev1.Namespace - var storageSample *api.Storage + var storageSample *v1alpha1.Storage BeforeEach(func() { storageSample = testobjects.DefaultStorage(filepath.Join("..", "..", "..", "e2e", "tests", "data", "storage-block-4-2-config.yaml")) - storageSample.Spec.NodeSets = append(storageSample.Spec.NodeSets, api.StorageNodeSetSpecInline{ + storageSample.Spec.NodeSets = append(storageSample.Spec.NodeSets, v1alpha1.StorageNodeSetSpecInline{ Name: testNodeSetName + "-local", - StorageNodeSpec: api.StorageNodeSpec{ + StorageNodeSpec: v1alpha1.StorageNodeSpec{ Nodes: 4, }, }) - storageSample.Spec.NodeSets = append(storageSample.Spec.NodeSets, api.StorageNodeSetSpecInline{ + storageSample.Spec.NodeSets = append(storageSample.Spec.NodeSets, v1alpha1.StorageNodeSetSpecInline{ + Name: testNodeSetName + "-remote-static", + Remote: &v1alpha1.RemoteSpec{ + Cluster: testRemoteCluster, + }, + StorageNodeSpec: v1alpha1.StorageNodeSpec{ + Nodes: 8, + }, + }) + storageSample.Spec.NodeSets = append(storageSample.Spec.NodeSets, v1alpha1.StorageNodeSetSpecInline{ Name: testNodeSetName + "-remote", - Remote: &api.RemoteSpec{ + Remote: &v1alpha1.RemoteSpec{ Cluster: testRemoteCluster, }, - StorageNodeSpec: api.StorageNodeSpec{ + StorageNodeSpec: v1alpha1.StorageNodeSpec{ Nodes: 4, }, }) @@ -209,66 +226,61 @@ var _ = Describe("RemoteStorageNodeSet controller tests", func() { By("issuing create Storage commands...") Expect(localClient.Create(ctx, storageSample)).Should(Succeed()) + By("checking that Storage created on local cluster...") - foundStorage := api.Storage{} Eventually(func() bool { + foundStorage := v1alpha1.Storage{} Expect(localClient.Get(ctx, types.NamespacedName{ Name: storageSample.Name, Namespace: testobjects.YdbNamespace, }, &foundStorage)) return foundStorage.Status.State == StorageProvisioning }, test.Timeout, test.Interval).Should(BeTrue()) - By("set status Ready to Storage...") - foundStorage.Status.State = StorageReady - Expect(localClient.Status().Update(ctx, &foundStorage)).Should(Succeed()) By("checking that StorageNodeSet created on local cluster...") - Eventually(func() bool { - foundStorageNodeSet := api.StorageNodeSetList{} - - Expect(localClient.List(ctx, &foundStorageNodeSet, client.InNamespace( - testobjects.YdbNamespace, - ))).Should(Succeed()) - - for _, nodeset := range foundStorageNodeSet.Items { - if nodeset.Name == storageSample.Name+"-"+testNodeSetName+"-local" { - return true - } - } - return false - }, test.Timeout, test.Interval).Should(BeTrue()) + Eventually(func() error { + foundStorageNodeSet := &v1alpha1.StorageNodeSet{} + return localClient.Get(ctx, types.NamespacedName{ + Name: storageSample.Name + "-" + testNodeSetName + "-local", + Namespace: testobjects.YdbNamespace, + }, foundStorageNodeSet) + }, test.Timeout, test.Interval).ShouldNot(HaveOccurred()) By("checking that RemoteStorageNodeSet created on local cluster...") - Eventually(func() bool { - foundRemoteStorageNodeSet := api.RemoteStorageNodeSetList{} - - Expect(localClient.List(ctx, &foundRemoteStorageNodeSet, client.InNamespace( - testobjects.YdbNamespace, - ))).Should(Succeed()) - - for _, nodeset := range foundRemoteStorageNodeSet.Items { - if nodeset.Name == storageSample.Name+"-"+testNodeSetName+"-remote" { - return true - } - } - return false - }, test.Timeout, test.Interval).Should(BeTrue()) + Eventually(func() error { + foundRemoteStorageNodeSet := &v1alpha1.RemoteStorageNodeSet{} + return localClient.Get(ctx, types.NamespacedName{ + Name: storageSample.Name + "-" + testNodeSetName + "-remote", + Namespace: testobjects.YdbNamespace, + }, foundRemoteStorageNodeSet) + }, test.Timeout, test.Interval).ShouldNot(HaveOccurred()) + + By("checking that static RemoteStorageNodeSet created on local cluster...") + Eventually(func() error { + foundStaticRemoteStorageNodeSet := &v1alpha1.RemoteStorageNodeSet{} + return localClient.Get(ctx, types.NamespacedName{ + Name: storageSample.Name + "-" + testNodeSetName + "-remote-static", + Namespace: testobjects.YdbNamespace, + }, foundStaticRemoteStorageNodeSet) + }, test.Timeout, test.Interval).ShouldNot(HaveOccurred()) By("checking that StorageNodeSet created on remote cluster...") - Eventually(func() bool { - foundStorageNodeSetOnRemote := api.StorageNodeSetList{} - - Expect(remoteClient.List(ctx, &foundStorageNodeSetOnRemote, client.InNamespace( - testobjects.YdbNamespace, - ))).Should(Succeed()) - - for _, nodeset := range foundStorageNodeSetOnRemote.Items { - if nodeset.Name == storageSample.Name+"-"+testNodeSetName+"-remote" { - return true - } - } - return false - }, test.Timeout, test.Interval).Should(BeTrue()) + Eventually(func() error { + foundStorageNodeSetOnRemote := &v1alpha1.StorageNodeSet{} + return remoteClient.Get(ctx, types.NamespacedName{ + Name: storageSample.Name + "-" + testNodeSetName + "-remote", + Namespace: testobjects.YdbNamespace, + }, foundStorageNodeSetOnRemote) + }, test.Timeout, test.Interval).ShouldNot(HaveOccurred()) + + By("checking that static StorageNodeSet created on remote cluster...") + Eventually(func() error { + foundStaticStorageNodeSetOnRemote := &v1alpha1.StorageNodeSet{} + return remoteClient.Get(ctx, types.NamespacedName{ + Name: storageSample.Name + "-" + testNodeSetName + "-remote-static", + Namespace: testobjects.YdbNamespace, + }, foundStaticStorageNodeSetOnRemote) + }, test.Timeout, test.Interval).ShouldNot(HaveOccurred()) }) AfterEach(func() { @@ -276,41 +288,257 @@ var _ = Describe("RemoteStorageNodeSet controller tests", func() { deleteAll(remoteEnv, remoteClient, &localNamespace) }) - When("Create Storage with RemoteStorageNodeSet in k8s-mgmt-cluster", func() { - It("Should create StorageNodeSet and sync resources in k8s-data-cluster", func() { + When("Created RemoteStorageNodeSet in k8s-mgmt-cluster", func() { + It("Should receive status from k8s-data-cluster", func() { + By("set static StorageNodeSet status to Ready on remote cluster...") + Eventually(func() error { + foundStaticStorageNodeSetOnRemote := &v1alpha1.StorageNodeSet{} + Expect(remoteClient.Get(ctx, types.NamespacedName{ + Name: storageSample.Name + "-" + testNodeSetName + "-remote-static", + Namespace: testobjects.YdbNamespace, + }, foundStaticStorageNodeSetOnRemote)).Should(Succeed()) + foundStaticStorageNodeSetOnRemote.Status.State = StorageNodeSetReady + foundStaticStorageNodeSetOnRemote.Status.Conditions = append( + foundStaticStorageNodeSetOnRemote.Status.Conditions, + metav1.Condition{ + Type: StorageNodeSetReadyCondition, + Status: "True", + Reason: ReasonCompleted, + LastTransitionTime: metav1.NewTime(time.Now()), + Message: fmt.Sprintf("Scaled StorageNodeSet to %d successfully", foundStaticStorageNodeSetOnRemote.Spec.Nodes), + }, + ) + return remoteClient.Status().Update(ctx, foundStaticStorageNodeSetOnRemote) + }, test.Timeout, test.Interval).ShouldNot(HaveOccurred()) + By("set StorageNodeSet status to Ready on remote cluster...") - foundStorageNodeSetOnRemote := api.StorageNodeSet{} - Expect(remoteClient.Get(ctx, types.NamespacedName{ - Name: storageSample.Name + "-" + testNodeSetName + "-remote", - Namespace: testobjects.YdbNamespace, - }, &foundStorageNodeSetOnRemote)).Should(Succeed()) - - foundStorageNodeSetOnRemote.Status.State = StorageNodeSetReady - foundStorageNodeSetOnRemote.Status.Conditions = append( - foundStorageNodeSetOnRemote.Status.Conditions, - metav1.Condition{ - Type: StorageNodeSetReadyCondition, - Status: "True", - Reason: ReasonCompleted, - LastTransitionTime: metav1.NewTime(time.Now()), - Message: fmt.Sprintf("Scaled StorageNodeSet to %d successfully", foundStorageNodeSetOnRemote.Spec.Nodes), - }, - ) - Expect(remoteClient.Status().Update(ctx, &foundStorageNodeSetOnRemote)).Should(Succeed()) + Eventually(func() error { + foundStorageNodeSetOnRemote := &v1alpha1.StorageNodeSet{} + Expect(remoteClient.Get(ctx, types.NamespacedName{ + Name: storageSample.Name + "-" + testNodeSetName + "-remote", + Namespace: testobjects.YdbNamespace, + }, foundStorageNodeSetOnRemote)).Should(Succeed()) + foundStorageNodeSetOnRemote.Status.State = StorageNodeSetReady + foundStorageNodeSetOnRemote.Status.Conditions = append( + foundStorageNodeSetOnRemote.Status.Conditions, + metav1.Condition{ + Type: StorageNodeSetReadyCondition, + Status: "True", + Reason: ReasonCompleted, + LastTransitionTime: metav1.NewTime(time.Now()), + Message: fmt.Sprintf("Scaled StorageNodeSet to %d successfully", foundStorageNodeSetOnRemote.Spec.Nodes), + }, + ) + return remoteClient.Status().Update(ctx, foundStorageNodeSetOnRemote) + }, test.Timeout, test.Interval).ShouldNot(HaveOccurred()) + + By("checking that static RemoteStorageNodeSet status updated on local cluster...") + Eventually(func() bool { + foundStaticRemoteStorageNodeSet := v1alpha1.RemoteStorageNodeSet{} + Expect(localClient.Get(ctx, types.NamespacedName{ + Name: storageSample.Name + "-" + testNodeSetName + "-remote-static", + Namespace: testobjects.YdbNamespace, + }, &foundStaticRemoteStorageNodeSet)).Should(Succeed()) + + return meta.IsStatusConditionPresentAndEqual( + foundStaticRemoteStorageNodeSet.Status.Conditions, + StorageNodeSetReadyCondition, + metav1.ConditionTrue, + ) && foundStaticRemoteStorageNodeSet.Status.State == StorageNodeSetReady + }, test.Timeout, test.Interval).Should(BeTrue()) By("checking that RemoteStorageNodeSet status updated on local cluster...") Eventually(func() bool { - foundRemoteStorageNodeSetOnRemote := api.RemoteStorageNodeSet{} + foundRemoteStorageNodeSet := v1alpha1.RemoteStorageNodeSet{} Expect(localClient.Get(ctx, types.NamespacedName{ Name: storageSample.Name + "-" + testNodeSetName + "-remote", Namespace: testobjects.YdbNamespace, - }, &foundRemoteStorageNodeSetOnRemote)).Should(Succeed()) + }, &foundRemoteStorageNodeSet)).Should(Succeed()) return meta.IsStatusConditionPresentAndEqual( - foundRemoteStorageNodeSetOnRemote.Status.Conditions, + foundRemoteStorageNodeSet.Status.Conditions, StorageNodeSetReadyCondition, metav1.ConditionTrue, - ) && foundRemoteStorageNodeSetOnRemote.Status.State == StorageNodeSetReady + ) && foundRemoteStorageNodeSet.Status.State == StorageNodeSetReady + }, test.Timeout, test.Interval).Should(BeTrue()) + }) + }) + When("Created RemoteStorageNodeSet with Secrets in k8s-mgmt-cluster", func() { + It("Should sync Secrets into k8s-data-cluster", func() { + By("create simple Secret in Storage namespace") + simpleSecret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: testSecretName, + Namespace: testobjects.YdbNamespace, + }, + StringData: map[string]string{ + "message": "Hello from k8s-mgmt-cluster", + }, + } + Expect(localClient.Create(ctx, simpleSecret)) + + By("checking that Storage updated on local cluster...") + Eventually(func() error { + foundStorage := &v1alpha1.Storage{} + Expect(localClient.Get(ctx, types.NamespacedName{ + Name: storageSample.Name, + Namespace: testobjects.YdbNamespace, + }, foundStorage)) + + foundStorage.Spec.Secrets = append( + foundStorage.Spec.Secrets, + &corev1.LocalObjectReference{ + Name: testSecretName, + }, + ) + return localClient.Update(ctx, foundStorage) + }, test.Timeout, test.Interval).ShouldNot(HaveOccurred()) + + By("checking that Secrets are synced...") + Eventually(func() error { + foundRemoteStorageNodeSet := &v1alpha1.RemoteStorageNodeSet{} + Expect(localClient.Get(ctx, types.NamespacedName{ + Name: storageSample.Name + "-" + testNodeSetName + "-remote", + Namespace: testobjects.YdbNamespace, + }, foundRemoteStorageNodeSet)).Should(Succeed()) + + localSecret := &corev1.Secret{} + err := localClient.Get(ctx, types.NamespacedName{ + Name: testSecretName, + Namespace: testobjects.YdbNamespace, + }, localSecret) + if err != nil { + return err + } + + remoteSecret := &corev1.Secret{} + err = remoteClient.Get(ctx, types.NamespacedName{ + Name: testSecretName, + Namespace: testobjects.YdbNamespace, + }, remoteSecret) + if err != nil { + return err + } + + primaryResourceName, exist := remoteSecret.Annotations[ydbannotations.PrimaryResourceStorageAnnotation] + if !exist { + return fmt.Errorf("annotation %s does not exist on remoteSecret %s", ydbannotations.PrimaryResourceStorageAnnotation, remoteSecret.Name) + } + if primaryResourceName != foundRemoteStorageNodeSet.Spec.StorageRef.Name { + return fmt.Errorf("primaryResourceName %s does not equal storageRef name %s", primaryResourceName, foundRemoteStorageNodeSet.Spec.StorageRef.Name) + } + + remoteRV, exist := remoteSecret.Annotations[ydbannotations.RemoteResourceVersionAnnotation] + if !exist { + return fmt.Errorf("annotation %s does not exist on remoteSecret %s", ydbannotations.RemoteResourceVersionAnnotation, remoteSecret.Name) + } + if localSecret.GetResourceVersion() != remoteRV { + return fmt.Errorf("localRV %s does not equal remoteRV %s", localSecret.GetResourceVersion(), remoteRV) + } + + return nil + }, test.Timeout, test.Interval).ShouldNot(HaveOccurred()) + }) + }) + When("Created RemoteStorageNodeSet with Services in k8s-mgmt-cluster", func() { + It("Should sync Services into k8s-data-cluster", func() { + By("checking that Services are synced...") + Eventually(func() error { + foundRemoteStorageNodeSet := &v1alpha1.RemoteStorageNodeSet{} + Expect(localClient.Get(ctx, types.NamespacedName{ + Name: storageSample.Name + "-" + testNodeSetName + "-remote", + Namespace: testobjects.YdbNamespace, + }, foundRemoteStorageNodeSet)).Should(Succeed()) + + foundServices := corev1.ServiceList{} + Expect(localClient.List(ctx, &foundServices, client.InNamespace( + testobjects.YdbNamespace, + ))).Should(Succeed()) + for _, localService := range foundServices.Items { + remoteService := &corev1.Service{} + err := remoteClient.Get(ctx, types.NamespacedName{ + Name: localService.Name, + Namespace: localService.Namespace, + }, remoteService) + if err != nil { + return err + } + } + return nil + }, test.Timeout, test.Interval).ShouldNot(HaveOccurred()) + + By("checking that static RemoteStorageNodeSet RemoteStatus are updated...") + Eventually(func() bool { + foundRemoteStorageNodeSet := &v1alpha1.RemoteStorageNodeSet{} + Expect(localClient.Get(ctx, types.NamespacedName{ + Name: storageSample.Name + "-" + testNodeSetName + "-remote-static", + Namespace: testobjects.YdbNamespace, + }, foundRemoteStorageNodeSet)).Should(Succeed()) + + foundConfigMap := corev1.ConfigMap{} + Expect(remoteClient.Get(ctx, types.NamespacedName{ + Name: storageSample.Name, + Namespace: testobjects.YdbNamespace, + }, &foundConfigMap)).Should(Succeed()) + + gvk, err := apiutil.GVKForObject(foundConfigMap.DeepCopy(), scheme.Scheme) + Expect(err).ShouldNot(HaveOccurred()) + + for idx := range foundRemoteStorageNodeSet.Status.RemoteResources { + remoteResource := foundRemoteStorageNodeSet.Status.RemoteResources[idx] + if resources.EqualRemoteResourceWithObject( + &remoteResource, + testobjects.YdbNamespace, + foundConfigMap.DeepCopy(), + gvk, + ) { + if meta.IsStatusConditionPresentAndEqual( + remoteResource.Conditions, + RemoteResourceSyncedCondition, + metav1.ConditionTrue, + ) { + return true + } + } + } + return false + }, test.Timeout, test.Interval).Should(BeTrue()) + + By("checking that RemoteStorageNodeSet RemoteStatus are updated...") + Eventually(func() bool { + foundRemoteStorageNodeSet := &v1alpha1.RemoteStorageNodeSet{} + Expect(localClient.Get(ctx, types.NamespacedName{ + Name: storageSample.Name + "-" + testNodeSetName + "-remote", + Namespace: testobjects.YdbNamespace, + }, foundRemoteStorageNodeSet)).Should(Succeed()) + + foundConfigMap := corev1.ConfigMap{} + Expect(remoteClient.Get(ctx, types.NamespacedName{ + Name: storageSample.Name, + Namespace: testobjects.YdbNamespace, + }, &foundConfigMap)).Should(Succeed()) + + gvk, err := apiutil.GVKForObject(foundConfigMap.DeepCopy(), scheme.Scheme) + Expect(err).ShouldNot(HaveOccurred()) + + for idx := range foundRemoteStorageNodeSet.Status.RemoteResources { + remoteResource := foundRemoteStorageNodeSet.Status.RemoteResources[idx] + if resources.EqualRemoteResourceWithObject( + &remoteResource, + testobjects.YdbNamespace, + foundConfigMap.DeepCopy(), + gvk, + ) { + if meta.IsStatusConditionPresentAndEqual( + remoteResource.Conditions, + RemoteResourceSyncedCondition, + metav1.ConditionTrue, + ) { + return true + } + } + } + return false }, test.Timeout, test.Interval).Should(BeTrue()) }) }) @@ -318,25 +546,34 @@ var _ = Describe("RemoteStorageNodeSet controller tests", func() { It("Should delete all resources in k8s-data-cluster", func() { By("delete RemoteStorageNodeSet on local cluster...") Eventually(func() error { - foundStorage := api.Storage{} + foundStorage := v1alpha1.Storage{} Expect(localClient.Get(ctx, types.NamespacedName{ Name: storageSample.Name, Namespace: testobjects.YdbNamespace, }, &foundStorage)).Should(Succeed()) - foundStorage.Spec.NodeSets = []api.StorageNodeSetSpecInline{ + foundStorage.Spec.NodeSets = []v1alpha1.StorageNodeSetSpecInline{ { Name: testNodeSetName + "-local", - StorageNodeSpec: api.StorageNodeSpec{ + StorageNodeSpec: v1alpha1.StorageNodeSpec{ Nodes: 4, }, }, + { + Name: testNodeSetName + "-remote-static", + Remote: &v1alpha1.RemoteSpec{ + Cluster: testRemoteCluster, + }, + StorageNodeSpec: v1alpha1.StorageNodeSpec{ + Nodes: 8, + }, + }, } return localClient.Update(ctx, &foundStorage) }, test.Timeout, test.Interval).Should(Succeed()) By("checking that StorageNodeSet deleted from remote cluster...") Eventually(func() bool { - foundStorageNodeSetOnRemote := api.StorageNodeSet{} + foundStorageNodeSetOnRemote := v1alpha1.StorageNodeSet{} err := remoteClient.Get(ctx, types.NamespacedName{ Name: storageSample.Name + "-" + testNodeSetName + "-remote", @@ -348,7 +585,7 @@ var _ = Describe("RemoteStorageNodeSet controller tests", func() { By("checking that RemoteStorageNodeSet deleted from local cluster...") Eventually(func() bool { - foundRemoteStorageNodeSet := api.RemoteStorageNodeSet{} + foundRemoteStorageNodeSet := v1alpha1.RemoteStorageNodeSet{} err := localClient.Get(ctx, types.NamespacedName{ Name: storageSample.Name + "-" + testNodeSetName + "-remote", @@ -357,6 +594,30 @@ var _ = Describe("RemoteStorageNodeSet controller tests", func() { return apierrors.IsNotFound(err) }, test.Timeout, test.Interval).Should(BeTrue()) + + By("checking that Services for static RemoteStorageNodeSet exisiting in remote cluster...") + Eventually(func() error { + foundStaticStorageNodeSetOnRemote := &v1alpha1.StorageNodeSet{} + Expect(remoteClient.Get(ctx, types.NamespacedName{ + Name: storageSample.Name + "-" + testNodeSetName + "-remote-static", + Namespace: testobjects.YdbNamespace, + }, foundStaticStorageNodeSetOnRemote)).Should(Succeed()) + + storageServices := corev1.ServiceList{} + Expect(localClient.List(ctx, &storageServices, + client.InNamespace(testobjects.YdbNamespace), + )).Should(Succeed()) + for _, storageService := range storageServices.Items { + remoteService := &corev1.Service{} + if err := remoteClient.Get(ctx, types.NamespacedName{ + Name: storageService.GetName(), + Namespace: storageService.GetNamespace(), + }, remoteService); err != nil { + return err + } + } + return nil + }, test.Timeout, test.Interval).ShouldNot(HaveOccurred()) }) }) }) diff --git a/internal/controllers/remotestoragenodeset/remote_objects.go b/internal/controllers/remotestoragenodeset/remote_objects.go new file mode 100644 index 00000000..4fae4866 --- /dev/null +++ b/internal/controllers/remotestoragenodeset/remote_objects.go @@ -0,0 +1,508 @@ +package remotestoragenodeset + +import ( + "context" + "fmt" + "reflect" + + "github.com/banzaicloud/k8s-objectmatcher/patch" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/selection" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/apiutil" + + "github.com/ydb-platform/ydb-kubernetes-operator/api/v1alpha1" + ydbannotations "github.com/ydb-platform/ydb-kubernetes-operator/internal/annotations" + . "github.com/ydb-platform/ydb-kubernetes-operator/internal/controllers/constants" //nolint:revive,stylecheck + ydblabels "github.com/ydb-platform/ydb-kubernetes-operator/internal/labels" + "github.com/ydb-platform/ydb-kubernetes-operator/internal/resources" +) + +var ( + annotator = patch.NewAnnotator(ydbannotations.LastAppliedAnnotation) + patchMaker = patch.NewPatchMaker(annotator) +) + +func (r *Reconciler) initRemoteResourcesStatus( + ctx context.Context, + remoteStorageNodeSet *resources.RemoteStorageNodeSetResource, + remoteObjects []client.Object, +) (bool, ctrl.Result, error) { + r.Log.Info("running step initRemoteResourcesStatus") + syncedResources := []v1alpha1.RemoteResource{} + // copy actual slice to local variable + if remoteStorageNodeSet.Status.RemoteResources != nil { + syncedResources = append(syncedResources, remoteStorageNodeSet.Status.RemoteResources...) + } + + for idx := range remoteObjects { + remoteObj := remoteObjects[idx] + remoteObjGVK, err := apiutil.GVKForObject(remoteObj, r.Scheme) + if err != nil { + r.Recorder.Event( + remoteStorageNodeSet, + corev1.EventTypeWarning, + "ControllerError", + fmt.Sprintf("Failed to recognize GVK for remote object %s with name %s: %s", remoteObjGVK.Kind, remoteObj.GetName(), err), + ) + return Stop, ctrl.Result{RequeueAfter: DefaultRequeueDelay}, err + } + + existInStatus := false + for i := range syncedResources { + remoteResource := syncedResources[i] + if resources.EqualRemoteResourceWithObject( + &remoteResource, + remoteStorageNodeSet.Namespace, + remoteObj, + remoteObjGVK, + ) { + existInStatus = true + break + } + } + + if !existInStatus { + remoteStorageNodeSet.Status.RemoteResources = append( + remoteStorageNodeSet.Status.RemoteResources, + v1alpha1.RemoteResource{ + Group: remoteObjGVK.Group, + Version: remoteObjGVK.Version, + Kind: remoteObjGVK.Kind, + Name: remoteObj.GetName(), + State: ResourceSyncPending, + Conditions: []metav1.Condition{}, + }, + ) + } + } + + return r.updateRemoteResourcesStatus(ctx, remoteStorageNodeSet) +} + +func (r *Reconciler) syncRemoteObjects( + ctx context.Context, + remoteStorageNodeSet *resources.RemoteStorageNodeSetResource, + remoteObjects []client.Object, +) (bool, ctrl.Result, error) { + r.Log.Info("running step syncRemoteObjects") + + for _, remoteObj := range remoteObjects { + // Determine actual GVK for generic client.Object + remoteObjGVK, err := apiutil.GVKForObject(remoteObj, r.Scheme) + if err != nil { + r.Recorder.Event( + remoteStorageNodeSet, + corev1.EventTypeWarning, + "ControllerError", + fmt.Sprintf("Failed to recognize GVK for remote object %s with name %s: %s", remoteObjGVK.Kind, remoteObj.GetName(), err), + ) + return Stop, ctrl.Result{RequeueAfter: DefaultRequeueDelay}, err + } + + // Get object to sync from remote cluster + err = r.RemoteClient.Get(ctx, types.NamespacedName{ + Name: remoteObj.GetName(), + Namespace: remoteObj.GetNamespace(), + }, remoteObj) + if err != nil { + // Resource not found on remote cluster but we should retry + if apierrors.IsNotFound(err) { + r.Recorder.Event( + remoteStorageNodeSet, + corev1.EventTypeWarning, + "ProvisioningFailed", + fmt.Sprintf("Resource %s with name %s was not found on remote cluster: %s", remoteObjGVK.Kind, remoteObj.GetName(), err), + ) + r.RemoteRecorder.Event( + remoteStorageNodeSet, + corev1.EventTypeWarning, + "ProvisioningFailed", + fmt.Sprintf("Resource %s with name %s was not found: %s", remoteObjGVK.Kind, remoteObj.GetName(), err), + ) + } else { + r.Recorder.Event( + remoteStorageNodeSet, + corev1.EventTypeWarning, + "ControllerError", + fmt.Sprintf("Failed to get resource %s with name %s on remote cluster: %s", remoteObjGVK.Kind, remoteObj.GetName(), err), + ) + r.RemoteRecorder.Event( + remoteStorageNodeSet, + corev1.EventTypeWarning, + "ControllerError", + fmt.Sprintf("Failed to get resource %s with name %s: %s", remoteObjGVK.Kind, remoteObj.GetName(), err), + ) + } + return Stop, ctrl.Result{RequeueAfter: DefaultRequeueDelay}, err + } + + // Create client.Object from api.RemoteResource struct + localObj := resources.CreateResource(remoteObj) + remoteStorageNodeSet.SetPrimaryResourceAnnotations(localObj) + // Check object existence in local cluster + err = r.Client.Get(ctx, types.NamespacedName{ + Name: remoteObj.GetName(), + Namespace: remoteObj.GetNamespace(), + }, localObj) + //nolint:nestif + if err != nil { + if !apierrors.IsNotFound(err) { + r.Recorder.Event( + remoteStorageNodeSet, + corev1.EventTypeWarning, + "ControllerError", + fmt.Sprintf("Failed to get resource %s with name %s: %s", remoteObjGVK.Kind, remoteObj.GetName(), err), + ) + return Stop, ctrl.Result{RequeueAfter: DefaultRequeueDelay}, err + } + // Object does not exist in local cluster + // Try to create resource in remote cluster + if err := r.Client.Create(ctx, localObj); err != nil { + r.Recorder.Event( + remoteStorageNodeSet, + corev1.EventTypeWarning, + "ControllerError", + fmt.Sprintf("Failed to create resource %s with name %s: %s", remoteObjGVK.Kind, remoteObj.GetName(), err), + ) + return Stop, ctrl.Result{RequeueAfter: DefaultRequeueDelay}, nil + } + r.Recorder.Event( + remoteStorageNodeSet, + corev1.EventTypeNormal, + "Provisioning", + fmt.Sprintf("RemoteSync CREATE resource %s with name %s", remoteObjGVK.Kind, remoteObj.GetName()), + ) + } else { + // Update client.Object for local object with spec from remote object + updatedObj := resources.UpdateResource(localObj, remoteObj) + remoteStorageNodeSet.SetPrimaryResourceAnnotations(updatedObj) + // Remote object existing in local cluster, сheck the need for an update + // Get diff resources and compare bytes by k8s-objectmatcher PatchMaker + patchResult, err := patchMaker.Calculate(localObj, updatedObj, + []patch.CalculateOption{ + patch.IgnoreStatusFields(), + }..., + ) + if err != nil { + r.Recorder.Event( + remoteStorageNodeSet, + corev1.EventTypeWarning, + "ControllerError", + fmt.Sprintf("Failed to get diff for remote resource %s with name %s: %s", remoteObjGVK.Kind, remoteObj.GetName(), err), + ) + return Stop, ctrl.Result{RequeueAfter: DefaultRequeueDelay}, nil + } + // We need to check patchResult by k8s-objectmatcherand and resourceVersion from annotation + // And update if localObj does not match updatedObj from remote cluster + if !patchResult.IsEmpty() || + remoteObj.GetResourceVersion() != localObj.GetAnnotations()[ydbannotations.RemoteResourceVersionAnnotation] { + r.Recorder.Event( + remoteStorageNodeSet, + corev1.EventTypeNormal, + "Provisioning", + fmt.Sprintf("Patch for resource %s with name %s: %s", remoteObjGVK.Kind, remoteObj.GetName(), string(patchResult.Patch)), + ) + // Try to update resource in local cluster + if err := r.Client.Update(ctx, updatedObj); err != nil { + r.Recorder.Event( + remoteStorageNodeSet, + corev1.EventTypeWarning, + "ControllerError", + fmt.Sprintf("Failed to update resource %s with name %s: %s", remoteObjGVK.Kind, remoteObj.GetName(), err), + ) + return Stop, ctrl.Result{RequeueAfter: DefaultRequeueDelay}, nil + } + r.Recorder.Event( + remoteStorageNodeSet, + corev1.EventTypeNormal, + "Provisioning", + fmt.Sprintf("RemoteSync UPDATE resource %s with name %s resourceVersion %s", remoteObjGVK.Kind, remoteObj.GetName(), remoteObj.GetResourceVersion()), + ) + } + } + // Update status for remote resource in RemoteStorageNodeSet object + remoteStorageNodeSet.SetRemoteResourceStatus(localObj, remoteObjGVK) + } + + return r.updateRemoteResourcesStatus(ctx, remoteStorageNodeSet) +} + +func (r *Reconciler) removeUnusedRemoteObjects( + ctx context.Context, + remoteStorageNodeSet *resources.RemoteStorageNodeSetResource, + remoteObjects []client.Object, +) (bool, ctrl.Result, error) { + r.Log.Info("running step removeUnusedRemoteObjects") + // We should check every remote resource to need existence in cluster + // Get processed remote resources from object Status + candidatesToDelete := []v1alpha1.RemoteResource{} + + // Remove remote resource from candidates to delete if it declared + // to using in current RemoteStorageNodeSet spec + for idx := range remoteStorageNodeSet.Status.RemoteResources { + remoteResource := remoteStorageNodeSet.Status.RemoteResources[idx] + existInSpec := false + for i := range remoteObjects { + declaredObj := remoteObjects[i] + declaredObjGVK, err := apiutil.GVKForObject(declaredObj, r.Scheme) + if err != nil { + return Stop, ctrl.Result{RequeueAfter: DefaultRequeueDelay}, err + } + if resources.EqualRemoteResourceWithObject( + &remoteResource, + remoteStorageNodeSet.Namespace, + declaredObj, + declaredObjGVK, + ) { + existInSpec = true + break + } + } + if !existInSpec { + candidatesToDelete = append(candidatesToDelete, remoteResource) + } + } + + // Check resources usage in another StorageNodeSet and make List request + // only if we have candidates to Delete + resourcesToDelete := []v1alpha1.RemoteResource{} + if len(candidatesToDelete) > 0 { + remoteObjectsUsed, err := r.getRemoteObjectsUsedInNamespace(ctx, remoteStorageNodeSet, remoteObjects) + if err != nil { + r.Recorder.Event( + remoteStorageNodeSet, + corev1.EventTypeWarning, + "ProvisioningFailed", + fmt.Sprintf("Failed to get resources used in another object: %s", err), + ) + return Stop, ctrl.Result{RequeueAfter: DefaultRequeueDelay}, err + } + for idx := range candidatesToDelete { + remoteResource := candidatesToDelete[idx] + isCandidateExistInANotherObject := false + // Remove resource from cadidates to Delete if another object using it now + for i := range remoteObjectsUsed { + usedObj := remoteObjectsUsed[i] + usedObjGVK, err := apiutil.GVKForObject(usedObj, r.Scheme) + if err != nil { + return Stop, ctrl.Result{RequeueAfter: DefaultRequeueDelay}, err + } + if resources.EqualRemoteResourceWithObject( + &remoteResource, + remoteStorageNodeSet.Namespace, + usedObj, + usedObjGVK, + ) { + isCandidateExistInANotherObject = true + break + } + } + if !isCandidateExistInANotherObject { + resourcesToDelete = append(resourcesToDelete, remoteResource) + } + } + } + + // Remove unused remote resource from cluster and make API call DELETE + // for every candidate to Delete + for _, recourceToDelete := range resourcesToDelete { + // Convert RemoteResource struct from Status to client.Object + remoteObj, err := resources.ConvertRemoteResourceToObject( + recourceToDelete, + remoteStorageNodeSet.Namespace, + ) + if err != nil { + return Stop, ctrl.Result{RequeueAfter: DefaultRequeueDelay}, err + } + + // Determine actual GVK for generic client.Object + remoteResourceGVK, err := apiutil.GVKForObject(remoteObj, r.Scheme) + if err != nil { + r.Recorder.Event( + remoteStorageNodeSet, + corev1.EventTypeWarning, + "ControllerError", + fmt.Sprintf("Failed to recognize GVK for remote object %v: %s", remoteObj, err), + ) + return Stop, ctrl.Result{RequeueAfter: DefaultRequeueDelay}, err + } + + // Try to get resource in local cluster + if err := r.Client.Get(ctx, types.NamespacedName{ + Name: remoteObj.GetName(), + Namespace: remoteObj.GetNamespace(), + }, remoteObj); err != nil { + if !apierrors.IsNotFound(err) { + r.Recorder.Event( + remoteStorageNodeSet, + corev1.EventTypeWarning, + "ControllerError", + fmt.Sprintf("Failed to get resource %s with name %s: %s", remoteResourceGVK.Kind, remoteObj.GetName(), err), + ) + return Stop, ctrl.Result{RequeueAfter: DefaultRequeueDelay}, err + } + } + + // Skip resource deletion because it using in some Database + // check by existence of annotation `ydb.tech/primary-resource-database` + if _, exist := remoteObj.GetAnnotations()[ydbannotations.PrimaryResourceDatabaseAnnotation]; exist { + continue + } + + // Try to delete unused resource from local cluster + if err := r.Client.Delete(ctx, remoteObj); err != nil { + if !apierrors.IsNotFound(err) { + r.Recorder.Event( + remoteStorageNodeSet, + corev1.EventTypeWarning, + "ControllerError", + fmt.Sprintf("Failed to delete resource %s with name %s: %s", remoteResourceGVK.Kind, remoteObj.GetName(), err), + ) + return Stop, ctrl.Result{RequeueAfter: DefaultRequeueDelay}, err + } + } + r.Recorder.Event( + remoteStorageNodeSet, + corev1.EventTypeNormal, + "Provisioning", + fmt.Sprintf("RemoteSync DELETE resource %s with name %s", remoteResourceGVK.Kind, remoteObj.GetName()), + ) + // Remove status for remote resource from RemoteStorageNodeSet object + remoteStorageNodeSet.RemoveRemoteResourceStatus(remoteObj, remoteResourceGVK) + } + + return r.updateRemoteResourcesStatus(ctx, remoteStorageNodeSet) +} + +func (r *Reconciler) getRemoteObjectsUsedInNamespace( + ctx context.Context, + remoteStorageNodeSet *resources.RemoteStorageNodeSetResource, + remoteObjects []client.Object, +) ([]client.Object, error) { + remoteObjectsUsedInNamespace := []client.Object{} + + // Create label requirement that label `ydb.tech/storage-nodeset` which not equal + // to current StorageNodeSet object for exclude current nodeSet from List result + labelRequirement, err := labels.NewRequirement( + ydblabels.StorageNodeSetComponent, + selection.NotEquals, + []string{remoteStorageNodeSet.Labels[ydblabels.StorageNodeSetComponent]}, + ) + if err != nil { + return nil, err + } + + // Search another StorageNodeSets in current namespace with the same StorageRef + storageNodeSets := &v1alpha1.StorageNodeSetList{} + if err := r.Client.List( + ctx, + storageNodeSets, + client.InNamespace(remoteStorageNodeSet.Namespace), + client.MatchingLabelsSelector{ + Selector: labels.NewSelector().Add(*labelRequirement), + }, + client.MatchingFields{ + StorageRefField: remoteStorageNodeSet.Spec.StorageRef.Name, + }, + ); err != nil { + return nil, err + } + + // We found some StorageNodeSet and should check objects usage + if len(storageNodeSets.Items) > 0 { + for _, remoteObj := range remoteObjects { + switch obj := remoteObj.(type) { + // If client.Object typed by Secret search existence + // in another StorageNodeSet spec.secrets + case *corev1.Secret: + for _, storageNodeSet := range storageNodeSets.Items { + for _, secret := range storageNodeSet.Spec.Secrets { + if obj.GetName() == secret.Name { + remoteObjectsUsedInNamespace = append( + remoteObjectsUsedInNamespace, + obj, + ) + } + } + } + // Else client.Object typed by ConfigMap or Service + // which always used in another StorageNodeSet + default: + remoteObjectsUsedInNamespace = append( + remoteObjectsUsedInNamespace, + obj, + ) + } + } + } + + return remoteObjectsUsedInNamespace, nil +} + +func (r *Reconciler) updateRemoteResourcesStatus( + ctx context.Context, + remoteStorageNodeSet *resources.RemoteStorageNodeSetResource, +) (bool, ctrl.Result, error) { + crRemoteStorageNodeSet := &v1alpha1.RemoteStorageNodeSet{} + err := r.RemoteClient.Get(ctx, types.NamespacedName{ + Name: remoteStorageNodeSet.Name, + Namespace: remoteStorageNodeSet.Namespace, + }, crRemoteStorageNodeSet) + if err != nil { + r.Recorder.Event( + remoteStorageNodeSet, + corev1.EventTypeWarning, + "ControllerError", + "Failed fetching RemoteStorageNodeSet on remote cluster before remote status update", + ) + r.RemoteRecorder.Event( + remoteStorageNodeSet, + corev1.EventTypeWarning, + "ControllerError", + "Failed fetching RemoteStorageNodeSet before status update", + ) + return Stop, ctrl.Result{RequeueAfter: DefaultRequeueDelay}, err + } + + oldSyncResources := append([]v1alpha1.RemoteResource{}, crRemoteStorageNodeSet.Status.RemoteResources...) + crRemoteStorageNodeSet.Status.RemoteResources = append([]v1alpha1.RemoteResource{}, remoteStorageNodeSet.Status.RemoteResources...) + + if !reflect.DeepEqual(oldSyncResources, remoteStorageNodeSet.Status.RemoteResources) { + if err = r.RemoteClient.Status().Update(ctx, crRemoteStorageNodeSet); err != nil { + r.Recorder.Event( + remoteStorageNodeSet, + corev1.EventTypeWarning, + "ControllerError", + fmt.Sprintf("Failed to update status for remote resources on remote cluster: %s", err), + ) + r.RemoteRecorder.Event( + remoteStorageNodeSet, + corev1.EventTypeWarning, + "ControllerError", + fmt.Sprintf("Failed to update status for remote resources: %s", err), + ) + return Stop, ctrl.Result{RequeueAfter: DefaultRequeueDelay}, err + } + r.Recorder.Event( + remoteStorageNodeSet, + corev1.EventTypeNormal, + "StatusChanged", + "Status updated for remote resources on remote cluster", + ) + r.RemoteRecorder.Event( + remoteStorageNodeSet, + corev1.EventTypeNormal, + "StatusChanged", + "Status updated for remote resources", + ) + return Stop, ctrl.Result{RequeueAfter: StatusUpdateRequeueDelay}, nil + } + + return Continue, ctrl.Result{Requeue: false}, nil +} diff --git a/internal/controllers/remotestoragenodeset/sync.go b/internal/controllers/remotestoragenodeset/sync.go index 63fc931c..22da20e9 100644 --- a/internal/controllers/remotestoragenodeset/sync.go +++ b/internal/controllers/remotestoragenodeset/sync.go @@ -11,23 +11,35 @@ import ( ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" - ydbv1alpha1 "github.com/ydb-platform/ydb-kubernetes-operator/api/v1alpha1" + "github.com/ydb-platform/ydb-kubernetes-operator/api/v1alpha1" . "github.com/ydb-platform/ydb-kubernetes-operator/internal/controllers/constants" //nolint:revive,stylecheck "github.com/ydb-platform/ydb-kubernetes-operator/internal/resources" ) -func (r *Reconciler) Sync(ctx context.Context, crRemoteStorageNodeSet *ydbv1alpha1.RemoteStorageNodeSet) (ctrl.Result, error) { +func (r *Reconciler) Sync(ctx context.Context, crRemoteStorageNodeSet *v1alpha1.RemoteStorageNodeSet) (ctrl.Result, error) { var stop bool var result ctrl.Result var err error remoteStorageNodeSet := resources.NewRemoteStorageNodeSet(crRemoteStorageNodeSet) + remoteObjects := remoteStorageNodeSet.GetRemoteObjects() + + stop, result, err = r.initRemoteResourcesStatus(ctx, &remoteStorageNodeSet, remoteObjects) + if stop { + return result, err + } + + stop, result, err = r.syncRemoteObjects(ctx, &remoteStorageNodeSet, remoteObjects) + if stop { + return result, err + } + stop, result, err = r.handleResourcesSync(ctx, &remoteStorageNodeSet) if stop { return result, err } - stop, result, err = r.updateStatus(ctx, crRemoteStorageNodeSet) + stop, result, err = r.removeUnusedRemoteObjects(ctx, &remoteStorageNodeSet, remoteObjects) if stop { return result, err } @@ -85,24 +97,23 @@ func (r *Reconciler) handleResourcesSync( ) } } - r.Log.Info("resource sync complete") - return Continue, ctrl.Result{Requeue: false}, nil + + return r.updateRemoteStatus(ctx, remoteStorageNodeSet) } -func (r *Reconciler) updateStatus( +func (r *Reconciler) updateRemoteStatus( ctx context.Context, - crRemoteStorageNodeSet *ydbv1alpha1.RemoteStorageNodeSet, + remoteStorageNodeSet *resources.RemoteStorageNodeSetResource, ) (bool, ctrl.Result, error) { - r.Log.Info("running step updateStatus") - - storageNodeSet := &ydbv1alpha1.StorageNodeSet{} - err := r.Client.Get(ctx, types.NamespacedName{ - Name: crRemoteStorageNodeSet.Name, - Namespace: crRemoteStorageNodeSet.Namespace, - }, storageNodeSet) - if err != nil { + r.Log.Info("running step updateRemoteStatus") + + crStorageNodeSet := &v1alpha1.StorageNodeSet{} + if err := r.Client.Get(ctx, types.NamespacedName{ + Name: remoteStorageNodeSet.Name, + Namespace: remoteStorageNodeSet.Namespace, + }, crStorageNodeSet); err != nil { r.Recorder.Event( - crRemoteStorageNodeSet, + remoteStorageNodeSet, corev1.EventTypeWarning, "ControllerError", "Failed fetching StorageNodeSet before status update", @@ -110,39 +121,61 @@ func (r *Reconciler) updateStatus( return Stop, ctrl.Result{RequeueAfter: DefaultRequeueDelay}, err } - oldStatus := crRemoteStorageNodeSet.Status.State - crRemoteStorageNodeSet.Status.State = storageNodeSet.Status.State - crRemoteStorageNodeSet.Status.Conditions = storageNodeSet.Status.Conditions - - err = r.RemoteClient.Status().Update(ctx, crRemoteStorageNodeSet) - if err != nil { + crRemoteStorageNodeSet := &v1alpha1.RemoteStorageNodeSet{} + if err := r.RemoteClient.Get(ctx, types.NamespacedName{ + Name: remoteStorageNodeSet.Name, + Namespace: remoteStorageNodeSet.Namespace, + }, crRemoteStorageNodeSet); err != nil { r.Recorder.Event( - crRemoteStorageNodeSet, + remoteStorageNodeSet, corev1.EventTypeWarning, "ControllerError", - fmt.Sprintf("Failed setting status on remote cluster: %s", err), + "Failed fetching RemoteStorageNodeSet on remote cluster before status update", ) r.RemoteRecorder.Event( - crRemoteStorageNodeSet, + remoteStorageNodeSet, corev1.EventTypeWarning, "ControllerError", - fmt.Sprintf("Failed setting status: %s", err), + "Failed fetching RemoteStorageNodeSet before status update", ) return Stop, ctrl.Result{RequeueAfter: DefaultRequeueDelay}, err - } else if oldStatus != crRemoteStorageNodeSet.Status.State { + } + + oldStatus := crRemoteStorageNodeSet.Status.State + if oldStatus != crStorageNodeSet.Status.State { + crRemoteStorageNodeSet.Status.State = crStorageNodeSet.Status.State + crRemoteStorageNodeSet.Status.Conditions = crStorageNodeSet.Status.Conditions + if err := r.RemoteClient.Status().Update(ctx, crRemoteStorageNodeSet); err != nil { + r.Recorder.Event( + remoteStorageNodeSet, + corev1.EventTypeWarning, + "ControllerError", + fmt.Sprintf("Failed to update status on remote cluster: %s", err), + ) + r.RemoteRecorder.Event( + remoteStorageNodeSet, + corev1.EventTypeWarning, + "ControllerError", + fmt.Sprintf("Failed to update status: %s", err), + ) + return Stop, ctrl.Result{RequeueAfter: DefaultRequeueDelay}, err + } r.Recorder.Event( - crRemoteStorageNodeSet, + remoteStorageNodeSet, corev1.EventTypeNormal, "StatusChanged", - fmt.Sprintf("StorageNodeSet moved from %s to %s on remote cluster", oldStatus, crRemoteStorageNodeSet.Status.State), + "StorageNodeSet status updated on remote cluster", ) r.RemoteRecorder.Event( - crRemoteStorageNodeSet, + remoteStorageNodeSet, corev1.EventTypeNormal, "StatusChanged", - fmt.Sprintf("StorageNodeSet moved from %s to %s", oldStatus, crRemoteStorageNodeSet.Status.State), + "RemoteStorageNodeSet status updated", ) + r.Log.Info("step updateRemoteStatus requeue reconcile") + return Stop, ctrl.Result{RequeueAfter: StatusUpdateRequeueDelay}, nil } + r.Log.Info("step updateRemoteStatus completed") return Continue, ctrl.Result{Requeue: false}, nil } diff --git a/internal/controllers/storage/controller.go b/internal/controllers/storage/controller.go index 936b97ed..90eada89 100644 --- a/internal/controllers/storage/controller.go +++ b/internal/controllers/storage/controller.go @@ -10,17 +10,20 @@ import ( apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/rest" "k8s.io/client-go/tools/record" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/handler" "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/predicate" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + "sigs.k8s.io/controller-runtime/pkg/source" - ydbv1alpha1 "github.com/ydb-platform/ydb-kubernetes-operator/api/v1alpha1" - "github.com/ydb-platform/ydb-kubernetes-operator/internal/annotations" + "github.com/ydb-platform/ydb-kubernetes-operator/api/v1alpha1" . "github.com/ydb-platform/ydb-kubernetes-operator/internal/controllers/constants" //nolint:revive,stylecheck + "github.com/ydb-platform/ydb-kubernetes-operator/internal/resources" ) // Reconciler reconciles a Storage object @@ -63,7 +66,7 @@ type Reconciler struct { func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { r.Log = log.FromContext(ctx) - resource := &ydbv1alpha1.Storage{} + resource := &v1alpha1.Storage{} err := r.Get(ctx, req.NamespacedName, resource) if err != nil { if apierrors.IsNotFound(err) { @@ -81,41 +84,20 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu return result, err } -func ignoreDeletionPredicate() predicate.Predicate { - return predicate.Funcs{ - UpdateFunc: func(e event.UpdateEvent) bool { - generationChanged := e.ObjectOld.GetGeneration() != e.ObjectNew.GetGeneration() - annotationsChanged := !annotations.CompareYdbTechAnnotations(e.ObjectOld.GetAnnotations(), e.ObjectNew.GetAnnotations()) - _, isService := e.ObjectOld.(*corev1.Service) - - return generationChanged || annotationsChanged || isService - }, - DeleteFunc: func(e event.DeleteEvent) bool { - // Evaluates to false if the object has been confirmed deleted. - return !e.DeleteStateUnknown - }, - } -} - -// SetupWithManager sets up the controller with the Manager. -func (r *Reconciler) SetupWithManager(mgr ctrl.Manager) error { - storage := &ydbv1alpha1.Storage{} - - r.Recorder = mgr.GetEventRecorderFor(StorageKind) - controller := ctrl.NewControllerManagedBy(mgr) +func createFieldIndexers(mgr ctrl.Manager) error { if err := mgr.GetFieldIndexer().IndexField( context.Background(), - &ydbv1alpha1.RemoteStorageNodeSet{}, - OwnerControllerKey, + &v1alpha1.RemoteStorageNodeSet{}, + OwnerControllerField, func(obj client.Object) []string { // grab the RemoteStorageNodeSet object, extract the owner... - remoteStorageNodeSet := obj.(*ydbv1alpha1.RemoteStorageNodeSet) + remoteStorageNodeSet := obj.(*v1alpha1.RemoteStorageNodeSet) owner := metav1.GetControllerOf(remoteStorageNodeSet) if owner == nil { return nil } // ...make sure it's a Storage... - if owner.APIVersion != ydbv1alpha1.GroupVersion.String() || owner.Kind != StorageKind { + if owner.APIVersion != v1alpha1.GroupVersion.String() || owner.Kind != StorageKind { return nil } @@ -124,19 +106,20 @@ func (r *Reconciler) SetupWithManager(mgr ctrl.Manager) error { }); err != nil { return err } + if err := mgr.GetFieldIndexer().IndexField( context.Background(), - &ydbv1alpha1.StorageNodeSet{}, - OwnerControllerKey, + &v1alpha1.StorageNodeSet{}, + OwnerControllerField, func(obj client.Object) []string { // grab the StorageNodeSet object, extract the owner... - storageNodeSet := obj.(*ydbv1alpha1.StorageNodeSet) + storageNodeSet := obj.(*v1alpha1.StorageNodeSet) owner := metav1.GetControllerOf(storageNodeSet) if owner == nil { return nil } // ...make sure it's a Storage... - if owner.APIVersion != ydbv1alpha1.GroupVersion.String() || owner.Kind != StorageKind { + if owner.APIVersion != v1alpha1.GroupVersion.String() || owner.Kind != StorageKind { return nil } @@ -146,18 +129,77 @@ func (r *Reconciler) SetupWithManager(mgr ctrl.Manager) error { return err } + return mgr.GetFieldIndexer().IndexField( + context.Background(), + &v1alpha1.Storage{}, + SecretField, + func(obj client.Object) []string { + secrets := []string{} + storage := obj.(*v1alpha1.Storage) + for _, secret := range storage.Spec.Secrets { + secrets = append(secrets, secret.Name) + } + + return secrets + }) +} + +// SetupWithManager sets up the controller with the Manager. +func (r *Reconciler) SetupWithManager(mgr ctrl.Manager) error { + r.Recorder = mgr.GetEventRecorderFor(StorageKind) + controller := ctrl.NewControllerManagedBy(mgr) + + if err := createFieldIndexers(mgr); err != nil { + r.Log.Error(err, "unexpected FieldIndexer error") + return err + } + if r.WithServiceMonitors { controller = controller. Owns(&monitoringv1.ServiceMonitor{}) } return controller. - For(storage). - Owns(&ydbv1alpha1.RemoteStorageNodeSet{}). - Owns(&ydbv1alpha1.StorageNodeSet{}). - Owns(&corev1.Service{}). + For(&v1alpha1.Storage{}). + Owns(&v1alpha1.RemoteStorageNodeSet{}). + Owns(&v1alpha1.StorageNodeSet{}). Owns(&appsv1.StatefulSet{}). Owns(&corev1.ConfigMap{}). - WithEventFilter(ignoreDeletionPredicate()). + Owns(&corev1.Service{}). + Watches( + &source.Kind{Type: &corev1.Secret{}}, + handler.EnqueueRequestsFromMapFunc(r.findStoragesForSecret), + ). + WithEventFilter(predicate.Or( + predicate.GenerationChangedPredicate{}, + resources.IgnoreDeletetionPredicate(), + resources.LastAppliedAnnotationPredicate(), + resources.IsServicePredicate(), + resources.IsSecretPredicate(), + )). Complete(r) } + +func (r *Reconciler) findStoragesForSecret(secret client.Object) []reconcile.Request { + attachedStorages := &v1alpha1.StorageList{} + err := r.List( + context.Background(), + attachedStorages, + client.InNamespace(secret.GetNamespace()), + client.MatchingFields{SecretField: secret.GetName()}, + ) + if err != nil { + return []reconcile.Request{} + } + + requests := make([]reconcile.Request, len(attachedStorages.Items)) + for i, item := range attachedStorages.Items { + requests[i] = reconcile.Request{ + NamespacedName: types.NamespacedName{ + Name: item.GetName(), + Namespace: item.GetNamespace(), + }, + } + } + return requests +} diff --git a/internal/controllers/storage/init.go b/internal/controllers/storage/init.go index f28cfa25..99bca944 100644 --- a/internal/controllers/storage/init.go +++ b/internal/controllers/storage/init.go @@ -68,22 +68,16 @@ func (r *Reconciler) setInitialStatus( return Stop, ctrl.Result{RequeueAfter: StorageInitializationRequeueDelay}, nil } - changed := false - if meta.FindStatusCondition(storage.Status.Conditions, StorageInitializedCondition) == nil { + if storage.Status.State == StoragePending || + meta.FindStatusCondition(storage.Status.Conditions, StorageInitializedCondition) == nil { meta.SetStatusCondition(&storage.Status.Conditions, metav1.Condition{ Type: StorageInitializedCondition, Status: "False", Reason: ReasonInProgress, Message: "Storage is not ready yet", }) - changed = true - } - if storage.Status.State == StoragePending { storage.Status.State = StoragePreparing - changed = true - } - if changed { - return r.setState(ctx, storage) + return r.updateStatus(ctx, storage) } return Continue, ctrl.Result{Requeue: false}, nil } @@ -101,7 +95,7 @@ func (r *Reconciler) setInitStorageCompleted( }) storage.Status.State = StorageReady - return r.setState(ctx, storage) + return r.updateStatus(ctx, storage) } func (r *Reconciler) initializeStorage( @@ -112,7 +106,7 @@ func (r *Reconciler) initializeStorage( if storage.Status.State == StorageProvisioning { storage.Status.State = StorageInitializing - return r.setState(ctx, storage) + return r.updateStatus(ctx, storage) } initJob := &batchv1.Job{} diff --git a/internal/controllers/storage/sync.go b/internal/controllers/storage/sync.go index 94f026d9..9ed50500 100644 --- a/internal/controllers/storage/sync.go +++ b/internal/controllers/storage/sync.go @@ -76,7 +76,7 @@ func (r *Reconciler) waitForStatefulSetToScale( fmt.Sprintf("Starting to track number of running storage pods, expected: %d", storage.Spec.Nodes), ) storage.Status.State = StorageProvisioning - return r.setState(ctx, storage) + return r.updateStatus(ctx, storage) } found := &appsv1.StatefulSet{} @@ -92,7 +92,7 @@ func (r *Reconciler) waitForStatefulSetToScale( "ProvisioningFailed", fmt.Sprintf("StatefulSet with name %s was not found: %s", storage.Name, err), ) - return Stop, ctrl.Result{RequeueAfter: DefaultRequeueDelay}, nil + return Stop, ctrl.Result{RequeueAfter: DefaultRequeueDelay}, err } r.Recorder.Event( storage, @@ -164,7 +164,7 @@ func (r *Reconciler) waitForStorageNodeSetsToReady( fmt.Sprintf("Starting to track readiness of running nodeSets objects, expected: %d", storage.Spec.Nodes), ) storage.Status.State = StorageProvisioning - return r.setState(ctx, storage) + return r.updateStatus(ctx, storage) } var nodeSetObject client.Object @@ -191,7 +191,7 @@ func (r *Reconciler) waitForStorageNodeSetsToReady( "ProvisioningFailed", fmt.Sprintf("%s with name %s was not found: %s", nodeSetKind, nodeSetName, err), ) - return Stop, ctrl.Result{RequeueAfter: DefaultRequeueDelay}, nil + return Stop, ctrl.Result{RequeueAfter: DefaultRequeueDelay}, err } r.Recorder.Event( storage, @@ -299,7 +299,6 @@ func (r *Reconciler) handleResourcesSync( } } - r.Log.Info("resource sync complete") return Continue, ctrl.Result{Requeue: false}, nil } @@ -309,7 +308,7 @@ func (r *Reconciler) syncNodeSetSpecInline( ) (bool, ctrl.Result, error) { r.Log.Info("running step syncNodeSetSpecInline") matchingFields := client.MatchingFields{ - OwnerControllerKey: storage.Name, + OwnerControllerField: storage.Name, } storageNodeSets := &v1alpha1.StorageNodeSetList{} @@ -410,7 +409,6 @@ func (r *Reconciler) syncNodeSetSpecInline( } } - r.Log.Info("syncNodeSetSpecInline complete") return Continue, ctrl.Result{Requeue: false}, nil } @@ -461,18 +459,20 @@ func (r *Reconciler) runSelfCheck( return Continue, ctrl.Result{}, nil } -func (r *Reconciler) setState( +func (r *Reconciler) updateStatus( ctx context.Context, storage *resources.StorageClusterBuilder, ) (bool, ctrl.Result, error) { + r.Log.Info("running step updateStatus") + storageCr := &v1alpha1.Storage{} - err := r.Get(ctx, client.ObjectKey{ + err := r.Get(ctx, types.NamespacedName{ Namespace: storage.Namespace, Name: storage.Name, }, storageCr) if err != nil { r.Recorder.Event( - storageCr, + storage, corev1.EventTypeWarning, "ControllerError", "Failed fetching CR before status update", @@ -481,21 +481,20 @@ func (r *Reconciler) setState( } oldStatus := storageCr.Status.State - storageCr.Status.State = storage.Status.State - storageCr.Status.Conditions = storage.Status.Conditions - - err = r.Status().Update(ctx, storageCr) - if err != nil { - r.Recorder.Event( - storageCr, - corev1.EventTypeWarning, - "ControllerError", - fmt.Sprintf("Failed setting status: %s", err), - ) - return Stop, ctrl.Result{RequeueAfter: DefaultRequeueDelay}, err - } else if oldStatus != storageCr.Status.State { + if oldStatus != storage.Status.State { + storageCr.Status.State = storage.Status.State + storageCr.Status.Conditions = storage.Status.Conditions + if err = r.Status().Update(ctx, storageCr); err != nil { + r.Recorder.Event( + storage, + corev1.EventTypeWarning, + "ControllerError", + fmt.Sprintf("Failed setting status: %s", err), + ) + return Stop, ctrl.Result{RequeueAfter: DefaultRequeueDelay}, err + } r.Recorder.Event( - storageCr, + storage, corev1.EventTypeNormal, "StatusChanged", fmt.Sprintf("Storage moved from %s to %s", oldStatus, storageCr.Status.State), @@ -519,14 +518,14 @@ func (r *Reconciler) handlePauseResume( Message: "State Storage set to Paused", }) storage.Status.State = StoragePaused - return r.setState(ctx, storage) + return r.updateStatus(ctx, storage) } if storage.Status.State == StoragePaused && !storage.Spec.Pause { r.Log.Info("`pause: false` was noticed, moving Storage to state `Ready`") meta.RemoveStatusCondition(&storage.Status.Conditions, StoragePausedCondition) storage.Status.State = StorageReady - return r.setState(ctx, storage) + return r.updateStatus(ctx, storage) } return Continue, ctrl.Result{}, nil diff --git a/internal/controllers/storagenodeset/controller.go b/internal/controllers/storagenodeset/controller.go index af5405a9..79b9fabb 100644 --- a/internal/controllers/storagenodeset/controller.go +++ b/internal/controllers/storagenodeset/controller.go @@ -11,13 +11,12 @@ import ( "k8s.io/client-go/tools/record" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/event" "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/predicate" - api "github.com/ydb-platform/ydb-kubernetes-operator/api/v1alpha1" - "github.com/ydb-platform/ydb-kubernetes-operator/internal/annotations" + "github.com/ydb-platform/ydb-kubernetes-operator/api/v1alpha1" . "github.com/ydb-platform/ydb-kubernetes-operator/internal/controllers/constants" //nolint:revive,stylecheck + "github.com/ydb-platform/ydb-kubernetes-operator/internal/resources" ) // Reconciler reconciles a Storage object @@ -41,11 +40,11 @@ type Reconciler struct { func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { logger := log.FromContext(ctx) - crStorageNodeSet := &api.StorageNodeSet{} + crStorageNodeSet := &v1alpha1.StorageNodeSet{} err := r.Get(ctx, req.NamespacedName, crStorageNodeSet) if err != nil { if apierrors.IsNotFound(err) { - logger.Info("StorageNodeSet has been deleted") + logger.Info("StorageNodeSet resource not found") return ctrl.Result{Requeue: false}, nil } logger.Error(err, "unable to get StorageNodeSet") @@ -60,29 +59,18 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu return result, err } -func ignoreDeletionPredicate() predicate.Predicate { - return predicate.Funcs{ - UpdateFunc: func(e event.UpdateEvent) bool { - generationChanged := e.ObjectOld.GetGeneration() != e.ObjectNew.GetGeneration() - annotationsChanged := !annotations.CompareYdbTechAnnotations(e.ObjectOld.GetAnnotations(), e.ObjectNew.GetAnnotations()) - - return generationChanged || annotationsChanged - }, - DeleteFunc: func(e event.DeleteEvent) bool { - // Evaluates to false if the object has been confirmed deleted. - return !e.DeleteStateUnknown - }, - } -} - // SetupWithManager sets up the controller with the Manager. func (r *Reconciler) SetupWithManager(mgr ctrl.Manager) error { r.Recorder = mgr.GetEventRecorderFor(StorageNodeSetKind) controller := ctrl.NewControllerManagedBy(mgr) return controller. - For(&api.StorageNodeSet{}). + For(&v1alpha1.StorageNodeSet{}). Owns(&appsv1.StatefulSet{}). - WithEventFilter(ignoreDeletionPredicate()). + WithEventFilter(predicate.Or( + predicate.GenerationChangedPredicate{}, + resources.IgnoreDeletetionPredicate(), + resources.LastAppliedAnnotationPredicate()), + ). Complete(r) } diff --git a/internal/controllers/storagenodeset/sync.go b/internal/controllers/storagenodeset/sync.go index 60a35b68..0a038a6a 100644 --- a/internal/controllers/storagenodeset/sync.go +++ b/internal/controllers/storagenodeset/sync.go @@ -7,7 +7,7 @@ import ( appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/errors" + apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -16,12 +16,12 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" - ydbv1alpha1 "github.com/ydb-platform/ydb-kubernetes-operator/api/v1alpha1" + "github.com/ydb-platform/ydb-kubernetes-operator/api/v1alpha1" . "github.com/ydb-platform/ydb-kubernetes-operator/internal/controllers/constants" //nolint:revive,stylecheck "github.com/ydb-platform/ydb-kubernetes-operator/internal/resources" ) -func (r *Reconciler) Sync(ctx context.Context, crStorageNodeSet *ydbv1alpha1.StorageNodeSet) (ctrl.Result, error) { +func (r *Reconciler) Sync(ctx context.Context, crStorageNodeSet *v1alpha1.StorageNodeSet) (ctrl.Result, error) { var stop bool var result ctrl.Result var err error @@ -131,7 +131,7 @@ func (r *Reconciler) waitForStatefulSetToScale( string(StorageNodeSetProvisioning), fmt.Sprintf("Starting to track number of running storageNodeSet pods, expected: %d", storageNodeSet.Spec.Nodes)) storageNodeSet.Status.State = StorageNodeSetProvisioning - return r.setState(ctx, storageNodeSet) + return r.updateStatus(ctx, storageNodeSet) } foundStatefulSet := &appsv1.StatefulSet{} @@ -140,7 +140,7 @@ func (r *Reconciler) waitForStatefulSetToScale( Namespace: storageNodeSet.Namespace, }, foundStatefulSet) if err != nil { - if errors.IsNotFound(err) { + if apierrors.IsNotFound(err) { r.Recorder.Event( storageNodeSet, corev1.EventTypeWarning, @@ -195,39 +195,34 @@ func (r *Reconciler) waitForStatefulSetToScale( return Stop, ctrl.Result{RequeueAfter: DefaultRequeueDelay}, nil } - if storageNodeSet.Spec.Pause { - meta.SetStatusCondition(&storageNodeSet.Status.Conditions, metav1.Condition{ - Type: StoragePausedCondition, - Status: "True", - Reason: ReasonCompleted, - Message: "Scaled StorageNodeSet to 0 successfully", - }) - storageNodeSet.Status.State = DatabaseNodeSetPaused - } else { + if storageNodeSet.Status.State == StorageNodeSetProvisioning { meta.SetStatusCondition(&storageNodeSet.Status.Conditions, metav1.Condition{ Type: StorageNodeSetReadyCondition, Status: "True", Reason: ReasonCompleted, - Message: fmt.Sprintf("Scaled DatabaseNodeSet to %d successfully", storageNodeSet.Spec.Nodes), + Message: fmt.Sprintf("Scaled StorageNodeSet to %d successfully", storageNodeSet.Spec.Nodes), }) - storageNodeSet.Status.State = DatabaseNodeSetReady + storageNodeSet.Status.State = StorageNodeSetReady + return r.updateStatus(ctx, storageNodeSet) } - return r.setState(ctx, storageNodeSet) + return Continue, ctrl.Result{Requeue: false}, nil } -func (r *Reconciler) setState( +func (r *Reconciler) updateStatus( ctx context.Context, storageNodeSet *resources.StorageNodeSetResource, ) (bool, ctrl.Result, error) { - crStorageNodeSet := &ydbv1alpha1.StorageNodeSet{} - err := r.Get(ctx, client.ObjectKey{ + r.Log.Info("running step updateStatus") + + crStorageNodeSet := &v1alpha1.StorageNodeSet{} + err := r.Get(ctx, types.NamespacedName{ Namespace: storageNodeSet.Namespace, Name: storageNodeSet.Name, }, crStorageNodeSet) if err != nil { r.Recorder.Event( - crStorageNodeSet, + storageNodeSet, corev1.EventTypeWarning, "ControllerError", "Failed fetching CR before status update", @@ -236,21 +231,20 @@ func (r *Reconciler) setState( } oldStatus := crStorageNodeSet.Status.State - crStorageNodeSet.Status.State = storageNodeSet.Status.State - crStorageNodeSet.Status.Conditions = storageNodeSet.Status.Conditions - - err = r.Status().Update(ctx, crStorageNodeSet) - if err != nil { - r.Recorder.Event( - crStorageNodeSet, - corev1.EventTypeWarning, - "ControllerError", - fmt.Sprintf("Failed setting status: %s", err), - ) - return Stop, ctrl.Result{RequeueAfter: DefaultRequeueDelay}, err - } else if oldStatus != storageNodeSet.Status.State { + if oldStatus != storageNodeSet.Status.State { + crStorageNodeSet.Status.State = storageNodeSet.Status.State + crStorageNodeSet.Status.Conditions = storageNodeSet.Status.Conditions + if err = r.Status().Update(ctx, crStorageNodeSet); err != nil { + r.Recorder.Event( + storageNodeSet, + corev1.EventTypeWarning, + "ControllerError", + fmt.Sprintf("Failed setting status: %s", err), + ) + return Stop, ctrl.Result{RequeueAfter: DefaultRequeueDelay}, err + } r.Recorder.Event( - crStorageNodeSet, + storageNodeSet, corev1.EventTypeNormal, "StatusChanged", fmt.Sprintf("StorageNodeSet moved from %s to %s", oldStatus, storageNodeSet.Status.State), @@ -276,30 +270,24 @@ func (r *Reconciler) handlePauseResume( storageNodeSet *resources.StorageNodeSetResource, ) (bool, ctrl.Result, error) { r.Log.Info("running step handlePauseResume") + if storageNodeSet.Status.State == StorageReady && storageNodeSet.Spec.Pause { r.Log.Info("`pause: true` was noticed, moving StorageNodeSet to state `Paused`") - meta.RemoveStatusCondition(&storageNodeSet.Status.Conditions, StorageNodeSetReadyCondition) meta.SetStatusCondition(&storageNodeSet.Status.Conditions, metav1.Condition{ Type: StoragePausedCondition, - Status: "False", - Reason: ReasonInProgress, - Message: "Transitioning StorageNodeSet to Paused state", + Status: "True", + Reason: ReasonCompleted, + Message: "State StorageNodeSet set to Paused", }) storageNodeSet.Status.State = StorageNodeSetPaused - return r.setState(ctx, storageNodeSet) + return r.updateStatus(ctx, storageNodeSet) } if storageNodeSet.Status.State == StoragePaused && !storageNodeSet.Spec.Pause { r.Log.Info("`pause: false` was noticed, moving Storage to state `Ready`") meta.RemoveStatusCondition(&storageNodeSet.Status.Conditions, StoragePausedCondition) - meta.SetStatusCondition(&storageNodeSet.Status.Conditions, metav1.Condition{ - Type: StorageNodeSetReadyCondition, - Status: "False", - Reason: ReasonInProgress, - Message: "Recovering StorageNodeSet from Paused state", - }) storageNodeSet.Status.State = StorageNodeSetReady - return r.setState(ctx, storageNodeSet) + return r.updateStatus(ctx, storageNodeSet) } return Continue, ctrl.Result{}, nil diff --git a/internal/resources/database.go b/internal/resources/database.go index f0c252ba..794f6305 100644 --- a/internal/resources/database.go +++ b/internal/resources/database.go @@ -76,7 +76,8 @@ func (b *DatabaseBuilder) GetResourceBuilders(restConfig *rest.Config) []Resourc optionalBuilders, &ConfigMapBuilder{ Object: b, - Name: b.GetName(), + + Name: b.GetName(), Data: map[string]string{ api.ConfigFileName: b.Spec.Configuration, }, @@ -110,8 +111,9 @@ func (b *DatabaseBuilder) GetResourceBuilders(restConfig *rest.Config) []Resourc optionalBuilders, &EncryptionSecretBuilder{ Object: b, - Labels: databaseLabels, + Pin: pin, + Labels: databaseLabels, }, ) } @@ -120,7 +122,7 @@ func (b *DatabaseBuilder) GetResourceBuilders(restConfig *rest.Config) []Resourc optionalBuilders, &ServiceBuilder{ Object: b, - NameFormat: grpcServiceNameFormat, + NameFormat: GRPCServiceNameFormat, Labels: grpcServiceLabels, SelectorLabels: databaseLabels, Annotations: b.Spec.Service.GRPC.AdditionalAnnotations, @@ -133,7 +135,7 @@ func (b *DatabaseBuilder) GetResourceBuilders(restConfig *rest.Config) []Resourc }, &ServiceBuilder{ Object: b, - NameFormat: interconnectServiceNameFormat, + NameFormat: InterconnectServiceNameFormat, Labels: interconnectServiceLabels, SelectorLabels: databaseLabels, Annotations: b.Spec.Service.Interconnect.AdditionalAnnotations, @@ -147,7 +149,7 @@ func (b *DatabaseBuilder) GetResourceBuilders(restConfig *rest.Config) []Resourc }, &ServiceBuilder{ Object: b, - NameFormat: statusServiceNameFormat, + NameFormat: StatusServiceNameFormat, Labels: statusServiceLabels, SelectorLabels: databaseLabels, Annotations: b.Spec.Service.Status.AdditionalAnnotations, @@ -165,7 +167,7 @@ func (b *DatabaseBuilder) GetResourceBuilders(restConfig *rest.Config) []Resourc optionalBuilders, &ServiceBuilder{ Object: b, - NameFormat: datastreamsServiceNameFormat, + NameFormat: DatastreamsServiceNameFormat, Labels: datastreamsServiceLabels, SelectorLabels: databaseLabels, Annotations: b.Spec.Service.Datastreams.AdditionalAnnotations, @@ -202,8 +204,8 @@ func (b *DatabaseBuilder) getNodeSetBuilders(databaseLabels labels.Labels) []Res for _, nodeSetSpecInline := range b.Spec.NodeSets { nodeSetLabels := databaseLabels.Copy() - nodeSetLabels = nodeSetLabels.Merge(nodeSetSpecInline.AdditionalLabels) - nodeSetLabels = nodeSetLabels.Merge(map[string]string{labels.DatabaseNodeSetComponent: nodeSetSpecInline.Name}) + nodeSetLabels.Merge(nodeSetSpecInline.AdditionalLabels) + nodeSetLabels.Merge(map[string]string{labels.DatabaseNodeSetComponent: nodeSetSpecInline.Name}) databaseNodeSetSpec := b.recastDatabaseNodeSetSpecInline(nodeSetSpecInline.DeepCopy()) if nodeSetSpecInline.Remote != nil { diff --git a/internal/resources/database_statefulset.go b/internal/resources/database_statefulset.go index be37c606..0b95b75b 100644 --- a/internal/resources/database_statefulset.go +++ b/internal/resources/database_statefulset.go @@ -15,12 +15,12 @@ import ( "k8s.io/client-go/rest" "sigs.k8s.io/controller-runtime/pkg/client" - "github.com/ydb-platform/ydb-kubernetes-operator/api/v1alpha1" + api "github.com/ydb-platform/ydb-kubernetes-operator/api/v1alpha1" "github.com/ydb-platform/ydb-kubernetes-operator/internal/ptr" ) type DatabaseStatefulSetBuilder struct { - *v1alpha1.Database + *api.Database RestConfig *rest.Config Name string @@ -53,11 +53,11 @@ func (b *DatabaseStatefulSetBuilder) Build(obj client.Object) error { }, PodManagementPolicy: appsv1.ParallelPodManagement, RevisionHistoryLimit: ptr.Int32(10), - ServiceName: fmt.Sprintf(interconnectServiceNameFormat, b.Database.Name), + ServiceName: fmt.Sprintf(InterconnectServiceNameFormat, b.Database.Name), Template: b.buildPodTemplateSpec(), } - if value, ok := b.ObjectMeta.Annotations[v1alpha1.AnnotationUpdateStrategyOnDelete]; ok && value == v1alpha1.AnnotationValueTrue { + if value, ok := b.ObjectMeta.Annotations[api.AnnotationUpdateStrategyOnDelete]; ok && value == api.AnnotationValueTrue { sts.Spec.UpdateStrategy = appsv1.StatefulSetUpdateStrategy{ Type: "OnDelete", } @@ -101,7 +101,7 @@ func (b *DatabaseStatefulSetBuilder) buildPodTemplateSpec() corev1.PodTemplateSp DNSConfig: &corev1.PodDNSConfig{ Searches: []string{ - fmt.Sprintf(v1alpha1.InterconnectServiceFQDNFormat, b.Spec.StorageClusterRef.Name, b.Spec.StorageClusterRef.Namespace), + fmt.Sprintf(api.InterconnectServiceFQDNFormat, b.Spec.StorageClusterRef.Name, b.Spec.StorageClusterRef.Namespace), }, }, }, @@ -122,7 +122,7 @@ func (b *DatabaseStatefulSetBuilder) buildPodTemplateSpec() corev1.PodTemplateSp podTemplate.Spec.ImagePullSecrets = []corev1.LocalObjectReference{{Name: *b.Spec.Image.PullSecret}} } - if value, ok := b.ObjectMeta.Annotations[v1alpha1.AnnotationUpdateDNSPolicy]; ok { + if value, ok := b.ObjectMeta.Annotations[api.AnnotationUpdateDNSPolicy]; ok { switch value { case string(corev1.DNSClusterFirstWithHostNet), string(corev1.DNSClusterFirst), string(corev1.DNSDefault), string(corev1.DNSNone): podTemplate.Spec.DNSPolicy = corev1.DNSPolicy(value) @@ -284,7 +284,7 @@ func (b *DatabaseStatefulSetBuilder) buildCaStorePatchingInitContainerVolumeMoun return volumeMounts } -func buildTLSVolume(name string, configuration *v1alpha1.TLSConfiguration) corev1.Volume { // fixme move somewhere? +func buildTLSVolume(name string, configuration *api.TLSConfiguration) corev1.Volume { // fixme move somewhere? volume := corev1.Volume{ Name: name, VolumeSource: corev1.VolumeSource{ @@ -329,7 +329,7 @@ func (b *DatabaseStatefulSetBuilder) buildEncryptionVolume() corev1.Volume { Items: []corev1.KeyToPath{ { Key: secretKey, - Path: v1alpha1.DatabaseEncryptionKeyFile, + Path: api.DatabaseEncryptionKeyFile, }, }, }, @@ -346,7 +346,7 @@ func (b *DatabaseStatefulSetBuilder) buildDatastreamsIAMServiceAccountKeyVolume( Items: []corev1.KeyToPath{ { Key: b.Spec.Datastreams.IAMServiceAccountKey.Key, - Path: v1alpha1.DatastreamsIAMServiceAccountKeyFile, + Path: api.DatastreamsIAMServiceAccountKeyFile, }, }, }, @@ -377,27 +377,27 @@ func (b *DatabaseStatefulSetBuilder) buildContainer() corev1.Container { }, } - if value, ok := b.ObjectMeta.Annotations[v1alpha1.AnnotationDisableLivenessProbe]; !ok || value != v1alpha1.AnnotationValueTrue { + if value, ok := b.ObjectMeta.Annotations[api.AnnotationDisableLivenessProbe]; !ok || value != api.AnnotationValueTrue { container.LivenessProbe = &corev1.Probe{ ProbeHandler: corev1.ProbeHandler{ TCPSocket: &corev1.TCPSocketAction{ - Port: intstr.FromInt(v1alpha1.GRPCPort), + Port: intstr.FromInt(api.GRPCPort), }, }, } } ports := []corev1.ContainerPort{{ - Name: "grpc", ContainerPort: v1alpha1.GRPCPort, + Name: "grpc", ContainerPort: api.GRPCPort, }, { - Name: "interconnect", ContainerPort: v1alpha1.InterconnectPort, + Name: "interconnect", ContainerPort: api.InterconnectPort, }, { - Name: "status", ContainerPort: v1alpha1.StatusPort, + Name: "status", ContainerPort: api.StatusPort, }} if b.Spec.Datastreams != nil && b.Spec.Datastreams.Enabled { ports = append(ports, corev1.ContainerPort{ - Name: "datastreams", ContainerPort: v1alpha1.DatastreamsPort, + Name: "datastreams", ContainerPort: api.DatastreamsPort, }) } @@ -417,7 +417,7 @@ func (b *DatabaseStatefulSetBuilder) buildVolumeMounts() []corev1.VolumeMount { volumeMounts = append(volumeMounts, corev1.VolumeMount{ Name: configVolumeName, ReadOnly: true, - MountPath: v1alpha1.ConfigDir, + MountPath: api.ConfigDir, }) if b.Spec.Service.GRPC.TLSConfiguration.Enabled { @@ -440,7 +440,7 @@ func (b *DatabaseStatefulSetBuilder) buildVolumeMounts() []corev1.VolumeMount { volumeMounts = append(volumeMounts, corev1.VolumeMount{ Name: encryptionVolumeName, ReadOnly: true, - MountPath: v1alpha1.DatabaseEncryptionKeyPath, + MountPath: api.DatabaseEncryptionKeyPath, }) } @@ -448,7 +448,7 @@ func (b *DatabaseStatefulSetBuilder) buildVolumeMounts() []corev1.VolumeMount { volumeMounts = append(volumeMounts, corev1.VolumeMount{ Name: datastreamsIAMServiceAccountKeyVolumeName, ReadOnly: true, - MountPath: v1alpha1.DatastreamsIAMServiceAccountKeyPath, + MountPath: api.DatastreamsIAMServiceAccountKeyPath, }) if b.Spec.Service.Datastreams.TLSConfiguration.Enabled { volumeMounts = append(volumeMounts, corev1.VolumeMount{ @@ -489,19 +489,19 @@ func (b *DatabaseStatefulSetBuilder) buildVolumeMounts() []corev1.VolumeMount { } func (b *DatabaseStatefulSetBuilder) buildContainerArgs() ([]string, []string) { - command := []string{fmt.Sprintf("%s/%s", v1alpha1.BinariesDir, v1alpha1.DaemonBinaryName)} + command := []string{fmt.Sprintf("%s/%s", api.BinariesDir, api.DaemonBinaryName)} args := []string{ "server", "--mon-port", - fmt.Sprintf("%d", v1alpha1.StatusPort), + fmt.Sprintf("%d", api.StatusPort), "--ic-port", - fmt.Sprintf("%d", v1alpha1.InterconnectPort), + fmt.Sprintf("%d", api.InterconnectPort), "--yaml-config", - fmt.Sprintf("%s/%s", v1alpha1.ConfigDir, v1alpha1.ConfigFileName), + fmt.Sprintf("%s/%s", api.ConfigDir, api.ConfigFileName), "--tenant", b.GetDatabasePath(), @@ -519,7 +519,7 @@ func (b *DatabaseStatefulSetBuilder) buildContainerArgs() ([]string, []string) { LocalObjectReference: corev1.LocalObjectReference{ Name: secret.Name, }, - Key: v1alpha1.YdbAuthToken, + Key: api.YdbAuthToken, }, ) if err != nil { @@ -533,19 +533,19 @@ func (b *DatabaseStatefulSetBuilder) buildContainerArgs() ([]string, []string) { "%s/%s/%s", wellKnownDirForAdditionalSecrets, secret.Name, - v1alpha1.YdbAuthToken, + api.YdbAuthToken, ), ) } } publicHostOption := "--grpc-public-host" - publicHost := fmt.Sprintf(v1alpha1.InterconnectServiceFQDNFormat, b.Database.Name, b.GetNamespace()) // FIXME .svc.cluster.local + publicHost := fmt.Sprintf(api.InterconnectServiceFQDNFormat, b.Database.Name, b.GetNamespace()) // FIXME .svc.cluster.local if b.Spec.Service.GRPC.ExternalHost != "" { publicHost = b.Spec.Service.GRPC.ExternalHost } publicPortOption := "--grpc-public-port" - publicPort := v1alpha1.GRPCPort + publicPort := api.GRPCPort args = append( args, @@ -557,7 +557,7 @@ func (b *DatabaseStatefulSetBuilder) buildContainerArgs() ([]string, []string) { strconv.Itoa(publicPort), ) - if value, ok := b.ObjectMeta.Annotations[v1alpha1.AnnotationDataCenter]; ok { + if value, ok := b.ObjectMeta.Annotations[api.AnnotationDataCenter]; ok { if annotationDataCenterPattern.MatchString(value) { args = append(args, "--data-center", @@ -566,14 +566,14 @@ func (b *DatabaseStatefulSetBuilder) buildContainerArgs() ([]string, []string) { } } - if value, ok := b.ObjectMeta.Annotations[v1alpha1.AnnotationNodeHost]; ok { + if value, ok := b.ObjectMeta.Annotations[api.AnnotationNodeHost]; ok { args = append(args, "--node-host", value, ) } - if value, ok := b.ObjectMeta.Annotations[v1alpha1.AnnotationNodeDomain]; ok { + if value, ok := b.ObjectMeta.Annotations[api.AnnotationNodeDomain]; ok { args = append(args, "--node-domain", value, diff --git a/internal/resources/remotedatabasenodeset.go b/internal/resources/remotedatabasenodeset.go index 02342a35..33efacdb 100644 --- a/internal/resources/remotedatabasenodeset.go +++ b/internal/resources/remotedatabasenodeset.go @@ -2,12 +2,17 @@ package resources import ( "errors" + "fmt" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" "sigs.k8s.io/controller-runtime/pkg/client" api "github.com/ydb-platform/ydb-kubernetes-operator/api/v1alpha1" ydbannotations "github.com/ydb-platform/ydb-kubernetes-operator/internal/annotations" + . "github.com/ydb-platform/ydb-kubernetes-operator/internal/controllers/constants" //nolint:revive,stylecheck ) type RemoteDatabaseNodeSetBuilder struct { @@ -51,6 +56,7 @@ func (b *RemoteDatabaseNodeSetBuilder) Placeholder(cr client.Object) client.Obje func (b *RemoteDatabaseNodeSetResource) GetResourceBuilders() []ResourceBuilder { var resourceBuilders []ResourceBuilder + resourceBuilders = append(resourceBuilders, &DatabaseNodeSetBuilder{ Object: b, @@ -61,6 +67,7 @@ func (b *RemoteDatabaseNodeSetResource) GetResourceBuilders() []ResourceBuilder DatabaseNodeSetSpec: b.Spec, }, ) + return resourceBuilders } @@ -70,6 +77,63 @@ func NewRemoteDatabaseNodeSet(remoteDatabaseNodeSet *api.RemoteDatabaseNodeSet) return RemoteDatabaseNodeSetResource{RemoteDatabaseNodeSet: crRemoteDatabaseNodeSet} } +func (b *RemoteDatabaseNodeSetResource) GetRemoteObjects() []client.Object { + objects := []client.Object{} + + // sync Secrets + for _, secret := range b.Spec.Secrets { + objects = append(objects, + &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: secret.Name, + Namespace: b.Namespace, + }, + }) + } + + // sync ConfigMap + objects = append(objects, + &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: b.Spec.DatabaseRef.Name, + Namespace: b.Namespace, + }, + }) + + // sync Services + objects = append(objects, + &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf(GRPCServiceNameFormat, b.Spec.DatabaseRef.Name), + Namespace: b.Namespace, + }, + }, + &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf(InterconnectServiceNameFormat, b.Spec.DatabaseRef.Name), + Namespace: b.Namespace, + }, + }, + &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf(StatusServiceNameFormat, b.Spec.DatabaseRef.Name), + Namespace: b.Namespace, + }, + }, + ) + if b.Spec.Datastreams != nil && b.Spec.Datastreams.Enabled { + objects = append(objects, + &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf(DatastreamsServiceNameFormat, b.Spec.DatabaseRef.Name), + Namespace: b.Namespace, + }, + }) + } + + return objects +} + func (b *RemoteDatabaseNodeSetResource) SetPrimaryResourceAnnotations(obj client.Object) { annotations := make(map[string]string) for key, value := range obj.GetAnnotations() { @@ -82,3 +146,31 @@ func (b *RemoteDatabaseNodeSetResource) SetPrimaryResourceAnnotations(obj client obj.SetAnnotations(annotations) } + +func (b *RemoteDatabaseNodeSetResource) SetRemoteResourceStatus(remoteObj client.Object, remoteObjGVK schema.GroupVersionKind) { + for idx := range b.Status.RemoteResources { + if EqualRemoteResourceWithObject(&b.Status.RemoteResources[idx], b.Namespace, remoteObj, remoteObjGVK) { + meta.SetStatusCondition(&b.Status.RemoteResources[idx].Conditions, + metav1.Condition{ + Type: RemoteResourceSyncedCondition, + Status: "True", + Reason: ReasonCompleted, + Message: fmt.Sprintf("Resource updated with resourceVersion %s", remoteObj.GetResourceVersion()), + }) + b.Status.RemoteResources[idx].State = ResourceSyncSuccess + } + } +} + +func (b *RemoteDatabaseNodeSetResource) RemoveRemoteResourceStatus(remoteObj client.Object, remoteObjGVK schema.GroupVersionKind) { + syncedResources := append([]api.RemoteResource{}, b.Status.RemoteResources...) + for idx := range syncedResources { + if EqualRemoteResourceWithObject(&syncedResources[idx], b.Namespace, remoteObj, remoteObjGVK) { + b.Status.RemoteResources = append( + b.Status.RemoteResources[:idx], + b.Status.RemoteResources[idx+1:]..., + ) + break + } + } +} diff --git a/internal/resources/remotestoragenodeset.go b/internal/resources/remotestoragenodeset.go index 07e6fba8..7734dc8d 100644 --- a/internal/resources/remotestoragenodeset.go +++ b/internal/resources/remotestoragenodeset.go @@ -2,12 +2,17 @@ package resources import ( "errors" + "fmt" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" "sigs.k8s.io/controller-runtime/pkg/client" api "github.com/ydb-platform/ydb-kubernetes-operator/api/v1alpha1" ydbannotations "github.com/ydb-platform/ydb-kubernetes-operator/internal/annotations" + . "github.com/ydb-platform/ydb-kubernetes-operator/internal/controllers/constants" //nolint:revive,stylecheck ) type RemoteStorageNodeSetBuilder struct { @@ -51,12 +56,14 @@ func (b *RemoteStorageNodeSetBuilder) Placeholder(cr client.Object) client.Objec func (b *RemoteStorageNodeSetResource) GetResourceBuilders() []ResourceBuilder { var resourceBuilders []ResourceBuilder + resourceBuilders = append(resourceBuilders, &StorageNodeSetBuilder{ Object: b, - Name: b.Name, - Labels: b.Labels, + Name: b.Name, + Labels: b.Labels, + StorageNodeSetSpec: b.Spec, }, ) @@ -69,6 +76,54 @@ func NewRemoteStorageNodeSet(remoteStorageNodeSet *api.RemoteStorageNodeSet) Rem return RemoteStorageNodeSetResource{RemoteStorageNodeSet: crRemoteStorageNodeSet} } +func (b *RemoteStorageNodeSetResource) GetRemoteObjects() []client.Object { + objects := []client.Object{} + + // sync Secrets + for _, secret := range b.Spec.Secrets { + objects = append(objects, + &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: secret.Name, + Namespace: b.Namespace, + }, + }) + } + + // sync ConfigMap + objects = append(objects, + &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: b.Spec.StorageRef.Name, + Namespace: b.Namespace, + }, + }) + + // sync Services + objects = append(objects, + &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf(GRPCServiceNameFormat, b.Spec.StorageRef.Name), + Namespace: b.Namespace, + }, + }, + &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf(InterconnectServiceNameFormat, b.Spec.StorageRef.Name), + Namespace: b.Namespace, + }, + }, + &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf(StatusServiceNameFormat, b.Spec.StorageRef.Name), + Namespace: b.Namespace, + }, + }, + ) + + return objects +} + func (b *RemoteStorageNodeSetResource) SetPrimaryResourceAnnotations(obj client.Object) { annotations := make(map[string]string) for key, value := range obj.GetAnnotations() { @@ -81,3 +136,31 @@ func (b *RemoteStorageNodeSetResource) SetPrimaryResourceAnnotations(obj client. obj.SetAnnotations(annotations) } + +func (b *RemoteStorageNodeSetResource) SetRemoteResourceStatus(remoteObj client.Object, remoteObjGVK schema.GroupVersionKind) { + for idx := range b.Status.RemoteResources { + if EqualRemoteResourceWithObject(&b.Status.RemoteResources[idx], b.Namespace, remoteObj, remoteObjGVK) { + meta.SetStatusCondition(&b.Status.RemoteResources[idx].Conditions, + metav1.Condition{ + Type: RemoteResourceSyncedCondition, + Status: "True", + Reason: ReasonCompleted, + Message: fmt.Sprintf("Resource updated with resourceVersion %s", remoteObj.GetResourceVersion()), + }) + b.Status.RemoteResources[idx].State = ResourceSyncSuccess + } + } +} + +func (b *RemoteStorageNodeSetResource) RemoveRemoteResourceStatus(remoteObj client.Object, remoteObjGVK schema.GroupVersionKind) { + syncedResources := append([]api.RemoteResource{}, b.Status.RemoteResources...) + for idx := range syncedResources { + if EqualRemoteResourceWithObject(&syncedResources[idx], b.Namespace, remoteObj, remoteObjGVK) { + b.Status.RemoteResources = append( + b.Status.RemoteResources[:idx], + b.Status.RemoteResources[idx+1:]..., + ) + break + } + } +} diff --git a/internal/resources/resource.go b/internal/resources/resource.go index 4b35016e..c55f149d 100644 --- a/internal/resources/resource.go +++ b/internal/resources/resource.go @@ -8,24 +8,31 @@ import ( "github.com/banzaicloud/k8s-objectmatcher/patch" ydbCredentials "github.com/ydb-platform/ydb-go-sdk/v3/credentials" appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/client-go/rest" + "k8s.io/kubectl/pkg/scheme" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" ctrlutil "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/event" "sigs.k8s.io/controller-runtime/pkg/predicate" api "github.com/ydb-platform/ydb-kubernetes-operator/api/v1alpha1" + ydbannotations "github.com/ydb-platform/ydb-kubernetes-operator/internal/annotations" "github.com/ydb-platform/ydb-kubernetes-operator/internal/connection" ) const ( - grpcServiceNameFormat = "%s-grpc" - interconnectServiceNameFormat = "%s-interconnect" - statusServiceNameFormat = "%s-status" - datastreamsServiceNameFormat = "%s-datastreams" + GRPCServiceNameFormat = "%s-grpc" + InterconnectServiceNameFormat = "%s-interconnect" + StatusServiceNameFormat = "%s-status" + DatastreamsServiceNameFormat = "%s-datastreams" grpcTLSVolumeName = "grpc-tls-volume" interconnectTLSVolumeName = "interconnect-tls-volume" @@ -53,7 +60,6 @@ const ( localCertsDir = "/usr/local/share/ca-certificates" systemCertsDir = "/etc/ssl/certs" - lastAppliedAnnotation = "ydb.tech/last-applied" encryptionVolumeName = "encryption" datastreamsIAMServiceAccountKeyVolumeName = "datastreams-iam-sa-key" defaultEncryptionSecretKey = "key" @@ -66,7 +72,7 @@ type ResourceBuilder interface { } var ( - annotator = patch.NewAnnotator(lastAppliedAnnotation) + annotator = patch.NewAnnotator(ydbannotations.LastAppliedAnnotation) patchMaker = patch.NewPatchMaker(annotator) ) @@ -189,6 +195,146 @@ func CopyDict(src map[string]string) map[string]string { return dst } +func CreateResource(obj client.Object) client.Object { + createdObj := obj.DeepCopyObject().(client.Object) + + // Remove or reset fields + createdObj.SetResourceVersion("") + createdObj.SetCreationTimestamp(metav1.Time{}) + createdObj.SetUID("") + createdObj.SetOwnerReferences([]metav1.OwnerReference{}) + createdObj.SetFinalizers([]string{}) + + if svc, ok := createdObj.(*corev1.Service); ok { + svc.Spec.ClusterIP = "" + svc.Spec.ClusterIPs = nil + } + + setRemoteResourceVersionAnnotation(createdObj, obj.GetResourceVersion()) + + return createdObj +} + +func UpdateResource(oldObj, newObj client.Object) client.Object { + updatedObj := newObj.DeepCopyObject().(client.Object) + + // Save current fields + updatedObj.SetResourceVersion(oldObj.GetResourceVersion()) + updatedObj.SetCreationTimestamp(oldObj.GetCreationTimestamp()) + updatedObj.SetUID(oldObj.GetUID()) + updatedObj.SetOwnerReferences(oldObj.GetOwnerReferences()) + updatedObj.SetFinalizers(oldObj.GetFinalizers()) + + if svc, ok := updatedObj.(*corev1.Service); ok { + svc.Spec.ClusterIP = oldObj.(*corev1.Service).Spec.ClusterIP + svc.Spec.ClusterIPs = append([]string{}, oldObj.(*corev1.Service).Spec.ClusterIPs...) + } + + setRemoteResourceVersionAnnotation(updatedObj, newObj.GetResourceVersion()) + + return updatedObj +} + +func setRemoteResourceVersionAnnotation(obj client.Object, resourceVersion string) { + annotations := make(map[string]string) + for key, value := range obj.GetAnnotations() { + annotations[key] = value + } + annotations[ydbannotations.RemoteResourceVersionAnnotation] = resourceVersion + obj.SetAnnotations(annotations) +} + +func ConvertRemoteResourceToObject(remoteResource api.RemoteResource, namespace string) (client.Object, error) { + // Create an unstructured object + obj := &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": remoteResource.Version, + "group": remoteResource.Group, + "kind": remoteResource.Kind, + "metadata": map[string]interface{}{ + "name": remoteResource.Name, + "namespace": namespace, + }, + }, + } + + // Convert unstructured object to runtime.Object + var runtimeObj runtime.Object + runtimeObj, err := scheme.Scheme.New( + schema.GroupVersionKind{ + Group: remoteResource.Group, + Version: remoteResource.Version, + Kind: remoteResource.Kind, + }, + ) + if err != nil { + return nil, err + } + + // Copy data from unstructured to runtime object + err = scheme.Scheme.Convert(obj, runtimeObj, nil) + if err != nil { + return nil, err + } + + // Assert runtime.Object to client.Object + return runtimeObj.(client.Object), nil +} + +func EqualRemoteResourceWithObject( + remoteResource *api.RemoteResource, + namespace string, + remoteObj client.Object, + remoteObjGVK schema.GroupVersionKind, +) bool { + if remoteObj.GetName() == remoteResource.Name && + remoteObj.GetNamespace() == namespace && + remoteObjGVK.Kind == remoteResource.Kind && + remoteObjGVK.Group == remoteResource.Group && + remoteObjGVK.Version == remoteResource.Version { + return true + } + return false +} + +func LastAppliedAnnotationPredicate() predicate.Predicate { + return predicate.Funcs{ + UpdateFunc: func(e event.UpdateEvent) bool { + return !ydbannotations.CompareYdbTechAnnotations( + e.ObjectOld.GetAnnotations(), + e.ObjectNew.GetAnnotations(), + ) + }, + } +} + +func IgnoreDeletetionPredicate() predicate.Predicate { + return predicate.Funcs{ + DeleteFunc: func(e event.DeleteEvent) bool { + // Evaluates to false if the object has been confirmed deleted. + return !e.DeleteStateUnknown + }, + } +} + +func IsServicePredicate() predicate.Predicate { + return predicate.Funcs{ + UpdateFunc: func(e event.UpdateEvent) bool { + _, isService := e.ObjectOld.(*corev1.Service) + return isService + }, + } +} + +func IsSecretPredicate() predicate.Predicate { + return predicate.Funcs{ + UpdateFunc: func(e event.UpdateEvent) bool { + _, isSecret := e.ObjectOld.(*corev1.Secret) + return isSecret + }, + } +} + func LabelExistsPredicate(selector labels.Selector) predicate.Predicate { return predicate.NewPredicateFuncs(func(o client.Object) bool { return selector.Matches(labels.Set(o.GetLabels())) diff --git a/internal/resources/servicemonitor.go b/internal/resources/servicemonitor.go index be77ce9b..3656acef 100644 --- a/internal/resources/servicemonitor.go +++ b/internal/resources/servicemonitor.go @@ -8,7 +8,7 @@ import ( "k8s.io/apimachinery/pkg/util/intstr" "sigs.k8s.io/controller-runtime/pkg/client" - "github.com/ydb-platform/ydb-kubernetes-operator/api/v1alpha1" + api "github.com/ydb-platform/ydb-kubernetes-operator/api/v1alpha1" "github.com/ydb-platform/ydb-kubernetes-operator/internal/labels" "github.com/ydb-platform/ydb-kubernetes-operator/internal/metrics" ) @@ -19,7 +19,7 @@ type ServiceMonitorBuilder struct { Name string MetricsServices []metrics.Service TargetPort int - Options *v1alpha1.MonitoringOptions + Options *api.MonitoringOptions Labels labels.Labels SelectorLabels labels.Labels diff --git a/internal/resources/storage.go b/internal/resources/storage.go index a4c52099..26671de0 100644 --- a/internal/resources/storage.go +++ b/internal/resources/storage.go @@ -49,8 +49,19 @@ func (b *StorageClusterBuilder) Unwrap() *api.Storage { func (b *StorageClusterBuilder) GetResourceBuilders(restConfig *rest.Config) []ResourceBuilder { storageLabels := labels.StorageLabels(b.Unwrap()) - var optionalBuilders []ResourceBuilder + grpcServiceLabels := storageLabels.Copy() + grpcServiceLabels.Merge(b.Spec.Service.GRPC.AdditionalLabels) + grpcServiceLabels.Merge(map[string]string{labels.ServiceComponent: labels.GRPCComponent}) + interconnectServiceLabels := storageLabels.Copy() + interconnectServiceLabels.Merge(b.Spec.Service.Interconnect.AdditionalLabels) + interconnectServiceLabels.Merge(map[string]string{labels.ServiceComponent: labels.InterconnectComponent}) + + statusServiceLabels := storageLabels.Copy() + statusServiceLabels.Merge(b.Spec.Service.Status.AdditionalLabels) + statusServiceLabels.Merge(map[string]string{labels.ServiceComponent: labels.StatusComponent}) + + var optionalBuilders []ResourceBuilder optionalBuilders = append( optionalBuilders, &ConfigMapBuilder{ @@ -63,18 +74,6 @@ func (b *StorageClusterBuilder) GetResourceBuilders(restConfig *rest.Config) []R }, ) - grpcServiceLabels := storageLabels.Copy() - grpcServiceLabels.Merge(b.Spec.Service.GRPC.AdditionalLabels) - grpcServiceLabels.Merge(map[string]string{labels.ServiceComponent: labels.GRPCComponent}) - - interconnectServiceLabels := storageLabels.Copy() - interconnectServiceLabels.Merge(b.Spec.Service.Interconnect.AdditionalLabels) - interconnectServiceLabels.Merge(map[string]string{labels.ServiceComponent: labels.InterconnectComponent}) - - statusServiceLabels := storageLabels.Copy() - statusServiceLabels.Merge(b.Spec.Service.Status.AdditionalLabels) - statusServiceLabels.Merge(map[string]string{labels.ServiceComponent: labels.StatusComponent}) - if b.Spec.Monitoring.Enabled { optionalBuilders = append(optionalBuilders, &ServiceMonitorBuilder{ @@ -109,7 +108,7 @@ func (b *StorageClusterBuilder) GetResourceBuilders(restConfig *rest.Config) []R optionalBuilders, &ServiceBuilder{ Object: b, - NameFormat: grpcServiceNameFormat, + NameFormat: GRPCServiceNameFormat, Labels: grpcServiceLabels, SelectorLabels: storageLabels, Annotations: b.Spec.Service.GRPC.AdditionalAnnotations, @@ -122,7 +121,7 @@ func (b *StorageClusterBuilder) GetResourceBuilders(restConfig *rest.Config) []R }, &ServiceBuilder{ Object: b, - NameFormat: interconnectServiceNameFormat, + NameFormat: InterconnectServiceNameFormat, Labels: interconnectServiceLabels, SelectorLabels: storageLabels, Annotations: b.Spec.Service.Interconnect.AdditionalAnnotations, @@ -136,7 +135,7 @@ func (b *StorageClusterBuilder) GetResourceBuilders(restConfig *rest.Config) []R }, &ServiceBuilder{ Object: b, - NameFormat: statusServiceNameFormat, + NameFormat: StatusServiceNameFormat, Labels: statusServiceLabels, SelectorLabels: storageLabels, Annotations: b.Spec.Service.Status.AdditionalAnnotations, @@ -155,8 +154,8 @@ func (b *StorageClusterBuilder) getNodeSetBuilders(storageLabels labels.Labels) for _, nodeSetSpecInline := range b.Spec.NodeSets { nodeSetLabels := storageLabels.Copy() - nodeSetLabels = nodeSetLabels.Merge(nodeSetSpecInline.AdditionalLabels) - nodeSetLabels = nodeSetLabels.Merge(map[string]string{labels.StorageNodeSetComponent: nodeSetSpecInline.Name}) + nodeSetLabels.Merge(nodeSetSpecInline.AdditionalLabels) + nodeSetLabels.Merge(map[string]string{labels.StorageNodeSetComponent: nodeSetSpecInline.Name}) storageNodeSetSpec := b.recastStorageNodeSetSpecInline(nodeSetSpecInline.DeepCopy()) if nodeSetSpecInline.Remote != nil { diff --git a/internal/resources/storage_statefulset.go b/internal/resources/storage_statefulset.go index e4128fcc..6977ade7 100644 --- a/internal/resources/storage_statefulset.go +++ b/internal/resources/storage_statefulset.go @@ -14,7 +14,7 @@ import ( "k8s.io/client-go/rest" "sigs.k8s.io/controller-runtime/pkg/client" - "github.com/ydb-platform/ydb-kubernetes-operator/api/v1alpha1" + api "github.com/ydb-platform/ydb-kubernetes-operator/api/v1alpha1" "github.com/ydb-platform/ydb-kubernetes-operator/internal/ptr" ) @@ -23,7 +23,7 @@ const ( ) type StorageStatefulSetBuilder struct { - *v1alpha1.Storage + *api.Storage RestConfig *rest.Config Name string @@ -40,11 +40,11 @@ func StringRJust(str, pad string, length int) string { } func (b *StorageStatefulSetBuilder) GeneratePVCName(index int) string { - return b.Name + "-" + StringRJust(strconv.Itoa(index), "0", v1alpha1.DiskNumberMaxDigits) + return b.Name + "-" + StringRJust(strconv.Itoa(index), "0", api.DiskNumberMaxDigits) } func (b *StorageStatefulSetBuilder) GenerateDeviceName(index int) string { - return v1alpha1.DiskPathPrefix + "_" + StringRJust(strconv.Itoa(index), "0", v1alpha1.DiskNumberMaxDigits) + return api.DiskPathPrefix + "_" + StringRJust(strconv.Itoa(index), "0", api.DiskNumberMaxDigits) } func (b *StorageStatefulSetBuilder) Build(obj client.Object) error { @@ -71,11 +71,11 @@ func (b *StorageStatefulSetBuilder) Build(obj client.Object) error { }, PodManagementPolicy: appsv1.ParallelPodManagement, RevisionHistoryLimit: ptr.Int32(10), - ServiceName: fmt.Sprintf(interconnectServiceNameFormat, b.Storage.Name), + ServiceName: fmt.Sprintf(InterconnectServiceNameFormat, b.Storage.Name), Template: b.buildPodTemplateSpec(), } - if value, ok := b.ObjectMeta.Annotations[v1alpha1.AnnotationUpdateStrategyOnDelete]; ok && value == v1alpha1.AnnotationValueTrue { + if value, ok := b.ObjectMeta.Annotations[api.AnnotationUpdateStrategyOnDelete]; ok && value == api.AnnotationValueTrue { sts.Spec.UpdateStrategy = appsv1.StatefulSetUpdateStrategy{ Type: "OnDelete", } @@ -100,7 +100,7 @@ func (b *StorageStatefulSetBuilder) Build(obj client.Object) error { func (b *StorageStatefulSetBuilder) buildPodTemplateSpec() corev1.PodTemplateSpec { dnsConfigSearches := []string{ - fmt.Sprintf(v1alpha1.InterconnectServiceFQDNFormat, b.Storage.Name, b.GetNamespace()), + fmt.Sprintf(api.InterconnectServiceFQDNFormat, b.Storage.Name, b.GetNamespace()), } podTemplate := corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ @@ -143,7 +143,7 @@ func (b *StorageStatefulSetBuilder) buildPodTemplateSpec() corev1.PodTemplateSpe podTemplate.Spec.ImagePullSecrets = []corev1.LocalObjectReference{{Name: *b.Spec.Image.PullSecret}} } - if value, ok := b.ObjectMeta.Annotations[v1alpha1.AnnotationUpdateDNSPolicy]; ok { + if value, ok := b.ObjectMeta.Annotations[api.AnnotationUpdateDNSPolicy]; ok { switch value { case string(corev1.DNSClusterFirstWithHostNet), string(corev1.DNSClusterFirst), string(corev1.DNSDefault), string(corev1.DNSNone): podTemplate.Spec.DNSPolicy = corev1.DNSPolicy(value) @@ -161,7 +161,7 @@ func (b *StorageStatefulSetBuilder) buildTopologySpreadConstraints() []corev1.To return b.Spec.TopologySpreadConstraints } - if b.Spec.Erasure != v1alpha1.ErasureMirror3DC { + if b.Spec.Erasure != api.ErasureMirror3DC { return []corev1.TopologySpreadConstraint{} } @@ -335,22 +335,22 @@ func (b *StorageStatefulSetBuilder) buildContainer() corev1.Container { // todo }, Ports: []corev1.ContainerPort{{ - Name: "grpc", ContainerPort: v1alpha1.GRPCPort, + Name: "grpc", ContainerPort: api.GRPCPort, }, { - Name: "interconnect", ContainerPort: v1alpha1.InterconnectPort, + Name: "interconnect", ContainerPort: api.InterconnectPort, }, { - Name: "status", ContainerPort: v1alpha1.StatusPort, + Name: "status", ContainerPort: api.StatusPort, }}, VolumeMounts: b.buildVolumeMounts(), Resources: containerResources, } - if value, ok := b.ObjectMeta.Annotations[v1alpha1.AnnotationDisableLivenessProbe]; !ok || value != v1alpha1.AnnotationValueTrue { + if value, ok := b.ObjectMeta.Annotations[api.AnnotationDisableLivenessProbe]; !ok || value != api.AnnotationValueTrue { container.LivenessProbe = &corev1.Probe{ ProbeHandler: corev1.ProbeHandler{ TCPSocket: &corev1.TCPSocketAction{ - Port: intstr.FromInt(v1alpha1.GRPCPort), + Port: intstr.FromInt(api.GRPCPort), }, }, } @@ -364,7 +364,7 @@ func (b *StorageStatefulSetBuilder) buildContainer() corev1.Container { // todo volumeMountList, corev1.VolumeMount{ Name: b.GeneratePVCName(i), - MountPath: v1alpha1.DiskFilePath, + MountPath: api.DiskFilePath, }, ) } @@ -389,7 +389,7 @@ func (b *StorageStatefulSetBuilder) buildVolumeMounts() []corev1.VolumeMount { { Name: configVolumeName, ReadOnly: true, - MountPath: v1alpha1.ConfigDir, + MountPath: api.ConfigDir, }, } @@ -439,20 +439,20 @@ func (b *StorageStatefulSetBuilder) buildVolumeMounts() []corev1.VolumeMount { } func (b *StorageStatefulSetBuilder) buildContainerArgs() ([]string, []string) { - command := []string{fmt.Sprintf("%s/%s", v1alpha1.BinariesDir, v1alpha1.DaemonBinaryName)} + command := []string{fmt.Sprintf("%s/%s", api.BinariesDir, api.DaemonBinaryName)} var args []string args = append(args, "server", "--mon-port", - fmt.Sprintf("%d", v1alpha1.StatusPort), + fmt.Sprintf("%d", api.StatusPort), "--ic-port", - fmt.Sprintf("%d", v1alpha1.InterconnectPort), + fmt.Sprintf("%d", api.InterconnectPort), "--yaml-config", - fmt.Sprintf("%s/%s", v1alpha1.ConfigDir, v1alpha1.ConfigFileName), + fmt.Sprintf("%s/%s", api.ConfigDir, api.ConfigFileName), "--node", "static", @@ -467,7 +467,7 @@ func (b *StorageStatefulSetBuilder) buildContainerArgs() ([]string, []string) { LocalObjectReference: corev1.LocalObjectReference{ Name: secret.Name, }, - Key: v1alpha1.YdbAuthToken, + Key: api.YdbAuthToken, }, ) if err != nil { @@ -481,7 +481,7 @@ func (b *StorageStatefulSetBuilder) buildContainerArgs() ([]string, []string) { "%s/%s/%s", wellKnownDirForAdditionalSecrets, secret.Name, - v1alpha1.YdbAuthToken, + api.YdbAuthToken, ), ) } diff --git a/samples/remote-rbac.yml b/samples/remote-rbac.yml new file mode 100644 index 00000000..68834d8b --- /dev/null +++ b/samples/remote-rbac.yml @@ -0,0 +1,92 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: yc-dev + namespace: ydb +--- +apiVersion: v1 +kind: Secret +metadata: + name: yc-dev-token + namespace: ydb + annotations: + kubernetes.io/service-account.name: yc-dev +type: kubernetes.io/service-account-token +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: ydb-operator-remote + namespace: ydb +rules: +- apiGroups: + - "" + resources: + - configmaps + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - secrets + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - services + verbs: + - get + - list + - watch +- apiGroups: + - ydb.tech + resources: + - remotedatabasenodesets + - remotestoragenodesets + verbs: + - get + - list + - watch +- apiGroups: + - ydb.tech + resources: + - remotedatabasenodesets + - remotestoragenodesets + verbs: + - update +- apiGroups: + - ydb.tech + resources: + - remotedatabasenodesets/status + - remotestoragenodesets/status + verbs: + - get + - patch + - update +- apiGroups: + - "" + resources: + - events + verbs: + - create + - patch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: ydb-operator-remote-rolebinding + namespace: ydb +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ydb-operator-remote +subjects: +- kind: ServiceAccount + name: yc-dev + namespace: ydb