Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Introduce a nutanix prism client cache #425

Closed
wants to merge 2 commits into from
Closed
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Next Next commit
Introduce a nutanix prism client cache
The cache stores a prismgoclient.V3 client instance for each NutanixCluster instance.
The cache is shared between nutanixcluster and nutanixmachine controllers.
  • Loading branch information
thunderboltsid committed May 3, 2024
commit 9b21e06f6f8afc3df83ca314387776f06e7f128d
8 changes: 8 additions & 0 deletions api/v1beta1/nutanixcluster_types.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,9 +17,11 @@ limitations under the License.
package v1beta1

import (
"cmp"
"fmt"

credentialTypes "github.com/nutanix-cloud-native/prism-go-client/environment/credentials"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
capiv1 "sigs.k8s.io/cluster-api/api/v1beta1"
"sigs.k8s.io/cluster-api/errors"
Expand Down Expand Up @@ -118,6 +120,12 @@ func (ncl *NutanixCluster) GetPrismCentralCredentialRef() (*credentialTypes.Nuta
return prismCentralInfo.CredentialRef, nil
}

// GetNamespacedName returns the namespaced name of the NutanixCluster.
func (ncl *NutanixCluster) GetNamespacedName() string {
namespace := cmp.Or(ncl.Namespace, corev1.NamespaceDefault)
return fmt.Sprintf("%s/%s", namespace, ncl.Name)
}

// +kubebuilder:object:root=true

// NutanixClusterList contains a list of NutanixCluster
Expand Down
105 changes: 96 additions & 9 deletions controllers/helpers.go
Original file line number Diff line number Diff line change
Expand Up @@ -25,9 +25,16 @@ import (
"github.com/google/uuid"
"github.com/nutanix-cloud-native/prism-go-client/utils"
nutanixClientV3 "github.com/nutanix-cloud-native/prism-go-client/v3"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"
coreinformers "k8s.io/client-go/informers/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
v1 "k8s.io/client-go/informers/core/v1"
capiv1 "sigs.k8s.io/cluster-api/api/v1beta1"
capiutil "sigs.k8s.io/cluster-api/util"
"sigs.k8s.io/cluster-api/util/conditions"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
ctrlutil "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"

infrav1 "github.com/nutanix-cloud-native/cluster-api-provider-nutanix/api/v1beta1"
nutanixClient "github.com/nutanix-cloud-native/cluster-api-provider-nutanix/pkg/client"
Expand All @@ -44,14 +51,6 @@ const (
gpuUnused = "UNUSED"
)

// CreateNutanixClient creates a new Nutanix client from the environment
func CreateNutanixClient(ctx context.Context, secretInformer coreinformers.SecretInformer, cmInformer coreinformers.ConfigMapInformer, nutanixCluster *infrav1.NutanixCluster) (*nutanixClientV3.Client, error) {
log := ctrl.LoggerFrom(ctx)
log.V(1).Info("creating nutanix client")
helper := nutanixClient.NewHelper(secretInformer, cmInformer)
return helper.BuildClientForNutanixClusterWithFallback(ctx, nutanixCluster)
}

// DeleteVM deletes a VM and is invoked by the NutanixMachineReconciler
func DeleteVM(ctx context.Context, client *nutanixClientV3.Client, vmName, vmUUID string) (string, error) {
log := ctrl.LoggerFrom(ctx)
Expand Down Expand Up @@ -741,3 +740,91 @@ func GetGPUsForPE(ctx context.Context, client *nutanixClientV3.Client, peUUID st
}
return gpus, nil
}

func reconcileCredentialRef(ctx context.Context, k8sClient client.Client, nutanixCluster *infrav1.NutanixCluster) error {
log := ctrl.LoggerFrom(ctx)
credentialRef, err := getPrismCentralCredentialRefForCluster(nutanixCluster)
if err != nil {
return err
}

secret := &corev1.Secret{}
if credentialRef == nil {
return nil
}

log.V(1).Info(fmt.Sprintf("credential ref is kind Secret for cluster %s", nutanixCluster.Name))
secretKey := client.ObjectKey{
Namespace: nutanixCluster.Namespace,
Name: credentialRef.Name,
}

if err := k8sClient.Get(ctx, secretKey, secret); err != nil {
errorMsg := fmt.Errorf("error occurred while fetching cluster %s secret for credential ref: %v", nutanixCluster.Name, err)
log.Error(errorMsg, "error occurred fetching cluster")
return errorMsg
}

// Check if ownerRef is already set on nutanixCluster object
if !capiutil.IsOwnedByObject(secret, nutanixCluster) {
// Check if another nutanixCluster already has set ownerRef. Secret can only be owned by one nutanixCluster object
if capiutil.HasOwner(secret.OwnerReferences, infrav1.GroupVersion.String(), []string{
nutanixCluster.Kind,
}) {
return fmt.Errorf("secret %s already owned by another nutanixCluster object", secret.Name)
}
// Set nutanixCluster ownerRef on the secret
secret.OwnerReferences = capiutil.EnsureOwnerRef(secret.OwnerReferences, metav1.OwnerReference{
APIVersion: infrav1.GroupVersion.String(),
Kind: nutanixCluster.Kind,
UID: nutanixCluster.UID,
Name: nutanixCluster.Name,
})
}

if !ctrlutil.ContainsFinalizer(secret, infrav1.NutanixClusterCredentialFinalizer) {
ctrlutil.AddFinalizer(secret, infrav1.NutanixClusterCredentialFinalizer)
}

err = k8sClient.Update(ctx, secret)
if err != nil {
errorMsg := fmt.Errorf("failed to update secret for cluster %s: %v", nutanixCluster.Name, err)
log.Error(errorMsg, "failed to update secret")
return errorMsg
}

return nil
}

func getPrismCentralClientForCluster(ctx context.Context, k8sClient client.Client, cluster *infrav1.NutanixCluster, capiCluster *capiv1.Cluster,
secretInformer v1.SecretInformer, mapInformer v1.ConfigMapInformer) (*nutanixClientV3.Client, error) {
log := ctrl.LoggerFrom(ctx)

if err := reconcileCredentialRef(ctx, k8sClient, cluster); err != nil {
log.Error(err, fmt.Sprintf("error occurred while reconciling credential ref for cluster %s", capiCluster.Name))
conditions.MarkFalse(cluster, infrav1.CredentialRefSecretOwnerSetCondition, infrav1.CredentialRefSecretOwnerSetFailed, capiv1.ConditionSeverityError, err.Error())
return nil, err
}
conditions.MarkTrue(cluster, infrav1.CredentialRefSecretOwnerSetCondition)

clientHelper := nutanixClient.NewHelper(secretInformer, mapInformer)
mep, err := clientHelper.BuildManagementEndpoint(ctx, cluster)
if err != nil {
log.Error(err, fmt.Sprintf("error occurred while getting management endpoint for cluster %q", cluster.GetNamespacedName()))
conditions.MarkFalse(cluster, infrav1.PrismCentralClientCondition, infrav1.PrismCentralClientInitializationFailed, capiv1.ConditionSeverityError, err.Error())
return nil, err
}

v3Client, err := nutanixClient.NutanixClientCache.GetOrCreate(&nutanixClient.CacheParams{
NutanixCluster: cluster,
PrismManagementEndpoint: mep,
})
if err != nil {
log.Error(err, "error occurred while getting nutanix prism client from cache")
conditions.MarkFalse(cluster, infrav1.PrismCentralClientCondition, infrav1.PrismCentralClientInitializationFailed, capiv1.ConditionSeverityError, err.Error())
return nil, fmt.Errorf("nutanix prism client error: %w", err)
}

conditions.MarkTrue(cluster, infrav1.PrismCentralClientCondition)
return v3Client, nil
}
63 changes: 7 additions & 56 deletions controllers/nutanixcluster_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,6 @@ import (
credentialTypes "github.com/nutanix-cloud-native/prism-go-client/environment/credentials"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
apitypes "k8s.io/apimachinery/pkg/types"
kerrors "k8s.io/apimachinery/pkg/util/errors"
Expand All @@ -44,6 +43,7 @@ import (
"sigs.k8s.io/controller-runtime/pkg/source"

infrav1 "github.com/nutanix-cloud-native/cluster-api-provider-nutanix/api/v1beta1"
nutanixClient "github.com/nutanix-cloud-native/cluster-api-provider-nutanix/pkg/client"
nctx "github.com/nutanix-cloud-native/cluster-api-provider-nutanix/pkg/context"
)

Expand Down Expand Up @@ -170,20 +170,11 @@ func (r *NutanixClusterReconciler) Reconcile(ctx context.Context, req ctrl.Reque
log.V(1).Info(fmt.Sprintf("Patched NutanixCluster. Status: %+v", cluster.Status))
}()

err = r.reconcileCredentialRef(ctx, cluster)
v3Client, err := getPrismCentralClientForCluster(ctx, r.Client, cluster, capiCluster, r.SecretInformer, r.ConfigMapInformer)
if err != nil {
log.Error(err, fmt.Sprintf("error occurred while reconciling credential ref for cluster %s", capiCluster.Name))
conditions.MarkFalse(cluster, infrav1.CredentialRefSecretOwnerSetCondition, infrav1.CredentialRefSecretOwnerSetFailed, capiv1.ConditionSeverityError, err.Error())
log.Error(err, "error occurred while fetching prism central client")
return reconcile.Result{}, err
}
conditions.MarkTrue(cluster, infrav1.CredentialRefSecretOwnerSetCondition)

v3Client, err := CreateNutanixClient(ctx, r.SecretInformer, r.ConfigMapInformer, cluster)
if err != nil {
conditions.MarkFalse(cluster, infrav1.PrismCentralClientCondition, infrav1.PrismCentralClientInitializationFailed, capiv1.ConditionSeverityError, err.Error())
return ctrl.Result{Requeue: true}, fmt.Errorf("nutanix client error: %v", err)
}
conditions.MarkTrue(cluster, infrav1.PrismCentralClientCondition)

rctx := &nctx.ClusterContext{
Context: ctx,
Expand Down Expand Up @@ -224,6 +215,10 @@ func (r *NutanixClusterReconciler) reconcileDelete(rctx *nctx.ClusterContext) (r
return reconcile.Result{}, err
}

// delete the client from the cache
log.Info(fmt.Sprintf("deleting nutanix prism client for cluster %s from cache", rctx.NutanixCluster.GetNamespacedName()))
nutanixClient.NutanixClientCache.Delete(&nutanixClient.CacheParams{NutanixCluster: rctx.NutanixCluster})

err = r.reconcileCredentialRefDelete(rctx.Context, rctx.NutanixCluster)
if err != nil {
log.Error(err, fmt.Sprintf("error occurred while reconciling credential ref deletion for cluster %s", rctx.Cluster.Name))
Expand Down Expand Up @@ -343,50 +338,6 @@ func (r *NutanixClusterReconciler) reconcileCredentialRefDelete(ctx context.Cont
return nil
}

func (r *NutanixClusterReconciler) reconcileCredentialRef(ctx context.Context, nutanixCluster *infrav1.NutanixCluster) error {
log := ctrl.LoggerFrom(ctx)
credentialRef, err := getPrismCentralCredentialRefForCluster(nutanixCluster)
if err != nil {
return err
}
if credentialRef == nil {
return nil
}
log.V(1).Info(fmt.Sprintf("credential ref is kind Secret for cluster %s", nutanixCluster.Name))
secret := &corev1.Secret{}
secretKey := client.ObjectKey{
Namespace: nutanixCluster.Namespace,
Name: credentialRef.Name,
}
err = r.Client.Get(ctx, secretKey, secret)
if err != nil {
errorMsg := fmt.Errorf("error occurred while fetching cluster %s secret for credential ref: %v", nutanixCluster.Name, err)
log.Error(errorMsg, "error occurred fetching cluster")
return errorMsg
}
if !capiutil.IsOwnedByObject(secret, nutanixCluster) {
if len(secret.GetOwnerReferences()) > 0 {
return fmt.Errorf("secret for cluster %s already has other owners set", nutanixCluster.Name)
}
secret.SetOwnerReferences([]metav1.OwnerReference{{
APIVersion: infrav1.GroupVersion.String(),
Kind: nutanixCluster.Kind,
UID: nutanixCluster.UID,
Name: nutanixCluster.Name,
}})
}
if !ctrlutil.ContainsFinalizer(secret, infrav1.NutanixClusterCredentialFinalizer) {
ctrlutil.AddFinalizer(secret, infrav1.NutanixClusterCredentialFinalizer)
}
err = r.Client.Update(ctx, secret)
if err != nil {
errorMsg := fmt.Errorf("failed to update secret for cluster %s: %v", nutanixCluster.Name, err)
log.Error(errorMsg, "failed to update secret")
return errorMsg
}
return nil
}

// getPrismCentralCredentialRefForCluster calls nutanixCluster.GetPrismCentralCredentialRef() function
// and returns an error if nutanixCluster is nil
func getPrismCentralCredentialRefForCluster(nutanixCluster *infrav1.NutanixCluster) (*credentialTypes.NutanixCredentialReference, error) {
Expand Down
Loading