Track k8s server error

kubernetes · May 2, 2023 · fa9bd69 · fa9bd69
1 parent e019327
commit fa9bd69
Show file tree

Hide file tree

Showing 4 changed files with 20 additions and 5 deletions.
diff --git a/pkg/neg/controller.go b/pkg/neg/controller.go
@@ -677,7 +677,9 @@ func (c *Controller) syncNegStatusAnnotation(namespace, name string, portMap neg
 			newSvcObjectMeta := service.ObjectMeta.DeepCopy()
 			delete(newSvcObjectMeta.Annotations, annotations.NEGStatusKey)
 			c.logger.V(2).Info("Removing NEG status annotation from service", "service", klog.KRef(namespace, name))
-			return patch.PatchServiceObjectMetadata(c.client.CoreV1(), service, *newSvcObjectMeta)
+			err := patch.PatchServiceObjectMetadata(c.client.CoreV1(), service, *newSvcObjectMeta)
+			metrics.PublishNegSyncErrorCountMetrics(err)
+			return err
 		}
 		// service doesn't have the expose NEG annotation and doesn't need update
 		return nil
@@ -699,7 +701,9 @@ func (c *Controller) syncNegStatusAnnotation(namespace, name string, portMap neg
 	}
 	newSvcObjectMeta.Annotations[annotations.NEGStatusKey] = annotation
 	c.logger.V(2).Info("Updating NEG visibility annotation on service", "annotation", annotation, "service", klog.KRef(namespace, name))
-	return patch.PatchServiceObjectMetadata(c.client.CoreV1(), service, *newSvcObjectMeta)
+	err = patch.PatchServiceObjectMetadata(c.client.CoreV1(), service, *newSvcObjectMeta)
+	metrics.PublishNegSyncErrorCountMetrics(err)
+	return err
 }
 
 func (c *Controller) handleErr(err error, key interface{}) {

diff --git a/pkg/neg/manager.go b/pkg/neg/manager.go
@@ -458,6 +458,7 @@ func (manager *syncerManager) ensureDeleteSvcNegCR(namespace, negName string) er
 
 	if neg.GetDeletionTimestamp().IsZero() {
 		if err = manager.svcNegClient.NetworkingV1beta1().ServiceNetworkEndpointGroups(namespace).Delete(context.Background(), negName, metav1.DeleteOptions{}); err != nil {
+			metrics.PublishNegSyncErrorCountMetrics(err)
 			return fmt.Errorf("errored while deleting neg cr %s/%s: %w", negName, namespace, err)
 		}
 		manager.logger.V(2).Info("Deleted neg cr", "svcneg", klog.KRef(namespace, negName))
@@ -747,6 +748,7 @@ func (manager *syncerManager) ensureSvcNegCR(svcKey serviceKey, portInfo negtype
 	if !exists {
 		// Neg does not exist so create it
 		_, err = manager.svcNegClient.NetworkingV1beta1().ServiceNetworkEndpointGroups(svcKey.namespace).Create(context.Background(), &newCR, metav1.CreateOptions{})
+		metrics.PublishNegSyncErrorCountMetrics(err)
 		manager.logger.V(2).Info("Created ServiceNetworkEndpointGroup CR for neg", "svcneg", klog.KRef(svcKey.namespace, portInfo.NegName))
 		return err
 	}
@@ -761,6 +763,7 @@ func (manager *syncerManager) ensureSvcNegCR(svcKey serviceKey, portInfo negtype
 
 	if needUpdate {
 		_, err = manager.svcNegClient.NetworkingV1beta1().ServiceNetworkEndpointGroups(svcKey.namespace).Update(context.Background(), negCR, metav1.UpdateOptions{})
+		metrics.PublishNegSyncErrorCountMetrics(err)
 		return err
 	}
 	return nil
@@ -810,7 +813,9 @@ func deleteSvcNegCR(svcNegClient svcnegclient.Interface, negCR *negv1beta1.Servi
 	// If CR does not have a deletion timestamp, delete
 	if negCR.GetDeletionTimestamp().IsZero() {
 		logger.V(2).Info("Deleting ServiceNetworkEndpointGroup CR", "svcneg", klog.KRef(negCR.Namespace, negCR.Name))
-		return svcNegClient.NetworkingV1beta1().ServiceNetworkEndpointGroups(negCR.Namespace).Delete(context.Background(), negCR.Name, metav1.DeleteOptions{})
+		err := svcNegClient.NetworkingV1beta1().ServiceNetworkEndpointGroups(negCR.Namespace).Delete(context.Background(), negCR.Name, metav1.DeleteOptions{})
+		metrics.PublishNegSyncErrorCountMetrics(err)
+		return err
 	}
 	return nil
 }
@@ -822,7 +827,9 @@ func patchNegStatus(svcNegClient svcnegclient.Interface, oldNeg, newNeg negv1bet
 		return nil, fmt.Errorf("failed to prepare patch bytes: %s", err)
 	}
 
-	return svcNegClient.NetworkingV1beta1().ServiceNetworkEndpointGroups(oldNeg.Namespace).Patch(context.Background(), oldNeg.Name, types.MergePatchType, patchBytes, metav1.PatchOptions{})
+	neg, err := svcNegClient.NetworkingV1beta1().ServiceNetworkEndpointGroups(oldNeg.Namespace).Patch(context.Background(), oldNeg.Name, types.MergePatchType, patchBytes, metav1.PatchOptions{})
+	metrics.PublishNegSyncErrorCountMetrics(err)
+	return neg, err
 }
 
 // getSyncerKey encodes a service namespace, name, service port and targetPort into a string key

diff --git a/pkg/neg/readiness/utils.go b/pkg/neg/readiness/utils.go
@@ -25,6 +25,7 @@ import (
 	"k8s.io/apimachinery/pkg/types"
 	clientset "k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/tools/cache"
+	"k8s.io/ingress-gce/pkg/neg/metrics"
 	negtypes "k8s.io/ingress-gce/pkg/neg/types"
 	"k8s.io/ingress-gce/pkg/neg/types/shared"
 	"k8s.io/ingress-gce/pkg/utils/patch"
@@ -106,6 +107,7 @@ func SetNegReadinessConditionStatus(pod *v1.Pod, condition v1.PodCondition) {
 func patchPodStatus(c clientset.Interface, namespace, name string, patchBytes []byte) (*v1.Pod, []byte, error) {
 	updatedPod, err := c.CoreV1().Pods(namespace).Patch(context.TODO(), name, types.StrategicMergePatchType, patchBytes, metav1.PatchOptions{}, "status")
 	if err != nil {
+		metrics.PublishNegSyncErrorCountMetrics(err)
 		return nil, nil, fmt.Errorf("failed to patch status %q for pod %q/%q: %v", patchBytes, namespace, name, err)
 	}
 	return updatedPod, patchBytes, nil

diff --git a/pkg/neg/syncers/transaction.go b/pkg/neg/syncers/transaction.go
@@ -822,7 +822,9 @@ func patchNegStatus(svcNegClient svcnegclient.Interface, oldStatus, newStatus ne
 		return nil, fmt.Errorf("failed to prepare patch bytes: %w", err)
 	}
 
-	return svcNegClient.NetworkingV1beta1().ServiceNetworkEndpointGroups(namespace).Patch(context.Background(), negName, types.MergePatchType, patchBytes, metav1.PatchOptions{})
+	neg, err := svcNegClient.NetworkingV1beta1().ServiceNetworkEndpointGroups(namespace).Patch(context.Background(), negName, types.MergePatchType, patchBytes, metav1.PatchOptions{})
+	metrics.PublishNegSyncErrorCountMetrics(err)
+	return neg, err
 }
 
 // ensureCondition will update the condition on the neg object if necessary