From fa9bd6910abd00ab89be64b431c740e962388659 Mon Sep 17 00:00:00 2001 From: David Cheung Date: Tue, 2 May 2023 21:16:06 +0000 Subject: [PATCH] Track k8s server error --- pkg/neg/controller.go | 8 ++++++-- pkg/neg/manager.go | 11 +++++++++-- pkg/neg/readiness/utils.go | 2 ++ pkg/neg/syncers/transaction.go | 4 +++- 4 files changed, 20 insertions(+), 5 deletions(-) diff --git a/pkg/neg/controller.go b/pkg/neg/controller.go index 49c1a5933a..5b34dcca3b 100644 --- a/pkg/neg/controller.go +++ b/pkg/neg/controller.go @@ -677,7 +677,9 @@ func (c *Controller) syncNegStatusAnnotation(namespace, name string, portMap neg newSvcObjectMeta := service.ObjectMeta.DeepCopy() delete(newSvcObjectMeta.Annotations, annotations.NEGStatusKey) c.logger.V(2).Info("Removing NEG status annotation from service", "service", klog.KRef(namespace, name)) - return patch.PatchServiceObjectMetadata(c.client.CoreV1(), service, *newSvcObjectMeta) + err := patch.PatchServiceObjectMetadata(c.client.CoreV1(), service, *newSvcObjectMeta) + metrics.PublishNegSyncErrorCountMetrics(err) + return err } // service doesn't have the expose NEG annotation and doesn't need update return nil @@ -699,7 +701,9 @@ func (c *Controller) syncNegStatusAnnotation(namespace, name string, portMap neg } newSvcObjectMeta.Annotations[annotations.NEGStatusKey] = annotation c.logger.V(2).Info("Updating NEG visibility annotation on service", "annotation", annotation, "service", klog.KRef(namespace, name)) - return patch.PatchServiceObjectMetadata(c.client.CoreV1(), service, *newSvcObjectMeta) + err = patch.PatchServiceObjectMetadata(c.client.CoreV1(), service, *newSvcObjectMeta) + metrics.PublishNegSyncErrorCountMetrics(err) + return err } func (c *Controller) handleErr(err error, key interface{}) { diff --git a/pkg/neg/manager.go b/pkg/neg/manager.go index 6a9b87c681..3e6b052bd1 100644 --- a/pkg/neg/manager.go +++ b/pkg/neg/manager.go @@ -458,6 +458,7 @@ func (manager *syncerManager) ensureDeleteSvcNegCR(namespace, negName string) er if neg.GetDeletionTimestamp().IsZero() { if err = manager.svcNegClient.NetworkingV1beta1().ServiceNetworkEndpointGroups(namespace).Delete(context.Background(), negName, metav1.DeleteOptions{}); err != nil { + metrics.PublishNegSyncErrorCountMetrics(err) return fmt.Errorf("errored while deleting neg cr %s/%s: %w", negName, namespace, err) } manager.logger.V(2).Info("Deleted neg cr", "svcneg", klog.KRef(namespace, negName)) @@ -747,6 +748,7 @@ func (manager *syncerManager) ensureSvcNegCR(svcKey serviceKey, portInfo negtype if !exists { // Neg does not exist so create it _, err = manager.svcNegClient.NetworkingV1beta1().ServiceNetworkEndpointGroups(svcKey.namespace).Create(context.Background(), &newCR, metav1.CreateOptions{}) + metrics.PublishNegSyncErrorCountMetrics(err) manager.logger.V(2).Info("Created ServiceNetworkEndpointGroup CR for neg", "svcneg", klog.KRef(svcKey.namespace, portInfo.NegName)) return err } @@ -761,6 +763,7 @@ func (manager *syncerManager) ensureSvcNegCR(svcKey serviceKey, portInfo negtype if needUpdate { _, err = manager.svcNegClient.NetworkingV1beta1().ServiceNetworkEndpointGroups(svcKey.namespace).Update(context.Background(), negCR, metav1.UpdateOptions{}) + metrics.PublishNegSyncErrorCountMetrics(err) return err } return nil @@ -810,7 +813,9 @@ func deleteSvcNegCR(svcNegClient svcnegclient.Interface, negCR *negv1beta1.Servi // If CR does not have a deletion timestamp, delete if negCR.GetDeletionTimestamp().IsZero() { logger.V(2).Info("Deleting ServiceNetworkEndpointGroup CR", "svcneg", klog.KRef(negCR.Namespace, negCR.Name)) - return svcNegClient.NetworkingV1beta1().ServiceNetworkEndpointGroups(negCR.Namespace).Delete(context.Background(), negCR.Name, metav1.DeleteOptions{}) + err := svcNegClient.NetworkingV1beta1().ServiceNetworkEndpointGroups(negCR.Namespace).Delete(context.Background(), negCR.Name, metav1.DeleteOptions{}) + metrics.PublishNegSyncErrorCountMetrics(err) + return err } return nil } @@ -822,7 +827,9 @@ func patchNegStatus(svcNegClient svcnegclient.Interface, oldNeg, newNeg negv1bet return nil, fmt.Errorf("failed to prepare patch bytes: %s", err) } - return svcNegClient.NetworkingV1beta1().ServiceNetworkEndpointGroups(oldNeg.Namespace).Patch(context.Background(), oldNeg.Name, types.MergePatchType, patchBytes, metav1.PatchOptions{}) + neg, err := svcNegClient.NetworkingV1beta1().ServiceNetworkEndpointGroups(oldNeg.Namespace).Patch(context.Background(), oldNeg.Name, types.MergePatchType, patchBytes, metav1.PatchOptions{}) + metrics.PublishNegSyncErrorCountMetrics(err) + return neg, err } // getSyncerKey encodes a service namespace, name, service port and targetPort into a string key diff --git a/pkg/neg/readiness/utils.go b/pkg/neg/readiness/utils.go index 5555a76422..2e87cd6165 100644 --- a/pkg/neg/readiness/utils.go +++ b/pkg/neg/readiness/utils.go @@ -25,6 +25,7 @@ import ( "k8s.io/apimachinery/pkg/types" clientset "k8s.io/client-go/kubernetes" "k8s.io/client-go/tools/cache" + "k8s.io/ingress-gce/pkg/neg/metrics" negtypes "k8s.io/ingress-gce/pkg/neg/types" "k8s.io/ingress-gce/pkg/neg/types/shared" "k8s.io/ingress-gce/pkg/utils/patch" @@ -106,6 +107,7 @@ func SetNegReadinessConditionStatus(pod *v1.Pod, condition v1.PodCondition) { func patchPodStatus(c clientset.Interface, namespace, name string, patchBytes []byte) (*v1.Pod, []byte, error) { updatedPod, err := c.CoreV1().Pods(namespace).Patch(context.TODO(), name, types.StrategicMergePatchType, patchBytes, metav1.PatchOptions{}, "status") if err != nil { + metrics.PublishNegSyncErrorCountMetrics(err) return nil, nil, fmt.Errorf("failed to patch status %q for pod %q/%q: %v", patchBytes, namespace, name, err) } return updatedPod, patchBytes, nil diff --git a/pkg/neg/syncers/transaction.go b/pkg/neg/syncers/transaction.go index 878f190a8f..c72c431483 100644 --- a/pkg/neg/syncers/transaction.go +++ b/pkg/neg/syncers/transaction.go @@ -822,7 +822,9 @@ func patchNegStatus(svcNegClient svcnegclient.Interface, oldStatus, newStatus ne return nil, fmt.Errorf("failed to prepare patch bytes: %w", err) } - return svcNegClient.NetworkingV1beta1().ServiceNetworkEndpointGroups(namespace).Patch(context.Background(), negName, types.MergePatchType, patchBytes, metav1.PatchOptions{}) + neg, err := svcNegClient.NetworkingV1beta1().ServiceNetworkEndpointGroups(namespace).Patch(context.Background(), negName, types.MergePatchType, patchBytes, metav1.PatchOptions{}) + metrics.PublishNegSyncErrorCountMetrics(err) + return neg, err } // ensureCondition will update the condition on the neg object if necessary