From d2af05f78b9f85a80b58d8c5593e635c946067a1 Mon Sep 17 00:00:00 2001
From: David Cheung <cheungdavid@google.com>
Date: Thu, 26 Jan 2023 18:14:07 +0000
Subject: [PATCH] Add metrics for sync result

Created sync metrics collector to collect sync related metrics. Added
metrics to collect the cumulative count of sync results. Specified
metrics emit interval in flags.
---
 pkg/neg/metrics/neg_metrics_collector.go | 23 ++++++++++++++++-------
 pkg/neg/syncers/transaction.go           |  9 +++++++++
 2 files changed, 25 insertions(+), 7 deletions(-)

diff --git a/pkg/neg/metrics/neg_metrics_collector.go b/pkg/neg/metrics/neg_metrics_collector.go
index 9986234bde..8be9c91ab0 100644
--- a/pkg/neg/metrics/neg_metrics_collector.go
+++ b/pkg/neg/metrics/neg_metrics_collector.go
@@ -30,10 +30,6 @@ import (
 const (
 	syncResultLabel = "result"
 	syncResultKey   = "sync_result"
-
-	EPSDup     = "EPSWithDuplicateEP"
-	EPSMissing = "EPSWithMissingEP"
-	EPSTotal   = "TotalEPS"
 )
 
 var (
@@ -69,8 +65,22 @@ func init() {
 // NewNEGMetricsCollector initializes SyncerMetrics and starts a go routine to compute and export metrics periodically.
 func NewNegMetricsCollector(exportInterval time.Duration) *SyncerMetrics {
 	return &SyncerMetrics{
-		countSinceLastExport: make(map[syncError]int),
-		metricsInterval:      exportInterval,
+		countSinceLastExport: map[syncError]int{
+			ErrEPCountsDiffer:         0,
+			ErrEPMissingNodeName:      0,
+			ErrEPMissingZone:          0,
+			ErrInvalidEPAttach:        0,
+			ErrInvalidEPDetach:        0,
+			ErrEPSEndpointCountZero:   0,
+			ErrEPCalculationCountZero: 0,
+			ErrNegNotFound:            0,
+			ErrCurrentEPNotFound:      0,
+			ErrEPSNotFound:            0,
+			ErrNodeNotFound:           0,
+			ErrOtherError:             0,
+			Success:                   0,
+		},
+		metricsInterval: exportInterval,
 	}
 }
 
@@ -98,7 +108,6 @@ func (im *SyncerMetrics) Run(stopCh <-chan struct{}) {
 	klog.V(3).Infof("Syncer Metrics initialized. Metrics will be exported at an interval of %v", im.metricsInterval)
 	// Compute and export metrics periodically.
 	go func() {
-		// Wait for ingress states to be populated in the cache before computing metrics.
 		time.Sleep(im.metricsInterval)
 		wait.Until(im.export, im.metricsInterval, stopCh)
 	}()
diff --git a/pkg/neg/syncers/transaction.go b/pkg/neg/syncers/transaction.go
index c3963aed71..8773aa3a26 100644
--- a/pkg/neg/syncers/transaction.go
+++ b/pkg/neg/syncers/transaction.go
@@ -104,6 +104,9 @@ type transactionSyncer struct {
 	// 4. Endpoint count from EPS or calculated endpoint list is 0
 	// Need to grab syncLock first for any reads or writes based on this value
 	inError bool
+
+	// syncCollector collect sync related metrics
+	syncCollector metrics.SyncerMetricsCollector
 }
 
 func NewTransactionSyncer(
@@ -121,6 +124,7 @@ func NewTransactionSyncer(
 	epc negtypes.NetworkEndpointsCalculator,
 	kubeSystemUID string,
 	svcNegClient svcnegclient.Interface,
+	syncerMetrics *metrics.SyncerMetrics,
 	customName bool,
 	enableEndpointSlices bool,
 	log klog.Logger) negtypes.NegSyncer {
@@ -145,6 +149,7 @@ func NewTransactionSyncer(
 		reflector:            reflector,
 		kubeSystemUID:        kubeSystemUID,
 		svcNegClient:         svcNegClient,
+		syncCollector:        syncerMetrics,
 		customName:           customName,
 		enableEndpointSlices: enableEndpointSlices,
 		inError:              false,
@@ -192,6 +197,9 @@ func (s *transactionSyncer) syncInternal() error {
 
 	s.updateStatus(err)
 	metrics.PublishNegSyncMetrics(string(s.NegSyncerKey.NegType), string(s.endpointsCalculator.Mode()), err, start)
+	if s.enableEndpointSlices {
+		s.syncCollector.UpdateSyncer(s.NegSyncerKey, err)
+	}
 	return err
 }
 
@@ -510,6 +518,7 @@ func (s *transactionSyncer) operationInternal(operation transactionOp, zone stri
 			if operation == detachOp {
 				err = fmt.Errorf("%w, reason: %v", metrics.ErrInvalidEPDetach, err)
 			}
+			s.syncCollector.UpdateSyncer(s.NegSyncerKey, err)
 		}
 	}