Merge pull request #1229 from swetharepakula/metrics

Cherry Pick #1222 [Neg crd metrics]

pkg/metrics/features.go

@@ -89,6 +89,7 @@ const (
 	vmIpNeg        = feature("VmIpNEG")
 	vmIpNegLocal   = feature("VmIpNegLocal")
 	vmIpNegCluster = feature("VmIpNegCluster")
+	customNamedNeg = feature("CustomNamedNEG")
 
 	l4ILBService      = feature("L4ILBService")
 	l4ILBGlobalAccess = feature("L4ILBGlobalAccess")

pkg/metrics/metrics.go

@@ -286,6 +286,7 @@ func (im *ControllerMetrics) computeNegMetrics() map[feature]int {
 		vmIpNeg:        0,
 		vmIpNegLocal:   0,
 		vmIpNegCluster: 0,
+		customNamedNeg: 0,
 	}
 
 	for key, negState := range im.negMap {
@@ -295,6 +296,7 @@ func (im *ControllerMetrics) computeNegMetrics() map[feature]int {
 		counts[ingressNeg] += negState.IngressNeg
 		counts[asmNeg] += negState.AsmNeg
 		counts[neg] += negState.AsmNeg + negState.StandaloneNeg + negState.IngressNeg
+		counts[customNamedNeg] += negState.CustomNamedNeg
 		if negState.VmIpNeg != nil {
 			counts[neg] += 1
 			counts[vmIpNeg] += 1

pkg/metrics/metrics_test.go

@@ -789,12 +789,13 @@ func TestComputeNegMetrics(t *testing.T) {
 				vmIpNeg:        0,
 				vmIpNegLocal:   0,
 				vmIpNegCluster: 0,
+				customNamedNeg: 0,
 			},
 		},
 		{
 			"one neg service",
 			[]NegServiceState{
-				newNegState(0, 0, 1, nil),
+				newNegState(0, 0, 1, 0, nil),
 			},
 			map[feature]int{
 				standaloneNeg:  0,
@@ -804,12 +805,13 @@ func TestComputeNegMetrics(t *testing.T) {
 				vmIpNeg:        0,
 				vmIpNegLocal:   0,
 				vmIpNegCluster: 0,
+				customNamedNeg: 0,
 			},
 		},
 		{
 			"vm primary ip neg in traffic policy cluster mode",
 			[]NegServiceState{
-				newNegState(0, 0, 1, &VmIpNegType{trafficPolicyLocal: false}),
+				newNegState(0, 0, 1, 0, &VmIpNegType{trafficPolicyLocal: false}),
 			},
 			map[feature]int{
 				standaloneNeg:  0,
@@ -819,15 +821,32 @@ func TestComputeNegMetrics(t *testing.T) {
 				vmIpNeg:        1,
 				vmIpNegLocal:   0,
 				vmIpNegCluster: 1,
+				customNamedNeg: 0,
+			},
+		},
+		{
+			"custom named neg",
+			[]NegServiceState{
+				newNegState(1, 0, 0, 1, nil),
+			},
+			map[feature]int{
+				standaloneNeg:  1,
+				ingressNeg:     0,
+				asmNeg:         0,
+				neg:            1,
+				vmIpNeg:        0,
+				vmIpNegLocal:   0,
+				vmIpNegCluster: 0,
+				customNamedNeg: 1,
 			},
 		},
 		{
 			"many neg services",
 			[]NegServiceState{
-				newNegState(0, 0, 1, nil),
-				newNegState(0, 1, 0, &VmIpNegType{trafficPolicyLocal: false}),
-				newNegState(5, 0, 0, &VmIpNegType{trafficPolicyLocal: true}),
-				newNegState(5, 3, 2, nil),
+				newNegState(0, 0, 1, 0, nil),
+				newNegState(0, 1, 0, 0, &VmIpNegType{trafficPolicyLocal: false}),
+				newNegState(5, 0, 0, 0, &VmIpNegType{trafficPolicyLocal: true}),
+				newNegState(5, 3, 2, 0, nil),
 			},
 			map[feature]int{
 				standaloneNeg:  10,
@@ -837,6 +856,7 @@ func TestComputeNegMetrics(t *testing.T) {
 				vmIpNeg:        2,
 				vmIpNegLocal:   1,
 				vmIpNegCluster: 1,
+				customNamedNeg: 0,
 			},
 		},
 	} {
@@ -856,12 +876,13 @@ func TestComputeNegMetrics(t *testing.T) {
 	}
 }
 
-func newNegState(standalone, ingress, asm int, negType *VmIpNegType) NegServiceState {
+func newNegState(standalone, ingress, asm, customNamed int, negType *VmIpNegType) NegServiceState {
 	return NegServiceState{
-		IngressNeg:    ingress,
-		StandaloneNeg: standalone,
-		AsmNeg:        asm,
-		VmIpNeg:       negType,
+		IngressNeg:     ingress,
+		StandaloneNeg:  standalone,
+		AsmNeg:         asm,
+		VmIpNeg:        negType,
+		CustomNamedNeg: customNamed,
 	}
 }
 

pkg/metrics/types.go

@@ -37,6 +37,8 @@ type NegServiceState struct {
 	AsmNeg int
 	// VmIpNeg specifies if a service uses GCE_VM_IP NEG.
 	VmIpNeg *VmIpNegType
+	// CustomNamedNeg is the count of standalone negs with custom names
+	CustomNamedNeg int
 }
 
 // VmIpNegType contains whether a GCE_VM_IP NEG is requesting for

pkg/neg/controller.go

@@ -333,7 +333,7 @@ func (c *Controller) nodeWorker() {
 func (c *Controller) processNode() {
 	defer func() {
 		now := c.nodeSyncTracker.Track()
-		metrics.LastSyncTimestamp.WithLabelValues().Set(float64(now.UTC().UnixNano()))
+		metrics.LastSyncTimestamp.Set(float64(now.UTC().UnixNano()))
 	}()
 	c.manager.SyncNodes()
 }
@@ -342,7 +342,7 @@ func (c *Controller) processNode() {
 func (c *Controller) processEndpoint(key string) {
 	defer func() {
 		now := c.syncTracker.Track()
-		metrics.LastSyncTimestamp.WithLabelValues().Set(float64(now.UTC().UnixNano()))
+		metrics.LastSyncTimestamp.Set(float64(now.UTC().UnixNano()))
 	}()
 
 	namespace, name, err := cache.SplitMetaNamespaceKey(key)
@@ -371,7 +371,7 @@ func (c *Controller) serviceWorker() {
 func (c *Controller) processService(key string) error {
 	defer func() {
 		now := c.syncTracker.Track()
-		metrics.LastSyncTimestamp.WithLabelValues().Set(float64(now.UTC().UnixNano()))
+		metrics.LastSyncTimestamp.Set(float64(now.UTC().UnixNano()))
 	}()
 
 	namespace, name, err := cache.SplitMetaNamespaceKey(key)
@@ -403,7 +403,7 @@ func (c *Controller) processService(key string) error {
 		return err
 	}
 	negUsage.IngressNeg = len(svcPortInfoMap)
-	if err := c.mergeStandaloneNEGsPortInfo(service, types.NamespacedName{Namespace: namespace, Name: name}, svcPortInfoMap); err != nil {
+	if err := c.mergeStandaloneNEGsPortInfo(service, types.NamespacedName{Namespace: namespace, Name: name}, svcPortInfoMap, &negUsage); err != nil {
 		return err
 	}
 	negUsage.StandaloneNeg = len(svcPortInfoMap) - negUsage.IngressNeg
@@ -470,7 +470,7 @@ func (c *Controller) mergeIngressPortInfo(service *apiv1.Service, name types.Nam
 }
 
 // mergeStandaloneNEGsPortInfo merge Standalone NEG PortInfo into portInfoMap
-func (c *Controller) mergeStandaloneNEGsPortInfo(service *apiv1.Service, name types.NamespacedName, portInfoMap negtypes.PortInfoMap) error {
+func (c *Controller) mergeStandaloneNEGsPortInfo(service *apiv1.Service, name types.NamespacedName, portInfoMap negtypes.PortInfoMap, negUsage *usage.NegServiceState) error {
 	negAnnotation, foundNEGAnnotation, err := annotations.FromService(service).NEGAnnotation()
 	if err != nil {
 		return err
@@ -502,6 +502,7 @@ func (c *Controller) mergeStandaloneNEGsPortInfo(service *apiv1.Service, name ty
 		if negAnnotation.NEGEnabledForIngress() && len(customNames) != 0 {
 			return fmt.Errorf("configuration for negs in service (%s) is invalid, custom neg name cannot be used with ingress enabled", name.String())
 		}
+		negUsage.CustomNamedNeg = len(customNames)
 
 		if err := portInfoMap.Merge(negtypes.NewPortInfoMap(name.Namespace, name.Name, exposedNegSvcPort, c.namer /*readinessGate*/, true, customNames)); err != nil {
 			return fmt.Errorf("failed to merge service ports exposed as standalone NEGs (%v) into ingress referenced service ports (%v): %v", exposedNegSvcPort, portInfoMap, err)

pkg/neg/manager.go

@@ -21,6 +21,7 @@ import (
 	"fmt"
 	"reflect"
 	"sync"
+	"time"
 
 	"github.com/GoogleCloudPlatform/k8s-cloud-provider/pkg/cloud"
 	"github.com/GoogleCloudPlatform/k8s-cloud-provider/pkg/cloud/meta"
@@ -36,6 +37,7 @@ import (
 	"k8s.io/client-go/tools/record"
 	negv1beta1 "k8s.io/ingress-gce/pkg/apis/svcneg/v1beta1"
 	"k8s.io/ingress-gce/pkg/flags"
+	"k8s.io/ingress-gce/pkg/neg/metrics"
 	"k8s.io/ingress-gce/pkg/neg/readiness"
 	negsyncer "k8s.io/ingress-gce/pkg/neg/syncers"
 	negtypes "k8s.io/ingress-gce/pkg/neg/types"
@@ -108,6 +110,7 @@ func newSyncerManager(namer negtypes.NetworkEndpointGroupNamer, recorder record.
 func (manager *syncerManager) EnsureSyncers(namespace, name string, newPorts negtypes.PortInfoMap) error {
 	manager.mu.Lock()
 	defer manager.mu.Unlock()
+	start := time.Now()
 	key := getServiceKey(namespace, name)
 	currentPorts, ok := manager.svcPortMap[key]
 	if !ok {
@@ -177,7 +180,9 @@ func (manager *syncerManager) EnsureSyncers(namespace, name string, newPorts neg
 			}
 		}
 	}
-	return utilerrors.NewAggregate(errList)
+	err := utilerrors.NewAggregate(errList)
+	metrics.PublishNegManagerProcessMetrics(metrics.SyncProcess, err, start)
+	return err
 }
 
 // StopSyncer stops all syncers for the input service.
@@ -236,6 +241,7 @@ func (manager *syncerManager) ShutDown() {
 func (manager *syncerManager) GC() error {
 	klog.V(2).Infof("Start NEG garbage collection.")
 	defer klog.V(2).Infof("NEG garbage collection finished.")
+	start := time.Now()
 	// Garbage collect Syncers
 	manager.garbageCollectSyncer()
 
@@ -249,6 +255,7 @@ func (manager *syncerManager) GC() error {
 	if err != nil {
 		err = fmt.Errorf("failed to garbage collect negs: %v", err)
 	}
+	metrics.PublishNegManagerProcessMetrics(metrics.GCProcess, err, start)
 	return err
 }