From 0e76ed7e7c41dc1b1ba162aa644810eae3629821 Mon Sep 17 00:00:00 2001 From: mmamczur Date: Wed, 25 Jan 2023 17:24:55 +0100 Subject: [PATCH] Export various stats about services in the metrics exported by this controller. --- cmd/glbc/main.go | 7 + pkg/flags/flags.go | 2 + pkg/servicemetrics/servicemetrics.go | 369 +++++++++++ pkg/servicemetrics/servicemetrics_test.go | 731 ++++++++++++++++++++++ 4 files changed, 1109 insertions(+) create mode 100644 pkg/servicemetrics/servicemetrics.go create mode 100644 pkg/servicemetrics/servicemetrics_test.go diff --git a/cmd/glbc/main.go b/cmd/glbc/main.go index fc221d2fed..a55b642227 100644 --- a/cmd/glbc/main.go +++ b/cmd/glbc/main.go @@ -31,6 +31,7 @@ import ( "k8s.io/ingress-gce/pkg/l4lb" "k8s.io/ingress-gce/pkg/psc" "k8s.io/ingress-gce/pkg/serviceattachment" + "k8s.io/ingress-gce/pkg/servicemetrics" "k8s.io/ingress-gce/pkg/svcneg" "k8s.io/klog/v2" @@ -293,6 +294,12 @@ func runControllers(ctx *ingctx.ControllerContext) { klog.V(0).Infof("PSC Controller started") } + if flags.F.EnableServiceMetrics { + metricsController := servicemetrics.NewController(ctx, flags.F.MetricsExportInterval, stopCh) + go metricsController.Run() + klog.V(0).Infof("Service Metrics Controller started") + } + var zoneGetter negtypes.ZoneGetter zoneGetter = lbc.Translator // In NonGCP mode, use the zone specified in gce.conf directly. diff --git a/pkg/flags/flags.go b/pkg/flags/flags.go index 5c1e340831..a0ca5df732 100644 --- a/pkg/flags/flags.go +++ b/pkg/flags/flags.go @@ -111,6 +111,7 @@ var ( EnablePinhole bool EnableL4ILBDualStack bool EnableMultipleIGs bool + EnableServiceMetrics bool MaxIGSize int }{ GCERateLimitScale: 1.0, @@ -244,6 +245,7 @@ L7 load balancing. CSV values accepted. Example: -node-port-ranges=80,8080,400-5 flag.BoolVar(&F.RunIngressController, "run-ingress-controller", true, `Optional, whether or not to run IngressController as part of glbc. If set to false, ingress resources will not be processed. Only the L4 Service controller will be run, if that flag is set to true.`) flag.BoolVar(&F.RunL4Controller, "run-l4-controller", false, `Optional, whether or not to run L4 Service Controller as part of glbc. If set to true, services of Type:LoadBalancer with Internal annotation will be processed by this controller.`) flag.BoolVar(&F.RunL4NetLBController, "run-l4-netlb-controller", false, `Optional, f enabled then the L4NetLbController will be run.`) + flag.BoolVar(&F.EnableServiceMetrics, "enable-service-metrics", false, `Optional, if enabled then the service metrics controller will be run.`) flag.BoolVar(&F.EnableBackendConfigHealthCheck, "enable-backendconfig-healthcheck", false, "Enable configuration of HealthChecks from the BackendConfig") flag.BoolVar(&F.EnablePSC, "enable-psc", false, "Enable PSC controller") flag.BoolVar(&F.EnableIngressGAFields, "enable-ingress-ga-fields", false, "Enable using Ingress Class GA features") diff --git a/pkg/servicemetrics/servicemetrics.go b/pkg/servicemetrics/servicemetrics.go new file mode 100644 index 0000000000..e86729e638 --- /dev/null +++ b/pkg/servicemetrics/servicemetrics.go @@ -0,0 +1,369 @@ +package servicemetrics + +import ( + "fmt" + "k8s.io/utils/net" + "strconv" + "time" + + "github.com/prometheus/client_golang/prometheus" + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/util/wait" + listers "k8s.io/client-go/listers/core/v1" + "k8s.io/client-go/tools/cache" + "k8s.io/ingress-gce/pkg/annotations" + "k8s.io/ingress-gce/pkg/context" + "k8s.io/ingress-gce/pkg/utils" + "k8s.io/ingress-gce/pkg/utils/common" + "k8s.io/klog/v2" + "k8s.io/legacy-cloud-providers/gce" +) + +const ( + // Names of the labels used by the service metrics. + labelType = "type" + labelExternalTrafficPolicy = "external_traffic_policy" + labelInternalTrafficPolicy = "internal_traffic_policy" + labelSessionAffinityConfig = "session_affinity_config" + labelProtocol = "protocol" + labelIPFamilies = "ip_families" + labelIPFamilyPolicy = "ip_family_policy" + labelIsStaticIPv4 = "is_static_ip_v4" + labelIsStaticIPv6 = "is_static_ip_v6" + labelNetworkTier = "network_tier" + labelGlobalAccess = "global_access" + labelCustomSubnet = "custom_subnet" + labelNumberOfPorts = "number_of_ports" + + // possible values for the service_type label + serviceTypeSubsettingILB = "SubsettingILB" + serviceTypeRBSXLB = "RBSXLB" + serviceTypeLegacyILB = "LegacyILB" + serviceTypeLegacyXLB = "LegacyXLB" + + // sessionAffinityTimeoutDefault is the default timeout value for a service session affinity. + sessionAffinityTimeoutDefault = 10800 + + // possible values for the session_affinity_config label. + sessionAffinityBucketMoreThanDefault = "10800+" + sessionAffinityBucketDefault = "10800" + sessionAffinityBucketLessThanDefault = "0-10799" + sessionAffinityBucketNone = "None" +) + +var ( + serviceL4ProtocolStatsCount = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "service_l4_protocol_stats", + Help: "Number of services broken down by various stats", + }, + []string{ + labelType, + labelExternalTrafficPolicy, + labelInternalTrafficPolicy, + labelSessionAffinityConfig, + labelProtocol, + labelNumberOfPorts, + }, + ) + serviceIPStackStatsCount = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "service_ip_stack_stats", + Help: "Number of services broken down by various stats", + }, + []string{ + labelType, + labelExternalTrafficPolicy, + labelInternalTrafficPolicy, + labelIPFamilies, + labelIPFamilyPolicy, + labelIsStaticIPv4, + labelIsStaticIPv6, + }, + ) + serviceGCPFeaturesStatsCount = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "service_gcp_features_stats", + Help: "Number of services broken down by various stats", + }, + []string{ + labelType, + labelNetworkTier, + labelGlobalAccess, + labelCustomSubnet, + }, + ) +) + +func init() { + klog.V(3).Infof("Registering Service stats usage metrics %v", serviceL4ProtocolStatsCount) + prometheus.MustRegister(serviceL4ProtocolStatsCount) + prometheus.MustRegister(serviceIPStackStatsCount) + prometheus.MustRegister(serviceGCPFeaturesStatsCount) +} + +// Controller is the controller that exposes and populates metrics containing various stats about Services in the cluster. +type Controller struct { + ctx *context.ControllerContext + stopCh chan struct{} + svcQueue utils.TaskQueue + metricsInterval time.Duration + serviceInformer cache.SharedIndexInformer +} + +// NewController creates a new Controller. +func NewController(ctx *context.ControllerContext, exportInterval time.Duration, stopCh chan struct{}) *Controller { + svcMetrics := &Controller{ + ctx: ctx, + stopCh: stopCh, + serviceInformer: ctx.ServiceInformer, + metricsInterval: exportInterval, + } + return svcMetrics +} + +// Run starts the controller until stopped via the stop channel. +func (c *Controller) Run() { + klog.Infof("Starting Service Metric Stats controller") + go func() { + time.Sleep(c.metricsInterval) + wait.Until(c.export, c.metricsInterval, c.stopCh) + }() + <-c.stopCh +} + +// serviceL4ProtocolMetricState defines metric state related to the L4 protocol +// related part of services. +type serviceL4ProtocolMetricState struct { + Type string + ExternalTrafficPolicy string + InternalTrafficPolicy string + SessionAffinityConfig string + NumberOfPorts string + Protocol string +} + +// serviceIPStackMetricState defines metric state related to the IP stack of services. +type serviceIPStackMetricState struct { + Type string + ExternalTrafficPolicy string + InternalTrafficPolicy string + IPFamilies string + IPFamilyPolicy string + IsStaticIPv4 bool + IsStaticIPv6 bool +} + +// serviceGCPFeaturesMetricState defines metric state related to the GCP +// specific features of services. +type serviceGCPFeaturesMetricState struct { + Type string + NetworkTier string + GlobalAccess bool + CustomSubnet bool +} + +func (c *Controller) export() { + serviceLister := c.serviceInformer.GetIndexer() + allServices, err := listers.NewServiceLister(serviceLister).List(labels.Everything()) + if err != nil { + klog.Errorf("failed to list services err=%v", err) + return + } + + l4ProtocolState, ipStackState, gcpFeaturesState := calculateMetrics(allServices) + + updatePrometheusMetrics(l4ProtocolState, ipStackState, gcpFeaturesState) +} + +func calculateMetrics(services []*v1.Service) (map[serviceL4ProtocolMetricState]int64, map[serviceIPStackMetricState]int64, map[serviceGCPFeaturesMetricState]int64) { + l4ProtocolState := make(map[serviceL4ProtocolMetricState]int64) + ipStackState := make(map[serviceIPStackMetricState]int64) + gcpFeaturesState := make(map[serviceGCPFeaturesMetricState]int64) + + for _, service := range services { + l4Protocol, ipStack, gcpFeatures := metricsFromService(service) + l4ProtocolState[*l4Protocol]++ + ipStackState[*ipStack]++ + gcpFeaturesState[*gcpFeatures]++ + } + return l4ProtocolState, ipStackState, gcpFeaturesState +} + +func updatePrometheusMetrics(l4ProtocolState map[serviceL4ProtocolMetricState]int64, ipStackState map[serviceIPStackMetricState]int64, gcpFeaturesState map[serviceGCPFeaturesMetricState]int64) { + for serviceStat, count := range l4ProtocolState { + serviceL4ProtocolStatsCount.With(prometheus.Labels{ + labelType: serviceStat.Type, + labelExternalTrafficPolicy: serviceStat.ExternalTrafficPolicy, + labelInternalTrafficPolicy: serviceStat.InternalTrafficPolicy, + labelSessionAffinityConfig: serviceStat.SessionAffinityConfig, + labelProtocol: serviceStat.Protocol, + labelNumberOfPorts: serviceStat.NumberOfPorts, + }).Set(float64(count)) + } + + for serviceStat, count := range ipStackState { + serviceIPStackStatsCount.With(prometheus.Labels{ + labelType: serviceStat.Type, + labelExternalTrafficPolicy: serviceStat.ExternalTrafficPolicy, + labelInternalTrafficPolicy: serviceStat.InternalTrafficPolicy, + labelIPFamilies: serviceStat.IPFamilies, + labelIPFamilyPolicy: serviceStat.IPFamilyPolicy, + labelIsStaticIPv4: strconv.FormatBool(serviceStat.IsStaticIPv4), + labelIsStaticIPv6: strconv.FormatBool(serviceStat.IsStaticIPv6), + }).Set(float64(count)) + } + + for serviceStat, count := range gcpFeaturesState { + serviceGCPFeaturesStatsCount.With(prometheus.Labels{ + labelType: serviceStat.Type, + labelNetworkTier: serviceStat.NetworkTier, + labelGlobalAccess: strconv.FormatBool(serviceStat.GlobalAccess), + labelCustomSubnet: strconv.FormatBool(serviceStat.CustomSubnet), + }).Set(float64(count)) + } +} + +func metricsFromService(service *v1.Service) (*serviceL4ProtocolMetricState, *serviceIPStackMetricState, *serviceGCPFeaturesMetricState) { + serviceType := getServiceType(service) + internalTrafficPolicy := getInternalTrafficPolicy(service) + externalTrafficPolicy := getExternalTrafficPolicy(service) + l4Protocol := &serviceL4ProtocolMetricState{ + Type: serviceType, + ExternalTrafficPolicy: externalTrafficPolicy, + InternalTrafficPolicy: internalTrafficPolicy, + SessionAffinityConfig: getSessionAffinityConfig(service), + NumberOfPorts: getPortsBucket(service.Spec.Ports), + Protocol: getProtocol(service.Spec.Ports), + } + ipStack := &serviceIPStackMetricState{ + Type: serviceType, + ExternalTrafficPolicy: externalTrafficPolicy, + InternalTrafficPolicy: internalTrafficPolicy, + IPFamilies: getIPFamilies(service.Spec.IPFamilies), + IPFamilyPolicy: getIPFamilyPolicy(service.Spec.IPFamilyPolicy), + IsStaticIPv4: isStaticIPv4(service.Spec.LoadBalancerIP), + IsStaticIPv6: isStaticIPv6(service.Spec.LoadBalancerIP), + } + netTier, _ := utils.GetNetworkTier(service) + gcpFeatures := &serviceGCPFeaturesMetricState{ + Type: serviceType, + NetworkTier: string(netTier), + GlobalAccess: gce.GetLoadBalancerAnnotationAllowGlobalAccess(service), + CustomSubnet: gce.GetLoadBalancerAnnotationSubnet(service) != "", + } + return l4Protocol, ipStack, gcpFeatures +} + +func isStaticIPv6(loadBalancerIP string) bool { + return loadBalancerIP != "" && net.IsIPv6String(loadBalancerIP) +} + +func isStaticIPv4(loadBalancerIP string) bool { + return loadBalancerIP != "" && net.IsIPv4String(loadBalancerIP) +} + +func getExternalTrafficPolicy(service *v1.Service) string { + if service.Spec.ExternalTrafficPolicy == "" { + return string(v1.ServiceExternalTrafficPolicyTypeCluster) + } + return string(service.Spec.ExternalTrafficPolicy) +} + +func getInternalTrafficPolicy(service *v1.Service) string { + if service.Spec.InternalTrafficPolicy == nil { + return string(v1.ServiceInternalTrafficPolicyCluster) + } + return string(*service.Spec.InternalTrafficPolicy) +} + +func getPortsBucket(ports []v1.ServicePort) string { + n := len(ports) + if n <= 1 { + return fmt.Sprint(n) + } + if n <= 5 { + return "2-5" + } + if n <= 100 { + return "6-100" + } + return "100+" +} + +func protocolOrDefault(port v1.ServicePort) string { + if port.Protocol == "" { + return string(v1.ProtocolTCP) + } + return string(port.Protocol) +} + +func getProtocol(ports []v1.ServicePort) string { + if len(ports) == 0 { + return "" + } + protocol := protocolOrDefault(ports[0]) + for _, port := range ports { + if protocol != protocolOrDefault(port) { + return "mixed" + } + } + return protocol +} + +func getIPFamilies(families []v1.IPFamily) string { + if len(families) == 2 { + return fmt.Sprintf("%s-%s", string(families[0]), string(families[1])) + } + return string(families[0]) +} + +func getIPFamilyPolicy(policyType *v1.IPFamilyPolicyType) string { + if policyType == nil { + return string(v1.IPFamilyPolicySingleStack) + } + return string(*policyType) +} + +func getServiceType(service *v1.Service) string { + if service.Spec.Type != v1.ServiceTypeLoadBalancer { + return string(service.Spec.Type) + } + wantsL4ILB, _ := annotations.WantsL4ILB(service) + + if wantsL4ILB { + if common.HasGivenFinalizer(service.ObjectMeta, common.ILBFinalizerV2) { + return serviceTypeSubsettingILB + } + return serviceTypeLegacyILB + } + wantsL4NetLB, _ := annotations.WantsL4NetLB(service) + if wantsL4NetLB { + if common.HasGivenFinalizer(service.ObjectMeta, common.NetLBFinalizerV2) { + return serviceTypeRBSXLB + } + return serviceTypeLegacyXLB + } + return "" +} + +func getSessionAffinityConfig(service *v1.Service) string { + if service.Spec.SessionAffinity != v1.ServiceAffinityClientIP { + return sessionAffinityBucketNone + } + if service.Spec.SessionAffinityConfig == nil || + service.Spec.SessionAffinityConfig.ClientIP == nil || + service.Spec.SessionAffinityConfig.ClientIP.TimeoutSeconds == nil { + return sessionAffinityBucketDefault + } + timeout := *service.Spec.SessionAffinityConfig.ClientIP.TimeoutSeconds + + if timeout < sessionAffinityTimeoutDefault { + return sessionAffinityBucketLessThanDefault + } + if timeout == sessionAffinityTimeoutDefault { + return sessionAffinityBucketDefault + } + return sessionAffinityBucketMoreThanDefault +} diff --git a/pkg/servicemetrics/servicemetrics_test.go b/pkg/servicemetrics/servicemetrics_test.go new file mode 100644 index 0000000000..1c8c25dd60 --- /dev/null +++ b/pkg/servicemetrics/servicemetrics_test.go @@ -0,0 +1,731 @@ +package servicemetrics + +import ( + "github.com/GoogleCloudPlatform/k8s-cloud-provider/pkg/cloud" + "github.com/google/go-cmp/cmp" + apiv1 "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/ingress-gce/pkg/annotations" + "k8s.io/ingress-gce/pkg/utils/common" + "k8s.io/legacy-cloud-providers/gce" + "testing" +) + +func TestMetricsFromService(t *testing.T) { + var timeout int32 = 10 + internalPolicyLocal := v1.ServiceInternalTrafficPolicyLocal + ipFamilyPolicyRequireDualStack := v1.IPFamilyPolicyRequireDualStack + + cases := []struct { + desc string + service *v1.Service + wantL4Protocol *serviceL4ProtocolMetricState + wantIPStack *serviceIPStackMetricState + wantGCPFeatures *serviceGCPFeaturesMetricState + }{ + { + desc: "default netXLB", + service: &v1.Service{ + TypeMeta: metav1.TypeMeta{ + Kind: "Service", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "testsvc", + Namespace: "testns", + Finalizers: []string{common.NetLBFinalizerV2}, + }, + Spec: apiv1.ServiceSpec{ + Type: apiv1.ServiceTypeLoadBalancer, + Ports: []apiv1.ServicePort{ + { + Name: "http", + Port: 80, + }, + }, + IPFamilies: []v1.IPFamily{v1.IPv4Protocol}, + }, + }, + wantL4Protocol: &serviceL4ProtocolMetricState{ + Type: serviceTypeRBSXLB, + ExternalTrafficPolicy: string(v1.ServiceExternalTrafficPolicyTypeCluster), + InternalTrafficPolicy: string(v1.ServiceInternalTrafficPolicyCluster), + SessionAffinityConfig: sessionAffinityBucketNone, + NumberOfPorts: "1", + Protocol: "TCP", + }, + wantIPStack: &serviceIPStackMetricState{ + Type: serviceTypeRBSXLB, + ExternalTrafficPolicy: string(v1.ServiceExternalTrafficPolicyTypeCluster), + InternalTrafficPolicy: string(v1.ServiceInternalTrafficPolicyCluster), + IPFamilies: "IPv4", + IPFamilyPolicy: "SingleStack", + IsStaticIPv4: false, + IsStaticIPv6: false, + }, + wantGCPFeatures: &serviceGCPFeaturesMetricState{ + Type: serviceTypeRBSXLB, + NetworkTier: "Premium", + GlobalAccess: false, + CustomSubnet: false, + }, + }, + { + desc: "non defaults", + service: &v1.Service{ + TypeMeta: metav1.TypeMeta{ + Kind: "Service", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "testsvc", + Namespace: "testns", + Finalizers: []string{common.NetLBFinalizerV2}, + Annotations: map[string]string{ + annotations.NetworkTierAnnotationKey: string(cloud.NetworkTierStandard), + gce.ServiceAnnotationILBAllowGlobalAccess: "true", + gce.ServiceAnnotationILBSubnet: "testcustomsubnet", + }, + }, + Spec: apiv1.ServiceSpec{ + Type: apiv1.ServiceTypeLoadBalancer, + Ports: []apiv1.ServicePort{ + { + Name: "http", + Port: 80, + }, + { + Name: "udp", + Port: 80, + Protocol: v1.ProtocolUDP, + }, + }, + ExternalTrafficPolicy: v1.ServiceExternalTrafficPolicyTypeLocal, + InternalTrafficPolicy: &internalPolicyLocal, + IPFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol}, + IPFamilyPolicy: &ipFamilyPolicyRequireDualStack, + SessionAffinity: v1.ServiceAffinityClientIP, + SessionAffinityConfig: &v1.SessionAffinityConfig{ + ClientIP: &v1.ClientIPConfig{ + TimeoutSeconds: &timeout, + }, + }, + LoadBalancerIP: "10.0.0.1", + }, + }, + wantL4Protocol: &serviceL4ProtocolMetricState{ + Type: serviceTypeRBSXLB, + ExternalTrafficPolicy: string(v1.ServiceExternalTrafficPolicyTypeLocal), + InternalTrafficPolicy: string(v1.ServiceInternalTrafficPolicyLocal), + SessionAffinityConfig: sessionAffinityBucketLessThanDefault, + NumberOfPorts: "2-5", + Protocol: "mixed", + }, + wantIPStack: &serviceIPStackMetricState{ + Type: serviceTypeRBSXLB, + ExternalTrafficPolicy: string(v1.ServiceExternalTrafficPolicyTypeLocal), + InternalTrafficPolicy: string(v1.ServiceInternalTrafficPolicyLocal), + IPFamilies: "IPv6-IPv4", + IPFamilyPolicy: string(v1.IPFamilyPolicyRequireDualStack), + IsStaticIPv4: true, + IsStaticIPv6: false, + }, + wantGCPFeatures: &serviceGCPFeaturesMetricState{ + Type: serviceTypeRBSXLB, + NetworkTier: "Standard", + GlobalAccess: true, + CustomSubnet: true, + }, + }, + } + + for _, tc := range cases { + t.Run(tc.desc, func(t *testing.T) { + ports, ipStack, policy := metricsFromService(tc.service) + if diff := cmp.Diff(tc.wantL4Protocol, ports); diff != "" { + t.Errorf("ports metrics mismatch (-want +got):\n%s", diff) + } + if diff := cmp.Diff(tc.wantIPStack, ipStack); diff != "" { + t.Errorf("IP stack metrics mismatch (-want +got):\n%s", diff) + } + if diff := cmp.Diff(tc.wantGCPFeatures, policy); diff != "" { + t.Errorf("policy metrics mismatch (-want +got):\n%s", diff) + } + }) + } +} + +func TestMetricCounting(t *testing.T) { + ipFamilyPolicyRequireDualStack := v1.IPFamilyPolicyRequireDualStack + services := []*v1.Service{ + { + TypeMeta: metav1.TypeMeta{ + Kind: "Service", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "testsvc1", + Namespace: "testns", + Finalizers: []string{common.NetLBFinalizerV2}, + }, + Spec: apiv1.ServiceSpec{ + Type: apiv1.ServiceTypeLoadBalancer, + Ports: []apiv1.ServicePort{ + { + Name: "http", + Port: 80, + }, + }, + IPFamilies: []v1.IPFamily{v1.IPv4Protocol}, + }, + }, + { + TypeMeta: metav1.TypeMeta{ + Kind: "Service", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "testsvc2", + Namespace: "testns", + Finalizers: []string{common.NetLBFinalizerV2}, + }, + Spec: apiv1.ServiceSpec{ + Type: apiv1.ServiceTypeLoadBalancer, + Ports: []apiv1.ServicePort{ + { + Name: "http", + Port: 80, + }, + }, + IPFamilyPolicy: &ipFamilyPolicyRequireDualStack, + IPFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, + LoadBalancerIP: "34.12.153.11", + }, + }, + } + + l4ProtocolKeyS1AndS2 := serviceL4ProtocolMetricState{ + Type: serviceTypeRBSXLB, + ExternalTrafficPolicy: string(v1.ServiceExternalTrafficPolicyTypeCluster), + InternalTrafficPolicy: string(v1.ServiceInternalTrafficPolicyCluster), + SessionAffinityConfig: sessionAffinityBucketNone, + NumberOfPorts: "1", + Protocol: "TCP", + } + ipStackKeyS1 := serviceIPStackMetricState{ + Type: serviceTypeRBSXLB, + ExternalTrafficPolicy: string(v1.ServiceExternalTrafficPolicyTypeCluster), + InternalTrafficPolicy: string(v1.ServiceInternalTrafficPolicyCluster), + IPFamilies: "IPv4", + IPFamilyPolicy: string(v1.IPFamilyPolicySingleStack), + IsStaticIPv4: false, + IsStaticIPv6: false, + } + ipStackKeyS2 := serviceIPStackMetricState{ + Type: serviceTypeRBSXLB, + ExternalTrafficPolicy: string(v1.ServiceExternalTrafficPolicyTypeCluster), + InternalTrafficPolicy: string(v1.ServiceInternalTrafficPolicyCluster), + IPFamilies: "IPv4-IPv6", + IPFamilyPolicy: string(v1.IPFamilyPolicyRequireDualStack), + IsStaticIPv4: true, + IsStaticIPv6: false, + } + gcpFeaturesKeyS1AndS2 := serviceGCPFeaturesMetricState{ + Type: serviceTypeRBSXLB, + NetworkTier: "Premium", + GlobalAccess: false, + CustomSubnet: false, + } + + l4ProtocolState, ipStackState, gcpFeaturesState := calculateMetrics(services) + + l4ProtocolVal, ok := l4ProtocolState[l4ProtocolKeyS1AndS2] + if !ok || l4ProtocolVal != 2 { + t.Errorf("l4Protocol metrics were missing or invalid, present=%t, value=%d", ok, l4ProtocolVal) + } + ipStackVal, ok := ipStackState[ipStackKeyS1] + if !ok || ipStackVal != 1 { + t.Errorf("IP stack metrics were missing or invalid, present=%t, value=%d", ok, ipStackVal) + } + ipStackVal2, ok := ipStackState[ipStackKeyS2] + if !ok || ipStackVal2 != 1 { + t.Errorf("IP stack metrics were missing or invalid, present=%t, value=%d", ok, ipStackVal2) + } + gcpFeaturesVal, ok := gcpFeaturesState[gcpFeaturesKeyS1AndS2] + if !ok || gcpFeaturesVal != 2 { + t.Errorf("l4Protocol metrics were missing or invalid, present=%t, value=%d", ok, gcpFeaturesVal) + } + +} + +func TestGetExternalTrafficPolicy(t *testing.T) { + cases := []struct { + desc string + service *v1.Service + want string + }{ + { + desc: "default", + service: &v1.Service{}, + want: string(v1.ServiceExternalTrafficPolicyTypeCluster), + }, + { + desc: "cluster", + service: &v1.Service{ + Spec: v1.ServiceSpec{ + ExternalTrafficPolicy: v1.ServiceExternalTrafficPolicyTypeCluster, + }, + }, + want: string(v1.ServiceExternalTrafficPolicyTypeCluster), + }, + { + desc: "local", + service: &v1.Service{ + Spec: v1.ServiceSpec{ + ExternalTrafficPolicy: v1.ServiceExternalTrafficPolicyTypeLocal, + }, + }, + want: string(v1.ServiceExternalTrafficPolicyTypeLocal), + }, + } + + for _, tc := range cases { + t.Run(tc.desc, func(t *testing.T) { + got := getExternalTrafficPolicy(tc.service) + if tc.want != got { + t.Errorf("getExternalTrafficPolicy output differed, want=%q, got=%q", tc.want, got) + } + }) + } +} + +func TestGetInternalTrafficPolicy(t *testing.T) { + policyCluster := v1.ServiceInternalTrafficPolicyCluster + policyLocal := v1.ServiceInternalTrafficPolicyLocal + cases := []struct { + desc string + service *v1.Service + want string + }{ + { + desc: "default", + service: &v1.Service{}, + want: string(v1.ServiceInternalTrafficPolicyCluster), + }, + { + desc: "cluster", + service: &v1.Service{ + Spec: v1.ServiceSpec{ + InternalTrafficPolicy: &policyCluster, + }, + }, + want: string(v1.ServiceInternalTrafficPolicyCluster), + }, + { + desc: "local", + service: &v1.Service{ + Spec: v1.ServiceSpec{ + InternalTrafficPolicy: &policyLocal, + }, + }, + want: string(v1.ServiceInternalTrafficPolicyLocal), + }, + } + + for _, tc := range cases { + t.Run(tc.desc, func(t *testing.T) { + got := getInternalTrafficPolicy(tc.service) + if tc.want != got { + t.Errorf("getInternalTrafficPolicy output differed, want=%q, got=%q", tc.want, got) + } + }) + } + +} + +func TestGetPortsBucket(t *testing.T) { + cases := []struct { + desc string + ports []v1.ServicePort + want string + }{ + { + desc: "0", + ports: []v1.ServicePort{}, + want: "0", + }, + { + desc: "1", + ports: []v1.ServicePort{ + { + Name: "p1", + Port: 80, + }, + }, + want: "1", + }, + { + desc: "2-5", + ports: []v1.ServicePort{ + { + Name: "p1", + Port: 80, + }, + { + Name: "p2", + Port: 123456, + }, + }, + want: "2-5", + }, + { + desc: "6-100", + ports: make([]v1.ServicePort, 10), + want: "6-100", + }, + { + desc: "100+", + ports: make([]v1.ServicePort, 101), + want: "100+", + }, + } + for _, tc := range cases { + t.Run(tc.desc, func(t *testing.T) { + got := getPortsBucket(tc.ports) + if tc.want != got { + t.Errorf("getPortsBucket output differed, want=%q, got=%q", tc.want, got) + } + }) + } +} + +func TestGetProtocol(t *testing.T) { + cases := []struct { + desc string + ports []v1.ServicePort + want string + }{ + { + desc: "default", + ports: []v1.ServicePort{{Name: "default port"}}, + want: "TCP", + }, + { + desc: "TCP", + ports: []v1.ServicePort{{Protocol: v1.ProtocolTCP}}, + want: "TCP", + }, + { + desc: "UDP", + ports: []v1.ServicePort{{Protocol: v1.ProtocolUDP}, {Protocol: v1.ProtocolUDP}}, + want: "UDP", + }, + { + desc: "different", + ports: []v1.ServicePort{{Protocol: v1.ProtocolTCP}, {Protocol: v1.ProtocolUDP}}, + want: "mixed", + }, + } + for _, tc := range cases { + t.Run(tc.desc, func(t *testing.T) { + got := getProtocol(tc.ports) + if tc.want != got { + t.Errorf("getProtocol output differed, want=%q, got=%q", tc.want, got) + } + }) + } +} + +func TestGetIPFamilies(t *testing.T) { + cases := []struct { + desc string + families []v1.IPFamily + want string + }{ + { + desc: "IPv4", + families: []v1.IPFamily{v1.IPv4Protocol}, + want: "IPv4", + }, + { + desc: "IPv6", + families: []v1.IPFamily{v1.IPv6Protocol}, + want: "IPv6", + }, + { + desc: "IPv6-IPv4", + families: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol}, + want: "IPv6-IPv4", + }, + { + desc: "IPv4-IPv6", + families: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, + want: "IPv4-IPv6", + }, + } + for _, tc := range cases { + t.Run(tc.desc, func(t *testing.T) { + got := getIPFamilies(tc.families) + if tc.want != got { + t.Errorf("getIPFamilies output differed, want=%q, got=%q", tc.want, got) + } + }) + } +} + +func TestGetIPFamilyPolicy(t *testing.T) { + singleStack := v1.IPFamilyPolicySingleStack + preferDualStack := v1.IPFamilyPolicyPreferDualStack + requireDualStack := v1.IPFamilyPolicyRequireDualStack + cases := []struct { + desc string + policyType *v1.IPFamilyPolicyType + want string + }{ + { + desc: "default", + policyType: nil, + want: string(v1.IPFamilyPolicySingleStack), + }, + { + desc: "single stack", + policyType: &singleStack, + want: string(v1.IPFamilyPolicySingleStack), + }, + { + desc: "prefer dual stack", + policyType: &preferDualStack, + want: string(v1.IPFamilyPolicyPreferDualStack), + }, + { + desc: "require dual stack", + policyType: &requireDualStack, + want: string(v1.IPFamilyPolicyRequireDualStack), + }, + } + for _, tc := range cases { + t.Run(tc.desc, func(t *testing.T) { + got := getIPFamilyPolicy(tc.policyType) + if tc.want != got { + t.Errorf("getIPFamilyPolicy output differed, want=%q, got=%q", tc.want, got) + } + }) + } +} + +func TestGetLBType(t *testing.T) { + cases := []struct { + desc string + service *v1.Service + want string + }{ + { + desc: "non LB", + service: &v1.Service{}, + want: "", + }, + { + desc: "SubsettingILB", + service: &v1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{gce.ServiceAnnotationLoadBalancerType: string(gce.LBTypeInternal)}, + Finalizers: []string{common.ILBFinalizerV2}, + }, + Spec: v1.ServiceSpec{ + Type: v1.ServiceTypeLoadBalancer, + }, + }, + want: serviceTypeSubsettingILB, + }, + { + desc: "LegacyILB", + service: &v1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{gce.ServiceAnnotationLoadBalancerType: string(gce.LBTypeInternal)}, + }, + Spec: v1.ServiceSpec{ + Type: v1.ServiceTypeLoadBalancer, + }, + }, + want: serviceTypeLegacyILB, + }, + { + desc: "RBSXLB", + service: &v1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{}, + Finalizers: []string{common.NetLBFinalizerV2}, + }, + Spec: v1.ServiceSpec{ + Type: v1.ServiceTypeLoadBalancer, + }, + }, + want: serviceTypeRBSXLB, + }, + { + desc: "LegacyXLB", + service: &v1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{}, + }, + Spec: v1.ServiceSpec{ + Type: v1.ServiceTypeLoadBalancer, + }, + }, + want: serviceTypeLegacyXLB, + }, + } + + for _, tc := range cases { + t.Run(tc.desc, func(t *testing.T) { + got := getServiceType(tc.service) + if tc.want != got { + t.Errorf("getServiceType output differed, want=%q, got=%q", tc.want, got) + } + }) + } +} + +func TestGetSessionAffinityConfig(t *testing.T) { + var oneSecond int32 = 1 + var defaultPlus1 int32 = 10801 + cases := []struct { + desc string + service *v1.Service + want string + }{ + { + desc: "no session affinity", + service: &v1.Service{}, + want: sessionAffinityBucketNone, + }, + { + desc: "default client IP", + service: &v1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{gce.ServiceAnnotationLoadBalancerType: string(gce.LBTypeInternal)}, + Finalizers: []string{common.ILBFinalizerV2}, + }, + Spec: v1.ServiceSpec{ + SessionAffinity: v1.ServiceAffinityClientIP, + }, + }, + want: sessionAffinityBucketDefault, + }, + { + desc: "0-10799", + service: &v1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{gce.ServiceAnnotationLoadBalancerType: string(gce.LBTypeInternal)}, + Finalizers: []string{common.ILBFinalizerV2}, + }, + Spec: v1.ServiceSpec{ + SessionAffinity: v1.ServiceAffinityClientIP, + SessionAffinityConfig: &v1.SessionAffinityConfig{ + ClientIP: &v1.ClientIPConfig{ + TimeoutSeconds: &oneSecond, + }, + }, + }, + }, + want: sessionAffinityBucketLessThanDefault, + }, + { + desc: "10800+", + service: &v1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{gce.ServiceAnnotationLoadBalancerType: string(gce.LBTypeInternal)}, + Finalizers: []string{common.ILBFinalizerV2}, + }, + Spec: v1.ServiceSpec{ + SessionAffinity: v1.ServiceAffinityClientIP, + SessionAffinityConfig: &v1.SessionAffinityConfig{ + ClientIP: &v1.ClientIPConfig{ + TimeoutSeconds: &defaultPlus1, + }, + }, + }, + }, + want: sessionAffinityBucketMoreThanDefault, + }, + } + + for _, tc := range cases { + t.Run(tc.desc, func(t *testing.T) { + got := getSessionAffinityConfig(tc.service) + if tc.want != got { + t.Errorf("getSessionAffinityConfig output differed, want=%q, got=%q", tc.want, got) + } + }) + } +} + +func TestIsStaticIPv4(t *testing.T) { + cases := []struct { + desc string + loadBalancerIP string + want bool + }{ + { + desc: "empty", + loadBalancerIP: "", + want: false, + }, + { + desc: "valid IPv4", + loadBalancerIP: "10.0.0.1", + want: true, + }, + { + desc: "invalid IPv4", + loadBalancerIP: "500.0.0.1", + want: false, + }, + { + desc: "non IPv4", + loadBalancerIP: "2001:db8::8a2e:370:7334", + want: false, + }, + } + for _, tc := range cases { + t.Run(tc.desc, func(t *testing.T) { + got := isStaticIPv4(tc.loadBalancerIP) + if tc.want != got { + t.Errorf("isStaticIPv4 output differed, want=%t, got=%t", tc.want, got) + } + }) + } +} + +func TestIsStaticIPv6(t *testing.T) { + cases := []struct { + desc string + loadBalancerIP string + want bool + }{ + { + desc: "empty", + loadBalancerIP: "", + want: false, + }, + { + desc: "valid IPv6", + loadBalancerIP: "2001:db8::8a2e:370:7334", + want: true, + }, + { + desc: "non IPv6", + loadBalancerIP: "10.0.0.1", + want: false, + }, + } + for _, tc := range cases { + t.Run(tc.desc, func(t *testing.T) { + got := isStaticIPv6(tc.loadBalancerIP) + if tc.want != got { + t.Errorf("isStaticIPv6 output differed, want=%t, got=%t", tc.want, got) + } + }) + } +}