diff --git a/pkg/annotations/service.go b/pkg/annotations/service.go index 536ff5cc00..55b94bb213 100644 --- a/pkg/annotations/service.go +++ b/pkg/annotations/service.go @@ -73,6 +73,7 @@ const ( // ProtocolHTTP2 protocol for a service ProtocolHTTP2 AppProtocol = "HTTP2" + IPv6Suffix = "-ipv6" // ServiceStatusPrefix is the prefix used in annotations used to record // debug information in the Service annotations. This is applicable to L4 ILB services. ServiceStatusPrefix = "service.kubernetes.io" @@ -82,24 +83,39 @@ const ( // UDPForwardingRuleKey is the annotation key used by l4 controller to record // GCP UDP forwarding rule name. UDPForwardingRuleKey = ServiceStatusPrefix + "/udp-" + ForwardingRuleResource + // TCPForwardingRuleIPv6Key is the annotation key used by l4 controller to record + // GCP IPv6 TCP forwarding rule name. + TCPForwardingRuleIPv6Key = TCPForwardingRuleKey + IPv6Suffix + // UDPForwardingRuleIPv6Key is the annotation key used by l4 controller to record + // GCP IPv6 UDP forwarding rule name. + UDPForwardingRuleIPv6Key = UDPForwardingRuleKey + IPv6Suffix // BackendServiceKey is the annotation key used by l4 controller to record // GCP Backend service name. BackendServiceKey = ServiceStatusPrefix + "/" + BackendServiceResource // FirewallRuleKey is the annotation key used by l4 controller to record // GCP Firewall rule name. FirewallRuleKey = ServiceStatusPrefix + "/" + FirewallRuleResource + // FirewallRuleIPv6Key is the annotation key used by l4 controller to record + // GCP IPv6 Firewall rule name. + FirewallRuleIPv6Key = FirewallRuleKey + IPv6Suffix // HealthcheckKey is the annotation key used by l4 controller to record // GCP Healthcheck name. HealthcheckKey = ServiceStatusPrefix + "/" + HealthcheckResource // FirewallRuleForHealthcheckKey is the annotation key used by l4 controller to record // the firewall rule name that allows healthcheck traffic. - FirewallRuleForHealthcheckKey = ServiceStatusPrefix + "/" + FirewallForHealthcheckResource - ForwardingRuleResource = "forwarding-rule" - BackendServiceResource = "backend-service" - FirewallRuleResource = "firewall-rule" - HealthcheckResource = "healthcheck" - FirewallForHealthcheckResource = "firewall-rule-for-hc" - AddressResource = "address" + FirewallRuleForHealthcheckKey = ServiceStatusPrefix + "/" + FirewallForHealthcheckResource + // FirewallRuleForHealthcheckIPv6Key is the annotation key used by l4 controller to record + // the firewall rule name that allows IPv6 healthcheck traffic. + FirewallRuleForHealthcheckIPv6Key = FirewallRuleForHealthcheckKey + IPv6Suffix + ForwardingRuleResource = "forwarding-rule" + ForwardingRuleIPv6Resource = ForwardingRuleResource + IPv6Suffix + BackendServiceResource = "backend-service" + FirewallRuleResource = "firewall-rule" + FirewallRuleIPv6Resource = FirewallRuleResource + IPv6Suffix + HealthcheckResource = "healthcheck" + FirewallForHealthcheckResource = "firewall-rule-for-hc" + FirewallForHealthcheckIPv6Resource = FirewallRuleForHealthcheckKey + IPv6Suffix + AddressResource = "address" // TODO(slavik): import this from gce_annotations when it will be merged in k8s RBSAnnotationKey = "cloud.google.com/l4-rbs" RBSEnabled = "enabled" diff --git a/pkg/flags/flags.go b/pkg/flags/flags.go index 8edd9936dd..7af7611f4f 100644 --- a/pkg/flags/flags.go +++ b/pkg/flags/flags.go @@ -108,6 +108,7 @@ var ( EnableTrafficScaling bool EnableEndpointSlices bool EnablePinhole bool + EnableL4ILBDualStack bool EnableMultipleIGs bool MaxIGSize int }{ @@ -249,6 +250,7 @@ L7 load balancing. CSV values accepted. Example: -node-port-ranges=80,8080,400-5 flag.BoolVar(&F.EnableTrafficScaling, "enable-traffic-scaling", false, "Enable support for Service {max-rate-per-endpoint, capacity-scaler}") flag.BoolVar(&F.EnableEndpointSlices, "enable-endpoint-slices", false, "Enable using Endpoint Slices API instead of Endpoints API") flag.BoolVar(&F.EnablePinhole, "enable-pinhole", false, "Enable Pinhole firewall feature") + flag.BoolVar(&F.EnableL4ILBDualStack, "enable-l4ilb-dual-stack", false, "Enable Dual-Stack handling for L4 ILB load balancers") flag.BoolVar(&F.EnableMultipleIGs, "enable-multiple-igs", false, "Enable using multiple unmanaged instance groups") flag.IntVar(&F.MaxIGSize, "max-ig-size", 1000, "Max number of instances in Instance Group") flag.DurationVar(&F.MetricsExportInterval, "metrics-export-interval", 10*time.Minute, `Period for calculating and exporting metrics related to state of managed objects.`) diff --git a/pkg/healthchecks/healthchecks_l4.go b/pkg/healthchecks/healthchecks_l4.go index 6d7fc1effc..5333bd27ab 100644 --- a/pkg/healthchecks/healthchecks_l4.go +++ b/pkg/healthchecks/healthchecks_l4.go @@ -104,6 +104,10 @@ func L4() *l4HealthChecks { // Services of different scope (Global vs Regional). func (l4hc *l4HealthChecks) EnsureL4HealthCheck(svc *corev1.Service, namer namer.L4ResourcesNamer, sharedHC bool, scope meta.KeyType, l4Type utils.L4LBType, nodeNames []string) *EnsureL4HealthCheckResult { + return l4hc.EnsureL4DualStackHealthCheck(svc, namer, sharedHC, scope, l4Type, nodeNames, true, false) +} + +func (l4hc *l4HealthChecks) EnsureL4DualStackHealthCheck(svc *corev1.Service, namer namer.L4ResourcesNamer, sharedHC bool, scope meta.KeyType, l4Type utils.L4LBType, nodeNames []string, needsIPv4 bool, needsIPv6 bool) *EnsureL4HealthCheckResult { namespacedName := types.NamespacedName{Name: svc.Name, Namespace: svc.Namespace} hcName, hcFwName := namer.L4HealthCheck(svc.Namespace, svc.Name, sharedHC) @@ -126,24 +130,57 @@ func (l4hc *l4HealthChecks) EnsureL4HealthCheck(svc *corev1.Service, namer namer } } - klog.V(3).Infof("Healthcheck created, ensuring firewall rule %s", hcFwName) - err = l4hc.ensureFirewall(svc, hcFwName, hcPort, sharedHC, nodeNames) - if err != nil { - return &EnsureL4HealthCheckResult{ - GceResourceInError: annotations.HealthcheckResource, - Err: err, + hcResult := &EnsureL4HealthCheckResult{ + HCName: hcName, + HCLink: hcLink, + } + + if needsIPv4 { + klog.V(3).Infof("Healthcheck created, ensuring firewall rule %s", hcFwName) + err = l4hc.ensureFirewall(svc, hcFwName, hcPort, sharedHC, nodeNames) + if err != nil { + return &EnsureL4HealthCheckResult{ + GceResourceInError: annotations.FirewallForHealthcheckResource, + Err: err, + } + } + hcResult.HCFirewallRuleName = hcFwName + } + + if needsIPv6 { + ipv6HCFWName := namer.L4IPv6HealthCheckFirewall(svc.Namespace, svc.Name, sharedHC) + klog.V(3).Infof("Healthcheck created, ensuring ipv6 firewall rule %s", ipv6HCFWName) + err = l4hc.ensureIPv6Firewall(svc, ipv6HCFWName, hcPort, sharedHC, nodeNames) + if err != nil { + return &EnsureL4HealthCheckResult{ + GceResourceInError: annotations.FirewallForHealthcheckIPv6Resource, + Err: err, + } } + hcResult.HCFirewallRuleIPv6Name = ipv6HCFWName } - return &EnsureL4HealthCheckResult{ - HCName: hcName, - HCLink: hcLink, - HCFirewallRuleName: hcFwName, + + return hcResult +} + +func (l4hc *l4HealthChecks) ensureIPv6Firewall(svc *corev1.Service, ipv6HCFWName string, hcPort int32, isSharedHC bool, nodeNames []string) error { + hcFWRParams := firewalls.FirewallParams{ + PortRanges: []string{strconv.Itoa(int(hcPort))}, + SourceRanges: L4ILBIPv6HCRange.StringSlice(), + Protocol: string(corev1.ProtocolTCP), + Name: ipv6HCFWName, + NodeNames: nodeNames, } + return firewalls.EnsureL4LBFirewallForHc(svc, isSharedHC, &hcFWRParams, l4hc.cloud, l4hc.recorderFactory.Recorder(svc.Namespace)) } // DeleteHealthCheck deletes health check (and firewall rule) for l4 service. Checks if shared resources are safe to delete. func (l4hc *l4HealthChecks) DeleteHealthCheck(svc *corev1.Service, namer namer.L4ResourcesNamer, sharedHC bool, scope meta.KeyType, l4Type utils.L4LBType) (string, error) { + return l4hc.DeleteDualStackHealthCheck(svc, namer, sharedHC, scope, l4Type, false) +} +// DeleteDualStackHealthCheck deletes health check (and firewall rule) for l4 service. Checks if shared resources are safe to delete. +func (l4hc *l4HealthChecks) DeleteDualStackHealthCheck(svc *corev1.Service, namer namer.L4ResourcesNamer, sharedHC bool, scope meta.KeyType, l4Type utils.L4LBType, handleIPv6 bool) (string, error) { hcName, hcFwName := namer.L4HealthCheck(svc.Namespace, svc.Name, sharedHC) namespacedName := types.NamespacedName{Name: svc.Name, Namespace: svc.Namespace} klog.V(3).Infof("Trying to delete L4 healthcheck: %s and firewall rule %s from service %s, shared: %v", hcName, hcFwName, namespacedName.String(), sharedHC) @@ -164,6 +201,12 @@ func (l4hc *l4HealthChecks) DeleteHealthCheck(svc *corev1.Service, namer namer.L return "", nil } // Health check deleted, now delete the firewall rule + if handleIPv6 { + errResource, err := l4hc.deleteIPv6HealthCheckFirewall(svc, hcName, namer, sharedHC, l4Type) + if err != nil { + return errResource, err + } + } return l4hc.deleteHealthCheckFirewall(svc, hcName, hcFwName, sharedHC, l4Type) } @@ -256,6 +299,11 @@ func (l4hc *l4HealthChecks) deleteHealthCheckFirewall(svc *corev1.Service, hcNam return "", nil } +func (l4hc *l4HealthChecks) deleteIPv6HealthCheckFirewall(svc *corev1.Service, hcName string, namer namer.L4ResourcesNamer, sharedHC bool, l4type utils.L4LBType) (string, error) { + ipv6hcFwName := namer.L4IPv6HealthCheckFirewall(svc.Namespace, svc.Name, sharedHC) + return l4hc.deleteHealthCheckFirewall(svc, hcName, ipv6hcFwName, sharedHC, l4type) +} + func (l4hc *l4HealthChecks) healthCheckFirewallSafeToDelete(hcName string, sharedHC bool, l4Type utils.L4LBType) (bool, error) { if !sharedHC { return true, nil diff --git a/pkg/healthchecks/interfaces.go b/pkg/healthchecks/interfaces.go index 0b9956be13..43339316f7 100644 --- a/pkg/healthchecks/interfaces.go +++ b/pkg/healthchecks/interfaces.go @@ -63,14 +63,19 @@ type HealthChecker interface { type L4HealthChecks interface { // EnsureL4HealthCheck creates health check (and firewall rule) for l4 service EnsureL4HealthCheck(svc *v1.Service, namer namer.L4ResourcesNamer, sharedHC bool, scope meta.KeyType, l4Type utils.L4LBType, nodeNames []string) *EnsureL4HealthCheckResult + // EnsureL4DualStackHealthCheck creates health check (and firewall rule) for l4 service. Handles both IPv4 and IPv6 + EnsureL4DualStackHealthCheck(svc *v1.Service, namer namer.L4ResourcesNamer, sharedHC bool, scope meta.KeyType, l4Type utils.L4LBType, nodeNames []string, needsIPv4 bool, needsIPv6 bool) *EnsureL4HealthCheckResult // DeleteHealthCheck deletes health check (and firewall rule) for l4 service DeleteHealthCheck(svc *v1.Service, namer namer.L4ResourcesNamer, sharedHC bool, scope meta.KeyType, l4Type utils.L4LBType) (string, error) + // DeleteDualStackHealthCheck deletes health check (and firewall rule) for l4 service, deletes IPv6 if asked + DeleteDualStackHealthCheck(svc *v1.Service, namer namer.L4ResourcesNamer, sharedHC bool, scope meta.KeyType, l4Type utils.L4LBType, handleIPv6 bool) (string, error) } type EnsureL4HealthCheckResult struct { - HCName string - HCLink string - HCFirewallRuleName string - GceResourceInError string - Err error + HCName string + HCLink string + HCFirewallRuleName string + HCFirewallRuleIPv6Name string + GceResourceInError string + Err error } diff --git a/pkg/healthchecks/ipv6.go b/pkg/healthchecks/ipv6.go new file mode 100644 index 0000000000..f049a542bd --- /dev/null +++ b/pkg/healthchecks/ipv6.go @@ -0,0 +1,31 @@ +package healthchecks + +import ( + "fmt" + + "k8s.io/klog/v2" + utilnet "k8s.io/utils/net" +) + +const ( + l4ELBIPv6HCRangeString = "2600:1901:8001::/48" + l4ILBIPv6HCRangeString = "2600:2d00:1:b029::/64" +) + +var ( + L4ELBIPv6HCRange utilnet.IPNetSet + L4ILBIPv6HCRange utilnet.IPNetSet +) + +func init() { + var err error + L4ELBIPv6HCRange, err = utilnet.ParseIPNets([]string{l4ELBIPv6HCRangeString}...) + if err != nil { + klog.Fatalf(fmt.Sprintf("utilnet.ParseIPNets([]string{%s}...) returned error %v, want nil", l4ELBIPv6HCRangeString, err)) + } + + L4ILBIPv6HCRange, err = utilnet.ParseIPNets([]string{l4ILBIPv6HCRangeString}...) + if err != nil { + klog.Fatalf(fmt.Sprintf("utilnet.ParseIPNets([]string{%s}...) returned error %v, want nil", l4ILBIPv6HCRangeString, err)) + } +} diff --git a/pkg/l4lb/l4controller.go b/pkg/l4lb/l4controller.go index 0d180070a0..f9c3ac2af2 100644 --- a/pkg/l4lb/l4controller.go +++ b/pkg/l4lb/l4controller.go @@ -33,6 +33,7 @@ import ( "k8s.io/ingress-gce/pkg/backends" "k8s.io/ingress-gce/pkg/context" "k8s.io/ingress-gce/pkg/controller/translator" + "k8s.io/ingress-gce/pkg/flags" "k8s.io/ingress-gce/pkg/forwardingrules" l4metrics "k8s.io/ingress-gce/pkg/l4lb/metrics" "k8s.io/ingress-gce/pkg/loadbalancers" @@ -208,7 +209,7 @@ func (l4c *L4Controller) processServiceCreateOrUpdate(key string, service *v1.Se } // Use the same function for both create and updates. If controller crashes and restarts, // all existing services will show up as Service Adds. - l4 := loadbalancers.NewL4Handler(service, l4c.ctx.Cloud, meta.Regional, l4c.namer, l4c.ctx.Recorder(service.Namespace)) + l4 := loadbalancers.NewL4Handler(service, l4c.ctx.Cloud, meta.Regional, l4c.namer, l4c.ctx.Recorder(service.Namespace), flags.F.EnableL4ILBDualStack) syncResult := l4.EnsureInternalLoadBalancer(nodeNames, service) // syncResult will not be nil if syncResult.Error != nil { @@ -248,7 +249,7 @@ func (l4c *L4Controller) processServiceCreateOrUpdate(key string, service *v1.Se } func (l4c *L4Controller) processServiceDeletion(key string, svc *v1.Service) *loadbalancers.L4ILBSyncResult { - l4 := loadbalancers.NewL4Handler(svc, l4c.ctx.Cloud, meta.Regional, l4c.namer, l4c.ctx.Recorder(svc.Namespace)) + l4 := loadbalancers.NewL4Handler(svc, l4c.ctx.Cloud, meta.Regional, l4c.namer, l4c.ctx.Recorder(svc.Namespace), flags.F.EnableL4ILBDualStack) l4c.ctx.Recorder(svc.Namespace).Eventf(svc, v1.EventTypeNormal, "DeletingLoadBalancer", "Deleting load balancer for %s", key) result := l4.EnsureInternalLoadBalancerDeleted(svc) if result.Error != nil { diff --git a/pkg/l4lb/l4netlbcontroller_test.go b/pkg/l4lb/l4netlbcontroller_test.go index 31acee4f8f..0f4cd79370 100644 --- a/pkg/l4lb/l4netlbcontroller_test.go +++ b/pkg/l4lb/l4netlbcontroller_test.go @@ -877,7 +877,7 @@ func TestHealthCheckWhenExternalTrafficPolicyWasUpdated(t *testing.T) { // Update ExternalTrafficPolicy to Cluster check if shared HC was created err = updateAndAssertExternalTrafficPolicy(newSvc, lc, v1.ServiceExternalTrafficPolicyTypeCluster, hcNameShared) if err != nil { - t.Errorf("Error asserthing shared health check %v", err) + t.Errorf("Error asserting shared health check %v", err) } newSvc.DeletionTimestamp = &metav1.Time{} updateNetLBService(lc, newSvc) @@ -1005,7 +1005,7 @@ func TestIsRBSBasedService(t *testing.T) { func TestIsRBSBasedServiceWithILBServices(t *testing.T) { controller := newL4NetLBServiceController() ilbSvc := test.NewL4ILBService(false, 8080) - ilbFrName := loadbalancers.NewL4Handler(ilbSvc, controller.ctx.Cloud, meta.Regional, controller.namer, record.NewFakeRecorder(100)).GetFRName() + ilbFrName := loadbalancers.NewL4Handler(ilbSvc, controller.ctx.Cloud, meta.Regional, controller.namer, record.NewFakeRecorder(100), false).GetFRName() ilbSvc.Annotations = map[string]string{ annotations.TCPForwardingRuleKey: ilbFrName, annotations.UDPForwardingRuleKey: ilbFrName, diff --git a/pkg/loadbalancers/forwarding_rules_ipv6.go b/pkg/loadbalancers/forwarding_rules_ipv6.go new file mode 100644 index 0000000000..969031e852 --- /dev/null +++ b/pkg/loadbalancers/forwarding_rules_ipv6.go @@ -0,0 +1,120 @@ +package loadbalancers + +import ( + "fmt" + + "github.com/GoogleCloudPlatform/k8s-cloud-provider/pkg/cloud" + "github.com/GoogleCloudPlatform/k8s-cloud-provider/pkg/cloud/meta" + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + corev1 "k8s.io/api/core/v1" + "k8s.io/ingress-gce/pkg/composite" + "k8s.io/ingress-gce/pkg/events" + "k8s.io/ingress-gce/pkg/utils" + "k8s.io/klog/v2" + "k8s.io/legacy-cloud-providers/gce" +) + +func (l *L4) ensureIPv6ForwardingRule(bsLink string, options gce.ILBOptions) (*composite.ForwardingRule, error) { + expectedIPv6FwdRule, err := l.buildExpectedIPv6ForwardingRule(bsLink, options) + if err != nil { + return nil, fmt.Errorf("l.buildExpectedIPv6ForwardingRule(%s, %v) returned error %w, want nil", bsLink, options, err) + } + + existingIPv6FwdRule, err := l.forwardingRules.Get(expectedIPv6FwdRule.Name) + if err != nil { + return nil, fmt.Errorf("l.forwardingRules.GetForwardingRule(%s) returned error %w, want nil", expectedIPv6FwdRule.Name, err) + } + + if existingIPv6FwdRule != nil { + equal, err := EqualIPv6ForwardingRules(existingIPv6FwdRule, expectedIPv6FwdRule) + if err != nil { + return existingIPv6FwdRule, err + } + if equal { + klog.V(2).Infof("ensureIPv6ForwardingRule: Skipping update of unchanged ipv6 forwarding rule - %s", expectedIPv6FwdRule.Name) + return existingIPv6FwdRule, nil + } + + frDiff := cmp.Diff(existingIPv6FwdRule, expectedIPv6FwdRule, cmpopts.IgnoreFields(composite.ForwardingRule{}, "IPAddress")) + klog.V(2).Infof("ensureIPv6ForwardingRule: forwarding rule changed - Existing - %+v\n, New - %+v\n, Diff(-existing, +new) - %s\n. Deleting existing ipv6 forwarding rule.", existingIPv6FwdRule, expectedIPv6FwdRule, frDiff) + + err = l.forwardingRules.Delete(existingIPv6FwdRule.Name) + if err != nil { + return nil, err + } + l.recorder.Eventf(l.Service, corev1.EventTypeNormal, events.SyncIngress, "ForwardingRule %q deleted", existingIPv6FwdRule.Name) + } + klog.V(2).Infof("ensureIPv6ForwardingRule: Creating/Recreating forwarding rule - %s", expectedIPv6FwdRule.Name) + err = l.forwardingRules.Create(expectedIPv6FwdRule) + if err != nil { + return nil, err + } + + createdFr, err := l.forwardingRules.Get(expectedIPv6FwdRule.Name) + return createdFr, err +} + +func (l *L4) buildExpectedIPv6ForwardingRule(bsLink string, options gce.ILBOptions) (*composite.ForwardingRule, error) { + frName := l.getIPv6FRName() + + frDesc, err := utils.MakeL4IPv6ForwardingRuleDescription(l.Service) + if err != nil { + return nil, fmt.Errorf("failed to compute description for forwarding rule %s, err: %w", frName, err) + } + + subnetworkURL := l.cloud.SubnetworkURL() + + if options.SubnetName != "" { + key, err := l.CreateKey(frName) + if err != nil { + return nil, err + } + subnetKey := *key + subnetKey.Name = options.SubnetName + subnetworkURL = cloud.SelfLink(meta.VersionGA, l.cloud.NetworkProjectID(), "subnetworks", &subnetKey) + } + + svcPorts := l.Service.Spec.Ports + ports := utils.GetPorts(svcPorts) + protocol := utils.GetProtocol(svcPorts) + + fr := &composite.ForwardingRule{ + Name: frName, + Description: frDesc, + IPProtocol: string(protocol), + Ports: ports, + LoadBalancingScheme: string(cloud.SchemeInternal), + BackendService: bsLink, + IpVersion: "IPV6", + Network: l.cloud.NetworkURL(), + Subnetwork: subnetworkURL, + AllowGlobalAccess: options.AllowGlobalAccess, + NetworkTier: cloud.NetworkTierPremium.ToGCEValue(), + } + if len(ports) > maxL4ILBPorts { + fr.Ports = nil + fr.AllPorts = true + } + + return fr, nil +} + +func EqualIPv6ForwardingRules(fr1, fr2 *composite.ForwardingRule) (bool, error) { + id1, err := cloud.ParseResourceURL(fr1.BackendService) + if err != nil { + return false, fmt.Errorf("EqualIPv6ForwardingRules(): failed to parse backend resource URL from FR, err - %w", err) + } + id2, err := cloud.ParseResourceURL(fr2.BackendService) + if err != nil { + return false, fmt.Errorf("EqualIPv6ForwardingRules(): failed to parse resource URL from FR, err - %w", err) + } + return fr1.IPProtocol == fr2.IPProtocol && + fr1.LoadBalancingScheme == fr2.LoadBalancingScheme && + utils.EqualStringSets(fr1.Ports, fr2.Ports) && + id1.Equal(id2) && + fr1.AllowGlobalAccess == fr2.AllowGlobalAccess && + fr1.AllPorts == fr2.AllPorts && + fr1.Subnetwork == fr2.Subnetwork && + fr1.NetworkTier == fr2.NetworkTier, nil +} diff --git a/pkg/loadbalancers/l4.go b/pkg/loadbalancers/l4.go index 0938bfc1b2..54345cbeba 100644 --- a/pkg/loadbalancers/l4.go +++ b/pkg/loadbalancers/l4.go @@ -54,6 +54,7 @@ type L4 struct { NamespacedName types.NamespacedName l4HealthChecks healthchecks.L4HealthChecks forwardingRules ForwardingRulesProvider + enableDualStack bool } // L4ILBSyncResult contains information about the outcome of an L4 ILB sync. It stores the list of resource name annotations, @@ -69,7 +70,7 @@ type L4ILBSyncResult struct { } // NewL4Handler creates a new L4Handler for the given L4 service. -func NewL4Handler(service *corev1.Service, cloud *gce.Cloud, scope meta.KeyType, namer namer.L4ResourcesNamer, recorder record.EventRecorder) *L4 { +func NewL4Handler(service *corev1.Service, cloud *gce.Cloud, scope meta.KeyType, namer namer.L4ResourcesNamer, recorder record.EventRecorder, dualStackEnabled bool) *L4 { l := &L4{ cloud: cloud, scope: scope, @@ -78,6 +79,7 @@ func NewL4Handler(service *corev1.Service, cloud *gce.Cloud, scope meta.KeyType, Service: service, l4HealthChecks: healthchecks.L4(), forwardingRules: forwardingrules.New(cloud, meta.VersionGA, scope), + enableDualStack: dualStackEnabled, } l.NamespacedName = types.NamespacedName{Name: service.Name, Namespace: service.Namespace} l.backendPool = backends.NewPool(l.cloud, l.namer) @@ -107,12 +109,51 @@ func (l *L4) EnsureInternalLoadBalancerDeleted(svc *corev1.Service) *L4ILBSyncRe result.Error = fmt.Errorf("Namer does not support L4 Backends") return result } + + l.deleteIPv4Resources(result, name) + if l.enableDualStack { + l.deleteIPv6Resources(result, name) + } + + // Delete backend service + err := utils.IgnoreHTTPNotFound(l.backendPool.Delete(name, meta.VersionGA, meta.Regional)) + if err != nil { + klog.Errorf("Failed to delete backends for internal loadbalancer service %s, err %v", l.NamespacedName.String(), err) + result.GCEResourceInError = annotations.BackendServiceResource + result.Error = err + } + + // Delete healthcheck + // We don't delete health check during service update so + // it is possible that there might be some health check leak + // when externalTrafficPolicy is changed from Local to Cluster and a new health check was created. + // When service is deleted we need to check both health checks shared and non-shared + // and delete them if needed. + for _, isShared := range []bool{true, false} { + if l.enableDualStack { + resourceInError, err := l.l4HealthChecks.DeleteDualStackHealthCheck(svc, l.namer, isShared, meta.Global, utils.ILB, true) + if err != nil { + result.GCEResourceInError = resourceInError + result.Error = err + } + } else { + resourceInError, err := l.l4HealthChecks.DeleteHealthCheck(svc, l.namer, isShared, meta.Global, utils.ILB) + if err != nil { + result.GCEResourceInError = resourceInError + result.Error = err + } + } + } + return result +} + +func (l *L4) deleteIPv4Resources(result *L4ILBSyncResult, bsName string) { frName := l.GetFRName() key, err := l.CreateKey(frName) if err != nil { klog.Errorf("Failed to create key for LoadBalancer resources with name %s for service %s, err %v", frName, l.NamespacedName.String(), err) result.Error = err - return result + return } // If any resource deletion fails, log the error and continue cleanup. if err = utils.IgnoreHTTPNotFound(composite.DeleteForwardingRule(l.cloud, key, meta.VersionGA)); err != nil { @@ -120,41 +161,20 @@ func (l *L4) EnsureInternalLoadBalancerDeleted(svc *corev1.Service) *L4ILBSyncRe result.Error = err result.GCEResourceInError = annotations.ForwardingRuleResource } - if err = ensureAddressDeleted(l.cloud, name, l.cloud.Region()); err != nil { + + if err = ensureAddressDeleted(l.cloud, bsName, l.cloud.Region()); err != nil { klog.Errorf("Failed to delete address for internal loadbalancer service %s, err %v", l.NamespacedName.String(), err) result.Error = err result.GCEResourceInError = annotations.AddressResource } // delete firewall rule allowing load balancer source ranges - err = l.deleteFirewall(name) + err = l.deleteFirewall(bsName) if err != nil { - klog.Errorf("Failed to delete firewall rule %s for internal loadbalancer service %s, err %v", name, l.NamespacedName.String(), err) + klog.Errorf("Failed to delete firewall rule %s for internal loadbalancer service %s, err %v", bsName, l.NamespacedName.String(), err) result.GCEResourceInError = annotations.FirewallRuleResource result.Error = err } - // Delete backend service - err = utils.IgnoreHTTPNotFound(l.backendPool.Delete(name, meta.VersionGA, meta.Regional)) - if err != nil { - klog.Errorf("Failed to delete backends for internal loadbalancer service %s, err %v", l.NamespacedName.String(), err) - result.GCEResourceInError = annotations.BackendServiceResource - result.Error = err - } - - // Delete healthcheck - // We don't delete health check during service update so - // it is possible that there might be some health check leak - // when externalTrafficPolicy is changed from Local to Cluster and a new health check was created. - // When service is deleted we need to check both health checks shared and non-shared - // and delete them if needed. - for _, isShared := range []bool{true, false} { - resourceInError, err := l.l4HealthChecks.DeleteHealthCheck(svc, l.namer, isShared, meta.Global, utils.ILB) - if err != nil { - result.GCEResourceInError = resourceInError - result.Error = err - } - } - return result } func (l *L4) deleteFirewall(name string) error { @@ -207,8 +227,19 @@ func (l *L4) EnsureInternalLoadBalancer(nodeNames []string, svc *corev1.Service) // create healthcheck sharedHC := !helpers.RequestsOnlyLocalTraffic(l.Service) - hcResult := l.l4HealthChecks.EnsureL4HealthCheck(l.Service, l.namer, sharedHC, meta.Global, utils.ILB, nodeNames) - + var hcResult *healthchecks.EnsureL4HealthCheckResult + if l.enableDualStack { + hcResult = l.l4HealthChecks.EnsureL4DualStackHealthCheck(l.Service, l.namer, sharedHC, meta.Global, utils.ILB, nodeNames, utils.NeedsIPv4(l.Service), utils.NeedsIPv6(l.Service)) + if hcResult.HCFirewallRuleName != "" { + result.Annotations[annotations.FirewallRuleForHealthcheckKey] = hcResult.HCFirewallRuleName + } + if hcResult.HCFirewallRuleIPv6Name != "" { + result.Annotations[annotations.FirewallRuleForHealthcheckIPv6Key] = hcResult.HCFirewallRuleIPv6Name + } + } else { + hcResult = l.l4HealthChecks.EnsureL4HealthCheck(l.Service, l.namer, sharedHC, meta.Global, utils.ILB, nodeNames) + result.Annotations[annotations.FirewallRuleForHealthcheckKey] = hcResult.HCFirewallRuleName + } if hcResult.Err != nil { result.GCEResourceInError = hcResult.GceResourceInError result.Error = hcResult.Err @@ -217,7 +248,6 @@ func (l *L4) EnsureInternalLoadBalancer(nodeNames []string, svc *corev1.Service) result.Annotations[annotations.HealthcheckKey] = hcResult.HCName servicePorts := l.Service.Spec.Ports - portRanges := utils.GetServicePortRanges(servicePorts) protocol := utils.GetProtocol(servicePorts) // Check if protocol has changed for this service. In this case, forwarding rule should be deleted before @@ -227,6 +257,7 @@ func (l *L4) EnsureInternalLoadBalancer(nodeNames []string, svc *corev1.Service) if err != nil { klog.Errorf("Failed to lookup existing backend service, ignoring err: %v", err) } + existingFR, err := l.forwardingRules.Get(l.GetFRName()) if existingBS != nil && existingBS.Protocol != string(protocol) { klog.Infof("Protocol changed from %q to %q for service %s", existingBS.Protocol, string(protocol), l.NamespacedName) @@ -240,6 +271,15 @@ func (l *L4) EnsureInternalLoadBalancer(nodeNames []string, svc *corev1.Service) if err != nil { klog.Errorf("Failed to delete forwarding rule %s, err %v", frName, err) } + + if l.enableDualStack { + // Delete ipv6 forwarding rule if it exists + ipv6FrName := l.getIPv6FRNameWithProtocol(existingBS.Protocol) + err = l.forwardingRules.Delete(ipv6FrName) + if err != nil { + klog.Errorf("Failed to delete ipv6 forwarding rule %s, err %v", ipv6FrName, err) + } + } } // ensure backend service @@ -251,14 +291,49 @@ func (l *L4) EnsureInternalLoadBalancer(nodeNames []string, svc *corev1.Service) return result } result.Annotations[annotations.BackendServiceKey] = name - // create fr rule + + if l.enableDualStack { + if utils.NeedsIPv4(l.Service) { + l.ensureIPv4Resources(result, nodeNames, options, bs, existingFR) + if result.Error != nil { + return result + } + } else { + l.deleteIPv4Resources(result, name) + } + if utils.NeedsIPv6(l.Service) { + l.ensureIPv6Resources(result, nodeNames, options, bs.SelfLink, name) + if result.Error != nil { + return result + } + } else { + l.deleteIPv6Resources(result, name) + } + } else { + l.ensureIPv4Resources(result, nodeNames, options, bs, existingFR) + } + + result.MetricsState.InSuccess = true + if options.AllowGlobalAccess { + result.MetricsState.EnabledGlobalAccess = true + } + // SubnetName is overwritten to nil value if Alpha feature gate for custom subnet + // is not enabled. So, a non empty subnet name at this point implies that the + // feature is in use. + if options.SubnetName != "" { + result.MetricsState.EnabledCustomSubnet = true + } + return result +} + +func (l *L4) ensureIPv4Resources(result *L4ILBSyncResult, nodeNames []string, options gce.ILBOptions, bs *composite.BackendService, existingFR *composite.ForwardingRule) { frName := l.GetFRName() fr, err := l.ensureForwardingRule(frName, bs.SelfLink, options, existingFR) if err != nil { klog.Errorf("EnsureInternalLoadBalancer: Failed to create forwarding rule - %v", err) result.GCEResourceInError = annotations.ForwardingRuleResource result.Error = err - return result + return } if fr.IPProtocol == string(corev1.ProtocolTCP) { result.Annotations[annotations.TCPForwardingRuleKey] = frName @@ -270,15 +345,19 @@ func (l *L4) EnsureInternalLoadBalancer(nodeNames []string, svc *corev1.Service) sourceRanges, err := helpers.GetLoadBalancerSourceRanges(l.Service) if err != nil { result.Error = err - return result + return } + + servicePorts := l.Service.Spec.Ports + protocol := utils.GetProtocol(servicePorts) + portRanges := utils.GetServicePortRanges(servicePorts) // Add firewall rule for ILB traffic to nodes nodesFWRParams := firewalls.FirewallParams{ PortRanges: portRanges, SourceRanges: sourceRanges.StringSlice(), DestinationRanges: []string{fr.IPAddress}, Protocol: string(protocol), - Name: name, + Name: bs.Name, NodeNames: nodeNames, L4Type: utils.ILB, } @@ -286,21 +365,9 @@ func (l *L4) EnsureInternalLoadBalancer(nodeNames []string, svc *corev1.Service) if err := firewalls.EnsureL4LBFirewallForNodes(l.Service, &nodesFWRParams, l.cloud, l.recorder); err != nil { result.GCEResourceInError = annotations.FirewallRuleResource result.Error = err - return result + return } - result.Annotations[annotations.FirewallRuleKey] = name - result.Annotations[annotations.FirewallRuleForHealthcheckKey] = hcResult.HCFirewallRuleName + result.Annotations[annotations.FirewallRuleKey] = bs.Name - result.MetricsState.InSuccess = true - if options.AllowGlobalAccess { - result.MetricsState.EnabledGlobalAccess = true - } - // SubnetName is overwritten to nil value if Alpha feature gate for custom subnet - // is not enabled. So, a non empty subnet name at this point implies that the - // feature is in use. - if options.SubnetName != "" { - result.MetricsState.EnabledCustomSubnet = true - } - result.Status = &corev1.LoadBalancerStatus{Ingress: []corev1.LoadBalancerIngress{{IP: fr.IPAddress}}} - return result + result.Status = utils.AddIPToLBStatus(result.Status, fr.IPAddress) } diff --git a/pkg/loadbalancers/l4_test.go b/pkg/loadbalancers/l4_test.go index b0dbcf613e..cffaaa9e9e 100644 --- a/pkg/loadbalancers/l4_test.go +++ b/pkg/loadbalancers/l4_test.go @@ -23,11 +23,11 @@ import ( "strings" "testing" - "k8s.io/ingress-gce/pkg/healthchecks" - + "github.com/google/go-cmp/cmp" "google.golang.org/api/compute/v1" "k8s.io/ingress-gce/pkg/backends" "k8s.io/ingress-gce/pkg/firewalls" + "k8s.io/ingress-gce/pkg/healthchecks" "k8s.io/ingress-gce/pkg/utils" "github.com/GoogleCloudPlatform/k8s-cloud-provider/pkg/cloud" @@ -68,7 +68,7 @@ func TestEnsureInternalBackendServiceUpdates(t *testing.T) { svc := test.NewL4ILBService(false, 8080) namer := namer_util.NewL4Namer(kubeSystemUID, nil) - l := NewL4Handler(svc, fakeGCE, meta.Regional, namer, record.NewFakeRecorder(100)) + l := NewL4Handler(svc, fakeGCE, meta.Regional, namer, record.NewFakeRecorder(100), false) l.l4HealthChecks = healthchecks.FakeL4(fakeGCE, &test.FakeRecorderSource{}) bsName, _ := l.namer.L4Backend(l.Service.Namespace, l.Service.Name) @@ -119,7 +119,7 @@ func TestEnsureInternalLoadBalancer(t *testing.T) { svc := test.NewL4ILBService(false, 8080) namer := namer_util.NewL4Namer(kubeSystemUID, nil) - l := NewL4Handler(svc, fakeGCE, meta.Regional, namer, record.NewFakeRecorder(100)) + l := NewL4Handler(svc, fakeGCE, meta.Regional, namer, record.NewFakeRecorder(100), false) l.l4HealthChecks = healthchecks.FakeL4(fakeGCE, &test.FakeRecorderSource{}) if _, err := test.CreateAndInsertNodes(l.cloud, nodeNames, vals.ZoneName); err != nil { @@ -176,7 +176,7 @@ func TestEnsureInternalLoadBalancerTypeChange(t *testing.T) { svc := test.NewL4ILBService(false, 8080) namer := namer_util.NewL4Namer(kubeSystemUID, nil) - l := NewL4Handler(svc, fakeGCE, meta.Regional, namer, record.NewFakeRecorder(100)) + l := NewL4Handler(svc, fakeGCE, meta.Regional, namer, record.NewFakeRecorder(100), false) l.l4HealthChecks = healthchecks.FakeL4(fakeGCE, &test.FakeRecorderSource{}) if _, err := test.CreateAndInsertNodes(l.cloud, nodeNames, vals.ZoneName); err != nil { @@ -210,7 +210,7 @@ func TestEnsureInternalLoadBalancerWithExistingResources(t *testing.T) { svc := test.NewL4ILBService(false, 8080) namer := namer_util.NewL4Namer(kubeSystemUID, nil) - l := NewL4Handler(svc, fakeGCE, meta.Regional, namer, record.NewFakeRecorder(100)) + l := NewL4Handler(svc, fakeGCE, meta.Regional, namer, record.NewFakeRecorder(100), false) l.l4HealthChecks = healthchecks.FakeL4(fakeGCE, &test.FakeRecorderSource{}) if _, err := test.CreateAndInsertNodes(l.cloud, nodeNames, vals.ZoneName); err != nil { @@ -253,7 +253,7 @@ func TestEnsureInternalLoadBalancerClearPreviousResources(t *testing.T) { svc := test.NewL4ILBService(true, 8080) namer := namer_util.NewL4Namer(kubeSystemUID, nil) - l := NewL4Handler(svc, fakeGCE, meta.Regional, namer, record.NewFakeRecorder(100)) + l := NewL4Handler(svc, fakeGCE, meta.Regional, namer, record.NewFakeRecorder(100), false) l.l4HealthChecks = healthchecks.FakeL4(fakeGCE, &test.FakeRecorderSource{}) _, err := test.CreateAndInsertNodes(l.cloud, nodeNames, vals.ZoneName) @@ -373,7 +373,7 @@ func TestUpdateResourceLinks(t *testing.T) { svc := test.NewL4ILBService(true, 8080) namer := namer_util.NewL4Namer(kubeSystemUID, nil) - l := NewL4Handler(svc, fakeGCE, meta.Regional, namer, record.NewFakeRecorder(100)) + l := NewL4Handler(svc, fakeGCE, meta.Regional, namer, record.NewFakeRecorder(100), false) l.l4HealthChecks = healthchecks.FakeL4(fakeGCE, &test.FakeRecorderSource{}) _, err := test.CreateAndInsertNodes(l.cloud, nodeNames, vals.ZoneName) @@ -451,7 +451,7 @@ func TestEnsureInternalLoadBalancerHealthCheckConfigurable(t *testing.T) { svc := test.NewL4ILBService(true, 8080) namer := namer_util.NewL4Namer(kubeSystemUID, nil) - l := NewL4Handler(svc, fakeGCE, meta.Regional, namer, record.NewFakeRecorder(100)) + l := NewL4Handler(svc, fakeGCE, meta.Regional, namer, record.NewFakeRecorder(100), false) l.l4HealthChecks = healthchecks.FakeL4(fakeGCE, &test.FakeRecorderSource{}) _, err := test.CreateAndInsertNodes(l.cloud, nodeNames, vals.ZoneName) @@ -494,7 +494,7 @@ func TestEnsureInternalLoadBalancerDeleted(t *testing.T) { nodeNames := []string{"test-node-1"} svc := test.NewL4ILBService(false, 8080) namer := namer_util.NewL4Namer(kubeSystemUID, nil) - l := NewL4Handler(svc, fakeGCE, meta.Regional, namer, record.NewFakeRecorder(100)) + l := NewL4Handler(svc, fakeGCE, meta.Regional, namer, record.NewFakeRecorder(100), false) l.l4HealthChecks = healthchecks.FakeL4(fakeGCE, &test.FakeRecorderSource{}) if _, err := test.CreateAndInsertNodes(l.cloud, nodeNames, vals.ZoneName); err != nil { @@ -526,7 +526,7 @@ func TestEnsureInternalLoadBalancerDeletedTwiceDoesNotError(t *testing.T) { nodeNames := []string{"test-node-1"} svc := test.NewL4ILBService(false, 8080) namer := namer_util.NewL4Namer(kubeSystemUID, nil) - l := NewL4Handler(svc, fakeGCE, meta.Regional, namer, record.NewFakeRecorder(100)) + l := NewL4Handler(svc, fakeGCE, meta.Regional, namer, record.NewFakeRecorder(100), false) l.l4HealthChecks = healthchecks.FakeL4(fakeGCE, &test.FakeRecorderSource{}) if _, err := test.CreateAndInsertNodes(l.cloud, nodeNames, vals.ZoneName); err != nil { @@ -643,7 +643,7 @@ func TestHealthCheckFirewallDeletionWithNetLB(t *testing.T) { func ensureService(fakeGCE *gce.Cloud, namer *namer_util.L4Namer, nodeNames []string, zoneName string, port int, t *testing.T) (*v1.Service, *L4, *L4ILBSyncResult) { svc := test.NewL4ILBService(false, 8080) - l := NewL4Handler(svc, fakeGCE, meta.Regional, namer, record.NewFakeRecorder(100)) + l := NewL4Handler(svc, fakeGCE, meta.Regional, namer, record.NewFakeRecorder(100), false) l.l4HealthChecks = healthchecks.FakeL4(fakeGCE, &test.FakeRecorderSource{}) if _, err := test.CreateAndInsertNodes(l.cloud, nodeNames, zoneName); err != nil { @@ -668,7 +668,7 @@ func TestEnsureInternalLoadBalancerWithSpecialHealthCheck(t *testing.T) { nodeNames := []string{"test-node-1"} svc := test.NewL4ILBService(false, 8080) namer := namer_util.NewL4Namer(kubeSystemUID, nil) - l := NewL4Handler(svc, fakeGCE, meta.Regional, namer, record.NewFakeRecorder(100)) + l := NewL4Handler(svc, fakeGCE, meta.Regional, namer, record.NewFakeRecorder(100), false) l.l4HealthChecks = healthchecks.FakeL4(fakeGCE, &test.FakeRecorderSource{}) if _, err := test.CreateAndInsertNodes(l.cloud, nodeNames, vals.ZoneName); err != nil { @@ -775,7 +775,7 @@ func TestEnsureInternalLoadBalancerErrors(t *testing.T) { namer := namer_util.NewL4Namer(kubeSystemUID, nil) fakeGCE := getFakeGCECloud(gce.DefaultTestClusterValues()) - l := NewL4Handler(params.service, fakeGCE, meta.Regional, namer, record.NewFakeRecorder(100)) + l := NewL4Handler(params.service, fakeGCE, meta.Regional, namer, record.NewFakeRecorder(100), false) l.l4HealthChecks = healthchecks.FakeL4(fakeGCE, &test.FakeRecorderSource{}) //lbName := l.namer.L4Backend(params.service.Namespace, params.service.Name) @@ -858,7 +858,7 @@ func TestEnsureInternalLoadBalancerEnableGlobalAccess(t *testing.T) { nodeNames := []string{"test-node-1"} svc := test.NewL4ILBService(false, 8080) namer := namer_util.NewL4Namer(kubeSystemUID, nil) - l := NewL4Handler(svc, fakeGCE, meta.Regional, namer, record.NewFakeRecorder(100)) + l := NewL4Handler(svc, fakeGCE, meta.Regional, namer, record.NewFakeRecorder(100), false) l.l4HealthChecks = healthchecks.FakeL4(fakeGCE, &test.FakeRecorderSource{}) if _, err := test.CreateAndInsertNodes(l.cloud, nodeNames, vals.ZoneName); err != nil { @@ -940,7 +940,7 @@ func TestEnsureInternalLoadBalancerCustomSubnet(t *testing.T) { svc := test.NewL4ILBService(false, 8080) namer := namer_util.NewL4Namer(kubeSystemUID, nil) - l := NewL4Handler(svc, fakeGCE, meta.Regional, namer, record.NewFakeRecorder(100)) + l := NewL4Handler(svc, fakeGCE, meta.Regional, namer, record.NewFakeRecorder(100), false) l.l4HealthChecks = healthchecks.FakeL4(fakeGCE, &test.FakeRecorderSource{}) if _, err := test.CreateAndInsertNodes(l.cloud, nodeNames, vals.ZoneName); err != nil { @@ -1038,7 +1038,7 @@ func TestEnsureInternalFirewallPortRanges(t *testing.T) { svc := test.NewL4ILBService(false, 8080) namer := namer_util.NewL4Namer(kubeSystemUID, nil) - l := NewL4Handler(svc, fakeGCE, meta.Regional, namer, record.NewFakeRecorder(100)) + l := NewL4Handler(svc, fakeGCE, meta.Regional, namer, record.NewFakeRecorder(100), false) l.l4HealthChecks = healthchecks.FakeL4(fakeGCE, &test.FakeRecorderSource{}) fwName, _ := l.namer.L4Backend(l.Service.Namespace, l.Service.Name) @@ -1093,7 +1093,7 @@ func TestEnsureInternalLoadBalancerModifyProtocol(t *testing.T) { nodeNames := []string{"test-node-1"} svc := test.NewL4ILBService(false, 8080) namer := namer_util.NewL4Namer(kubeSystemUID, nil) - l := NewL4Handler(svc, fakeGCE, meta.Regional, namer, record.NewFakeRecorder(100)) + l := NewL4Handler(svc, fakeGCE, meta.Regional, namer, record.NewFakeRecorder(100), false) l.l4HealthChecks = healthchecks.FakeL4(fakeGCE, &test.FakeRecorderSource{}) _, err := test.CreateAndInsertNodes(l.cloud, nodeNames, vals.ZoneName) @@ -1185,7 +1185,7 @@ func TestEnsureInternalLoadBalancerAllPorts(t *testing.T) { nodeNames := []string{"test-node-1"} svc := test.NewL4ILBService(false, 8080) namer := namer_util.NewL4Namer(kubeSystemUID, nil) - l := NewL4Handler(svc, fakeGCE, meta.Regional, namer, record.NewFakeRecorder(100)) + l := NewL4Handler(svc, fakeGCE, meta.Regional, namer, record.NewFakeRecorder(100), false) l.l4HealthChecks = healthchecks.FakeL4(fakeGCE, &test.FakeRecorderSource{}) if _, err := test.CreateAndInsertNodes(l.cloud, nodeNames, vals.ZoneName); err != nil { @@ -1272,6 +1272,254 @@ func TestEnsureInternalLoadBalancerAllPorts(t *testing.T) { assertInternalLbResourcesDeleted(t, svc, true, l) } +func TestEnsureInternalDualStackLoadBalancer(t *testing.T) { + t.Parallel() + nodeNames := []string{"test-node-1"} + vals := gce.DefaultTestClusterValues() + + namer := namer_util.NewL4Namer(kubeSystemUID, nil) + + testCases := []struct { + ipFamilies []v1.IPFamily + localPolicy bool + desc string + }{ + { + ipFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, + desc: "Test ipv4 ipv6 service", + }, + { + ipFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, + localPolicy: true, + desc: "Test ipv4 ipv6 local service", + }, + { + ipFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol}, + desc: "Test ipv6 ipv4 service", + }, + { + ipFamilies: []v1.IPFamily{v1.IPv4Protocol}, + desc: "Test ipv4 service", + }, + { + ipFamilies: []v1.IPFamily{v1.IPv6Protocol}, + desc: "Test ipv6 service", + }, + } + + for _, tc := range testCases { + tc := tc + + t.Run(tc.desc, func(t *testing.T) { + t.Parallel() + + fakeGCE := getFakeGCECloud(vals) + + svc := test.NewL4ILBDualStackService(tc.localPolicy, 8080, tc.ipFamilies) + + l := NewL4Handler(svc, fakeGCE, meta.Regional, namer, record.NewFakeRecorder(100), true) + l.l4HealthChecks = healthchecks.FakeL4(fakeGCE, &test.FakeRecorderSource{}) + + if _, err := test.CreateAndInsertNodes(l.cloud, nodeNames, vals.ZoneName); err != nil { + t.Errorf("Unexpected error when adding nodes %v", err) + } + + result := l.EnsureInternalLoadBalancer(nodeNames, svc) + if result.Error != nil { + t.Errorf("Failed to ensure loadBalancer, err %v", result.Error) + } + if len(result.Status.Ingress) == 0 { + t.Errorf("Got empty loadBalancer status using handler %v", l) + } + assertInternalDualStackLbResources(t, svc, l, nodeNames, result.Annotations) + + backendServiceName, _ := l.namer.L4Backend(l.Service.Namespace, l.Service.Name) + key := meta.RegionalKey(backendServiceName, l.cloud.Region()) + bs, err := composite.GetBackendService(l.cloud, key, meta.VersionGA) + if err != nil { + t.Errorf("Failed to lookup backend service, err %v", err) + } + if len(bs.Backends) != 0 { + // Backends are populated by NEG linker. + t.Errorf("Unexpected backends list - %v, expected empty", bs.Backends) + } + // Add a backend list to simulate NEG linker populating the backends. + bs.Backends = []*composite.Backend{{Group: "test"}} + if err := composite.UpdateBackendService(l.cloud, key, bs); err != nil { + t.Errorf("Failed updating backend service, err %v", err) + } + // Simulate a periodic sync. The backends list should not be reconciled. + result = l.EnsureInternalLoadBalancer(nodeNames, svc) + if result.Error != nil { + t.Errorf("Failed to ensure loadBalancer, err %v", result.Error) + } + if len(result.Status.Ingress) == 0 { + t.Errorf("Got empty loadBalancer status using handler %v", l) + } + assertInternalDualStackLbResources(t, svc, l, nodeNames, result.Annotations) + bs, err = composite.GetBackendService(l.cloud, meta.RegionalKey(backendServiceName, l.cloud.Region()), meta.VersionGA) + if err != nil { + t.Errorf("Failed to lookup backend service, err %v", err) + } + if len(bs.Backends) == 0 { + t.Errorf("Backends got reconciled by the periodic sync") + } + }) + } +} + +func assertInternalDualStackLbResources(t *testing.T, apiService *v1.Service, l *L4, nodeNames []string, resourceAnnotations map[string]string) { + // Check that Firewalls are created for the LoadBalancer and the HealthCheck + sharedHC := !servicehelper.RequestsOnlyLocalTraffic(apiService) + resourceName, _ := l.namer.L4Backend(l.Service.Namespace, l.Service.Name) + resourceDesc, err := utils.MakeL4LBServiceDescription(utils.ServiceKeyFunc(apiService.Namespace, apiService.Name), "", meta.VersionGA, false, utils.ILB) + + if err != nil { + t.Errorf("Failed to create description for resources, err %v", err) + } + sharedResourceDesc, err := utils.MakeL4LBServiceDescription(utils.ServiceKeyFunc(apiService.Namespace, apiService.Name), "", meta.VersionGA, true, utils.ILB) + if err != nil { + t.Errorf("Failed to create description for shared resources, err %v", err) + } + proto := utils.GetProtocol(apiService.Spec.Ports) + expectedAnnotations := make(map[string]string) + hcName, hcFwName := l.namer.L4HealthCheck(apiService.Namespace, apiService.Name, sharedHC) + ipv6hcFwName := l.namer.L4IPv6HealthCheckFirewall(apiService.Namespace, apiService.Name, sharedHC) + // hcDesc is the resource description for healthcheck and firewall rule allowing healthcheck. + hcDesc := resourceDesc + if sharedHC { + hcDesc = sharedResourceDesc + } + + type nameAndDesc struct { + fwName string + fwDesc string + } + var fwNamesAndDesc []nameAndDesc + + if utils.NeedsIPv4(apiService) { + fwNamesAndDesc = append(fwNamesAndDesc, nameAndDesc{resourceName, resourceDesc}) + fwNamesAndDesc = append(fwNamesAndDesc, nameAndDesc{hcFwName, hcDesc}) + expectedAnnotations[annotations.FirewallRuleForHealthcheckKey] = hcFwName + expectedAnnotations[annotations.FirewallRuleKey] = resourceName + } + if utils.NeedsIPv6(apiService) { + ipv6FirewallName := l.getIPv6FirewallName(resourceName) + fwNamesAndDesc = append(fwNamesAndDesc, nameAndDesc{ipv6FirewallName, resourceDesc}) + fwNamesAndDesc = append(fwNamesAndDesc, nameAndDesc{ipv6hcFwName, hcDesc}) + expectedAnnotations[annotations.FirewallRuleForHealthcheckIPv6Key] = ipv6hcFwName + expectedAnnotations[annotations.FirewallRuleIPv6Key] = ipv6FirewallName + } + + for _, info := range fwNamesAndDesc { + firewall, err := l.cloud.GetFirewall(info.fwName) + if err != nil { + t.Fatalf("Failed to fetch firewall rule %q - err %v", info.fwName, err) + } + if !utils.EqualStringSets(nodeNames, firewall.TargetTags) { + t.Fatalf("Expected firewall rule target tags '%v', Got '%v'", nodeNames, firewall.TargetTags) + } + if len(firewall.SourceRanges) == 0 { + t.Fatalf("Unexpected empty source range for firewall rule %v", firewall) + } + if !sharedHC && firewall.Description != info.fwDesc { + t.Errorf("Unexpected description in firewall %q - Expected %s, Got %s", info.fwName, firewall.Description, info.fwDesc) + } + } + + // Check that HealthCheck is created + healthcheck, err := composite.GetHealthCheck(l.cloud, meta.GlobalKey(hcName), meta.VersionGA) + if err != nil { + t.Errorf("Failed to fetch healthcheck %s - err %v", hcName, err) + } + if healthcheck.Name != hcName { + t.Errorf("Unexpected name for healthcheck '%s' - expected '%s'", healthcheck.Name, hcName) + } + expectedAnnotations[annotations.HealthcheckKey] = hcName + // Only non-shared Healthchecks get a description. + if healthcheck.Description != hcDesc { + t.Errorf("Unexpected description in healthcheck - Expected %s, Got %s", healthcheck.Description, resourceDesc) + } + + // Check that BackendService exists + backendServiceName := resourceName + key := meta.RegionalKey(backendServiceName, l.cloud.Region()) + backendServiceLink := cloud.SelfLink(meta.VersionGA, l.cloud.ProjectID(), "backendServices", key) + bs, err := composite.GetBackendService(l.cloud, key, meta.VersionGA) + if err != nil { + t.Errorf("Failed to fetch backend service %s - err %v", backendServiceName, err) + } + if bs.Protocol != string(proto) { + t.Errorf("Unexpected protocol '%s' for backend service %v", bs.Protocol, bs) + } + if bs.SelfLink != backendServiceLink { + t.Errorf("Unexpected self link in backend service - Expected %s, Got %s", bs.SelfLink, backendServiceLink) + } + if bs.Description != resourceDesc { + t.Errorf("Unexpected description in backend service - Expected %s, Got %s", bs.Description, resourceDesc) + } + if !utils.EqualStringSets(bs.HealthChecks, []string{healthcheck.SelfLink}) { + t.Errorf("Unexpected healthcheck reference '%v' in backend service, expected '%s'", bs.HealthChecks, + healthcheck.SelfLink) + } + expectedAnnotations[annotations.BackendServiceKey] = backendServiceName + subnet := apiService.Annotations[gce.ServiceAnnotationILBSubnet] + if subnet == "" { + subnet = l.cloud.SubnetworkURL() + } else { + key.Name = subnet + subnet = cloud.SelfLink(meta.VersionGA, l.cloud.ProjectID(), "subnetworks", key) + } + // Check that ForwardingRule is created + var expectedFwdRules []string + if utils.NeedsIPv4(apiService) { + ipv4FRName := l.GetFRName() + expectedFwdRules = append(expectedFwdRules, ipv4FRName) + if proto == v1.ProtocolTCP { + expectedAnnotations[annotations.TCPForwardingRuleKey] = ipv4FRName + } else { + expectedAnnotations[annotations.UDPForwardingRuleKey] = ipv4FRName + } + } + if utils.NeedsIPv6(apiService) { + ipv6FRName := l.getIPv6FRName() + expectedFwdRules = append(expectedFwdRules, ipv6FRName) + if proto == v1.ProtocolTCP { + expectedAnnotations[annotations.TCPForwardingRuleIPv6Key] = ipv6FRName + } else { + expectedAnnotations[annotations.UDPForwardingRuleIPv6Key] = ipv6FRName + } + } + for _, frName := range expectedFwdRules { + fwdRule, err := composite.GetForwardingRule(l.cloud, meta.RegionalKey(frName, l.cloud.Region()), meta.VersionGA) + if err != nil { + t.Errorf("Failed to fetch forwarding rule %s - err %v", frName, err) + + } + if fwdRule.Name != frName { + t.Errorf("Unexpected name for forwarding rule '%s' - expected '%s'", fwdRule.Name, frName) + } + if fwdRule.IPProtocol != string(proto) { + t.Errorf("Unexpected protocol '%s' for forwarding rule %v", fwdRule.IPProtocol, fwdRule) + } + if fwdRule.BackendService != backendServiceLink { + t.Errorf("Unexpected backend service link '%s' for forwarding rule, expected '%s'", fwdRule.BackendService, backendServiceLink) + } + if fwdRule.Subnetwork != subnet { + t.Errorf("Unexpected subnetwork %q in forwarding rule, expected %q", + fwdRule.Subnetwork, subnet) + } + addr, err := l.cloud.GetRegionAddress(frName, l.cloud.Region()) + if err == nil || addr != nil { + t.Errorf("Expected error when looking up ephemeral address, got %v", addr) + } + } + if !reflect.DeepEqual(expectedAnnotations, resourceAnnotations) { + diff := cmp.Diff(expectedAnnotations, resourceAnnotations) + t.Fatalf("Expected annotations %v, got %v, diff %v", expectedAnnotations, resourceAnnotations, diff) + } +} + func assertInternalLbResources(t *testing.T, apiService *v1.Service, l *L4, nodeNames []string, resourceAnnotations map[string]string) { // Check that Firewalls are created for the LoadBalancer and the HealthCheck sharedHC := !servicehelper.RequestsOnlyLocalTraffic(apiService) diff --git a/pkg/loadbalancers/l4ipv6.go b/pkg/loadbalancers/l4ipv6.go new file mode 100644 index 0000000000..a3fa379b1f --- /dev/null +++ b/pkg/loadbalancers/l4ipv6.go @@ -0,0 +1,102 @@ +package loadbalancers + +import ( + "strings" + + "github.com/GoogleCloudPlatform/k8s-cloud-provider/pkg/cloud/meta" + corev1 "k8s.io/api/core/v1" + "k8s.io/ingress-gce/pkg/annotations" + "k8s.io/ingress-gce/pkg/composite" + "k8s.io/ingress-gce/pkg/firewalls" + "k8s.io/ingress-gce/pkg/utils" + "k8s.io/klog/v2" + "k8s.io/legacy-cloud-providers/gce" +) + +func (l *L4) ensureIPv6Resources(syncResult *L4ILBSyncResult, nodeNames []string, options gce.ILBOptions, bsLink string, bsName string) { + ipv6fr, err := l.ensureIPv6ForwardingRule(bsLink, options) + if err != nil { + klog.Errorf("ensureIPv6Resources: Failed to create ipv6 forwarding rule - %v", err) + syncResult.GCEResourceInError = annotations.ForwardingRuleIPv6Resource + syncResult.Error = err + } + + if ipv6fr.IPProtocol == string(corev1.ProtocolTCP) { + syncResult.Annotations[annotations.TCPForwardingRuleIPv6Key] = ipv6fr.Name + } else { + syncResult.Annotations[annotations.UDPForwardingRuleIPv6Key] = ipv6fr.Name + } + + firewallName := l.getIPv6FirewallName(bsName) + err = l.ensureIPv6Firewall(ipv6fr, firewallName, nodeNames) + if err != nil { + syncResult.GCEResourceInError = annotations.FirewallRuleIPv6Resource + syncResult.Error = err + return + } + syncResult.Annotations[annotations.FirewallRuleIPv6Key] = firewallName + + trimmedIPv6Address := strings.Split(ipv6fr.IPAddress, "/")[0] + syncResult.Status = utils.AddIPToLBStatus(syncResult.Status, trimmedIPv6Address) +} + +func (l *L4) deleteIPv6Resources(syncResult *L4ILBSyncResult, bsName string) { + err := l.deleteIPv6ForwardingRule() + if err != nil { + klog.Errorf("Failed to delete ipv6 forwarding rule for internal loadbalancer service %s, err %v", l.NamespacedName.String(), err) + syncResult.Error = err + syncResult.GCEResourceInError = annotations.ForwardingRuleIPv6Resource + } + + err = l.deleteIPv6Firewall(bsName) + if err != nil { + klog.Errorf("Failed to delete ipv6 firewall rule for internal loadbalancer service %s, err %v", l.NamespacedName.String(), err) + syncResult.GCEResourceInError = annotations.FirewallRuleIPv6Resource + syncResult.Error = err + } +} + +func (l *L4) getIPv6FRName() string { + protocol := utils.GetProtocol(l.Service.Spec.Ports) + return l.getIPv6FRNameWithProtocol(string(protocol)) +} + +func (l *L4) getIPv6FRNameWithProtocol(protocol string) string { + return l.namer.L4IPv6ForwardingRule(l.Service.Namespace, l.Service.Name, strings.ToLower(protocol)) +} + +func (l *L4) getIPv6FirewallName(bsName string) string { + return bsName + "-ipv6" +} + +func (l *L4) ensureIPv6Firewall(forwardingRule *composite.ForwardingRule, firewallName string, nodeNames []string) error { + svcPorts := l.Service.Spec.Ports + portRanges := utils.GetServicePortRanges(svcPorts) + protocol := utils.GetProtocol(svcPorts) + + ipv6nodesFWRParams := firewalls.FirewallParams{ + PortRanges: portRanges, + // TODO(panslava): support .spec.loadBalancerSourceRanges + SourceRanges: []string{"0::0/0"}, + DestinationRanges: []string{forwardingRule.IPAddress}, + Protocol: string(protocol), + Name: firewallName, + NodeNames: nodeNames, + L4Type: utils.ILB, + } + + return firewalls.EnsureL4LBFirewallForNodes(l.Service, &ipv6nodesFWRParams, l.cloud, l.recorder) +} + +func (l *L4) deleteIPv6ForwardingRule() error { + ipv6FrName := l.getIPv6FRName() + ipv6key, err := l.CreateKey(ipv6FrName) + if err != nil { + return err + } + return utils.IgnoreHTTPNotFound(composite.DeleteForwardingRule(l.cloud, ipv6key, meta.VersionGA)) +} + +func (l *L4) deleteIPv6Firewall(bsName string) error { + return l.deleteFirewall(l.getIPv6FirewallName(bsName)) +} diff --git a/pkg/loadbalancers/l4netlb.go b/pkg/loadbalancers/l4netlb.go index 8e108b7184..cb96062d08 100644 --- a/pkg/loadbalancers/l4netlb.go +++ b/pkg/loadbalancers/l4netlb.go @@ -158,7 +158,8 @@ func (l4netlb *L4NetLB) EnsureFrontend(nodeNames []string, svc *corev1.Service) } else { result.Annotations[annotations.UDPForwardingRuleKey] = fr.Name } - result.Status = &corev1.LoadBalancerStatus{Ingress: []corev1.LoadBalancerIngress{{IP: fr.IPAddress}}} + + result.Status = utils.AddIPToLBStatus(result.Status, fr.IPAddress) result.MetricsState.IsPremiumTier = fr.NetworkTier == cloud.NetworkTierPremium.ToGCEValue() result.MetricsState.IsManagedIP = ipAddrType == IPAddrManaged if fr.NetworkTier == cloud.NetworkTierPremium.ToGCEValue() { diff --git a/pkg/test/utils.go b/pkg/test/utils.go index af7294e3f5..1c749b48c1 100644 --- a/pkg/test/utils.go +++ b/pkg/test/utils.go @@ -97,6 +97,13 @@ func NewL4ILBService(onlyLocal bool, port int) *api_v1.Service { return svc } +// NewL4ILBDualStackService creates a Service of type LoadBalancer with the Internal annotation and provided ipFamilies and ipFamilyPolicy +func NewL4ILBDualStackService(onlyLocal bool, port int, ipFamilies []api_v1.IPFamily) *api_v1.Service { + svc := NewL4ILBService(onlyLocal, port) + svc.Spec.IPFamilies = ipFamilies + return svc +} + func NewL4LegacyNetLBServiceWithoutPorts() *api_v1.Service { svc := &api_v1.Service{ ObjectMeta: meta_v1.ObjectMeta{ diff --git a/pkg/utils/ipfamily.go b/pkg/utils/ipfamily.go new file mode 100644 index 0000000000..6cb119d8cb --- /dev/null +++ b/pkg/utils/ipfamily.go @@ -0,0 +1,33 @@ +package utils + +import "k8s.io/api/core/v1" + +func NeedsIPv6(service *v1.Service) bool { + return supportsIPFamily(service, v1.IPv6Protocol) +} + +func NeedsIPv4(service *v1.Service) bool { + if service == nil { + return false + } + // Should never happen, defensive coding if kube-api did not populate IPFamilies + if len(service.Spec.IPFamilies) == 0 { + return true + } + + return supportsIPFamily(service, v1.IPv4Protocol) +} + +func supportsIPFamily(service *v1.Service, ipFamily v1.IPFamily) bool { + if service == nil { + return false + } + + ipFamilies := service.Spec.IPFamilies + for _, ipf := range ipFamilies { + if ipf == ipFamily { + return true + } + } + return false +} diff --git a/pkg/utils/ipfamily_test.go b/pkg/utils/ipfamily_test.go new file mode 100644 index 0000000000..9df524bbc8 --- /dev/null +++ b/pkg/utils/ipfamily_test.go @@ -0,0 +1,104 @@ +package utils + +import ( + "testing" + + "k8s.io/api/core/v1" +) + +func TestNeedsIPv6(t *testing.T) { + testCases := []struct { + service *v1.Service + wantNeedsIPv6 bool + desc string + }{ + { + service: nil, + wantNeedsIPv6: false, + desc: "Should return false for nil pointer", + }, + { + service: &v1.Service{Spec: v1.ServiceSpec{ + IPFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, + }}, + wantNeedsIPv6: true, + desc: "Should detect ipv6 for dual-stack ip families", + }, + { + service: &v1.Service{Spec: v1.ServiceSpec{ + IPFamilies: []v1.IPFamily{v1.IPv4Protocol}, + }}, + wantNeedsIPv6: false, + desc: "Should not detect ipv6 for only ipv4 families", + }, + { + service: &v1.Service{Spec: v1.ServiceSpec{ + IPFamilies: []v1.IPFamily{v1.IPv6Protocol}, + }}, + wantNeedsIPv6: true, + desc: "Should detect ipv6 for only ipv6 families", + }, + } + + for _, tc := range testCases { + t.Run(tc.desc, func(t *testing.T) { + needsIPv6 := NeedsIPv6(tc.service) + + if needsIPv6 != tc.wantNeedsIPv6 { + t.Errorf("NeedsIPv6(%v) returned %t, not equal to expected wantNeedsIPv6 = %t", tc.service, needsIPv6, tc.wantNeedsIPv6) + } + }) + } +} + +func TestNeedsIPv4(t *testing.T) { + testCases := []struct { + service *v1.Service + wantNeedsIPv4 bool + desc string + }{ + { + service: nil, + wantNeedsIPv4: false, + desc: "Should return false for nil pointer", + }, + { + service: &v1.Service{Spec: v1.ServiceSpec{ + IPFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, + }}, + wantNeedsIPv4: true, + desc: "Should handle dual-stack ip families", + }, + { + service: &v1.Service{Spec: v1.ServiceSpec{ + IPFamilies: []v1.IPFamily{v1.IPv4Protocol}, + }}, + wantNeedsIPv4: true, + desc: "Should handle only ipv4 family", + }, + { + service: &v1.Service{Spec: v1.ServiceSpec{ + IPFamilies: []v1.IPFamily{v1.IPv6Protocol}, + }}, + wantNeedsIPv4: false, + desc: "Should not handle only ipv6 family", + }, + { + service: &v1.Service{Spec: v1.ServiceSpec{ + IPFamilies: []v1.IPFamily{}, + }}, + wantNeedsIPv4: true, + desc: "Empty families should be recognized as IPv4. Should never happen in real life", + }, + } + + for _, tc := range testCases { + t.Run(tc.desc, func(t *testing.T) { + needsIPv4 := NeedsIPv4(tc.service) + + if needsIPv4 != tc.wantNeedsIPv4 { + t.Errorf("NeedsIPv4(%v) returned %t, not equal to expected wantNeedsIPv6 = %t", tc.service, needsIPv4, tc.wantNeedsIPv4) + } + }) + } +} diff --git a/pkg/utils/namer/interfaces.go b/pkg/utils/namer/interfaces.go index 5394970bd6..28e1ebb281 100644 --- a/pkg/utils/namer/interfaces.go +++ b/pkg/utils/namer/interfaces.go @@ -87,8 +87,12 @@ type L4ResourcesNamer interface { BackendNamer // L4ForwardingRule returns the name of the forwarding rule for the given service and protocol. L4ForwardingRule(namespace, name, protocol string) string + // L4IPv6ForwardingRule returns the name of the IPv6 forwarding rule for the given service and protocol. + L4IPv6ForwardingRule(namespace, name, protocol string) string // L4HealthCheck returns the names of the Healthcheck and HC-firewall rule. L4HealthCheck(namespace, name string, shared bool) (string, string) + // L4IPv6HealthCheckFirewall returns the name of the IPv6 L4 LB health check firewall rule. + L4IPv6HealthCheckFirewall(namespace, name string, shared bool) string // IsNEG returns if the given name is a VM_IP_NEG name. IsNEG(name string) bool } diff --git a/pkg/utils/namer/l4_namer.go b/pkg/utils/namer/l4_namer.go index 19ca7b1474..07e4e97735 100644 --- a/pkg/utils/namer/l4_namer.go +++ b/pkg/utils/namer/l4_namer.go @@ -17,6 +17,7 @@ const ( maximumL4CombinedLength = 39 sharedHcSuffix = "l4-shared-hc" firewallHcSuffix = "-fw" + ipv6Suffix = "ipv6" sharedFirewallHcSuffix = sharedHcSuffix + firewallHcSuffix maxResourceNameLength = 63 ) @@ -47,19 +48,22 @@ func NewL4Namer(kubeSystemUID string, namer *Namer) *L4Namer { // L4Backend returns the gce L4 Backend name based on the service namespace and name // Naming convention: -// k8s2-{uid}-{ns}-{name}-{suffix} +// +// k8s2-{uid}-{ns}-{name}-{suffix} +// // Output name is at most 63 characters. func (namer *L4Namer) L4Backend(namespace, name string) (string, bool) { truncFields := TrimFieldsEvenly(maximumL4CombinedLength, namespace, name) truncNamespace := truncFields[0] truncName := truncFields[1] return strings.Join([]string{namer.v2Prefix, namer.v2ClusterUID, truncNamespace, truncName, namer.suffix(namespace, name)}, "-"), true - } // L4ForwardingRule returns the name of the L4 forwarding rule name based on the service namespace, name and protocol. // Naming convention: -// k8s2-{protocol}-{uid}-{ns}-{name}-{suffix} +// +// k8s2-{protocol}-{uid}-{ns}-{name}-{suffix} +// // Output name is at most 63 characters. func (namer *L4Namer) L4ForwardingRule(namespace, name, protocol string) string { // add 1 for hyphen @@ -70,6 +74,16 @@ func (namer *L4Namer) L4ForwardingRule(namespace, name, protocol string) string return strings.Join([]string{namer.v2Prefix, protocol, namer.v2ClusterUID, truncNamespace, truncName, namer.suffix(namespace, name)}, "-") } +// L4IPv6ForwardingRule returns the name of the L4 forwarding rule name based on the service namespace, name and protocol. +// Naming convention: +// +// k8s2-{protocol}-{uid}-{ns}-{name}-{suffix} +// +// Output name is at most 63 characters. +func (namer *L4Namer) L4IPv6ForwardingRule(namespace, name, protocol string) string { + return strings.Join([]string{namer.L4ForwardingRule(namespace, name, protocol), ipv6Suffix}, "-") +} + // L4HealthCheck returns the name of the L4 LB Healthcheck and the associated firewall rule. func (namer *L4Namer) L4HealthCheck(namespace, name string, shared bool) (string, string) { if !shared { @@ -80,6 +94,15 @@ func (namer *L4Namer) L4HealthCheck(namespace, name string, shared bool) (string strings.Join([]string{namer.v2Prefix, namer.v2ClusterUID, sharedFirewallHcSuffix}, "-") } +// L4IPv6HealthCheckFirewall returns the name of the IPv6 L4 LB health check firewall rule. +func (namer *L4Namer) L4IPv6HealthCheckFirewall(namespace, name string, shared bool) string { + if !shared { + l4Name, _ := namer.L4Backend(namespace, name) + return strings.Join([]string{namer.hcFirewallName(l4Name), ipv6Suffix}, "-") + } + return strings.Join([]string{namer.v2Prefix, namer.v2ClusterUID, sharedFirewallHcSuffix, ipv6Suffix}, "-") +} + // IsNEG indicates if the given name is a NEG following the L4 naming convention. func (namer *L4Namer) IsNEG(name string) bool { return strings.HasPrefix(name, namer.v2Prefix+"-"+namer.v2ClusterUID) diff --git a/pkg/utils/utils.go b/pkg/utils/utils.go index 674882647e..4f780619fd 100644 --- a/pkg/utils/utils.go +++ b/pkg/utils/utils.go @@ -756,6 +756,10 @@ func MakeL4LBServiceDescription(svcName, ip string, version meta.Version, shared return (&L4LBResourceDescription{ServiceName: svcName, ServiceIP: ip, APIVersion: version}).Marshal() } +func MakeL4IPv6ForwardingRuleDescription(service *api_v1.Service) (string, error) { + return (&L4LBResourceDescription{ServiceName: ServiceKeyFunc(service.Namespace, service.Name)}).Marshal() +} + // NewStringPointer returns a pointer to the provided string literal func NewStringPointer(s string) *string { return &s @@ -819,3 +823,16 @@ func GetServiceNodePort(service *api_v1.Service) int64 { } return int64(service.Spec.Ports[0].NodePort) } + +func AddIPToLBStatus(status *api_v1.LoadBalancerStatus, ips ...string) *api_v1.LoadBalancerStatus { + if status == nil { + status = &api_v1.LoadBalancerStatus{ + Ingress: []api_v1.LoadBalancerIngress{}, + } + } + + for _, ip := range ips { + status.Ingress = append(status.Ingress, api_v1.LoadBalancerIngress{IP: ip}) + } + return status +} diff --git a/pkg/utils/utils_test.go b/pkg/utils/utils_test.go index 939f4efb95..b050fc0c10 100644 --- a/pkg/utils/utils_test.go +++ b/pkg/utils/utils_test.go @@ -1522,3 +1522,47 @@ func TestGetServicePortRanges(t *testing.T) { }) } } + +func TestAddIPToLBStatus(t *testing.T) { + testCases := []struct { + status *api_v1.LoadBalancerStatus + ipsToAdd []string + expectedStatus *api_v1.LoadBalancerStatus + desc string + }{ + { + status: nil, + ipsToAdd: []string{}, + expectedStatus: &api_v1.LoadBalancerStatus{Ingress: []api_v1.LoadBalancerIngress{}}, + desc: "Should create empty status ingress if no IPs provided", + }, + { + status: nil, + ipsToAdd: []string{"1.1.1.1", "0::0"}, + expectedStatus: &api_v1.LoadBalancerStatus{Ingress: []api_v1.LoadBalancerIngress{ + {IP: "1.1.1.1"}, {IP: "0::0"}, + }}, + desc: "Should add IPs to the empty status", + }, + { + status: &api_v1.LoadBalancerStatus{Ingress: []api_v1.LoadBalancerIngress{ + {IP: "0::0"}, + }}, + ipsToAdd: []string{"1.1.1.1"}, + expectedStatus: &api_v1.LoadBalancerStatus{Ingress: []api_v1.LoadBalancerIngress{ + {IP: "0::0"}, {IP: "1.1.1.1"}, + }}, + desc: "Should add IP to the existing status", + }, + } + + for _, tc := range testCases { + t.Run(tc.desc, func(t *testing.T) { + newStatus := AddIPToLBStatus(tc.status, tc.ipsToAdd...) + + if !reflect.DeepEqual(tc.expectedStatus, newStatus) { + t.Errorf("newStatus = %v, not equal to expectedStatus = %v", newStatus, tc.expectedStatus) + } + }) + } +}