Add Dual-Stack support to L4 ILB

pkg/annotations/service.go

@@ -73,6 +73,7 @@ const (
 	// ProtocolHTTP2 protocol for a service
 	ProtocolHTTP2 AppProtocol = "HTTP2"
 
+	IPv6Suffix = "-ipv6"
 	// ServiceStatusPrefix is the prefix used in annotations used to record
 	// debug information in the Service annotations. This is applicable to L4 ILB services.
 	ServiceStatusPrefix = "service.kubernetes.io"
@@ -82,24 +83,39 @@ const (
 	// UDPForwardingRuleKey is the annotation key used by l4 controller to record
 	// GCP UDP forwarding rule name.
 	UDPForwardingRuleKey = ServiceStatusPrefix + "/udp-" + ForwardingRuleResource
+	// TCPForwardingRuleIPv6Key is the annotation key used by l4 controller to record
+	// GCP IPv6 TCP forwarding rule name.
+	TCPForwardingRuleIPv6Key = TCPForwardingRuleKey + IPv6Suffix
+	// UDPForwardingRuleIPv6Key is the annotation key used by l4 controller to record
+	// GCP IPv6 UDP forwarding rule name.
+	UDPForwardingRuleIPv6Key = UDPForwardingRuleKey + IPv6Suffix
 	// BackendServiceKey is the annotation key used by l4 controller to record
 	// GCP Backend service name.
 	BackendServiceKey = ServiceStatusPrefix + "/" + BackendServiceResource
 	// FirewallRuleKey is the annotation key used by l4 controller to record
 	// GCP Firewall rule name.
 	FirewallRuleKey = ServiceStatusPrefix + "/" + FirewallRuleResource
+	// FirewallRuleIPv6Key is the annotation key used by l4 controller to record
+	// GCP IPv6 Firewall rule name.
+	FirewallRuleIPv6Key = FirewallRuleKey + IPv6Suffix
 	// HealthcheckKey is the annotation key used by l4 controller to record
 	// GCP Healthcheck name.
 	HealthcheckKey = ServiceStatusPrefix + "/" + HealthcheckResource
 	// FirewallRuleForHealthcheckKey is the annotation key used by l4 controller to record
 	// the firewall rule name that allows healthcheck traffic.
-	FirewallRuleForHealthcheckKey  = ServiceStatusPrefix + "/" + FirewallForHealthcheckResource
-	ForwardingRuleResource         = "forwarding-rule"
-	BackendServiceResource         = "backend-service"
-	FirewallRuleResource           = "firewall-rule"
-	HealthcheckResource            = "healthcheck"
-	FirewallForHealthcheckResource = "firewall-rule-for-hc"
-	AddressResource                = "address"
+	FirewallRuleForHealthcheckKey = ServiceStatusPrefix + "/" + FirewallForHealthcheckResource
+	// FirewallRuleForHealthcheckIPv6Key is the annotation key used by l4 controller to record
+	// the firewall rule name that allows IPv6 healthcheck traffic.
+	FirewallRuleForHealthcheckIPv6Key  = FirewallRuleForHealthcheckKey + IPv6Suffix
+	ForwardingRuleResource             = "forwarding-rule"
+	ForwardingRuleIPv6Resource         = ForwardingRuleResource + IPv6Suffix
+	BackendServiceResource             = "backend-service"
+	FirewallRuleResource               = "firewall-rule"
+	FirewallRuleIPv6Resource           = FirewallRuleResource + IPv6Suffix
+	HealthcheckResource                = "healthcheck"
+	FirewallForHealthcheckResource     = "firewall-rule-for-hc"
+	FirewallForHealthcheckIPv6Resource = FirewallRuleForHealthcheckKey + IPv6Suffix
+	AddressResource                    = "address"
 	// TODO(slavik): import this from gce_annotations when it will be merged in k8s
 	RBSAnnotationKey = "cloud.google.com/l4-rbs"
 	RBSEnabled       = "enabled"

pkg/flags/flags.go

@@ -108,6 +108,7 @@ var (
 		EnableTrafficScaling           bool
 		EnableEndpointSlices           bool
 		EnablePinhole                  bool
+		EnableL4ILBDualStack           bool
 		EnableMultipleIGs              bool
 		MaxIGSize                      int
 	}{
@@ -249,6 +250,7 @@ L7 load balancing. CSV values accepted. Example: -node-port-ranges=80,8080,400-5
 	flag.BoolVar(&F.EnableTrafficScaling, "enable-traffic-scaling", false, "Enable support for Service {max-rate-per-endpoint, capacity-scaler}")
 	flag.BoolVar(&F.EnableEndpointSlices, "enable-endpoint-slices", false, "Enable using Endpoint Slices API instead of Endpoints API")
 	flag.BoolVar(&F.EnablePinhole, "enable-pinhole", false, "Enable Pinhole firewall feature")
+	flag.BoolVar(&F.EnableL4ILBDualStack, "enable-l4ilb-dual-stack", false, "Enable Dual-Stack handling for L4 ILB load balancers")
 	flag.BoolVar(&F.EnableMultipleIGs, "enable-multiple-igs", false, "Enable using multiple unmanaged instance groups")
 	flag.IntVar(&F.MaxIGSize, "max-ig-size", 1000, "Max number of instances in Instance Group")
 	flag.DurationVar(&F.MetricsExportInterval, "metrics-export-interval", 10*time.Minute, `Period for calculating and exporting metrics related to state of managed objects.`)

pkg/healthchecks/healthchecks_l4.go

@@ -104,6 +104,10 @@ func L4() *l4HealthChecks {
 // Services of different scope (Global vs Regional).
 
 func (l4hc *l4HealthChecks) EnsureL4HealthCheck(svc *corev1.Service, namer namer.L4ResourcesNamer, sharedHC bool, scope meta.KeyType, l4Type utils.L4LBType, nodeNames []string) *EnsureL4HealthCheckResult {
+	return l4hc.EnsureL4DualStackHealthCheck(svc, namer, sharedHC, scope, l4Type, nodeNames, true, false)
+}
+
+func (l4hc *l4HealthChecks) EnsureL4DualStackHealthCheck(svc *corev1.Service, namer namer.L4ResourcesNamer, sharedHC bool, scope meta.KeyType, l4Type utils.L4LBType, nodeNames []string, needsIPv4 bool, needsIPv6 bool) *EnsureL4HealthCheckResult {
 	namespacedName := types.NamespacedName{Name: svc.Name, Namespace: svc.Namespace}
 
 	hcName := namer.L4HealthCheck(svc.Namespace, svc.Name, sharedHC)
@@ -127,23 +131,58 @@ func (l4hc *l4HealthChecks) EnsureL4HealthCheck(svc *corev1.Service, namer namer
 		}
 	}
 
-	klog.V(3).Infof("Healthcheck created, ensuring firewall rule %s", hcFwName)
-	err = l4hc.ensureFirewall(svc, hcFwName, hcPort, sharedHC, nodeNames)
-	if err != nil {
-		return &EnsureL4HealthCheckResult{
-			GceResourceInError: annotations.HealthcheckResource,
-			Err:                err,
+	hcResult := &EnsureL4HealthCheckResult{
+		HCName: hcName,
+		HCLink: hcLink,
+	}
+
+	if needsIPv4 {
+		klog.V(3).Infof("Healthcheck created, ensuring firewall rule %s", hcFwName)
+		err = l4hc.ensureFirewall(svc, hcFwName, hcPort, sharedHC, nodeNames)
+		if err != nil {
+			return &EnsureL4HealthCheckResult{
+				GceResourceInError: annotations.FirewallForHealthcheckResource,
+				Err:                err,
+			}
 		}
+		hcResult.HCFirewallRuleName = hcFwName
 	}
-	return &EnsureL4HealthCheckResult{
-		HCName:             hcName,
-		HCLink:             hcLink,
-		HCFirewallRuleName: hcFwName,
+
+	if needsIPv6 {
+		ipv6HCFWName := namer.L4IPv6HealthCheckFirewall(svc.Namespace, svc.Name, sharedHC)
+		klog.V(3).Infof("Healthcheck created, ensuring ipv6 firewall rule %s", ipv6HCFWName)
+		err = l4hc.ensureIPv6Firewall(svc, ipv6HCFWName, hcPort, sharedHC, nodeNames)
+		if err != nil {
+			return &EnsureL4HealthCheckResult{
+				GceResourceInError: annotations.FirewallForHealthcheckIPv6Resource,
+				Err:                err,
+			}
+		}
+		hcResult.HCFirewallRuleIPv6Name = ipv6HCFWName
 	}
+
+	return hcResult
+}
+
+func (l4hc *l4HealthChecks) ensureIPv6Firewall(svc *corev1.Service, ipv6HCFWName string, hcPort int32, isSharedHC bool, nodeNames []string) error {
+	hcFWRParams := firewalls.FirewallParams{
+		PortRanges:   []string{strconv.Itoa(int(hcPort))},
+		SourceRanges: L4ILBIPv6HCRange.StringSlice(),
+		Protocol:     string(corev1.ProtocolTCP),
+		Name:         ipv6HCFWName,
+		NodeNames:    nodeNames,
+	}
+	return firewalls.EnsureL4LBFirewallForHc(svc, isSharedHC, &hcFWRParams, l4hc.cloud, l4hc.recorderFactory.Recorder(svc.Namespace))
 }
 
 // DeleteHealthCheck deletes health check (and firewall rule) for l4 service. Checks if shared resources are safe to delete.
 func (l4hc *l4HealthChecks) DeleteHealthCheck(svc *corev1.Service, namer namer.L4ResourcesNamer, sharedHC bool, scope meta.KeyType, l4Type utils.L4LBType) (string, error) {
+	return l4hc.DeleteDualStackHealthCheck(svc, namer, sharedHC, scope, l4Type, false)
+}
+
+// DeleteDualStackHealthCheck deletes health check (and firewall rule) for l4 service.
+// Checks if shared resources are safe to delete. Deletes IPv6 firewall if asked
+func (l4hc *l4HealthChecks) DeleteDualStackHealthCheck(svc *corev1.Service, namer namer.L4ResourcesNamer, sharedHC bool, scope meta.KeyType, l4Type utils.L4LBType, handleIPv6 bool) (string, error) {
 	hcName := namer.L4HealthCheck(svc.Namespace, svc.Name, sharedHC)
 	hcFwName := namer.L4HealthCheckFirewall(svc.Namespace, svc.Name, sharedHC)
 	namespacedName := types.NamespacedName{Name: svc.Name, Namespace: svc.Namespace}
@@ -165,6 +204,12 @@ func (l4hc *l4HealthChecks) DeleteHealthCheck(svc *corev1.Service, namer namer.L
 		return "", nil
 	}
 	// Health check deleted, now delete the firewall rule
+	if handleIPv6 {
+		errResource, err := l4hc.deleteIPv6HealthCheckFirewall(svc, hcName, namer, sharedHC, l4Type)
+		if err != nil {
+			return errResource, err
+		}
+	}
 	return l4hc.deleteHealthCheckFirewall(svc, hcName, hcFwName, sharedHC, l4Type)
 }
 
@@ -257,6 +302,11 @@ func (l4hc *l4HealthChecks) deleteHealthCheckFirewall(svc *corev1.Service, hcNam
 	return "", nil
 }
 
+func (l4hc *l4HealthChecks) deleteIPv6HealthCheckFirewall(svc *corev1.Service, hcName string, namer namer.L4ResourcesNamer, sharedHC bool, l4type utils.L4LBType) (string, error) {
+	ipv6hcFwName := namer.L4IPv6HealthCheckFirewall(svc.Namespace, svc.Name, sharedHC)
+	return l4hc.deleteHealthCheckFirewall(svc, hcName, ipv6hcFwName, sharedHC, l4type)
+}
+
 func (l4hc *l4HealthChecks) healthCheckFirewallSafeToDelete(hcName string, sharedHC bool, l4Type utils.L4LBType) (bool, error) {
 	if !sharedHC {
 		return true, nil

pkg/healthchecks/interfaces.go

@@ -63,14 +63,19 @@ type HealthChecker interface {
 type L4HealthChecks interface {
 	// EnsureL4HealthCheck creates health check (and firewall rule) for l4 service
 	EnsureL4HealthCheck(svc *v1.Service, namer namer.L4ResourcesNamer, sharedHC bool, scope meta.KeyType, l4Type utils.L4LBType, nodeNames []string) *EnsureL4HealthCheckResult
+	// EnsureL4DualStackHealthCheck creates health check (and firewall rule) for l4 service. Handles both IPv4 and IPv6
+	EnsureL4DualStackHealthCheck(svc *v1.Service, namer namer.L4ResourcesNamer, sharedHC bool, scope meta.KeyType, l4Type utils.L4LBType, nodeNames []string, needsIPv4 bool, needsIPv6 bool) *EnsureL4HealthCheckResult
 	// DeleteHealthCheck deletes health check (and firewall rule) for l4 service
 	DeleteHealthCheck(svc *v1.Service, namer namer.L4ResourcesNamer, sharedHC bool, scope meta.KeyType, l4Type utils.L4LBType) (string, error)
+	// DeleteDualStackHealthCheck deletes health check (and firewall rule) for l4 service, deletes IPv6 if asked
+	DeleteDualStackHealthCheck(svc *v1.Service, namer namer.L4ResourcesNamer, sharedHC bool, scope meta.KeyType, l4Type utils.L4LBType, handleIPv6 bool) (string, error)
 }
 
 type EnsureL4HealthCheckResult struct {
-	HCName             string
-	HCLink             string
-	HCFirewallRuleName string
-	GceResourceInError string
-	Err                error
+	HCName                 string
+	HCLink                 string
+	HCFirewallRuleName     string
+	HCFirewallRuleIPv6Name string
+	GceResourceInError     string
+	Err                    error
 }

pkg/healthchecks/ipv6.go

@@ -0,0 +1,31 @@
+package healthchecks
+
+import (
+	"fmt"
+
+	"k8s.io/klog/v2"
+	utilnet "k8s.io/utils/net"
+)
+
+const (
+	l4ELBIPv6HCRangeString = "2600:1901:8001::/48"
+	l4ILBIPv6HCRangeString = "2600:2d00:1:b029::/64"
+)
+
+var (
+	L4ELBIPv6HCRange utilnet.IPNetSet
+	L4ILBIPv6HCRange utilnet.IPNetSet
+)
+
+func init() {
+	var err error
+	L4ELBIPv6HCRange, err = utilnet.ParseIPNets([]string{l4ELBIPv6HCRangeString}...)
+	if err != nil {
+		klog.Fatalf(fmt.Sprintf("utilnet.ParseIPNets([]string{%s}...) returned error %v, want nil", l4ELBIPv6HCRangeString, err))
+	}
+
+	L4ILBIPv6HCRange, err = utilnet.ParseIPNets([]string{l4ILBIPv6HCRangeString}...)
+	if err != nil {
+		klog.Fatalf(fmt.Sprintf("utilnet.ParseIPNets([]string{%s}...) returned error %v, want nil", l4ILBIPv6HCRangeString, err))
+	}
+}

pkg/l4lb/l4controller.go

@@ -33,6 +33,7 @@ import (
 	"k8s.io/ingress-gce/pkg/backends"
 	"k8s.io/ingress-gce/pkg/context"
 	"k8s.io/ingress-gce/pkg/controller/translator"
+	"k8s.io/ingress-gce/pkg/flags"
 	"k8s.io/ingress-gce/pkg/forwardingrules"
 	l4metrics "k8s.io/ingress-gce/pkg/l4lb/metrics"
 	"k8s.io/ingress-gce/pkg/loadbalancers"
@@ -209,10 +210,11 @@ func (l4c *L4Controller) processServiceCreateOrUpdate(key string, service *v1.Se
 	// Use the same function for both create and updates. If controller crashes and restarts,
 	// all existing services will show up as Service Adds.
 	l4ilbParams := &loadbalancers.L4ILBParams{
-		Service:  service,
-		Cloud:    l4c.ctx.Cloud,
-		Namer:    l4c.namer,
-		Recorder: l4c.ctx.Recorder(service.Namespace),
+		Service:          service,
+		Cloud:            l4c.ctx.Cloud,
+		Namer:            l4c.namer,
+		Recorder:         l4c.ctx.Recorder(service.Namespace),
+		DualStackEnabled: flags.F.EnableL4ILBDualStack,
 	}
 	l4 := loadbalancers.NewL4Handler(l4ilbParams)
 	syncResult := l4.EnsureInternalLoadBalancer(nodeNames, service)
@@ -255,10 +257,11 @@ func (l4c *L4Controller) processServiceCreateOrUpdate(key string, service *v1.Se
 
 func (l4c *L4Controller) processServiceDeletion(key string, svc *v1.Service) *loadbalancers.L4ILBSyncResult {
 	l4ilbParams := &loadbalancers.L4ILBParams{
-		Service:  svc,
-		Cloud:    l4c.ctx.Cloud,
-		Namer:    l4c.namer,
-		Recorder: l4c.ctx.Recorder(svc.Namespace),
+		Service:          svc,
+		Cloud:            l4c.ctx.Cloud,
+		Namer:            l4c.namer,
+		Recorder:         l4c.ctx.Recorder(svc.Namespace),
+		DualStackEnabled: flags.F.EnableL4ILBDualStack,
 	}
 	l4 := loadbalancers.NewL4Handler(l4ilbParams)
 	l4c.ctx.Recorder(svc.Namespace).Eventf(svc, v1.EventTypeNormal, "DeletingLoadBalancer", "Deleting load balancer for %s", key)

pkg/l4lb/l4netlbcontroller_test.go

@@ -877,7 +877,7 @@ func TestHealthCheckWhenExternalTrafficPolicyWasUpdated(t *testing.T) {
 	// Update ExternalTrafficPolicy to Cluster check if shared HC was created
 	err = updateAndAssertExternalTrafficPolicy(newSvc, lc, v1.ServiceExternalTrafficPolicyTypeCluster, hcNameShared)
 	if err != nil {
-		t.Errorf("Error asserthing shared health check %v", err)
+		t.Errorf("Error asserting shared health check %v", err)
 	}
 	newSvc.DeletionTimestamp = &metav1.Time{}
 	updateNetLBService(lc, newSvc)