WIP

cmd/glbc/main.go

@@ -199,6 +199,7 @@ func main() {
 		ASMConfigMapName:      flags.F.ASMConfigMapBasedConfigCMName,
 		EndpointSlicesEnabled: flags.F.EnableEndpointSlices,
 		MaxIGSize:             flags.F.MaxIGSize,
+		EnableL4ILBDualStack:  flags.F.EnableL4ILBDualStack,
 	}
 	ctx := ingctx.NewControllerContext(kubeConfig, kubeClient, backendConfigClient, frontendConfigClient, svcNegClient, ingParamsClient, svcAttachmentClient, cloud, namer, kubeSystemUID, ctxConfig)
 	go app.RunHTTPServer(ctx.HealthCheck)

pkg/context/context.go

@@ -135,6 +135,7 @@ type ControllerContextConfig struct {
 	ASMConfigMapName      string
 	EndpointSlicesEnabled bool
 	MaxIGSize             int
+	EnableL4ILBDualStack  bool
 }
 
 // NewControllerContext returns a new shared set of informers.

pkg/healthchecks/healthchecks_l4.go

@@ -44,6 +44,7 @@ const (
 	gceHcHealthyThreshold = int64(1)
 	// Defaults to 3 * 8 = 24 seconds before the LB will steer traffic away.
 	gceHcUnhealthyThreshold = int64(3)
+	l4ILBIPv6HCRangeString  = "2600:2d00:1:b029::/64"
 )
 
 var (
@@ -167,7 +168,7 @@ func (l4hc *l4HealthChecks) EnsureL4DualStackHealthCheck(svc *corev1.Service, na
 func (l4hc *l4HealthChecks) ensureIPv6Firewall(svc *corev1.Service, ipv6HCFWName string, hcPort int32, isSharedHC bool, nodeNames []string) error {
 	hcFWRParams := firewalls.FirewallParams{
 		PortRanges:   []string{strconv.Itoa(int(hcPort))},
-		SourceRanges: L4ILBIPv6HCRange.StringSlice(),
+		SourceRanges: []string{l4ILBIPv6HCRangeString},
 		Protocol:     string(corev1.ProtocolTCP),
 		Name:         ipv6HCFWName,
 		NodeNames:    nodeNames,
@@ -177,12 +178,6 @@ func (l4hc *l4HealthChecks) ensureIPv6Firewall(svc *corev1.Service, ipv6HCFWName
 
 // DeleteHealthCheck deletes health check (and firewall rule) for l4 service. Checks if shared resources are safe to delete.
 func (l4hc *l4HealthChecks) DeleteHealthCheck(svc *corev1.Service, namer namer.L4ResourcesNamer, sharedHC bool, scope meta.KeyType, l4Type utils.L4LBType) (string, error) {
-	return l4hc.DeleteDualStackHealthCheck(svc, namer, sharedHC, scope, l4Type, false)
-}
-
-// DeleteDualStackHealthCheck deletes health check (and firewall rule) for l4 service.
-// Checks if shared resources are safe to delete. Deletes IPv6 firewall if asked
-func (l4hc *l4HealthChecks) DeleteDualStackHealthCheck(svc *corev1.Service, namer namer.L4ResourcesNamer, sharedHC bool, scope meta.KeyType, l4Type utils.L4LBType, handleIPv6 bool) (string, error) {
 	hcName := namer.L4HealthCheck(svc.Namespace, svc.Name, sharedHC)
 	hcFwName := namer.L4HealthCheckFirewall(svc.Namespace, svc.Name, sharedHC)
 	namespacedName := types.NamespacedName{Name: svc.Name, Namespace: svc.Namespace}
@@ -203,16 +198,19 @@ func (l4hc *l4HealthChecks) DeleteDualStackHealthCheck(svc *corev1.Service, name
 		klog.V(2).Infof("Failed to delete healthcheck %s: shared health check in use.", hcName)
 		return "", nil
 	}
-	// Health check deleted, now delete the firewall rule
-	if handleIPv6 {
-		errResource, err := l4hc.deleteIPv6HealthCheckFirewall(svc, hcName, namer, sharedHC, l4Type)
-		if err != nil {
-			return errResource, err
-		}
-	}
 	return l4hc.deleteHealthCheckFirewall(svc, hcName, hcFwName, sharedHC, l4Type)
 }
 
+// DeleteDualStackHealthCheck deletes health check, ipv4 and ipv6 firewall rules for l4 service.
+// Checks if shared resources are safe to delete.
+func (l4hc *l4HealthChecks) DeleteDualStackHealthCheck(svc *corev1.Service, namer namer.L4ResourcesNamer, sharedHC bool, scope meta.KeyType, l4Type utils.L4LBType) (string, error) {
+	errResource, err := l4hc.DeleteHealthCheck(svc, namer, sharedHC, scope, l4Type)
+	if err != nil {
+		return errResource, err
+	}
+	return l4hc.deleteIPv6HealthCheckFirewall(svc, namer, sharedHC, l4Type)
+}
+
 func (l4hc *l4HealthChecks) ensureL4HealthCheckInternal(hcName string, svcName types.NamespacedName, shared bool, path string, port int32, scope meta.KeyType, l4Type utils.L4LBType) (*composite.HealthCheck, string, error) {
 	selfLink := ""
 	key, err := composite.CreateKey(l4hc.cloud, hcName, scope)
@@ -302,7 +300,8 @@ func (l4hc *l4HealthChecks) deleteHealthCheckFirewall(svc *corev1.Service, hcNam
 	return "", nil
 }
 
-func (l4hc *l4HealthChecks) deleteIPv6HealthCheckFirewall(svc *corev1.Service, hcName string, namer namer.L4ResourcesNamer, sharedHC bool, l4type utils.L4LBType) (string, error) {
+func (l4hc *l4HealthChecks) deleteIPv6HealthCheckFirewall(svc *corev1.Service, namer namer.L4ResourcesNamer, sharedHC bool, l4type utils.L4LBType) (string, error) {
+	hcName := namer.L4HealthCheck(svc.Namespace, svc.Name, sharedHC)
 	ipv6hcFwName := namer.L4IPv6HealthCheckFirewall(svc.Namespace, svc.Name, sharedHC)
 	return l4hc.deleteHealthCheckFirewall(svc, hcName, ipv6hcFwName, sharedHC, l4type)
 }

pkg/healthchecks/interfaces.go

@@ -68,7 +68,7 @@ type L4HealthChecks interface {
 	// DeleteHealthCheck deletes health check (and firewall rule) for l4 service
 	DeleteHealthCheck(svc *v1.Service, namer namer.L4ResourcesNamer, sharedHC bool, scope meta.KeyType, l4Type utils.L4LBType) (string, error)
 	// DeleteDualStackHealthCheck deletes health check (and firewall rule) for l4 service, deletes IPv6 if asked
-	DeleteDualStackHealthCheck(svc *v1.Service, namer namer.L4ResourcesNamer, sharedHC bool, scope meta.KeyType, l4Type utils.L4LBType, handleIPv6 bool) (string, error)
+	DeleteDualStackHealthCheck(svc *v1.Service, namer namer.L4ResourcesNamer, sharedHC bool, scope meta.KeyType, l4Type utils.L4LBType) (string, error)
 }
 
 type EnsureL4HealthCheckResult struct {

pkg/l4lb/l4controller.go

@@ -33,7 +33,6 @@ import (
 	"k8s.io/ingress-gce/pkg/backends"
 	"k8s.io/ingress-gce/pkg/context"
 	"k8s.io/ingress-gce/pkg/controller/translator"
-	"k8s.io/ingress-gce/pkg/flags"
 	"k8s.io/ingress-gce/pkg/forwardingrules"
 	l4metrics "k8s.io/ingress-gce/pkg/l4lb/metrics"
 	"k8s.io/ingress-gce/pkg/loadbalancers"
@@ -72,6 +71,7 @@ type L4Controller struct {
 	syncTracker         utils.TimeTracker
 	forwardingRules     ForwardingRulesGetter
 	sharedResourcesLock sync.Mutex
+	enableDualStack     bool
 }
 
 // NewILBController creates a new instance of the L4 ILB controller.
@@ -90,6 +90,7 @@ func NewILBController(ctx *context.ControllerContext, stopCh chan struct{}) *L4C
 		namer:           ctx.L4Namer,
 		translator:      ctx.Translator,
 		forwardingRules: forwardingrules.New(ctx.Cloud, meta.VersionGA, meta.Regional),
+		enableDualStack: ctx.EnableL4ILBDualStack,
 	}
 	l4c.backendPool = backends.NewPool(ctx.Cloud, l4c.namer)
 	l4c.NegLinker = backends.NewNEGLinker(l4c.backendPool, negtypes.NewAdapter(ctx.Cloud), ctx.Cloud, ctx.SvcNegInformer.GetIndexer())
@@ -214,7 +215,7 @@ func (l4c *L4Controller) processServiceCreateOrUpdate(key string, service *v1.Se
 		Cloud:            l4c.ctx.Cloud,
 		Namer:            l4c.namer,
 		Recorder:         l4c.ctx.Recorder(service.Namespace),
-		DualStackEnabled: flags.F.EnableL4ILBDualStack,
+		DualStackEnabled: l4c.enableDualStack,
 	}
 	l4 := loadbalancers.NewL4Handler(l4ilbParams)
 	syncResult := l4.EnsureInternalLoadBalancer(nodeNames, service)
@@ -261,7 +262,7 @@ func (l4c *L4Controller) processServiceDeletion(key string, svc *v1.Service) *lo
 		Cloud:            l4c.ctx.Cloud,
 		Namer:            l4c.namer,
 		Recorder:         l4c.ctx.Recorder(svc.Namespace),
-		DualStackEnabled: flags.F.EnableL4ILBDualStack,
+		DualStackEnabled: l4c.enableDualStack,
 	}
 	l4 := loadbalancers.NewL4Handler(l4ilbParams)
 	l4c.ctx.Recorder(svc.Namespace).Eventf(svc, v1.EventTypeNormal, "DeletingLoadBalancer", "Deleting load balancer for %s", key)

pkg/loadbalancers/forwarding_rules_ipv6.go

@@ -4,7 +4,6 @@ import (
 	"fmt"
 
 	"github.com/GoogleCloudPlatform/k8s-cloud-provider/pkg/cloud"
-	"github.com/GoogleCloudPlatform/k8s-cloud-provider/pkg/cloud/meta"
 	"github.com/google/go-cmp/cmp"
 	"github.com/google/go-cmp/cmp/cmpopts"
 	corev1 "k8s.io/api/core/v1"
@@ -15,15 +14,15 @@ import (
 	"k8s.io/legacy-cloud-providers/gce"
 )
 
-func (l *L4) ensureIPv6ForwardingRule(bsLink string, options gce.ILBOptions) (*composite.ForwardingRule, error) {
-	expectedIPv6FwdRule, err := l.buildExpectedIPv6ForwardingRule(bsLink, options)
+func (l4 *L4) ensureIPv6ForwardingRule(bsLink string, options gce.ILBOptions) (*composite.ForwardingRule, error) {
+	expectedIPv6FwdRule, err := l4.buildExpectedIPv6ForwardingRule(bsLink, options)
 	if err != nil {
-		return nil, fmt.Errorf("l.buildExpectedIPv6ForwardingRule(%s, %v) returned error %w, want nil", bsLink, options, err)
+		return nil, fmt.Errorf("l4.buildExpectedIPv6ForwardingRule(%s, %v) returned error %w, want nil", bsLink, options, err)
 	}
 
-	existingIPv6FwdRule, err := l.forwardingRules.Get(expectedIPv6FwdRule.Name)
+	existingIPv6FwdRule, err := l4.forwardingRules.Get(expectedIPv6FwdRule.Name)
 	if err != nil {
-		return nil, fmt.Errorf("l.forwardingRules.GetForwardingRule(%s) returned error %w, want nil", expectedIPv6FwdRule.Name, err)
+		return nil, fmt.Errorf("l4.forwardingRules.GetForwardingRule(%s) returned error %w, want nil", expectedIPv6FwdRule.Name, err)
 	}
 
 	if existingIPv6FwdRule != nil {
@@ -39,43 +38,40 @@ func (l *L4) ensureIPv6ForwardingRule(bsLink string, options gce.ILBOptions) (*c
 		frDiff := cmp.Diff(existingIPv6FwdRule, expectedIPv6FwdRule, cmpopts.IgnoreFields(composite.ForwardingRule{}, "IPAddress"))
 		klog.V(2).Infof("ensureIPv6ForwardingRule: forwarding rule changed - Existing - %+v\n, New - %+v\n, Diff(-existing, +new) - %s\n. Deleting existing ipv6 forwarding rule.", existingIPv6FwdRule, expectedIPv6FwdRule, frDiff)
 
-		err = l.forwardingRules.Delete(existingIPv6FwdRule.Name)
+		err = l4.forwardingRules.Delete(existingIPv6FwdRule.Name)
 		if err != nil {
 			return nil, err
 		}
-		l.recorder.Eventf(l.Service, corev1.EventTypeNormal, events.SyncIngress, "ForwardingRule %q deleted", existingIPv6FwdRule.Name)
+		l4.recorder.Eventf(l4.Service, corev1.EventTypeNormal, events.SyncIngress, "ForwardingRule %q deleted", existingIPv6FwdRule.Name)
 	}
 	klog.V(2).Infof("ensureIPv6ForwardingRule: Creating/Recreating forwarding rule - %s", expectedIPv6FwdRule.Name)
-	err = l.forwardingRules.Create(expectedIPv6FwdRule)
+	err = l4.forwardingRules.Create(expectedIPv6FwdRule)
 	if err != nil {
 		return nil, err
 	}
 
-	createdFr, err := l.forwardingRules.Get(expectedIPv6FwdRule.Name)
+	createdFr, err := l4.forwardingRules.Get(expectedIPv6FwdRule.Name)
 	return createdFr, err
 }
 
-func (l *L4) buildExpectedIPv6ForwardingRule(bsLink string, options gce.ILBOptions) (*composite.ForwardingRule, error) {
-	frName := l.getIPv6FRName()
+func (l4 *L4) buildExpectedIPv6ForwardingRule(bsLink string, options gce.ILBOptions) (*composite.ForwardingRule, error) {
+	frName := l4.getIPv6FRName()
 
-	frDesc, err := utils.MakeL4IPv6ForwardingRuleDescription(l.Service)
+	frDesc, err := utils.MakeL4IPv6ForwardingRuleDescription(l4.Service)
 	if err != nil {
 		return nil, fmt.Errorf("failed to compute description for forwarding rule %s, err: %w", frName, err)
 	}
 
-	subnetworkURL := l.cloud.SubnetworkURL()
+	subnetworkURL := l4.cloud.SubnetworkURL()
 
 	if options.SubnetName != "" {
-		key, err := l.CreateKey(frName)
+		subnetworkURL, err = l4.getSubnetworkURLByName(options.SubnetName, frName)
 		if err != nil {
 			return nil, err
 		}
-		subnetKey := *key
-		subnetKey.Name = options.SubnetName
-		subnetworkURL = cloud.SelfLink(meta.VersionGA, l.cloud.NetworkProjectID(), "subnetworks", &subnetKey)
 	}
 
-	svcPorts := l.Service.Spec.Ports
+	svcPorts := l4.Service.Spec.Ports
 	ports := utils.GetPorts(svcPorts)
 	protocol := utils.GetProtocol(svcPorts)
 
@@ -87,7 +83,7 @@ func (l *L4) buildExpectedIPv6ForwardingRule(bsLink string, options gce.ILBOptio
 		LoadBalancingScheme: string(cloud.SchemeInternal),
 		BackendService:      bsLink,
 		IpVersion:           "IPV6",
-		Network:             l.cloud.NetworkURL(),
+		Network:             l4.cloud.NetworkURL(),
 		Subnetwork:          subnetworkURL,
 		AllowGlobalAccess:   options.AllowGlobalAccess,
 		NetworkTier:         cloud.NetworkTierPremium.ToGCEValue(),

pkg/loadbalancers/l4.go

@@ -140,7 +140,7 @@ func (l4 *L4) EnsureInternalLoadBalancerDeleted(svc *corev1.Service) *L4ILBSyncR
 	// and delete them if needed.
 	for _, isShared := range []bool{true, false} {
 		if l4.enableDualStack {
-			resourceInError, err := l4.l4HealthChecks.DeleteDualStackHealthCheck(svc, l4.namer, isShared, meta.Global, utils.ILB, true)
+			resourceInError, err := l4.l4HealthChecks.DeleteDualStackHealthCheck(svc, l4.namer, isShared, meta.Global, utils.ILB)
 			if err != nil {
 				result.GCEResourceInError = resourceInError
 				result.Error = err
@@ -156,31 +156,31 @@ func (l4 *L4) EnsureInternalLoadBalancerDeleted(svc *corev1.Service) *L4ILBSyncR
 	return result
 }
 
-func (l *L4) deleteIPv4Resources(result *L4ILBSyncResult, bsName string) {
-	frName := l.GetFRName()
-	key, err := l.CreateKey(frName)
+func (l4 *L4) deleteIPv4Resources(result *L4ILBSyncResult, bsName string) {
+	frName := l4.GetFRName()
+	key, err := l4.CreateKey(frName)
 	if err != nil {
-		klog.Errorf("Failed to create key for LoadBalancer resources with name %s for service %s, err %v", frName, l.NamespacedName.String(), err)
+		klog.Errorf("Failed to create key for LoadBalancer resources with name %s for service %s, err %v", frName, l4.NamespacedName.String(), err)
 		result.Error = err
 		return
 	}
 	// If any resource deletion fails, log the error and continue cleanup.
-	if err = utils.IgnoreHTTPNotFound(composite.DeleteForwardingRule(l.cloud, key, meta.VersionGA)); err != nil {
-		klog.Errorf("Failed to delete forwarding rule for internal loadbalancer service %s, err %v", l.NamespacedName.String(), err)
+	if err = utils.IgnoreHTTPNotFound(composite.DeleteForwardingRule(l4.cloud, key, meta.VersionGA)); err != nil {
+		klog.Errorf("Failed to delete forwarding rule for internal loadbalancer service %s, err %v", l4.NamespacedName.String(), err)
 		result.Error = err
 		result.GCEResourceInError = annotations.ForwardingRuleResource
 	}
 
-	if err = ensureAddressDeleted(l.cloud, bsName, l.cloud.Region()); err != nil {
-		klog.Errorf("Failed to delete address for internal loadbalancer service %s, err %v", l.NamespacedName.String(), err)
+	if err = ensureAddressDeleted(l4.cloud, bsName, l4.cloud.Region()); err != nil {
+		klog.Errorf("Failed to delete address for internal loadbalancer service %s, err %v", l4.NamespacedName.String(), err)
 		result.Error = err
 		result.GCEResourceInError = annotations.AddressResource
 	}
 
 	// delete firewall rule allowing load balancer source ranges
-	err = l.deleteFirewall(bsName)
+	err = l4.deleteFirewall(bsName)
 	if err != nil {
-		klog.Errorf("Failed to delete firewall rule %s for internal loadbalancer service %s, err %v", bsName, l.NamespacedName.String(), err)
+		klog.Errorf("Failed to delete firewall rule %s for internal loadbalancer service %s, err %v", bsName, l4.NamespacedName.String(), err)
 		result.GCEResourceInError = annotations.FirewallRuleResource
 		result.Error = err
 	}
@@ -266,6 +266,7 @@ func (l4 *L4) EnsureInternalLoadBalancer(nodeNames []string, svc *corev1.Service
 	if err != nil {
 		klog.Errorf("Failed to lookup existing backend service, ignoring err: %v", err)
 	}
+
 	existingFR, err := l4.forwardingRules.Get(l4.GetFRName())
 	if existingBS != nil && existingBS.Protocol != string(protocol) {
 		klog.Infof("Protocol changed from %q to %q for service %s", existingBS.Protocol, string(protocol), l4.NamespacedName)
@@ -342,9 +343,9 @@ func (l4 *L4) getSubnetworkURLByName(subnetName string) (string, error) {
 	return cloud.SelfLink(meta.VersionGA, l4.cloud.NetworkProjectID(), "subnetworks", subnetKey), nil
 }
 
-func (l *L4) ensureIPv4Resources(result *L4ILBSyncResult, nodeNames []string, options gce.ILBOptions, bs *composite.BackendService, existingFR *composite.ForwardingRule) {
-	frName := l.GetFRName()
-	fr, err := l.ensureForwardingRule(frName, bs.SelfLink, options, existingFR)
+func (l4 *L4) ensureIPv4Resources(result *L4ILBSyncResult, nodeNames []string, options gce.ILBOptions, bs *composite.BackendService, existingFR *composite.ForwardingRule) {
+	frName := l4.GetFRName()
+	fr, err := l4.ensureForwardingRule(frName, bs.SelfLink, options, existingFR)
 	if err != nil {
 		klog.Errorf("EnsureInternalLoadBalancer: Failed to create forwarding rule - %v", err)
 		result.GCEResourceInError = annotations.ForwardingRuleResource
@@ -358,13 +359,13 @@ func (l *L4) ensureIPv4Resources(result *L4ILBSyncResult, nodeNames []string, op
 	}
 
 	// ensure firewalls
-	sourceRanges, err := helpers.GetLoadBalancerSourceRanges(l.Service)
+	sourceRanges, err := helpers.GetLoadBalancerSourceRanges(l4.Service)
 	if err != nil {
 		result.Error = err
 		return
 	}
 
-	servicePorts := l.Service.Spec.Ports
+	servicePorts := l4.Service.Spec.Ports
 	protocol := utils.GetProtocol(servicePorts)
 	portRanges := utils.GetServicePortRanges(servicePorts)
 	// Add firewall rule for ILB traffic to nodes
@@ -378,7 +379,7 @@ func (l *L4) ensureIPv4Resources(result *L4ILBSyncResult, nodeNames []string, op
 		L4Type:            utils.ILB,
 	}
 
-	if err := firewalls.EnsureL4LBFirewallForNodes(l.Service, &nodesFWRParams, l.cloud, l.recorder); err != nil {
+	if err := firewalls.EnsureL4LBFirewallForNodes(l4.Service, &nodesFWRParams, l4.cloud, l4.recorder); err != nil {
 		result.GCEResourceInError = annotations.FirewallRuleResource
 		result.Error = err
 		return