WIP

cmd/glbc/main.go

@@ -199,6 +199,7 @@ func main() {
 		ASMConfigMapName:      flags.F.ASMConfigMapBasedConfigCMName,
 		EndpointSlicesEnabled: flags.F.EnableEndpointSlices,
 		MaxIGSize:             flags.F.MaxIGSize,
+		EnableL4ILBDualStack:  flags.F.EnableL4ILBDualStack,
 	}
 	ctx := ingctx.NewControllerContext(kubeConfig, kubeClient, backendConfigClient, frontendConfigClient, svcNegClient, ingParamsClient, svcAttachmentClient, cloud, namer, kubeSystemUID, ctxConfig)
 	go app.RunHTTPServer(ctx.HealthCheck)

pkg/context/context.go

@@ -135,6 +135,7 @@ type ControllerContextConfig struct {
 	ASMConfigMapName      string
 	EndpointSlicesEnabled bool
 	MaxIGSize             int
+	EnableL4ILBDualStack  bool
 }
 
 // NewControllerContext returns a new shared set of informers.

pkg/healthchecks/healthchecks_l4.go

@@ -44,6 +44,7 @@ const (
 	gceHcHealthyThreshold = int64(1)
 	// Defaults to 3 * 8 = 24 seconds before the LB will steer traffic away.
 	gceHcUnhealthyThreshold = int64(3)
+	l4ILBIPv6HCRangeString  = "2600:2d00:1:b029::/64"
 )
 
 var (
@@ -166,7 +167,7 @@ func (l4hc *l4HealthChecks) EnsureL4DualStackHealthCheck(svc *corev1.Service, na
 func (l4hc *l4HealthChecks) ensureIPv6Firewall(svc *corev1.Service, ipv6HCFWName string, hcPort int32, isSharedHC bool, nodeNames []string) error {
 	hcFWRParams := firewalls.FirewallParams{
 		PortRanges:   []string{strconv.Itoa(int(hcPort))},
-		SourceRanges: L4ILBIPv6HCRange.StringSlice(),
+		SourceRanges: []string{l4ILBIPv6HCRangeString},
 		Protocol:     string(corev1.ProtocolTCP),
 		Name:         ipv6HCFWName,
 		NodeNames:    nodeNames,
@@ -176,12 +177,6 @@ func (l4hc *l4HealthChecks) ensureIPv6Firewall(svc *corev1.Service, ipv6HCFWName
 
 // DeleteHealthCheck deletes health check (and firewall rule) for l4 service. Checks if shared resources are safe to delete.
 func (l4hc *l4HealthChecks) DeleteHealthCheck(svc *corev1.Service, namer namer.L4ResourcesNamer, sharedHC bool, scope meta.KeyType, l4Type utils.L4LBType) (string, error) {
-	return l4hc.DeleteDualStackHealthCheck(svc, namer, sharedHC, scope, l4Type, false)
-}
-
-// DeleteDualStackHealthCheck deletes health check (and firewall rule) for l4 service.
-// Checks if shared resources are safe to delete. Deletes IPv6 firewall if asked
-func (l4hc *l4HealthChecks) DeleteDualStackHealthCheck(svc *corev1.Service, namer namer.L4ResourcesNamer, sharedHC bool, scope meta.KeyType, l4Type utils.L4LBType, handleIPv6 bool) (string, error) {
 	hcName, hcFwName := namer.L4HealthCheck(svc.Namespace, svc.Name, sharedHC)
 	namespacedName := types.NamespacedName{Name: svc.Name, Namespace: svc.Namespace}
 	klog.V(3).Infof("Trying to delete L4 healthcheck: %s and firewall rule %s from service %s, shared: %v", hcName, hcFwName, namespacedName.String(), sharedHC)
@@ -201,16 +196,19 @@ func (l4hc *l4HealthChecks) DeleteDualStackHealthCheck(svc *corev1.Service, name
 		klog.V(2).Infof("Failed to delete healthcheck %s: shared health check in use.", hcName)
 		return "", nil
 	}
-	// Health check deleted, now delete the firewall rule
-	if handleIPv6 {
-		errResource, err := l4hc.deleteIPv6HealthCheckFirewall(svc, hcName, namer, sharedHC, l4Type)
-		if err != nil {
-			return errResource, err
-		}
-	}
 	return l4hc.deleteHealthCheckFirewall(svc, hcName, hcFwName, sharedHC, l4Type)
 }
 
+// DeleteDualStackHealthCheck deletes health check, ipv4 and ipv6 firewall rules for l4 service.
+// Checks if shared resources are safe to delete.
+func (l4hc *l4HealthChecks) DeleteDualStackHealthCheck(svc *corev1.Service, namer namer.L4ResourcesNamer, sharedHC bool, scope meta.KeyType, l4Type utils.L4LBType) (string, error) {
+	errResource, err := l4hc.DeleteHealthCheck(svc, namer, sharedHC, scope, l4Type)
+	if err != nil {
+		return errResource, err
+	}
+	return l4hc.deleteIPv6HealthCheckFirewall(svc, namer, sharedHC, l4Type)
+}
+
 func (l4hc *l4HealthChecks) ensureL4HealthCheckInternal(hcName string, svcName types.NamespacedName, shared bool, path string, port int32, scope meta.KeyType, l4Type utils.L4LBType) (*composite.HealthCheck, string, error) {
 	selfLink := ""
 	key, err := composite.CreateKey(l4hc.cloud, hcName, scope)
@@ -300,7 +298,8 @@ func (l4hc *l4HealthChecks) deleteHealthCheckFirewall(svc *corev1.Service, hcNam
 	return "", nil
 }
 
-func (l4hc *l4HealthChecks) deleteIPv6HealthCheckFirewall(svc *corev1.Service, hcName string, namer namer.L4ResourcesNamer, sharedHC bool, l4type utils.L4LBType) (string, error) {
+func (l4hc *l4HealthChecks) deleteIPv6HealthCheckFirewall(svc *corev1.Service, namer namer.L4ResourcesNamer, sharedHC bool, l4type utils.L4LBType) (string, error) {
+	hcName, _ := namer.L4HealthCheck(svc.Namespace, svc.Name, sharedHC)
 	ipv6hcFwName := namer.L4IPv6HealthCheckFirewall(svc.Namespace, svc.Name, sharedHC)
 	return l4hc.deleteHealthCheckFirewall(svc, hcName, ipv6hcFwName, sharedHC, l4type)
 }

pkg/healthchecks/interfaces.go

@@ -68,7 +68,7 @@ type L4HealthChecks interface {
 	// DeleteHealthCheck deletes health check (and firewall rule) for l4 service
 	DeleteHealthCheck(svc *v1.Service, namer namer.L4ResourcesNamer, sharedHC bool, scope meta.KeyType, l4Type utils.L4LBType) (string, error)
 	// DeleteDualStackHealthCheck deletes health check (and firewall rule) for l4 service, deletes IPv6 if asked
-	DeleteDualStackHealthCheck(svc *v1.Service, namer namer.L4ResourcesNamer, sharedHC bool, scope meta.KeyType, l4Type utils.L4LBType, handleIPv6 bool) (string, error)
+	DeleteDualStackHealthCheck(svc *v1.Service, namer namer.L4ResourcesNamer, sharedHC bool, scope meta.KeyType, l4Type utils.L4LBType) (string, error)
 }
 
 type EnsureL4HealthCheckResult struct {

pkg/l4lb/l4controller.go

@@ -33,7 +33,6 @@ import (
 	"k8s.io/ingress-gce/pkg/backends"
 	"k8s.io/ingress-gce/pkg/context"
 	"k8s.io/ingress-gce/pkg/controller/translator"
-	"k8s.io/ingress-gce/pkg/flags"
 	"k8s.io/ingress-gce/pkg/forwardingrules"
 	l4metrics "k8s.io/ingress-gce/pkg/l4lb/metrics"
 	"k8s.io/ingress-gce/pkg/loadbalancers"
@@ -72,6 +71,7 @@ type L4Controller struct {
 	syncTracker         utils.TimeTracker
 	forwardingRules     ForwardingRulesGetter
 	sharedResourcesLock sync.Mutex
+	enableDualStack     bool
 }
 
 // NewILBController creates a new instance of the L4 ILB controller.
@@ -90,6 +90,7 @@ func NewILBController(ctx *context.ControllerContext, stopCh chan struct{}) *L4C
 		namer:           ctx.L4Namer,
 		translator:      ctx.Translator,
 		forwardingRules: forwardingrules.New(ctx.Cloud, meta.VersionGA, meta.Regional),
+		enableDualStack: ctx.EnableL4ILBDualStack,
 	}
 	l4c.backendPool = backends.NewPool(ctx.Cloud, l4c.namer)
 	l4c.NegLinker = backends.NewNEGLinker(l4c.backendPool, negtypes.NewAdapter(ctx.Cloud), ctx.Cloud, ctx.SvcNegInformer.GetIndexer())
@@ -209,7 +210,7 @@ func (l4c *L4Controller) processServiceCreateOrUpdate(key string, service *v1.Se
 	}
 	// Use the same function for both create and updates. If controller crashes and restarts,
 	// all existing services will show up as Service Adds.
-	l4 := loadbalancers.NewL4Handler(service, l4c.ctx.Cloud, meta.Regional, l4c.namer, l4c.ctx.Recorder(service.Namespace), flags.F.EnableL4ILBDualStack)
+	l4 := loadbalancers.NewL4Handler(service, l4c.ctx.Cloud, meta.Regional, l4c.namer, l4c.ctx.Recorder(service.Namespace), l4c.enableDualStack)
 	syncResult := l4.EnsureInternalLoadBalancer(nodeNames, service)
 	// syncResult will not be nil
 	if syncResult.Error != nil {
@@ -249,7 +250,7 @@ func (l4c *L4Controller) processServiceCreateOrUpdate(key string, service *v1.Se
 }
 
 func (l4c *L4Controller) processServiceDeletion(key string, svc *v1.Service) *loadbalancers.L4ILBSyncResult {
-	l4 := loadbalancers.NewL4Handler(svc, l4c.ctx.Cloud, meta.Regional, l4c.namer, l4c.ctx.Recorder(svc.Namespace), flags.F.EnableL4ILBDualStack)
+	l4 := loadbalancers.NewL4Handler(svc, l4c.ctx.Cloud, meta.Regional, l4c.namer, l4c.ctx.Recorder(svc.Namespace), l4c.enableDualStack)
 	l4c.ctx.Recorder(svc.Namespace).Eventf(svc, v1.EventTypeNormal, "DeletingLoadBalancer", "Deleting load balancer for %s", key)
 	result := l4.EnsureInternalLoadBalancerDeleted(svc)
 	if result.Error != nil {

pkg/loadbalancers/forwarding_rules.go

@@ -195,39 +195,37 @@ func (l *L7) getEffectiveIP() (string, bool, error) {
 
 // ensureForwardingRule creates a forwarding rule with the given name, if it does not exist. It updates the existing
 // forwarding rule if needed.
-func (l *L4) ensureForwardingRule(loadBalancerName, bsLink string, options gce.ILBOptions, existingFwdRule *composite.ForwardingRule) (*composite.ForwardingRule, error) {
+func (l4 *L4) ensureForwardingRule(loadBalancerName, bsLink string, options gce.ILBOptions, existingFwdRule *composite.ForwardingRule) (*composite.ForwardingRule, error) {
 	// version used for creating the existing forwarding rule.
 	version := meta.VersionGA
 
-	if l.cloud.IsLegacyNetwork() {
-		l.recorder.Event(l.Service, v1.EventTypeWarning, "ILBOptionsIgnored", "Internal LoadBalancer options are not supported with Legacy Networks.")
+	if l4.cloud.IsLegacyNetwork() {
+		l4.recorder.Event(l4.Service, v1.EventTypeWarning, "ILBOptionsIgnored", "Internal LoadBalancer options are not supported with Legacy Networks.")
 		options = gce.ILBOptions{}
 	}
-	subnetworkURL := l.cloud.SubnetworkURL()
+	subnetworkURL := l4.cloud.SubnetworkURL()
 
 	// Custom subnet feature is always enabled when running L4 controller.
 	// Changes to subnet annotation will be picked up and reflected in the forwarding rule.
 	// Removing the annotation will set the forwarding rule to use the default subnet.
 	if options.SubnetName != "" {
-		key, err := l.CreateKey(loadBalancerName)
+		var err error
+		subnetworkURL, err = l4.getSubnetworkURLByName(options.SubnetName, loadBalancerName)
 		if err != nil {
 			return nil, err
 		}
-		subnetKey := *key
-		subnetKey.Name = options.SubnetName
-		subnetworkURL = cloud.SelfLink(meta.VersionGA, l.cloud.NetworkProjectID(), "subnetworks", &subnetKey)
 	}
 	// Determine IP which will be used for this LB. If no forwarding rule has been established
 	// or specified in the Service spec, then requestedIP = "".
-	ipToUse := l4lbIPToUse(l.Service, existingFwdRule, subnetworkURL)
+	ipToUse := l4lbIPToUse(l4.Service, existingFwdRule, subnetworkURL)
 	klog.V(2).Infof("ensureForwardingRule(%v): Using subnet %q for LoadBalancer IP %s", loadBalancerName, subnetworkURL, ipToUse)
 
 	var addrMgr *addressManager
 	// If the network is not a legacy network, use the address manager
-	if !l.cloud.IsLegacyNetwork() {
-		nm := types.NamespacedName{Namespace: l.Service.Namespace, Name: l.Service.Name}.String()
+	if !l4.cloud.IsLegacyNetwork() {
+		nm := types.NamespacedName{Namespace: l4.Service.Namespace, Name: l4.Service.Name}.String()
 		// ILB can be created only in Premium Tier
-		addrMgr = newAddressManager(l.cloud, nm, l.cloud.Region(), subnetworkURL, loadBalancerName, ipToUse, cloud.SchemeInternal, cloud.NetworkTierPremium)
+		addrMgr = newAddressManager(l4.cloud, nm, l4.cloud.Region(), subnetworkURL, loadBalancerName, ipToUse, cloud.SchemeInternal, cloud.NetworkTierPremium)
 		var err error
 		ipToUse, _, err = addrMgr.HoldAddress()
 		if err != nil {
@@ -243,11 +241,11 @@ func (l *L4) ensureForwardingRule(loadBalancerName, bsLink string, options gce.I
 		}()
 	}
 
-	servicePorts := l.Service.Spec.Ports
+	servicePorts := l4.Service.Spec.Ports
 	ports := utils.GetPorts(servicePorts)
 	protocol := utils.GetProtocol(servicePorts)
 	// Create the forwarding rule
-	frDesc, err := utils.MakeL4LBServiceDescription(utils.ServiceKeyFunc(l.Service.Namespace, l.Service.Name), ipToUse,
+	frDesc, err := utils.MakeL4LBServiceDescription(utils.ServiceKeyFunc(l4.Service.Namespace, l4.Service.Name), ipToUse,
 		version, false, utils.ILB)
 	if err != nil {
 		return nil, fmt.Errorf("Failed to compute description for forwarding rule %s, err: %w", loadBalancerName,
@@ -261,7 +259,7 @@ func (l *L4) ensureForwardingRule(loadBalancerName, bsLink string, options gce.I
 		IPProtocol:          string(protocol),
 		LoadBalancingScheme: string(cloud.SchemeInternal),
 		Subnetwork:          subnetworkURL,
-		Network:             l.cloud.NetworkURL(),
+		Network:             l4.cloud.NetworkURL(),
 		NetworkTier:         cloud.NetworkTierDefault.ToGCEValue(),
 		Version:             version,
 		BackendService:      bsLink,
@@ -287,17 +285,17 @@ func (l *L4) ensureForwardingRule(loadBalancerName, bsLink string, options gce.I
 		// If the forwarding rule pointed to a backend service which does not match the controller naming scheme,
 		// that resouce could be leaked. It is not being deleted here because that is a user-managed resource.
 		klog.V(2).Infof("ensureForwardingRule: forwarding rule changed - Existing - %+v\n, New - %+v\n, Diff(-existing, +new) - %s\n. Deleting existing forwarding rule.", existingFwdRule, fr, frDiff)
-		if err = l.forwardingRules.Delete(existingFwdRule.Name); err != nil {
+		if err = l4.forwardingRules.Delete(existingFwdRule.Name); err != nil {
 			return nil, err
 		}
-		l.recorder.Eventf(l.Service, corev1.EventTypeNormal, events.SyncIngress, "ForwardingRule %q deleted", existingFwdRule.Name)
+		l4.recorder.Eventf(l4.Service, corev1.EventTypeNormal, events.SyncIngress, "ForwardingRule %q deleted", existingFwdRule.Name)
 	}
 	klog.V(2).Infof("ensureForwardingRule: Creating/Recreating forwarding rule - %s", fr.Name)
-	if err = l.forwardingRules.Create(fr); err != nil {
+	if err = l4.forwardingRules.Create(fr); err != nil {
 		return nil, err
 	}
 
-	fr, err = l.forwardingRules.Get(fr.Name)
+	fr, err = l4.forwardingRules.Get(fr.Name)
 	if err != nil {
 		return nil, err
 	}

pkg/loadbalancers/forwarding_rules_ipv6.go

@@ -4,7 +4,6 @@ import (
 	"fmt"
 
 	"github.com/GoogleCloudPlatform/k8s-cloud-provider/pkg/cloud"
-	"github.com/GoogleCloudPlatform/k8s-cloud-provider/pkg/cloud/meta"
 	"github.com/google/go-cmp/cmp"
 	"github.com/google/go-cmp/cmp/cmpopts"
 	corev1 "k8s.io/api/core/v1"
@@ -15,15 +14,15 @@ import (
 	"k8s.io/legacy-cloud-providers/gce"
 )
 
-func (l *L4) ensureIPv6ForwardingRule(bsLink string, options gce.ILBOptions) (*composite.ForwardingRule, error) {
-	expectedIPv6FwdRule, err := l.buildExpectedIPv6ForwardingRule(bsLink, options)
+func (l4 *L4) ensureIPv6ForwardingRule(bsLink string, options gce.ILBOptions) (*composite.ForwardingRule, error) {
+	expectedIPv6FwdRule, err := l4.buildExpectedIPv6ForwardingRule(bsLink, options)
 	if err != nil {
-		return nil, fmt.Errorf("l.buildExpectedIPv6ForwardingRule(%s, %v) returned error %w, want nil", bsLink, options, err)
+		return nil, fmt.Errorf("l4.buildExpectedIPv6ForwardingRule(%s, %v) returned error %w, want nil", bsLink, options, err)
 	}
 
-	existingIPv6FwdRule, err := l.forwardingRules.Get(expectedIPv6FwdRule.Name)
+	existingIPv6FwdRule, err := l4.forwardingRules.Get(expectedIPv6FwdRule.Name)
 	if err != nil {
-		return nil, fmt.Errorf("l.forwardingRules.GetForwardingRule(%s) returned error %w, want nil", expectedIPv6FwdRule.Name, err)
+		return nil, fmt.Errorf("l4.forwardingRules.GetForwardingRule(%s) returned error %w, want nil", expectedIPv6FwdRule.Name, err)
 	}
 
 	if existingIPv6FwdRule != nil {
@@ -39,43 +38,40 @@ func (l *L4) ensureIPv6ForwardingRule(bsLink string, options gce.ILBOptions) (*c
 		frDiff := cmp.Diff(existingIPv6FwdRule, expectedIPv6FwdRule, cmpopts.IgnoreFields(composite.ForwardingRule{}, "IPAddress"))
 		klog.V(2).Infof("ensureIPv6ForwardingRule: forwarding rule changed - Existing - %+v\n, New - %+v\n, Diff(-existing, +new) - %s\n. Deleting existing ipv6 forwarding rule.", existingIPv6FwdRule, expectedIPv6FwdRule, frDiff)
 
-		err = l.forwardingRules.Delete(existingIPv6FwdRule.Name)
+		err = l4.forwardingRules.Delete(existingIPv6FwdRule.Name)
 		if err != nil {
 			return nil, err
 		}
-		l.recorder.Eventf(l.Service, corev1.EventTypeNormal, events.SyncIngress, "ForwardingRule %q deleted", existingIPv6FwdRule.Name)
+		l4.recorder.Eventf(l4.Service, corev1.EventTypeNormal, events.SyncIngress, "ForwardingRule %q deleted", existingIPv6FwdRule.Name)
 	}
 	klog.V(2).Infof("ensureIPv6ForwardingRule: Creating/Recreating forwarding rule - %s", expectedIPv6FwdRule.Name)
-	err = l.forwardingRules.Create(expectedIPv6FwdRule)
+	err = l4.forwardingRules.Create(expectedIPv6FwdRule)
 	if err != nil {
 		return nil, err
 	}
 
-	createdFr, err := l.forwardingRules.Get(expectedIPv6FwdRule.Name)
+	createdFr, err := l4.forwardingRules.Get(expectedIPv6FwdRule.Name)
 	return createdFr, err
 }
 
-func (l *L4) buildExpectedIPv6ForwardingRule(bsLink string, options gce.ILBOptions) (*composite.ForwardingRule, error) {
-	frName := l.getIPv6FRName()
+func (l4 *L4) buildExpectedIPv6ForwardingRule(bsLink string, options gce.ILBOptions) (*composite.ForwardingRule, error) {
+	frName := l4.getIPv6FRName()
 
-	frDesc, err := utils.MakeL4IPv6ForwardingRuleDescription(l.Service)
+	frDesc, err := utils.MakeL4IPv6ForwardingRuleDescription(l4.Service)
 	if err != nil {
 		return nil, fmt.Errorf("failed to compute description for forwarding rule %s, err: %w", frName, err)
 	}
 
-	subnetworkURL := l.cloud.SubnetworkURL()
+	subnetworkURL := l4.cloud.SubnetworkURL()
 
 	if options.SubnetName != "" {
-		key, err := l.CreateKey(frName)
+		subnetworkURL, err = l4.getSubnetworkURLByName(options.SubnetName, frName)
 		if err != nil {
 			return nil, err
 		}
-		subnetKey := *key
-		subnetKey.Name = options.SubnetName
-		subnetworkURL = cloud.SelfLink(meta.VersionGA, l.cloud.NetworkProjectID(), "subnetworks", &subnetKey)
 	}
 
-	svcPorts := l.Service.Spec.Ports
+	svcPorts := l4.Service.Spec.Ports
 	ports := utils.GetPorts(svcPorts)
 	protocol := utils.GetProtocol(svcPorts)
 
@@ -87,7 +83,7 @@ func (l *L4) buildExpectedIPv6ForwardingRule(bsLink string, options gce.ILBOptio
 		LoadBalancingScheme: string(cloud.SchemeInternal),
 		BackendService:      bsLink,
 		IpVersion:           "IPV6",
-		Network:             l.cloud.NetworkURL(),
+		Network:             l4.cloud.NetworkURL(),
 		Subnetwork:          subnetworkURL,
 		AllowGlobalAccess:   options.AllowGlobalAccess,
 		NetworkTier:         cloud.NetworkTierPremium.ToGCEValue(),