Merge pull request #36 from hhyasdf/bugfix/add-routes-for-excluded-ips

bugfix: add routes for excluded ips

pkg/daemon/containernetwork/containernetwork.go

@@ -154,28 +154,34 @@ func ConfigureHostNic(nicName string, allocatedIPs map[ramav1.IPVersion]*IPInfo,
 
 // ipAddr is a CIDR notation IP address and prefix length
 func ConfigureContainerNic(containerNicName, hostNicName, nodeIfName string, allocatedIPs map[ramav1.IPVersion]*IPInfo,
-	macAddr net.HardwareAddr, vlanID *uint32, netns ns.NetNS, mtu int, vlanCheckTimeout time.Duration,
+	macAddr net.HardwareAddr, netID *uint32, netns ns.NetNS, mtu int, vlanCheckTimeout time.Duration,
 	networkType ramav1.NetworkType, neighGCThresh1, neighGCThresh2, neighGCThresh3 int) error {
 
 	var defaultRouteNets []*types.Route
 	var ipConfigs []*current.IPConfig
+	var forwardNodeIfName string
+	var err error
 
 	ipv4AddressAllocated := false
 	ipv6AddressAllocated := false
 
-	var vlanIf *net.Interface
 	if networkType == ramav1.NetworkTypeUnderlay {
-		vlanIfName, err := EnsureVlanIf(nodeIfName, vlanID)
+		forwardNodeIfName, err = GenerateVlanNetIfName(nodeIfName, netID)
 		if err != nil {
-			return fmt.Errorf("ensure vlan interface %v error: %v", vlanIfName, err)
+			return fmt.Errorf("generate vlan forward node interface name failed: %v", err)
 		}
-
-		vlanIf, err = net.InterfaceByName(vlanIfName)
+	} else {
+		forwardNodeIfName, err = GenerateVxlanNetIfName(nodeIfName, netID)
 		if err != nil {
-			return fmt.Errorf("get interface by name %v failed: %v", vlanIfName, err)
+			return fmt.Errorf("generate vxlan forward node interface name failed: %v", err)
 		}
 	}
 
+	forwardNodeIf, err := net.InterfaceByName(forwardNodeIfName)
+	if err != nil {
+		return fmt.Errorf("get forward node interface %v failed: %v; if not exist, waiting for daemon to create it", forwardNodeIfName, err)
+	}
+
 	if allocatedIPs[ramav1.IPv4] != nil {
 
 		ipv4AddressAllocated = true
@@ -185,11 +191,14 @@ func ConfigureContainerNic(containerNicName, hostNicName, nodeIfName string, all
 			GW:  allocatedIPs[ramav1.IPv4].Gw,
 		})
 
+		podIP := allocatedIPs[ramav1.IPv4].Addr
+		podCidr := allocatedIPs[ramav1.IPv4].Cidr
+
 		ipConfigs = append(ipConfigs, &current.IPConfig{
 			Version: "4",
 			Address: net.IPNet{
-				IP:   allocatedIPs[ramav1.IPv4].Addr,
-				Mask: allocatedIPs[ramav1.IPv4].Cidr.Mask,
+				IP:   podIP,
+				Mask: podCidr.Mask,
 			},
 			Gateway:   allocatedIPs[ramav1.IPv4].Gw,
 			Interface: current.Int(0),
@@ -210,11 +219,15 @@ func ConfigureContainerNic(containerNicName, hostNicName, nodeIfName string, all
 		// Underlay gw ipv4 ip should be resolved here.
 		// Only underlay network need to do this.
 		if networkType == ramav1.NetworkTypeUnderlay {
-			if err := arp.CheckWithTimeout(vlanIf, allocatedIPs[ramav1.IPv4].Addr,
+			if err := arp.CheckWithTimeout(forwardNodeIf, podIP,
 				allocatedIPs[ramav1.IPv4].Gw, vlanCheckTimeout); err != nil {
 				return fmt.Errorf("ipv4 vlan check failed: %v", err)
 			}
 		}
+
+		if err := checkPodNetConfigReady(podIP, podCidr, forwardNodeIf.Index, netlink.FAMILY_V4); err != nil {
+			return fmt.Errorf("check pod ip %v network configuration failed: %v", podIP, err)
+		}
 	}
 
 	if allocatedIPs[ramav1.IPv6] != nil {
@@ -226,11 +239,14 @@ func ConfigureContainerNic(containerNicName, hostNicName, nodeIfName string, all
 			GW:  allocatedIPs[ramav1.IPv6].Gw,
 		})
 
+		podIP := allocatedIPs[ramav1.IPv6].Addr
+		podCidr := allocatedIPs[ramav1.IPv6].Cidr
+
 		ipConfigs = append(ipConfigs, &current.IPConfig{
 			Version: "6",
 			Address: net.IPNet{
-				IP:   allocatedIPs[ramav1.IPv6].Addr,
-				Mask: allocatedIPs[ramav1.IPv6].Cidr.Mask,
+				IP:   podIP,
+				Mask: podCidr.Mask,
 			},
 			Gateway:   allocatedIPs[ramav1.IPv6].Gw,
 			Interface: current.Int(0),
@@ -245,11 +261,15 @@ func ConfigureContainerNic(containerNicName, hostNicName, nodeIfName string, all
 		}
 
 		if networkType == ramav1.NetworkTypeUnderlay {
-			if err := ndp.CheckWithTimeout(vlanIf, allocatedIPs[ramav1.IPv6].Addr,
+			if err := ndp.CheckWithTimeout(forwardNodeIf, podIP,
 				allocatedIPs[ramav1.IPv6].Gw, vlanCheckTimeout); err != nil {
 				return fmt.Errorf("ipv6 vlan check failed: %v", err)
 			}
 		}
+
+		if err := checkPodNetConfigReady(podIP, podCidr, forwardNodeIf.Index, netlink.FAMILY_V6); err != nil {
+			return fmt.Errorf("check pod ip %v network configuration failed: %v", podIP, err)
+		}
 	}
 
 	if err := ns.WithNetNSPath(netns.Path(), func(_ ns.NetNS) error {

pkg/daemon/containernetwork/utils.go

@@ -21,6 +21,7 @@ import (
 	"fmt"
 	"net"
 	"strings"
+	"time"
 
 	daemonutils "github.com/oecp/rama/pkg/daemon/utils"
 
@@ -235,3 +236,69 @@ func ListLocalAddressExceptLink(exceptLinkName string) ([]netlink.Addr, error) {
 func CheckIPIsGlobalUnicast(ip net.IP) bool {
 	return !ip.IsInterfaceLocalMulticast() && ip.IsGlobalUnicast()
 }
+
+func checkPodRuleExist(podCidr *net.IPNet, family int) (bool, error) {
+	ruleList, err := netlink.RuleList(family)
+	if err != nil {
+		return false, fmt.Errorf("list rule failed: %v", err)
+	}
+
+	for _, rule := range ruleList {
+		if rule.Src != nil && podCidr.String() == rule.Src.String() {
+			return true, nil
+		}
+	}
+
+	return false, nil
+}
+
+func checkPodNeighExist(podIP net.IP, forwardNodeIfIndex int, family int) (bool, error) {
+	neighList, err := netlink.NeighProxyList(forwardNodeIfIndex, family)
+	if err != nil {
+		return false, fmt.Errorf("list neighs for forward node if index %v failed: %v", forwardNodeIfIndex, err)
+	}
+
+	for _, neigh := range neighList {
+		if neigh.IP.Equal(podIP) {
+			return true, nil
+		}
+	}
+
+	return false, nil
+}
+
+func checkPodNetConfigReady(podIP net.IP, podCidr *net.IPNet, forwardNodeIfIndex int, family int) error {
+	backOffBase := 100 * time.Microsecond
+	retries := 4
+
+	for i := 0; i < retries; i++ {
+		time.Sleep(backOffBase)
+		backOffBase = backOffBase * 2
+
+		neighExist, err := checkPodNeighExist(podIP, forwardNodeIfIndex, family)
+		if err != nil {
+			return fmt.Errorf("check pod ip %v neigh exist failed: %v", podIP, err)
+		}
+
+		ruleExist, err := checkPodRuleExist(podCidr, family)
+		if err != nil {
+			return fmt.Errorf("check cidr %v rule exist failed: %v", podCidr, err)
+		}
+
+		if neighExist && ruleExist {
+			break
+		}
+
+		if i == retries-1 {
+			if !neighExist {
+				return fmt.Errorf("proxy neigh for %v is not created, waiting for daemon to create it", podIP)
+			}
+
+			if !ruleExist {
+				return fmt.Errorf("policy rule for %v is not created, waiting for daemon to create it", podCidr)
+			}
+		}
+	}
+
+	return nil
+}

pkg/daemon/controller/subnet.go

@@ -18,13 +18,10 @@ package controller
 
 import (
 	"fmt"
-	"net"
 
 	ramav1 "github.com/oecp/rama/pkg/apis/networking/v1"
 	"github.com/oecp/rama/pkg/daemon/containernetwork"
 
-	"github.com/vishvananda/netlink"
-
 	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/klog"
 )
@@ -124,14 +121,11 @@ func (c *Controller) reconcileSubnet() error {
 			netID = network.Spec.NetID
 		}
 
-		subnetCidr, err := netlink.ParseIPNet(subnet.Spec.Range.CIDR)
-		if err != nil {
-			return fmt.Errorf("failed to parse subnet cidr %v error: %v", subnet.Spec.Range.CIDR, err)
-		}
+		subnetCidr, gatewayIP, startIP, endIP, excludeIPs,
+			_, err := parseSubnetSpecRangeMeta(&subnet.Spec.Range)
 
-		gatewayIP := net.ParseIP(subnet.Spec.Range.Gateway)
-		if gatewayIP == nil {
-			return fmt.Errorf("invalid gateway ip %v", subnet.Spec.Range.Gateway)
+		if err != nil {
+			return fmt.Errorf("parse subnet %v spec range meta failed: %v", subnet.Name, err)
 		}
 
 		var forwardNodeIfName string
@@ -154,7 +148,8 @@ func (c *Controller) reconcileSubnet() error {
 
 		// create policy route
 		routeManager := c.getRouterManager(subnet.Spec.Range.Version)
-		routeManager.AddSubnetInfo(subnetCidr, gatewayIP, forwardNodeIfName, autoNatOutgoing, isOverlay)
+		routeManager.AddSubnetInfo(subnetCidr, gatewayIP, startIP, endIP, excludeIPs,
+			forwardNodeIfName, autoNatOutgoing, isOverlay)
 	}
 
 	if err := c.routeV4Manager.SyncRoutes(); err != nil {

pkg/daemon/controller/utils.go

@@ -20,6 +20,8 @@ import (
 	"fmt"
 	"net"
 
+	"github.com/vishvananda/netlink"
+
 	ramav1 "github.com/oecp/rama/pkg/apis/networking/v1"
 	"github.com/oecp/rama/pkg/daemon/iptables"
 	"github.com/oecp/rama/pkg/daemon/neigh"
@@ -79,3 +81,60 @@ func initErrorMessageWrapper(prefix string) func(string, ...interface{}) string
 		return prefix + fmt.Sprintf(format, args...)
 	}
 }
+
+func parseSubnetSpecRangeMeta(addressRange *ramav1.AddressRange) (cidr *net.IPNet, gateway, start, end net.IP,
+	excludeIPs, reservedIPs []net.IP, err error) {
+
+	if addressRange == nil {
+		return nil, nil, nil, nil, nil, nil,
+			fmt.Errorf("cannot parse a nil range")
+	}
+
+	cidr, err = netlink.ParseIPNet(addressRange.CIDR)
+	if err != nil {
+		return nil, nil, nil, nil, nil, nil,
+			fmt.Errorf("failed to parse subnet cidr %v error: %v", addressRange.CIDR, err)
+	}
+
+	gateway = net.ParseIP(addressRange.Gateway)
+	if gateway == nil {
+		return nil, nil, nil, nil, nil, nil,
+			fmt.Errorf("invalid gateway ip %v", addressRange.Gateway)
+	}
+
+	if addressRange.Start != "" {
+		start = net.ParseIP(addressRange.Start)
+		if start == nil {
+			return nil, nil, nil, nil, nil, nil,
+				fmt.Errorf("invalid start ip %v", addressRange.Start)
+		}
+	}
+
+	if addressRange.End != "" {
+		end = net.ParseIP(addressRange.End)
+		if end == nil {
+			return nil, nil, nil, nil, nil, nil,
+				fmt.Errorf("invalid end ip %v", addressRange.End)
+		}
+	}
+
+	for _, ipString := range addressRange.ExcludeIPs {
+		excludeIP := net.ParseIP(ipString)
+		if excludeIP == nil {
+			return nil, nil, nil, nil, nil, nil,
+				fmt.Errorf("invalid exclude ip %v", ipString)
+		}
+		excludeIPs = append(excludeIPs, excludeIP)
+	}
+
+	for _, ipString := range addressRange.ReservedIPs {
+		reservedIP := net.ParseIP(ipString)
+		if reservedIP == nil {
+			return nil, nil, nil, nil, nil, nil,
+				fmt.Errorf("invalid reserved ip %v", ipString)
+		}
+		reservedIPs = append(reservedIPs, reservedIP)
+	}
+
+	return
+}