Kindnetd 0.6.0 DualStack

cmd/kindnetd/main.go

@@ -18,10 +18,13 @@ package main
 
 import (
 	"fmt"
+	"net"
 	"os"
 	"strings"
 	"time"
 
+	"k8s.io/apimachinery/pkg/util/sets"
+
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/kubernetes"
@@ -76,20 +79,6 @@ func main() {
 			hostIP, podIP,
 		))
 	}
-	// obtain all pod ip addresses if they exist
-	podIPs := strings.Split(os.Getenv("POD_IPS"), ",")
-	// detect the cluster IP family based on the IP addresses that exist in a POD
-	// if is not dual stack we check only first address of the cluster
-	// since multiple addresses of the same family will be supported in the future
-	var ipFamily IPFamily
-	dualstack, _ := utilsnet.IsDualStackIPStrings(podIPs)
-	if dualstack {
-		ipFamily = DualStackFamily
-	} else if utilsnet.IsIPv6String(podIP) {
-		ipFamily = IPv6Family
-	} else {
-		ipFamily = IPv4Family
-	}
 
 	// used to track if the cni config inputs changed and write the config
 	cniConfigWriter := &CNIConfigWriter{
@@ -98,6 +87,17 @@ func main() {
 
 	// enforce ip masquerade rules
 	noMaskIPv4Subnets, noMaskIPv6Subnets := getNoMasqueradeSubnets(clientset)
+	// detect the cluster IP family based on the Cluster CIDR akka PodSubnet
+	var ipFamily IPFamily
+	if len(noMaskIPv4Subnets) > 0 && len(noMaskIPv6Subnets) > 0 {
+		ipFamily = DualStackFamily
+	} else if len(noMaskIPv6Subnets) > 0 {
+		ipFamily = IPv6Family
+	} else if len(noMaskIPv4Subnets) > 0 {
+		ipFamily = IPv4Family
+	} else {
+		panic(fmt.Sprint("Cluster CIDR is not defined"))
+	}
 
 	// create an ipMasqAgent for IPv4
 	if len(noMaskIPv4Subnets) > 0 {
@@ -129,7 +129,7 @@ func main() {
 	}
 
 	// setup nodes reconcile function, closes over arguments
-	reconcileNodes := makeNodesReconciler(cniConfigWriter, hostIP, ipFamily)
+	reconcileNodes := makeNodesReconciler(cniConfigWriter, hostIP, ipFamily, clientset)
 
 	// main control loop
 	for {
@@ -151,19 +151,19 @@ func main() {
 }
 
 // nodeNodesReconciler returns a reconciliation func for nodes
-func makeNodesReconciler(cniConfig *CNIConfigWriter, hostIP string, ipFamily IPFamily) func(*corev1.NodeList) error {
+func makeNodesReconciler(cniConfig *CNIConfigWriter, hostIP string, ipFamily IPFamily, clientset *kubernetes.Clientset) func(*corev1.NodeList) error {
 	// reconciles a node
 	reconcileNode := func(node corev1.Node) error {
 		// first get this node's IPs
 		// we don't support more than one IP address per IP family for simplification
 		nodeIPs := internalIPs(node)
-		if len(nodeIPs) == 0 || len(nodeIPs) > 2 {
-			fmt.Printf("Node %v has wrong number of Internal IPs, ignoring\n", node.Name)
+		if (ipFamily == DualStackFamily && len(nodeIPs) != 2) || (ipFamily != DualStackFamily && len(nodeIPs) != 1) {
+			fmt.Printf("Node %v has wrong number of Internal IPs: %v , ignoring\n", node.Name, nodeIPs)
 			return nil
 		}
 
 		var nodeIPv4, nodeIPv6 string
-		for _, ip := range nodeIPs {
+		for _, ip := range nodeIPs.List() {
 			if utilsnet.IsIPv6String(ip) {
 				nodeIPv6 = ip
 			} else {
@@ -185,14 +185,24 @@ func makeNodesReconciler(cniConfig *CNIConfigWriter, hostIP string, ipFamily IPF
 		podCIDRsv4, podCIDRsv6 := splitCIDRs(podCIDRs)
 		// This is our node. We don't need to add routes, but we might need to
 		// update the cni config.
-		if stringInSlice(hostIP, nodeIPs) {
+		if nodeIPs.Has(hostIP) {
 			fmt.Printf("handling current node\n")
 			// compute the current cni config inputs
 			if err := cniConfig.Write(
 				ComputeCNIConfigInputs(node),
 			); err != nil {
 				return err
 			}
+			// update internal IPs node annotations until #42125 is fixed
+			// TODO: https://github.com/kubernetes/kubernetes/issues/4A2125
+			// in kind we can hardcode the outer interface but we can be smarter
+			// and find the interface matching the HOST_IP
+			node.Annotations["io.k8s.sigs.kind.kindnet/addresses"] = getNodeExternalIPs(hostIP)
+			fmt.Printf("current node IPs: %q\n", node.Annotations["io.k8s.sigs.kind.kindnet/addresses"])
+			_, err := clientset.CoreV1().Nodes().Update(&node)
+			if err != nil {
+				return err
+			}
 			// we're done handling this node
 			return nil
 		}
@@ -224,22 +234,86 @@ func makeNodesReconciler(cniConfig *CNIConfigWriter, hostIP string, ipFamily IPF
 }
 
 // internalIPs returns the internal IP address for node
-func internalIPs(node corev1.Node) []string {
-	var ips []string
+func internalIPs(node corev1.Node) sets.String {
+	ips := sets.NewString()
+	// check the node.Status.Addresses
 	for _, address := range node.Status.Addresses {
 		if address.Type == "InternalIP" {
-			ips = append(ips, address.Address)
+			ips.Insert(address.Address)
 		}
 	}
+	// check the node.annotations.Internal.Addresses
+	for _, address := range strings.Split(node.Annotations["io.k8s.sigs.kind.kindnet/addresses"], ",") {
+		ips.Insert(address)
+	}
 	return ips
 }
 
-// stringInSlice find if a strings exist in an array of strings
-func stringInSlice(a string, list []string) bool {
-	for _, b := range list {
-		if b == a {
-			return true
+// getNodeExternalIPs return a comma separated list with the
+// node external IP addresses
+func getNodeExternalIPs(ip string) string {
+	return getInterfaceIPs(interfaceByAddress(ip))
+}
+
+// interfaceByAddress return the name of the interface
+// that hosts a given IP address
+func interfaceByAddress(address string) string {
+	ifaces, err := net.Interfaces()
+	if err != nil {
+		panic(err.Error())
+	}
+	for _, i := range ifaces {
+		addrs, err := i.Addrs()
+		if err != nil {
+			panic(err.Error())
+		}
+		for _, addr := range addrs {
+			var ip net.IP
+			switch v := addr.(type) {
+			case *net.IPAddr:
+				ip = v.IP
+			case *net.IPNet:
+				ip = v.IP
+			default:
+				continue
+			}
+			if ip.String() == address {
+				return i.Name
+			}
+
+		}
+	}
+	return ""
+}
+
+// getInterfaceIPs returns an array with all the global addresses
+// of the interfaces passed as a parameter
+func getInterfaceIPs(ifazName string) string {
+	var ips string
+
+	ifaz, err := net.InterfaceByName(ifazName)
+	if err != nil {
+		panic(err.Error())
+	}
+
+	addrs, err := ifaz.Addrs()
+	if err != nil {
+		panic(err.Error())
+	}
+	for _, addr := range addrs {
+		var ip net.IP
+		switch v := addr.(type) {
+		case *net.IPAddr:
+			ip = v.IP
+		case *net.IPNet:
+			ip = v.IP
+		default:
+			continue
 		}
+		if ip.IsGlobalUnicast() {
+			ips = ips + "," + ip.String()
+		}
+
 	}
-	return false
+	return strings.Trim(ips, ",")
 }

cmd/kindnetd/routes.go

@@ -25,6 +25,13 @@ import (
 )
 
 func syncRoute(nodeIP string, podCIDRs []string) error {
+	ip := net.ParseIP(nodeIP)
+	// return if is not a global unicast address
+	// we can't use to route our traffic
+	if !ip.IsGlobalUnicast() {
+		// TODO fail and handle the error
+		return nil
+	}
 	for _, podCIDR := range podCIDRs {
 		// parse subnet
 		dst, err := netlink.ParseIPNet(podCIDR)
@@ -33,7 +40,6 @@ func syncRoute(nodeIP string, podCIDRs []string) error {
 		}
 
 		// Check if the route exists to the other node's PodCIDR
-		ip := net.ParseIP(nodeIP)
 		routeToDst := netlink.Route{Dst: dst, Gw: ip}
 		route, err := netlink.RouteListFiltered(nl.GetIPFamily(ip), &routeToDst, netlink.RT_FILTER_DST)
 		if err != nil {

cmd/kindnetd/subnets.go

@@ -27,8 +27,8 @@ import (
 )
 
 const (
-	kubeadmClusterCIDRRegex   = `\s+podSubnet: (.*)$`
-	kubeproxyClusterCIDRRegex = `\s+clusterCIDR: (.*)$`
+	kubeadmClusterCIDRRegex   = `\s+podSubnet: (.*)\n`
+	kubeproxyClusterCIDRRegex = `\s+clusterCIDR: (.*)\n`
 )
 
 type kubeSubnets struct {

pkg/build/node/cni.go

@@ -96,6 +96,7 @@ rules:
     verbs:
       - list
       - watch
+      - update
   - apiGroups:
      - ""
     resources: