WIP kindnet 0.6

cmd/kindnetd/cni.go

@@ -32,20 +32,32 @@ import (
 
 // CNIConfigInputs is supplied to the CNI config template
 type CNIConfigInputs struct {
-	PodCIDR      string
-	DefaultRoute string
+	PodCIDRs      []string
+	DefaultRoutes []string
 }
 
 // ComputeCNIConfigInputs computes the template inputs for CNIConfigWriter
 func ComputeCNIConfigInputs(node corev1.Node) CNIConfigInputs {
-	podCIDR := node.Spec.PodCIDR
-	defaultRoute := "0.0.0.0/0"
-	if net.IsIPv6CIDRString(podCIDR) {
-		defaultRoute = "::/0"
+
+	defaultRoutes := []string{"0.0.0.0/0", "::/0"}
+	// check if is a dual-stack cluster
+	if len(node.Spec.PodCIDRs) > 1 {
+		return CNIConfigInputs{
+			PodCIDRs:      node.Spec.PodCIDRs,
+			DefaultRoutes: defaultRoutes,
+		}
+	}
+	// the cluster is single stack
+	// we use the legacy node.Spec.PodCIDR for backwards compatibility
+	podCIDRs := []string{"node.Spec.PodCIDR"}
+	// This is a single stack cluster
+	defaultRoute := defaultRoutes[:1]
+	if net.IsIPv6CIDRString(podCIDRs[0]) {
+		defaultRoute := defaultRoutes[1:]
 	}
 	return CNIConfigInputs{
-		PodCIDR:      podCIDR,
-		DefaultRoute: defaultRoute,
+		PodCIDRs:      podCIDRs,
+		DefaultRoutes: defaultRoute,
 	}
 }
 
@@ -64,15 +76,19 @@ const cniConfigTemplate = `
 			"type": "host-local",
 			"dataDir": "/run/cni-ipam-state",
 			"routes": [
-				{
-					"dst": "{{ .DefaultRoute }}"
-				}
+				{{range $route := .DefaultRoutes}}
+					{
+						"dst": "{{ $route }}"
+					},
+				{{end}}
 			],
 			"ranges": [
 			[
-				{
-					"subnet": "{{ .PodCIDR }}"
-				}
+				{{range $cidr := .PodCIDRs}}
+					{
+						"subnet": "{{ $cidr }}"
+					},
+				{{end}}
 			]
 		]
 		}

cmd/kindnetd/main.go

@@ -19,6 +19,7 @@ package main
 import (
 	"fmt"
 	"os"
+	"strings"
 	"time"
 
 	corev1 "k8s.io/api/core/v1"
@@ -36,11 +37,25 @@ import (
 // input envs:
 // - HOST_IP: hould be populated by downward API
 // - POD_IP: should be populated by downward API
+// - POD_IPS: should be populated by downward API (only with dual-stack clusters)
 // - CNI_CONFIG_TEMPLATE: the cni .conflist template, run with {{ .PodCIDR }}
 
 // TODO: improve logging & error handling
 
+// IPFamily defines kindnet networking operating model
+type IPFamily string
+
+const (
+	// IPv4Family sets IPFamily to ipv4
+	IPv4Family IPFamily = "ipv4"
+	// IPv6Family sets IPFamily to ipv6
+	IPv6Family IPFamily = "ipv6"
+	// DualStackFamily sets ClusterIPFamily to dual-stack
+	DualStackFamily IPFamily = "dual-stack"
+)
+
 func main() {
+
 	// create a Kubernetes client
 	config, err := rest.InClusterConfig()
 	if err != nil {
@@ -52,36 +67,65 @@ func main() {
 	}
 
 	// obtain the host and pod ip addresses
-	// if both ips are different we are not using the host network
 	hostIP, podIP := os.Getenv("HOST_IP"), os.Getenv("POD_IP")
+	// if both ips are different we are not using the host network
 	fmt.Printf("hostIP = %s\npodIP = %s\n", hostIP, podIP)
 	if hostIP != podIP {
 		panic(fmt.Sprintf(
 			"hostIP(= %q) != podIP(= %q) but must be running with host network: ",
 			hostIP, podIP,
 		))
 	}
+	// obtain all pod ip addresses if they exist
+	podIPs := strings.Split(os.Getenv("POD_IPS"), ",")
+	// detect the cluster IP family based on the IP addresses that exist in a POD
+	if len(podIPs) > 1 && net.IsDualStackIPStrings(podIPs) {
+		ipFamily := DualStackFamily
+	} else if net.IsIPv6String(podIP) {
+		ipFamily := IPv6Family
+	} else {
+		ipFamily := IPv4Family
+	}
 
 	// used to track if the cni config inputs changed and write the config
 	cniConfigWriter := &CNIConfigWriter{
 		path: cniConfigPath,
 	}
 
 	// enforce ip masquerade rules
-	// TODO: dual stack...?
-	masqAgent, _ := NewIPMasqAgent(
-		net.IsIPv6String(hostIP),
-		getNoMasqueradeSubnets(clientset),
-	)
-	go func() {
-		// TODO: use logging and continue retrying instead...
-		if err := masqAgent.SyncRulesForever(time.Second * 60); err != nil {
-			panic(err)
-		}
-	}()
+	noMaskIPv4Subnets, noMaskIPv6Subnets := getNoMasqueradeSubnets(clientset)
+
+	// create an ipMasqAgent for IPv4
+	if len(noMaskIPv4Subnets) > 0 {
+		masqAgentIPv4, _ := NewIPMasqAgent(
+			false,
+			noMaskIPv4Subnets,
+		)
+		go func() {
+			// TODO: use logging and continue retrying instead...
+			if err := masqAgentIPv4.SyncRulesForever(time.Second * 60); err != nil {
+				panic(err)
+			}
+		}()
+	}
+
+	// create an ipMasqAgent for IPv6
+	if len(noMaskIPv6Subnets) > 0 {
+		masqAgentIPv6, _ := NewIPMasqAgent(
+			true,
+			noMaskIPv6Subnets,
+		)
+
+		go func() {
+			// TODO: use logging and continue retrying instead...
+			if err := masqAgentIPv6.SyncRulesForever(time.Second * 60); err != nil {
+				panic(err)
+			}
+		}()
+	}
 
 	// setup nodes reconcile function, closes over arguments
-	reconcileNodes := makeNodesReconciler(cniConfigWriter, hostIP)
+	reconcileNodes := makeNodesReconciler(cniConfigWriter, hostIP, ipFamily)
 
 	// main control loop
 	for {
@@ -103,26 +147,40 @@ func main() {
 }
 
 // nodeNodesReconciler returns a reconciliation func for nodes
-func makeNodesReconciler(cniConfig *CNIConfigWriter, hostIP string) func(*corev1.NodeList) error {
+func makeNodesReconciler(cniConfig *CNIConfigWriter, hostIP string, ipFamily IPFamily) func(*corev1.NodeList) error {
 	// reconciles a node
 	reconcileNode := func(node corev1.Node) error {
-		// first get this node's IP
-		nodeIP := internalIP(node)
-		if nodeIP == "" {
-			fmt.Printf("Node %v has no Internal IP, ignoring\n", node.Name)
+		// first get this node's IPs
+		// we don't support more than one IP address per IP family for simplification
+		nodeIPs := internalIPs(node)
+		if len(nodeIPs) == 0 || len(nodeIPs) > 2 {
+			fmt.Printf("Node %v has wrong number of Internal IPs, ignoring\n", node.Name)
 			return nil
 		}
 
+		var nodeIPv4, nodeIPv6 string
+		for _, ip := range nodeIPs {
+			if net.IsIPv6String(ip) {
+				nodeIPv6 = ip
+			} else {
+				nodeIPv4 = ip
+			}
+		}
+
 		// don't do anything unless there is a PodCIDR
-		podCIDR := node.Spec.PodCIDR
-		if podCIDR == "" {
+		if ipFamily == DualStackFamily {
+			podCIDRs := node.Spec.PodCIDRS
+		} else {
+			podCIDRs := []string{node.Spec.PodCIDR}
+		}
+		if len(podCIDRs) == 0 {
 			fmt.Printf("Node %v has no CIDR, ignoring\n", node.Name)
 			return nil
 		}
-
+		podCIDRsv4, podCIDRsv6 := splitCIDRs(podCIDRs)
 		// This is our node. We don't need to add routes, but we might need to
 		// update the cni config.
-		if nodeIP == hostIP {
+		if stringInSlice(hostIP, nodeIPs) {
 			fmt.Printf("handling current node\n")
 			// compute the current cni config inputs
 			if err := cniConfig.Write(
@@ -134,12 +192,18 @@ func makeNodesReconciler(cniConfig *CNIConfigWriter, hostIP string) func(*corev1
 			return nil
 		}
 
-		fmt.Printf("Handling node with IP: %s\n", nodeIP)
-		fmt.Printf("Node %v has CIDR %s \n", node.Name, podCIDR)
-		if err := syncRoute(nodeIP, podCIDR); err != nil {
-			return err
+		fmt.Printf("Handling node with IPs: %v\n", nodeIPs)
+		fmt.Printf("Node %v has CIDRs %v \n", node.Name, podCIDRs)
+		if ipFamily == DualStackFamily || ipFamily == IPv4Family {
+			if err := syncRoute(nodeIPv4, podCIDRsv4); err != nil {
+				return err
+			}
+		}
+		if ipFamily == DualStackFamily || ipFamily == IPv6Family {
+			if err := syncRoute(nodeIPv6, podCIDRsV6); err != nil {
+				return err
+			}
 		}
-
 		return nil
 	}
 
@@ -154,12 +218,23 @@ func makeNodesReconciler(cniConfig *CNIConfigWriter, hostIP string) func(*corev1
 	}
 }
 
-// internalIP returns the internalIP address for node
-func internalIP(node corev1.Node) string {
+// internalIPs returns the internal IP address for node
+func internalIPs(node corev1.Node) []string {
+	var ips []string
 	for _, address := range node.Status.Addresses {
 		if address.Type == "InternalIP" {
-			return address.Address
+			ips = append(ips, address.Address)
+		}
+	}
+	return ips
+}
+
+// stringInSlice find if a strings exist in an array of strings
+func stringInSlice(a string, list []string) bool {
+	for _, b := range list {
+		if b == a {
+			return true
 		}
 	}
-	return ""
+	return false
 }

cmd/kindnetd/routes.go

@@ -24,28 +24,29 @@ import (
 	"github.com/vishvananda/netlink/nl"
 )
 
-func syncRoute(nodeIP, podCIDR string) error {
-	// parse subnet
-	dst, err := netlink.ParseIPNet(podCIDR)
-	if err != nil {
-		return err
-	}
-
-	// Check if the route exists to the other node's PodCIDR
-	ip := net.ParseIP(nodeIP)
-	routeToDst := netlink.Route{Dst: dst, Gw: ip}
-	route, err := netlink.RouteListFiltered(nl.GetIPFamily(ip), &routeToDst, netlink.RT_FILTER_DST)
-	if err != nil {
-		return err
-	}
+func syncRoute(nodeIP string, podCIDRs []string) error {
+	for _, podCIDR := range podCIDRs {
+		// parse subnet
+		dst, err := netlink.ParseIPNet(podCIDR)
+		if err != nil {
+			return err
+		}
 
-	// Add route if not present
-	if len(route) == 0 {
-		if err := netlink.RouteAdd(&routeToDst); err != nil {
+		// Check if the route exists to the other node's PodCIDR
+		ip := net.ParseIP(nodeIP)
+		routeToDst := netlink.Route{Dst: dst, Gw: ip}
+		route, err := netlink.RouteListFiltered(nl.GetIPFamily(ip), &routeToDst, netlink.RT_FILTER_DST)
+		if err != nil {
 			return err
 		}
-		fmt.Printf("Adding route %v \n", routeToDst)
-	}
 
+		// Add route if not present
+		if len(route) == 0 {
+			if err := netlink.RouteAdd(&routeToDst); err != nil {
+				return err
+			}
+			fmt.Printf("Adding route %v \n", routeToDst)
+		}
+	}
 	return nil
 }