Add dual stack support

images/kindnetd/Dockerfile

@@ -14,7 +14,7 @@
 
 # first stage build kindnetd binary
 # NOTE: tentatively follow upstream kubernetes go version based on k8s in go.mod
-FROM golang:1.13
+FROM golang:1.15
 WORKDIR /go/src
 # make deps fetching cacheable
 COPY go.mod go.sum ./

images/kindnetd/cmd/kindnetd/cni.go

@@ -26,29 +26,39 @@ import (
 	"github.com/pkg/errors"
 
 	corev1 "k8s.io/api/core/v1"
-	"k8s.io/utils/net"
 )
 
 /* cni config management */
 
 // CNIConfigInputs is supplied to the CNI config template
 type CNIConfigInputs struct {
-	PodCIDR      string
-	DefaultRoute string
-	Mtu          int
+	PodCIDRs      []string
+	DefaultRoutes []string
+	Mtu           int
 }
 
 // ComputeCNIConfigInputs computes the template inputs for CNIConfigWriter
 func ComputeCNIConfigInputs(node corev1.Node) CNIConfigInputs {
-	podCIDR := node.Spec.PodCIDR
-	defaultRoute := "0.0.0.0/0"
 
-	if net.IsIPv6CIDRString(podCIDR) {
-		defaultRoute = "::/0"
+	defaultRoutes := []string{"0.0.0.0/0", "::/0"}
+	// check if is a dualstack cluster
+	if len(node.Spec.PodCIDRs) > 1 {
+		return CNIConfigInputs{
+			PodCIDRs:      node.Spec.PodCIDRs,
+			DefaultRoutes: defaultRoutes,
+		}
+	}
+	// the cluster is single stack
+	// we use the legacy node.Spec.PodCIDR for backwards compatibility
+	podCIDRs := []string{node.Spec.PodCIDR}
+	// This is a single stack cluster
+	defaultRoute := defaultRoutes[:1]
+	if isIPv6CIDRString(podCIDRs[0]) {
+		defaultRoute = defaultRoutes[1:]
 	}
 	return CNIConfigInputs{
-		PodCIDR:      podCIDR,
-		DefaultRoute: defaultRoute,
+		PodCIDRs:      podCIDRs,
+		DefaultRoutes: defaultRoute,
 	}
 }
 
@@ -82,17 +92,19 @@ const cniConfigTemplate = `
 			"type": "host-local",
 			"dataDir": "/run/cni-ipam-state",
 			"routes": [
-				{
-					"dst": "{{ .DefaultRoute }}"
-				}
+				{{$first := true}}
+				{{- range $route := .DefaultRoutes}}
+				{{if $first}}{{$first = false}}{{else}},{{end}}
+				{ "dst": "{{ $route }}" }
+				{{- end}}
 			],
 			"ranges": [
-			[
-				{
-					"subnet": "{{ .PodCIDR }}"
-				}
+				{{$first := true}}
+				{{- range $cidr := .PodCIDRs}}
+				{{if $first}}{{$first = false}}{{else}},{{end}}
+				[ { "subnet": "{{ $cidr }}" } ]
+				{{- end}}
 			]
-		]
 		}
 		{{if .Mtu}},
 		"mtu": {{ .Mtu }}

images/kindnetd/cmd/kindnetd/main.go

@@ -22,15 +22,17 @@ import (
 	"fmt"
 	"net"
 	"os"
+	"strings"
 	"syscall"
 	"time"
 
+	"k8s.io/apimachinery/pkg/util/sets"
+
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/rest"
 	"k8s.io/klog/v2"
-	utilsnet "k8s.io/utils/net"
 )
 
 const (
@@ -50,6 +52,18 @@ const (
 
 // TODO: improve logging & error handling
 
+// IPFamily defines kindnet networking operating model
+type IPFamily string
+
+const (
+	// IPv4Family sets IPFamily to ipv4
+	IPv4Family IPFamily = "ipv4"
+	// IPv6Family sets IPFamily to ipv6
+	IPv6Family IPFamily = "ipv6"
+	// DualStackFamily sets ClusterIPFamily to DualStack
+	DualStackFamily IPFamily = "dualstack"
+)
+
 func main() {
 	// enable logging
 	klog.InitFlags(nil)
@@ -92,10 +106,10 @@ func main() {
 	hostIP, podIP := os.Getenv("HOST_IP"), os.Getenv("POD_IP")
 	klog.Infof("hostIP = %s\npodIP = %s\n", hostIP, podIP)
 	if hostIP != podIP {
-		panic(fmt.Sprintf(
+		klog.Warningf(
 			"hostIP(= %q) != podIP(= %q) but must be running with host network: ",
 			hostIP, podIP,
-		))
+		)
 	}
 
 	mtu, err := computeBridgeMTU()
@@ -110,20 +124,58 @@ func main() {
 	}
 
 	// enforce ip masquerade rules
-	// TODO: dual stack...?
-	masqAgent, err := NewIPMasqAgent(utilsnet.IsIPv6String(hostIP), []string{os.Getenv("POD_SUBNET")})
-	if err != nil {
-		panic(err.Error())
+	podSubnetEnv := os.Getenv("POD_SUBNET")
+	if podSubnetEnv == "" {
+		panic("missing environment variable POD_SUBNET")
+	}
+	podSubnetEnv = strings.TrimSpace(podSubnetEnv)
+	podSubnets := strings.Split(podSubnetEnv, ",")
+	clusterIPv4Subnets, clusterIPv6Subnets := splitCIDRs(podSubnets)
+
+	// detect the cluster IP family based on the Cluster CIDR aka PodSubnet
+	var ipFamily IPFamily
+	if len(clusterIPv4Subnets) > 0 && len(clusterIPv6Subnets) > 0 {
+		ipFamily = DualStackFamily
+	} else if len(clusterIPv6Subnets) > 0 {
+		ipFamily = IPv6Family
+	} else if len(clusterIPv4Subnets) > 0 {
+		ipFamily = IPv4Family
+	} else {
+		panic(fmt.Sprintf("podSubnets ClusterCIDR/Pod_Subnet: %v", podSubnetEnv))
+	}
+	klog.Infof("kindnetd IP family: %q", ipFamily)
+
+	// create an ipMasqAgent for IPv4
+	if len(clusterIPv4Subnets) > 0 {
+		klog.Infof("noMask IPv4 subnets: %v", clusterIPv4Subnets)
+		masqAgentIPv4, err := NewIPMasqAgent(false, clusterIPv4Subnets)
+		if err != nil {
+			panic(err.Error())
+		}
+		go func() {
+			if err := masqAgentIPv4.SyncRulesForever(time.Second * 60); err != nil {
+				panic(err)
+			}
+		}()
 	}
-	// run the masqAgent and panic if is not able to install the rules to no masquerade the pod to pod traffic
-	go func() {
-		if err := masqAgent.SyncRulesForever(time.Second * 60); err != nil {
+
+	// create an ipMasqAgent for IPv6
+	if len(clusterIPv6Subnets) > 0 {
+		klog.Infof("noMask IPv6 subnets: %v", clusterIPv6Subnets)
+		masqAgentIPv6, err := NewIPMasqAgent(true, clusterIPv6Subnets)
+		if err != nil {
 			panic(err.Error())
 		}
-	}()
+
+		go func() {
+			if err := masqAgentIPv6.SyncRulesForever(time.Second * 60); err != nil {
+				panic(err)
+			}
+		}()
+	}
 
 	// setup nodes reconcile function, closes over arguments
-	reconcileNodes := makeNodesReconciler(cniConfigWriter, hostIP)
+	reconcileNodes := makeNodesReconciler(cniConfigWriter, hostIP, ipFamily)
 
 	// main control loop
 	for {
@@ -162,27 +214,17 @@ func main() {
 }
 
 // nodeNodesReconciler returns a reconciliation func for nodes
-func makeNodesReconciler(cniConfig *CNIConfigWriter, hostIP string) func(*corev1.NodeList) error {
+func makeNodesReconciler(cniConfig *CNIConfigWriter, hostIP string, ipFamily IPFamily) func(*corev1.NodeList) error {
 	// reconciles a node
 	reconcileNode := func(node corev1.Node) error {
-		// first get this node's IP
-		nodeIP := internalIP(node)
-		if nodeIP == "" {
-			klog.Infof("Node %v has no Internal IP, ignoring\n", node.Name)
-			return nil
-		}
-
-		// don't do anything unless there is a PodCIDR
-		podCIDR := node.Spec.PodCIDR
-		if podCIDR == "" {
-			klog.Infof("Node %v has no CIDR, ignoring\n", node.Name)
-			return nil
-		}
-
-		// This is our node. We don't need to add routes, but we might need to
-		// update the cni config.
-		if nodeIP == hostIP {
-			klog.Infof("handling current node\n")
+		// first get this node's IPs
+		// we don't support more than one IP address per IP family for simplification
+		nodeIPs := internalIPs(node)
+		klog.Infof("Handling node with IPs: %v\n", nodeIPs)
+		// This is our node. We don't need to add routes,
+		// but we might need to update the cni config
+		if nodeIPs.Has(hostIP) {
+			klog.Info("handling current node\n")
 			// compute the current cni config inputs
 			if err := cniConfig.Write(
 				ComputeCNIConfigInputs(node),
@@ -193,12 +235,41 @@ func makeNodesReconciler(cniConfig *CNIConfigWriter, hostIP string) func(*corev1
 			return nil
 		}
 
-		klog.Infof("Handling node with IP: %s\n", nodeIP)
-		klog.Infof("Node %v has CIDR %s \n", node.Name, podCIDR)
-		if err := syncRoute(nodeIP, podCIDR); err != nil {
-			return err
+		// This is another node. Add routes to the POD subnets in the other nodes
+		// don't do anything unless there is a PodCIDR
+		var podCIDRs []string
+		if ipFamily == DualStackFamily {
+			podCIDRs = node.Spec.PodCIDRs
+		} else {
+			podCIDRs = []string{node.Spec.PodCIDR}
+		}
+		if len(podCIDRs) == 0 {
+			fmt.Printf("Node %v has no CIDR, ignoring\n", node.Name)
+			return nil
+		}
+		klog.Infof("Node %v has CIDR %s \n", node.Name, podCIDRs)
+		podCIDRsv4, podCIDRsv6 := splitCIDRs(podCIDRs)
+
+		// obtain the PodCIDR gateway
+		var nodeIPv4, nodeIPv6 string
+		for _, ip := range nodeIPs.List() {
+			if isIPv6String(ip) {
+				nodeIPv6 = ip
+			} else {
+				nodeIPv4 = ip
+			}
 		}
 
+		if nodeIPv4 != "" && len(podCIDRsv4) > 0 {
+			if err := syncRoute(nodeIPv4, podCIDRsv4); err != nil {
+				return err
+			}
+		}
+		if nodeIPv6 != "" && len(podCIDRsv6) > 0 {
+			if err := syncRoute(nodeIPv6, podCIDRsv6); err != nil {
+				return err
+			}
+		}
 		return nil
 	}
 
@@ -213,14 +284,30 @@ func makeNodesReconciler(cniConfig *CNIConfigWriter, hostIP string) func(*corev1
 	}
 }
 
-// internalIP returns the internalIP address for node
-func internalIP(node corev1.Node) string {
+// internalIPs returns the internal IP addresses for node
+func internalIPs(node corev1.Node) sets.String {
+	ips := sets.NewString()
+	// check the node.Status.Addresses
 	for _, address := range node.Status.Addresses {
 		if address.Type == "InternalIP" {
-			return address.Address
+			ips.Insert(address.Address)
 		}
 	}
-	return ""
+	return ips
+}
+
+// splitCIDRs given a slice of strings with CIDRs it returns 2 slice of strings per IP family
+// The order returned is always v4 v6
+func splitCIDRs(cidrs []string) ([]string, []string) {
+	var v4subnets, v6subnets []string
+	for _, subnet := range cidrs {
+		if isIPv6CIDRString(subnet) {
+			v6subnets = append(v6subnets, subnet)
+		} else {
+			v4subnets = append(v4subnets, subnet)
+		}
+	}
+	return v4subnets, v6subnets
 }
 
 // Modified from agnhost connect command in k/k
@@ -251,3 +338,16 @@ func probeTCP(address string, timeout time.Duration) bool {
 	klog.Warningf("OTHER %s: %v", address, err)
 	return false
 }
+
+// isIPv6String returns if ip is IPv6.
+func isIPv6String(ip string) bool {
+	netIP := net.ParseIP(ip)
+	return netIP != nil && netIP.To4() == nil
+}
+
+// isIPv6CIDRString returns if cidr is IPv6.
+// This assumes cidr is a valid CIDR.
+func isIPv6CIDRString(cidr string) bool {
+	ip, _, _ := net.ParseCIDR(cidr)
+	return ip != nil && ip.To4() == nil
+}

images/kindnetd/cmd/kindnetd/routes.go

@@ -24,28 +24,30 @@ import (
 	"k8s.io/klog/v2"
 )
 
-func syncRoute(nodeIP, podCIDR string) error {
-	// parse subnet
-	dst, err := netlink.ParseIPNet(podCIDR)
-	if err != nil {
-		return err
-	}
-
-	// Check if the route exists to the other node's PodCIDR
+func syncRoute(nodeIP string, podCIDRs []string) error {
 	ip := net.ParseIP(nodeIP)
-	routeToDst := netlink.Route{Dst: dst, Gw: ip}
-	route, err := netlink.RouteListFiltered(nl.GetIPFamily(ip), &routeToDst, netlink.RT_FILTER_DST)
-	if err != nil {
-		return err
-	}
 
-	// Add route if not present
-	if len(route) == 0 {
-		if err := netlink.RouteAdd(&routeToDst); err != nil {
+	for _, podCIDR := range podCIDRs {
+		// parse subnet
+		dst, err := netlink.ParseIPNet(podCIDR)
+		if err != nil {
 			return err
 		}
-		klog.Infof("Adding route %v \n", routeToDst)
-	}
 
+		// Check if the route exists to the other node's PodCIDR
+		routeToDst := netlink.Route{Dst: dst, Gw: ip}
+		route, err := netlink.RouteListFiltered(nl.GetIPFamily(ip), &routeToDst, netlink.RT_FILTER_DST)
+		if err != nil {
+			return err
+		}
+
+		// Add route if not present
+		if len(route) == 0 {
+			klog.Infof("Adding route %v \n", routeToDst)
+			if err := netlink.RouteAdd(&routeToDst); err != nil {
+				return err
+			}
+		}
+	}
 	return nil
 }