Skip to content

Commit

Permalink
Merge pull request #692 from aojea/dualstack
Browse files Browse the repository at this point in the history 
Add dual stack support
  • Loading branch information
@k8s-ci-robot
k8s-ci-robot authored Mar 27, 2021
2 parents d0df6d2 + 04bdfc7 commit 5e58a0d
Show file tree
Hide file tree
Showing 18 changed files with 553 additions and 159 deletions.
2 changes: 1 addition & 1 deletion images/kindnetd/Dockerfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@

# first stage build kindnetd binary
# NOTE: tentatively follow upstream kubernetes go version based on k8s in go.mod
FROM golang:1.13
FROM golang:1.15
WORKDIR /go/src
# make deps fetching cacheable
COPY go.mod go.sum ./
Expand Down
48 changes: 30 additions & 18 deletions images/kindnetd/cmd/kindnetd/cni.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,29 +26,39 @@ import (
"github.com/pkg/errors"

corev1 "k8s.io/api/core/v1"
"k8s.io/utils/net"
)

/* cni config management */

// CNIConfigInputs is supplied to the CNI config template
type CNIConfigInputs struct {
PodCIDR string
DefaultRoute string
Mtu int
PodCIDRs []string
DefaultRoutes []string
Mtu int
}

// ComputeCNIConfigInputs computes the template inputs for CNIConfigWriter
func ComputeCNIConfigInputs(node corev1.Node) CNIConfigInputs {
podCIDR := node.Spec.PodCIDR
defaultRoute := "0.0.0.0/0"

if net.IsIPv6CIDRString(podCIDR) {
defaultRoute = "::/0"
defaultRoutes := []string{"0.0.0.0/0", "::/0"}
// check if is a dualstack cluster
if len(node.Spec.PodCIDRs) > 1 {
return CNIConfigInputs{
PodCIDRs: node.Spec.PodCIDRs,
DefaultRoutes: defaultRoutes,
}
}
// the cluster is single stack
// we use the legacy node.Spec.PodCIDR for backwards compatibility
podCIDRs := []string{node.Spec.PodCIDR}
// This is a single stack cluster
defaultRoute := defaultRoutes[:1]
if isIPv6CIDRString(podCIDRs[0]) {
defaultRoute = defaultRoutes[1:]
}
return CNIConfigInputs{
PodCIDR: podCIDR,
DefaultRoute: defaultRoute,
PodCIDRs: podCIDRs,
DefaultRoutes: defaultRoute,
}
}

Expand Down Expand Up @@ -82,17 +92,19 @@ const cniConfigTemplate = `
"type": "host-local",
"dataDir": "/run/cni-ipam-state",
"routes": [
{
"dst": "{{ .DefaultRoute }}"
}
{{$first := true}}
{{- range $route := .DefaultRoutes}}
{{if $first}}{{$first = false}}{{else}},{{end}}
{ "dst": "{{ $route }}" }
{{- end}}
],
"ranges": [
[
{
"subnet": "{{ .PodCIDR }}"
}
{{$first := true}}
{{- range $cidr := .PodCIDRs}}
{{if $first}}{{$first = false}}{{else}},{{end}}
[ { "subnet": "{{ $cidr }}" } ]
{{- end}}
]
]
}
{{if .Mtu}},
"mtu": {{ .Mtu }}
Expand Down
178 changes: 139 additions & 39 deletions images/kindnetd/cmd/kindnetd/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,15 +22,17 @@ import (
"fmt"
"net"
"os"
"strings"
"syscall"
"time"

"k8s.io/apimachinery/pkg/util/sets"

corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/rest"
"k8s.io/klog/v2"
utilsnet "k8s.io/utils/net"
)

const (
Expand All @@ -50,6 +52,18 @@ const (

// TODO: improve logging & error handling

// IPFamily defines kindnet networking operating model
type IPFamily string

const (
// IPv4Family sets IPFamily to ipv4
IPv4Family IPFamily = "ipv4"
// IPv6Family sets IPFamily to ipv6
IPv6Family IPFamily = "ipv6"
// DualStackFamily sets ClusterIPFamily to DualStack
DualStackFamily IPFamily = "dualstack"
)

func main() {
// enable logging
klog.InitFlags(nil)
Expand Down Expand Up @@ -92,10 +106,10 @@ func main() {
hostIP, podIP := os.Getenv("HOST_IP"), os.Getenv("POD_IP")
klog.Infof("hostIP = %s\npodIP = %s\n", hostIP, podIP)
if hostIP != podIP {
panic(fmt.Sprintf(
klog.Warningf(
"hostIP(= %q) != podIP(= %q) but must be running with host network: ",
hostIP, podIP,
))
)
}

mtu, err := computeBridgeMTU()
Expand All @@ -110,20 +124,58 @@ func main() {
}

// enforce ip masquerade rules
// TODO: dual stack...?
masqAgent, err := NewIPMasqAgent(utilsnet.IsIPv6String(hostIP), []string{os.Getenv("POD_SUBNET")})
if err != nil {
panic(err.Error())
podSubnetEnv := os.Getenv("POD_SUBNET")
if podSubnetEnv == "" {
panic("missing environment variable POD_SUBNET")
}
podSubnetEnv = strings.TrimSpace(podSubnetEnv)
podSubnets := strings.Split(podSubnetEnv, ",")
clusterIPv4Subnets, clusterIPv6Subnets := splitCIDRs(podSubnets)

// detect the cluster IP family based on the Cluster CIDR aka PodSubnet
var ipFamily IPFamily
if len(clusterIPv4Subnets) > 0 && len(clusterIPv6Subnets) > 0 {
ipFamily = DualStackFamily
} else if len(clusterIPv6Subnets) > 0 {
ipFamily = IPv6Family
} else if len(clusterIPv4Subnets) > 0 {
ipFamily = IPv4Family
} else {
panic(fmt.Sprintf("podSubnets ClusterCIDR/Pod_Subnet: %v", podSubnetEnv))
}
klog.Infof("kindnetd IP family: %q", ipFamily)

// create an ipMasqAgent for IPv4
if len(clusterIPv4Subnets) > 0 {
klog.Infof("noMask IPv4 subnets: %v", clusterIPv4Subnets)
masqAgentIPv4, err := NewIPMasqAgent(false, clusterIPv4Subnets)
if err != nil {
panic(err.Error())
}
go func() {
if err := masqAgentIPv4.SyncRulesForever(time.Second * 60); err != nil {
panic(err)
}
}()
}
// run the masqAgent and panic if is not able to install the rules to no masquerade the pod to pod traffic
go func() {
if err := masqAgent.SyncRulesForever(time.Second * 60); err != nil {

// create an ipMasqAgent for IPv6
if len(clusterIPv6Subnets) > 0 {
klog.Infof("noMask IPv6 subnets: %v", clusterIPv6Subnets)
masqAgentIPv6, err := NewIPMasqAgent(true, clusterIPv6Subnets)
if err != nil {
panic(err.Error())
}
}()

go func() {
if err := masqAgentIPv6.SyncRulesForever(time.Second * 60); err != nil {
panic(err)
}
}()
}

// setup nodes reconcile function, closes over arguments
reconcileNodes := makeNodesReconciler(cniConfigWriter, hostIP)
reconcileNodes := makeNodesReconciler(cniConfigWriter, hostIP, ipFamily)

// main control loop
for {
Expand Down Expand Up @@ -162,27 +214,17 @@ func main() {
}

// nodeNodesReconciler returns a reconciliation func for nodes
func makeNodesReconciler(cniConfig *CNIConfigWriter, hostIP string) func(*corev1.NodeList) error {
func makeNodesReconciler(cniConfig *CNIConfigWriter, hostIP string, ipFamily IPFamily) func(*corev1.NodeList) error {
// reconciles a node
reconcileNode := func(node corev1.Node) error {
// first get this node's IP
nodeIP := internalIP(node)
if nodeIP == "" {
klog.Infof("Node %v has no Internal IP, ignoring\n", node.Name)
return nil
}

// don't do anything unless there is a PodCIDR
podCIDR := node.Spec.PodCIDR
if podCIDR == "" {
klog.Infof("Node %v has no CIDR, ignoring\n", node.Name)
return nil
}

// This is our node. We don't need to add routes, but we might need to
// update the cni config.
if nodeIP == hostIP {
klog.Infof("handling current node\n")
// first get this node's IPs
// we don't support more than one IP address per IP family for simplification
nodeIPs := internalIPs(node)
klog.Infof("Handling node with IPs: %v\n", nodeIPs)
// This is our node. We don't need to add routes,
// but we might need to update the cni config
if nodeIPs.Has(hostIP) {
klog.Info("handling current node\n")
// compute the current cni config inputs
if err := cniConfig.Write(
ComputeCNIConfigInputs(node),
Expand All @@ -193,12 +235,41 @@ func makeNodesReconciler(cniConfig *CNIConfigWriter, hostIP string) func(*corev1
return nil
}

klog.Infof("Handling node with IP: %s\n", nodeIP)
klog.Infof("Node %v has CIDR %s \n", node.Name, podCIDR)
if err := syncRoute(nodeIP, podCIDR); err != nil {
return err
// This is another node. Add routes to the POD subnets in the other nodes
// don't do anything unless there is a PodCIDR
var podCIDRs []string
if ipFamily == DualStackFamily {
podCIDRs = node.Spec.PodCIDRs
} else {
podCIDRs = []string{node.Spec.PodCIDR}
}
if len(podCIDRs) == 0 {
fmt.Printf("Node %v has no CIDR, ignoring\n", node.Name)
return nil
}
klog.Infof("Node %v has CIDR %s \n", node.Name, podCIDRs)
podCIDRsv4, podCIDRsv6 := splitCIDRs(podCIDRs)

// obtain the PodCIDR gateway
var nodeIPv4, nodeIPv6 string
for _, ip := range nodeIPs.List() {
if isIPv6String(ip) {
nodeIPv6 = ip
} else {
nodeIPv4 = ip
}
}

if nodeIPv4 != "" && len(podCIDRsv4) > 0 {
if err := syncRoute(nodeIPv4, podCIDRsv4); err != nil {
return err
}
}
if nodeIPv6 != "" && len(podCIDRsv6) > 0 {
if err := syncRoute(nodeIPv6, podCIDRsv6); err != nil {
return err
}
}
return nil
}

Expand All @@ -213,14 +284,30 @@ func makeNodesReconciler(cniConfig *CNIConfigWriter, hostIP string) func(*corev1
}
}

// internalIP returns the internalIP address for node
func internalIP(node corev1.Node) string {
// internalIPs returns the internal IP addresses for node
func internalIPs(node corev1.Node) sets.String {
ips := sets.NewString()
// check the node.Status.Addresses
for _, address := range node.Status.Addresses {
if address.Type == "InternalIP" {
return address.Address
ips.Insert(address.Address)
}
}
return ""
return ips
}

// splitCIDRs given a slice of strings with CIDRs it returns 2 slice of strings per IP family
// The order returned is always v4 v6
func splitCIDRs(cidrs []string) ([]string, []string) {
var v4subnets, v6subnets []string
for _, subnet := range cidrs {
if isIPv6CIDRString(subnet) {
v6subnets = append(v6subnets, subnet)
} else {
v4subnets = append(v4subnets, subnet)
}
}
return v4subnets, v6subnets
}

// Modified from agnhost connect command in k/k
Expand Down Expand Up @@ -251,3 +338,16 @@ func probeTCP(address string, timeout time.Duration) bool {
klog.Warningf("OTHER %s: %v", address, err)
return false
}

// isIPv6String returns if ip is IPv6.
func isIPv6String(ip string) bool {
netIP := net.ParseIP(ip)
return netIP != nil && netIP.To4() == nil
}

// isIPv6CIDRString returns if cidr is IPv6.
// This assumes cidr is a valid CIDR.
func isIPv6CIDRString(cidr string) bool {
ip, _, _ := net.ParseCIDR(cidr)
return ip != nil && ip.To4() == nil
}
38 changes: 20 additions & 18 deletions images/kindnetd/cmd/kindnetd/routes.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,28 +24,30 @@ import (
"k8s.io/klog/v2"
)

func syncRoute(nodeIP, podCIDR string) error {
// parse subnet
dst, err := netlink.ParseIPNet(podCIDR)
if err != nil {
return err
}

// Check if the route exists to the other node's PodCIDR
func syncRoute(nodeIP string, podCIDRs []string) error {
ip := net.ParseIP(nodeIP)
routeToDst := netlink.Route{Dst: dst, Gw: ip}
route, err := netlink.RouteListFiltered(nl.GetIPFamily(ip), &routeToDst, netlink.RT_FILTER_DST)
if err != nil {
return err
}

// Add route if not present
if len(route) == 0 {
if err := netlink.RouteAdd(&routeToDst); err != nil {
for _, podCIDR := range podCIDRs {
// parse subnet
dst, err := netlink.ParseIPNet(podCIDR)
if err != nil {
return err
}
klog.Infof("Adding route %v \n", routeToDst)
}

// Check if the route exists to the other node's PodCIDR
routeToDst := netlink.Route{Dst: dst, Gw: ip}
route, err := netlink.RouteListFiltered(nl.GetIPFamily(ip), &routeToDst, netlink.RT_FILTER_DST)
if err != nil {
return err
}

// Add route if not present
if len(route) == 0 {
klog.Infof("Adding route %v \n", routeToDst)
if err := netlink.RouteAdd(&routeToDst); err != nil {
return err
}
}
}
return nil
}
Loading

0 comments on commit 5e58a0d

Please sign in to comment.