From 3dbeb894e3092a336ab4278d3823e73a1d66aff7 Mon Sep 17 00:00:00 2001 From: Benjamin Elder Date: Fri, 26 Jul 2019 00:07:34 -0700 Subject: [PATCH 1/4] Kindnetd with dual-stack support Kindnet supports dual-stack --- images/kindnetd/Dockerfile | 2 +- images/kindnetd/cmd/kindnetd/cni.go | 48 +++--- images/kindnetd/cmd/kindnetd/main.go | 178 +++++++++++++++++----- images/kindnetd/cmd/kindnetd/routes.go | 38 ++--- images/kindnetd/go.mod | 10 +- images/kindnetd/go.sum | 198 ++++++++++++++++++------- 6 files changed, 340 insertions(+), 134 deletions(-) diff --git a/images/kindnetd/Dockerfile b/images/kindnetd/Dockerfile index ee51c62ff3..295796f713 100644 --- a/images/kindnetd/Dockerfile +++ b/images/kindnetd/Dockerfile @@ -14,7 +14,7 @@ # first stage build kindnetd binary # NOTE: tentatively follow upstream kubernetes go version based on k8s in go.mod -FROM golang:1.13 +FROM golang:1.15 WORKDIR /go/src # make deps fetching cacheable COPY go.mod go.sum ./ diff --git a/images/kindnetd/cmd/kindnetd/cni.go b/images/kindnetd/cmd/kindnetd/cni.go index 990755cd2d..b19419f2e7 100644 --- a/images/kindnetd/cmd/kindnetd/cni.go +++ b/images/kindnetd/cmd/kindnetd/cni.go @@ -26,29 +26,39 @@ import ( "github.com/pkg/errors" corev1 "k8s.io/api/core/v1" - "k8s.io/utils/net" ) /* cni config management */ // CNIConfigInputs is supplied to the CNI config template type CNIConfigInputs struct { - PodCIDR string - DefaultRoute string - Mtu int + PodCIDRs []string + DefaultRoutes []string + Mtu int } // ComputeCNIConfigInputs computes the template inputs for CNIConfigWriter func ComputeCNIConfigInputs(node corev1.Node) CNIConfigInputs { - podCIDR := node.Spec.PodCIDR - defaultRoute := "0.0.0.0/0" - if net.IsIPv6CIDRString(podCIDR) { - defaultRoute = "::/0" + defaultRoutes := []string{"0.0.0.0/0", "::/0"} + // check if is a dualstack cluster + if len(node.Spec.PodCIDRs) > 1 { + return CNIConfigInputs{ + PodCIDRs: node.Spec.PodCIDRs, + DefaultRoutes: defaultRoutes, + } + } + // the cluster is single stack + // we use the legacy node.Spec.PodCIDR for backwards compatibility + podCIDRs := []string{node.Spec.PodCIDR} + // This is a single stack cluster + defaultRoute := defaultRoutes[:1] + if isIPv6CIDRString(podCIDRs[0]) { + defaultRoute = defaultRoutes[1:] } return CNIConfigInputs{ - PodCIDR: podCIDR, - DefaultRoute: defaultRoute, + PodCIDRs: podCIDRs, + DefaultRoutes: defaultRoute, } } @@ -82,17 +92,19 @@ const cniConfigTemplate = ` "type": "host-local", "dataDir": "/run/cni-ipam-state", "routes": [ - { - "dst": "{{ .DefaultRoute }}" - } + {{$first := true}} + {{- range $route := .DefaultRoutes}} + {{if $first}}{{$first = false}}{{else}},{{end}} + { "dst": "{{ $route }}" } + {{- end}} ], "ranges": [ - [ - { - "subnet": "{{ .PodCIDR }}" - } + {{$first := true}} + {{- range $cidr := .PodCIDRs}} + {{if $first}}{{$first = false}}{{else}},{{end}} + [ { "subnet": "{{ $cidr }}" } ] + {{- end}} ] - ] } {{if .Mtu}}, "mtu": {{ .Mtu }} diff --git a/images/kindnetd/cmd/kindnetd/main.go b/images/kindnetd/cmd/kindnetd/main.go index 2c5f5d7b6f..1038632492 100644 --- a/images/kindnetd/cmd/kindnetd/main.go +++ b/images/kindnetd/cmd/kindnetd/main.go @@ -22,15 +22,17 @@ import ( "fmt" "net" "os" + "strings" "syscall" "time" + "k8s.io/apimachinery/pkg/util/sets" + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" "k8s.io/klog/v2" - utilsnet "k8s.io/utils/net" ) const ( @@ -50,6 +52,18 @@ const ( // TODO: improve logging & error handling +// IPFamily defines kindnet networking operating model +type IPFamily string + +const ( + // IPv4Family sets IPFamily to ipv4 + IPv4Family IPFamily = "ipv4" + // IPv6Family sets IPFamily to ipv6 + IPv6Family IPFamily = "ipv6" + // DualStackFamily sets ClusterIPFamily to DualStack + DualStackFamily IPFamily = "dualstack" +) + func main() { // enable logging klog.InitFlags(nil) @@ -92,10 +106,10 @@ func main() { hostIP, podIP := os.Getenv("HOST_IP"), os.Getenv("POD_IP") klog.Infof("hostIP = %s\npodIP = %s\n", hostIP, podIP) if hostIP != podIP { - panic(fmt.Sprintf( + klog.Warningf( "hostIP(= %q) != podIP(= %q) but must be running with host network: ", hostIP, podIP, - )) + ) } mtu, err := computeBridgeMTU() @@ -110,20 +124,58 @@ func main() { } // enforce ip masquerade rules - // TODO: dual stack...? - masqAgent, err := NewIPMasqAgent(utilsnet.IsIPv6String(hostIP), []string{os.Getenv("POD_SUBNET")}) - if err != nil { - panic(err.Error()) + podSubnetEnv := os.Getenv("POD_SUBNET") + if podSubnetEnv == "" { + panic("missing environment variable POD_SUBNET") + } + podSubnetEnv = strings.TrimSpace(podSubnetEnv) + podSubnets := strings.Split(podSubnetEnv, ",") + clusterIPv4Subnets, clusterIPv6Subnets := splitCIDRs(podSubnets) + + // detect the cluster IP family based on the Cluster CIDR aka PodSubnet + var ipFamily IPFamily + if len(clusterIPv4Subnets) > 0 && len(clusterIPv6Subnets) > 0 { + ipFamily = DualStackFamily + } else if len(clusterIPv6Subnets) > 0 { + ipFamily = IPv6Family + } else if len(clusterIPv4Subnets) > 0 { + ipFamily = IPv4Family + } else { + panic(fmt.Sprintf("podSubnets ClusterCIDR/Pod_Subnet: %v", podSubnetEnv)) + } + klog.Infof("kindnetd IP family: %q", ipFamily) + + // create an ipMasqAgent for IPv4 + if len(clusterIPv4Subnets) > 0 { + klog.Infof("noMask IPv4 subnets: %v", clusterIPv4Subnets) + masqAgentIPv4, err := NewIPMasqAgent(false, clusterIPv4Subnets) + if err != nil { + panic(err.Error()) + } + go func() { + if err := masqAgentIPv4.SyncRulesForever(time.Second * 60); err != nil { + panic(err) + } + }() } - // run the masqAgent and panic if is not able to install the rules to no masquerade the pod to pod traffic - go func() { - if err := masqAgent.SyncRulesForever(time.Second * 60); err != nil { + + // create an ipMasqAgent for IPv6 + if len(clusterIPv6Subnets) > 0 { + klog.Infof("noMask IPv6 subnets: %v", clusterIPv6Subnets) + masqAgentIPv6, err := NewIPMasqAgent(true, clusterIPv6Subnets) + if err != nil { panic(err.Error()) } - }() + + go func() { + if err := masqAgentIPv6.SyncRulesForever(time.Second * 60); err != nil { + panic(err) + } + }() + } // setup nodes reconcile function, closes over arguments - reconcileNodes := makeNodesReconciler(cniConfigWriter, hostIP) + reconcileNodes := makeNodesReconciler(cniConfigWriter, hostIP, ipFamily) // main control loop for { @@ -162,27 +214,17 @@ func main() { } // nodeNodesReconciler returns a reconciliation func for nodes -func makeNodesReconciler(cniConfig *CNIConfigWriter, hostIP string) func(*corev1.NodeList) error { +func makeNodesReconciler(cniConfig *CNIConfigWriter, hostIP string, ipFamily IPFamily) func(*corev1.NodeList) error { // reconciles a node reconcileNode := func(node corev1.Node) error { - // first get this node's IP - nodeIP := internalIP(node) - if nodeIP == "" { - klog.Infof("Node %v has no Internal IP, ignoring\n", node.Name) - return nil - } - - // don't do anything unless there is a PodCIDR - podCIDR := node.Spec.PodCIDR - if podCIDR == "" { - klog.Infof("Node %v has no CIDR, ignoring\n", node.Name) - return nil - } - - // This is our node. We don't need to add routes, but we might need to - // update the cni config. - if nodeIP == hostIP { - klog.Infof("handling current node\n") + // first get this node's IPs + // we don't support more than one IP address per IP family for simplification + nodeIPs := internalIPs(node) + klog.Infof("Handling node with IPs: %v\n", nodeIPs) + // This is our node. We don't need to add routes, + // but we might need to update the cni config + if nodeIPs.Has(hostIP) { + klog.Info("handling current node\n") // compute the current cni config inputs if err := cniConfig.Write( ComputeCNIConfigInputs(node), @@ -193,12 +235,41 @@ func makeNodesReconciler(cniConfig *CNIConfigWriter, hostIP string) func(*corev1 return nil } - klog.Infof("Handling node with IP: %s\n", nodeIP) - klog.Infof("Node %v has CIDR %s \n", node.Name, podCIDR) - if err := syncRoute(nodeIP, podCIDR); err != nil { - return err + // This is another node. Add routes to the POD subnets in the other nodes + // don't do anything unless there is a PodCIDR + var podCIDRs []string + if ipFamily == DualStackFamily { + podCIDRs = node.Spec.PodCIDRs + } else { + podCIDRs = []string{node.Spec.PodCIDR} + } + if len(podCIDRs) == 0 { + fmt.Printf("Node %v has no CIDR, ignoring\n", node.Name) + return nil + } + klog.Infof("Node %v has CIDR %s \n", node.Name, podCIDRs) + podCIDRsv4, podCIDRsv6 := splitCIDRs(podCIDRs) + + // obtain the PodCIDR gateway + var nodeIPv4, nodeIPv6 string + for _, ip := range nodeIPs.List() { + if isIPv6String(ip) { + nodeIPv6 = ip + } else { + nodeIPv4 = ip + } } + if nodeIPv4 != "" && len(podCIDRsv4) > 0 { + if err := syncRoute(nodeIPv4, podCIDRsv4); err != nil { + return err + } + } + if nodeIPv6 != "" && len(podCIDRsv6) > 0 { + if err := syncRoute(nodeIPv6, podCIDRsv6); err != nil { + return err + } + } return nil } @@ -213,14 +284,30 @@ func makeNodesReconciler(cniConfig *CNIConfigWriter, hostIP string) func(*corev1 } } -// internalIP returns the internalIP address for node -func internalIP(node corev1.Node) string { +// internalIPs returns the internal IP addresses for node +func internalIPs(node corev1.Node) sets.String { + ips := sets.NewString() + // check the node.Status.Addresses for _, address := range node.Status.Addresses { if address.Type == "InternalIP" { - return address.Address + ips.Insert(address.Address) } } - return "" + return ips +} + +// splitCIDRs given a slice of strings with CIDRs it returns 2 slice of strings per IP family +// The order returned is always v4 v6 +func splitCIDRs(cidrs []string) ([]string, []string) { + var v4subnets, v6subnets []string + for _, subnet := range cidrs { + if isIPv6CIDRString(subnet) { + v6subnets = append(v6subnets, subnet) + } else { + v4subnets = append(v4subnets, subnet) + } + } + return v4subnets, v6subnets } // Modified from agnhost connect command in k/k @@ -251,3 +338,16 @@ func probeTCP(address string, timeout time.Duration) bool { klog.Warningf("OTHER %s: %v", address, err) return false } + +// isIPv6String returns if ip is IPv6. +func isIPv6String(ip string) bool { + netIP := net.ParseIP(ip) + return netIP != nil && netIP.To4() == nil +} + +// isIPv6CIDRString returns if cidr is IPv6. +// This assumes cidr is a valid CIDR. +func isIPv6CIDRString(cidr string) bool { + ip, _, _ := net.ParseCIDR(cidr) + return ip != nil && ip.To4() == nil +} diff --git a/images/kindnetd/cmd/kindnetd/routes.go b/images/kindnetd/cmd/kindnetd/routes.go index 7c7d3d64c9..39f628d161 100644 --- a/images/kindnetd/cmd/kindnetd/routes.go +++ b/images/kindnetd/cmd/kindnetd/routes.go @@ -24,28 +24,30 @@ import ( "k8s.io/klog/v2" ) -func syncRoute(nodeIP, podCIDR string) error { - // parse subnet - dst, err := netlink.ParseIPNet(podCIDR) - if err != nil { - return err - } - - // Check if the route exists to the other node's PodCIDR +func syncRoute(nodeIP string, podCIDRs []string) error { ip := net.ParseIP(nodeIP) - routeToDst := netlink.Route{Dst: dst, Gw: ip} - route, err := netlink.RouteListFiltered(nl.GetIPFamily(ip), &routeToDst, netlink.RT_FILTER_DST) - if err != nil { - return err - } - // Add route if not present - if len(route) == 0 { - if err := netlink.RouteAdd(&routeToDst); err != nil { + for _, podCIDR := range podCIDRs { + // parse subnet + dst, err := netlink.ParseIPNet(podCIDR) + if err != nil { return err } - klog.Infof("Adding route %v \n", routeToDst) - } + // Check if the route exists to the other node's PodCIDR + routeToDst := netlink.Route{Dst: dst, Gw: ip} + route, err := netlink.RouteListFiltered(nl.GetIPFamily(ip), &routeToDst, netlink.RT_FILTER_DST) + if err != nil { + return err + } + + // Add route if not present + if len(route) == 0 { + klog.Infof("Adding route %v \n", routeToDst) + if err := netlink.RouteAdd(&routeToDst); err != nil { + return err + } + } + } return nil } diff --git a/images/kindnetd/go.mod b/images/kindnetd/go.mod index eacaaec061..ccfc77ad33 100644 --- a/images/kindnetd/go.mod +++ b/images/kindnetd/go.mod @@ -7,10 +7,8 @@ require ( github.com/pkg/errors v0.9.1 github.com/vishvananda/netlink v1.1.0 github.com/vishvananda/netns v0.0.0-20200520041808-52d707b772fe // indirect - golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d // indirect - k8s.io/api v0.19.2 - k8s.io/apimachinery v0.19.2 - k8s.io/client-go v0.19.2 - k8s.io/klog/v2 v2.3.0 - k8s.io/utils v0.0.0-20200912215256-4140de9c8800 + k8s.io/api v0.20.5 + k8s.io/apimachinery v0.20.5 + k8s.io/client-go v0.20.5 + k8s.io/klog/v2 v2.4.0 ) diff --git a/images/kindnetd/go.sum b/images/kindnetd/go.sum index 69806d6077..753644236a 100644 --- a/images/kindnetd/go.sum +++ b/images/kindnetd/go.sum @@ -5,28 +5,37 @@ cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6A cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= -cloud.google.com/go v0.51.0/go.mod h1:hWtGJ6gnXH+KgDv+V0zFGDvpi07n3z8ZNj3T1RW0Gcw= +cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= +cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= +cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= +cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= +cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= +cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= +cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= +cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= +cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= +cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= +cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -github.com/Azure/go-autorest/autorest v0.9.0/go.mod h1:xyHB1BMZT0cuDHU7I0+g046+BFDTQ8rEZB0s4Yfa6bI= -github.com/Azure/go-autorest/autorest v0.9.6/go.mod h1:/FALq9T/kS7b5J5qsQ+RSTUdAmGFqi0vUdVNNx8q630= -github.com/Azure/go-autorest/autorest/adal v0.5.0/go.mod h1:8Z9fGy2MpX0PvDjB1pEgQTmVqjGhiHBW7RJJEciWzS0= -github.com/Azure/go-autorest/autorest/adal v0.8.2/go.mod h1:ZjhuQClTqx435SRJ2iMlOxPYt3d2C/T/7TiQCVZSn3Q= -github.com/Azure/go-autorest/autorest/date v0.1.0/go.mod h1:plvfp3oPSKwf2DNjlBjWF/7vwR+cUD/ELuzDCXwHUVA= -github.com/Azure/go-autorest/autorest/date v0.2.0/go.mod h1:vcORJHLJEh643/Ioh9+vPmf1Ij9AEBM5FuBIXLmIy0g= -github.com/Azure/go-autorest/autorest/mocks v0.1.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0= -github.com/Azure/go-autorest/autorest/mocks v0.2.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0= -github.com/Azure/go-autorest/autorest/mocks v0.3.0/go.mod h1:a8FDP3DYzQ4RYfVAxAN3SVSiiO77gL2j2ronKKP0syM= -github.com/Azure/go-autorest/logger v0.1.0/go.mod h1:oExouG+K6PryycPJfVSxi/koC6LSNgds39diKLz7Vrc= -github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk= +github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= +github.com/Azure/go-autorest/autorest v0.11.1/go.mod h1:JFgpikqFJ/MleTTxwepExTKnFUKKszPS8UavbQYUMuw= +github.com/Azure/go-autorest/autorest/adal v0.9.0/go.mod h1:/c022QCutn2P7uY+/oQWWNcK9YU+MH96NgK+jErpbcg= +github.com/Azure/go-autorest/autorest/adal v0.9.5/go.mod h1:B7KF7jKIeC9Mct5spmyCB/A8CG/sEz1vwIRGv/bbw7A= +github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74= +github.com/Azure/go-autorest/autorest/mocks v0.4.0/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= +github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= +github.com/Azure/go-autorest/logger v0.2.0/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= +github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= -github.com/PuerkitoBio/purell v1.0.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= -github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= +github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= +github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= +github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= @@ -45,44 +54,56 @@ github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= github.com/go-logr/logr v0.2.0 h1:QvGt2nLcHH0WK9orKa+ppBPAxREcH364nPUedEpK0TY= github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= -github.com/go-openapi/jsonpointer v0.0.0-20160704185906-46af16f9f7b1/go.mod h1:+35s3my2LFTysnkMfxsJBAMHj/DoqoB9knIWoYG/Vk0= -github.com/go-openapi/jsonreference v0.0.0-20160704190145-13c6e3589ad9/go.mod h1:W3Z9FmVs9qj+KR4zFKmDPGiLdk1D9Rlm7cyMvf57TTg= -github.com/go-openapi/spec v0.0.0-20160808142527-6aced65f8501/go.mod h1:J8+jY1nAiCcj+friV/PDoE1/3eeccG9LYBs0tYvLOWc= -github.com/go-openapi/swag v0.0.0-20160704191624-1d0bd113de87/go.mod h1:DXUve3Dpr1UfpPtxFw+EFuQ41HhCWZfha5jSVRG7C7I= +github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg= +github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= +github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc= +github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8= +github.com/go-openapi/spec v0.19.3/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8Lj9mJglo= +github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= +github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls= github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= +github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= -github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0= -github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.4.3 h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.2 h1:X2ev0eStA3AbceY54o37/0PQ/UWqKEiiO2dKL5OPaFM= +github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g= github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= @@ -90,8 +111,11 @@ github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXi github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/gnostic v0.4.1 h1:DLJCy1n/vrD4HPjOvYcT8aYQXpPIzoRZONaYwyycI+I= @@ -113,9 +137,12 @@ github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORN github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -141,9 +168,11 @@ github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzu github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/vishvananda/netlink v1.1.0 h1:1iyaYNBLmP6L0220aDnYQpo1QEV4t4hJ+xEEhhJH8j0= github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE= github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU= @@ -152,18 +181,25 @@ github.com/vishvananda/netns v0.0.0-20200520041808-52d707b772fe/go.mod h1:DD4vA1 go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0 h1:hb9wdF1z5waM+dSIICn1l0DkLVDT3hqhhQsDNUmHPRE= +golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= +golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= +golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= +golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -172,12 +208,17 @@ golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTk golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= +golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -188,16 +229,25 @@ golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200707034311-ab3426394381 h1:VXak5I6aEWmAXeQjA+QSZzlgNrpq9mjcfDemuexIKsU= -golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b h1:uwuIcX0g4Yl1NC5XAz37xsr2lTtcqevgzYNVt49waME= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6 h1:pE8b58s1HRDMi8RDc79m0HISf9D4TzseP40cEA6IGfs= golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d h1:TzXSXBo42m9gQenoE3b9BGiEpg5IG2JkU5FkPIawgtw= +golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -213,26 +263,37 @@ golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190606203320-7fc4e5ec1444/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200217220822-9197077df867/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200622214017-ed371f2e16b4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d h1:L/IKR6COd7ubZrs2oTnTi73IhgqJ71c9s80WsQnh0Es= -golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201112073958-5cba982894dd h1:5CtCZbICpIOFdgO940moixOPjc0178IU44m4EjOO5IY= +golang.org/x/sys v0.0.0-20201112073958-5cba982894dd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= -golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.4 h1:0YWbFKbhXG/wIiuHDSKpS0Iy7FSA+u45VtBMfQcFTTc= +golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20191024005414-555d28b269f0 h1:/5xXl8Y5W96D+TtHSlonuFqGHIWVuyCkGJLwGh9JJFs= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e h1:EHBhcS0mlXEAVwNyO2dLfjToGsyY4j24pTs2ScHnX7s= +golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20181011042414-1f849cf54d09/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= @@ -243,22 +304,42 @@ golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBn golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -273,7 +354,16 @@ google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRn google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= +google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= +google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= @@ -281,6 +371,7 @@ google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ij google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -289,8 +380,8 @@ google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzi google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.24.0 h1:UhZDfRO8JRQru4/+LlLE0BRKGF8L+PICnvYZmx/fEGA= -google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= +google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= @@ -304,29 +395,32 @@ gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= -k8s.io/api v0.19.2 h1:q+/krnHWKsL7OBZg/rxnycsl9569Pud76UJ77MvKXms= -k8s.io/api v0.19.2/go.mod h1:IQpK0zFQ1xc5iNIQPqzgoOwuFugaYHK4iCknlAQP9nI= -k8s.io/apimachinery v0.19.2 h1:5Gy9vQpAGTKHPVOh5c4plE274X8D/6cuEiTO2zve7tc= -k8s.io/apimachinery v0.19.2/go.mod h1:DnPGDnARWFvYa3pMHgSxtbZb7gpzzAZ1pTfaUNDVlmA= -k8s.io/client-go v0.19.2 h1:gMJuU3xJZs86L1oQ99R4EViAADUPMHHtS9jFshasHSc= -k8s.io/client-go v0.19.2/go.mod h1:S5wPhCqyDNAlzM9CnEdgTGV4OqhsW3jGO1UM1epwfJA= +honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +k8s.io/api v0.20.5 h1:zsMTffV0Le2EiI0aKvlTHEnXGxk1HiqGRhJcCPiI7JI= +k8s.io/api v0.20.5/go.mod h1:FQjAceXnVaWDeov2YUWhOb6Yt+5UjErkp6UO3nczO1Y= +k8s.io/apimachinery v0.20.5 h1:wO/FxMVRn223rAKxnBbwCyuN96bS9MFTIvP0e/V7cps= +k8s.io/apimachinery v0.20.5/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRpU= +k8s.io/client-go v0.20.5 h1:dJGtYUvFrFGjQ+GjXEIby0gZWdlAOc0xJBJqY3VyDxA= +k8s.io/client-go v0.20.5/go.mod h1:Ee5OOMMYvlH8FCZhDsacjMlCBwetbGZETwo1OA+e6Zw= k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= -k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= -k8s.io/klog/v2 v2.3.0 h1:WmkrnW7fdrm0/DMClc+HIxtftvxVIPAhlVwMQo5yLco= -k8s.io/klog/v2 v2.3.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= -k8s.io/kube-openapi v0.0.0-20200805222855-6aeccd4b50c6/go.mod h1:UuqjUnNftUyPE5H64/qeyjQoUZhGpeFDVdxjTeEVN2o= -k8s.io/utils v0.0.0-20200729134348-d5654de09c73/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= -k8s.io/utils v0.0.0-20200912215256-4140de9c8800 h1:9ZNvfPvVIEsp/T1ez4GQuzCcCTEQWhovSofhqR73A6g= -k8s.io/utils v0.0.0-20200912215256-4140de9c8800/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= +k8s.io/klog/v2 v2.4.0 h1:7+X0fUguPyrKEC4WjH8iGDg3laWgMo5tMnRTIGTTxGQ= +k8s.io/klog/v2 v2.4.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= +k8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd/go.mod h1:WOJ3KddDSol4tAGcJo0Tvi+dK12EcqSLqcWsryKMpfM= +k8s.io/utils v0.0.0-20201110183641-67b214c5f920 h1:CbnUZsM497iRC5QMVkHwyl8s2tB3g7yaSHkYPkpgelw= +k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= -sigs.k8s.io/structured-merge-diff/v4 v4.0.1 h1:YXTMot5Qz/X1iBRJhAt+vI+HVttY0WkSqqhKxQ0xVbA= -sigs.k8s.io/structured-merge-diff/v4 v4.0.1/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= +rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= +rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= +sigs.k8s.io/structured-merge-diff/v4 v4.0.2 h1:YHQV7Dajm86OuqnIR6zAelnDWBRjo+YhYV9PmGrh1s8= +sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q= sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= From 5657682609bc9ae52c1adf7164fa05d394d8f9ca Mon Sep 17 00:00:00 2001 From: Antonio Ojea Date: Fri, 26 Mar 2021 10:41:41 +0100 Subject: [PATCH 2/4] Add dual stack support: APIs Enable ipv6 on nodes if dual-stack --- pkg/apis/config/v1alpha4/default.go | 14 +++- pkg/apis/config/v1alpha4/types.go | 2 + .../internal/create/actions/config/config.go | 7 +- pkg/cluster/internal/kubeadm/config.go | 15 ++-- .../internal/providers/docker/provision.go | 5 +- .../internal/providers/podman/provision.go | 5 +- pkg/internal/apis/config/default.go | 14 +++- pkg/internal/apis/config/types.go | 2 + pkg/internal/apis/config/validate.go | 73 ++++++++++++++++- pkg/internal/apis/config/validate_test.go | 81 +++++++++++++++++++ 10 files changed, 195 insertions(+), 23 deletions(-) diff --git a/pkg/apis/config/v1alpha4/default.go b/pkg/apis/config/v1alpha4/default.go index 3bd00f0be5..4626fdd41b 100644 --- a/pkg/apis/config/v1alpha4/default.go +++ b/pkg/apis/config/v1alpha4/default.go @@ -37,24 +37,27 @@ func SetDefaultsCluster(obj *Cluster) { SetDefaultsNode(a) } if obj.Networking.IPFamily == "" { - obj.Networking.IPFamily = "ipv4" + obj.Networking.IPFamily = IPv4Family } // default to listening on 127.0.0.1:randomPort on ipv4 // and [::1]:randomPort on ipv6 if obj.Networking.APIServerAddress == "" { obj.Networking.APIServerAddress = "127.0.0.1" - if obj.Networking.IPFamily == "ipv6" { + if obj.Networking.IPFamily == IPv6Family { obj.Networking.APIServerAddress = "::1" } } // default the pod CIDR if obj.Networking.PodSubnet == "" { obj.Networking.PodSubnet = "10.244.0.0/16" - if obj.Networking.IPFamily == "ipv6" { + if obj.Networking.IPFamily == IPv6Family { // node-mask cidr default is /64 so we need a larger subnet, we use /56 following best practices // xref: https://www.ripe.net/publications/docs/ripe-690#4--size-of-end-user-prefix-assignment---48---56-or-something-else- obj.Networking.PodSubnet = "fd00:10:244::/56" } + if obj.Networking.IPFamily == DualStackFamily { + obj.Networking.PodSubnet = "10.244.0.0/16,fd00:10:244::/56" + } } // default the service CIDR using a different subnet than kubeadm default // https://github.com/kubernetes/kubernetes/blob/746404f82a28e55e0b76ffa7e40306fb88eb3317/cmd/kubeadm/app/apis/kubeadm/v1beta2/defaults.go#L32 @@ -62,9 +65,12 @@ func SetDefaultsCluster(obj *Cluster) { // we allocate a /16 subnet that allows 65535 services (current Kubernetes tested limit is O(10k) services) if obj.Networking.ServiceSubnet == "" { obj.Networking.ServiceSubnet = "10.96.0.0/16" - if obj.Networking.IPFamily == "ipv6" { + if obj.Networking.IPFamily == IPv6Family { obj.Networking.ServiceSubnet = "fd00:10:96::/112" } + if obj.Networking.IPFamily == DualStackFamily { + obj.Networking.ServiceSubnet = "10.96.0.0/16,fd00:10:96::/112" + } } // default the KubeProxyMode using iptables as it's already the default if obj.Networking.KubeProxyMode == "" { diff --git a/pkg/apis/config/v1alpha4/types.go b/pkg/apis/config/v1alpha4/types.go index 1916b56250..eef673673f 100644 --- a/pkg/apis/config/v1alpha4/types.go +++ b/pkg/apis/config/v1alpha4/types.go @@ -199,6 +199,8 @@ const ( IPv4Family ClusterIPFamily = "ipv4" // IPv6Family sets ClusterIPFamily to ipv6 IPv6Family ClusterIPFamily = "ipv6" + // DualStackFamily sets ClusterIPFamily to dual + DualStackFamily ClusterIPFamily = "dual" ) // ProxyMode defines a proxy mode for kube-proxy diff --git a/pkg/cluster/internal/create/actions/config/config.go b/pkg/cluster/internal/create/actions/config/config.go index 64beac53cc..a5ab77b200 100644 --- a/pkg/cluster/internal/create/actions/config/config.go +++ b/pkg/cluster/internal/create/actions/config/config.go @@ -78,7 +78,7 @@ func (a *Action) Execute(ctx *actions.ActionContext) error { KubeProxyMode: string(ctx.Config.Networking.KubeProxyMode), ServiceSubnet: ctx.Config.Networking.ServiceSubnet, ControlPlane: true, - IPv6: ctx.Config.Networking.IPFamily == "ipv6", + IPv6: ctx.Config.Networking.IPFamily == config.IPv6Family, FeatureGates: ctx.Config.FeatureGates, RuntimeConfig: ctx.Config.RuntimeConfig, RootlessProvider: providerInfo.Rootless, @@ -237,11 +237,14 @@ func getKubeadmConfig(cfg *config.Cluster, data kubeadm.ConfigData, node nodes.N data.NodeAddress = nodeAddress // configure the right protocol addresses - if cfg.Networking.IPFamily == "ipv6" { + if cfg.Networking.IPFamily == config.IPv6Family || cfg.Networking.IPFamily == config.DualStackFamily { if ip := net.ParseIP(nodeAddressIPv6); ip.To16() == nil { return "", errors.Errorf("failed to get IPv6 address for node %s; is %s configured to use IPv6 correctly?", node.String(), provider) } data.NodeAddress = nodeAddressIPv6 + if cfg.Networking.IPFamily == config.DualStackFamily { + data.NodeAddress = fmt.Sprintf("%s,%s", nodeAddress, nodeAddressIPv6) + } } // generate the config contents diff --git a/pkg/cluster/internal/kubeadm/config.go b/pkg/cluster/internal/kubeadm/config.go index 82f97e8556..80c22586b5 100644 --- a/pkg/cluster/internal/kubeadm/config.go +++ b/pkg/cluster/internal/kubeadm/config.go @@ -46,7 +46,7 @@ type ConfigData struct { // ControlPlane flag specifies the node belongs to the control plane ControlPlane bool - // The main IP address of the node + // The IP address or comma separated list IP addresses of of the node NodeAddress string // The name for the node (not the address) NodeName string @@ -86,6 +86,8 @@ type ConfigData struct { // DerivedConfigData fields are automatically derived by // ConfigData.Derive if they are not specified / zero valued type DerivedConfigData struct { + // AdvertiseAddress is the first address in NodeAddress + AdvertiseAddress string // DockerStableTag is automatically derived from KubernetesVersion DockerStableTag string // SortedFeatureGateKeys allows us to iterate FeatureGates deterministically @@ -98,6 +100,9 @@ type DerivedConfigData struct { // Derive automatically derives DockerStableTag if not specified func (c *ConfigData) Derive() { + // get the first address to use it as the API advertised address + c.AdvertiseAddress = strings.Split(c.NodeAddress, ",")[0] + if c.DockerStableTag == "" { c.DockerStableTag = strings.Replace(c.KubernetesVersion, "+", "_", -1) } @@ -194,7 +199,7 @@ bootstrapTokens: # we use a well know port for making the API server discoverable inside docker network. # from the host machine such port will be accessible via a random local port instead. localAPIEndpoint: - advertiseAddress: "{{ .NodeAddress }}" + advertiseAddress: "{{ .AdvertiseAddress }}" bindPort: {{.APIBindPort}} nodeRegistration: criSocket: "/run/containerd/containerd.sock" @@ -211,7 +216,7 @@ metadata: {{ if .ControlPlane -}} controlPlane: localAPIEndpoint: - advertiseAddress: "{{ .NodeAddress }}" + advertiseAddress: "{{ .AdvertiseAddress }}" bindPort: {{.APIBindPort}} {{- end }} nodeRegistration: @@ -321,7 +326,7 @@ bootstrapTokens: # we use a well know port for making the API server discoverable inside docker network. # from the host machine such port will be accessible via a random local port instead. localAPIEndpoint: - advertiseAddress: "{{ .NodeAddress }}" + advertiseAddress: "{{ .AdvertiseAddress }}" bindPort: {{.APIBindPort}} nodeRegistration: criSocket: "unix:///run/containerd/containerd.sock" @@ -339,7 +344,7 @@ metadata: {{ if .ControlPlane -}} controlPlane: localAPIEndpoint: - advertiseAddress: "{{ .NodeAddress }}" + advertiseAddress: "{{ .AdvertiseAddress }}" bindPort: {{.APIBindPort}} {{- end }} nodeRegistration: diff --git a/pkg/cluster/internal/providers/docker/provision.go b/pkg/cluster/internal/providers/docker/provision.go index 3502cd0946..501618615e 100644 --- a/pkg/cluster/internal/providers/docker/provision.go +++ b/pkg/cluster/internal/providers/docker/provision.go @@ -63,7 +63,8 @@ func planCreation(cfg *config.Cluster, networkName string) (createContainerFuncs // For now remote docker + multi control plane is not supported apiServerPort = 0 // replaced with random ports apiServerAddress = "127.0.0.1" // only the LB needs to be non-local - if clusterIsIPv6(cfg) { + // only for IPv6 only clusters + if cfg.Networking.IPFamily == config.IPv6Family { apiServerAddress = "::1" // only the LB needs to be non-local } // plan loadbalancer node @@ -134,7 +135,7 @@ func createContainer(args []string) error { } func clusterIsIPv6(cfg *config.Cluster) bool { - return cfg.Networking.IPFamily == "ipv6" + return cfg.Networking.IPFamily == config.IPv6Family || cfg.Networking.IPFamily == config.DualStackFamily } func clusterHasImplicitLoadBalancer(cfg *config.Cluster) bool { diff --git a/pkg/cluster/internal/providers/podman/provision.go b/pkg/cluster/internal/providers/podman/provision.go index 97b9413a83..fc04fd92c2 100644 --- a/pkg/cluster/internal/providers/podman/provision.go +++ b/pkg/cluster/internal/providers/podman/provision.go @@ -51,7 +51,8 @@ func planCreation(cfg *config.Cluster, networkName string) (createContainerFuncs // For now remote podman + multi control plane is not supported apiServerPort = 0 // replaced with random ports apiServerAddress = "127.0.0.1" // only the LB needs to be non-local - if clusterIsIPv6(cfg) { + // only for IPv6 only clusters + if cfg.Networking.IPFamily == config.IPv6Family { apiServerAddress = "::1" // only the LB needs to be non-local } // plan loadbalancer node @@ -120,7 +121,7 @@ func createContainer(args []string) error { } func clusterIsIPv6(cfg *config.Cluster) bool { - return cfg.Networking.IPFamily == "ipv6" + return cfg.Networking.IPFamily == config.IPv6Family || cfg.Networking.IPFamily == config.DualStackFamily } func clusterHasImplicitLoadBalancer(cfg *config.Cluster) bool { diff --git a/pkg/internal/apis/config/default.go b/pkg/internal/apis/config/default.go index ad24ce4895..7a93df8022 100644 --- a/pkg/internal/apis/config/default.go +++ b/pkg/internal/apis/config/default.go @@ -48,14 +48,14 @@ func SetDefaultsCluster(obj *Cluster) { SetDefaultsNode(a) } if obj.Networking.IPFamily == "" { - obj.Networking.IPFamily = "ipv4" + obj.Networking.IPFamily = IPv4Family } // default to listening on 127.0.0.1:randomPort on ipv4 // and [::1]:randomPort on ipv6 if obj.Networking.APIServerAddress == "" { obj.Networking.APIServerAddress = "127.0.0.1" - if obj.Networking.IPFamily == "ipv6" { + if obj.Networking.IPFamily == IPv6Family { obj.Networking.APIServerAddress = "::1" } } @@ -63,11 +63,14 @@ func SetDefaultsCluster(obj *Cluster) { // default the pod CIDR if obj.Networking.PodSubnet == "" { obj.Networking.PodSubnet = "10.244.0.0/16" - if obj.Networking.IPFamily == "ipv6" { + if obj.Networking.IPFamily == IPv6Family { // node-mask cidr default is /64 so we need a larger subnet, we use /56 following best practices // xref: https://www.ripe.net/publications/docs/ripe-690#4--size-of-end-user-prefix-assignment---48---56-or-something-else- obj.Networking.PodSubnet = "fd00:10:244::/56" } + if obj.Networking.IPFamily == DualStackFamily { + obj.Networking.PodSubnet = "10.244.0.0/16,fd00:10:244::/56" + } } // default the service CIDR using the kubeadm default @@ -76,9 +79,12 @@ func SetDefaultsCluster(obj *Cluster) { // we allocate a /16 subnet that allows 65535 services (current Kubernetes tested limit is O(10k) services) if obj.Networking.ServiceSubnet == "" { obj.Networking.ServiceSubnet = "10.96.0.0/16" - if obj.Networking.IPFamily == "ipv6" { + if obj.Networking.IPFamily == IPv6Family { obj.Networking.ServiceSubnet = "fd00:10:96::/112" } + if obj.Networking.IPFamily == DualStackFamily { + obj.Networking.ServiceSubnet = "10.96.0.0/16,fd00:10:96::/112" + } } // default the KubeProxyMode using iptables as it's already the default if obj.Networking.KubeProxyMode == "" { diff --git a/pkg/internal/apis/config/types.go b/pkg/internal/apis/config/types.go index 7c9786b1ab..36ef76eacd 100644 --- a/pkg/internal/apis/config/types.go +++ b/pkg/internal/apis/config/types.go @@ -160,6 +160,8 @@ const ( IPv4Family ClusterIPFamily = "ipv4" // IPv6Family sets ClusterIPFamily to ipv6 IPv6Family ClusterIPFamily = "ipv6" + // DualStackFamily sets ClusterIPFamily to dual + DualStackFamily ClusterIPFamily = "dual" ) // ProxyMode defines a proxy mode for kube-proxy diff --git a/pkg/internal/apis/config/validate.go b/pkg/internal/apis/config/validate.go index ed9c4e2c84..59c8cf5cd4 100644 --- a/pkg/internal/apis/config/validate.go +++ b/pkg/internal/apis/config/validate.go @@ -17,8 +17,10 @@ limitations under the License. package config import ( + "fmt" "net" "regexp" + "strings" "sigs.k8s.io/kind/pkg/errors" ) @@ -49,13 +51,15 @@ func (c *Cluster) Validate() error { } } + isDualStack := c.Networking.IPFamily == DualStackFamily // podSubnet should be a valid CIDR - if _, _, err := net.ParseCIDR(c.Networking.PodSubnet); err != nil { - errs = append(errs, errors.Wrapf(err, "invalid podSubnet")) + if err := validateSubnets(c.Networking.PodSubnet, isDualStack); err != nil { + errs = append(errs, errors.Errorf("invalid pod subnet %v", err)) } + // serviceSubnet should be a valid CIDR - if _, _, err := net.ParseCIDR(c.Networking.ServiceSubnet); err != nil { - errs = append(errs, errors.Wrapf(err, "invalid serviceSubnet")) + if err := validateSubnets(c.Networking.ServiceSubnet, isDualStack); err != nil { + errs = append(errs, errors.Errorf("invalid service subnet %v", err)) } // KubeProxyMode should be iptables or ipvs @@ -135,3 +139,64 @@ func validatePort(port int32) error { } return nil } + +func validateSubnets(subnetStr string, dualstack bool) error { + allErrs := []error{} + + cidrsString := strings.Split(subnetStr, ",") + subnets := make([]*net.IPNet, 0, len(cidrsString)) + for _, cidrString := range cidrsString { + _, cidr, err := net.ParseCIDR(cidrString) + if err != nil { + return fmt.Errorf("failed to parse cidr value:%q with error: %v", cidrString, err) + } + subnets = append(subnets, cidr) + } + + switch { + // if DualStack only 2 CIDRs allowed + case dualstack && len(subnets) > 2: + allErrs = append(allErrs, errors.New("expected one (IPv4 or IPv6) CIDR or two CIDRs from each family for dual-stack networking")) + // if DualStack and there are 2 CIDRs validate if there is at least one of each IP family + case dualstack && len(subnets) == 2: + areDualStackCIDRs, err := isDualStackCIDRs(subnets) + if err != nil { + allErrs = append(allErrs, err) + } else if !areDualStackCIDRs { + allErrs = append(allErrs, errors.New("expected one (IPv4 or IPv6) CIDR or two CIDRs from each family for dual-stack networking")) + } + // if not DualStack only one CIDR allowed + case !dualstack && len(subnets) > 1: + allErrs = append(allErrs, errors.New("only one CIDR allowed for single-stack networking")) + } + + if len(allErrs) > 0 { + return errors.NewAggregate(allErrs) + } + return nil +} + +// isDualStackCIDRs returns if +// - all are valid cidrs +// - at least one cidr from each family (v4 or v6) +func isDualStackCIDRs(cidrs []*net.IPNet) (bool, error) { + v4Found := false + v6Found := false + for _, cidr := range cidrs { + if cidr == nil { + return false, fmt.Errorf("cidr %v is invalid", cidr) + } + + if v4Found && v6Found { + continue + } + + if cidr.IP != nil && cidr.IP.To4() == nil { + v6Found = true + continue + } + v4Found = true + } + + return v4Found && v6Found, nil +} diff --git a/pkg/internal/apis/config/validate_test.go b/pkg/internal/apis/config/validate_test.go index 5e46aa7397..9f69331f59 100644 --- a/pkg/internal/apis/config/validate_test.go +++ b/pkg/internal/apis/config/validate_test.go @@ -95,6 +95,87 @@ func TestClusterValidate(t *testing.T) { }(), ExpectErrors: 1, }, + { + Name: "bogus serviceSubnet", + Cluster: func() Cluster { + c := Cluster{} + SetDefaultsCluster(&c) + c.Networking.ServiceSubnet = "aa" + return c + }(), + ExpectErrors: 1, + }, + { + Name: "invalid number of podSubnet", + Cluster: func() Cluster { + c := Cluster{} + SetDefaultsCluster(&c) + c.Networking.PodSubnet = "192.168.0.2/24,2.2.2.0/24" + return c + }(), + ExpectErrors: 1, + }, + { + Name: "valid dual stack podSubnet and serviceSubnet", + Cluster: func() Cluster { + c := Cluster{} + SetDefaultsCluster(&c) + c.Networking.PodSubnet = "192.168.0.2/24,fd00:1::/25" + c.Networking.ServiceSubnet = "192.168.0.2/24,fd00:1::/25" + c.Networking.IPFamily = DualStackFamily + return c + }(), + ExpectErrors: 0, + }, + { + Name: "invalid dual stack podSubnet and multiple serviceSubnet", + Cluster: func() Cluster { + c := Cluster{} + SetDefaultsCluster(&c) + c.Networking.PodSubnet = "192.168.0.2/24,fd00:1::/25" + c.Networking.ServiceSubnet = "192.168.0.2/24,fd00:1::/25,10.0.0.0/16" + c.Networking.IPFamily = DualStackFamily + return c + }(), + ExpectErrors: 1, + }, + { + Name: "valid dual stack podSubnet and single stack serviceSubnet", + Cluster: func() Cluster { + c := Cluster{} + SetDefaultsCluster(&c) + c.Networking.PodSubnet = "192.168.0.2/24,fd00:1::/25" + c.Networking.ServiceSubnet = "192.168.0.2/24" + c.Networking.IPFamily = DualStackFamily + return c + }(), + ExpectErrors: 0, + }, + { + Name: "valid dual stack serviceSubnet and single stack podSubnet", + Cluster: func() Cluster { + c := Cluster{} + SetDefaultsCluster(&c) + c.Networking.PodSubnet = "192.168.0.2/24" + c.Networking.ServiceSubnet = "192.168.0.2/24,fd00:1::/25" + c.Networking.IPFamily = DualStackFamily + return c + }(), + ExpectErrors: 0, + }, + + { + Name: "bad dual stack podSubnet and serviceSubnet", + Cluster: func() Cluster { + c := Cluster{} + SetDefaultsCluster(&c) + c.Networking.PodSubnet = "192.168.0.2/24,2.2.2.0/25" + c.Networking.ServiceSubnet = "192.168.0.2/24,2.2.2.0/25" + c.Networking.IPFamily = DualStackFamily + return c + }(), + ExpectErrors: 2, + }, { Name: "missing control-plane", Cluster: func() Cluster { From ec617af085e2dc363d2c43bba4a8341de10fa44e Mon Sep 17 00:00:00 2001 From: Antonio Ojea Date: Mon, 19 Aug 2019 18:58:24 +0200 Subject: [PATCH 3/4] Add dual stack support: Docs Fix dual docs --- site/content/docs/user/quick-start.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/site/content/docs/user/quick-start.md b/site/content/docs/user/quick-start.md index 016c179ed6..8f96601a78 100644 --- a/site/content/docs/user/quick-start.md +++ b/site/content/docs/user/quick-start.md @@ -370,6 +370,22 @@ networking: ipFamily: ipv6 ``` +#### Dual Stack clusters +You can run dual stack clusters using `kind`, on kubernetes versions +1.21 +but first you need to [enable ipv6 in your docker daemon][docker enable ipv6]. + +```yaml +# a dual-stack cluster +kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 +networking: + ipFamily: dual +nodes: +- role: control-plane +- role: worker +- role: worker +``` + ### Configure kind to use a proxy If you are running kind in an environment that requires a proxy, you may need to configure kind to use it. From 04bdfc73c42f96f4647269422bdb5c3ee742c87b Mon Sep 17 00:00:00 2001 From: Antonio Ojea Date: Fri, 26 Mar 2021 11:17:08 +0100 Subject: [PATCH 4/4] use kindnetd dualstack --- pkg/build/nodeimage/const_cni.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/build/nodeimage/const_cni.go b/pkg/build/nodeimage/const_cni.go index 10803cc166..dbd927e0ec 100644 --- a/pkg/build/nodeimage/const_cni.go +++ b/pkg/build/nodeimage/const_cni.go @@ -20,7 +20,7 @@ package nodeimage The default CNI manifest and images are our own tiny kindnet */ -var defaultCNIImages = []string{"kindest/kindnetd:v20210220-5b7e6d01"} +var defaultCNIImages = []string{"kindest/kindnetd:v20210326-1e038dc5"} // TODO: migrate to fully patching and deprecate the template const defaultCNIManifest = ` @@ -95,7 +95,7 @@ spec: serviceAccountName: kindnet containers: - name: kindnet-cni - image: kindest/kindnetd:v20210220-5b7e6d01 + image: kindest/kindnetd:v20210326-1e038dc5 env: - name: HOST_IP valueFrom: