Merge pull request kubernetes#100722 from cici37/automated-cherry-pic…

…k-of-#98210-upstream-release-1.20

Automated cherry pick of kubernetes#98210 upstream release 1.20

cmd/cloud-controller-manager/.import-restrictions

@@ -1,6 +1,8 @@
 rules:
   - selectorRegexp: k8s[.]io/kubernetes
     allowedPrefixes:
+      - k8s.io/kubernetes/cmd/kube-controller-manager/app/options
+      - k8s.io/kubernetes/cmd/kube-controller-manager/app/config
       - k8s.io/kubernetes/pkg/api/legacyscheme
       - k8s.io/kubernetes/pkg/api/service
       - k8s.io/kubernetes/pkg/api/v1/pod
@@ -34,4 +36,7 @@ rules:
       - k8s.io/kubernetes/pkg/util/hash
       - k8s.io/kubernetes/pkg/util/node
       - k8s.io/kubernetes/pkg/util/parsers
-      - k8s.io/kubernetes/pkg/util/taints
+      - k8s.io/kubernetes/pkg/util/taints
+      - k8s.io/kubernetes/pkg/proxy/util
+      - k8s.io/kubernetes/pkg/proxy/util/testing
+      - k8s.io/kubernetes/pkg/util/sysctl

cmd/cloud-controller-manager/BUILD

@@ -23,9 +23,11 @@ go_library(
     ],
     importpath = "k8s.io/kubernetes/cmd/cloud-controller-manager",
     deps = [
+        "//cmd/kube-controller-manager/app/options:go_default_library",
         "//pkg/controller/nodeipam:go_default_library",
         "//pkg/controller/nodeipam/config:go_default_library",
         "//pkg/controller/nodeipam/ipam:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library",
         "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library",
         "//staging/src/k8s.io/cloud-provider:go_default_library",
         "//staging/src/k8s.io/cloud-provider/app:go_default_library",

cmd/cloud-controller-manager/README.md

@@ -0,0 +1,14 @@
+# cloud-controller-manager/example
+
+This directory provides an example of how to leverage CCM extension mechanism.
+
+## Purpose
+
+Begin with 1.20, all cloud providers should not copy over or vendor in `k8s.io/kubernetes/cmd/cloud-controller-manager`. Inside this directory, an example is included to demonstrate how to leverage CCM extension mechanism to add a controller.
+Please refer to `k8s.io/cloud-provider/sample` if you do not have the requirement of adding/deleting controllers in CCM.
+
+## Things you should NOT do
+
+1. Vendor in `k8s.io/cmd/cloud-controller-manager`.
+2. Directly modify anything under `k8s.io/cmd/cloud-controller-manager` in this repo. 
+3. Make specific cloud provider changes here.

cmd/cloud-controller-manager/main.go

@@ -19,128 +19,89 @@ limitations under the License.
 
 // This file should be written by each cloud provider.
 // For an minimal working example, please refer to k8s.io/cloud-provider/sample/basic_main.go
-// For an advanced example, please refer to k8s.io/cloud-provider/sample/advanced_main.go
 // For more details, please refer to k8s.io/kubernetes/cmd/cloud-controller-manager/main.go
 // The current file demonstrate how other cloud provider should leverage CCM and it uses fake parameters. Please modify for your own use.
 
 package main
 
 import (
-	"fmt"
 	"math/rand"
-	"net/http"
 	"os"
 	"time"
 
 	"github.com/spf13/pflag"
-	cloudprovider "k8s.io/cloud-provider"
+
+	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/cloud-provider"
 	"k8s.io/cloud-provider/app"
 	cloudcontrollerconfig "k8s.io/cloud-provider/app/config"
 	"k8s.io/cloud-provider/options"
-	"k8s.io/component-base/cli/flag"
+	cliflag "k8s.io/component-base/cli/flag"
 	"k8s.io/component-base/logs"
 	_ "k8s.io/component-base/metrics/prometheus/clientgo" // load all the prometheus client-go plugins
 	_ "k8s.io/component-base/metrics/prometheus/version"  // for version metric registration
-	genericcontrollermanager "k8s.io/controller-manager/app"
 	"k8s.io/klog/v2"
-	nodeipamconfig "k8s.io/kubernetes/pkg/controller/nodeipam/config"
 	// For existing cloud providers, the option to import legacy providers is still available.
 	// e.g. _"k8s.io/legacy-cloud-providers/<provider>"
 )
 
-const (
-	// cloudProviderName shows an sample of using hard coded parameter, please edit the value for your case.
-	cloudProviderName = "SampleCloudProviderName"
-)
-
 func main() {
 	rand.Seed(time.Now().UnixNano())
 
-	// cloudProviderConfigFile shows an sample of parse config file from flag option
-	var flagset *pflag.FlagSet = pflag.NewFlagSet("flagSet", pflag.ContinueOnError)
-	var cloudProviderConfigFile *string = flagset.String("cloud-provider-configfile", "", "This is the sample input for cloud provider config file")
-	pflag.CommandLine.ParseErrorsWhitelist.UnknownFlags = true
-	_ = pflag.CommandLine.Parse(os.Args[1:])
-
-	// this is an example of allow-listing specific controller loops
-	controllerList := []string{"cloud-node", "cloud-node-lifecycle", "service", "route"}
-
-	s, err := options.NewCloudControllerManagerOptions()
+	ccmOptions, err := options.NewCloudControllerManagerOptions()
 	if err != nil {
 		klog.Fatalf("unable to initialize command options: %v", err)
 	}
-	c, err := s.Config(controllerList, app.ControllersDisabledByDefault.List())
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "%v\n", err)
-		os.Exit(1)
-	}
-
-	cloud, err := cloudprovider.InitCloudProvider(cloudProviderName, *cloudProviderConfigFile)
-	if err != nil {
-		klog.Fatalf("Cloud provider could not be initialized: %v", err)
-	}
-	if cloud == nil {
-		klog.Fatalf("cloud provider is nil")
-	}
-
-	if !cloud.HasClusterID() {
-		if c.ComponentConfig.KubeCloudShared.AllowUntaggedCloud {
-			klog.Warning("detected a cluster without a ClusterID.  A ClusterID will be required in the future.  Please tag your cluster to avoid any future issues")
-		} else {
-			klog.Fatalf("no ClusterID found.  A ClusterID is required for the cloud provider to function properly.  This check can be bypassed by setting the allow-untagged-cloud option")
-		}
-	}
-
-	// Initialize the cloud provider with a reference to the clientBuilder
-	cloud.Initialize(c.ClientBuilder, make(chan struct{}))
-	// Set the informer on the user cloud object
-	if informerUserCloud, ok := cloud.(cloudprovider.InformerUser); ok {
-		informerUserCloud.SetInformers(c.SharedInformers)
-	}
-
-	controllerInitializers := app.DefaultControllerInitializers(c.Complete(), cloud)
 
+	controllerInitializers := app.DefaultInitFuncConstructors
 	// Here is an example to remove the controller which is not needed.
 	// e.g. remove the cloud-node-lifecycle controller which current cloud provider does not need.
 	//delete(controllerInitializers, "cloud-node-lifecycle")
 
 	// Here is an example to add an controller(NodeIpamController) which will be used by cloud provider
-	// generate nodeipamconfig. Here is an sample code. Please pass the right parameter in your code.
+	// generate nodeIPAMConfig. Here is an sample code.
 	// If you do not need additional controller, please ignore.
-	nodeipamconfig := nodeipamconfig.NodeIPAMControllerConfiguration{
-		ServiceCIDR:          "sample",
-		SecondaryServiceCIDR: "sample",
-		NodeCIDRMaskSize:     11,
-		NodeCIDRMaskSizeIPv4: 11,
-		NodeCIDRMaskSizeIPv6: 111,
-	}
-	controllerInitializers["nodeipam"] = startNodeIpamControllerWrapper(c.Complete(), nodeipamconfig, cloud)
 
-	command := app.NewCloudControllerManagerCommand(s, c, controllerInitializers)
+	nodeIpamController := nodeIPAMController{}
+	nodeIpamController.nodeIPAMControllerOptions.NodeIPAMControllerConfiguration = &nodeIpamController.nodeIPAMControllerConfiguration
+	fss := cliflag.NamedFlagSets{}
+	nodeIpamController.nodeIPAMControllerOptions.AddFlags(fss.FlagSet("nodeipam controller"))
+	controllerInitializers["nodeipam"] = nodeIpamController.startNodeIpamControllerWrapper
+
+	command := app.NewCloudControllerManagerCommand(ccmOptions, cloudInitializer, controllerInitializers, fss, wait.NeverStop)
 
 	// TODO: once we switch everything over to Cobra commands, we can go back to calling
 	// utilflag.InitFlags() (by removing its pflag.Parse() call). For now, we have to set the
 	// normalize func and add the go flag set by hand.
 	// Here is an sample
-	pflag.CommandLine.SetNormalizeFunc(flag.WordSepNormalizeFunc)
+	pflag.CommandLine.SetNormalizeFunc(cliflag.WordSepNormalizeFunc)
 	// utilflag.InitFlags()
 	logs.InitLogs()
 	defer logs.FlushLogs()
 
-	// the flags could be set before execute
-	command.Flags().VisitAll(func(flag *pflag.Flag) {
-		if flag.Name == "cloud-provider" {
-			flag.Value.Set("SampleCloudProviderFlagValue")
-			return
-		}
-	})
 	if err := command.Execute(); err != nil {
 		os.Exit(1)
 	}
 }
 
-func startNodeIpamControllerWrapper(ccmconfig *cloudcontrollerconfig.CompletedConfig, nodeipamconfig nodeipamconfig.NodeIPAMControllerConfiguration, cloud cloudprovider.Interface) func(ctx genericcontrollermanager.ControllerContext) (http.Handler, bool, error) {
-	return func(ctx genericcontrollermanager.ControllerContext) (http.Handler, bool, error) {
-		return startNodeIpamController(ccmconfig, nodeipamconfig, ctx, cloud)
+func cloudInitializer(config *cloudcontrollerconfig.CompletedConfig) cloudprovider.Interface {
+	cloudConfig := config.ComponentConfig.KubeCloudShared.CloudProvider
+	// initialize cloud provider with the cloud provider name and config file provided
+	cloud, err := cloudprovider.InitCloudProvider(cloudConfig.Name, cloudConfig.CloudConfigFile)
+	if err != nil {
+		klog.Fatalf("Cloud provider could not be initialized: %v", err)
 	}
+	if cloud == nil {
+		klog.Fatalf("Cloud provider is nil")
+	}
+
+	if !cloud.HasClusterID() {
+		if config.ComponentConfig.KubeCloudShared.AllowUntaggedCloud {
+			klog.Warning("detected a cluster without a ClusterID.  A ClusterID will be required in the future.  Please tag your cluster to avoid any future issues")
+		} else {
+			klog.Fatalf("no ClusterID found.  A ClusterID is required for the cloud provider to function properly.  This check can be bypassed by setting the allow-untagged-cloud option")
+		}
+	}
+
+	return cloud
 }

cmd/cloud-controller-manager/nodeipamcontroller.go

@@ -32,6 +32,7 @@ import (
 	cloudcontrollerconfig "k8s.io/cloud-provider/app/config"
 	genericcontrollermanager "k8s.io/controller-manager/app"
 	"k8s.io/klog/v2"
+	nodeipamcontrolleroptions "k8s.io/kubernetes/cmd/kube-controller-manager/app/options"
 	nodeipamcontroller "k8s.io/kubernetes/pkg/controller/nodeipam"
 	nodeipamconfig "k8s.io/kubernetes/pkg/controller/nodeipam/config"
 	"k8s.io/kubernetes/pkg/controller/nodeipam/ipam"
@@ -45,17 +46,34 @@ const (
 	defaultNodeMaskCIDRIPv6 = 64
 )
 
-func startNodeIpamController(ccmconfig *cloudcontrollerconfig.CompletedConfig, nodeipamconfig nodeipamconfig.NodeIPAMControllerConfiguration, ctx genericcontrollermanager.ControllerContext, cloud cloudprovider.Interface) (http.Handler, bool, error) {
+type nodeIPAMController struct {
+	nodeIPAMControllerConfiguration nodeipamconfig.NodeIPAMControllerConfiguration
+	nodeIPAMControllerOptions       nodeipamcontrolleroptions.NodeIPAMControllerOptions
+}
+
+func (nodeIpamController *nodeIPAMController) startNodeIpamControllerWrapper(completedConfig *cloudcontrollerconfig.CompletedConfig, cloud cloudprovider.Interface) app.InitFunc {
+	errors := nodeIpamController.nodeIPAMControllerOptions.Validate()
+	if len(errors) > 0 {
+		klog.Fatal("NodeIPAM controller values are not properly set.")
+	}
+	nodeIpamController.nodeIPAMControllerOptions.ApplyTo(&nodeIpamController.nodeIPAMControllerConfiguration)
+
+	return func(ctx genericcontrollermanager.ControllerContext) (http.Handler, bool, error) {
+		return startNodeIpamController(completedConfig, nodeIpamController.nodeIPAMControllerConfiguration, ctx, cloud)
+	}
+}
+
+func startNodeIpamController(ccmConfig *cloudcontrollerconfig.CompletedConfig, nodeIPAMConfig nodeipamconfig.NodeIPAMControllerConfiguration, ctx genericcontrollermanager.ControllerContext, cloud cloudprovider.Interface) (http.Handler, bool, error) {
 	var serviceCIDR *net.IPNet
 	var secondaryServiceCIDR *net.IPNet
 
 	// should we start nodeIPAM
-	if !ccmconfig.ComponentConfig.KubeCloudShared.AllocateNodeCIDRs {
+	if !ccmConfig.ComponentConfig.KubeCloudShared.AllocateNodeCIDRs {
 		return nil, false, nil
 	}
 
 	// failure: bad cidrs in config
-	clusterCIDRs, dualStack, err := processCIDRs(ccmconfig.ComponentConfig.KubeCloudShared.ClusterCIDR)
+	clusterCIDRs, dualStack, err := processCIDRs(ccmConfig.ComponentConfig.KubeCloudShared.ClusterCIDR)
 	if err != nil {
 		return nil, false, err
 	}
@@ -76,17 +94,17 @@ func startNodeIpamController(ccmconfig *cloudcontrollerconfig.CompletedConfig, n
 	}
 
 	// service cidr processing
-	if len(strings.TrimSpace(nodeipamconfig.ServiceCIDR)) != 0 {
-		_, serviceCIDR, err = net.ParseCIDR(nodeipamconfig.ServiceCIDR)
+	if len(strings.TrimSpace(nodeIPAMConfig.ServiceCIDR)) != 0 {
+		_, serviceCIDR, err = net.ParseCIDR(nodeIPAMConfig.ServiceCIDR)
 		if err != nil {
-			klog.Warningf("Unsuccessful parsing of service CIDR %v: %v", nodeipamconfig.ServiceCIDR, err)
+			klog.Warningf("Unsuccessful parsing of service CIDR %v: %v", nodeIPAMConfig.ServiceCIDR, err)
 		}
 	}
 
-	if len(strings.TrimSpace(nodeipamconfig.SecondaryServiceCIDR)) != 0 {
-		_, secondaryServiceCIDR, err = net.ParseCIDR(nodeipamconfig.SecondaryServiceCIDR)
+	if len(strings.TrimSpace(nodeIPAMConfig.SecondaryServiceCIDR)) != 0 {
+		_, secondaryServiceCIDR, err = net.ParseCIDR(nodeIPAMConfig.SecondaryServiceCIDR)
 		if err != nil {
-			klog.Warningf("Unsuccessful parsing of service CIDR %v: %v", nodeipamconfig.SecondaryServiceCIDR, err)
+			klog.Warningf("Unsuccessful parsing of service CIDR %v: %v", nodeIPAMConfig.SecondaryServiceCIDR, err)
 		}
 	}
 
@@ -111,11 +129,11 @@ func startNodeIpamController(ccmconfig *cloudcontrollerconfig.CompletedConfig, n
 	if utilfeature.DefaultFeatureGate.Enabled(app.IPv6DualStack) {
 		// only --node-cidr-mask-size-ipv4 and --node-cidr-mask-size-ipv6 supported with dual stack clusters.
 		// --node-cidr-mask-size flag is incompatible with dual stack clusters.
-		nodeCIDRMaskSizeIPv4, nodeCIDRMaskSizeIPv6, err = setNodeCIDRMaskSizesDualStack(nodeipamconfig)
+		nodeCIDRMaskSizeIPv4, nodeCIDRMaskSizeIPv6, err = setNodeCIDRMaskSizesDualStack(nodeIPAMConfig)
 	} else {
 		// only --node-cidr-mask-size supported with single stack clusters.
 		// --node-cidr-mask-size-ipv4 and --node-cidr-mask-size-ipv6 flags are incompatible with dual stack clusters.
-		nodeCIDRMaskSizeIPv4, nodeCIDRMaskSizeIPv6, err = setNodeCIDRMaskSizes(nodeipamconfig)
+		nodeCIDRMaskSizeIPv4, nodeCIDRMaskSizeIPv6, err = setNodeCIDRMaskSizes(nodeIPAMConfig)
 	}
 
 	if err != nil {
@@ -133,7 +151,7 @@ func startNodeIpamController(ccmconfig *cloudcontrollerconfig.CompletedConfig, n
 		serviceCIDR,
 		secondaryServiceCIDR,
 		nodeCIDRMaskSizes,
-		ipam.CIDRAllocatorType(ccmconfig.ComponentConfig.KubeCloudShared.CIDRAllocatorType),
+		ipam.CIDRAllocatorType(ccmConfig.ComponentConfig.KubeCloudShared.CIDRAllocatorType),
 	)
 	if err != nil {
 		return nil, true, err

hack/.staticcheck_failures

@@ -47,5 +47,4 @@ vendor/k8s.io/client-go/discovery
 vendor/k8s.io/client-go/rest
 vendor/k8s.io/client-go/rest/watch
 vendor/k8s.io/client-go/transport
-vendor/k8s.io/cloud-provider/sample
 vendor/k8s.io/kubectl/pkg/cmd/scale