From a1071ac88fa5eb3d76b5616641e251d585dc703a Mon Sep 17 00:00:00 2001 From: Lukasz Szaszkiewicz Date: Wed, 24 Mar 2021 10:18:02 +0100 Subject: [PATCH] UPSTREAM: : allows for switching KS to talk to Kube API over localhost to force KS to use localhost set the following flag in kubescheduler (oc edit kubescheduler cluster) unsupportedConfigOverrides: arguments: unsupported-kube-api-over-localhost:: - "true" UPSTREAM: : allows for switching KS to talk to Kube API over localhost-squash to other This commit is addendum to https://github.com/openshift/kubernetes/commit/04eabe53d2a4a5a26b4c448fc6b983a5324bd0c6 to stop using cc and start relying on scheduler config options OpenShift-Rebase-Source: aa9dde2bfb2 --- cmd/kube-scheduler/app/config/config.go | 3 + cmd/kube-scheduler/app/config/patch.go | 15 +++++ cmd/kube-scheduler/app/options/options.go | 11 ++++ cmd/kube-scheduler/app/options/patch.go | 7 +++ cmd/kube-scheduler/app/patch.go | 70 +++++++++++++++++++++++ cmd/kube-scheduler/app/server.go | 9 +++ 6 files changed, 115 insertions(+) create mode 100644 cmd/kube-scheduler/app/config/patch.go create mode 100644 cmd/kube-scheduler/app/options/patch.go create mode 100644 cmd/kube-scheduler/app/patch.go diff --git a/cmd/kube-scheduler/app/config/config.go b/cmd/kube-scheduler/app/config/config.go index 774cab62dc666..34803dcac449c 100644 --- a/cmd/kube-scheduler/app/config/config.go +++ b/cmd/kube-scheduler/app/config/config.go @@ -57,6 +57,9 @@ type Config struct { // value, the pod will be moved from unschedulablePods to backoffQ or activeQ. // If this value is empty, the default value (5min) will be used. PodMaxInUnschedulablePodsDuration time.Duration + + // OpenShiftContext is additional context that we need to launch the kube-scheduler for openshift + OpenShiftContext OpenShiftContext } type completedConfig struct { diff --git a/cmd/kube-scheduler/app/config/patch.go b/cmd/kube-scheduler/app/config/patch.go new file mode 100644 index 0000000000000..1f2e3ea2c6d3b --- /dev/null +++ b/cmd/kube-scheduler/app/config/patch.go @@ -0,0 +1,15 @@ +package config + +import ( + "k8s.io/client-go/transport" + + "github.com/openshift/library-go/pkg/monitor/health" +) + +// OpenShiftContext is additional context that we need to launch the kube-scheduler for openshift. +// Basically, this holds our additional config information. +type OpenShiftContext struct { + UnsupportedKubeAPIOverPreferredHost bool + PreferredHostRoundTripperWrapperFn transport.WrapperFunc + PreferredHostHealthMonitor *health.Prober +} diff --git a/cmd/kube-scheduler/app/options/options.go b/cmd/kube-scheduler/app/options/options.go index dfc7ce86300f8..b10642c611739 100644 --- a/cmd/kube-scheduler/app/options/options.go +++ b/cmd/kube-scheduler/app/options/options.go @@ -47,6 +47,8 @@ import ( kubeschedulerconfig "k8s.io/kubernetes/pkg/scheduler/apis/config" "k8s.io/kubernetes/pkg/scheduler/apis/config/validation" netutils "k8s.io/utils/net" + + libgorestclient "github.com/openshift/library-go/pkg/config/client" ) // Options has all the params needed to run a Scheduler @@ -72,6 +74,9 @@ type Options struct { // Flags hold the parsed CLI flags. Flags *cliflag.NamedFlagSets + + // OpenShiftContext is additional context that we need to launch the kube-scheduler for openshift. + OpenShiftContext schedulerappconfig.OpenShiftContext } // NewOptions returns default scheduler app options. @@ -187,6 +192,7 @@ func (o *Options) initFlags() { fs.StringVar(&o.ConfigFile, "config", o.ConfigFile, "The path to the configuration file.") fs.StringVar(&o.WriteConfigTo, "write-config-to", o.WriteConfigTo, "If set, write the configuration values to this file and exit.") fs.StringVar(&o.Master, "master", o.Master, "The address of the Kubernetes API server (overrides any value in kubeconfig)") + fs.BoolVar(&o.OpenShiftContext.UnsupportedKubeAPIOverPreferredHost, "unsupported-kube-api-over-localhost", false, "when set makes KS prefer talking to localhost kube-apiserver (when available) instead of an LB") o.SecureServing.AddFlags(nfs.FlagSet("secure serving")) o.Authentication.AddFlags(nfs.FlagSet("authentication")) @@ -229,6 +235,10 @@ func (o *Options) ApplyTo(c *schedulerappconfig.Config) error { if err != nil { return err } + if c.OpenShiftContext.PreferredHostRoundTripperWrapperFn != nil { + libgorestclient.DefaultServerName(kubeConfig) + kubeConfig.Wrap(c.OpenShiftContext.PreferredHostRoundTripperWrapperFn) + } c.KubeConfig = kubeConfig if err := o.SecureServing.ApplyTo(&c.SecureServing, &c.LoopbackClientConfig); err != nil { @@ -276,6 +286,7 @@ func (o *Options) Config() (*schedulerappconfig.Config, error) { } c := &schedulerappconfig.Config{} + c.OpenShiftContext = o.OpenShiftContext if err := o.ApplyTo(c); err != nil { return nil, err } diff --git a/cmd/kube-scheduler/app/options/patch.go b/cmd/kube-scheduler/app/options/patch.go new file mode 100644 index 0000000000000..fcd9c6e405f33 --- /dev/null +++ b/cmd/kube-scheduler/app/options/patch.go @@ -0,0 +1,7 @@ +package options + +import kubeschedulerconfig "k8s.io/kubernetes/pkg/scheduler/apis/config" + +func LoadKubeSchedulerConfiguration(file string) (*kubeschedulerconfig.KubeSchedulerConfiguration, error) { + return loadConfigFromFile(file) +} diff --git a/cmd/kube-scheduler/app/patch.go b/cmd/kube-scheduler/app/patch.go new file mode 100644 index 0000000000000..f377f22f7fa6d --- /dev/null +++ b/cmd/kube-scheduler/app/patch.go @@ -0,0 +1,70 @@ +package app + +import ( + "time" + + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" + "k8s.io/component-base/metrics/legacyregistry" + "k8s.io/kubernetes/cmd/kube-scheduler/app/options" + + libgorestclient "github.com/openshift/library-go/pkg/config/client" + "github.com/openshift/library-go/pkg/monitor/health" +) + +func setUpPreferredHostForOpenShift(kubeSchedulerOptions *options.Options) error { + if !kubeSchedulerOptions.OpenShiftContext.UnsupportedKubeAPIOverPreferredHost { + return nil + } + + master := kubeSchedulerOptions.Master + var kubeConfig string + + // We cannot load component config anymore as the options are not being initialized. + // if there was no kubeconfig specified we won't be able to get cluster info. + // in that case try to load the configuration and read kubeconfig directly from it if it was provided. + if len(kubeSchedulerOptions.ConfigFile) > 0 { + cfg, err := options.LoadKubeSchedulerConfiguration(kubeSchedulerOptions.ConfigFile) + if err != nil { + return err + } + kubeConfig = cfg.ClientConnection.Kubeconfig + } + + config, err := clientcmd.BuildConfigFromFlags(master, kubeConfig) + if err != nil { + return err + } + libgorestclient.DefaultServerName(config) + + targetProvider := health.StaticTargetProvider{"localhost:6443"} + kubeSchedulerOptions.OpenShiftContext.PreferredHostHealthMonitor, err = health.New(targetProvider, createRestConfigForHealthMonitor(config)) + if err != nil { + return err + } + kubeSchedulerOptions.OpenShiftContext.PreferredHostHealthMonitor. + WithHealthyProbesThreshold(3). + WithUnHealthyProbesThreshold(5). + WithProbeInterval(5 * time.Second). + WithProbeResponseTimeout(2 * time.Second). + WithMetrics(health.Register(legacyregistry.MustRegister)) + + kubeSchedulerOptions.OpenShiftContext.PreferredHostRoundTripperWrapperFn = libgorestclient.NewPreferredHostRoundTripper(func() string { + healthyTargets, _ := kubeSchedulerOptions.OpenShiftContext.PreferredHostHealthMonitor.Targets() + if len(healthyTargets) == 1 { + return healthyTargets[0] + } + return "" + }) + + kubeSchedulerOptions.Authentication.WithCustomRoundTripper(kubeSchedulerOptions.OpenShiftContext.PreferredHostRoundTripperWrapperFn) + kubeSchedulerOptions.Authorization.WithCustomRoundTripper(kubeSchedulerOptions.OpenShiftContext.PreferredHostRoundTripperWrapperFn) + return nil +} + +func createRestConfigForHealthMonitor(restConfig *rest.Config) *rest.Config { + restConfigCopy := *restConfig + rest.AddUserAgent(&restConfigCopy, "kube-scheduler-health-monitor") + + return &restConfigCopy +} diff --git a/cmd/kube-scheduler/app/server.go b/cmd/kube-scheduler/app/server.go index 1e23dda87a98c..4ff7bfb4f3c4e 100644 --- a/cmd/kube-scheduler/app/server.go +++ b/cmd/kube-scheduler/app/server.go @@ -137,6 +137,10 @@ func runCommand(cmd *cobra.Command, opts *options.Options, registryOptions ...Op cancel() }() + if err := setUpPreferredHostForOpenShift(opts); err != nil { + return err + } + cc, sched, err := Setup(ctx, opts, registryOptions...) if err != nil { return err @@ -153,6 +157,11 @@ func Run(ctx context.Context, cc *schedulerserverconfig.CompletedConfig, sched * klog.InfoS("Golang settings", "GOGC", os.Getenv("GOGC"), "GOMAXPROCS", os.Getenv("GOMAXPROCS"), "GOTRACEBACK", os.Getenv("GOTRACEBACK")) + // start the localhost health monitor early so that it can be used by the LE client + if cc.OpenShiftContext.PreferredHostHealthMonitor != nil { + go cc.OpenShiftContext.PreferredHostHealthMonitor.Run(ctx) + } + // Configz registration. if cz, err := configz.New("componentconfig"); err == nil { cz.Set(cc.ComponentConfig)