diff --git a/cmd/yurt-controller-manager/app/controllermanager.go b/cmd/yurt-controller-manager/app/controllermanager.go index 58724e8d3cc..ff2513b7f1f 100644 --- a/cmd/yurt-controller-manager/app/controllermanager.go +++ b/cmd/yurt-controller-manager/app/controllermanager.go @@ -287,6 +287,7 @@ var ControllersDisabledByDefault = sets.NewString() func NewControllerInitializers() map[string]InitFunc { controllers := map[string]InitFunc{} controllers["nodelifecycle"] = startNodeLifecycleController + controllers["yurthubcsrapprover"] = startYurtHubCSRApproverController return controllers } diff --git a/cmd/yurt-controller-manager/app/core.go b/cmd/yurt-controller-manager/app/core.go index 1cf4a341dde..25e5082e11b 100644 --- a/cmd/yurt-controller-manager/app/core.go +++ b/cmd/yurt-controller-manager/app/core.go @@ -25,7 +25,10 @@ import ( "net/http" "time" + "k8s.io/client-go/informers" + lifecyclecontroller "github.com/openyurtio/openyurt/pkg/controller/nodelifecycle" + "github.com/openyurtio/openyurt/pkg/controller/yurthub" ) func startNodeLifecycleController(ctx ControllerContext) (http.Handler, bool, error) { @@ -53,3 +56,15 @@ func startNodeLifecycleController(ctx ControllerContext) (http.Handler, bool, er go lifecycleController.Run(ctx.Stop) return nil, true, nil } + +func startYurtHubCSRApproverController(ctx ControllerContext) (http.Handler, bool, error) { + clientSet := ctx.ClientBuilder.ClientOrDie("node-controller") + sharedInformerFactory := informers.NewSharedInformerFactory(clientSet, 10*time.Second) + go yurthub.NewCSRApprover(clientSet, sharedInformerFactory.Certificates().V1beta1().CertificateSigningRequests()). + Run(yurthub.YurtHubCSRApproverThreadiness, ctx.Stop) + + // after all of informers are configured completed, start the shared index informer + sharedInformerFactory.Start(ctx.Stop) + + return nil, true, nil +} diff --git a/cmd/yurthub/app/config/config.go b/cmd/yurthub/app/config/config.go index c7a1a0d5463..2590233d367 100644 --- a/cmd/yurthub/app/config/config.go +++ b/cmd/yurthub/app/config/config.go @@ -17,6 +17,7 @@ limitations under the License. package config import ( + "crypto/tls" "fmt" "net" "net/url" @@ -33,28 +34,31 @@ import ( // YurtHubConfiguration represents configuration of yurthub type YurtHubConfiguration struct { - LBMode string - RemoteServers []*url.URL - YurtHubServerAddr string - YurtHubProxyServerAddr string - YurtHubProxyServerDummyAddr string - GCFrequency int - CertMgrMode string - KubeletRootCAFilePath string - KubeletPairFilePath string - NodeName string - HeartbeatFailedRetry int - HeartbeatHealthyThreshold int - HeartbeatTimeoutSeconds int - MaxRequestInFlight int - JoinToken string - RootDir string - EnableProfiling bool - EnableDummyIf bool - EnableIptables bool - HubAgentDummyIfName string - StorageWrapper cachemanager.StorageWrapper - SerializerManager *serializer.SerializerManager + LBMode string + RemoteServers []*url.URL + YurtHubServerAddr string + YurtHubProxyServerAddr string + YurtHubProxyServerSecureAddr string + YurtHubProxyServerDummyAddr string + YurtHubProxyServerSecureDummyAddr string + GCFrequency int + CertMgrMode string + KubeletRootCAFilePath string + KubeletPairFilePath string + NodeName string + HeartbeatFailedRetry int + HeartbeatHealthyThreshold int + HeartbeatTimeoutSeconds int + MaxRequestInFlight int + JoinToken string + RootDir string + EnableProfiling bool + EnableDummyIf bool + EnableIptables bool + HubAgentDummyIfName string + StorageWrapper cachemanager.StorageWrapper + SerializerManager *serializer.SerializerManager + TLSConfig *tls.Config } // Complete converts *options.YurtHubOptions to *YurtHubConfiguration @@ -74,30 +78,34 @@ func Complete(options *options.YurtHubOptions) (*YurtHubConfiguration, error) { hubServerAddr := net.JoinHostPort(options.YurtHubHost, options.YurtHubPort) proxyServerAddr := net.JoinHostPort(options.YurtHubHost, options.YurtHubProxyPort) + proxySecureServerAddr := net.JoinHostPort(options.YurtHubHost, options.YurtHubProxySecurePort) proxyServerDummyAddr := net.JoinHostPort(options.HubAgentDummyIfIP, options.YurtHubProxyPort) + proxySecureServerDummyAddr := net.JoinHostPort(options.HubAgentDummyIfIP, options.YurtHubProxySecurePort) cfg := &YurtHubConfiguration{ - LBMode: options.LBMode, - RemoteServers: us, - YurtHubServerAddr: hubServerAddr, - YurtHubProxyServerAddr: proxyServerAddr, - YurtHubProxyServerDummyAddr: proxyServerDummyAddr, - GCFrequency: options.GCFrequency, - CertMgrMode: options.CertMgrMode, - KubeletRootCAFilePath: options.KubeletRootCAFilePath, - KubeletPairFilePath: options.KubeletPairFilePath, - NodeName: options.NodeName, - HeartbeatFailedRetry: options.HeartbeatFailedRetry, - HeartbeatHealthyThreshold: options.HeartbeatHealthyThreshold, - HeartbeatTimeoutSeconds: options.HeartbeatTimeoutSeconds, - MaxRequestInFlight: options.MaxRequestInFlight, - JoinToken: options.JoinToken, - RootDir: options.RootDir, - EnableProfiling: options.EnableProfiling, - EnableDummyIf: options.EnableDummyIf, - EnableIptables: options.EnableIptables, - HubAgentDummyIfName: options.HubAgentDummyIfName, - StorageWrapper: storageWrapper, - SerializerManager: serializerManager, + LBMode: options.LBMode, + RemoteServers: us, + YurtHubServerAddr: hubServerAddr, + YurtHubProxyServerAddr: proxyServerAddr, + YurtHubProxyServerSecureAddr: proxySecureServerAddr, + YurtHubProxyServerDummyAddr: proxyServerDummyAddr, + YurtHubProxyServerSecureDummyAddr: proxySecureServerDummyAddr, + GCFrequency: options.GCFrequency, + CertMgrMode: options.CertMgrMode, + KubeletRootCAFilePath: options.KubeletRootCAFilePath, + KubeletPairFilePath: options.KubeletPairFilePath, + NodeName: options.NodeName, + HeartbeatFailedRetry: options.HeartbeatFailedRetry, + HeartbeatHealthyThreshold: options.HeartbeatHealthyThreshold, + HeartbeatTimeoutSeconds: options.HeartbeatTimeoutSeconds, + MaxRequestInFlight: options.MaxRequestInFlight, + JoinToken: options.JoinToken, + RootDir: options.RootDir, + EnableProfiling: options.EnableProfiling, + EnableDummyIf: options.EnableDummyIf, + EnableIptables: options.EnableIptables, + HubAgentDummyIfName: options.HubAgentDummyIfName, + StorageWrapper: storageWrapper, + SerializerManager: serializerManager, } return cfg, nil diff --git a/cmd/yurthub/app/options/options.go b/cmd/yurthub/app/options/options.go index 21b9bab0c6e..2e323e7ed11 100644 --- a/cmd/yurthub/app/options/options.go +++ b/cmd/yurthub/app/options/options.go @@ -38,6 +38,7 @@ type YurtHubOptions struct { YurtHubHost string YurtHubPort string YurtHubProxyPort string + YurtHubProxySecurePort string GCFrequency int CertMgrMode string KubeletRootCAFilePath string @@ -65,6 +66,7 @@ func NewYurtHubOptions() *YurtHubOptions { YurtHubHost: "127.0.0.1", YurtHubProxyPort: "10261", YurtHubPort: "10267", + YurtHubProxySecurePort: "10268", GCFrequency: 120, CertMgrMode: util.YurtHubCertificateManagerName, KubeletRootCAFilePath: util.DefaultKubeletRootCAFilePath, @@ -116,6 +118,7 @@ func (o *YurtHubOptions) AddFlags(fs *pflag.FlagSet) { fs.StringVar(&o.YurtHubHost, "bind-address", o.YurtHubHost, "the IP address on which to listen for the --serve-port port.") fs.StringVar(&o.YurtHubPort, "serve-port", o.YurtHubPort, "the port on which to serve HTTP requests(like profiling, metrics) for hub agent.") fs.StringVar(&o.YurtHubProxyPort, "proxy-port", o.YurtHubProxyPort, "the port on which to proxy HTTP requests to kube-apiserver") + fs.StringVar(&o.YurtHubProxySecurePort, "proxy-secure-port", o.YurtHubProxySecurePort, "the port on which to proxy HTTPS requests to kube-apiserver") fs.StringVar(&o.ServerAddr, "server-addr", o.ServerAddr, "the address of Kubernetes kube-apiserver,the format is: \"server1,server2,...\"") fs.StringVar(&o.CertMgrMode, "cert-mgr-mode", o.CertMgrMode, "the cert manager mode, kubelet: use certificates that belongs to kubelet, hubself: auto generate client cert for hub agent.") fs.StringVar(&o.KubeletRootCAFilePath, "kubelet-ca-file", o.KubeletRootCAFilePath, "the ca file path used by kubelet.") diff --git a/cmd/yurthub/app/start.go b/cmd/yurthub/app/start.go index aa2901808a2..608375480cd 100644 --- a/cmd/yurthub/app/start.go +++ b/cmd/yurthub/app/start.go @@ -18,6 +18,7 @@ package app import ( "fmt" + "path/filepath" "github.com/openyurtio/openyurt/cmd/yurthub/app/config" "github.com/openyurtio/openyurt/cmd/yurthub/app/options" @@ -119,6 +120,14 @@ func Run(cfg *config.YurtHubConfiguration, stopCh <-chan struct{}) error { } trace++ + klog.Infof("%d. create tls config for secure servers ", trace) + cfg.TLSConfig, err = server.GenUseCertMgrAndTLSConfig(restConfigMgr, certManager, filepath.Join(cfg.RootDir, "pki"), stopCh) + if err != nil { + klog.Errorf("could not create tls config, %v", err) + return err + } + trace++ + klog.Infof("%d. new cache manager with storage wrapper and serializer manager", trace) cacheMgr, err := cachemanager.NewCacheManager(cfg.StorageWrapper, cfg.SerializerManager) if err != nil { @@ -153,11 +162,11 @@ func Run(cfg *config.YurtHubConfiguration, stopCh <-chan struct{}) error { } networkMgr.Run(stopCh) trace++ - klog.Infof("%d. new %s server and begin to serve, dummy proxy server: %s", trace, projectinfo.GetHubName(), cfg.YurtHubProxyServerDummyAddr) + klog.Infof("%d. new %s server and begin to serve, dummy proxy server: %s, secure dummy proxy server: %s", trace, projectinfo.GetHubName(), cfg.YurtHubProxyServerDummyAddr, cfg.YurtHubProxyServerSecureDummyAddr) } - klog.Infof("%d. new %s server and begin to serve, proxy server: %s, hub server: %s", trace, projectinfo.GetHubName(), cfg.YurtHubProxyServerAddr, cfg.YurtHubServerAddr) - s, err := server.NewYurtHubServer(cfg, certManager, yurtProxyHandler) + klog.Infof("%d. new %s server and begin to serve, proxy server: %s, secure proxy server: %s, hub server: %s", trace, projectinfo.GetHubName(), cfg.YurtHubProxyServerAddr, cfg.YurtHubProxyServerSecureAddr, cfg.YurtHubServerAddr) + s, err := server.NewYurtHubServer(cfg, certManager, yurtProxyHandler, stopCh) if err != nil { klog.Errorf("could not create hub server, %v", err) return err diff --git a/pkg/controller/yurthub/crsapprover.go b/pkg/controller/yurthub/crsapprover.go new file mode 100644 index 00000000000..eedb4110855 --- /dev/null +++ b/pkg/controller/yurthub/crsapprover.go @@ -0,0 +1,219 @@ +/* +Copyright 2020 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package yurthub + +import ( + "context" + "crypto/x509" + "encoding/pem" + "fmt" + "time" + + certificates "k8s.io/api/certificates/v1beta1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/apimachinery/pkg/util/wait" + certinformer "k8s.io/client-go/informers/certificates/v1beta1" + certv1beta1 "k8s.io/client-go/informers/certificates/v1beta1" + "k8s.io/client-go/kubernetes" + typev1beta1 "k8s.io/client-go/kubernetes/typed/certificates/v1beta1" + "k8s.io/client-go/tools/cache" + "k8s.io/client-go/util/workqueue" + "k8s.io/klog/v2" + + "github.com/openyurtio/openyurt/pkg/projectinfo" + "github.com/openyurtio/openyurt/pkg/yurthub/pki/certmanager" +) + +const ( + YurtHubCSRApproverThreadiness = 2 +) + +// YurtHubCSRApprover is the controller that auto approve all +// yurthub related CSR +type YurtHubCSRApprover struct { + csrInformer certv1beta1.CertificateSigningRequestInformer + csrClient typev1beta1.CertificateSigningRequestInterface + workqueue workqueue.RateLimitingInterface +} + +// Run starts the YurtHubCSRApprover +func (yca *YurtHubCSRApprover) Run(threadiness int, stopCh <-chan struct{}) { + defer runtime.HandleCrash() + defer yca.workqueue.ShutDown() + klog.Info("starting the crsapprover") + if !cache.WaitForCacheSync(stopCh, + yca.csrInformer.Informer().HasSynced) { + klog.Error("sync csr timeout") + return + } + for i := 0; i < threadiness; i++ { + go wait.Until(yca.runWorker, time.Second, stopCh) + } + <-stopCh + klog.Info("stoping the csrapprover") +} + +func (yca *YurtHubCSRApprover) runWorker() { + for yca.processNextItem() { + } +} + +func (yca *YurtHubCSRApprover) processNextItem() bool { + key, quit := yca.workqueue.Get() + if quit { + return false + } + csrName, ok := key.(string) + if !ok { + yca.workqueue.Forget(key) + runtime.HandleError( + fmt.Errorf("expected string in workqueue but got %#v", key)) + return true + } + defer yca.workqueue.Done(key) + + csr, err := yca.csrInformer.Lister().Get(csrName) + if err != nil { + runtime.HandleError(err) + if !apierrors.IsNotFound(err) { + yca.workqueue.AddRateLimited(key) + } + return true + } + + if err := approveYurtHubCSR(csr, yca.csrClient); err != nil { + runtime.HandleError(err) + enqueueObj(yca.workqueue, csr) + return true + } + + return true +} + +func enqueueObj(wq workqueue.RateLimitingInterface, obj interface{}) { + key, err := cache.MetaNamespaceKeyFunc(obj) + if err != nil { + runtime.HandleError(err) + return + } + wq.AddRateLimited(key) +} + +// NewCSRApprover creates a new YurtHubCSRApprover +func NewCSRApprover( + clientset kubernetes.Interface, + csrInformer certinformer.CertificateSigningRequestInformer) *YurtHubCSRApprover { + + wq := workqueue.NewRateLimitingQueue(workqueue.DefaultControllerRateLimiter()) + csrInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ + AddFunc: func(obj interface{}) { + enqueueObj(wq, obj) + }, + UpdateFunc: func(old, new interface{}) { + enqueueObj(wq, new) + }, + }) + return &YurtHubCSRApprover{ + csrInformer: csrInformer, + csrClient: clientset.CertificatesV1beta1().CertificateSigningRequests(), + workqueue: wq, + } +} + +// approveYurtHubCSR checks the csr status, if it is neither approved nor +// denied, it will try to approve the csr. +func approveYurtHubCSR( + obj interface{}, + csrClient typev1beta1.CertificateSigningRequestInterface) error { + csr, ok := obj.(*certificates.CertificateSigningRequest) + if !ok { + return nil + } + + if !isYurtHubCSR(csr) { + klog.Infof("csr(%s) is not %s csr", csr.GetName(), projectinfo.GetTunnelName()) + return nil + } + + approved, denied := checkCertApprovalCondition(&csr.Status) + if approved { + klog.V(4).Infof("csr(%s) is approved", csr.GetName()) + return nil + } + + if denied { + klog.V(4).Infof("csr(%s) is denied", csr.GetName()) + return nil + } + + // approve the yurthub related csr + csr.Status.Conditions = append(csr.Status.Conditions, + certificates.CertificateSigningRequestCondition{ + Type: certificates.CertificateApproved, + Reason: "AutoApproved", + Message: fmt.Sprintf("self-approving %s csr", projectinfo.GetTunnelName()), + }) + + result, err := csrClient.UpdateApproval(context.Background(), csr, metav1.UpdateOptions{}) + if err != nil { + klog.Errorf("failed to approve %s csr(%s), %v", projectinfo.GetTunnelName(), csr.GetName(), err) + return err + } + klog.Infof("successfully approve %s csr(%s)", projectinfo.GetTunnelName(), result.Name) + return nil +} + +// isYurtHubCSR checks if given csr is a yurthub related csr, i.e., +// the organizations' list contains "openyurt:yurthub" +func isYurtHubCSR(csr *certificates.CertificateSigningRequest) bool { + pemBytes := csr.Spec.Request + block, _ := pem.Decode(pemBytes) + if block == nil || block.Type != "CERTIFICATE REQUEST" { + return false + } + x509cr, err := x509.ParseCertificateRequest(block.Bytes) + if err != nil { + return false + } + for i, org := range x509cr.Subject.Organization { + if org == certmanager.YurtHubCSROrg { + break + } + if i == len(x509cr.Subject.Organization)-1 { + return false + } + } + return true +} + +// checkCertApprovalCondition checks if the given csr's status is +// approved or denied +func checkCertApprovalCondition( + status *certificates.CertificateSigningRequestStatus) ( + approved bool, denied bool) { + for _, c := range status.Conditions { + if c.Type == certificates.CertificateApproved { + approved = true + } + if c.Type == certificates.CertificateDenied { + denied = true + } + } + return +} diff --git a/pkg/yurthub/network/network.go b/pkg/yurthub/network/network.go index 55f3aed6dc0..b47e610e894 100644 --- a/pkg/yurthub/network/network.go +++ b/pkg/yurthub/network/network.go @@ -54,6 +54,12 @@ func NewNetworkManager(cfg *config.YurtHubConfiguration) (*NetworkManager, error dummyIfName: cfg.HubAgentDummyIfName, enableIptables: cfg.EnableIptables, } + // secure port + _, securePort, err := net.SplitHostPort(cfg.YurtHubProxyServerSecureDummyAddr) + if err != nil { + return nil, err + } + m.iptablesManager.rules = append(m.iptablesManager.rules, makeupIptablesRules(ip, securePort)...) if err = m.configureNetwork(); err != nil { return nil, err } diff --git a/pkg/yurthub/pki/certmanager/certmanager.go b/pkg/yurthub/pki/certmanager/certmanager.go new file mode 100644 index 00000000000..615036e43bc --- /dev/null +++ b/pkg/yurthub/pki/certmanager/certmanager.go @@ -0,0 +1,96 @@ +/* +Copyright 2020 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package certmanager + +import ( + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "fmt" + "net" + + "github.com/openyurtio/openyurt/pkg/projectinfo" + + certificates "k8s.io/api/certificates/v1beta1" + "k8s.io/client-go/kubernetes" + clicert "k8s.io/client-go/kubernetes/typed/certificates/v1beta1" + "k8s.io/client-go/util/certificate" + "k8s.io/klog/v2" +) + +const ( + YurtHubServerCSROrg = "system:masters" + YurtHubCSROrg = "openyurt:yurthub" + YurtHubServerCSRCN = "kube-apiserver-kubelet-client" +) + +// NewYurtHubServerCertManager creates a certificate manager for +// the yurthub-server +func NewYurtHubServerCertManager( + clientset kubernetes.Interface, + certDir string) (certificate.Manager, error) { + + klog.Infof("subject of yurthub server certificate") + return newCertManager( + clientset, + projectinfo.GetHubName(), + certDir, + YurtHubServerCSRCN, + []string{YurtHubServerCSROrg, YurtHubCSROrg}) +} + +// NewCertManager creates a certificate manager that will generates a +// certificate by sending a csr to the apiserver +func newCertManager( + clientset kubernetes.Interface, + componentName, + certDir, + commonName string, + organizations []string) (certificate.Manager, error) { + certificateStore, err := + certificate.NewFileStore(componentName, certDir, certDir, "", "") + if err != nil { + return nil, fmt.Errorf("failed to initialize the server certificate store: %v", err) + } + + getTemplate := func() *x509.CertificateRequest { + return &x509.CertificateRequest{ + Subject: pkix.Name{ + CommonName: commonName, + Organization: organizations, + }, + IPAddresses: []net.IP{net.ParseIP("127.0.0.1")}, + } + } + + certManager, err := certificate.NewManager(&certificate.Config{ + ClientFn: func(current *tls.Certificate) (clicert.CertificateSigningRequestInterface, error) { + return clientset.CertificatesV1beta1().CertificateSigningRequests(), nil + }, + SignerName: certificates.LegacyUnknownSignerName, + GetTemplate: getTemplate, + Usages: []certificates.KeyUsage{ + certificates.UsageAny, + }, + CertificateStore: certificateStore, + }) + if err != nil { + return nil, fmt.Errorf("failed to initialize server certificate manager: %v", err) + } + + return certManager, nil +} diff --git a/pkg/yurthub/pki/pki.go b/pkg/yurthub/pki/pki.go new file mode 100644 index 00000000000..57584b13fff --- /dev/null +++ b/pkg/yurthub/pki/pki.go @@ -0,0 +1,85 @@ +/* +Copyright 2020 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package pki + +import ( + "crypto/tls" + "crypto/x509" + "errors" + "fmt" + "io/ioutil" + "os" + + "k8s.io/client-go/util/certificate" +) + +// GenTGenTLSConfigUseCertMgrAndCertPool generates a TLS configuration +// using the given certificate manager and x509 CertPool +func GenTLSConfigUseCertMgrAndCertPool( + m certificate.Manager, + root *x509.CertPool) (*tls.Config, error) { + tlsConfig := &tls.Config{ + // Can't use SSLv3 because of POODLE and BEAST + // Can't use TLSv1.0 because of POODLE and BEAST using CBC cipher + // Can't use TLSv1.1 because of RC4 cipher usage + MinVersion: tls.VersionTLS12, + RootCAs: root, + ClientAuth: tls.RequireAnyClientCert, + } + + tlsConfig.GetClientCertificate = + func(*tls.CertificateRequestInfo) (*tls.Certificate, error) { + cert := m.Current() + if cert == nil { + return &tls.Certificate{Certificate: nil}, nil + } + return cert, nil + } + tlsConfig.GetCertificate = + func(*tls.ClientHelloInfo) (*tls.Certificate, error) { + cert := m.Current() + if cert == nil { + return &tls.Certificate{Certificate: nil}, nil + } + return cert, nil + } + + return tlsConfig, nil +} + +// GenCertPoolUseCA generates a x509 CertPool based on the given CA file +func GenCertPoolUseCA(caFile string) (*x509.CertPool, error) { + if caFile == "" { + return nil, errors.New("CA file is not set") + } + + if _, err := os.Stat(caFile); err != nil { + if os.IsNotExist(err) { + return nil, fmt.Errorf("CA file(%s) doesn't exist", caFile) + } + return nil, fmt.Errorf("fail to stat the CA file(%s): %s", caFile, err) + } + + caData, err := ioutil.ReadFile(caFile) + if err != nil { + return nil, err + } + + certPool := x509.NewCertPool() + certPool.AppendCertsFromPEM(caData) + return certPool, nil +} diff --git a/pkg/yurthub/server/server.go b/pkg/yurthub/server/server.go index 4753966062f..d1d3f2897bd 100644 --- a/pkg/yurthub/server/server.go +++ b/pkg/yurthub/server/server.go @@ -17,16 +17,25 @@ limitations under the License. package server import ( + "crypto/tls" "fmt" "net" "net/http" + "time" "github.com/openyurtio/openyurt/cmd/yurthub/app/config" "github.com/openyurtio/openyurt/pkg/profile" + "github.com/openyurtio/openyurt/pkg/projectinfo" "github.com/openyurtio/openyurt/pkg/yurthub/certificate/interfaces" + "github.com/openyurtio/openyurt/pkg/yurthub/kubernetes/rest" + "github.com/openyurtio/openyurt/pkg/yurthub/pki" + "github.com/openyurtio/openyurt/pkg/yurthub/pki/certmanager" "github.com/gorilla/mux" "github.com/prometheus/client_golang/prometheus/promhttp" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/kubernetes" + "k8s.io/klog" ) // Server is an interface for providing http service for yurthub @@ -38,15 +47,17 @@ type Server interface { // and hubServer handles requests by hub agent itself, like profiling, metrics, healthz // and proxyServer does not handle requests locally and proxy requests to kube-apiserver type yurtHubServer struct { - hubServer *http.Server - proxyServer *http.Server - dummyProxyServer *http.Server + hubServer *http.Server + proxyServer *http.Server + secureProxyServer *http.Server + dummyProxyServer *http.Server + dummySecureProxyServer *http.Server } // NewYurtHubServer creates a Server object func NewYurtHubServer(cfg *config.YurtHubConfiguration, certificateMgr interfaces.YurtCertificateManager, - proxyHandler http.Handler) (Server, error) { + proxyHandler http.Handler, stopCh <-chan struct{}) (Server, error) { hubMux := mux.NewRouter() registerHandlers(hubMux, cfg, certificateMgr) hubServer := &http.Server{ @@ -56,12 +67,19 @@ func NewYurtHubServer(cfg *config.YurtHubConfiguration, } proxyServer := &http.Server{ - Addr: cfg.YurtHubProxyServerAddr, + Addr: cfg.YurtHubProxyServerAddr, + Handler: proxyHandler, + } + + secureProxyServer := &http.Server{ + Addr: cfg.YurtHubProxyServerSecureAddr, Handler: proxyHandler, + TLSConfig: cfg.TLSConfig, + TLSNextProto: make(map[string]func(*http.Server, *tls.Conn, http.Handler)), MaxHeaderBytes: 1 << 20, } - var dummyProxyServer *http.Server + var dummyProxyServer, secureDummyProxyServer *http.Server if cfg.EnableDummyIf { if _, err := net.InterfaceByName(cfg.HubAgentDummyIfName); err != nil { return nil, err @@ -72,12 +90,22 @@ func NewYurtHubServer(cfg *config.YurtHubConfiguration, Handler: proxyHandler, MaxHeaderBytes: 1 << 20, } + + secureDummyProxyServer = &http.Server{ + Addr: cfg.YurtHubProxyServerSecureDummyAddr, + Handler: proxyHandler, + TLSConfig: cfg.TLSConfig, + TLSNextProto: make(map[string]func(*http.Server, *tls.Conn, http.Handler)), + MaxHeaderBytes: 1 << 20, + } } return &yurtHubServer{ - hubServer: hubServer, - proxyServer: proxyServer, - dummyProxyServer: dummyProxyServer, + hubServer: hubServer, + proxyServer: proxyServer, + secureProxyServer: secureProxyServer, + dummyProxyServer: dummyProxyServer, + dummySecureProxyServer: secureDummyProxyServer, }, nil } @@ -97,8 +125,21 @@ func (s *yurtHubServer) Run() { panic(err) } }() + go func() { + err := s.dummySecureProxyServer.ListenAndServeTLS("", "") + if err != nil { + panic(err) + } + }() } + go func() { + err := s.secureProxyServer.ListenAndServeTLS("", "") + if err != nil { + panic(err) + } + }() + err := s.proxyServer.ListenAndServe() if err != nil { panic(err) @@ -127,3 +168,41 @@ func healthz(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusOK) fmt.Fprintf(w, "OK") } + +// create a certificate manager for the yurthub server and run the csr approver for both yurthub +// and generate a TLS configuration +func GenUseCertMgrAndTLSConfig(restConfigMgr *rest.RestConfigManager, certificateMgr interfaces.YurtCertificateManager, certDir string, stopCh <-chan struct{}) (*tls.Config, error) { + clientSet, err := kubernetes.NewForConfig(restConfigMgr.GetRestConfig()) + if err != nil { + return nil, err + } + // create a certificate manager for the yurthub server and run the csr approver for both yurthub + serverCertMgr, err := certmanager.NewYurtHubServerCertManager(clientSet, certDir) + if err != nil { + return nil, err + } + serverCertMgr.Start() + + // generate the TLS configuration based on the latest certificate + rootCert, err := pki.GenCertPoolUseCA(certificateMgr.GetCaFile()) + if err != nil { + klog.Errorf("could not generate a x509 CertPool based on the given CA file, %v", err) + return nil, err + } + tlsCfg, err := pki.GenTLSConfigUseCertMgrAndCertPool(serverCertMgr, rootCert) + if err != nil { + return nil, err + } + + // waiting for the certificate is generated + _ = wait.PollUntil(5*time.Second, func() (bool, error) { + // keep polling until the certificate is signed + if serverCertMgr.Current() != nil { + return true, nil + } + klog.Infof("waiting for the master to sign the %s certificate", projectinfo.GetHubName()) + return false, nil + }, stopCh) + + return tlsCfg, nil +}