From 2a9460f6b30d019970e2c91b34388ca644581727 Mon Sep 17 00:00:00 2001 From: Cookie Wang Date: Thu, 24 Jun 2021 00:01:41 +0800 Subject: [PATCH 1/2] change yurthub's protocol from http to https --- cmd/yurthub/app/config/config.go | 6 ++++++ cmd/yurthub/app/options/options.go | 18 +++++++++++++++++ pkg/yurthub/server/server.go | 31 +++++++++++++++++++++++++++--- 3 files changed, 52 insertions(+), 3 deletions(-) diff --git a/cmd/yurthub/app/config/config.go b/cmd/yurthub/app/config/config.go index 9c12610301d..67db8e82762 100644 --- a/cmd/yurthub/app/config/config.go +++ b/cmd/yurthub/app/config/config.go @@ -53,6 +53,9 @@ type YurtHubConfiguration struct { HubAgentDummyIfName string StorageWrapper cachemanager.StorageWrapper SerializerManager *serializer.SerializerManager + CAFile string + CertFile string + KeyFile string } // Complete converts *options.YurtHubOptions to *YurtHubConfiguration @@ -94,6 +97,9 @@ func Complete(options *options.YurtHubOptions) (*YurtHubConfiguration, error) { HubAgentDummyIfName: options.HubAgentDummyIfName, StorageWrapper: storageWrapper, SerializerManager: serializerManager, + CAFile: options.CAFile, + CertFile: options.CertFile, + KeyFile: options.KeyFile, } return cfg, nil diff --git a/cmd/yurthub/app/options/options.go b/cmd/yurthub/app/options/options.go index eb98f2121b5..3e22e70e8c9 100644 --- a/cmd/yurthub/app/options/options.go +++ b/cmd/yurthub/app/options/options.go @@ -55,6 +55,9 @@ type YurtHubOptions struct { HubAgentDummyIfIP string HubAgentDummyIfName string DiskCachePath string + CAFile string + CertFile string + KeyFile string } // NewYurtHubOptions creates a new YurtHubOptions with a default config. @@ -92,6 +95,18 @@ func ValidateOptions(options *YurtHubOptions) error { return fmt.Errorf("server-address is empty") } + if len(options.CAFile) == 0 { + return fmt.Errorf("CA is empty") + } + + if len(options.CertFile) == 0 { + return fmt.Errorf("tls cert is empty") + } + + if len(options.KeyFile) == 0 { + return fmt.Errorf("tls key is empty") + } + if !util.IsSupportedLBMode(options.LBMode) { return fmt.Errorf("lb mode(%s) is not supported", options.LBMode) } @@ -130,6 +145,9 @@ func (o *YurtHubOptions) AddFlags(fs *pflag.FlagSet) { fs.StringVar(&o.HubAgentDummyIfIP, "dummy-if-ip", o.HubAgentDummyIfIP, "the ip address of dummy interface that used for container connect hub agent(exclusive ips: 169.254.31.0/24, 169.254.1.1/32)") fs.StringVar(&o.HubAgentDummyIfName, "dummy-if-name", o.HubAgentDummyIfName, "the name of dummy interface that is used for hub agent") fs.StringVar(&o.DiskCachePath, "disk-cache-path", o.DiskCachePath, "the path for kubernetes to storage metadata") + fs.StringVar(&o.CAFile, "ca-file", "", "the CA for yurthub to verify client") + fs.StringVar(&o.CertFile, "tls-cert-file", "", "the tls cert of yurthub") + fs.StringVar(&o.KeyFile, "tls-private-key-file", "", "the tls key of yurthub") } // verifyDummyIP verify the specified ip is valid or not diff --git a/pkg/yurthub/server/server.go b/pkg/yurthub/server/server.go index 4753966062f..007e71dfc71 100644 --- a/pkg/yurthub/server/server.go +++ b/pkg/yurthub/server/server.go @@ -17,7 +17,10 @@ limitations under the License. package server import ( + "crypto/tls" + "crypto/x509" "fmt" + "io/ioutil" "net" "net/http" @@ -27,6 +30,7 @@ import ( "github.com/gorilla/mux" "github.com/prometheus/client_golang/prometheus/promhttp" + "k8s.io/klog" ) // Server is an interface for providing http service for yurthub @@ -38,6 +42,12 @@ type Server interface { // and hubServer handles requests by hub agent itself, like profiling, metrics, healthz // and proxyServer does not handle requests locally and proxy requests to kube-apiserver type yurtHubServer struct { + CAFile string + // CertFile the tls cert file for proxyhub's https server + CertFile string + // KeyFile the tls key file for proxyhub's https server + KeyFile string + hubServer *http.Server proxyServer *http.Server dummyProxyServer *http.Server @@ -55,9 +65,21 @@ func NewYurtHubServer(cfg *config.YurtHubConfiguration, MaxHeaderBytes: 1 << 20, } + caFile, err := ioutil.ReadFile(cfg.CAFile) + if err != nil { + klog.Errorf("Read ca file err: %v", err) + return nil, err + } + certPool := x509.NewCertPool() + certPool.AppendCertsFromPEM([]byte(caFile)) + proxyServer := &http.Server{ - Addr: cfg.YurtHubProxyServerAddr, - Handler: proxyHandler, + Addr: cfg.YurtHubProxyServerAddr, + Handler: proxyHandler, + TLSConfig: &tls.Config{ + ClientCAs: certPool, + ClientAuth: tls.VerifyClientCertIfGiven, + }, MaxHeaderBytes: 1 << 20, } @@ -75,6 +97,9 @@ func NewYurtHubServer(cfg *config.YurtHubConfiguration, } return &yurtHubServer{ + CAFile: cfg.CAFile, + CertFile: cfg.CertFile, + KeyFile: cfg.KeyFile, hubServer: hubServer, proxyServer: proxyServer, dummyProxyServer: dummyProxyServer, @@ -99,7 +124,7 @@ func (s *yurtHubServer) Run() { }() } - err := s.proxyServer.ListenAndServe() + err := s.proxyServer.ListenAndServeTLS(s.CertFile, s.KeyFile) if err != nil { panic(err) } From 0a0e31369b4baed753634f9f01bf170e5cc1d513 Mon Sep 17 00:00:00 2001 From: Cookie Wang Date: Thu, 24 Jun 2021 00:01:41 +0800 Subject: [PATCH 2/2] change yurthub's protocol from http to https --- cmd/yurthub/app/config/config.go | 89 ++++++++++++++++-------------- cmd/yurthub/app/options/options.go | 21 +++++++ cmd/yurthub/app/start.go | 2 +- pkg/yurthub/server/server.go | 55 +++++++++++++++--- 4 files changed, 118 insertions(+), 49 deletions(-) diff --git a/cmd/yurthub/app/config/config.go b/cmd/yurthub/app/config/config.go index 9c12610301d..3459d8094b6 100644 --- a/cmd/yurthub/app/config/config.go +++ b/cmd/yurthub/app/config/config.go @@ -33,26 +33,30 @@ import ( // YurtHubConfiguration represents configuration of yurthub type YurtHubConfiguration struct { - LBMode string - RemoteServers []*url.URL - YurtHubServerAddr string - YurtHubProxyServerAddr string - YurtHubProxyServerDummyAddr string - GCFrequency int - CertMgrMode string - NodeName string - HeartbeatFailedRetry int - HeartbeatHealthyThreshold int - HeartbeatTimeoutSeconds int - MaxRequestInFlight int - JoinToken string - RootDir string - EnableProfiling bool - EnableDummyIf bool - EnableIptables bool - HubAgentDummyIfName string - StorageWrapper cachemanager.StorageWrapper - SerializerManager *serializer.SerializerManager + LBMode string + RemoteServers []*url.URL + YurtHubServerAddr string + YurtHubProxyServerAddr string + YurtHubProxyServerSecureAddr string + YurtHubProxyServerDummyAddr string + GCFrequency int + CertMgrMode string + NodeName string + HeartbeatFailedRetry int + HeartbeatHealthyThreshold int + HeartbeatTimeoutSeconds int + MaxRequestInFlight int + JoinToken string + RootDir string + EnableProfiling bool + EnableDummyIf bool + EnableIptables bool + HubAgentDummyIfName string + StorageWrapper cachemanager.StorageWrapper + SerializerManager *serializer.SerializerManager + CAFile string + CertFile string + KeyFile string } // Complete converts *options.YurtHubOptions to *YurtHubConfiguration @@ -72,28 +76,33 @@ func Complete(options *options.YurtHubOptions) (*YurtHubConfiguration, error) { hubServerAddr := net.JoinHostPort(options.YurtHubHost, options.YurtHubPort) proxyServerAddr := net.JoinHostPort(options.YurtHubHost, options.YurtHubProxyPort) + proxySecureServerAddr := net.JoinHostPort(options.YurtHubHost, options.YurtHubProxySecurePort) proxyServerDummyAddr := net.JoinHostPort(options.HubAgentDummyIfIP, options.YurtHubProxyPort) cfg := &YurtHubConfiguration{ - LBMode: options.LBMode, - RemoteServers: us, - YurtHubServerAddr: hubServerAddr, - YurtHubProxyServerAddr: proxyServerAddr, - YurtHubProxyServerDummyAddr: proxyServerDummyAddr, - GCFrequency: options.GCFrequency, - CertMgrMode: options.CertMgrMode, - NodeName: options.NodeName, - HeartbeatFailedRetry: options.HeartbeatFailedRetry, - HeartbeatHealthyThreshold: options.HeartbeatHealthyThreshold, - HeartbeatTimeoutSeconds: options.HeartbeatTimeoutSeconds, - MaxRequestInFlight: options.MaxRequestInFlight, - JoinToken: options.JoinToken, - RootDir: options.RootDir, - EnableProfiling: options.EnableProfiling, - EnableDummyIf: options.EnableDummyIf, - EnableIptables: options.EnableIptables, - HubAgentDummyIfName: options.HubAgentDummyIfName, - StorageWrapper: storageWrapper, - SerializerManager: serializerManager, + LBMode: options.LBMode, + RemoteServers: us, + YurtHubServerAddr: hubServerAddr, + YurtHubProxyServerAddr: proxyServerAddr, + YurtHubProxyServerSecureAddr: proxySecureServerAddr, + YurtHubProxyServerDummyAddr: proxyServerDummyAddr, + GCFrequency: options.GCFrequency, + CertMgrMode: options.CertMgrMode, + NodeName: options.NodeName, + HeartbeatFailedRetry: options.HeartbeatFailedRetry, + HeartbeatHealthyThreshold: options.HeartbeatHealthyThreshold, + HeartbeatTimeoutSeconds: options.HeartbeatTimeoutSeconds, + MaxRequestInFlight: options.MaxRequestInFlight, + JoinToken: options.JoinToken, + RootDir: options.RootDir, + EnableProfiling: options.EnableProfiling, + EnableDummyIf: options.EnableDummyIf, + EnableIptables: options.EnableIptables, + HubAgentDummyIfName: options.HubAgentDummyIfName, + StorageWrapper: storageWrapper, + SerializerManager: serializerManager, + CAFile: options.CAFile, + CertFile: options.CertFile, + KeyFile: options.KeyFile, } return cfg, nil diff --git a/cmd/yurthub/app/options/options.go b/cmd/yurthub/app/options/options.go index eb98f2121b5..1ab787bc2f8 100644 --- a/cmd/yurthub/app/options/options.go +++ b/cmd/yurthub/app/options/options.go @@ -38,6 +38,7 @@ type YurtHubOptions struct { YurtHubHost string YurtHubPort string YurtHubProxyPort string + YurtHubProxySecurePort string GCFrequency int CertMgrMode string NodeName string @@ -55,12 +56,16 @@ type YurtHubOptions struct { HubAgentDummyIfIP string HubAgentDummyIfName string DiskCachePath string + CAFile string + CertFile string + KeyFile string } // NewYurtHubOptions creates a new YurtHubOptions with a default config. func NewYurtHubOptions() *YurtHubOptions { o := &YurtHubOptions{ YurtHubHost: "127.0.0.1", + YurtHubProxySecurePort: "10260", YurtHubProxyPort: "10261", YurtHubPort: "10267", GCFrequency: 120, @@ -92,6 +97,18 @@ func ValidateOptions(options *YurtHubOptions) error { return fmt.Errorf("server-address is empty") } + if len(options.CAFile) == 0 { + return fmt.Errorf("CA is empty") + } + + if len(options.CertFile) == 0 { + return fmt.Errorf("tls cert is empty") + } + + if len(options.KeyFile) == 0 { + return fmt.Errorf("tls key is empty") + } + if !util.IsSupportedLBMode(options.LBMode) { return fmt.Errorf("lb mode(%s) is not supported", options.LBMode) } @@ -112,6 +129,7 @@ func (o *YurtHubOptions) AddFlags(fs *pflag.FlagSet) { fs.StringVar(&o.YurtHubHost, "bind-address", o.YurtHubHost, "the IP address on which to listen for the --serve-port port.") fs.StringVar(&o.YurtHubPort, "serve-port", o.YurtHubPort, "the port on which to serve HTTP requests(like profiling, metrics) for hub agent.") fs.StringVar(&o.YurtHubProxyPort, "proxy-port", o.YurtHubProxyPort, "the port on which to proxy HTTP requests to kube-apiserver") + fs.StringVar(&o.YurtHubProxySecurePort, "proxy-secure-port", o.YurtHubProxySecurePort, "the port on which to proxy HTTPS requests to kube-apiserver") fs.StringVar(&o.ServerAddr, "server-addr", o.ServerAddr, "the address of Kubernetes kube-apiserver,the format is: \"server1,server2,...\"") fs.StringVar(&o.CertMgrMode, "cert-mgr-mode", o.CertMgrMode, "the cert manager mode, kubelet: use certificates that belongs to kubelet, hubself: auto generate client cert for hub agent.") fs.IntVar(&o.GCFrequency, "gc-frequency", o.GCFrequency, "the frequency to gc cache in storage(unit: minute).") @@ -130,6 +148,9 @@ func (o *YurtHubOptions) AddFlags(fs *pflag.FlagSet) { fs.StringVar(&o.HubAgentDummyIfIP, "dummy-if-ip", o.HubAgentDummyIfIP, "the ip address of dummy interface that used for container connect hub agent(exclusive ips: 169.254.31.0/24, 169.254.1.1/32)") fs.StringVar(&o.HubAgentDummyIfName, "dummy-if-name", o.HubAgentDummyIfName, "the name of dummy interface that is used for hub agent") fs.StringVar(&o.DiskCachePath, "disk-cache-path", o.DiskCachePath, "the path for kubernetes to storage metadata") + fs.StringVar(&o.CAFile, "ca-file", "", "the CA for yurthub to verify client") + fs.StringVar(&o.CertFile, "tls-cert-file", "", "the tls cert of yurthub") + fs.StringVar(&o.KeyFile, "tls-private-key-file", "", "the tls key of yurthub") } // verifyDummyIP verify the specified ip is valid or not diff --git a/cmd/yurthub/app/start.go b/cmd/yurthub/app/start.go index 137dfe8cc13..9acf4be0528 100644 --- a/cmd/yurthub/app/start.go +++ b/cmd/yurthub/app/start.go @@ -147,7 +147,7 @@ func Run(cfg *config.YurtHubConfiguration, stopCh <-chan struct{}) error { klog.Infof("%d. new %s server and begin to serve, dummy proxy server: %s", trace, projectinfo.GetHubName(), cfg.YurtHubProxyServerDummyAddr) } - klog.Infof("%d. new %s server and begin to serve, proxy server: %s, hub server: %s", trace, projectinfo.GetHubName(), cfg.YurtHubProxyServerAddr, cfg.YurtHubServerAddr) + klog.Infof("%d. new %s server and begin to serve, proxy server: %s, proxy secure server: %s, hub server: %s", trace, projectinfo.GetHubName(), cfg.YurtHubProxyServerAddr, cfg.YurtHubProxyServerSecureAddr, cfg.YurtHubServerAddr) s, err := server.NewYurtHubServer(cfg, certManager, yurtProxyHandler) if err != nil { klog.Errorf("could not create hub server, %v", err) diff --git a/pkg/yurthub/server/server.go b/pkg/yurthub/server/server.go index 4753966062f..bd4d79cf35f 100644 --- a/pkg/yurthub/server/server.go +++ b/pkg/yurthub/server/server.go @@ -17,7 +17,10 @@ limitations under the License. package server import ( + "crypto/tls" + "crypto/x509" "fmt" + "io/ioutil" "net" "net/http" @@ -27,6 +30,7 @@ import ( "github.com/gorilla/mux" "github.com/prometheus/client_golang/prometheus/promhttp" + "k8s.io/klog" ) // Server is an interface for providing http service for yurthub @@ -38,9 +42,16 @@ type Server interface { // and hubServer handles requests by hub agent itself, like profiling, metrics, healthz // and proxyServer does not handle requests locally and proxy requests to kube-apiserver type yurtHubServer struct { - hubServer *http.Server - proxyServer *http.Server - dummyProxyServer *http.Server + CAFile string + // CertFile the tls cert file for proxyhub's https server + CertFile string + // KeyFile the tls key file for proxyhub's https server + KeyFile string + + hubServer *http.Server + proxyServer *http.Server + secureProxyServer *http.Server + dummyProxyServer *http.Server } // NewYurtHubServer creates a Server object @@ -56,8 +67,25 @@ func NewYurtHubServer(cfg *config.YurtHubConfiguration, } proxyServer := &http.Server{ - Addr: cfg.YurtHubProxyServerAddr, - Handler: proxyHandler, + Addr: cfg.YurtHubProxyServerAddr, + Handler: proxyHandler, + } + + caFile, err := ioutil.ReadFile(cfg.CAFile) + if err != nil { + klog.Errorf("Read ca file err: %v", err) + return nil, err + } + certPool := x509.NewCertPool() + certPool.AppendCertsFromPEM([]byte(caFile)) + + secureProxyServer := &http.Server{ + Addr: cfg.YurtHubProxyServerSecureAddr, + Handler: proxyHandler, + TLSConfig: &tls.Config{ + ClientCAs: certPool, + ClientAuth: tls.VerifyClientCertIfGiven, + }, MaxHeaderBytes: 1 << 20, } @@ -75,9 +103,13 @@ func NewYurtHubServer(cfg *config.YurtHubConfiguration, } return &yurtHubServer{ - hubServer: hubServer, - proxyServer: proxyServer, - dummyProxyServer: dummyProxyServer, + CAFile: cfg.CAFile, + CertFile: cfg.CertFile, + KeyFile: cfg.KeyFile, + hubServer: hubServer, + proxyServer: proxyServer, + secureProxyServer: secureProxyServer, + dummyProxyServer: dummyProxyServer, }, nil } @@ -99,6 +131,13 @@ func (s *yurtHubServer) Run() { }() } + go func() { + err := s.secureProxyServer.ListenAndServeTLS(s.CertFile, s.KeyFile) + if err != nil { + panic(err) + } + }() + err := s.proxyServer.ListenAndServe() if err != nil { panic(err)