From 3e694686da0a1e724f7e2fdd0bdbbca15e8e0aee Mon Sep 17 00:00:00 2001 From: Yifei Zhang Date: Wed, 7 Dec 2022 19:36:33 +0800 Subject: [PATCH] pool-coordinator implementation of yurthub (#1073) * pool-coordinator implementation of yurthub Signed-off-by: Congrool --- cmd/yurthub/app/config/config.go | 22 +- cmd/yurthub/app/options/options.go | 83 +-- cmd/yurthub/app/start.go | 87 ++- cmd/yurthub/yurthub.go | 2 +- go.mod | 5 +- pkg/yurthub/cachemanager/cache_manager.go | 27 +- .../cachemanager/cache_manager_test.go | 56 ++ pkg/yurthub/cachemanager/storage_wrapper.go | 5 + pkg/yurthub/healthchecker/health_checker.go | 6 +- .../healthchecker/health_checker_test.go | 2 +- .../poolcoordinator/constants/constants.go | 39 ++ pkg/yurthub/poolcoordinator/coordinator.go | 611 ++++++++++++++++-- pkg/yurthub/poolcoordinator/informer_lease.go | 178 +++++ .../poolcoordinator/leader_election.go | 19 +- pkg/yurthub/proxy/local/local.go | 24 +- pkg/yurthub/proxy/local/local_test.go | 14 +- pkg/yurthub/proxy/pool/pool.go | 256 ++++++++ pkg/yurthub/proxy/proxy.go | 168 ++++- pkg/yurthub/proxy/remote/loadbalancer.go | 276 +++++++- pkg/yurthub/proxy/remote/loadbalancer_test.go | 93 ++- pkg/yurthub/proxy/{remote => util}/remote.go | 126 +--- pkg/yurthub/proxy/util/util.go | 68 ++ pkg/yurthub/storage/disk/storage.go | 2 +- pkg/yurthub/storage/etcd/etcd_suite_test.go | 58 ++ pkg/yurthub/storage/etcd/key.go | 78 +++ pkg/yurthub/storage/etcd/key_test.go | 103 +++ pkg/yurthub/storage/etcd/keycache.go | 280 ++++++++ pkg/yurthub/storage/etcd/keycache_test.go | 311 +++++++++ pkg/yurthub/storage/etcd/storage.go | 500 ++++++++++++++ pkg/yurthub/storage/etcd/storage_test.go | 548 ++++++++++++++++ pkg/yurthub/storage/store.go | 5 +- pkg/yurthub/storage/utils/validate.go | 6 +- pkg/yurthub/util/util.go | 93 +++ 33 files changed, 3813 insertions(+), 338 deletions(-) create mode 100644 pkg/yurthub/poolcoordinator/constants/constants.go create mode 100644 pkg/yurthub/poolcoordinator/informer_lease.go create mode 100644 pkg/yurthub/proxy/pool/pool.go rename pkg/yurthub/proxy/{remote => util}/remote.go (50%) create mode 100644 pkg/yurthub/storage/etcd/etcd_suite_test.go create mode 100644 pkg/yurthub/storage/etcd/key.go create mode 100644 pkg/yurthub/storage/etcd/key_test.go create mode 100644 pkg/yurthub/storage/etcd/keycache.go create mode 100644 pkg/yurthub/storage/etcd/keycache_test.go create mode 100644 pkg/yurthub/storage/etcd/storage.go create mode 100644 pkg/yurthub/storage/etcd/storage_test.go diff --git a/cmd/yurthub/app/config/config.go b/cmd/yurthub/app/config/config.go index 984cb3c224c..f4b20c50120 100644 --- a/cmd/yurthub/app/config/config.go +++ b/cmd/yurthub/app/config/config.go @@ -23,8 +23,6 @@ import ( "strings" "time" - componentbaseconfig "k8s.io/component-base/config" - corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/fields" @@ -40,6 +38,7 @@ import ( core "k8s.io/client-go/testing" "k8s.io/client-go/tools/cache" "k8s.io/client-go/tools/clientcmd" + componentbaseconfig "k8s.io/component-base/config" "k8s.io/klog/v2" "github.com/openyurtio/openyurt/cmd/yurthub/app/options" @@ -91,6 +90,17 @@ type YurtHubConfiguration struct { YurtHubProxyServerServing *apiserver.DeprecatedInsecureServingInfo YurtHubDummyProxyServerServing *apiserver.DeprecatedInsecureServingInfo YurtHubSecureProxyServerServing *apiserver.SecureServingInfo + YurtHubProxyServerAddr string + DiskCachePath string + CoordinatorPKIDir string + EnableCoordinator bool + CoordinatorServerURL *url.URL + CoordinatorStoragePrefix string + CoordinatorStorageAddr string // ip:port + CoordinatorStorageCaFile string + CoordinatorStorageCertFile string + CoordinatorStorageKeyFile string + CoordinatorClient kubernetes.Interface LeaderElection componentbaseconfig.LeaderElectionConfiguration } @@ -148,6 +158,14 @@ func Complete(options *options.YurtHubOptions) (*YurtHubConfiguration, error) { FilterManager: filterManager, MinRequestTimeout: options.MinRequestTimeout, TenantNs: tenantNs, + YurtHubProxyServerAddr: fmt.Sprintf("http://%s:%d", options.YurtHubProxyHost, options.YurtHubProxyPort), + DiskCachePath: options.DiskCachePath, + CoordinatorPKIDir: filepath.Join(options.RootDir, "poolcoordinator"), + EnableCoordinator: options.EnableCoordinator, + CoordinatorServerURL: coordinatorServerURL, + CoordinatorStoragePrefix: options.CoordinatorStoragePrefix, + CoordinatorStorageAddr: options.CoordinatorStorageAddr, + LeaderElection: options.LeaderElection, } certMgr, err := createCertManager(options, us) diff --git a/cmd/yurthub/app/options/options.go b/cmd/yurthub/app/options/options.go index 508a7245a98..e7d4fa16ade 100644 --- a/cmd/yurthub/app/options/options.go +++ b/cmd/yurthub/app/options/options.go @@ -44,42 +44,47 @@ const ( // YurtHubOptions is the main settings for the yurthub type YurtHubOptions struct { - ServerAddr string - YurtHubHost string // YurtHub server host (e.g.: expose metrics API) - YurtHubProxyHost string // YurtHub proxy server host - YurtHubPort int - YurtHubProxyPort int - YurtHubProxySecurePort int - GCFrequency int - YurtHubCertOrganizations []string - NodeName string - NodePoolName string - LBMode string - HeartbeatFailedRetry int - HeartbeatHealthyThreshold int - HeartbeatTimeoutSeconds int - HeartbeatIntervalSeconds int - MaxRequestInFlight int - JoinToken string - RootDir string - Version bool - EnableProfiling bool - EnableDummyIf bool - EnableIptables bool - HubAgentDummyIfIP string - HubAgentDummyIfName string - DiskCachePath string - AccessServerThroughHub bool - EnableResourceFilter bool - DisabledResourceFilters []string - WorkingMode string - KubeletHealthGracePeriod time.Duration - EnableNodePool bool - MinRequestTimeout time.Duration - CACertHashes []string - UnsafeSkipCAVerification bool - ClientForTest kubernetes.Interface - LeaderElection componentbaseconfig.LeaderElectionConfiguration + ServerAddr string + YurtHubHost string // YurtHub server host (e.g.: expose metrics API) + YurtHubProxyHost string // YurtHub proxy server host + YurtHubPort int + YurtHubProxyPort int + YurtHubProxySecurePort int + GCFrequency int + YurtHubCertOrganizations []string + NodeName string + NodePoolName string + LBMode string + HeartbeatFailedRetry int + HeartbeatHealthyThreshold int + HeartbeatTimeoutSeconds int + HeartbeatIntervalSeconds int + MaxRequestInFlight int + JoinToken string + RootDir string + Version bool + EnableProfiling bool + EnableDummyIf bool + EnableIptables bool + HubAgentDummyIfIP string + HubAgentDummyIfName string + DiskCachePath string + AccessServerThroughHub bool + EnableResourceFilter bool + DisabledResourceFilters []string + WorkingMode string + KubeletHealthGracePeriod time.Duration + EnableNodePool bool + MinRequestTimeout time.Duration + CACertHashes []string + UnsafeSkipCAVerification bool + ClientForTest kubernetes.Interface + CoordinatorStoragePrefix string + CoordinatorStorageAddr string + CoordinatorStorageCaFile string + CoordinatorStorageCertFile string + CoordinatorStorageKeyFile string + LeaderElection componentbaseconfig.LeaderElectionConfiguration } // NewYurtHubOptions creates a new YurtHubOptions with a default config. @@ -113,6 +118,7 @@ func NewYurtHubOptions() *YurtHubOptions { MinRequestTimeout: time.Second * 1800, CACertHashes: make([]string, 0), UnsafeSkipCAVerification: true, + CoordinatorStoragePrefix: "/registry", LeaderElection: componentbaseconfig.LeaderElectionConfiguration{ LeaderElect: true, LeaseDuration: metav1.Duration{Duration: 15 * time.Second}, @@ -195,6 +201,11 @@ func (o *YurtHubOptions) AddFlags(fs *pflag.FlagSet) { fs.DurationVar(&o.MinRequestTimeout, "min-request-timeout", o.MinRequestTimeout, "An optional field indicating at least how long a proxy handler must keep a request open before timing it out. Currently only honored by the local watch request handler(use request parameter timeoutSeconds firstly), which picks a randomized value above this number as the connection timeout, to spread out load.") fs.StringSliceVar(&o.CACertHashes, "discovery-token-ca-cert-hash", o.CACertHashes, "For token-based discovery, validate that the root CA public key matches this hash (format: \":\").") fs.BoolVar(&o.UnsafeSkipCAVerification, "discovery-token-unsafe-skip-ca-verification", o.UnsafeSkipCAVerification, "For token-based discovery, allow joining without --discovery-token-ca-cert-hash pinning.") + fs.StringVar(&o.CoordinatorStoragePrefix, "coordinator-storage-prefix", o.CoordinatorStoragePrefix, "Pool-Coordinator etcd storage prefix, same as etcd-prefix of Kube-APIServer") + fs.StringVar(&o.CoordinatorStorageAddr, "coordinator-storage-addr", o.CoordinatorStorageAddr, "Address of Pool-Coordinator etcd, in the format ip:port") + fs.StringVar(&o.CoordinatorStorageCaFile, "coordinator-storage-ca", o.CoordinatorStorageCaFile, "CA file path to communicate with Pool-Coordinator etcd") + fs.StringVar(&o.CoordinatorStorageCertFile, "coordinator-storage-cert", o.CoordinatorStorageCertFile, "Cert file path to communicate with Pool-Coordinator etcd") + fs.StringVar(&o.CoordinatorStorageKeyFile, "coordinator-storage-key", o.CoordinatorStorageKeyFile, "Key file path to communicate with Pool-Coordinator etcd") bindFlags(&o.LeaderElection, fs) } diff --git a/cmd/yurthub/app/start.go b/cmd/yurthub/app/start.go index 548999b1082..00b97b1baff 100644 --- a/cmd/yurthub/app/start.go +++ b/cmd/yurthub/app/start.go @@ -17,12 +17,14 @@ limitations under the License. package app import ( + "context" "fmt" "net/url" "time" "github.com/spf13/cobra" "github.com/spf13/pflag" + "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" "k8s.io/klog/v2" @@ -34,6 +36,7 @@ import ( "github.com/openyurtio/openyurt/pkg/yurthub/gc" "github.com/openyurtio/openyurt/pkg/yurthub/healthchecker" hubrest "github.com/openyurtio/openyurt/pkg/yurthub/kubernetes/rest" + "github.com/openyurtio/openyurt/pkg/yurthub/poolcoordinator" "github.com/openyurtio/openyurt/pkg/yurthub/proxy" "github.com/openyurtio/openyurt/pkg/yurthub/server" "github.com/openyurtio/openyurt/pkg/yurthub/tenant" @@ -42,7 +45,7 @@ import ( ) // NewCmdStartYurtHub creates a *cobra.Command object with default parameters -func NewCmdStartYurtHub(stopCh <-chan struct{}) *cobra.Command { +func NewCmdStartYurtHub(ctx context.Context) *cobra.Command { yurtHubOptions := options.NewYurtHubOptions() cmd := &cobra.Command{ @@ -69,7 +72,7 @@ func NewCmdStartYurtHub(stopCh <-chan struct{}) *cobra.Command { } klog.Infof("%s cfg: %#+v", projectinfo.GetHubName(), yurtHubCfg) - if err := Run(yurtHubCfg, stopCh); err != nil { + if err := Run(ctx, yurtHubCfg); err != nil { klog.Fatalf("run %s failed, %v", projectinfo.GetHubName(), err) } }, @@ -80,40 +83,47 @@ func NewCmdStartYurtHub(stopCh <-chan struct{}) *cobra.Command { } // Run runs the YurtHubConfiguration. This should never exit -func Run(cfg *config.YurtHubConfiguration, stopCh <-chan struct{}) error { +func Run(ctx context.Context, cfg *config.YurtHubConfiguration) error { defer cfg.CertManager.Stop() trace := 1 klog.Infof("%d. new transport manager", trace) - transportManager, err := transport.NewTransportManager(cfg.CertManager, stopCh) + transportManager, err := transport.NewTransportManager(cfg.CertManager, ctx.Done()) if err != nil { return fmt.Errorf("could not new transport manager, %w", err) } trace++ klog.Infof("%d. prepare for health checker clients", trace) - healthCheckerClientsForCloud, _, err := createHealthCheckerClient(cfg.HeartbeatTimeoutSeconds, cfg.RemoteServers, cfg.CoordinatorServer, transportManager) + cloudClients, coordinatorClient, err := createClients(cfg.HeartbeatTimeoutSeconds, cfg.RemoteServers, cfg.CoordinatorServer, transportManager) if err != nil { return fmt.Errorf("failed to create health checker clients, %w", err) } trace++ - var healthChecker healthchecker.MultipleBackendsHealthChecker + var cloudHealthChecker healthchecker.MultipleBackendsHealthChecker + var coordinatorHealthChecker healthchecker.HealthChecker if cfg.WorkingMode == util.WorkingModeEdge { - klog.Infof("%d. create health checker for remote servers ", trace) - healthChecker, err = healthchecker.NewCloudAPIServerHealthChecker(cfg, healthCheckerClientsForCloud, stopCh) + klog.Infof("%d. create health checkers for remote servers and pool coordinator", trace) + cloudHealthChecker, err = healthchecker.NewCloudAPIServerHealthChecker(cfg, cloudClients, ctx.Done()) if err != nil { - return fmt.Errorf("could not new health checker, %w", err) + return fmt.Errorf("could not new cloud health checker, %w", err) } + coordinatorHealthChecker, err = healthchecker.NewCoordinatorHealthChecker(cfg, coordinatorClient, cloudHealthChecker, ctx.Done()) + if err != nil { + return fmt.Errorf("failed to create coordinator health checker, %v", err) + } + } else { klog.Infof("%d. disable health checker for node %s because it is a cloud node", trace, cfg.NodeName) - // In cloud mode, health checker is not needed. - // This fake checker will always report that the remote server is healthy. - healthChecker = healthchecker.NewFakeChecker(true, make(map[string]int)) + // In cloud mode, cloud health checker and pool coordinator health checker are not needed. + // This fake checker will always report that the cloud is healthy and pool coordinator is unhealthy. + cloudHealthChecker = healthchecker.NewFakeChecker(true, make(map[string]int)) + coordinatorHealthChecker = healthchecker.NewFakeChecker(false, make(map[string]int)) } trace++ klog.Infof("%d. new restConfig manager", trace) - restConfigMgr, err := hubrest.NewRestConfigManager(cfg.CertManager, healthChecker) + restConfigMgr, err := hubrest.NewRestConfigManager(cfg.CertManager, cloudHealthChecker) if err != nil { return fmt.Errorf("could not new restConfig manager, %w", err) } @@ -130,7 +140,7 @@ func Run(cfg *config.YurtHubConfiguration, stopCh <-chan struct{}) error { if cfg.WorkingMode == util.WorkingModeEdge { klog.Infof("%d. new gc manager for node %s, and gc frequency is a random time between %d min and %d min", trace, cfg.NodeName, cfg.GCFrequency, 3*cfg.GCFrequency) - gcMgr, err := gc.NewGCManager(cfg, restConfigMgr, stopCh) + gcMgr, err := gc.NewGCManager(cfg, restConfigMgr, ctx.Done()) if err != nil { return fmt.Errorf("could not new gc manager, %w", err) } @@ -141,36 +151,55 @@ func Run(cfg *config.YurtHubConfiguration, stopCh <-chan struct{}) error { trace++ klog.Infof("%d. new tenant sa manager", trace) - tenantMgr := tenant.New(cfg.TenantNs, cfg.SharedFactory, stopCh) + tenantMgr := tenant.New(cfg.TenantNs, cfg.SharedFactory, ctx.Done()) + trace++ + + klog.Infof("%d. create yurthub elector", trace) + elector, err := poolcoordinator.NewHubElector(cfg, coordinatorClient, coordinatorHealthChecker, cloudHealthChecker, ctx.Done()) + if err != nil { + klog.Errorf("failed to create hub elector, %v", err) + } + elector.Run(ctx.Done()) + trace++ + + // TODO: cloud client load balance + klog.Infof("%d. create coordinator", trace) + coordinator, err := poolcoordinator.NewCoordinator(ctx, cfg, restConfigMgr, transportManager, elector) + if err != nil { + klog.Errorf("failed to create coordinator, %v", err) + } + coordinator.Run() trace++ klog.Infof("%d. new reverse proxy handler for remote servers", trace) - yurtProxyHandler, err := proxy.NewYurtReverseProxyHandler(cfg, cacheMgr, transportManager, healthChecker, tenantMgr, stopCh) + yurtProxyHandler, err := proxy.NewYurtReverseProxyHandler(cfg, cacheMgr, transportManager, coordinator, cloudHealthChecker, coordinatorHealthChecker, tenantMgr, ctx.Done()) if err != nil { return fmt.Errorf("could not create reverse proxy handler, %w", err) } trace++ if cfg.NetworkMgr != nil { - cfg.NetworkMgr.Run(stopCh) + cfg.NetworkMgr.Run(ctx.Done()) } // start shared informers before start hub server - cfg.SharedFactory.Start(stopCh) - cfg.YurtSharedFactory.Start(stopCh) + cfg.SharedFactory.Start(ctx.Done()) + cfg.YurtSharedFactory.Start(ctx.Done()) klog.Infof("%d. new %s server and begin to serve", trace, projectinfo.GetHubName()) - if err := server.RunYurtHubServers(cfg, yurtProxyHandler, restConfigMgr, stopCh); err != nil { + if err := server.RunYurtHubServers(cfg, yurtProxyHandler, restConfigMgr, ctx.Done()); err != nil { return fmt.Errorf("could not run hub servers, %w", err) } - <-stopCh + <-ctx.Done() klog.Infof("hub agent exited") return nil } -func createHealthCheckerClient(heartbeatTimeoutSeconds int, remoteServers []*url.URL, coordinatorServer *url.URL, tp transport.Interface) (map[string]kubernetes.Interface, kubernetes.Interface, error) { - var healthCheckerClientForCoordinator kubernetes.Interface - healthCheckerClientsForCloud := make(map[string]kubernetes.Interface) +// createClients will create clients for all cloud APIServer and client for pool coordinator +// It will return a map, mapping cloud APIServer URL to its client, and a pool coordinator client +func createClients(heartbeatTimeoutSeconds int, remoteServers []*url.URL, coordinatorServer *url.URL, tp transport.Interface) (map[string]kubernetes.Interface, kubernetes.Interface, error) { + var coordinatorClient kubernetes.Interface + cloudClients := make(map[string]kubernetes.Interface) for i := range remoteServers { restConf := &rest.Config{ Host: remoteServers[i].String(), @@ -179,9 +208,9 @@ func createHealthCheckerClient(heartbeatTimeoutSeconds int, remoteServers []*url } c, err := kubernetes.NewForConfig(restConf) if err != nil { - return healthCheckerClientsForCloud, healthCheckerClientForCoordinator, err + return cloudClients, coordinatorClient, err } - healthCheckerClientsForCloud[remoteServers[i].String()] = c + cloudClients[remoteServers[i].String()] = c } cfg := &rest.Config{ @@ -191,9 +220,9 @@ func createHealthCheckerClient(heartbeatTimeoutSeconds int, remoteServers []*url } c, err := kubernetes.NewForConfig(cfg) if err != nil { - return healthCheckerClientsForCloud, healthCheckerClientForCoordinator, err + return cloudClients, coordinatorClient, err } - healthCheckerClientForCoordinator = c + coordinatorClient = c - return healthCheckerClientsForCloud, healthCheckerClientForCoordinator, nil + return cloudClients, coordinatorClient, nil } diff --git a/cmd/yurthub/yurthub.go b/cmd/yurthub/yurthub.go index fea2aa43148..439bb09339c 100644 --- a/cmd/yurthub/yurthub.go +++ b/cmd/yurthub/yurthub.go @@ -28,7 +28,7 @@ import ( func main() { rand.Seed(time.Now().UnixNano()) - cmd := app.NewCmdStartYurtHub(server.SetupSignalHandler()) + cmd := app.NewCmdStartYurtHub(server.SetupSignalContext()) cmd.Flags().AddGoFlagSet(flag.CommandLine) if err := cmd.Execute(); err != nil { panic(err) diff --git a/go.mod b/go.mod index b59d9a1d095..46ea2c50d24 100644 --- a/go.mod +++ b/go.mod @@ -26,7 +26,10 @@ require ( github.com/spf13/pflag v1.0.5 github.com/stretchr/testify v1.7.0 github.com/vishvananda/netlink v1.1.1-0.20200603190939-5a869a71f0cb - golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a + github.com/wI2L/jsondiff v0.3.0 + go.etcd.io/etcd/client/pkg/v3 v3.5.0 + go.etcd.io/etcd/client/v3 v3.5.0 + golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8 google.golang.org/grpc v1.40.0 gopkg.in/cheggaaa/pb.v1 v1.0.25 gopkg.in/square/go-jose.v2 v2.2.2 diff --git a/pkg/yurthub/cachemanager/cache_manager.go b/pkg/yurthub/cachemanager/cache_manager.go index 449b4da5603..021b76fa6bb 100644 --- a/pkg/yurthub/cachemanager/cache_manager.go +++ b/pkg/yurthub/cachemanager/cache_manager.go @@ -31,6 +31,8 @@ import ( v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/meta" + metainternalversion "k8s.io/apimachinery/pkg/apis/meta/internalversion" + metainternalversionscheme "k8s.io/apimachinery/pkg/apis/meta/internalversion/scheme" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" @@ -44,7 +46,6 @@ import ( hubmeta "github.com/openyurtio/openyurt/pkg/yurthub/kubernetes/meta" "github.com/openyurtio/openyurt/pkg/yurthub/kubernetes/serializer" - proxyutil "github.com/openyurtio/openyurt/pkg/yurthub/proxy/util" "github.com/openyurtio/openyurt/pkg/yurthub/storage" "github.com/openyurtio/openyurt/pkg/yurthub/util" ) @@ -68,7 +69,6 @@ type cacheManager struct { restMapperManager *hubmeta.RESTMapperManager cacheAgents *CacheAgent listSelectorCollector map[storage.Key]string - sharedFactory informers.SharedInformerFactory inMemoryCache map[string]runtime.Object } @@ -86,7 +86,6 @@ func NewCacheManager( cacheAgents: cacheAgents, restMapperManager: restMapperMgr, listSelectorCollector: make(map[storage.Key]string), - sharedFactory: sharedFactory, inMemoryCache: make(map[string]runtime.Object), } @@ -181,7 +180,7 @@ func (cm *cacheManager) queryListObject(req *http.Request) (runtime.Object, erro } objs, err := cm.storage.List(key) - if err == storage.ErrStorageNotFound && proxyutil.IsListRequestWithNameFieldSelector(req) { + if err == storage.ErrStorageNotFound && isListRequestWithNameFieldSelector(req) { // When the request is a list request with FieldSelector "metadata.name", we should not return error // when the specified resource is not found return an empty list object, to keep same as APIServer. return listObj, nil @@ -858,3 +857,23 @@ func inMemoryCacheKeyFunc(reqInfo *apirequest.RequestInfo) (string, error) { key := filepath.Join(res, ns, name) return key, nil } + +// isListRequestWithNameFieldSelector will check if the request has FieldSelector "metadata.name". +// If found, return true, otherwise false. +func isListRequestWithNameFieldSelector(req *http.Request) bool { + ctx := req.Context() + if info, ok := apirequest.RequestInfoFrom(ctx); ok { + if info.IsResourceRequest && info.Verb == "list" { + opts := metainternalversion.ListOptions{} + if err := metainternalversionscheme.ParameterCodec.DecodeParameters(req.URL.Query(), metav1.SchemeGroupVersion, &opts); err == nil { + if opts.FieldSelector == nil { + return false + } + if _, found := opts.FieldSelector.RequiresExactMatch("metadata.name"); found { + return true + } + } + } + } + return false +} diff --git a/pkg/yurthub/cachemanager/cache_manager_test.go b/pkg/yurthub/cachemanager/cache_manager_test.go index a0c3adfd8cb..e4f8a251aac 100644 --- a/pkg/yurthub/cachemanager/cache_manager_test.go +++ b/pkg/yurthub/cachemanager/cache_manager_test.go @@ -3275,4 +3275,60 @@ func newTestRequestInfoResolver() *request.RequestInfoFactory { } } +func TestIsListRequestWithNameFieldSelector(t *testing.T) { + testcases := map[string]struct { + Verb string + Path string + Expect bool + }{ + "request has metadata.name fieldSelector": { + Verb: "GET", + Path: "/api/v1/namespaces/kube-system/pods?resourceVersion=1494416105&fieldSelector=metadata.name=test", + Expect: true, + }, + "request has no metadata.name fieldSelector": { + Verb: "GET", + Path: "/api/v1/namespaces/kube-system/pods?resourceVersion=1494416105&fieldSelector=spec.nodeName=test", + Expect: false, + }, + "request only has labelSelector": { + Verb: "GET", + Path: "/api/v1/namespaces/kube-system/pods?resourceVersion=1494416105&labelSelector=foo=bar", + Expect: false, + }, + "request has both labelSelector and fieldSelector and fieldSelector has metadata.name": { + Verb: "GET", + Path: "/api/v1/namespaces/kube-system/pods?fieldSelector=metadata.name=test&labelSelector=foo=bar", + Expect: true, + }, + "request has both labelSelector and fieldSelector but fieldSelector has no metadata.name": { + Verb: "GET", + Path: "/api/v1/namespaces/kube-system/pods?fieldSelector=spec.nodeName=test&labelSelector=foo=bar", + Expect: false, + }, + } + + resolver := newTestRequestInfoResolver() + + for k, tc := range testcases { + t.Run(k, func(t *testing.T) { + req, _ := http.NewRequest(tc.Verb, tc.Path, nil) + req.RemoteAddr = "127.0.0.1" + + var isMetadataNameFieldSelector bool + var handler http.Handler = http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + isMetadataNameFieldSelector = isListRequestWithNameFieldSelector(req) + }) + + handler = proxyutil.WithListRequestSelector(handler) + handler = filters.WithRequestInfo(handler, resolver) + handler.ServeHTTP(httptest.NewRecorder(), req) + + if isMetadataNameFieldSelector != tc.Expect { + t.Errorf("failed at case %s, want: %v, got: %v", k, tc.Expect, isMetadataNameFieldSelector) + } + }) + } +} + // TODO: in-memory cache unit tests diff --git a/pkg/yurthub/cachemanager/storage_wrapper.go b/pkg/yurthub/cachemanager/storage_wrapper.go index d992ccde61d..f9c8db77e7e 100644 --- a/pkg/yurthub/cachemanager/storage_wrapper.go +++ b/pkg/yurthub/cachemanager/storage_wrapper.go @@ -46,6 +46,7 @@ type StorageWrapper interface { DeleteComponentResources(component string) error SaveClusterInfo(key storage.ClusterInfoKey, content []byte) error GetClusterInfo(key storage.ClusterInfoKey) ([]byte, error) + GetStorage() storage.Store } type storageWrapper struct { @@ -70,6 +71,10 @@ func (sw *storageWrapper) KeyFunc(info storage.KeyBuildInfo) (storage.Key, error return sw.store.KeyFunc(info) } +func (sw *storageWrapper) GetStorage() storage.Store { + return sw.store +} + // Create store runtime object into backend storage // if obj is nil, the storage used to represent the key // will be created. for example: for disk storage, diff --git a/pkg/yurthub/healthchecker/health_checker.go b/pkg/yurthub/healthchecker/health_checker.go index eb78ac7e8f9..855720b489c 100644 --- a/pkg/yurthub/healthchecker/health_checker.go +++ b/pkg/yurthub/healthchecker/health_checker.go @@ -34,7 +34,7 @@ import ( ) const ( - delegateHeartBeat = "openyurt.io/delegate-heartbeat" + DelegateHeartBeat = "openyurt.io/delegate-heartbeat" ) type setNodeLease func(*coordinationv1.Lease) error @@ -114,9 +114,9 @@ func (chc *coordinatorHealthChecker) getLastNodeLease() *coordinationv1.Lease { if chc.latestLease.Annotations == nil { chc.latestLease.Annotations = make(map[string]string) } - chc.latestLease.Annotations[delegateHeartBeat] = "true" + chc.latestLease.Annotations[DelegateHeartBeat] = "true" } else { - delete(chc.latestLease.Annotations, delegateHeartBeat) + delete(chc.latestLease.Annotations, DelegateHeartBeat) } } diff --git a/pkg/yurthub/healthchecker/health_checker_test.go b/pkg/yurthub/healthchecker/health_checker_test.go index 85659c32b89..ba82ffbaf06 100644 --- a/pkg/yurthub/healthchecker/health_checker_test.go +++ b/pkg/yurthub/healthchecker/health_checker_test.go @@ -210,7 +210,7 @@ func TestNewCoordinatorHealthChecker(t *testing.T) { if tt.cloudAPIServerUnhealthy { if delegateLease == nil || len(delegateLease.Annotations) == 0 { t.Errorf("expect delegate heartbeat annotaion, but got nil") - } else if v, ok := delegateLease.Annotations[delegateHeartBeat]; !ok || v != "true" { + } else if v, ok := delegateLease.Annotations[DelegateHeartBeat]; !ok || v != "true" { t.Errorf("expect delegate heartbeat annotaion and v is true, but got empty or %v", v) } } diff --git a/pkg/yurthub/poolcoordinator/constants/constants.go b/pkg/yurthub/poolcoordinator/constants/constants.go new file mode 100644 index 00000000000..bcd06c86e76 --- /dev/null +++ b/pkg/yurthub/poolcoordinator/constants/constants.go @@ -0,0 +1,39 @@ +/* +Copyright 2022 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package constants + +import ( + "k8s.io/apimachinery/pkg/runtime/schema" + + "github.com/openyurtio/openyurt/pkg/yurthub/storage" +) + +var ( + PoolScopedResources = map[schema.GroupVersionResource]struct{}{ + {Group: "", Version: "v1", Resource: "endpoints"}: {}, + {Group: "discovery.k8s.io", Version: "v1", Resource: "endpointslices"}: {}, + } + + UploadResourcesKeyBuildInfo = map[storage.KeyBuildInfo]struct{}{ + {Component: "kubelet", Resources: "pods", Group: "", Version: "v1"}: {}, + {Component: "kubelet", Resources: "nodes", Group: "", Version: "v1"}: {}, + } +) + +const ( + DefaultPoolScopedUserAgent = "leader-yurthub" +) diff --git a/pkg/yurthub/poolcoordinator/coordinator.go b/pkg/yurthub/poolcoordinator/coordinator.go index 6afdc8ffd84..ab3e8763fcb 100644 --- a/pkg/yurthub/poolcoordinator/coordinator.go +++ b/pkg/yurthub/poolcoordinator/coordinator.go @@ -17,73 +17,610 @@ limitations under the License. package poolcoordinator import ( + "context" + "encoding/json" + "fmt" + "strconv" + "sync" + "time" + + coordinationv1 "k8s.io/api/coordination/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/informers" + "k8s.io/client-go/kubernetes" + coordclientset "k8s.io/client-go/kubernetes/typed/coordination/v1" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/cache" + "k8s.io/client-go/tools/clientcmd" "k8s.io/klog/v2" + "github.com/openyurtio/openyurt/cmd/yurthub/app/config" + "github.com/openyurtio/openyurt/pkg/yurthub/cachemanager" "github.com/openyurtio/openyurt/pkg/yurthub/healthchecker" + "github.com/openyurtio/openyurt/pkg/yurthub/kubernetes/meta" + yurtrest "github.com/openyurtio/openyurt/pkg/yurthub/kubernetes/rest" + "github.com/openyurtio/openyurt/pkg/yurthub/kubernetes/serializer" + "github.com/openyurtio/openyurt/pkg/yurthub/poolcoordinator/constants" + "github.com/openyurtio/openyurt/pkg/yurthub/storage" + "github.com/openyurtio/openyurt/pkg/yurthub/storage/etcd" + "github.com/openyurtio/openyurt/pkg/yurthub/transport" +) + +const ( + leaseDelegateRetryTimes = 5 + defaultInformerLeaseRenewDuration = 10 * time.Second + defaultPoolCacheStaleDuration = 30 * time.Second + namespaceInformerLease = "kube-system" + nameInformerLease = "leader-informer-sync" ) type Coordinator struct { - coordinatorHealthChecker healthchecker.HealthChecker - hubElector *HubElector - informerStarted bool + sync.Mutex + ctx context.Context + cancelEtcdStorage func() + informerFactory informers.SharedInformerFactory + restMapperMgr *meta.RESTMapperManager + serializerMgr *serializer.SerializerManager + restConfigMgr *yurtrest.RestConfigManager + etcdStorageCfg *etcd.EtcdStorageConfig + poolCacheManager cachemanager.CacheManager + diskStorage storage.Store + etcdStorage storage.Store + hubElector *HubElector + electStatus int32 + isPoolCacheSynced bool + needUploadLocalCache bool + // poolScopeCacheSyncManager is used to sync pool-scoped resources from cloud to poolcoordinator. + poolScopeCacheSyncManager *poolScopedCacheSyncManager + // informerSyncLeaseManager is used to detect the leader-informer-sync lease + // to check its RenewTime. If its renewTime is not updated after defaultInformerLeaseRenewDuration + // we can think that the poolcoordinator cache is stale and the poolcoordinator is not ready. + // It will start if yurthub becomes leader or follower. + informerSyncLeaseManager *coordinatorLeaseInformerManager + // delegateNodeLeaseManager is used to list/watch kube-node-lease from poolcoordinator. If the + // node lease contains DelegateHeartBeat label, it will triger the eventhandler which will + // use cloud client to send it to cloud APIServer. + delegateNodeLeaseManager *coordinatorLeaseInformerManager } -func NewCoordinator(coordinatorHealthChecker healthchecker.HealthChecker, elector *HubElector, stopCh <-chan struct{}) *Coordinator { - return &Coordinator{ - coordinatorHealthChecker: coordinatorHealthChecker, - hubElector: elector, +func NewCoordinator( + ctx context.Context, + cfg *config.YurtHubConfiguration, + restMgr *yurtrest.RestConfigManager, + transportMgr transport.Interface, + elector *HubElector) (*Coordinator, error) { + etcdStorageCfg := &etcd.EtcdStorageConfig{ + Prefix: cfg.CoordinatorStoragePrefix, + EtcdEndpoints: []string{cfg.CoordinatorStorageAddr}, + CaFile: cfg.CoordinatorStorageCaFile, + CertFile: cfg.CoordinatorStorageCertFile, + KeyFile: cfg.CoordinatorStorageKeyFile, + LocalCacheDir: cfg.DiskCachePath, + } + + coordinatorRESTCfg := &rest.Config{ + Host: cfg.CoordinatorServer.String(), + Transport: transportMgr.CurrentTransport(), + Timeout: defaultInformerLeaseRenewDuration, } + coordinatorClient, err := kubernetes.NewForConfig(coordinatorRESTCfg) + if err != nil { + return nil, fmt.Errorf("failed to create client for pool coordinator, %v", err) + } + + coordinator := &Coordinator{ + ctx: ctx, + etcdStorageCfg: etcdStorageCfg, + restConfigMgr: restMgr, + informerFactory: cfg.SharedFactory, + diskStorage: cfg.StorageWrapper.GetStorage(), + serializerMgr: cfg.SerializerManager, + restMapperMgr: cfg.RESTMapperManager, + hubElector: elector, + } + + informerSyncLeaseManager := &coordinatorLeaseInformerManager{ + ctx: ctx, + coordinatorClient: coordinatorClient, + } + + delegateNodeLeaseManager := &coordinatorLeaseInformerManager{ + ctx: ctx, + coordinatorClient: coordinatorClient, + } + + proxiedClient, err := buildProxiedClientWithUserAgent(fmt.Sprintf("http://%s", cfg.YurtHubProxyServerAddr), constants.DefaultPoolScopedUserAgent) + if err != nil { + return nil, fmt.Errorf("failed to create proxied client, %v", err) + } + poolScopedCacheSyncManager := &poolScopedCacheSyncManager{ + ctx: ctx, + proxiedClient: proxiedClient, + coordinatorClient: cfg.CoordinatorClient, + nodeName: cfg.NodeName, + getEtcdStore: coordinator.getEtcdStore, + } + + coordinator.informerSyncLeaseManager = informerSyncLeaseManager + coordinator.delegateNodeLeaseManager = delegateNodeLeaseManager + coordinator.poolScopeCacheSyncManager = poolScopedCacheSyncManager + + return coordinator, nil } -func (coordinator *Coordinator) Run(stopCh <-chan struct{}) { +func (coordinator *Coordinator) Run() { for { + var poolCacheManager cachemanager.CacheManager + var cancelEtcdStorage func() + var needUploadLocalCache bool + var needCancelEtcdStorage bool + var isPoolCacheSynced bool + var etcdStorage storage.Store + var err error + select { - case <-stopCh: - klog.Infof("exit normally in coordinator loop.") - if coordinator.informerStarted { - // stop shared informer - // - coordinator.informerStarted = false - } + case <-coordinator.ctx.Done(): + coordinator.poolScopeCacheSyncManager.EnsureStop() + coordinator.delegateNodeLeaseManager.EnsureStop() + coordinator.informerSyncLeaseManager.EnsureStop() + klog.Info("exit normally in coordinator loop.") return case electorStatus, ok := <-coordinator.hubElector.StatusChan(): if !ok { return } - if electorStatus != PendingHub && !coordinator.cacheIsUploaded() { - // upload local cache, and make sure yurthub pod is the last resource uploaded - } + switch electorStatus { + case PendingHub: + coordinator.poolScopeCacheSyncManager.EnsureStop() + coordinator.delegateNodeLeaseManager.EnsureStop() + coordinator.informerSyncLeaseManager.EnsureStop() + needUploadLocalCache = true + needCancelEtcdStorage = true + isPoolCacheSynced = false + etcdStorage = nil + poolCacheManager = nil + case LeaderHub: + poolCacheManager, etcdStorage, cancelEtcdStorage, err = coordinator.buildPoolCacheStore() + if err != nil { + klog.Errorf("failed to create pool scoped cache store and manager, %v", err) + continue + } - if electorStatus == LeaderHub { - if !coordinator.informerStarted { - coordinator.informerStarted = true - // start shared informer for pool-scope data - // make sure + cloudLeaseClient, err := coordinator.newCloudLeaseClient() + if err != nil { + klog.Errorf("cloud not get cloud lease client when becoming leader yurthub, %v", err) + continue + } + if err := coordinator.poolScopeCacheSyncManager.EnsureStart(); err != nil { + klog.Errorf("failed to sync pool-scoped resource, %v", err) + continue + } - // start shared informer for lease delegating - // + coordinator.delegateNodeLeaseManager.EnsureStartWithHandler(cache.FilteringResourceEventHandler{ + FilterFunc: ifDelegateHeartBeat, + Handler: cache.ResourceEventHandlerFuncs{ + AddFunc: func(obj interface{}) { + coordinator.delegateNodeLease(cloudLeaseClient, obj) + }, + UpdateFunc: func(_, newObj interface{}) { + coordinator.delegateNodeLease(cloudLeaseClient, newObj) + }, + }, + }) + coordinator.informerSyncLeaseManager.EnsureStartWithHandler(cache.FilteringResourceEventHandler{ + FilterFunc: ifInformerSyncLease, + Handler: cache.ResourceEventHandlerFuncs{ + AddFunc: coordinator.detectPoolCacheSynced, + UpdateFunc: func(_, newObj interface{}) { + coordinator.detectPoolCacheSynced(newObj) + }, + DeleteFunc: func(_ interface{}) { + coordinator.Lock() + defer coordinator.Unlock() + coordinator.isPoolCacheSynced = false + }, + }, + }) + + if coordinator.needUploadLocalCache { + if err := coordinator.uploadLocalCache(etcdStorage); err != nil { + klog.Errorf("failed to upload local cache when yurthub becomes leader, %v", err) + } else { + needUploadLocalCache = false + } } - break - } + case FollowerHub: + poolCacheManager, etcdStorage, cancelEtcdStorage, err = coordinator.buildPoolCacheStore() + if err != nil { + klog.Errorf("failed to create pool scoped cache store and manager, %v", err) + continue + } + + coordinator.poolScopeCacheSyncManager.EnsureStop() + coordinator.delegateNodeLeaseManager.EnsureStop() + coordinator.informerSyncLeaseManager.EnsureStartWithHandler(cache.FilteringResourceEventHandler{ + FilterFunc: ifInformerSyncLease, + Handler: cache.ResourceEventHandlerFuncs{ + AddFunc: coordinator.detectPoolCacheSynced, + UpdateFunc: func(_, newObj interface{}) { + coordinator.detectPoolCacheSynced(newObj) + }, + DeleteFunc: func(_ interface{}) { + coordinator.Lock() + defer coordinator.Unlock() + coordinator.isPoolCacheSynced = false + }, + }, + }) - if electorStatus == FollowerHub { - if coordinator.informerStarted { - // stop shared informer - // - coordinator.informerStarted = false + if coordinator.needUploadLocalCache { + if err := coordinator.uploadLocalCache(etcdStorage); err != nil { + klog.Errorf("failed to upload local cache when yurthub becomes follower, %v", err) + } else { + needUploadLocalCache = false + } } } + + // We should make sure that all fields update should happen + // after acquire lock to avoid race condition. + // Because the caller of IsReady() may be concurrent. + coordinator.Lock() + if needCancelEtcdStorage { + coordinator.cancelEtcdStorage() + } + coordinator.electStatus = electorStatus + coordinator.poolCacheManager = poolCacheManager + coordinator.etcdStorage = etcdStorage + coordinator.cancelEtcdStorage = cancelEtcdStorage + coordinator.needUploadLocalCache = needUploadLocalCache + coordinator.isPoolCacheSynced = isPoolCacheSynced + coordinator.Unlock() + } + } +} + +// IsReady will return the poolCacheManager and true if the pool-coordinator is ready. +// Pool-Coordinator ready means it is ready to handle request. To be specific, it should +// satisfy the following 3 condition: +// 1. Pool-Coordinator is healthy +// 2. Pool-Scoped resources have been synced with cloud, through list/watch +// 3. local cache has been uploaded to pool-coordinator +func (coordinator *Coordinator) IsReady() (cachemanager.CacheManager, bool) { + // If electStatus is not PendingHub, it means pool-coordinator is healthy. + coordinator.Lock() + defer coordinator.Unlock() + if coordinator.electStatus != PendingHub && coordinator.isPoolCacheSynced && !coordinator.needUploadLocalCache { + return coordinator.poolCacheManager, true + } + return nil, false +} + +// IsCoordinatorHealthy will return the poolCacheManager and true if the pool-coordinator is healthy. +// We assume coordinator is healthy when the elect status is LeaderHub and FollowerHub. +func (coordinator *Coordinator) IsHealthy() (cachemanager.CacheManager, bool) { + coordinator.Lock() + defer coordinator.Unlock() + if coordinator.electStatus != PendingHub { + return coordinator.poolCacheManager, true + } + return nil, false +} + +func (coordinator *Coordinator) buildPoolCacheStore() (cachemanager.CacheManager, storage.Store, func(), error) { + ctx, cancel := context.WithCancel(coordinator.ctx) + etcdStore, err := etcd.NewStorage(ctx, coordinator.etcdStorageCfg) + if err != nil { + cancel() + return nil, nil, nil, fmt.Errorf("failed to create etcd storage, %v", err) + } + poolCacheManager := cachemanager.NewCacheManager( + cachemanager.NewStorageWrapper(etcdStore), + coordinator.serializerMgr, + coordinator.restMapperMgr, + coordinator.informerFactory, + ) + return poolCacheManager, etcdStore, cancel, nil +} + +func (coordinator *Coordinator) getEtcdStore() storage.Store { + return coordinator.etcdStorage +} + +func (coordinator *Coordinator) newCloudLeaseClient() (coordclientset.LeaseInterface, error) { + restCfg := coordinator.restConfigMgr.GetRestConfig(true) + if restCfg == nil { + return nil, fmt.Errorf("no cloud server is healthy") + } + cloudClient, err := kubernetes.NewForConfig(restCfg) + if err != nil { + return nil, fmt.Errorf("failed to create cloud client, %v", err) + } + + return cloudClient.CoordinationV1().Leases(corev1.NamespaceNodeLease), nil +} + +func (coordinator *Coordinator) uploadLocalCache(etcdStore storage.Store) error { + uploader := &localCacheUploader{ + diskStorage: coordinator.diskStorage, + etcdStorage: etcdStore, + } + klog.Info("uploading local cache") + uploader.Upload() + return nil +} + +func (coordinator *Coordinator) delegateNodeLease(cloudLeaseClient coordclientset.LeaseInterface, obj interface{}) { + newLease := obj.(*coordinationv1.Lease) + for i := 0; i < leaseDelegateRetryTimes; i++ { + // ResourceVersions of lease objects in pool-coordinator always have different rv + // from what of cloud lease. So we should get cloud lease first and then update + // it with lease from pool-coordinator. + cloudLease, err := cloudLeaseClient.Get(coordinator.ctx, newLease.Name, metav1.GetOptions{}) + if apierrors.IsNotFound(err) { + if _, err := cloudLeaseClient.Create(coordinator.ctx, cloudLease, metav1.CreateOptions{}); err != nil { + klog.Errorf("failed to create lease %s at cloud, %v", newLease.Name, err) + continue + } + } + + lease := newLease.DeepCopy() + lease.ResourceVersion = cloudLease.ResourceVersion + if _, err := cloudLeaseClient.Update(coordinator.ctx, lease, metav1.UpdateOptions{}); err != nil { + klog.Errorf("failed to update lease %s at cloud, %v", newLease.Name, err) + continue + } + } +} + +func (coordinator *Coordinator) detectPoolCacheSynced(obj interface{}) { + lease := obj.(*coordinationv1.Lease) + renewTime := lease.Spec.RenewTime + if time.Now().After(renewTime.Add(defaultPoolCacheStaleDuration)) { + coordinator.Lock() + defer coordinator.Unlock() + coordinator.isPoolCacheSynced = false + } +} + +// poolScopedCacheSyncManager will continuously sync pool-scoped resources from cloud to pool-coordinator. +// After resource sync is completed, it will periodically renew the informer synced lease, which is used by +// other yurthub to determine if pool-coordinator is ready to handle requests of pool-scoped resources. +// It uses proxied client to list/watch pool-scoped resources from cloud APIServer, which +// will be automatically cached into pool-coordinator through YurtProxyServer. +type poolScopedCacheSyncManager struct { + ctx context.Context + isRunning bool + // proxiedClient is a client of Cloud APIServer which is proxied by yurthub. + proxiedClient kubernetes.Interface + // coordinatorClient is a client of APIServer in pool-coordinator. + coordinatorClient kubernetes.Interface + // nodeName will be used to update the ownerReference of informer synced lease. + nodeName string + informerSyncedLease *coordinationv1.Lease + getEtcdStore func() storage.Store + cancel func() +} + +func (p *poolScopedCacheSyncManager) EnsureStart() error { + if !p.isRunning { + if err := p.coordinatorClient.CoordinationV1().Leases(namespaceInformerLease).Delete(p.ctx, nameInformerLease, metav1.DeleteOptions{}); err != nil { + return fmt.Errorf("failed to delete informer sync lease, %v", err) + } + + etcdStore := p.getEtcdStore() + if etcdStore == nil { + return fmt.Errorf("got empty etcd storage") + } + if err := etcdStore.DeleteComponentResources(constants.DefaultPoolScopedUserAgent); err != nil { + return fmt.Errorf("failed to clean old pool-scoped cache, %v", err) + } + + ctx, cancel := context.WithCancel(p.ctx) + hasInformersSynced := []cache.InformerSynced{} + informerFactory := informers.NewSharedInformerFactory(p.proxiedClient, 0) + for gvr := range constants.PoolScopedResources { + informer, err := informerFactory.ForResource(gvr) + if err != nil { + cancel() + return fmt.Errorf("failed to add informer for %s, %v", gvr.String(), err) + } + hasInformersSynced = append(hasInformersSynced, informer.Informer().HasSynced) } + + informerFactory.Start(ctx.Done()) + go p.holdInformerSync(ctx, hasInformersSynced) + p.cancel = cancel + p.isRunning = true } + return nil } -func (coordinator *Coordinator) cacheIsUploaded() bool { - // check yurthub pod is uploaded - return true +func (p *poolScopedCacheSyncManager) EnsureStop() { + if p.isRunning { + p.cancel() + p.cancel = nil + p.isRunning = false + } } -func (coordinator *Coordinator) IsReady() bool { +func (p *poolScopedCacheSyncManager) holdInformerSync(ctx context.Context, hasInformersSynced []cache.InformerSynced) { + if cache.WaitForCacheSync(ctx.Done(), hasInformersSynced...) { + informerLease := NewInformerLease( + p.coordinatorClient, + nameInformerLease, + namespaceInformerLease, + p.nodeName, + int32(defaultInformerLeaseRenewDuration.Seconds()), + 5) + p.renewInformerLease(ctx, informerLease) + return + } + klog.Error("failed to wait for cache synced, it was canceled") +} + +func (p *poolScopedCacheSyncManager) renewInformerLease(ctx context.Context, lease informerLease) { + for { + t := time.NewTicker(defaultInformerLeaseRenewDuration) + select { + case <-ctx.Done(): + klog.Info("cancel renew informer lease") + return + case <-t.C: + newLease, err := lease.Update(p.informerSyncedLease) + if err != nil { + klog.Errorf("failed to update informer lease, %v", err) + continue + } + p.informerSyncedLease = newLease + } + } +} - return true +// coordinatorLeaseInformerManager will use pool-coordinator client to list/watch +// lease in pool-coordinator. Through passing different event handler, it can either +// delegating node lease by leader yurthub or detecting the informer synced lease to +// check if pool-coordinator is ready for requests of pool-scoped resources. +type coordinatorLeaseInformerManager struct { + ctx context.Context + coordinatorClient kubernetes.Interface + name string + isRunning bool + cancel func() +} + +func (c *coordinatorLeaseInformerManager) Name() string { + return c.name +} + +func (c *coordinatorLeaseInformerManager) EnsureStartWithHandler(handler cache.FilteringResourceEventHandler) { + if !c.isRunning { + ctx, cancel := context.WithCancel(c.ctx) + informerFactory := informers.NewSharedInformerFactory(c.coordinatorClient, 0) + informerFactory.Coordination().V1().Leases().Informer().AddEventHandler(handler) + informerFactory.Start(ctx.Done()) + c.isRunning = true + c.cancel = cancel + } +} + +func (c *coordinatorLeaseInformerManager) EnsureStop() { + if c.isRunning { + c.cancel() + c.isRunning = false + } +} + +// localCacheUploader can upload resources in local cache to pool cache. +// Currently, we only upload pods and nodes to pool-coordinator. +type localCacheUploader struct { + diskStorage storage.Store + etcdStorage storage.Store +} + +func (l *localCacheUploader) Upload() { + objBytes := l.resourcesToUpload() + for k, b := range objBytes { + rv, err := getRv(b) + if err != nil { + klog.Errorf("failed to get name from bytes %s, %v", string(b), err) + continue + } + + if err := l.createOrUpdate(k, b, rv); err != nil { + klog.Errorf("failed to upload %s, %v", k.Key(), err) + } + } +} + +func (l *localCacheUploader) createOrUpdate(key storage.Key, objBytes []byte, rv uint64) error { + err := l.etcdStorage.Create(key, objBytes) + + if err == storage.ErrKeyExists { + // try to update + _, updateErr := l.etcdStorage.Update(key, objBytes, rv) + if updateErr == storage.ErrUpdateConflict { + return nil + } + return updateErr + } + + return err +} + +func (l *localCacheUploader) resourcesToUpload() map[storage.Key][]byte { + objBytes := map[storage.Key][]byte{} + for info := range constants.UploadResourcesKeyBuildInfo { + gvr := schema.GroupVersionResource{ + Group: info.Group, + Version: info.Version, + Resource: info.Resources, + } + keys, err := l.diskStorage.ListResourceKeysOfComponent(info.Component, gvr) + if err != nil { + klog.Errorf("failed to get object keys from disk for %s, %v", gvr.String(), err) + continue + } + + for _, k := range keys { + buf, err := l.diskStorage.Get(k) + if err != nil { + klog.Errorf("failed to read local cache of key %s, %v", k.Key(), err) + continue + } + objBytes[k] = buf + } + } + return objBytes +} + +func getRv(objBytes []byte) (uint64, error) { + obj := &unstructured.Unstructured{} + if err := json.Unmarshal(objBytes, obj); err != nil { + return 0, fmt.Errorf("failed to unmarshal json: %v", err) + } + + rv, err := strconv.ParseUint(obj.GetResourceVersion(), 10, 64) + if err != nil { + return 0, fmt.Errorf("failed to parse rv %s of pod %s, %v", obj.GetName(), obj.GetResourceVersion(), err) + } + + return rv, nil +} + +func ifDelegateHeartBeat(obj interface{}) bool { + lease, ok := obj.(*coordinationv1.Lease) + if !ok { + return false + } + v, ok := lease.Labels[healthchecker.DelegateHeartBeat] + return ok && v == "true" +} + +func ifInformerSyncLease(obj interface{}) bool { + lease, ok := obj.(*coordinationv1.Lease) + if !ok { + return false + } + + return lease.Name == nameInformerLease && lease.Namespace == namespaceInformerLease +} + +func buildProxiedClientWithUserAgent(proxyAddr string, userAgent string) (kubernetes.Interface, error) { + kubeConfig, err := clientcmd.BuildConfigFromFlags(proxyAddr, "") + if err != nil { + return nil, err + } + + kubeConfig.UserAgent = userAgent + client, err := kubernetes.NewForConfig(kubeConfig) + if err != nil { + return nil, err + } + return client, nil } diff --git a/pkg/yurthub/poolcoordinator/informer_lease.go b/pkg/yurthub/poolcoordinator/informer_lease.go new file mode 100644 index 00000000000..b48a9ff5f59 --- /dev/null +++ b/pkg/yurthub/poolcoordinator/informer_lease.go @@ -0,0 +1,178 @@ +/* +Copyright 2021 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package poolcoordinator + +import ( + "context" + "fmt" + "time" + + coordinationv1 "k8s.io/api/coordination/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + clientset "k8s.io/client-go/kubernetes" + coordclientset "k8s.io/client-go/kubernetes/typed/coordination/v1" + "k8s.io/klog/v2" + "k8s.io/utils/clock" + "k8s.io/utils/pointer" +) + +// TODO: reuse code of healthchecker.NodeLease +// Add the file temporarily for coordinator use, because healthchecker.NodeLease cannot +// be directly used by coordinator and modifying it will encounter a lot of changes. +// We currently want to focus on the implementation of coordinator, so making a copy of it +// and modifying it as we want. We can reuse the code of healthchecker.NodeLease in further work. + +const ( + maxBackoff = 1 * time.Second +) + +type informerLease interface { + Update(base *coordinationv1.Lease) (*coordinationv1.Lease, error) +} + +type informerLeaseTmpl struct { + client clientset.Interface + leaseClient coordclientset.LeaseInterface + leaseName string + leaseNamespace string + leaseDurationSeconds int32 + holderIdentity string + failedRetry int + clock clock.Clock +} + +func NewInformerLease(coordinatorClient clientset.Interface, leaseName string, leaseNamespace string, holderIdentity string, leaseDurationSeconds int32, failedRetry int) informerLease { + return &informerLeaseTmpl{ + client: coordinatorClient, + leaseClient: coordinatorClient.CoordinationV1().Leases(leaseNamespace), + leaseName: leaseName, + holderIdentity: holderIdentity, + failedRetry: failedRetry, + leaseDurationSeconds: leaseDurationSeconds, + clock: clock.RealClock{}, + } +} + +func (nl *informerLeaseTmpl) Update(base *coordinationv1.Lease) (*coordinationv1.Lease, error) { + if base != nil { + lease, err := nl.retryUpdateLease(base) + if err == nil { + return lease, nil + } + } + lease, created, err := nl.backoffEnsureLease() + if err != nil { + return nil, err + } + if !created { + return nl.retryUpdateLease(lease) + } + return lease, nil +} + +func (nl *informerLeaseTmpl) retryUpdateLease(base *coordinationv1.Lease) (*coordinationv1.Lease, error) { + var err error + var lease *coordinationv1.Lease + for i := 0; i < nl.failedRetry; i++ { + lease, err = nl.leaseClient.Update(context.Background(), nl.newLease(base), metav1.UpdateOptions{}) + if err == nil { + return lease, nil + } + if apierrors.IsConflict(err) { + base, _, err = nl.backoffEnsureLease() + if err != nil { + return nil, err + } + continue + } + klog.V(3).Infof("update node lease fail: %v, will try it.", err) + } + return nil, err +} + +func (nl *informerLeaseTmpl) backoffEnsureLease() (*coordinationv1.Lease, bool, error) { + var ( + lease *coordinationv1.Lease + created bool + err error + ) + + sleep := 100 * time.Millisecond + for { + lease, created, err = nl.ensureLease() + if err == nil { + break + } + sleep = sleep * 2 + if sleep > maxBackoff { + return nil, false, fmt.Errorf("backoff ensure lease error: %w", err) + } + nl.clock.Sleep(sleep) + } + return lease, created, err +} + +func (nl *informerLeaseTmpl) ensureLease() (*coordinationv1.Lease, bool, error) { + lease, err := nl.leaseClient.Get(context.Background(), nl.leaseName, metav1.GetOptions{}) + if apierrors.IsNotFound(err) { + lease, err := nl.leaseClient.Create(context.Background(), nl.newLease(nil), metav1.CreateOptions{}) + if err != nil { + return nil, false, err + } + return lease, true, nil + } else if err != nil { + return nil, false, err + } + return lease, false, nil +} + +func (nl *informerLeaseTmpl) newLease(base *coordinationv1.Lease) *coordinationv1.Lease { + var lease *coordinationv1.Lease + if base == nil { + lease = &coordinationv1.Lease{ + ObjectMeta: metav1.ObjectMeta{ + Name: nl.leaseName, + Namespace: nl.leaseNamespace, + }, + Spec: coordinationv1.LeaseSpec{ + HolderIdentity: pointer.StringPtr(nl.holderIdentity), + LeaseDurationSeconds: pointer.Int32Ptr(nl.leaseDurationSeconds), + }, + } + } else { + lease = base.DeepCopy() + } + + lease.Spec.RenewTime = &metav1.MicroTime{Time: nl.clock.Now()} + if lease.OwnerReferences == nil || len(lease.OwnerReferences) == 0 { + if node, err := nl.client.CoreV1().Nodes().Get(context.Background(), nl.holderIdentity, metav1.GetOptions{}); err == nil { + lease.OwnerReferences = []metav1.OwnerReference{ + { + APIVersion: corev1.SchemeGroupVersion.WithKind("Node").Version, + Kind: corev1.SchemeGroupVersion.WithKind("Node").Kind, + Name: nl.holderIdentity, + UID: node.UID, + }, + } + } else { + klog.Errorf("failed to get node %q when trying to set owner ref to the node lease: %v", nl.leaseName, err) + } + } + return lease +} diff --git a/pkg/yurthub/poolcoordinator/leader_election.go b/pkg/yurthub/poolcoordinator/leader_election.go index ed9eae6e620..565caf4c9ac 100644 --- a/pkg/yurthub/poolcoordinator/leader_election.go +++ b/pkg/yurthub/poolcoordinator/leader_election.go @@ -39,21 +39,21 @@ const ( type HubElector struct { coordinatorClient kubernetes.Interface coordinatorHealthChecker healthchecker.HealthChecker - cloudAPIServerHealthChecker healthchecker.HealthChecker + cloudAPIServerHealthChecker healthchecker.MultipleBackendsHealthChecker electorStatus chan int32 le *leaderelection.LeaderElector inElecting bool } -func NewHubElector(cfg *config.YurtHubConfiguration, coordinatorClient kubernetes.Interface, cloudAPIServerHealthyChecker healthchecker.HealthChecker, stopCh <-chan struct{}) (*HubElector, error) { - coordinatorHealthyChecker, err := healthchecker.NewCoordinatorHealthChecker(cfg, coordinatorClient, cloudAPIServerHealthyChecker, stopCh) - if err != nil { - return nil, err - } - +func NewHubElector( + cfg *config.YurtHubConfiguration, + coordinatorClient kubernetes.Interface, + coordinatorHealthChecker healthchecker.HealthChecker, + cloudAPIServerHealthyChecker healthchecker.MultipleBackendsHealthChecker, + stopCh <-chan struct{}) (*HubElector, error) { he := &HubElector{ coordinatorClient: coordinatorClient, - coordinatorHealthChecker: coordinatorHealthyChecker, + coordinatorHealthChecker: coordinatorHealthChecker, cloudAPIServerHealthChecker: cloudAPIServerHealthyChecker, electorStatus: make(chan int32), } @@ -79,7 +79,8 @@ func NewHubElector(cfg *config.YurtHubConfiguration, coordinatorClient kubernete he.electorStatus <- LeaderHub }, OnStoppedLeading: func() { - + klog.Infof("yurthub of %s is no more a leader", cfg.NodeName) + he.electorStatus <- PendingHub }, }, }) diff --git a/pkg/yurthub/proxy/local/local.go b/pkg/yurthub/proxy/local/local.go index 4b58dfb5521..77eff158a55 100644 --- a/pkg/yurthub/proxy/local/local.go +++ b/pkg/yurthub/proxy/local/local.go @@ -50,17 +50,19 @@ type IsHealthy func() bool // LocalProxy is responsible for handling requests when remote servers are unhealthy type LocalProxy struct { - cacheMgr manager.CacheManager - isHealthy IsHealthy - minRequestTimeout time.Duration + cacheMgr manager.CacheManager + isCloudHealthy IsHealthy + isCoordinatorReady IsHealthy + minRequestTimeout time.Duration } // NewLocalProxy creates a *LocalProxy -func NewLocalProxy(cacheMgr manager.CacheManager, isHealthy IsHealthy, minRequestTimeout time.Duration) *LocalProxy { +func NewLocalProxy(cacheMgr manager.CacheManager, isCloudHealthy IsHealthy, isCoordinatorHealthy IsHealthy, minRequestTimeout time.Duration) *LocalProxy { return &LocalProxy{ - cacheMgr: cacheMgr, - isHealthy: isHealthy, - minRequestTimeout: minRequestTimeout, + cacheMgr: cacheMgr, + isCloudHealthy: isCloudHealthy, + isCoordinatorReady: isCoordinatorHealthy, + minRequestTimeout: minRequestTimeout, } } @@ -182,6 +184,7 @@ func (lp *LocalProxy) localWatch(w http.ResponseWriter, req *http.Request) error timeout = time.Duration(float64(lp.minRequestTimeout) * (rand.Float64() + 1.0)) } + isPoolScopedListWatch := util.IsPoolScopedResouceListWatchRequest(req) watchTimer := time.NewTimer(timeout) intervalTicker := time.NewTicker(interval) defer watchTimer.Stop() @@ -196,7 +199,12 @@ func (lp *LocalProxy) localWatch(w http.ResponseWriter, req *http.Request) error return nil case <-intervalTicker.C: // if cluster becomes healthy, exit the watch wait - if lp.isHealthy() { + if lp.isCloudHealthy() { + return nil + } + + // if poolcoordinator becomes healthy, exit the watch wait + if isPoolScopedListWatch && lp.isCoordinatorReady() { return nil } } diff --git a/pkg/yurthub/proxy/local/local_test.go b/pkg/yurthub/proxy/local/local_test.go index 51481dcab5f..6ce5ca88361 100644 --- a/pkg/yurthub/proxy/local/local_test.go +++ b/pkg/yurthub/proxy/local/local_test.go @@ -70,7 +70,7 @@ func TestServeHTTPForWatch(t *testing.T) { return false } - lp := NewLocalProxy(cacheM, fn, 0) + lp := NewLocalProxy(cacheM, fn, fn, 0) testcases := map[string]struct { userAgent string @@ -164,7 +164,7 @@ func TestServeHTTPForWatchWithHealthyChange(t *testing.T) { return cnt > 2 // after 6 seconds, become healthy } - lp := NewLocalProxy(cacheM, fn, 0) + lp := NewLocalProxy(cacheM, fn, fn, 0) testcases := map[string]struct { userAgent string @@ -247,7 +247,7 @@ func TestServeHTTPForWatchWithMinRequestTimeout(t *testing.T) { return false } - lp := NewLocalProxy(cacheM, fn, 10*time.Second) + lp := NewLocalProxy(cacheM, fn, fn, 10*time.Second) testcases := map[string]struct { userAgent string @@ -339,7 +339,7 @@ func TestServeHTTPForPost(t *testing.T) { return false } - lp := NewLocalProxy(cacheM, fn, 0) + lp := NewLocalProxy(cacheM, fn, fn, 0) testcases := map[string]struct { userAgent string @@ -419,7 +419,7 @@ func TestServeHTTPForDelete(t *testing.T) { return false } - lp := NewLocalProxy(cacheM, fn, 0) + lp := NewLocalProxy(cacheM, fn, fn, 0) testcases := map[string]struct { userAgent string @@ -486,7 +486,7 @@ func TestServeHTTPForGetReqCache(t *testing.T) { return false } - lp := NewLocalProxy(cacheM, fn, 0) + lp := NewLocalProxy(cacheM, fn, fn, 0) testcases := map[string]struct { userAgent string @@ -639,7 +639,7 @@ func TestServeHTTPForListReqCache(t *testing.T) { return false } - lp := NewLocalProxy(cacheM, fn, 0) + lp := NewLocalProxy(cacheM, fn, fn, 0) testcases := map[string]struct { userAgent string diff --git a/pkg/yurthub/proxy/pool/pool.go b/pkg/yurthub/proxy/pool/pool.go new file mode 100644 index 00000000000..7d98982177c --- /dev/null +++ b/pkg/yurthub/proxy/pool/pool.go @@ -0,0 +1,256 @@ +/* +Copyright 2022 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package pool + +import ( + "context" + "fmt" + "io" + "net/http" + "net/url" + "time" + + "k8s.io/apimachinery/pkg/api/errors" + apirequest "k8s.io/apiserver/pkg/endpoints/request" + "k8s.io/klog/v2" + + "github.com/openyurtio/openyurt/pkg/yurthub/cachemanager" + "github.com/openyurtio/openyurt/pkg/yurthub/filter/manager" + "github.com/openyurtio/openyurt/pkg/yurthub/proxy/util" + "github.com/openyurtio/openyurt/pkg/yurthub/transport" + hubutil "github.com/openyurtio/openyurt/pkg/yurthub/util" +) + +const ( + watchCheckInterval = 5 * time.Second +) + +// PoolCoordinatorProxy is responsible for handling requests when remote servers are unhealthy +type PoolCoordinatorProxy struct { + poolCoordinatorProxy *util.RemoteProxy + localCacheMgr cachemanager.CacheManager + filterMgr *manager.Manager + isCoordinatorReady func() bool + stopCh <-chan struct{} +} + +func NewPoolCoordinatorProxy( + poolCoordinatorAddr *url.URL, + localCacheMgr cachemanager.CacheManager, + transportMgr transport.Interface, + filterMgr *manager.Manager, + isCoordinatorReady func() bool, + stopCh <-chan struct{}) (*PoolCoordinatorProxy, error) { + if poolCoordinatorAddr == nil { + return nil, fmt.Errorf("pool-coordinator addr cannot be nil") + } + + pp := &PoolCoordinatorProxy{ + localCacheMgr: localCacheMgr, + isCoordinatorReady: isCoordinatorReady, + filterMgr: filterMgr, + stopCh: stopCh, + } + + proxy, err := util.NewRemoteProxy( + poolCoordinatorAddr, + pp.modifyResponse, + pp.errorHandler, + transportMgr, + stopCh) + if err != nil { + return nil, fmt.Errorf("failed to create remote proxy for pool-coordinator, %v", err) + } + + pp.poolCoordinatorProxy = proxy + return pp, nil +} + +// ServeHTTP of PoolCoordinatorProxy is able to handle read-only request, including +// watch, list, get. Other verbs that will write data to the cache are not supported +// currently. +func (pp *PoolCoordinatorProxy) ServeHTTP(rw http.ResponseWriter, req *http.Request) { + var err error + ctx := req.Context() + reqInfo, ok := apirequest.RequestInfoFrom(ctx) + if !ok || reqInfo == nil { + klog.Errorf("pool-coordinator proxy cannot handle request(%s), cannot get requestInfo", hubutil.ReqString(req), reqInfo) + util.Err(errors.NewBadRequest(fmt.Sprintf("pool-coordinator proxy cannot handle request(%s), cannot get requestInfo", hubutil.ReqString(req))), rw, req) + return + } + if reqInfo.IsResourceRequest { + switch reqInfo.Verb { + case "create": + err = pp.poolPost(rw, req) + case "list", "get": + err = pp.poolQuery(rw, req) + case "watch": + err = pp.poolWatch(rw, req) + default: + err = fmt.Errorf("unsupported verb for pool coordinator proxy: %s", reqInfo.Verb) + } + if err != nil { + klog.Errorf("could not proxy to pool-coordinator for %s, %v", hubutil.ReqString(req), err) + util.Err(errors.NewBadRequest(err.Error()), rw, req) + } + } else { + klog.Errorf("pool-coordinator does not support request(%s) when cluster is unhealthy, requestInfo: %v", hubutil.ReqString(req), reqInfo) + util.Err(errors.NewBadRequest(fmt.Sprintf("pool-coordinator does not support request(%s) when cluster is unhealthy", hubutil.ReqString(req))), rw, req) + } +} + +func (pp *PoolCoordinatorProxy) poolPost(rw http.ResponseWriter, req *http.Request) error { + ctx := req.Context() + info, _ := apirequest.RequestInfoFrom(ctx) + klog.V(4).Infof("pool handle post, req=%s, reqInfo=%s", hubutil.ReqString(req), hubutil.ReqInfoString(info)) + if util.IsSubjectAccessReviewCreateGetRequest(req) || util.IsEventCreateRequest(req) { + // kubelet needs to create subjectaccessreviews for auth + pp.poolCoordinatorProxy.ServeHTTP(rw, req) + return nil + } + + return fmt.Errorf("unsupported post request") +} + +func (pp *PoolCoordinatorProxy) poolQuery(rw http.ResponseWriter, req *http.Request) error { + if util.IsPoolScopedResouceListWatchRequest(req) || util.IsSubjectAccessReviewCreateGetRequest(req) { + pp.poolCoordinatorProxy.ServeHTTP(rw, req) + return nil + } + return fmt.Errorf("unsupported query request") +} + +func (pp *PoolCoordinatorProxy) poolWatch(rw http.ResponseWriter, req *http.Request) error { + if util.IsPoolScopedResouceListWatchRequest(req) { + clientReqCtx := req.Context() + poolServeCtx, poolServeCancel := context.WithCancel(clientReqCtx) + + go func() { + t := time.NewTicker(watchCheckInterval) + defer t.Stop() + for { + select { + case <-t.C: + if !pp.isCoordinatorReady() { + klog.Infof("notified the pool coordinator is not ready for handling request, cancel watch %s", hubutil.ReqString(req)) + poolServeCancel() + return + } + case <-clientReqCtx.Done(): + klog.Infof("notified client canceled the watch request %s, stop proxy it to pool coordinator", hubutil.ReqString(req)) + return + } + } + }() + + newReq := req.Clone(poolServeCtx) + pp.poolCoordinatorProxy.ServeHTTP(rw, newReq) + klog.Infof("watch %s to pool coordinator exited", hubutil.ReqString(req)) + return nil + } + return fmt.Errorf("unsupported watch request") +} + +func (pp *PoolCoordinatorProxy) errorHandler(rw http.ResponseWriter, req *http.Request, err error) { + klog.Errorf("remote proxy error handler: %s, %v", hubutil.ReqString(req), err) + ctx := req.Context() + if info, ok := apirequest.RequestInfoFrom(ctx); ok { + if info.Verb == "get" || info.Verb == "list" { + if obj, err := pp.localCacheMgr.QueryCache(req); err == nil { + hubutil.WriteObject(http.StatusOK, obj, rw, req) + return + } + } + } + rw.WriteHeader(http.StatusBadGateway) +} + +func (pp *PoolCoordinatorProxy) modifyResponse(resp *http.Response) error { + if resp == nil || resp.Request == nil { + klog.Infof("no request info in response, skip cache response") + return nil + } + + req := resp.Request + ctx := req.Context() + + // re-added transfer-encoding=chunked response header for watch request + info, exists := apirequest.RequestInfoFrom(ctx) + if exists { + if info.Verb == "watch" { + klog.V(5).Infof("add transfer-encoding=chunked header into response for req %s", hubutil.ReqString(req)) + h := resp.Header + if hv := h.Get("Transfer-Encoding"); hv == "" { + h.Add("Transfer-Encoding", "chunked") + } + } + } + + if resp.StatusCode >= http.StatusOK && resp.StatusCode <= http.StatusPartialContent { + // prepare response content type + reqContentType, _ := hubutil.ReqContentTypeFrom(ctx) + respContentType := resp.Header.Get("Content-Type") + if len(respContentType) == 0 { + respContentType = reqContentType + } + ctx = hubutil.WithRespContentType(ctx, respContentType) + req = req.WithContext(ctx) + + // filter response data + if pp.filterMgr != nil { + if ok, runner := pp.filterMgr.FindRunner(req); ok { + wrapBody, needUncompressed := hubutil.NewGZipReaderCloser(resp.Header, resp.Body, req, "filter") + size, filterRc, err := runner.Filter(req, wrapBody, pp.stopCh) + if err != nil { + klog.Errorf("failed to filter response for %s, %v", hubutil.ReqString(req), err) + return err + } + resp.Body = filterRc + if size > 0 { + resp.ContentLength = int64(size) + resp.Header.Set("Content-Length", fmt.Sprint(size)) + } + + // after gunzip in filter, the header content encoding should be removed. + // because there's no need to gunzip response.body again. + if needUncompressed { + resp.Header.Del("Content-Encoding") + } + } + } + // cache resp with storage interface + pp.cacheResponse(req, resp) + } + + return nil +} + +func (pp *PoolCoordinatorProxy) cacheResponse(req *http.Request, resp *http.Response) { + if pp.localCacheMgr.CanCacheFor(req) { + ctx := req.Context() + req = req.WithContext(ctx) + wrapPrc, _ := hubutil.NewGZipReaderCloser(resp.Header, resp.Body, req, "cache-manager") + + rc, prc := hubutil.NewDualReadCloser(req, wrapPrc, false) + go func(req *http.Request, prc io.ReadCloser, stopCh <-chan struct{}) { + if err := pp.localCacheMgr.CacheResponse(req, prc, stopCh); err != nil { + klog.Errorf("failed to cache req %s in local cache when cluster is unhealthy, %v", hubutil.ReqString(req), err) + } + }(req, prc, ctx.Done()) + req.Body = rc + } +} diff --git a/pkg/yurthub/proxy/proxy.go b/pkg/yurthub/proxy/proxy.go index 49dba96c1dc..88fe2e3188a 100644 --- a/pkg/yurthub/proxy/proxy.go +++ b/pkg/yurthub/proxy/proxy.go @@ -17,6 +17,7 @@ limitations under the License. package proxy import ( + "errors" "net/http" "k8s.io/apimachinery/pkg/util/sets" @@ -28,31 +29,38 @@ import ( "github.com/openyurtio/openyurt/cmd/yurthub/app/config" "github.com/openyurtio/openyurt/pkg/yurthub/cachemanager" "github.com/openyurtio/openyurt/pkg/yurthub/healthchecker" + "github.com/openyurtio/openyurt/pkg/yurthub/poolcoordinator" "github.com/openyurtio/openyurt/pkg/yurthub/proxy/local" + "github.com/openyurtio/openyurt/pkg/yurthub/proxy/pool" "github.com/openyurtio/openyurt/pkg/yurthub/proxy/remote" "github.com/openyurtio/openyurt/pkg/yurthub/proxy/util" "github.com/openyurtio/openyurt/pkg/yurthub/tenant" "github.com/openyurtio/openyurt/pkg/yurthub/transport" + hubutil "github.com/openyurtio/openyurt/pkg/yurthub/util" ) type yurtReverseProxy struct { - resolver apirequest.RequestInfoResolver - loadBalancer remote.LoadBalancer - checker healthchecker.HealthChecker - localProxy http.Handler - cacheMgr cachemanager.CacheManager - maxRequestsInFlight int - tenantMgr tenant.Interface - stopCh <-chan struct{} + resolver apirequest.RequestInfoResolver + loadBalancer remote.LoadBalancer + cloudHealthChecker healthchecker.MultipleBackendsHealthChecker + coordinatorHealtChecker healthchecker.HealthChecker + localProxy http.Handler + poolProxy http.Handler + maxRequestsInFlight int + tenantMgr tenant.Interface + coordinator *poolcoordinator.Coordinator + workingMode hubutil.WorkingMode } // NewYurtReverseProxyHandler creates a http handler for proxying // all of incoming requests. func NewYurtReverseProxyHandler( yurtHubCfg *config.YurtHubConfiguration, - cacheMgr cachemanager.CacheManager, + localCacheMgr cachemanager.CacheManager, transportMgr transport.Interface, - healthChecker healthchecker.MultipleBackendsHealthChecker, + coordinator *poolcoordinator.Coordinator, + cloudHealthChecker healthchecker.MultipleBackendsHealthChecker, + coordinatorHealthChecker healthchecker.HealthChecker, tenantMgr tenant.Interface, stopCh <-chan struct{}) (http.Handler, error) { cfg := &server.Config{ @@ -63,32 +71,57 @@ func NewYurtReverseProxyHandler( lb, err := remote.NewLoadBalancer( yurtHubCfg.LBMode, yurtHubCfg.RemoteServers, - cacheMgr, + localCacheMgr, transportMgr, - healthChecker, + coordinator, + cloudHealthChecker, yurtHubCfg.FilterManager, + yurtHubCfg.WorkingMode, stopCh) if err != nil { return nil, err } - var localProxy http.Handler - // When yurthub is working in cloud mode, cacheMgr will be set to nil which means the local cache is disabled, - // so we don't need to create a LocalProxy. - if cacheMgr != nil { - localProxy = local.NewLocalProxy(cacheMgr, healthChecker.IsHealthy, yurtHubCfg.MinRequestTimeout) + var localProxy, poolProxy http.Handler + + if yurtHubCfg.WorkingMode == hubutil.WorkingModeEdge { + // When yurthub works in Edge mode, we may use local proxy or pool proxy to handle + // the request when offline. + localProxy = local.NewLocalProxy(localCacheMgr, + cloudHealthChecker.IsHealthy, + func() bool { + _, ready := coordinator.IsHealthy() + return ready + }, + yurtHubCfg.MinRequestTimeout, + ) localProxy = local.WithFakeTokenInject(localProxy, yurtHubCfg.SerializerManager) + poolProxy, err = pool.NewPoolCoordinatorProxy( + yurtHubCfg.CoordinatorServer, + localCacheMgr, + transportMgr, + yurtHubCfg.FilterManager, + func() bool { + _, isReady := coordinator.IsReady() + return isReady + }, + stopCh) + if err != nil { + return nil, err + } } yurtProxy := &yurtReverseProxy{ - resolver: resolver, - loadBalancer: lb, - checker: healthChecker, - localProxy: localProxy, - cacheMgr: cacheMgr, - maxRequestsInFlight: yurtHubCfg.MaxRequestInFlight, - tenantMgr: tenantMgr, - stopCh: stopCh, + resolver: resolver, + loadBalancer: lb, + cloudHealthChecker: cloudHealthChecker, + coordinatorHealtChecker: coordinatorHealthChecker, + localProxy: localProxy, + poolProxy: poolProxy, + maxRequestsInFlight: yurtHubCfg.MaxRequestInFlight, + coordinator: coordinator, + tenantMgr: tenantMgr, + workingMode: yurtHubCfg.WorkingMode, } return yurtProxy.buildHandlerChain(yurtProxy), nil @@ -97,16 +130,17 @@ func NewYurtReverseProxyHandler( func (p *yurtReverseProxy) buildHandlerChain(handler http.Handler) http.Handler { handler = util.WithRequestTrace(handler) handler = util.WithRequestContentType(handler) - if p.cacheMgr != nil { + if p.workingMode == hubutil.WorkingModeEdge { handler = util.WithCacheHeaderCheck(handler) } handler = util.WithRequestTimeout(handler) - if p.cacheMgr != nil { + if p.workingMode == hubutil.WorkingModeEdge { handler = util.WithListRequestSelector(handler) } handler = util.WithRequestTraceFull(handler) handler = util.WithMaxInFlightLimit(handler, p.maxRequestsInFlight) handler = util.WithRequestClientComponent(handler) + handler = util.WithIfPoolScopedResource(handler) if p.tenantMgr != nil && p.tenantMgr.GetTenantNs() != "" { handler = util.WithSaTokenSubstitute(handler, p.tenantMgr) @@ -120,13 +154,83 @@ func (p *yurtReverseProxy) buildHandlerChain(handler http.Handler) http.Handler } func (p *yurtReverseProxy) ServeHTTP(rw http.ResponseWriter, req *http.Request) { - isKubeletLeaseReq := util.IsKubeletLeaseReq(req) - if !isKubeletLeaseReq && p.checker.IsHealthy() || p.localProxy == nil { + if p.workingMode == hubutil.WorkingModeCloud { p.loadBalancer.ServeHTTP(rw, req) - } else { - if isKubeletLeaseReq { - p.checker.RenewKubeletLeaseTime() + return + } + + switch { + case util.IsKubeletLeaseReq(req): + p.handleKubeletLease(rw, req) + case util.IsEventCreateRequest(req): + p.eventHandler(rw, req) + case util.IsPoolScopedResouceListWatchRequest(req): + p.poolScopedResouceHandler(rw, req) + case util.IsSubjectAccessReviewCreateGetRequest(req): + p.subjectAccessReviewHandler(rw, req) + default: + // For resource request that do not need to be handled by pool-coordinator, + // handling the request with cloud apiserver or local cache. + if p.cloudHealthChecker.IsHealthy() { + p.loadBalancer.ServeHTTP(rw, req) + } else { + p.localProxy.ServeHTTP(rw, req) } + } +} + +func (p *yurtReverseProxy) handleKubeletLease(rw http.ResponseWriter, req *http.Request) { + p.cloudHealthChecker.RenewKubeletLeaseTime() + p.coordinatorHealtChecker.RenewKubeletLeaseTime() + if p.localProxy != nil { p.localProxy.ServeHTTP(rw, req) } } + +func (p *yurtReverseProxy) eventHandler(rw http.ResponseWriter, req *http.Request) { + if p.cloudHealthChecker.IsHealthy() { + p.loadBalancer.ServeHTTP(rw, req) + // TODO: We should also consider create the event in pool-coordinator when the cloud is healthy. + } else if _, isReady := p.coordinator.IsReady(); isReady { + p.poolProxy.ServeHTTP(rw, req) + } else { + p.localProxy.ServeHTTP(rw, req) + } +} + +func (p *yurtReverseProxy) poolScopedResouceHandler(rw http.ResponseWriter, req *http.Request) { + if _, isReady := p.coordinator.IsReady(); isReady { + p.poolProxy.ServeHTTP(rw, req) + } else if p.cloudHealthChecker.IsHealthy() { + p.loadBalancer.ServeHTTP(rw, req) + } else { + p.localProxy.ServeHTTP(rw, req) + } +} + +func (p *yurtReverseProxy) subjectAccessReviewHandler(rw http.ResponseWriter, req *http.Request) { + if isRequestFromPoolCoordinator(req) { + if _, isReady := p.coordinator.IsReady(); isReady { + p.poolProxy.ServeHTTP(rw, req) + } else { + err := errors.New("request is from pool-coordinator but it's currently not healthy") + klog.Errorf("could not handle SubjectAccessReview req %s, %v", hubutil.ReqString(req), err) + util.Err(err, rw, req) + } + } else { + if p.cloudHealthChecker.IsHealthy() { + p.loadBalancer.ServeHTTP(rw, req) + } else { + err := errors.New("request is from cloud APIServer but it's currently not healthy") + klog.Errorf("could not handle SubjectAccessReview req %s, %v", hubutil.ReqString(req), err) + util.Err(err, rw, req) + } + } +} + +func isRequestFromPoolCoordinator(req *http.Request) bool { + // TODO: need a way to check if the logs/exec request is from APIServer or PoolCoordinator. + // We should avoid sending SubjectAccessReview to Pool-Coordinator if the logs/exec requests + // come from APIServer, which may fail for RBAC differences, vise versa. + return false +} diff --git a/pkg/yurthub/proxy/remote/loadbalancer.go b/pkg/yurthub/proxy/remote/loadbalancer.go index 8158ef3b427..135a3e401e8 100644 --- a/pkg/yurthub/proxy/remote/loadbalancer.go +++ b/pkg/yurthub/proxy/remote/loadbalancer.go @@ -17,29 +17,41 @@ limitations under the License. package remote import ( + "context" "fmt" + "io" "net/http" "net/url" "sync" + "time" + "k8s.io/apimachinery/pkg/runtime/schema" + apirequest "k8s.io/apiserver/pkg/endpoints/request" "k8s.io/klog/v2" "github.com/openyurtio/openyurt/pkg/yurthub/cachemanager" "github.com/openyurtio/openyurt/pkg/yurthub/filter/manager" "github.com/openyurtio/openyurt/pkg/yurthub/healthchecker" + "github.com/openyurtio/openyurt/pkg/yurthub/poolcoordinator" + coordinatorconstants "github.com/openyurtio/openyurt/pkg/yurthub/poolcoordinator/constants" + "github.com/openyurtio/openyurt/pkg/yurthub/proxy/util" "github.com/openyurtio/openyurt/pkg/yurthub/transport" - "github.com/openyurtio/openyurt/pkg/yurthub/util" + hubutil "github.com/openyurtio/openyurt/pkg/yurthub/util" +) + +const ( + watchCheckInterval = 5 * time.Second ) type loadBalancerAlgo interface { - PickOne() *RemoteProxy + PickOne() *util.RemoteProxy Name() string } type rrLoadBalancerAlgo struct { sync.Mutex checker healthchecker.MultipleBackendsHealthChecker - backends []*RemoteProxy + backends []*util.RemoteProxy next int } @@ -47,7 +59,7 @@ func (rr *rrLoadBalancerAlgo) Name() string { return "rr algorithm" } -func (rr *rrLoadBalancerAlgo) PickOne() *RemoteProxy { +func (rr *rrLoadBalancerAlgo) PickOne() *util.RemoteProxy { if len(rr.backends) == 0 { return nil } else if len(rr.backends) == 1 { @@ -81,14 +93,14 @@ func (rr *rrLoadBalancerAlgo) PickOne() *RemoteProxy { type priorityLoadBalancerAlgo struct { sync.Mutex checker healthchecker.MultipleBackendsHealthChecker - backends []*RemoteProxy + backends []*util.RemoteProxy } func (prio *priorityLoadBalancerAlgo) Name() string { return "priority algorithm" } -func (prio *priorityLoadBalancerAlgo) PickOne() *RemoteProxy { +func (prio *priorityLoadBalancerAlgo) PickOne() *util.RemoteProxy { if len(prio.backends) == 0 { return nil } else if len(prio.backends) == 1 { @@ -116,22 +128,36 @@ type LoadBalancer interface { } type loadBalancer struct { - backends []*RemoteProxy - algo loadBalancerAlgo + backends []*util.RemoteProxy + algo loadBalancerAlgo + localCacheMgr cachemanager.CacheManager + filterManager *manager.Manager + coordinator *poolcoordinator.Coordinator + workingMode hubutil.WorkingMode + stopCh <-chan struct{} } // NewLoadBalancer creates a loadbalancer for specified remote servers func NewLoadBalancer( lbMode string, remoteServers []*url.URL, - cacheMgr cachemanager.CacheManager, + localCacheMgr cachemanager.CacheManager, transportMgr transport.Interface, + coordinator *poolcoordinator.Coordinator, healthChecker healthchecker.MultipleBackendsHealthChecker, filterManager *manager.Manager, + workingMode hubutil.WorkingMode, stopCh <-chan struct{}) (LoadBalancer, error) { - backends := make([]*RemoteProxy, 0, len(remoteServers)) + lb := &loadBalancer{ + localCacheMgr: localCacheMgr, + filterManager: filterManager, + coordinator: coordinator, + workingMode: workingMode, + stopCh: stopCh, + } + backends := make([]*util.RemoteProxy, 0, len(remoteServers)) for i := range remoteServers { - b, err := NewRemoteProxy(remoteServers[i], cacheMgr, transportMgr, filterManager, stopCh) + b, err := util.NewRemoteProxy(remoteServers[i], lb.modifyResponse, lb.errorHandler, transportMgr, stopCh) if err != nil { klog.Errorf("could not new proxy backend(%s), %v", remoteServers[i].String(), err) continue @@ -163,10 +189,234 @@ func (lb *loadBalancer) ServeHTTP(rw http.ResponseWriter, req *http.Request) { rp := lb.algo.PickOne() if rp == nil { // exceptional case - klog.Errorf("could not pick one healthy backends by %s for request %s", lb.algo.Name(), util.ReqString(req)) + klog.Errorf("could not pick one healthy backends by %s for request %s", lb.algo.Name(), hubutil.ReqString(req)) http.Error(rw, "could not pick one healthy backends, try again to go through local proxy.", http.StatusInternalServerError) return } - klog.V(3).Infof("picked backend %s by %s for request %s", rp.Name(), lb.algo.Name(), util.ReqString(req)) + klog.V(3).Infof("picked backend %s by %s for request %s", rp.Name(), lb.algo.Name(), hubutil.ReqString(req)) + if util.IsPoolScopedResouceListWatchRequest(req) { + // We get here possibly because the pool-coordinator is not ready. + // We should cancel the watch request when pool-coordinator becomes ready. + klog.Infof("pool-coordinator is not ready, we use cloud APIServer to temporarily handle the req: %s", hubutil.ReqString(req)) + clientReqCtx := req.Context() + cloudServeCtx, cloudServeCancel := context.WithCancel(clientReqCtx) + + go func() { + t := time.NewTicker(watchCheckInterval) + defer t.Stop() + for { + select { + case <-t.C: + if _, isReady := lb.coordinator.IsReady(); isReady { + klog.Infof("notified the pool coordinator is ready, cancel the req %s making it handled by pool coordinator", hubutil.ReqString(req)) + cloudServeCancel() + return + } + case <-clientReqCtx.Done(): + return + } + } + }() + + newReq := req.Clone(cloudServeCtx) + req = newReq + } + rp.ServeHTTP(rw, req) } + +func (lb *loadBalancer) errorHandler(rw http.ResponseWriter, req *http.Request, err error) { + klog.Errorf("remote proxy error handler: %s, %v", hubutil.ReqString(req), err) + if lb.localCacheMgr == nil || !lb.localCacheMgr.CanCacheFor(req) { + rw.WriteHeader(http.StatusBadGateway) + return + } + + ctx := req.Context() + if info, ok := apirequest.RequestInfoFrom(ctx); ok { + if info.Verb == "get" || info.Verb == "list" { + if obj, err := lb.localCacheMgr.QueryCache(req); err == nil { + hubutil.WriteObject(http.StatusOK, obj, rw, req) + return + } + } + } + rw.WriteHeader(http.StatusBadGateway) +} + +func (lb *loadBalancer) modifyResponse(resp *http.Response) error { + if resp == nil || resp.Request == nil { + klog.Infof("no request info in response, skip cache response") + return nil + } + + req := resp.Request + ctx := req.Context() + + // re-added transfer-encoding=chunked response header for watch request + info, exists := apirequest.RequestInfoFrom(ctx) + if exists { + if info.Verb == "watch" { + klog.V(5).Infof("add transfer-encoding=chunked header into response for req %s", hubutil.ReqString(req)) + h := resp.Header + if hv := h.Get("Transfer-Encoding"); hv == "" { + h.Add("Transfer-Encoding", "chunked") + } + } + } + + if resp.StatusCode >= http.StatusOK && resp.StatusCode <= http.StatusPartialContent { + // prepare response content type + reqContentType, _ := hubutil.ReqContentTypeFrom(ctx) + respContentType := resp.Header.Get("Content-Type") + if len(respContentType) == 0 { + respContentType = reqContentType + } + ctx = hubutil.WithRespContentType(ctx, respContentType) + req = req.WithContext(ctx) + + // filter response data + if lb.filterManager != nil { + if ok, runner := lb.filterManager.FindRunner(req); ok { + wrapBody, needUncompressed := hubutil.NewGZipReaderCloser(resp.Header, resp.Body, req, "filter") + size, filterRc, err := runner.Filter(req, wrapBody, lb.stopCh) + if err != nil { + klog.Errorf("failed to filter response for %s, %v", hubutil.ReqString(req), err) + return err + } + resp.Body = filterRc + if size > 0 { + resp.ContentLength = int64(size) + resp.Header.Set("Content-Length", fmt.Sprint(size)) + } + + // after gunzip in filter, the header content encoding should be removed. + // because there's no need to gunzip response.body again. + if needUncompressed { + resp.Header.Del("Content-Encoding") + } + } + } + + if lb.workingMode == hubutil.WorkingModeEdge { + // cache resp with storage interface + lb.cacheResponse(req, resp) + } + } else if resp.StatusCode == http.StatusNotFound && info.Verb == "list" && lb.localCacheMgr != nil { + // 404 Not Found: The CRD may have been unregistered and should be updated locally as well. + // Other types of requests may return a 404 response for other reasons (for example, getting a pod that doesn't exist). + // And the main purpose is to return 404 when list an unregistered resource locally, so here only consider the list request. + gvr := schema.GroupVersionResource{ + Group: info.APIGroup, + Version: info.APIVersion, + Resource: info.Resource, + } + + err := lb.localCacheMgr.DeleteKindFor(gvr) + if err != nil { + klog.Errorf("failed: %v", err) + } + } + return nil +} + +func (lb *loadBalancer) cacheResponse(req *http.Request, resp *http.Response) { + if lb.localCacheMgr.CanCacheFor(req) { + ctx := req.Context() + wrapPrc, _ := hubutil.NewGZipReaderCloser(resp.Header, resp.Body, req, "cache-manager") + resp.Body = wrapPrc + + poolCacheManager, isHealthy := lb.coordinator.IsHealthy() + if isHealthy && poolCacheManager != nil { + if !isLeaderHubUserAgent(ctx) { + if isRequestOfNodeAndPod(ctx) { + // Currently, for request that does not come from "leader-yurthub", + // we only cache pod and node resources to pool-coordinator. + // Note: We do not allow the non-leader yurthub to cache pool-scoped resources + // into pool-coordinator to ensure that only one yurthub can update pool-scoped + // cache to avoid inconsistency of data. + lb.cacheToLocalAndPool(req, resp, poolCacheManager) + } else { + lb.cacheToLocal(req, resp) + } + } else { + if isPoolScopedCtx(ctx) { + // Leader Yurthub will always list/watch all resources, which contain may resource this + // node does not need. + lb.cacheToPool(req, resp, poolCacheManager) + } else { + klog.Errorf("failed to cache response for request %s, leader yurthub does not cache non-poolscoped resources.", hubutil.ReqString(req)) + } + } + return + } + + // When pool-coordinator is not healthy or not be enabled, we can + // only cache the response at local. + lb.cacheToLocal(req, resp) + } +} + +func (lb *loadBalancer) cacheToLocal(req *http.Request, resp *http.Response) { + ctx := req.Context() + req = req.WithContext(ctx) + rc, prc := hubutil.NewDualReadCloser(req, resp.Body, false) + go func(req *http.Request, prc io.ReadCloser, stopCh <-chan struct{}) { + if err := lb.localCacheMgr.CacheResponse(req, prc, stopCh); err != nil { + klog.Errorf("failed to cache req %s in local cache when cluster is unhealthy, %v", hubutil.ReqString(req), err) + } + }(req, prc, ctx.Done()) + req.Body = rc +} + +func (lb *loadBalancer) cacheToPool(req *http.Request, resp *http.Response, poolCacheManager cachemanager.CacheManager) { + ctx := req.Context() + req = req.WithContext(ctx) + rc, prc := hubutil.NewDualReadCloser(req, resp.Body, false) + go func(req *http.Request, prc io.ReadCloser, stopCh <-chan struct{}) { + if err := poolCacheManager.CacheResponse(req, prc, stopCh); err != nil { + klog.Errorf("failed to cache req %s in local cache when cluster is unhealthy, %v", hubutil.ReqString(req), err) + } + }(req, prc, ctx.Done()) + req.Body = rc +} + +func (lb *loadBalancer) cacheToLocalAndPool(req *http.Request, resp *http.Response, poolCacheMgr cachemanager.CacheManager) { + ctx := req.Context() + req = req.WithContext(ctx) + rc, prc1, prc2 := hubutil.NewTripleReadCloser(req, resp.Body, false) + go func(req *http.Request, prc io.ReadCloser, stopCh <-chan struct{}) { + if err := lb.localCacheMgr.CacheResponse(req, prc, stopCh); err != nil { + klog.Errorf("failed to cache req %s in local cache when cluster is unhealthy, %v", hubutil.ReqString(req), err) + } + }(req, prc1, ctx.Done()) + + if poolCacheMgr != nil { + go func(req *http.Request, prc io.ReadCloser, stopCh <-chan struct{}) { + if err := poolCacheMgr.CacheResponse(req, prc, stopCh); err != nil { + klog.Errorf("failed to cache req %s in pool cache when cluster is unhealthy, %v", hubutil.ReqString(req), err) + } + }(req, prc2, ctx.Done()) + } + req.Body = rc +} + +func isLeaderHubUserAgent(reqCtx context.Context) bool { + comp, hasComp := hubutil.ClientComponentFrom(reqCtx) + return hasComp && comp == coordinatorconstants.DefaultPoolScopedUserAgent +} + +func isPoolScopedCtx(reqCtx context.Context) bool { + poolScoped, hasPoolScoped := hubutil.IfPoolScopedResourceFrom(reqCtx) + return hasPoolScoped && poolScoped +} + +func isRequestOfNodeAndPod(reqCtx context.Context) bool { + reqInfo, ok := apirequest.RequestInfoFrom(reqCtx) + if !ok { + return false + } + + return (reqInfo.Resource == "nodes" && reqInfo.APIGroup == "" && reqInfo.APIVersion == "v1") || + (reqInfo.Resource == "pods" && reqInfo.APIGroup == "" && reqInfo.APIVersion == "v1") +} diff --git a/pkg/yurthub/proxy/remote/loadbalancer_test.go b/pkg/yurthub/proxy/remote/loadbalancer_test.go index ceff84da8d4..f0477362978 100644 --- a/pkg/yurthub/proxy/remote/loadbalancer_test.go +++ b/pkg/yurthub/proxy/remote/loadbalancer_test.go @@ -17,12 +17,41 @@ limitations under the License. package remote import ( + "context" + "net/http" "net/url" "testing" "github.com/openyurtio/openyurt/pkg/yurthub/healthchecker" + "github.com/openyurtio/openyurt/pkg/yurthub/proxy/util" + "github.com/openyurtio/openyurt/pkg/yurthub/transport" ) +var neverStop <-chan struct{} = context.Background().Done() + +type nopRoundTrip struct{} + +func (n *nopRoundTrip) RoundTrip(r *http.Request) (*http.Response, error) { + return &http.Response{ + Status: http.StatusText(http.StatusOK), + StatusCode: http.StatusOK, + }, nil +} + +type fakeTransportManager struct{} + +func (f *fakeTransportManager) CurrentTransport() http.RoundTripper { + return &nopRoundTrip{} +} + +func (f *fakeTransportManager) BearerTransport() http.RoundTripper { + return &nopRoundTrip{} +} + +func (f *fakeTransportManager) Close(_ string) {} + +var transportMgr transport.Interface = &fakeTransportManager{} + type PickBackend struct { DeltaRequestsCnt int ReturnServer string @@ -66,11 +95,13 @@ func TestRrLoadBalancerAlgo(t *testing.T) { checker := healthchecker.NewFakeChecker(true, map[string]int{}) for k, tc := range testcases { - backends := make([]*RemoteProxy, len(tc.Servers)) + backends := make([]*util.RemoteProxy, len(tc.Servers)) for i := range tc.Servers { + var err error u, _ := url.Parse(tc.Servers[i]) - backends[i] = &RemoteProxy{ - remoteServer: u, + backends[i], err = util.NewRemoteProxy(u, nil, nil, transportMgr, neverStop) + if err != nil { + t.Errorf("failed to create remote server for %s, %v", u.String(), err) } } @@ -80,20 +111,20 @@ func TestRrLoadBalancerAlgo(t *testing.T) { } for i := range tc.PickBackends { - var b *RemoteProxy + var b *util.RemoteProxy for j := 0; j < tc.PickBackends[i].DeltaRequestsCnt; j++ { b = rr.PickOne() } if len(tc.PickBackends[i].ReturnServer) == 0 { if b != nil { - t.Errorf("%s rr lb pick: expect no backend server, but got %s", k, b.remoteServer.String()) + t.Errorf("%s rr lb pick: expect no backend server, but got %s", k, b.RemoteServer().String()) } } else { if b == nil { t.Errorf("%s rr lb pick: expect backend server: %s, but got no backend server", k, tc.PickBackends[i].ReturnServer) - } else if b.remoteServer.String() != tc.PickBackends[i].ReturnServer { - t.Errorf("%s rr lb pick(round %d): expect backend server: %s, but got %s", k, i+1, tc.PickBackends[i].ReturnServer, b.remoteServer.String()) + } else if b.RemoteServer().String() != tc.PickBackends[i].ReturnServer { + t.Errorf("%s rr lb pick(round %d): expect backend server: %s, but got %s", k, i+1, tc.PickBackends[i].ReturnServer, b.RemoteServer().String()) } } } @@ -127,11 +158,13 @@ func TestRrLoadBalancerAlgoWithReverseHealthy(t *testing.T) { "http://127.0.0.1:8081": 2, }) for k, tc := range testcases { - backends := make([]*RemoteProxy, len(tc.Servers)) + backends := make([]*util.RemoteProxy, len(tc.Servers)) for i := range tc.Servers { + var err error u, _ := url.Parse(tc.Servers[i]) - backends[i] = &RemoteProxy{ - remoteServer: u, + backends[i], err = util.NewRemoteProxy(u, nil, nil, transportMgr, neverStop) + if err != nil { + t.Errorf("failed to create remote server for %s, %v", u.String(), err) } } @@ -141,20 +174,20 @@ func TestRrLoadBalancerAlgoWithReverseHealthy(t *testing.T) { } for i := range tc.PickBackends { - var b *RemoteProxy + var b *util.RemoteProxy for j := 0; j < tc.PickBackends[i].DeltaRequestsCnt; j++ { b = rr.PickOne() } if len(tc.PickBackends[i].ReturnServer) == 0 { if b != nil { - t.Errorf("%s rr lb pick: expect no backend server, but got %s", k, b.remoteServer.String()) + t.Errorf("%s rr lb pick: expect no backend server, but got %s", k, b.RemoteServer().String()) } } else { if b == nil { t.Errorf("%s rr lb pick(round %d): expect backend server: %s, but got no backend server", k, i+1, tc.PickBackends[i].ReturnServer) - } else if b.remoteServer.String() != tc.PickBackends[i].ReturnServer { - t.Errorf("%s rr lb pick(round %d): expect backend server: %s, but got %s", k, i+1, tc.PickBackends[i].ReturnServer, b.remoteServer.String()) + } else if b.RemoteServer().String() != tc.PickBackends[i].ReturnServer { + t.Errorf("%s rr lb pick(round %d): expect backend server: %s, but got %s", k, i+1, tc.PickBackends[i].ReturnServer, b.RemoteServer().String()) } } } @@ -199,11 +232,13 @@ func TestPriorityLoadBalancerAlgo(t *testing.T) { checker := healthchecker.NewFakeChecker(true, map[string]int{}) for k, tc := range testcases { - backends := make([]*RemoteProxy, len(tc.Servers)) + backends := make([]*util.RemoteProxy, len(tc.Servers)) for i := range tc.Servers { + var err error u, _ := url.Parse(tc.Servers[i]) - backends[i] = &RemoteProxy{ - remoteServer: u, + backends[i], err = util.NewRemoteProxy(u, nil, nil, transportMgr, neverStop) + if err != nil { + t.Errorf("failed to create remote server for %s, %v", u.String(), err) } } @@ -213,20 +248,20 @@ func TestPriorityLoadBalancerAlgo(t *testing.T) { } for i := range tc.PickBackends { - var b *RemoteProxy + var b *util.RemoteProxy for j := 0; j < tc.PickBackends[i].DeltaRequestsCnt; j++ { b = rr.PickOne() } if len(tc.PickBackends[i].ReturnServer) == 0 { if b != nil { - t.Errorf("%s priority lb pick: expect no backend server, but got %s", k, b.remoteServer.String()) + t.Errorf("%s priority lb pick: expect no backend server, but got %s", k, b.RemoteServer().String()) } } else { if b == nil { t.Errorf("%s priority lb pick: expect backend server: %s, but got no backend server", k, tc.PickBackends[i].ReturnServer) - } else if b.remoteServer.String() != tc.PickBackends[i].ReturnServer { - t.Errorf("%s priority lb pick(round %d): expect backend server: %s, but got %s", k, i+1, tc.PickBackends[i].ReturnServer, b.remoteServer.String()) + } else if b.RemoteServer().String() != tc.PickBackends[i].ReturnServer { + t.Errorf("%s priority lb pick(round %d): expect backend server: %s, but got %s", k, i+1, tc.PickBackends[i].ReturnServer, b.RemoteServer().String()) } } } @@ -258,11 +293,13 @@ func TestPriorityLoadBalancerAlgoWithReverseHealthy(t *testing.T) { "http://127.0.0.1:8080": 2, "http://127.0.0.1:8081": 3}) for k, tc := range testcases { - backends := make([]*RemoteProxy, len(tc.Servers)) + backends := make([]*util.RemoteProxy, len(tc.Servers)) for i := range tc.Servers { + var err error u, _ := url.Parse(tc.Servers[i]) - backends[i] = &RemoteProxy{ - remoteServer: u, + backends[i], err = util.NewRemoteProxy(u, nil, nil, transportMgr, neverStop) + if err != nil { + t.Errorf("failed to create remote server for %s, %v", u.String(), err) } } @@ -272,20 +309,20 @@ func TestPriorityLoadBalancerAlgoWithReverseHealthy(t *testing.T) { } for i := range tc.PickBackends { - var b *RemoteProxy + var b *util.RemoteProxy for j := 0; j < tc.PickBackends[i].DeltaRequestsCnt; j++ { b = rr.PickOne() } if len(tc.PickBackends[i].ReturnServer) == 0 { if b != nil { - t.Errorf("%s priority lb pick: expect no backend server, but got %s", k, b.remoteServer.String()) + t.Errorf("%s priority lb pick: expect no backend server, but got %s", k, b.RemoteServer().String()) } } else { if b == nil { t.Errorf("%s priority lb pick: expect backend server: %s, but got no backend server", k, tc.PickBackends[i].ReturnServer) - } else if b.remoteServer.String() != tc.PickBackends[i].ReturnServer { - t.Errorf("%s priority lb pick(round %d): expect backend server: %s, but got %s", k, i+1, tc.PickBackends[i].ReturnServer, b.remoteServer.String()) + } else if b.RemoteServer().String() != tc.PickBackends[i].ReturnServer { + t.Errorf("%s priority lb pick(round %d): expect backend server: %s, but got %s", k, i+1, tc.PickBackends[i].ReturnServer, b.RemoteServer().String()) } } } diff --git a/pkg/yurthub/proxy/remote/remote.go b/pkg/yurthub/proxy/util/remote.go similarity index 50% rename from pkg/yurthub/proxy/remote/remote.go rename to pkg/yurthub/proxy/util/remote.go index 5a4e0f5ceff..ff30b688cd0 100644 --- a/pkg/yurthub/proxy/remote/remote.go +++ b/pkg/yurthub/proxy/util/remote.go @@ -14,27 +14,19 @@ See the License for the specific language governing permissions and limitations under the License. */ -package remote +package util import ( - "context" - "errors" "fmt" - "io" "net/http" "net/http/httputil" "net/url" "strings" - "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/util/httpstream" "k8s.io/apimachinery/pkg/util/proxy" - apirequest "k8s.io/apiserver/pkg/endpoints/request" "k8s.io/klog/v2" - "github.com/openyurtio/openyurt/pkg/yurthub/cachemanager" - "github.com/openyurtio/openyurt/pkg/yurthub/filter/manager" - "github.com/openyurtio/openyurt/pkg/yurthub/proxy/util" "github.com/openyurtio/openyurt/pkg/yurthub/transport" hubutil "github.com/openyurtio/openyurt/pkg/yurthub/util" ) @@ -42,9 +34,7 @@ import ( // RemoteProxy is an reverse proxy for remote server type RemoteProxy struct { reverseProxy *httputil.ReverseProxy - cacheMgr cachemanager.CacheManager remoteServer *url.URL - filterManager *manager.Manager currentTransport http.RoundTripper bearerTransport http.RoundTripper upgradeHandler *proxy.UpgradeAwareHandler @@ -61,9 +51,9 @@ func (r *responder) Error(w http.ResponseWriter, req *http.Request, err error) { // NewRemoteProxy creates an *RemoteProxy object, and will be used by LoadBalancer func NewRemoteProxy(remoteServer *url.URL, - cacheMgr cachemanager.CacheManager, + modifyResponse func(*http.Response) error, + errhandler func(http.ResponseWriter, *http.Request, error), transportMgr transport.Interface, - filterManager *manager.Manager, stopCh <-chan struct{}) (*RemoteProxy, error) { currentTransport := transportMgr.CurrentTransport() if currentTransport == nil { @@ -81,9 +71,7 @@ func NewRemoteProxy(remoteServer *url.URL, proxyBackend := &RemoteProxy{ reverseProxy: httputil.NewSingleHostReverseProxy(remoteServer), - cacheMgr: cacheMgr, remoteServer: remoteServer, - filterManager: filterManager, currentTransport: currentTransport, bearerTransport: bearerTransport, upgradeHandler: upgradeAwareHandler, @@ -92,9 +80,9 @@ func NewRemoteProxy(remoteServer *url.URL, } proxyBackend.reverseProxy.Transport = proxyBackend - proxyBackend.reverseProxy.ModifyResponse = proxyBackend.modifyResponse + proxyBackend.reverseProxy.ModifyResponse = modifyResponse proxyBackend.reverseProxy.FlushInterval = -1 - proxyBackend.reverseProxy.ErrorHandler = proxyBackend.errorHandler + proxyBackend.reverseProxy.ErrorHandler = errhandler return proxyBackend, nil } @@ -122,110 +110,6 @@ func (rp *RemoteProxy) ServeHTTP(rw http.ResponseWriter, req *http.Request) { rp.reverseProxy.ServeHTTP(rw, req) } -func (rp *RemoteProxy) modifyResponse(resp *http.Response) error { - if resp == nil || resp.Request == nil { - klog.Infof("no request info in response, skip cache response") - return nil - } - - req := resp.Request - ctx := req.Context() - - // re-added transfer-encoding=chunked response header for watch request - info, exists := apirequest.RequestInfoFrom(ctx) - if exists { - if info.Verb == "watch" { - klog.V(5).Infof("add transfer-encoding=chunked header into response for req %s", hubutil.ReqString(req)) - h := resp.Header - if hv := h.Get("Transfer-Encoding"); hv == "" { - h.Add("Transfer-Encoding", "chunked") - } - } - } - - if resp.StatusCode >= http.StatusOK && resp.StatusCode <= http.StatusPartialContent { - // prepare response content type - reqContentType, _ := hubutil.ReqContentTypeFrom(ctx) - respContentType := resp.Header.Get("Content-Type") - if len(respContentType) == 0 { - respContentType = reqContentType - } - ctx = hubutil.WithRespContentType(ctx, respContentType) - req = req.WithContext(ctx) - - // filter response data - if rp.filterManager != nil { - if ok, runner := rp.filterManager.FindRunner(req); ok { - wrapBody, needUncompressed := hubutil.NewGZipReaderCloser(resp.Header, resp.Body, req, "filter") - size, filterRc, err := runner.Filter(req, wrapBody, rp.stopCh) - if err != nil { - klog.Errorf("failed to filter response for %s, %v", hubutil.ReqString(req), err) - return err - } - resp.Body = filterRc - if size > 0 { - resp.ContentLength = int64(size) - resp.Header.Set("Content-Length", fmt.Sprint(size)) - } - - // after gunzip in filter, the header content encoding should be removed. - // because there's no need to gunzip response.body again. - if needUncompressed { - resp.Header.Del("Content-Encoding") - } - } - } - - // cache resp with storage interface - if rp.cacheMgr != nil && rp.cacheMgr.CanCacheFor(req) { - rc, prc := hubutil.NewDualReadCloser(req, resp.Body, true) - wrapPrc, _ := hubutil.NewGZipReaderCloser(resp.Header, prc, req, "cache-manager") - go func(req *http.Request, prc io.ReadCloser, stopCh <-chan struct{}) { - err := rp.cacheMgr.CacheResponse(req, prc, stopCh) - if err != nil && err != io.EOF && !errors.Is(err, context.Canceled) { - klog.Errorf("%s response cache ended with error, %v", hubutil.ReqString(req), err) - } - }(req, wrapPrc, rp.stopCh) - - resp.Body = rc - } - } else if resp.StatusCode == http.StatusNotFound && info.Verb == "list" && rp.cacheMgr != nil { - // 404 Not Found: The CRD may have been unregistered and should be updated locally as well. - // Other types of requests may return a 404 response for other reasons (for example, getting a pod that doesn't exist). - // And the main purpose is to return 404 when list an unregistered resource locally, so here only consider the list request. - gvr := schema.GroupVersionResource{ - Group: info.APIGroup, - Version: info.APIVersion, - Resource: info.Resource, - } - - err := rp.cacheMgr.DeleteKindFor(gvr) - if err != nil { - klog.Errorf("failed: %v", err) - } - } - return nil -} - -func (rp *RemoteProxy) errorHandler(rw http.ResponseWriter, req *http.Request, err error) { - klog.Errorf("remote proxy error handler: %s, %v", hubutil.ReqString(req), err) - if rp.cacheMgr == nil || !rp.cacheMgr.CanCacheFor(req) { - rw.WriteHeader(http.StatusBadGateway) - return - } - - ctx := req.Context() - if info, ok := apirequest.RequestInfoFrom(ctx); ok { - if info.Verb == "get" || info.Verb == "list" { - if obj, err := rp.cacheMgr.QueryCache(req); err == nil { - util.WriteObject(http.StatusOK, obj, rw, req) - return - } - } - } - rw.WriteHeader(http.StatusBadGateway) -} - // RoundTrip is used to implement http.RoundTripper for RemoteProxy. func (rp *RemoteProxy) RoundTrip(req *http.Request) (*http.Response, error) { // when edge client(like kube-proxy, flannel, etc) use service account(default InClusterConfig) to access yurthub, diff --git a/pkg/yurthub/proxy/util/util.go b/pkg/yurthub/proxy/util/util.go index ee5efeec407..76632db1605 100644 --- a/pkg/yurthub/proxy/util/util.go +++ b/pkg/yurthub/proxy/util/util.go @@ -176,6 +176,33 @@ func WithRequestClientComponent(handler http.Handler) http.Handler { }) } +func WithIfPoolScopedResource(handler http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + ctx := req.Context() + if info, ok := apirequest.RequestInfoFrom(ctx); ok { + var ifPoolScopedResource bool + if info.IsResourceRequest && isPoolScopedResource(info) { + ifPoolScopedResource = true + } + ctx = util.WithIfPoolScopedResource(ctx, ifPoolScopedResource) + req = req.WithContext(ctx) + } + handler.ServeHTTP(w, req) + }) +} + +func isPoolScopedResource(info *apirequest.RequestInfo) bool { + if info != nil { + if info.APIGroup == "" && info.APIVersion == "v1" && info.Resource == "endpoints" { + return true + } + if info.APIGroup == "discovery.k8s.io" && info.APIVersion == "v1" && info.Resource == "endpointslices" { + return true + } + } + return false +} + type wrapperResponseWriter struct { http.ResponseWriter http.Flusher @@ -433,3 +460,44 @@ func Err(err error, w http.ResponseWriter, req *http.Request) { klog.Errorf("request info is not found when err write, %s", util.ReqString(req)) } + +func IsPoolScopedResouceListWatchRequest(req *http.Request) bool { + ctx := req.Context() + info, ok := apirequest.RequestInfoFrom(ctx) + if !ok { + return false + } + + isPoolScopedResource, ok := util.IfPoolScopedResourceFrom(ctx) + return ok && isPoolScopedResource && (info.Verb == "list" || info.Verb == "watch") +} + +func IsSubjectAccessReviewCreateGetRequest(req *http.Request) bool { + ctx := req.Context() + info, ok := apirequest.RequestInfoFrom(ctx) + if !ok { + return false + } + + comp, ok := util.ClientComponentFrom(ctx) + if !ok { + return false + } + + return info.IsResourceRequest && + comp == "kubelet" && + info.Resource == "subjectaccessreviews" && + (info.Verb == "create" || info.Verb == "get") +} + +func IsEventCreateRequest(req *http.Request) bool { + ctx := req.Context() + info, ok := apirequest.RequestInfoFrom(ctx) + if !ok { + return false + } + + return info.IsResourceRequest && + info.Resource == "events" && + info.Verb == "create" +} diff --git a/pkg/yurthub/storage/disk/storage.go b/pkg/yurthub/storage/disk/storage.go index e7368904cee..794bff3b65f 100644 --- a/pkg/yurthub/storage/disk/storage.go +++ b/pkg/yurthub/storage/disk/storage.go @@ -104,7 +104,7 @@ func (ds *diskStorage) Name() string { // Create will create a new file with content. key indicates the path of the file. func (ds *diskStorage) Create(key storage.Key, content []byte) error { - if err := utils.ValidateKey(key, storageKey{}); err != nil { + if err := utils.ValidateKV(key, content, storageKey{}); err != nil { return err } storageKey := key.(storageKey) diff --git a/pkg/yurthub/storage/etcd/etcd_suite_test.go b/pkg/yurthub/storage/etcd/etcd_suite_test.go new file mode 100644 index 00000000000..160132c3aa4 --- /dev/null +++ b/pkg/yurthub/storage/etcd/etcd_suite_test.go @@ -0,0 +1,58 @@ +/* +Copyright 2022 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package etcd + +import ( + "os" + "os/exec" + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var keyCacheDir = "/tmp/etcd-test" +var etcdDataDir = "/tmp/storagetest.etcd" +var devNull *os.File +var etcdCmd *exec.Cmd + +var _ = BeforeSuite(func() { + Expect(os.RemoveAll(keyCacheDir)).To(BeNil()) + Expect(os.RemoveAll(etcdDataDir)).To(BeNil()) + + // start etcd + var err error + devNull, err = os.OpenFile("/dev/null", os.O_RDWR, 0755) + Expect(err).To(BeNil()) + etcdCmd = exec.Command("/usr/local/etcd/etcd", "--data-dir="+etcdDataDir) + etcdCmd.Stdout = devNull + etcdCmd.Stderr = devNull + Expect(etcdCmd.Start()).To(BeNil()) +}) + +var _ = AfterSuite(func() { + Expect(os.RemoveAll(keyCacheDir)).To(BeNil()) + + // stop etcd + Expect(etcdCmd.Process.Kill()).To(BeNil()) + Expect(devNull.Close()).To(BeNil()) +}) + +func TestEtcd(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "ComponentKeyCache Test Suite") +} diff --git a/pkg/yurthub/storage/etcd/key.go b/pkg/yurthub/storage/etcd/key.go new file mode 100644 index 00000000000..958ea8c7905 --- /dev/null +++ b/pkg/yurthub/storage/etcd/key.go @@ -0,0 +1,78 @@ +/* +Copyright 2022 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package etcd + +import ( + "errors" + "path/filepath" + + "k8s.io/apimachinery/pkg/api/validation/path" + "k8s.io/apimachinery/pkg/runtime/schema" + + "github.com/openyurtio/openyurt/pkg/yurthub/storage" +) + +// SpecialDefaultResourcePrefixes are prefixes compiled into Kubernetes. +// refer to SpecialDefaultResourcePrefixes in k8s.io/pkg/kubeapiserver/default_storage_factory_builder.go +var SpecialDefaultResourcePrefixes = map[schema.GroupResource]string{ + {Group: "", Resource: "replicationcontrollers"}: "controllers", + {Group: "", Resource: "endpoints"}: "services/endpoints", + {Group: "", Resource: "nodes"}: "minions", + {Group: "", Resource: "services"}: "services/specs", + {Group: "extensions", Resource: "ingresses"}: "ingress", + {Group: "networking.k8s.io", Resource: "ingresses"}: "ingress", + {Group: "extensions", Resource: "podsecuritypolicies"}: "podsecuritypolicy", + {Group: "policy", Resource: "podsecuritypolicies"}: "podsecuritypolicy", +} + +type storageKey struct { + comp string + path string +} + +func (k storageKey) Key() string { + return k.path +} + +func (k storageKey) component() string { + return k.comp +} + +func (s *etcdStorage) KeyFunc(info storage.KeyBuildInfo) (storage.Key, error) { + if info.Component == "" { + return nil, storage.ErrEmptyComponent + } + if info.Resources == "" { + return nil, storage.ErrEmptyResource + } + if errStrs := path.IsValidPathSegmentName(info.Name); len(errStrs) != 0 { + return nil, errors.New(errStrs[0]) + } + + resource := info.Resources + gr := schema.GroupResource{Group: info.Group, Resource: info.Resources} + if override, ok := SpecialDefaultResourcePrefixes[gr]; ok { + resource = override + } + + path := filepath.Join(s.prefix, resource, info.Namespace, info.Name) + + return storageKey{ + comp: info.Component, + path: path, + }, nil +} diff --git a/pkg/yurthub/storage/etcd/key_test.go b/pkg/yurthub/storage/etcd/key_test.go new file mode 100644 index 00000000000..bc59676282d --- /dev/null +++ b/pkg/yurthub/storage/etcd/key_test.go @@ -0,0 +1,103 @@ +/* +Copyright 2022 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package etcd + +import ( + "testing" + + "github.com/openyurtio/openyurt/pkg/yurthub/storage" +) + +var s = etcdStorage{ + prefix: "/registry", +} + +var keyFunc = s.KeyFunc + +func TestKeyFunc(t *testing.T) { + cases := map[string]struct { + info storage.KeyBuildInfo + key string + err error + }{ + "core group normal case": { + info: storage.KeyBuildInfo{ + Group: "", + Resources: "pods", + Version: "v1", + Namespace: "test", + Name: "test-pod", + }, + key: "/registry/pods/test/test-pod", + }, + + "special prefix for node resource": { + info: storage.KeyBuildInfo{ + Group: "", + Resources: "nodes", + Version: "v1", + Namespace: "", + Name: "test-node", + }, + key: "/registry/minions/test-node", + }, + "not core group": { + info: storage.KeyBuildInfo{ + Group: "apps", + Resources: "deployments", + Version: "v1", + Namespace: "test", + Name: "test-deploy", + }, + key: "/registry/deployments/test/test-deploy", + }, + "special prefix for service resource": { + info: storage.KeyBuildInfo{ + Group: "networking.k8s.io", + Resources: "ingresses", + Version: "v1", + Namespace: "test", + Name: "test-ingress", + }, + key: "/registry/ingress/test/test-ingress", + }, + "empty resources": { + info: storage.KeyBuildInfo{ + Group: "", + Resources: "", + Version: "v1", + Namespace: "", + Name: "", + }, + err: storage.ErrEmptyResource, + }, + } + + for n, c := range cases { + key, err := keyFunc(c.info) + if err != c.err { + t.Errorf("unexpected error in case %s, want: %v, got: %v", n, c.err, err) + continue + } + if err != nil { + continue + } + if key.Key() != c.key { + t.Errorf("unexpected key in case %s, want: %s, got: %s", n, c.key, key) + } + } +} diff --git a/pkg/yurthub/storage/etcd/keycache.go b/pkg/yurthub/storage/etcd/keycache.go new file mode 100644 index 00000000000..5a9c21ca227 --- /dev/null +++ b/pkg/yurthub/storage/etcd/keycache.go @@ -0,0 +1,280 @@ +/* +Copyright 2022 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package etcd + +import ( + "bytes" + "context" + "fmt" + "strings" + "sync" + + clientv3 "go.etcd.io/etcd/client/v3" + + coordinatorconstants "github.com/openyurtio/openyurt/pkg/yurthub/poolcoordinator/constants" + "github.com/openyurtio/openyurt/pkg/yurthub/storage" + "github.com/openyurtio/openyurt/pkg/yurthub/util/fs" +) + +// type state int + +// const ( +// done state = 0 +// processing state = 1 +// ) + +// type status struct + +type keySet struct { + m map[storageKey]struct{} +} + +// Difference will return keys in s but not in s2 +func (s keySet) Difference(s2 keySet) []storageKey { + keys := []storageKey{} + if s2.m == nil { + for k := range s.m { + keys = append(keys, k) + } + return keys + } + + for k := range s.m { + if _, ok := s2.m[k]; !ok { + keys = append(keys, k) + } + } + return keys +} + +// Do not directly modify value returned from functions of componentKeyCache, such as Load. +// Because it usually returns reference of internal objects for efficiency. +// The format in file is: +// component0:key0,key1... +// component1:key0,key1... +// ... +type componentKeyCache struct { + sync.Mutex + ctx context.Context + cache map[string]keySet + filePath string + keyFunc func(storage.KeyBuildInfo) (storage.Key, error) + fsOperator fs.FileSystemOperator + getEtcdClient func() *clientv3.Client +} + +func (c *componentKeyCache) Recover() error { + var buf []byte + var err error + if buf, err = c.fsOperator.Read(c.filePath); err == fs.ErrNotExists { + if err := c.fsOperator.CreateFile(c.filePath, []byte{}); err != nil { + return fmt.Errorf("failed to create cache file at %s, %v", c.filePath, err) + } + return nil + } else if err != nil { + return fmt.Errorf("failed to recover key cache from %s, %v", c.filePath, err) + } + + // successfully read from file + if len(buf) == 0 { + return nil + } + lines := strings.Split(string(buf), "\n") + for i, l := range lines { + s := strings.Split(l, ":") + if len(s) != 2 { + return fmt.Errorf("failed to parse line %d, invalid format", i) + } + comp, keys := s[0], strings.Split(s[1], ",") + ks := keySet{m: map[storageKey]struct{}{}} + for _, key := range keys { + ks.m[storageKey{path: key}] = struct{}{} + } + c.cache[comp] = ks + } + + poolScopedKeyset, err := c.getPoolScopedKeyset() + if err != nil { + return fmt.Errorf("failed to get pool-scoped keys, %v", err) + } + // Overwrite the data we recovered from local disk, if any. Because we + // only respect to the resources stored in pool-coordinator to recover the + // pool-scoped keys. + c.cache[coordinatorconstants.DefaultPoolScopedUserAgent] = *poolScopedKeyset + + return nil +} + +func (c *componentKeyCache) getPoolScopedKeyset() (*keySet, error) { + client := c.getEtcdClient() + if client == nil { + return nil, fmt.Errorf("got empty etcd client") + } + + keys := &keySet{m: map[storageKey]struct{}{}} + for gvr := range coordinatorconstants.PoolScopedResources { + getCtx, cancel := context.WithTimeout(c.ctx, defaultTimeout) + defer cancel() + rootKey, err := c.keyFunc(storage.KeyBuildInfo{ + Component: coordinatorconstants.DefaultPoolScopedUserAgent, + Group: gvr.Group, + Version: gvr.Version, + Resources: gvr.Resource, + }) + if err != nil { + return nil, fmt.Errorf("failed to generate keys for %s, %v", gvr.String(), err) + } + getResp, err := client.Get(getCtx, rootKey.Key(), clientv3.WithPrefix(), clientv3.WithKeysOnly()) + if err != nil { + return nil, fmt.Errorf("failed to get from etcd for %s, %v", gvr.String(), err) + } + + for _, kv := range getResp.Kvs { + ns, name, err := getNamespaceAndNameFromKeyPath(string(kv.Key)) + if err != nil { + return nil, fmt.Errorf("failed to parse namespace and name of %s", kv.Key) + } + key, err := c.keyFunc(storage.KeyBuildInfo{ + Component: coordinatorconstants.DefaultPoolScopedUserAgent, + Group: gvr.Group, + Version: gvr.Version, + Resources: gvr.Resource, + Namespace: ns, + Name: name, + }) + if err != nil { + return nil, fmt.Errorf("failed to create resource key for %v", kv.Key) + } + keys.m[key.(storageKey)] = struct{}{} + } + } + return keys, nil +} + +func (c *componentKeyCache) Load(component string) (keySet, bool) { + c.Lock() + defer c.Unlock() + cache, ok := c.cache[component] + return cache, ok +} + +func (c *componentKeyCache) AddKey(component string, key storageKey) { + c.Lock() + defer c.Unlock() + defer c.flush() + if _, ok := c.cache[component]; !ok { + c.cache[component] = keySet{m: map[storageKey]struct{}{ + key: {}, + }} + return + } + + keyset := c.cache[component] + if keyset.m == nil { + keyset.m = map[storageKey]struct{}{ + key: {}, + } + return + } + + c.cache[component].m[key] = struct{}{} +} + +func (c *componentKeyCache) DeleteKey(component string, key storageKey) { + c.Lock() + defer c.Unlock() + delete(c.cache[component].m, key) + c.flush() +} + +func (c *componentKeyCache) LoadOrStore(component string, keyset keySet) (keySet, bool) { + c.Lock() + defer c.Unlock() + if cache, ok := c.cache[component]; ok { + return cache, true + } else { + c.cache[component] = keyset + c.flush() + return keyset, false + } +} + +func (c *componentKeyCache) LoadAndDelete(component string) (keySet, bool) { + c.Lock() + defer c.Unlock() + if cache, ok := c.cache[component]; ok { + delete(c.cache, component) + c.flush() + return cache, true + } + return keySet{}, false +} + +func (c *componentKeyCache) DeleteAllKeysOfComponent(component string) { + c.Lock() + defer c.Unlock() + delete(c.cache, component) + c.flush() +} + +// func (c *componentKeyCache) MarkAsProcessing() { + +// } + +// func (c *componentKeyCache) MarkAsDone() { + +// } + +func (c *componentKeyCache) flush() error { + buf := bytes.NewBuffer(make([]byte, 0, 1024)) + for comp, ks := range c.cache { + line := bytes.NewBufferString(fmt.Sprintf("%s:", comp)) + keys := make([]string, 0, len(ks.m)) + for k := range ks.m { + keys = append(keys, k.Key()) + } + line.WriteString(strings.Join(keys, ",")) + line.WriteByte('\n') + buf.Write(line.Bytes()) + } + if buf.Len() != 0 { + // discard last '\n' + buf.Truncate(buf.Len() - 1) + } + if err := c.fsOperator.Write(c.filePath, buf.Bytes()); err != nil { + return fmt.Errorf("failed to flush cache to file %s, %v", c.filePath, err) + } + return nil +} + +func newComponentKeyCache(filePath string) *componentKeyCache { + return &componentKeyCache{ + filePath: filePath, + cache: map[string]keySet{}, + fsOperator: fs.FileSystemOperator{}, + } +} + +// We assume that path points to a namespaced resource. +func getNamespaceAndNameFromKeyPath(path string) (string, string, error) { + elems := strings.Split(path, "/") + if len(elems) < 2 { + return "", "", fmt.Errorf("unrecognized path: %v", path) + } + + return elems[len(elems)-2], elems[len(elems)-1], nil +} diff --git a/pkg/yurthub/storage/etcd/keycache_test.go b/pkg/yurthub/storage/etcd/keycache_test.go new file mode 100644 index 00000000000..c9512e221e3 --- /dev/null +++ b/pkg/yurthub/storage/etcd/keycache_test.go @@ -0,0 +1,311 @@ +/* +Copyright 2022 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package etcd + +import ( + "os" + "path/filepath" + "reflect" + "strings" + "testing" + + "github.com/google/uuid" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/openyurtio/openyurt/pkg/yurthub/util/fs" +) + +var _ = Describe("Test componentKeyCache setup", func() { + var cache *componentKeyCache + var fileName string + var f fs.FileSystemOperator + BeforeEach(func() { + fileName = uuid.New().String() + cache = newComponentKeyCache(filepath.Join(keyCacheDir, fileName)) + }) + AfterEach(func() { + Expect(os.RemoveAll(filepath.Join(keyCacheDir, fileName))) + }) + + It("should recover when cache file does not exist", func() { + Expect(cache.Recover()).To(BeNil()) + Expect(len(cache.cache)).To(BeZero()) + }) + + It("should recover when cache file is empty", func() { + Expect(f.CreateFile(filepath.Join(keyCacheDir, fileName), []byte{})).To(BeNil()) + Expect(cache.Recover()).To(BeNil()) + Expect(len(cache.cache)).To(BeZero()) + }) + + It("should recover when cache file exists and contains valid data", func() { + Expect(f.CreateFile(filepath.Join(keyCacheDir, fileName), []byte( + "kubelet:/registry/pods/default/pod1,/registry/pods/default/pod2\n"+ + "kube-proxy:/registry/configmaps/kube-system/kube-proxy", + ))).To(BeNil()) + Expect(cache.Recover()).To(BeNil()) + Expect(cache.cache).To(Equal(map[string]keySet{ + "kubelet": { + m: map[storageKey]struct{}{ + {path: "/registry/pods/default/pod1"}: {}, + {path: "/registry/pods/default/pod2"}: {}, + }, + }, + "kube-proxy": { + m: map[storageKey]struct{}{ + {path: "/registry/configmaps/kube-system/kube-proxy"}: {}, + }, + }, + })) + }) + + It("should return err when cache file contains invalid data", func() { + Expect(f.CreateFile(filepath.Join(keyCacheDir, fileName), []byte( + "kubelet,/registry/pods/default/pod1", + ))).To(BeNil()) + Expect(cache.Recover()).NotTo(BeNil()) + }) +}) + +var _ = Describe("Test componentKeyCache function", func() { + var cache *componentKeyCache + var fileName string + var key1, key2, key3 storageKey + BeforeEach(func() { + fileName = uuid.New().String() + cache = newComponentKeyCache(filepath.Join(keyCacheDir, fileName)) + key1 = storageKey{ + path: "/registry/pods/default/pod1", + } + key2 = storageKey{ + path: "/registry/pods/default/pod2", + } + key3 = storageKey{ + path: "/registry/pods/kube-system/kube-proxy", + } + }) + AfterEach(func() { + Expect(os.RemoveAll(filepath.Join(keyCacheDir, fileName))).To(BeNil()) + }) + + Context("Test Load", func() { + BeforeEach(func() { + cache.Recover() + cache.cache = map[string]keySet{ + "kubelet": { + m: map[storageKey]struct{}{ + key1: {}, + key2: {}, + }, + }, + } + cache.flush() + }) + It("should return nil,false if component is not in cache", func() { + c, found := cache.Load("kube-proxy") + Expect(c.m).To(BeNil()) + Expect(found).To(BeFalse()) + }) + It("should return keyset,true if component is in cache", func() { + c, found := cache.Load("kubelet") + Expect(c.m).To(Equal(map[storageKey]struct{}{ + key1: {}, + key2: {}, + })) + Expect(found).To(BeTrue()) + }) + }) + + Context("Test LoadAndDelete", func() { + BeforeEach(func() { + cache.Recover() + cache.cache = map[string]keySet{ + "kubelet": { + m: map[storageKey]struct{}{ + key1: {}, + key2: {}, + }, + }, + "kube-proxy": { + m: map[storageKey]struct{}{ + key3: {}, + }, + }, + } + cache.flush() + }) + It("should return nil,false if component is not in cache", func() { + c, found := cache.LoadAndDelete("foo") + Expect(c.m).To(BeNil()) + Expect(found).To(BeFalse()) + }) + It("should return keyset,true and delete cache for this component if exists", func() { + c, found := cache.LoadAndDelete("kubelet") + Expect(c.m).To(Equal(map[storageKey]struct{}{ + key1: {}, + key2: {}, + })) + Expect(found).To(BeTrue()) + Expect(cache.cache).To(Equal(map[string]keySet{ + "kube-proxy": { + m: map[storageKey]struct{}{ + key3: {}, + }, + }, + })) + data, err := os.ReadFile(cache.filePath) + Expect(err).To(BeNil()) + Expect(data).To(Equal([]byte( + "kube-proxy:" + key3.path, + ))) + }) + }) + Context("Test LoadOrStore", func() { + BeforeEach(func() { + cache.Recover() + cache.cache = map[string]keySet{ + "kubelet": { + m: map[storageKey]struct{}{ + key1: {}, + key2: {}, + }, + }, + } + cache.flush() + }) + It("should return data,false and store data if component currently does not in cache", func() { + c, found := cache.LoadOrStore("kube-proxy", keySet{ + m: map[storageKey]struct{}{ + key3: {}, + }, + }) + Expect(found).To(BeFalse()) + Expect(c.m).To(Equal(map[storageKey]struct{}{ + key3: {}, + })) + buf, err := os.ReadFile(cache.filePath) + Expect(err).To(BeNil()) + Expect(strings.Split(string(buf), "\n")).To(HaveLen(2)) + }) + It("should return original data and true if component already exists in cache", func() { + c, found := cache.LoadOrStore("kubelet", keySet{ + m: map[storageKey]struct{}{ + key3: {}, + }, + }) + Expect(found).To(BeTrue()) + Expect(c.m).To(Equal(map[storageKey]struct{}{ + key1: {}, + key2: {}, + })) + buf, err := os.ReadFile(cache.filePath) + Expect(err).To(BeNil()) + Expect(strings.Split(string(buf), "\n")).To(HaveLen(1)) + }) + }) +}) + +func TestKeySetDifference(t *testing.T) { + key1 := storageKey{path: "/registry/pods/test/test-pod"} + key2 := storageKey{path: "/registry/pods/test/test-pod2"} + key3 := storageKey{path: "/registry/pods/test/test-pod3"} + cases := []struct { + description string + s1 keySet + s2 keySet + want map[storageKey]struct{} + }{ + { + description: "s2 is nil", + s1: keySet{ + m: map[storageKey]struct{}{ + key1: {}, + key2: {}, + }, + }, + s2: keySet{ + m: nil, + }, + want: map[storageKey]struct{}{ + key1: {}, + key2: {}, + }, + }, { + description: "s2 is empty", + s1: keySet{ + m: map[storageKey]struct{}{ + key1: {}, + key2: {}, + }, + }, + s2: keySet{ + m: map[storageKey]struct{}{}, + }, + want: map[storageKey]struct{}{ + key1: {}, + key2: {}, + }, + }, + { + description: "s1 is empty", + s1: keySet{ + m: map[storageKey]struct{}{}, + }, + s2: keySet{ + m: map[storageKey]struct{}{ + key1: {}, + key2: {}, + }, + }, + want: map[storageKey]struct{}{}, + }, + { + description: "s1 has intersection with s2", + s1: keySet{ + m: map[storageKey]struct{}{ + key1: {}, + key2: {}, + }, + }, + s2: keySet{ + m: map[storageKey]struct{}{ + key2: {}, + key3: {}, + }, + }, + want: map[storageKey]struct{}{ + key1: {}, + }, + }, + } + + for _, c := range cases { + got := c.s1.Difference(c.s2) + if len(got) != len(c.want) { + t.Errorf("unexpected num of keys at case %s, got: %d, want: %d", c.description, len(got), len(c.want)) + } + gotm := map[storageKey]struct{}{} + for _, k := range got { + gotm[k] = struct{}{} + } + + if !reflect.DeepEqual(gotm, c.want) { + t.Errorf("failed at case %s, got: %v, want: %v", c.description, got, c.want) + } + } +} diff --git a/pkg/yurthub/storage/etcd/storage.go b/pkg/yurthub/storage/etcd/storage.go new file mode 100644 index 00000000000..fe0b9503b1f --- /dev/null +++ b/pkg/yurthub/storage/etcd/storage.go @@ -0,0 +1,500 @@ +/* +Copyright 2022 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package etcd + +import ( + "context" + "fmt" + "path/filepath" + "strings" + "time" + + "go.etcd.io/etcd/client/pkg/v3/transport" + clientv3 "go.etcd.io/etcd/client/v3" + healthpb "google.golang.org/grpc/health/grpc_health_v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/klog/v2" + + "github.com/openyurtio/openyurt/pkg/yurthub/storage" + "github.com/openyurtio/openyurt/pkg/yurthub/storage/utils" +) + +const ( + StorageName = "pool-coordinator" + defaultTimeout = 5 * time.Second + defaultHealthCheckPeriod = 10 * time.Second + defaultDialTimeout = 10 * time.Second + defaultComponentCacheFileName = "component-key-cache" + defaultRvLen = 32 +) + +type pathType string + +var ( + rvType pathType = "rv" +) + +type EtcdStorageConfig struct { + Prefix string + EtcdEndpoints []string + CertFile string + KeyFile string + CaFile string + LocalCacheDir string +} + +// TODO: consider how to recover the work if it was interrupted because of restart, in +// which case we've added/deleted key in local cache but failed to add/delete it in etcd. +type etcdStorage struct { + ctx context.Context + prefix string + mirrorPrefixMap map[pathType]string + client *clientv3.Client + clientConfig clientv3.Config + // localComponentKeyCache persistently records keys owned by different components + // It's useful to recover previous state when yurthub restarts. + // We need this cache at local host instead of in etcd, because we need to ensure each + // operation on etcd is atomic. If we store it in etcd, we have to get it first and then + // do the action, such as ReplaceComponentList, which makes it non-atomic. + // We assume that for resources listed by components on this node consist of two kinds: + // 1. common resources: which are also used by other nodes + // 2. special resources: which are only used by this nodes + // In local cache, we do not need to bother to distinguish these two kinds. + // For special resources, this node absolutely can create/update/delete them. + // For common resources, thanks to list/watch we can ensure that resources in pool-coordinator + // are finally consistent with the cloud, though there maybe a little jitter. + localComponentKeyCache *componentKeyCache + // For etcd storage, we do not need to cache cluster info, because + // we can get it form apiserver in pool-coordinator. + doNothingAboutClusterInfo +} + +func NewStorage(ctx context.Context, cfg *EtcdStorageConfig) (storage.Store, error) { + cacheFilePath := filepath.Join(cfg.LocalCacheDir, defaultComponentCacheFileName) + cache := newComponentKeyCache(cacheFilePath) + if err := cache.Recover(); err != nil { + return nil, fmt.Errorf("failed to recover component key cache from %s, %v", cacheFilePath, err) + } + + tlsInfo := transport.TLSInfo{ + CertFile: cfg.CertFile, + KeyFile: cfg.KeyFile, + TrustedCAFile: cfg.CaFile, + } + + tlsConfig, err := tlsInfo.ClientConfig() + if err != nil { + return nil, fmt.Errorf("failed to create tls config for etcd client, %v", err) + } + + clientConfig := clientv3.Config{ + Endpoints: cfg.EtcdEndpoints, + TLS: tlsConfig, + DialTimeout: defaultDialTimeout, + } + + client, err := clientv3.New(clientConfig) + if err != nil { + return nil, fmt.Errorf("failed to create etcd client, %v", err) + } + + s := &etcdStorage{ + ctx: ctx, + prefix: cfg.Prefix, + client: client, + clientConfig: clientConfig, + localComponentKeyCache: cache, + mirrorPrefixMap: map[pathType]string{ + rvType: "/mirror/rv", + }, + } + + go s.clientLifeCycleManagement() + return s, nil +} + +func (s *etcdStorage) mirrorPath(path string, pathType pathType) string { + return filepath.Join(s.mirrorPrefixMap[pathType], path) +} + +func (s *etcdStorage) Name() string { + return StorageName +} + +func (s *etcdStorage) clientLifeCycleManagement() { + reconnect := func(ctx context.Context) { + t := time.NewTicker(5 * time.Second) + for { + select { + case <-ctx.Done(): + return + case <-t.C: + if client, err := clientv3.New(s.clientConfig); err == nil { + klog.Infof("client reconnected to etcd server, %s", client.ActiveConnection().GetState().String()) + if err := s.client.Close(); err != nil { + klog.Errorf("failed to close old client, %v", err) + } + s.client = client + return + } + continue + } + } + } + + for { + select { + case <-s.ctx.Done(): + klog.Info("etcdstorage lifecycle routine exited") + return + default: + timeoutCtx, cancel := context.WithTimeout(s.ctx, defaultDialTimeout) + healthCli := healthpb.NewHealthClient(s.client.ActiveConnection()) + resp, err := healthCli.Check(timeoutCtx, &healthpb.HealthCheckRequest{}) + // We should call cancel in case Check request does not timeout, to release resource. + cancel() + if err != nil { + klog.Errorf("check health of etcd failed, err: %v, try to reconnect", err) + reconnect(s.ctx) + } else if resp != nil && resp.Status != healthpb.HealthCheckResponse_SERVING { + klog.Errorf("unexpected health status from etcd, status: %s", resp.Status.String()) + } + time.Sleep(defaultHealthCheckPeriod) + } + } +} + +func (s *etcdStorage) Create(key storage.Key, content []byte) error { + if err := utils.ValidateKV(key, content, storageKey{}); err != nil { + return err + } + + keyStr := key.Key() + originRv, err := getRvOfObject(content) + if err != nil { + return fmt.Errorf("failed to get rv from content when creating %s, %v", keyStr, err) + } + + ctx, cancel := context.WithTimeout(s.ctx, defaultTimeout) + defer cancel() + txnResp, err := s.client.KV.Txn(ctx).If( + notFound(keyStr), + ).Then( + clientv3.OpPut(keyStr, string(content)), + clientv3.OpPut(s.mirrorPath(keyStr, rvType), fixLenRvString(originRv)), + ).Commit() + + if err != nil { + return err + } + + if !txnResp.Succeeded { + return storage.ErrKeyExists + } + + storageKey := key.(storageKey) + s.localComponentKeyCache.AddKey(storageKey.component(), storageKey) + return nil +} + +func (s *etcdStorage) Delete(key storage.Key) error { + if err := utils.ValidateKey(key, storageKey{}); err != nil { + return err + } + + keyStr := key.Key() + ctx, cancel := context.WithTimeout(s.ctx, defaultTimeout) + defer cancel() + _, err := s.client.Txn(ctx).If().Then( + clientv3.OpDelete(keyStr), + clientv3.OpDelete(s.mirrorPath(keyStr, rvType)), + ).Commit() + if err != nil { + return err + } + + storageKey := key.(storageKey) + s.localComponentKeyCache.DeleteKey(storageKey.component(), storageKey) + return nil +} + +func (s *etcdStorage) Get(key storage.Key) ([]byte, error) { + if err := utils.ValidateKey(key, storageKey{}); err != nil { + return nil, err + } + + keyStr := key.Key() + ctx, cancel := context.WithTimeout(s.ctx, defaultTimeout) + defer cancel() + getResp, err := s.client.Get(ctx, keyStr) + if err != nil { + return nil, err + } + if len(getResp.Kvs) == 0 { + return nil, storage.ErrStorageNotFound + } + + return getResp.Kvs[0].Value, nil +} + +// TODO: When using etcd, do we have the case: +// "If the rootKey exists in the store but no keys has the prefix of rootKey"? +func (s *etcdStorage) List(key storage.Key) ([][]byte, error) { + if err := utils.ValidateKey(key, storageKey{}); err != nil { + return [][]byte{}, err + } + + rootKeyStr := key.Key() + ctx, cancel := context.WithTimeout(s.ctx, defaultTimeout) + defer cancel() + getResp, err := s.client.Get(ctx, rootKeyStr, clientv3.WithPrefix()) + if err != nil { + return nil, err + } + if len(getResp.Kvs) == 0 { + return nil, storage.ErrStorageNotFound + } + + values := make([][]byte, 0, len(getResp.Kvs)) + for _, kv := range getResp.Kvs { + values = append(values, kv.Value) + } + return values, nil +} + +func (s *etcdStorage) Update(key storage.Key, content []byte, rv uint64) ([]byte, error) { + if err := utils.ValidateKV(key, content, storageKey{}); err != nil { + return nil, err + } + + keyStr := key.Key() + ctx, cancel := context.WithTimeout(s.ctx, defaultTimeout) + defer cancel() + txnResp, err := s.client.KV.Txn(ctx).If( + found(keyStr), + fresherThan(fixLenRvUint64(rv), s.mirrorPath(keyStr, rvType)), + ).Then( + clientv3.OpPut(keyStr, string(content)), + clientv3.OpPut(s.mirrorPath(keyStr, rvType), fixLenRvUint64(rv)), + ).Else( + // Possibly we have two cases here: + // 1. key does not exist + // 2. key exists with a higher rv + // We can distinguish them by OpGet. If it gets no value back, it's case 1. + // Otherwise is case 2. + clientv3.OpGet(keyStr), + ).Commit() + + if err != nil { + return nil, err + } + + if !txnResp.Succeeded { + getResp := (*clientv3.GetResponse)(txnResp.Responses[0].GetResponseRange()) + if len(getResp.Kvs) == 0 { + return nil, storage.ErrStorageNotFound + } + return getResp.Kvs[0].Value, storage.ErrUpdateConflict + } + + return content, nil +} + +func (s *etcdStorage) ListResourceKeysOfComponent(component string, gvr schema.GroupVersionResource) ([]storage.Key, error) { + if component == "" { + return nil, storage.ErrEmptyComponent + } + + rootKey, err := s.KeyFunc(storage.KeyBuildInfo{ + Component: component, + Resources: gvr.Resource, + Group: gvr.Group, + Version: gvr.Version, + }) + if err != nil { + return nil, err + } + + keys := []storage.Key{} + keyCache, ok := s.localComponentKeyCache.Load(component) + if !ok { + return nil, storage.ErrStorageNotFound + } + for k := range keyCache.m { + if strings.HasPrefix(k.Key(), rootKey.Key()) { + keys = append(keys, k) + } + } + return keys, nil +} + +func (s *etcdStorage) ReplaceComponentList(component string, gvr schema.GroupVersionResource, namespace string, contents map[storage.Key][]byte) error { + if component == "" { + return storage.ErrEmptyComponent + } + rootKey, err := s.KeyFunc(storage.KeyBuildInfo{ + Component: component, + Resources: gvr.Resource, + Group: gvr.Group, + Version: gvr.Version, + Namespace: namespace, + }) + if err != nil { + return err + } + for key := range contents { + if !strings.HasPrefix(key.Key(), rootKey.Key()) { + return storage.ErrInvalidContent + } + } + + newKeyCache := keySet{m: map[storageKey]struct{}{}} + for k := range contents { + storageKey, ok := k.(storageKey) + if !ok { + return storage.ErrUnrecognizedKey + } + newKeyCache.m[storageKey] = struct{}{} + } + var addedOrUpdated, deleted []storageKey + oldKeyCache, loaded := s.localComponentKeyCache.LoadOrStore(component, newKeyCache) + addedOrUpdated = newKeyCache.Difference(keySet{}) + if loaded { + deleted = oldKeyCache.Difference(newKeyCache) + } + + ops := []clientv3.Op{} + for _, k := range addedOrUpdated { + rv, err := getRvOfObject(contents[k]) + if err != nil { + klog.Errorf("failed to process %s in list object, %v", k.Key(), err) + continue + } + createOrUpdateOp := clientv3.OpTxn( + []clientv3.Cmp{ + // if + found(k.Key()), + }, + []clientv3.Op{ + // then + clientv3.OpTxn([]clientv3.Cmp{ + // if + fresherThan(fixLenRvString(rv), s.mirrorPath(k.Key(), rvType)), + }, []clientv3.Op{ + // then + clientv3.OpPut(k.Key(), string(contents[k])), + clientv3.OpPut(s.mirrorPath(k.Key(), rvType), fixLenRvString(rv)), + }, []clientv3.Op{ + // else + // do nothing + }), + }, + []clientv3.Op{ + // else + clientv3.OpPut(k.Key(), string(contents[k])), + clientv3.OpPut(s.mirrorPath(k.Key(), rvType), fixLenRvString(rv)), + }, + ) + ops = append(ops, createOrUpdateOp) + } + for _, k := range deleted { + ops = append(ops, + clientv3.OpDelete(k.Key()), + clientv3.OpDelete(s.mirrorPath(k.Key(), rvType)), + ) + } + + ctx, cancel := context.WithTimeout(s.ctx, defaultTimeout) + defer cancel() + _, err = s.client.Txn(ctx).If().Then(ops...).Commit() + if err != nil { + return err + } + + return nil +} + +func (s *etcdStorage) DeleteComponentResources(component string) error { + if component == "" { + return storage.ErrEmptyComponent + } + keyCache, loaded := s.localComponentKeyCache.LoadAndDelete(component) + if !loaded { + // no need to delete + return nil + } + + ops := []clientv3.Op{} + for k := range keyCache.m { + ops = append(ops, + clientv3.OpDelete(k.Key()), + clientv3.OpDelete(s.mirrorPath(k.Key(), rvType)), + ) + } + + ctx, cancel := context.WithTimeout(s.ctx, defaultTimeout) + defer cancel() + _, err := s.client.Txn(ctx).If().Then(ops...).Commit() + if err != nil { + return err + } + return nil +} + +func fixLenRvUint64(rv uint64) string { + return fmt.Sprintf("%0*d", defaultRvLen, rv) +} + +func fixLenRvString(rv string) string { + return fmt.Sprintf("%0*s", defaultRvLen, rv) +} + +// TODO: do not get rv through decoding, which means we have to +// unmarshal bytes. We should not do any serialization in storage. +func getRvOfObject(object []byte) (string, error) { + decoder := scheme.Codecs.UniversalDeserializer() + unstructuredObj := new(unstructured.Unstructured) + _, _, err := decoder.Decode(object, nil, unstructuredObj) + if err != nil { + return "", err + } + + return unstructuredObj.GetResourceVersion(), nil +} + +func notFound(key string) clientv3.Cmp { + return clientv3.Compare(clientv3.ModRevision(key), "=", 0) +} + +func found(key string) clientv3.Cmp { + return clientv3.Compare(clientv3.ModRevision(key), ">", 0) +} + +func fresherThan(rv string, key string) clientv3.Cmp { + return clientv3.Compare(clientv3.Value(key), "<", rv) +} + +type doNothingAboutClusterInfo struct{} + +func (d doNothingAboutClusterInfo) SaveClusterInfo(_ storage.ClusterInfoKey, _ []byte) error { + return nil +} +func (d doNothingAboutClusterInfo) GetClusterInfo(_ storage.ClusterInfoKey) ([]byte, error) { + return nil, nil +} diff --git a/pkg/yurthub/storage/etcd/storage_test.go b/pkg/yurthub/storage/etcd/storage_test.go new file mode 100644 index 00000000000..ea3d0e3952c --- /dev/null +++ b/pkg/yurthub/storage/etcd/storage_test.go @@ -0,0 +1,548 @@ +/* +Copyright 2022 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package etcd + +import ( + "context" + "encoding/json" + "os" + "os/exec" + "path/filepath" + "time" + + "github.com/google/uuid" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + + "github.com/openyurtio/openyurt/pkg/yurthub/storage" +) + +var _ = Describe("Test EtcdStorage", func() { + var etcdstore *etcdStorage + var key1 storage.Key + var podObj *v1.Pod + var podJson []byte + var ctx context.Context + BeforeEach(func() { + ctx = context.Background() + randomize := uuid.New().String() + cfg := &EtcdStorageConfig{ + Prefix: "/" + randomize, + EtcdEndpoints: []string{"127.0.0.1:2379"}, + LocalCacheDir: filepath.Join(keyCacheDir, randomize), + } + s, err := NewStorage(context.Background(), cfg) + Expect(err).To(BeNil()) + etcdstore = s.(*etcdStorage) + key1, err = etcdstore.KeyFunc(storage.KeyBuildInfo{ + Component: "kubelet", + Resources: "pods", + Group: "", + Version: "v1", + Namespace: "default", + Name: "pod1-" + randomize, + }) + Expect(err).To(BeNil()) + podObj = &v1.Pod{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "Pod", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "pod1-" + randomize, + Namespace: "default", + ResourceVersion: "890", + }, + } + podJson, err = json.Marshal(podObj) + Expect(err).To(BeNil()) + }) + + Context("Test Lifecycle", Focus, func() { + It("should reconnect to etcd if connect once broken", Focus, func() { + Expect(etcdstore.Create(key1, podJson)).Should(BeNil()) + Expect(etcdCmd.Process.Kill()).To(BeNil()) + By("waiting for the etcd exited") + Eventually(func() bool { + _, err := etcdstore.Get(key1) + return err != nil + }, 10*time.Second, 1*time.Second).Should(BeTrue()) + + devNull, err := os.OpenFile("/dev/null", os.O_RDWR, 0755) + Expect(err).To(BeNil()) + etcdCmd = exec.Command("/usr/local/etcd/etcd", "--data-dir="+etcdDataDir) + etcdCmd.Stdout = devNull + etcdCmd.Stderr = devNull + Expect(etcdCmd.Start()).To(BeNil()) + By("waiting for storage function recovery") + Eventually(func() bool { + if _, err := etcdstore.Get(key1); err != nil { + return false + } + return true + }, 30*time.Second, 500*time.Microsecond).Should(BeTrue()) + }) + }) + + Context("Test Create", func() { + It("should return ErrKeyIsEmpty if key is nil", func() { + Expect(etcdstore.Create(nil, []byte("foo"))).To(Equal(storage.ErrKeyIsEmpty)) + Expect(etcdstore.Create(storageKey{}, []byte("foo"))).To(Equal(storage.ErrKeyIsEmpty)) + }) + It("should return ErrKeyHasNoContent if content is empty", func() { + Expect(etcdstore.Create(key1, []byte{})).To(Equal(storage.ErrKeyHasNoContent)) + Expect(etcdstore.Create(key1, nil)).To(Equal(storage.ErrKeyHasNoContent)) + }) + It("should return ErrKeyExists if key already exists in etcd", func() { + Expect(etcdstore.Create(key1, podJson)).To(BeNil()) + Expect(etcdstore.Create(key1, podJson)).To(Equal(storage.ErrKeyExists)) + }) + It("should create key with content in etcd", func() { + Expect(etcdstore.Create(key1, podJson)).To(BeNil()) + resp, err := etcdstore.client.Get(ctx, key1.Key()) + Expect(err).To(BeNil()) + Expect(resp.Kvs).To(HaveLen(1)) + Expect(resp.Kvs[0].Value).To(Equal([]byte(podJson))) + resp, err = etcdstore.client.Get(ctx, etcdstore.mirrorPath(key1.Key(), rvType)) + Expect(err).To(BeNil()) + Expect(resp.Kvs).To(HaveLen(1)) + Expect(resp.Kvs[0].Value).To(Equal([]byte(fixLenRvString(podObj.ResourceVersion)))) + }) + }) + + Context("Test Delete", func() { + It("should return ErrKeyIsEmpty if key is nil", func() { + Expect(etcdstore.Delete(nil)).To(Equal(storage.ErrKeyIsEmpty)) + Expect(etcdstore.Delete(storageKey{})).To(Equal(storage.ErrKeyIsEmpty)) + }) + It("should delete key from etcd if it exists", func() { + Expect(etcdstore.Create(key1, podJson)).To(BeNil()) + Expect(etcdstore.Delete(key1)).To(BeNil()) + resp, err := etcdstore.client.Get(ctx, key1.Key()) + Expect(err).To(BeNil()) + Expect(resp.Kvs).To(HaveLen(0)) + resp, err = etcdstore.client.Get(ctx, etcdstore.mirrorPath(key1.Key(), rvType)) + Expect(err).To(BeNil()) + Expect(resp.Kvs).To(HaveLen(0)) + }) + It("should not return error if key does not exist in etcd", func() { + Expect(etcdstore.Delete(key1)).To(BeNil()) + }) + }) + + Context("Test Get", func() { + It("should return ErrKeyIsEmpty if key is nil", func() { + _, err := etcdstore.Get(nil) + Expect(err).To(Equal(storage.ErrKeyIsEmpty)) + _, err = etcdstore.Get(storageKey{}) + Expect(err).To(Equal(storage.ErrKeyIsEmpty)) + }) + It("should return ErrStorageNotFound if key does not exist in etcd", func() { + _, err := etcdstore.Get(key1) + Expect(err).To(Equal(storage.ErrStorageNotFound)) + }) + It("should return content of key if it exists in etcd", func() { + Expect(etcdstore.Create(key1, podJson)).To(BeNil()) + buf, err := etcdstore.Get(key1) + Expect(err).To(BeNil()) + Expect(buf).To(Equal(podJson)) + }) + }) + + Context("Test List", func() { + var err error + var podsRootKey storage.Key + BeforeEach(func() { + podsRootKey, err = etcdstore.KeyFunc(storage.KeyBuildInfo{ + Component: "kubelet", + Resources: "pods", + }) + }) + It("should return ErrKeyIsEmpty if key is nil", func() { + _, err = etcdstore.List(nil) + Expect(err).To(Equal(storage.ErrKeyIsEmpty)) + _, err = etcdstore.List(storageKey{}) + Expect(err).To(Equal(storage.ErrKeyIsEmpty)) + }) + It("should return ErrStorageNotFound if key does not exist in etcd", func() { + _, err = etcdstore.List(podsRootKey) + Expect(err).To(Equal(storage.ErrStorageNotFound)) + _, err = etcdstore.List(key1) + Expect(err).To(Equal(storage.ErrStorageNotFound)) + }) + It("should return a single resource if key points to a specific resource", func() { + Expect(etcdstore.Create(key1, podJson)).To(BeNil()) + buf, err := etcdstore.List(key1) + Expect(err).To(BeNil()) + Expect(buf).To(Equal([][]byte{podJson})) + }) + It("should return a list of resources if its is a root key", func() { + Expect(etcdstore.Create(key1, podJson)).To(BeNil()) + info := storage.KeyBuildInfo{ + Component: "kubelet", + Resources: "pods", + Group: "", + Version: "v1", + Namespace: "default", + } + + info.Name = "pod2" + key2, err := etcdstore.KeyFunc(info) + Expect(err).To(BeNil()) + pod2Obj := podObj.DeepCopy() + pod2Obj.Name = "pod2" + pod2Json, err := json.Marshal(pod2Obj) + Expect(err).To(BeNil()) + Expect(etcdstore.Create(key2, pod2Json)).To(BeNil()) + + info.Name = "pod3" + info.Namespace = "kube-system" + key3, err := etcdstore.KeyFunc(info) + Expect(err).To(BeNil()) + pod3Obj := podObj.DeepCopy() + pod3Obj.Name = "pod3" + pod3Obj.Namespace = "kube-system" + pod3Json, err := json.Marshal(pod3Obj) + Expect(err).To(BeNil()) + Expect(etcdstore.Create(key3, pod3Json)).To(BeNil()) + + buf, err := etcdstore.List(podsRootKey) + Expect(err).To(BeNil()) + Expect(buf).To(HaveLen(len(buf))) + Expect(buf).To(ContainElements([][]byte{podJson, pod2Json, pod3Json})) + + namespacesRootKey, _ := etcdstore.KeyFunc(storage.KeyBuildInfo{ + Component: "kubelet", + Resources: "pods", + Namespace: "default", + }) + buf, err = etcdstore.List(namespacesRootKey) + Expect(err).To(BeNil()) + Expect(buf).To(ContainElements([][]byte{podJson, pod2Json})) + }) + }) + + Context("Test Update", func() { + It("should return ErrKeyIsEmpty if key is nil", func() { + _, err := etcdstore.Update(nil, []byte("foo"), 100) + Expect(err).To(Equal(storage.ErrKeyIsEmpty)) + _, err = etcdstore.Update(storageKey{}, []byte("foo"), 100) + Expect(err).To(Equal(storage.ErrKeyIsEmpty)) + }) + It("should return ErrStorageNotFound if key does not exist in etcd", func() { + _, err := etcdstore.Update(key1, podJson, 890) + Expect(err).To(Equal(storage.ErrStorageNotFound)) + }) + It("should return resource in etcd and ErrUpdateConflict if the provided resource has staler rv than resource in etcd", func() { + Expect(etcdstore.Create(key1, podJson)).To(BeNil()) + podObj.ResourceVersion = "100" + podObj.Labels = map[string]string{ + "new": "label", + } + newPodJson, err := json.Marshal(podObj) + Expect(err).To(BeNil()) + stored, err := etcdstore.Update(key1, newPodJson, 100) + Expect(err).To(Equal(storage.ErrUpdateConflict)) + Expect(stored).To(Equal(podJson)) + }) + It("should update resource in etcd and return the stored resource", func() { + Expect(etcdstore.Create(key1, podJson)).To(BeNil()) + + podObj.ResourceVersion = "900" + podObj.Labels = map[string]string{ + "rv": "900", + } + newPodJson, err := json.Marshal(podObj) + Expect(err).To(BeNil()) + stored, err := etcdstore.Update(key1, newPodJson, 900) + Expect(err).To(BeNil()) + Expect(stored).To(Equal(newPodJson)) + + podObj.ResourceVersion = "1000" + podObj.Labels = map[string]string{ + "rv": "1000", + } + newPodJson, err = json.Marshal(podObj) + Expect(err).To(BeNil()) + stored, err = etcdstore.Update(key1, newPodJson, 1000) + Expect(err).To(BeNil()) + Expect(stored).To(Equal(newPodJson)) + }) + }) + + Context("Test ComponentRelatedInterface", func() { + var cmKey, key2, key3 storage.Key + var cmObj *v1.ConfigMap + var pod2Obj, pod3Obj *v1.Pod + var cmJson, pod2Json, pod3Json []byte + var gvr schema.GroupVersionResource + var err error + BeforeEach(func() { + info := storage.KeyBuildInfo{ + Component: "kubelet", + Resources: "pods", + Group: "", + Version: "v1", + } + gvr = schema.GroupVersionResource{ + Group: "", + Version: "v1", + Resource: "pods", + } + + info.Namespace = "default" + info.Name = "pod2" + key2, _ = etcdstore.KeyFunc(info) + info.Namespace = "kube-system" + info.Name = "pod3" + key3, _ = etcdstore.KeyFunc(info) + cmKey, _ = etcdstore.KeyFunc(storage.KeyBuildInfo{ + Group: "", + Resources: "configmaps", + Version: "v1", + Namespace: "default", + Name: "cm", + Component: "kubelet", + }) + + pod2Obj = podObj.DeepCopy() + pod2Obj.Namespace = "default" + pod2Obj.Name = "pod2" + pod2Obj.ResourceVersion = "920" + pod3Obj = podObj.DeepCopy() + pod3Obj.Namespace = "kube-system" + pod3Obj.Name = "pod3" + pod3Obj.ResourceVersion = "930" + cmObj = &v1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cm", + Namespace: "default", + }, + TypeMeta: metav1.TypeMeta{ + Kind: "ConfigMap", + APIVersion: "v1", + }, + Data: map[string]string{ + "foo": "bar", + }, + } + + pod2Json, err = json.Marshal(pod2Obj) + Expect(err).To(BeNil()) + pod3Json, err = json.Marshal(pod3Obj) + Expect(err).To(BeNil()) + cmJson, err = json.Marshal(cmObj) + Expect(err).To(BeNil()) + }) + Context("Test ListResourceKeysOfComponent", func() { + It("should return ErrEmptyComponent if component is empty", func() { + _, err = etcdstore.ListResourceKeysOfComponent("", gvr) + Expect(err).To(Equal(storage.ErrEmptyComponent)) + }) + It("should return ErrEmptyResource if resource of gvr is empty", func() { + _, err = etcdstore.ListResourceKeysOfComponent("kubelet", schema.GroupVersionResource{ + Resource: "", + Version: "v1", + Group: "", + }) + Expect(err).To(Equal(storage.ErrEmptyResource)) + }) + It("should return ErrStorageNotFound if this component has no cache", func() { + _, err = etcdstore.ListResourceKeysOfComponent("flannel", gvr) + Expect(err).To(Equal(storage.ErrStorageNotFound)) + }) + It("should return all keys of gvr if cache of this component is found", func() { + By("creating objects in cache") + Expect(etcdstore.Create(key1, podJson)).To(BeNil()) + Expect(etcdstore.Create(key3, pod3Json)).To(BeNil()) + Expect(etcdstore.Create(cmKey, cmJson)).To(BeNil()) + + keys, err := etcdstore.ListResourceKeysOfComponent("kubelet", schema.GroupVersionResource{ + Group: "", + Version: "v1", + Resource: "pods", + }) + Expect(err).To(BeNil()) + Expect(keys).To(HaveLen(2)) + Expect(keys).To(ContainElements(key1, key3)) + }) + }) + + Context("Test ReplaceComponentList", func() { + BeforeEach(func() { + Expect(etcdstore.Create(key1, podJson)).To(BeNil()) + Expect(etcdstore.Create(key2, pod2Json)).To(BeNil()) + Expect(etcdstore.Create(key3, pod3Json)).To(BeNil()) + }) + It("should return ErrEmptyComponent if component is empty", func() { + Expect(etcdstore.ReplaceComponentList("", gvr, "", map[storage.Key][]byte{})).To(Equal(storage.ErrEmptyComponent)) + }) + It("should return ErrEmptyResource if resource of gvr is empty", func() { + gvr.Resource = "" + Expect(etcdstore.ReplaceComponentList("kubelet", gvr, "", map[storage.Key][]byte{})).To(Equal(storage.ErrEmptyResource)) + }) + It("should return ErrInvalidContent if it exists keys that are not passed-in gvr", func() { + invalidKey, err := etcdstore.KeyFunc(storage.KeyBuildInfo{ + Component: "kubelet", + Resources: "configmaps", + Group: "", + Version: "v1", + Namespace: "default", + Name: "cm", + }) + Expect(err).To(BeNil()) + Expect(etcdstore.ReplaceComponentList("kubelet", gvr, "default", map[storage.Key][]byte{ + key2: pod2Json, + invalidKey: {}, + })).To(Equal(storage.ErrInvalidContent)) + + invalidKey, err = etcdstore.KeyFunc(storage.KeyBuildInfo{ + Component: "kubelet", + Resources: "pods", + Group: "", + Version: "v1", + Namespace: "kube-system", + Name: "pod4", + }) + Expect(err).To(BeNil()) + Expect(etcdstore.ReplaceComponentList("kubelet", gvr, "default", map[storage.Key][]byte{ + key2: pod2Json, + key3: pod3Json, + invalidKey: {}, + })).To(Equal(storage.ErrInvalidContent)) + }) + It("should only use fresher resources in contents to update cache in etcd", func() { + pod2Obj.ResourceVersion = "921" + newPod2Json, err := json.Marshal(pod2Obj) + Expect(err).To(BeNil()) + pod3Obj.ResourceVersion = "1001" // case of different len(ResourceVersion) + newPod3Json, err := json.Marshal(pod3Obj) + Expect(err).To(BeNil()) + Expect(etcdstore.ReplaceComponentList("kubelet", gvr, "", map[storage.Key][]byte{ + key1: podJson, + key2: newPod2Json, + key3: newPod3Json, + })).To(BeNil()) + + buf, err := etcdstore.Get(key1) + Expect(err).To(BeNil()) + Expect(buf).To(Equal(podJson)) + buf, err = etcdstore.Get(key2) + Expect(err).To(BeNil()) + Expect(buf).To(Equal(newPod2Json)) + buf, err = etcdstore.Get(key3) + Expect(err).To(BeNil()) + Expect(buf).To(Equal(newPod3Json)) + }) + It("should create resource if it does not in etcd", func() { + key4, _ := etcdstore.KeyFunc(storage.KeyBuildInfo{ + Component: "kubelet", + Resources: "pods", + Version: "v1", + Group: "", + Namespace: "default", + Name: "pod4", + }) + pod4Obj := podObj.DeepCopy() + pod4Obj.ResourceVersion = "940" + pod4Obj.Name = "pod4" + pod4Json, err := json.Marshal(pod4Obj) + Expect(err).To(BeNil()) + Expect(etcdstore.ReplaceComponentList("kubelet", gvr, "", map[storage.Key][]byte{ + key1: podJson, + key2: pod2Json, + key3: pod3Json, + key4: pod4Json, + })).To(BeNil()) + + buf, err := etcdstore.Get(key1) + Expect(err).To(BeNil()) + Expect(buf).To(Equal(podJson)) + buf, err = etcdstore.Get(key2) + Expect(err).To(BeNil()) + Expect(buf).To(Equal(pod2Json)) + buf, err = etcdstore.Get(key3) + Expect(err).To(BeNil()) + Expect(buf).To(Equal(pod3Json)) + buf, err = etcdstore.Get(key4) + Expect(err).To(BeNil()) + Expect(buf).To(Equal(pod4Json)) + }) + It("should delete resources in etcd if they were in local cache but are not in current contents", func() { + Expect(etcdstore.ReplaceComponentList("kubelet", gvr, "", map[storage.Key][]byte{ + key1: podJson, + })).To(BeNil()) + buf, err := etcdstore.Get(key1) + Expect(err).To(BeNil()) + Expect(buf).To(Equal(podJson)) + _, err = etcdstore.Get(key2) + Expect(err).To(Equal(storage.ErrStorageNotFound)) + _, err = etcdstore.Get(key3) + Expect(err).To(Equal(storage.ErrStorageNotFound)) + }) + }) + + Context("Test DeleteComponentResources", func() { + It("should return ErrEmptyComponent if component is empty", func() { + Expect(etcdstore.DeleteComponentResources("")).To(Equal(storage.ErrEmptyComponent)) + }) + It("should not return err even there is no cache of component", func() { + Expect(etcdstore.DeleteComponentResources("flannel")).To(BeNil()) + }) + It("should delete cache of component from local cache and etcd", func() { + Expect(etcdstore.Create(cmKey, cmJson)).To(BeNil()) + Expect(etcdstore.Create(key1, podJson)).To(BeNil()) + Expect(etcdstore.Create(key3, pod3Json)).To(BeNil()) + keys := []storage.Key{cmKey, key1, key3} + cmKey, _ = etcdstore.KeyFunc(storage.KeyBuildInfo{ + Component: "kube-proxy", + Resources: "configmaps", + Group: "", + Version: "v1", + Namespace: "default", + Name: "cm-kube-proxy", + }) + cmObj.Name = "cm-kube-proxy" + cmJson, err = json.Marshal(cmObj) + Expect(err).To(BeNil()) + Expect(etcdstore.Create(cmKey, cmJson)).To(BeNil()) + + Expect(etcdstore.DeleteComponentResources("kubelet")).To(BeNil()) + for _, k := range keys { + _, err := etcdstore.Get(k) + Expect(err).To(Equal(storage.ErrStorageNotFound)) + } + buf, err := etcdstore.Get(cmKey) + Expect(err).To(BeNil()) + Expect(buf).To(Equal(cmJson)) + + _, found := etcdstore.localComponentKeyCache.Load("kubelet") + Expect(found).To(BeFalse()) + keyset, found := etcdstore.localComponentKeyCache.Load("kube-proxy") + Expect(found).To(BeTrue()) + Expect(keyset).To(Equal(keySet{ + m: map[storageKey]struct{}{ + cmKey.(storageKey): {}, + }, + })) + }) + }) + }) +}) diff --git a/pkg/yurthub/storage/store.go b/pkg/yurthub/storage/store.go index f0126a8669b..6aad24574e8 100644 --- a/pkg/yurthub/storage/store.go +++ b/pkg/yurthub/storage/store.go @@ -61,6 +61,7 @@ type objectRelatedHandler interface { // Create will create content of key in the store. // The key must indicate a specific resource. // If key is empty, ErrKeyIsEmpty will be returned. + // If content is empty, either nil or []byte{}, ErrKeyHasNoContent will be returned. // If this key has already existed in this store, ErrKeyExists will be returned. Create(key Key, content []byte) error @@ -78,7 +79,7 @@ type objectRelatedHandler interface { // List will retrieve all contents whose keys have the prefix of rootKey. // If key is empty, ErrKeyIsEmpty will be returned. // If the key does not exist in the store, ErrStorageNotFound will be returned. - // If the key exists in the store but no other keys has it as prefix, an empty slice + // If the key exists in the store but no other keys having it as prefix, an empty slice // of content will be returned. List(key Key) ([][]byte, error) @@ -88,7 +89,7 @@ type objectRelatedHandler interface { // The key must indicate a specific resource. // If key is empty, ErrKeyIsEmpty will be returned. // If the key does not exist in the store, ErrStorageNotFound will be returned. - // If force is not set and the rv is staler than what is in the store, ErrUpdateConflict will be returned. + // If rv is staler than what is in the store, ErrUpdateConflict will be returned. Update(key Key, contents []byte, rv uint64) ([]byte, error) // KeyFunc will generate the key used by this store. diff --git a/pkg/yurthub/storage/utils/validate.go b/pkg/yurthub/storage/utils/validate.go index 32ba6301b62..3ba8ae3e24a 100644 --- a/pkg/yurthub/storage/utils/validate.go +++ b/pkg/yurthub/storage/utils/validate.go @@ -24,12 +24,12 @@ import ( // TODO: should also valid the key format func ValidateKey(key storage.Key, validKeyType interface{}) error { + if key == nil || key.Key() == "" { + return storage.ErrKeyIsEmpty + } if reflect.TypeOf(key) != reflect.TypeOf(validKeyType) { return storage.ErrUnrecognizedKey } - if key.Key() == "" { - return storage.ErrKeyIsEmpty - } return nil } diff --git a/pkg/yurthub/util/util.go b/pkg/yurthub/util/util.go index e99af4d9f75..e6ae72bedb4 100644 --- a/pkg/yurthub/util/util.go +++ b/pkg/yurthub/util/util.go @@ -59,6 +59,8 @@ const ( ProxyReqCanCache // ProxyListSelector represents label selector and filed selector string for list request ProxyListSelector + // ProxyPoolScopedResource represents if this request is asking for pool-scoped resources + ProxyPoolScopedResource YurtHubNamespace = "kube-system" CacheUserAgentsKey = "cache_agents" @@ -132,6 +134,19 @@ func ListSelectorFrom(ctx context.Context) (string, bool) { return info, ok } +// WithIfPoolScopedResource returns a copy of parent in which IfPoolScopedResource is set, +// indicating whether this request is asking for pool-scoped resources. +func WithIfPoolScopedResource(parent context.Context, ifPoolScoped bool) context.Context { + return WithValue(parent, ProxyPoolScopedResource, ifPoolScoped) +} + +// IfPoolScopedResourceFrom returns the value of IfPoolScopedResource indicating whether this request +// is asking for pool-scoped resource. +func IfPoolScopedResourceFrom(ctx context.Context) (bool, bool) { + info, ok := ctx.Value(ProxyPoolScopedResource).(bool) + return info, ok +} + // ReqString formats a string for request func ReqString(req *http.Request) string { ctx := req.Context() @@ -168,6 +183,84 @@ func WriteObject(statusCode int, obj runtime.Object, w http.ResponseWriter, req return fmt.Errorf("request info is not found when write object, %s", ReqString(req)) } +func NewTripleReadCloser(req *http.Request, rc io.ReadCloser, isRespBody bool) (io.ReadCloser, io.ReadCloser, io.ReadCloser) { + pr1, pw1 := io.Pipe() + pr2, pw2 := io.Pipe() + tr := &tripleReadCloser{ + req: req, + rc: rc, + pw1: pw1, + pw2: pw2, + } + return tr, pr1, pr2 +} + +type tripleReadCloser struct { + req *http.Request + rc io.ReadCloser + pw1 *io.PipeWriter + pw2 *io.PipeWriter + // isRespBody shows rc(is.ReadCloser) is a response.Body + // or not(maybe a request.Body). if it is true(it's a response.Body), + // we should close the response body in Close func, else not, + // it(request body) will be closed by http request caller + isRespBody bool +} + +// Read read data into p and write into pipe +func (dr *tripleReadCloser) Read(p []byte) (n int, err error) { + defer func() { + if dr.req != nil && dr.isRespBody { + ctx := dr.req.Context() + info, _ := apirequest.RequestInfoFrom(ctx) + if info.IsResourceRequest { + comp, _ := ClientComponentFrom(ctx) + metrics.Metrics.AddProxyTrafficCollector(comp, info.Verb, info.Resource, info.Subresource, n) + } + } + }() + + n, err = dr.rc.Read(p) + if n > 0 { + var n1, n2 int + var err error + if n1, err = dr.pw1.Write(p[:n]); err != nil { + klog.Errorf("tripleReader: failed to write to pw1 %v", err) + return n1, err + } + if n2, err = dr.pw2.Write(p[:n]); err != nil { + klog.Errorf("tripleReader: failed to write to pw2 %v", err) + return n2, err + } + } + + return +} + +// Close close two readers +func (dr *tripleReadCloser) Close() error { + errs := make([]error, 0) + if dr.isRespBody { + if err := dr.rc.Close(); err != nil { + errs = append(errs, err) + } + } + + if err := dr.pw1.Close(); err != nil { + errs = append(errs, err) + } + + if err := dr.pw2.Close(); err != nil { + errs = append(errs, err) + } + + if len(errs) != 0 { + return fmt.Errorf("failed to close dualReader, %v", errs) + } + + return nil +} + // NewDualReadCloser create an dualReadCloser object func NewDualReadCloser(req *http.Request, rc io.ReadCloser, isRespBody bool) (io.ReadCloser, io.ReadCloser) { pr, pw := io.Pipe()