Support context-scoped plugin discovery for tanzu contexts behind f…

…eature-flag (#651)

* Support context-scoped plugin discovery for `tanzu` contexts
* Use kubeConfigBytes to create a k8s client instead of saving the kubeconfig to cache
* Use feature-flag 'features.global.context-plugin-discovery-for-tanzu-context' to enable feature. This feature is disabled by default.
* Use environment variable `TANZU_CLI_PLUGIN_DISCOVERY_PATH_FOR_TANZU_CONTEXT` to configure the different path for the discovery endpoint

go.mod

@@ -42,7 +42,7 @@ require (
 	github.com/vmware-tanzu/carvel-ytt v0.40.0
 	github.com/vmware-tanzu/tanzu-cli/test/e2e/framework v0.0.0-00010101000000-000000000000
 	github.com/vmware-tanzu/tanzu-framework/capabilities/client v0.0.0-20230523145612-1c6fbba34686
-	github.com/vmware-tanzu/tanzu-plugin-runtime v1.2.0-dev.0.20240118142028-778d24409033
+	github.com/vmware-tanzu/tanzu-plugin-runtime v1.2.0-dev.0.20240122215628-021d60b78abe
 	go.pinniped.dev v0.20.0
 	golang.org/x/mod v0.12.0
 	golang.org/x/oauth2 v0.8.0

go.sum

@@ -738,8 +738,8 @@ github.com/vmware-tanzu/tanzu-framework/apis/run v0.0.0-20230419030809-7081502eb
 github.com/vmware-tanzu/tanzu-framework/apis/run v0.0.0-20230419030809-7081502ebf68/go.mod h1:e1Uef+Ux5BIHpYwqbeP2ZZmOzehBcez2vUEWXHe+xHE=
 github.com/vmware-tanzu/tanzu-framework/capabilities/client v0.0.0-20230523145612-1c6fbba34686 h1:VcuXqUXFxm5WDqWkzAlU/6cJXua0ozELnqD59fy7J6E=
 github.com/vmware-tanzu/tanzu-framework/capabilities/client v0.0.0-20230523145612-1c6fbba34686/go.mod h1:AFGOXZD4tH+KhpmtV0VjWjllXhr8y57MvOsIxTtywc4=
-github.com/vmware-tanzu/tanzu-plugin-runtime v1.2.0-dev.0.20240118142028-778d24409033 h1:zd1HXrHCjQyEiqCrVwvl4RFhzpBmnKoAf+Cq5eOcRpk=
-github.com/vmware-tanzu/tanzu-plugin-runtime v1.2.0-dev.0.20240118142028-778d24409033/go.mod h1:M7WVZoItdyQp53tEprQIa6PZmhbrLe3CzuyQphWuRyI=
+github.com/vmware-tanzu/tanzu-plugin-runtime v1.2.0-dev.0.20240122215628-021d60b78abe h1:jlR24NEVKyBiTGlEtqWBSGULBqZe0LJd40VzybAStY0=
+github.com/vmware-tanzu/tanzu-plugin-runtime v1.2.0-dev.0.20240122215628-021d60b78abe/go.mod h1:M7WVZoItdyQp53tEprQIa6PZmhbrLe3CzuyQphWuRyI=
 github.com/xanzy/go-gitlab v0.83.0 h1:37p0MpTPNbsTMKX/JnmJtY8Ch1sFiJzVF342+RvZEGw=
 github.com/xanzy/go-gitlab v0.83.0/go.mod h1:5ryv+MnpZStBH8I/77HuQBsMbBGANtVpLWC15qOjWAw=
 github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=

pkg/cluster/client.go

@@ -24,6 +24,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/client/apiutil"
 
 	capdiscovery "github.com/vmware-tanzu/tanzu-framework/capabilities/client/pkg/discovery"
+	"github.com/vmware-tanzu/tanzu-plugin-runtime/log"
 
 	cliv1alpha1 "github.com/vmware-tanzu/tanzu-cli/apis/cli/v1alpha1"
 	"github.com/vmware-tanzu/tanzu-cli/pkg/constants"
@@ -88,6 +89,7 @@ type client struct {
 	CrtClient       CrtClient
 	DiscoveryClient DiscoveryClient
 	DynamicClient   DynamicClient
+	kubeConfigBytes []byte
 	kubeConfigPath  string
 	currentContext  string
 }
@@ -96,18 +98,25 @@ type client struct {
 // if kubeconfig path is empty it gets default path
 // if options.poller is nil it creates default poller. You should only pass custom poller for unit testing
 // if options.crtClientFactory is nil it creates default CrtClientFactory
-func NewClient(kubeConfigPath, contextStr string, options Options) (Client, error) {
+func NewClient(kubeConfigPath, contextStr string, kubeConfigBytes []byte, options Options) (Client, error) {
 	var err error
-	var rules *clientcmd.ClientConfigLoadingRules
-	if kubeConfigPath == "" {
-		rules = clientcmd.NewDefaultClientConfigLoadingRules()
-		kubeConfigPath = rules.GetDefaultFilename()
+	client := &client{}
+
+	if len(kubeConfigBytes) == 0 {
+		var rules *clientcmd.ClientConfigLoadingRules
+		if kubeConfigPath == "" {
+			rules = clientcmd.NewDefaultClientConfigLoadingRules()
+			kubeConfigPath = rules.GetDefaultFilename()
+		}
+		client.kubeConfigPath = kubeConfigPath
+		client.currentContext = contextStr
+		log.V(7).Infof("creating kubernetes client with kubeconfig %q, kubecontext %q", kubeConfigPath, contextStr)
+	} else {
+		client.kubeConfigBytes = kubeConfigBytes
+		log.V(7).Infof("creating kubernetes client with kubeconfigbytes %q", string(kubeConfigBytes))
 	}
+
 	InitializeOptions(&options)
-	client := &client{
-		kubeConfigPath: kubeConfigPath,
-		currentContext: contextStr,
-	}
 	err = client.getK8sClients(options.CrtClient, options.DiscoveryClientFactory, options.DynamicClientFactory, options.RequestTimeout)
 	if err != nil {
 		return nil, err
@@ -201,19 +210,29 @@ func ConsolidateImageRepoMaps(cmList *corev1.ConfigMapList) (map[string]string,
 
 func (c *client) getK8sClients(crtClient CrtClient, discoveryClientFactory DiscoveryClientFactory, dynamicClientFactory DynamicClientFactory, timeout time.Duration) error {
 	var discoveryClient discovery.DiscoveryInterface
-	config, err := clientcmd.LoadFromFile(c.kubeConfigPath)
-	if err != nil {
-		return errors.Errorf("Failed to load Kubeconfig file from %q", c.kubeConfigPath)
-	}
-	configOverrides := &clientcmd.ConfigOverrides{}
-	if c.currentContext != "" {
-		configOverrides.CurrentContext = c.currentContext
-	}
+	var restConfig *rest.Config
+	var err error
 
-	restConfig, err := clientcmd.NewDefaultClientConfig(*config, configOverrides).ClientConfig()
-	if err != nil {
-		return errors.Errorf("Unable to set up rest config due to : %v", err)
+	if len(c.kubeConfigBytes) != 0 {
+		restConfig, err = clientcmd.RESTConfigFromKubeConfig(c.kubeConfigBytes)
+		if err != nil {
+			return errors.Errorf("Unable to set up rest config due to : %v", err)
+		}
+	} else {
+		config, err := clientcmd.LoadFromFile(c.kubeConfigPath)
+		if err != nil {
+			return errors.Errorf("Failed to load Kubeconfig file from %q", c.kubeConfigPath)
+		}
+		configOverrides := &clientcmd.ConfigOverrides{}
+		if c.currentContext != "" {
+			configOverrides.CurrentContext = c.currentContext
+		}
+		restConfig, err = clientcmd.NewDefaultClientConfig(*config, configOverrides).ClientConfig()
+		if err != nil {
+			return errors.Errorf("Unable to set up rest config due to : %v", err)
+		}
 	}
+
 	// As there are many registered resources in the cluster, set the values for the maximum number of
 	// queries per second and the maximum burst for throttle to a high value to avoid throttling of messages
 	restConfig.QPS = constants.DefaultQPS
@@ -312,14 +331,14 @@ func NewOptions(crtClient CrtClient, discoveryClientFactory DiscoveryClientFacto
 
 // ClusterClientFactory a factory for creating cluster clients
 type ClusterClientFactory interface {
-	NewClient(kubeConfigPath, context string, options Options) (Client, error)
+	NewClient(kubeConfigPath, contextStr string, kubeConfigBytes []byte, options Options) (Client, error)
 }
 
 type clusterClientFactory struct{}
 
 // NewClient creates new clusterclient
-func (c *clusterClientFactory) NewClient(kubeConfigPath, contextStr string, options Options) (Client, error) {
-	return NewClient(kubeConfigPath, contextStr, options)
+func (c *clusterClientFactory) NewClient(kubeConfigPath, contextStr string, kubeConfigBytes []byte, options Options) (Client, error) {
+	return NewClient(kubeConfigPath, contextStr, kubeConfigBytes, options)
 }
 
 // NewClusterClientFactory creates new clusterclient factory

pkg/cluster/client_test.go

@@ -47,7 +47,7 @@ var _ = Describe("New Cluster Client Tests", func() {
 			discoveryClientFactoryFake.ServerVersionReturns(nil, nil)
 		})
 		It("return cluster client", func() {
-			client, err := cluster.NewClient(kubeconfigFile, "foo-context", options)
+			client, err := cluster.NewClient(kubeconfigFile, "foo-context", nil, options)
 			Expect(client).NotTo(BeNil())
 			Expect(err).To(BeNil())
 		})
@@ -61,7 +61,7 @@ var _ = Describe("New Cluster Client Tests", func() {
 			BeforeEach(func() {
 				discoveryClientFactoryFake.NewDiscoveryClientForConfigReturns(&discovery.DiscoveryClient{}, nil)
 				discoveryClientFactoryFake.ServerVersionReturns(nil, nil)
-				clusterClient, _ = cluster.NewClient(kubeconfigFile, "foo-context", options)
+				clusterClient, _ = cluster.NewClient(kubeconfigFile, "foo-context", nil, options)
 				crtClientFake.ListObjectsReturns(nil)
 			})
 			It("return empty plugins and no error", func() {
@@ -74,7 +74,7 @@ var _ = Describe("New Cluster Client Tests", func() {
 			BeforeEach(func() {
 				discoveryClientFactoryFake.NewDiscoveryClientForConfigReturns(&discovery.DiscoveryClient{}, nil)
 				discoveryClientFactoryFake.ServerVersionReturns(nil, nil)
-				clusterClient, _ = cluster.NewClient(kubeconfigFile, "foo-context", options)
+				clusterClient, _ = cluster.NewClient(kubeconfigFile, "foo-context", nil, options)
 				crtClientFake.ListObjectsReturns(nil)
 			})
 			It("return clusterQuery object and no errors", func() {
@@ -87,7 +87,7 @@ var _ = Describe("New Cluster Client Tests", func() {
 			BeforeEach(func() {
 				discoveryClientFactoryFake.NewDiscoveryClientForConfigReturns(&discovery.DiscoveryClient{}, nil)
 				discoveryClientFactoryFake.ServerVersionReturns(nil, nil)
-				clusterClient, _ = cluster.NewClient(kubeconfigFile, "foo-context", options)
+				clusterClient, _ = cluster.NewClient(kubeconfigFile, "foo-context", nil, options)
 				crtClientFake.ListObjectsReturns(nil)
 			})
 			It("return empty map and no error", func() {
@@ -105,7 +105,7 @@ var _ = Describe("New Cluster Client Tests", func() {
 			kubeconfigFile = "invalidkubeconfigfile.yaml"
 		})
 		It("should return error for NewClient()", func() {
-			client, err := cluster.NewClient(kubeconfigFile, "foo-context", options)
+			client, err := cluster.NewClient(kubeconfigFile, "foo-context", nil, options)
 			Expect(client).To(BeNil())
 			Expect(err.Error()).To(ContainSubstring("Failed to load Kubeconfig file from \"invalidkubeconfigfile.yaml\""))
 		})

pkg/constants/defaults.go

@@ -29,4 +29,11 @@ const (
 
 	// DefaultPluginDBCacheRefreshThreshold is the default value for db cache refresh
 	DefaultPluginDBCacheRefreshThreshold = 24 * time.Hour
+
+	// TanzuContextPluginDiscoveryEndpointPath specifies the default plugin discovery endpoint path
+	// Note: This path value needs to be updated once the Tanzu context backend support the context-scoped
+	// plugin discovery and the endpoint value gets finalized
+	// Until then for testing purpose, user can overwrite this path using `TANZU_CLI_PLUGIN_DISCOVERY_PATH_FOR_TANZU_CONTEXT`
+	// environment variable
+	TanzuContextPluginDiscoveryEndpointPath = "/discovery"
 )

pkg/constants/env_variables.go

@@ -60,4 +60,10 @@ const (
 
 	// TanzuCLIOAuthLocalListenerPort is the port to be used by local listener for OAuth authorization flow
 	TanzuCLIOAuthLocalListenerPort = "TANZU_CLI_OAUTH_LOCAL_LISTENER_PORT"
+
+	// TanzuPluginDiscoveryPathforTanzuContext specifies the custom endpoint path to use with the kubeconfig when talking
+	// to the tanzu context to get the recommended plugins by querying CLIPlugin resources
+	// If environment variable 'TANZU_CLI_PLUGIN_DISCOVERY_PATH_FOR_TANZU_CONTEXT' is not configured
+	// default discovery endpoint configured with TanzuContextPluginDiscoveryEndpointPath will be used
+	TanzuPluginDiscoveryPathforTanzuContext = "TANZU_CLI_PLUGIN_DISCOVERY_PATH_FOR_TANZU_CONTEXT"
 )

pkg/constants/featureflags.go

@@ -7,6 +7,10 @@ package constants
 const (
 	// FeatureContextCommand determines whether to surface the context command. This is disabled by default.
 	FeatureContextCommand = "features.global.context-target-v2"
+
+	// FeaturePluginDiscoveryForTanzuContext determines whether to enable context-scoped plugin discovery for Tanzu context.
+	// This is disabled by default
+	FeaturePluginDiscoveryForTanzuContext = "features.global.plugin-discovery-for-tanzu-context"
 )
 
 // DefaultCliFeatureFlags is used to populate an initially empty config file with default values for feature flags.

pkg/discovery/interface.go

@@ -119,7 +119,7 @@ func CreateDiscoveryFromV1alpha1(pd configtypes.PluginDiscovery, options ...Disc
 	case pd.Local != nil:
 		return NewLocalDiscovery(pd.Local.Name, pd.Local.Path), nil
 	case pd.Kubernetes != nil:
-		return NewKubernetesDiscovery(pd.Kubernetes.Name, pd.Kubernetes.Path, pd.Kubernetes.Context), nil
+		return NewKubernetesDiscovery(pd.Kubernetes.Name, pd.Kubernetes.Path, pd.Kubernetes.Context, pd.Kubernetes.KubeConfigBytes), nil
 	case pd.REST != nil:
 		return NewRESTDiscovery(pd.REST.Name, pd.REST.Endpoint, pd.REST.BasePath), nil
 	}

pkg/discovery/kubernetes.go

@@ -20,17 +20,19 @@ import (
 
 // KubernetesDiscovery is an artifact discovery utilizing CLIPlugin API in kubernetes cluster
 type KubernetesDiscovery struct {
-	name           string
-	kubeconfigPath string
-	kubecontext    string
+	name            string
+	kubeconfigPath  string
+	kubecontext     string
+	kubeconfigBytes []byte
 }
 
 // NewKubernetesDiscovery returns a new kubernetes repository
-func NewKubernetesDiscovery(name, kubeconfigPath, kubecontext string) Discovery {
+func NewKubernetesDiscovery(name, kubeconfigPath, kubecontext string, kubeconfigBytes []byte) Discovery {
 	return &KubernetesDiscovery{
-		name:           name,
-		kubeconfigPath: kubeconfigPath,
-		kubecontext:    kubecontext,
+		name:            name,
+		kubeconfigPath:  kubeconfigPath,
+		kubecontext:     kubecontext,
+		kubeconfigBytes: kubeconfigBytes,
 	}
 }
 
@@ -49,7 +51,7 @@ func (k *KubernetesDiscovery) Manifest() ([]Discovered, error) {
 	log.V(6).Infof("creating kubernetes client with kubeconfig %q, kubecontext %q", k.kubeconfigPath, k.kubecontext)
 
 	// Create cluster client
-	clusterClient, err := cluster.NewClient(k.kubeconfigPath, k.kubecontext, cluster.Options{RequestTimeout: defaultTimeout})
+	clusterClient, err := cluster.NewClient(k.kubeconfigPath, k.kubecontext, k.kubeconfigBytes, cluster.Options{RequestTimeout: defaultTimeout})
 	if err != nil {
 		return nil, err
 	}

pkg/pluginmanager/default_discoveries.go

@@ -9,7 +9,9 @@ import (
 	"strings"
 
 	"github.com/vmware-tanzu/tanzu-cli/pkg/constants"
+	"github.com/vmware-tanzu/tanzu-plugin-runtime/config"
 	configtypes "github.com/vmware-tanzu/tanzu-plugin-runtime/config/types"
+	"github.com/vmware-tanzu/tanzu-plugin-runtime/log"
 )
 
 const True = "true"
@@ -38,7 +40,15 @@ func defaultDiscoverySourceBasedOnContext(context *configtypes.Context) []config
 		defaultDiscoveries = append(defaultDiscoveries, defaultDiscoverySourceForK8sTargetedContext(context.Name, context.ClusterOpts.Path, context.ClusterOpts.Context))
 	} else if context.ContextType == configtypes.ContextTypeTMC && context.GlobalOpts != nil {
 		defaultDiscoveries = append(defaultDiscoveries, defaultDiscoverySourceForTMCTargetedContext(context))
+	} else if context.ContextType == configtypes.ContextTypeTanzu && config.IsFeatureActivated(constants.FeaturePluginDiscoveryForTanzuContext) {
+		discovery, err := defaultDiscoverySourceForTanzuTargetedContext(context.Name)
+		if err != nil {
+			log.V(6).Infof("error while getting default discovery for context %q, error: %s", context.Name, err.Error())
+		} else {
+			defaultDiscoveries = append(defaultDiscoveries, discovery)
+		}
 	}
+
 	return defaultDiscoveries
 }
 
@@ -62,6 +72,25 @@ func defaultDiscoverySourceForTMCTargetedContext(context *configtypes.Context) c
 	}
 }
 
+func defaultDiscoverySourceForTanzuTargetedContext(context string) (configtypes.PluginDiscovery, error) {
+	tanzuContextDiscoveryEndpointPath := strings.TrimSpace(os.Getenv(constants.TanzuPluginDiscoveryPathforTanzuContext))
+	if tanzuContextDiscoveryEndpointPath == "" {
+		tanzuContextDiscoveryEndpointPath = constants.TanzuContextPluginDiscoveryEndpointPath
+	}
+
+	kubeconfigBytes, err := config.GetKubeconfigForContext(context, config.ForCustomPath(tanzuContextDiscoveryEndpointPath))
+	if err != nil {
+		return configtypes.PluginDiscovery{}, err
+	}
+
+	return configtypes.PluginDiscovery{
+		Kubernetes: &configtypes.KubernetesDiscovery{
+			Name:            fmt.Sprintf("default-%s", context),
+			KubeConfigBytes: kubeconfigBytes,
+		},
+	}, nil
+}
+
 func appendURLScheme(endpoint string) string {
 	// At present, the e2e test environment lacks support for HTTPS, thus hardcoding HTTPS is being skipped.
 	if os.Getenv(constants.E2ETestEnvironment) == True {