Verify that the api-server serves the correct CRDs that this client expects

pkg/kudoctl/kudoinit/crd/crds.go

@@ -5,6 +5,7 @@ import (
 	"context"
 	"fmt"
 	"os"
+	"reflect"
 
 	apiextv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
 	"k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/typed/apiextensions/v1beta1"
@@ -66,15 +67,31 @@ func (c Initializer) PreUpgradeVerify(client *kube.Client, result *verifier.Resu
 	return nil
 }
 
-// VerifyInstallation ensures that the CRDs are installed and have the correct and expected version
+// VerifyInstallation ensures that all CRDs are installed and are the same as this CLI would install
 func (c Initializer) VerifyInstallation(client *kube.Client, result *verifier.Result) error {
-	if err := c.verifyInstallation(client.ExtClient.ApiextensionsV1beta1(), c.Operator, result); err != nil {
+	apiClient := client.ExtClient.ApiextensionsV1beta1()
+	if err := c.verifyInstallation(apiClient, c.Operator, result); err != nil {
 		return err
 	}
-	if err := c.verifyInstallation(client.ExtClient.ApiextensionsV1beta1(), c.OperatorVersion, result); err != nil {
+	if err := c.verifyInstallation(apiClient, c.OperatorVersion, result); err != nil {
 		return err
 	}
-	if err := c.verifyInstallation(client.ExtClient.ApiextensionsV1beta1(), c.Instance, result); err != nil {
+	if err := c.verifyInstallation(apiClient, c.Instance, result); err != nil {
+		return err
+	}
+	return nil
+}
+
+// VerifyServedVersion ensures that the api server provides the correct version of all CRDs that this client understands
+func (c Initializer) VerifyServedVersion(client *kube.Client, expectedVersion string, result *verifier.Result) error {
+	apiClient := client.ExtClient.ApiextensionsV1beta1()
+	if err := c.verifyServedVersion(apiClient, c.Operator.Name, expectedVersion, result); err != nil {
+		return err
+	}
+	if err := c.verifyServedVersion(apiClient, c.OperatorVersion.Name, expectedVersion, result); err != nil {
+		return err
+	}
+	if err := c.verifyServedVersion(apiClient, c.Instance.Name, expectedVersion, result); err != nil {
 		return err
 	}
 	return nil
@@ -94,6 +111,7 @@ func (c Initializer) Install(client *kube.Client) error {
 	return nil
 }
 
+// verifyIsNotInstalled is used to ensure that the cluster has no old KUDO version installed
 func (c Initializer) verifyIsNotInstalled(client v1beta1.CustomResourceDefinitionsGetter, crd *apiextv1beta1.CustomResourceDefinition, result *verifier.Result) error {
 	_, err := client.CustomResourceDefinitions().Get(context.TODO(), crd.Name, v1.GetOptions{})
 	if err != nil {
@@ -106,27 +124,66 @@ func (c Initializer) verifyIsNotInstalled(client v1beta1.CustomResourceDefinitio
 	return nil
 }
 
-func (c Initializer) verifyInstallation(client v1beta1.CustomResourceDefinitionsGetter, crd *apiextv1beta1.CustomResourceDefinition, result *verifier.Result) error {
-	existingCrd, err := client.CustomResourceDefinitions().Get(context.TODO(), crd.Name, v1.GetOptions{})
+func (c Initializer) getCrdForVerify(client v1beta1.CustomResourceDefinitionsGetter, crdName string, result *verifier.Result) (*apiextv1beta1.CustomResourceDefinition, error) {
+	existingCrd, err := client.CustomResourceDefinitions().Get(context.TODO(), crdName, v1.GetOptions{})
 	if err != nil {
 		if os.IsTimeout(err) {
-			return err
+			return nil, err
 		}
 		if kerrors.IsNotFound(err) {
-			result.AddErrors(fmt.Sprintf("CRD %s is not installed", crd.Name))
-			return nil
+			result.AddErrors(fmt.Sprintf("CRD %s is not installed", crdName))
+			return nil, nil
 		}
-		return fmt.Errorf("failed to retrieve CRD %s: %v", crd.Name, err)
+		return nil, fmt.Errorf("failed to retrieve CRD %s: %v", crdName, err)
 	}
-	if existingCrd.Spec.Version != crd.Spec.Version {
-		result.AddErrors(fmt.Sprintf("Installed CRD %s has invalid version %s, expected %s", crd.Name, existingCrd.Spec.Version, crd.Spec.Version))
-		return nil
+	return existingCrd, nil
+}
+
+// VerifyInstallation ensures that a single CRD is installed and is the same as this CLI would install
+func (c Initializer) verifyInstallation(client v1beta1.CustomResourceDefinitionsGetter, crd *apiextv1beta1.CustomResourceDefinition, result *verifier.Result) error {
+	existingCrd, err := c.getCrdForVerify(client, crd.Name, result)
+	if err != nil || existingCrd == nil {
+		return err
+	}
+	if !reflect.DeepEqual(existingCrd.Spec.Versions, crd.Spec.Versions) {
+		result.AddErrors(fmt.Sprintf("Installed CRD versions do not match expected CRD versions (%v vs %v).", existingCrd.Spec.Versions, crd.Spec.Versions))
 	}
 	if err := health.IsHealthy(existingCrd); err != nil {
 		result.AddErrors(fmt.Sprintf("Installed CRD %s is not healthy: %v", crd.Name, err))
 		return nil
 	}
-	clog.V(2).Printf("CRD %s is installed with version %s", crd.Name, existingCrd.Spec.Versions[0].Name)
+	clog.V(2).Printf("CRD %s is installed with versions %v", crd.Name, existingCrd.Spec.Versions)
+	return nil
+}
+
+// VerifyServedVersion ensures that the api server provides the correct version of a specific CRDs that this client understands
+func (c Initializer) verifyServedVersion(client v1beta1.CustomResourceDefinitionsGetter, crdName, version string, result *verifier.Result) error {
+	existingCrd, err := c.getCrdForVerify(client, crdName, result)
+	if err != nil || existingCrd == nil {
+		return err
+	}
+	if err := health.IsHealthy(existingCrd); err != nil {
+		result.AddErrors(err.Error())
+		return nil
+	}
+
+	var expectedVersion *apiextv1beta1.CustomResourceDefinitionVersion
+	var allNames = []string{}
+	for _, v := range existingCrd.Spec.Versions {
+		v := v
+		allNames = append(allNames, v.Name)
+		if v.Name == version {
+			expectedVersion = &v
+			break
+		}
+	}
+	if expectedVersion == nil {
+		result.AddErrors(fmt.Sprintf("Expected API version %s was not found for %s, api-server only supports %v. Please update your KUDO CLI.", version, crdName, allNames))
+		return nil
+	}
+	if !expectedVersion.Served {
+		result.AddErrors(fmt.Sprintf("Expected API version %s for %s is known to api-server, but is not served. Please update your KUDO CLI.", version, crdName))
+	}
 	return nil
 }
 

pkg/kudoctl/util/kudo/kudo.go

@@ -58,40 +58,29 @@ func NewClient(kubeConfigPath string, requestTimeout int64, validateInstall bool
 
 // NewClient creates new KUDO Client
 func NewClientForConfig(config *rest.Config, validateInstall bool) (*Client, error) {
-
 	kubeClient, err := kube.GetKubeClientForConfig(config)
 	if err != nil {
 		return nil, clog.Errorf("could not get Kubernetes client: %s", err)
 	}
 
-	result := verifier.NewResult()
-	err = crd.NewInitializer().VerifyInstallation(kubeClient, &result)
-	if err != nil {
-		return nil, fmt.Errorf("failed to run crd verification: %v", err)
-	}
-	if !result.IsValid() {
-		clog.V(0).Printf("KUDO CRDs are not set up correctly. Do you need to run kudo init?")
-
-		if validateInstall {
-			return nil, fmt.Errorf("CRDs invalid: %v", result.ErrorsAsString())
-		}
-	}
-
 	// create the kudo clientset
 	kudoClientset, err := versioned.NewForConfig(config)
 	if err != nil {
 		return nil, err
 	}
 
 	// create the kubernetes clientset
-	kubeClientset, err := kubernetes.NewForConfig(config)
-	if err != nil {
-		return nil, err
-	}
-	return &Client{
+	client := &Client{
 		kudoClientset: kudoClientset,
-		KubeClientset: kubeClientset,
-	}, nil
+		KubeClientset: kubeClient.KubeClient,
+	}
+
+	validationErr := client.VerifyServedCRDs(kubeClient)
+	if validateInstall && validationErr != nil {
+		return nil, validationErr
+	}
+
+	return client, nil
 }
 
 // NewClientFromK8s creates KUDO client from kubernetes client interface
@@ -102,6 +91,20 @@ func NewClientFromK8s(kudo versioned.Interface, kube kubernetes.Interface) *Clie
 	return &result
 }
 
+func (c *Client) VerifyServedCRDs(kubeClient *kube.Client) error {
+	result := verifier.NewResult()
+	err := crd.NewInitializer().VerifyServedVersion(kubeClient, v1beta1.SchemeGroupVersion.Version, &result)
+	if err != nil {
+		return fmt.Errorf("failed to run crd verification: %v", err)
+	}
+	if !result.IsValid() {
+		clog.V(0).Printf("KUDO CRDs are not served in the expected version.")
+		return fmt.Errorf("CRDs invalid: %v", result.ErrorsAsString())
+	}
+
+	return nil
+}
+
 // OperatorExistsInCluster checks if a given Operator object is installed on the current k8s cluster
 func (c *Client) OperatorExistsInCluster(name, namespace string) bool {
 	operator, err := c.kudoClientset.KudoV1beta1().Operators(namespace).Get(context.TODO(), name, v1.GetOptions{})

pkg/kudoctl/util/kudo/kudo_test.go

@@ -8,13 +8,17 @@ import (
 
 	"github.com/stretchr/testify/assert"
 	v1 "k8s.io/api/core/v1"
+	apiextv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
+	apiextensionfake "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/fake"
 	"k8s.io/apimachinery/pkg/api/meta"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
+	dynamicfake "k8s.io/client-go/dynamic/fake"
 	kubefake "k8s.io/client-go/kubernetes/fake"
 
 	"github.com/kudobuilder/kudo/pkg/apis/kudo/v1beta1"
 	"github.com/kudobuilder/kudo/pkg/client/clientset/versioned/fake"
+	"github.com/kudobuilder/kudo/pkg/kudoctl/kube"
 	"github.com/kudobuilder/kudo/pkg/util/convert"
 	"github.com/kudobuilder/kudo/pkg/util/kudo"
 )
@@ -25,6 +29,129 @@ func newTestSimpleK2o() *Client {
 	return NewClientFromK8s(fake.NewSimpleClientset(), kubefake.NewSimpleClientset())
 }
 
+func newFakeKubeClient() *kube.Client {
+	client := kubefake.NewSimpleClientset()
+	extClient := apiextensionfake.NewSimpleClientset()
+	dynamicClient := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme())
+
+	return &kube.Client{
+		KubeClient:    client,
+		ExtClient:     extClient,
+		DynamicClient: dynamicClient}
+}
+
+func TestKudoClient_ValidateServedCrds(t *testing.T) {
+	crdWrongVersion := apiextv1beta1.CustomResourceDefinition{
+		Spec: apiextv1beta1.CustomResourceDefinitionSpec{
+			Versions: []apiextv1beta1.CustomResourceDefinitionVersion{
+				{
+					Name:   "v1beta2",
+					Served: true,
+				},
+			},
+		},
+		Status: apiextv1beta1.CustomResourceDefinitionStatus{
+			Conditions: []apiextv1beta1.CustomResourceDefinitionCondition{
+				{
+					Type:   apiextv1beta1.Established,
+					Status: apiextv1beta1.ConditionTrue,
+				},
+			},
+		},
+	}
+	crdNotServed := apiextv1beta1.CustomResourceDefinition{
+		Spec: apiextv1beta1.CustomResourceDefinitionSpec{
+			Versions: []apiextv1beta1.CustomResourceDefinitionVersion{
+				{
+					Name:   "v1beta1",
+					Served: false,
+				},
+				{
+					Name:   "v1beta2",
+					Served: true,
+				},
+			},
+		},
+		Status: apiextv1beta1.CustomResourceDefinitionStatus{
+			Conditions: []apiextv1beta1.CustomResourceDefinitionCondition{
+				{
+					Type:   apiextv1beta1.Established,
+					Status: apiextv1beta1.ConditionTrue,
+				},
+			},
+		},
+	}
+	crdUnHealthy := apiextv1beta1.CustomResourceDefinition{
+		Spec: apiextv1beta1.CustomResourceDefinitionSpec{
+			Versions: []apiextv1beta1.CustomResourceDefinitionVersion{
+				{
+					Name:   "v1beta2",
+					Served: true,
+				},
+			},
+		},
+		Status: apiextv1beta1.CustomResourceDefinitionStatus{
+			Conditions: []apiextv1beta1.CustomResourceDefinitionCondition{
+				{
+					Type:   apiextv1beta1.Established,
+					Status: apiextv1beta1.ConditionFalse,
+				},
+				{
+					Type:   apiextv1beta1.Terminating,
+					Status: apiextv1beta1.ConditionTrue,
+				},
+			},
+		},
+	}
+
+	tests := []struct {
+		name    string
+		crdBase *apiextv1beta1.CustomResourceDefinition
+		err     string
+	}{
+		{name: "no crd", err: `CRDs invalid: CRD operators.kudo.dev is not installed       
+CRD operatorversions.kudo.dev is not installed
+CRD instances.kudo.dev is not installed       
+`},
+		{name: "wrong version", crdBase: &crdWrongVersion, err: `CRDs invalid: Expected API version v1beta1 was not found for operators.kudo.dev, api-server only supports [v1beta2]. Please update your KUDO CLI.       
+Expected API version v1beta1 was not found for operatorversions.kudo.dev, api-server only supports [v1beta2]. Please update your KUDO CLI.
+Expected API version v1beta1 was not found for instances.kudo.dev, api-server only supports [v1beta2]. Please update your KUDO CLI.       
+`},
+		{name: "not served", crdBase: &crdNotServed, err: `CRDs invalid: Expected API version v1beta1 for operators.kudo.dev is known to api-server, but is not served. Please update your KUDO CLI.       
+Expected API version v1beta1 for operatorversions.kudo.dev is known to api-server, but is not served. Please update your KUDO CLI.
+Expected API version v1beta1 for instances.kudo.dev is known to api-server, but is not served. Please update your KUDO CLI.       
+`},
+		{name: "unhealthy", crdBase: &crdUnHealthy, err: `CRDs invalid: CRD operators.kudo.dev is not healthy ( Conditions: [{Established False 0001-01-01 00:00:00 +0000 UTC  } {Terminating True 0001-01-01 00:00:00 +0000 UTC  }] )       
+CRD operatorversions.kudo.dev is not healthy ( Conditions: [{Established False 0001-01-01 00:00:00 +0000 UTC  } {Terminating True 0001-01-01 00:00:00 +0000 UTC  }] )
+CRD instances.kudo.dev is not healthy ( Conditions: [{Established False 0001-01-01 00:00:00 +0000 UTC  } {Terminating True 0001-01-01 00:00:00 +0000 UTC  }] )       
+`},
+	}
+
+	crdNames := []string{"instances.kudo.dev", "operators.kudo.dev", "operatorversions.kudo.dev"}
+
+	for _, tt := range tests {
+		tt := tt
+		t.Run(tt.name, func(t *testing.T) {
+			kudoClient := newTestSimpleK2o()
+			kubeClient := newFakeKubeClient()
+
+			if tt.crdBase != nil {
+				for _, crdName := range crdNames {
+					crd := tt.crdBase.DeepCopy()
+					crd.Name = crdName
+					_, _ = kubeClient.ExtClient.ApiextensionsV1beta1().CustomResourceDefinitions().Create(context.TODO(), crd, metav1.CreateOptions{})
+				}
+			}
+
+			err := kudoClient.VerifyServedCRDs(kubeClient)
+
+			assert.EqualError(t, err, tt.err)
+
+		})
+	}
+
+}
+
 func TestKudoClient_OperatorExistsInCluster(t *testing.T) {
 
 	obj := v1beta1.Operator{