diff --git a/pkg/kudoctl/kudoinit/crd/crds.go b/pkg/kudoctl/kudoinit/crd/crds.go index 14cb37833..f1f50d614 100644 --- a/pkg/kudoctl/kudoinit/crd/crds.go +++ b/pkg/kudoctl/kudoinit/crd/crds.go @@ -5,6 +5,7 @@ import ( "context" "fmt" "os" + "reflect" apiextv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/typed/apiextensions/v1beta1" @@ -66,15 +67,31 @@ func (c Initializer) PreUpgradeVerify(client *kube.Client, result *verifier.Resu return nil } -// VerifyInstallation ensures that the CRDs are installed and have the correct and expected version +// VerifyInstallation ensures that all CRDs are installed and are the same as this CLI would install func (c Initializer) VerifyInstallation(client *kube.Client, result *verifier.Result) error { - if err := c.verifyInstallation(client.ExtClient.ApiextensionsV1beta1(), c.Operator, result); err != nil { + apiClient := client.ExtClient.ApiextensionsV1beta1() + if err := c.verifyInstallation(apiClient, c.Operator, result); err != nil { return err } - if err := c.verifyInstallation(client.ExtClient.ApiextensionsV1beta1(), c.OperatorVersion, result); err != nil { + if err := c.verifyInstallation(apiClient, c.OperatorVersion, result); err != nil { return err } - if err := c.verifyInstallation(client.ExtClient.ApiextensionsV1beta1(), c.Instance, result); err != nil { + if err := c.verifyInstallation(apiClient, c.Instance, result); err != nil { + return err + } + return nil +} + +// VerifyServedVersion ensures that the api server provides the correct version of all CRDs that this client understands +func (c Initializer) VerifyServedVersion(client *kube.Client, expectedVersion string, result *verifier.Result) error { + apiClient := client.ExtClient.ApiextensionsV1beta1() + if err := c.verifyServedVersion(apiClient, c.Operator.Name, expectedVersion, result); err != nil { + return err + } + if err := c.verifyServedVersion(apiClient, c.OperatorVersion.Name, expectedVersion, result); err != nil { + return err + } + if err := c.verifyServedVersion(apiClient, c.Instance.Name, expectedVersion, result); err != nil { return err } return nil @@ -94,6 +111,7 @@ func (c Initializer) Install(client *kube.Client) error { return nil } +// verifyIsNotInstalled is used to ensure that the cluster has no old KUDO version installed func (c Initializer) verifyIsNotInstalled(client v1beta1.CustomResourceDefinitionsGetter, crd *apiextv1beta1.CustomResourceDefinition, result *verifier.Result) error { _, err := client.CustomResourceDefinitions().Get(context.TODO(), crd.Name, v1.GetOptions{}) if err != nil { @@ -106,21 +124,29 @@ func (c Initializer) verifyIsNotInstalled(client v1beta1.CustomResourceDefinitio return nil } -func (c Initializer) verifyInstallation(client v1beta1.CustomResourceDefinitionsGetter, crd *apiextv1beta1.CustomResourceDefinition, result *verifier.Result) error { - existingCrd, err := client.CustomResourceDefinitions().Get(context.TODO(), crd.Name, v1.GetOptions{}) +func (c Initializer) getCrdForVerify(client v1beta1.CustomResourceDefinitionsGetter, crdName string, result *verifier.Result) (*apiextv1beta1.CustomResourceDefinition, error) { + existingCrd, err := client.CustomResourceDefinitions().Get(context.TODO(), crdName, v1.GetOptions{}) if err != nil { if os.IsTimeout(err) { - return err + return nil, err } if kerrors.IsNotFound(err) { - result.AddErrors(fmt.Sprintf("CRD %s is not installed", crd.Name)) - return nil + result.AddErrors(fmt.Sprintf("CRD %s is not installed", crdName)) + return nil, nil } - return fmt.Errorf("failed to retrieve CRD %s: %v", crd.Name, err) + return nil, fmt.Errorf("failed to retrieve CRD %s: %v", crdName, err) } - if existingCrd.Spec.Version != crd.Spec.Version { - result.AddErrors(fmt.Sprintf("Installed CRD %s has invalid version %s, expected %s", crd.Name, existingCrd.Spec.Version, crd.Spec.Version)) - return nil + return existingCrd, nil +} + +// VerifyInstallation ensures that a single CRD is installed and is the same as this CLI would install +func (c Initializer) verifyInstallation(client v1beta1.CustomResourceDefinitionsGetter, crd *apiextv1beta1.CustomResourceDefinition, result *verifier.Result) error { + existingCrd, err := c.getCrdForVerify(client, crd.Name, result) + if err != nil || existingCrd == nil { + return err + } + if !reflect.DeepEqual(existingCrd.Spec.Versions, crd.Spec.Versions) { + result.AddErrors(fmt.Sprintf("Installed CRD versions do not match expected CRD versions (%v vs %v).", existingCrd.Spec.Versions, crd.Spec.Versions)) } if healthy, msg, err := status.IsHealthy(existingCrd); !healthy || err != nil { if err != nil { @@ -129,7 +155,38 @@ func (c Initializer) verifyInstallation(client v1beta1.CustomResourceDefinitions result.AddErrors(fmt.Sprintf("Installed CRD %s is not healthy: %v", crd.Name, msg)) return nil } - clog.V(2).Printf("CRD %s is installed with version %s", crd.Name, existingCrd.Spec.Versions[0].Name) + clog.V(2).Printf("CRD %s is installed with versions %v", crd.Name, existingCrd.Spec.Versions) + return nil +} + +// VerifyServedVersion ensures that the api server provides the correct version of a specific CRDs that this client understands +func (c Initializer) verifyServedVersion(client v1beta1.CustomResourceDefinitionsGetter, crdName, version string, result *verifier.Result) error { + existingCrd, err := c.getCrdForVerify(client, crdName, result) + if err != nil || existingCrd == nil { + return err + } + if err := health.IsHealthy(existingCrd); err != nil { + result.AddErrors(err.Error()) + return nil + } + + var expectedVersion *apiextv1beta1.CustomResourceDefinitionVersion + var allNames = []string{} + for _, v := range existingCrd.Spec.Versions { + v := v + allNames = append(allNames, v.Name) + if v.Name == version { + expectedVersion = &v + break + } + } + if expectedVersion == nil { + result.AddErrors(fmt.Sprintf("Expected API version %s was not found for %s, api-server only supports %v. Please update your KUDO CLI.", version, crdName, allNames)) + return nil + } + if !expectedVersion.Served { + result.AddErrors(fmt.Sprintf("Expected API version %s for %s is known to api-server, but is not served. Please update your KUDO CLI.", version, crdName)) + } return nil } diff --git a/pkg/kudoctl/util/kudo/kudo.go b/pkg/kudoctl/util/kudo/kudo.go index 638b55478..970776eb1 100644 --- a/pkg/kudoctl/util/kudo/kudo.go +++ b/pkg/kudoctl/util/kudo/kudo.go @@ -58,25 +58,11 @@ func NewClient(kubeConfigPath string, requestTimeout int64, validateInstall bool // NewClient creates new KUDO Client func NewClientForConfig(config *rest.Config, validateInstall bool) (*Client, error) { - kubeClient, err := kube.GetKubeClientForConfig(config) if err != nil { return nil, clog.Errorf("could not get Kubernetes client: %s", err) } - result := verifier.NewResult() - err = crd.NewInitializer().VerifyInstallation(kubeClient, &result) - if err != nil { - return nil, fmt.Errorf("failed to run crd verification: %v", err) - } - if !result.IsValid() { - clog.V(0).Printf("KUDO CRDs are not set up correctly. Do you need to run kudo init?") - - if validateInstall { - return nil, fmt.Errorf("CRDs invalid: %v", result.ErrorsAsString()) - } - } - // create the kudo clientset kudoClientset, err := versioned.NewForConfig(config) if err != nil { @@ -84,14 +70,17 @@ func NewClientForConfig(config *rest.Config, validateInstall bool) (*Client, err } // create the kubernetes clientset - kubeClientset, err := kubernetes.NewForConfig(config) - if err != nil { - return nil, err - } - return &Client{ + client := &Client{ kudoClientset: kudoClientset, - KubeClientset: kubeClientset, - }, nil + KubeClientset: kubeClient.KubeClient, + } + + validationErr := client.VerifyServedCRDs(kubeClient) + if validateInstall && validationErr != nil { + return nil, validationErr + } + + return client, nil } // NewClientFromK8s creates KUDO client from kubernetes client interface @@ -102,6 +91,20 @@ func NewClientFromK8s(kudo versioned.Interface, kube kubernetes.Interface) *Clie return &result } +func (c *Client) VerifyServedCRDs(kubeClient *kube.Client) error { + result := verifier.NewResult() + err := crd.NewInitializer().VerifyServedVersion(kubeClient, v1beta1.SchemeGroupVersion.Version, &result) + if err != nil { + return fmt.Errorf("failed to run crd verification: %v", err) + } + if !result.IsValid() { + clog.V(0).Printf("KUDO CRDs are not served in the expected version.") + return fmt.Errorf("CRDs invalid: %v", result.ErrorsAsString()) + } + + return nil +} + // OperatorExistsInCluster checks if a given Operator object is installed on the current k8s cluster func (c *Client) OperatorExistsInCluster(name, namespace string) bool { operator, err := c.kudoClientset.KudoV1beta1().Operators(namespace).Get(context.TODO(), name, v1.GetOptions{}) diff --git a/pkg/kudoctl/util/kudo/kudo_test.go b/pkg/kudoctl/util/kudo/kudo_test.go index 26d8f510f..2b40a7543 100644 --- a/pkg/kudoctl/util/kudo/kudo_test.go +++ b/pkg/kudoctl/util/kudo/kudo_test.go @@ -8,13 +8,17 @@ import ( "github.com/stretchr/testify/assert" v1 "k8s.io/api/core/v1" + apiextv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" + apiextensionfake "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/fake" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + dynamicfake "k8s.io/client-go/dynamic/fake" kubefake "k8s.io/client-go/kubernetes/fake" kudoapi "github.com/kudobuilder/kudo/pkg/apis/kudo/v1beta1" "github.com/kudobuilder/kudo/pkg/client/clientset/versioned/fake" + "github.com/kudobuilder/kudo/pkg/kudoctl/kube" "github.com/kudobuilder/kudo/pkg/util/convert" "github.com/kudobuilder/kudo/pkg/util/kudo" ) @@ -25,6 +29,129 @@ func newTestSimpleK2o() *Client { return NewClientFromK8s(fake.NewSimpleClientset(), kubefake.NewSimpleClientset()) } +func newFakeKubeClient() *kube.Client { + client := kubefake.NewSimpleClientset() + extClient := apiextensionfake.NewSimpleClientset() + dynamicClient := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme()) + + return &kube.Client{ + KubeClient: client, + ExtClient: extClient, + DynamicClient: dynamicClient} +} + +func TestKudoClient_ValidateServedCrds(t *testing.T) { + crdWrongVersion := apiextv1beta1.CustomResourceDefinition{ + Spec: apiextv1beta1.CustomResourceDefinitionSpec{ + Versions: []apiextv1beta1.CustomResourceDefinitionVersion{ + { + Name: "v1beta2", + Served: true, + }, + }, + }, + Status: apiextv1beta1.CustomResourceDefinitionStatus{ + Conditions: []apiextv1beta1.CustomResourceDefinitionCondition{ + { + Type: apiextv1beta1.Established, + Status: apiextv1beta1.ConditionTrue, + }, + }, + }, + } + crdNotServed := apiextv1beta1.CustomResourceDefinition{ + Spec: apiextv1beta1.CustomResourceDefinitionSpec{ + Versions: []apiextv1beta1.CustomResourceDefinitionVersion{ + { + Name: "v1beta1", + Served: false, + }, + { + Name: "v1beta2", + Served: true, + }, + }, + }, + Status: apiextv1beta1.CustomResourceDefinitionStatus{ + Conditions: []apiextv1beta1.CustomResourceDefinitionCondition{ + { + Type: apiextv1beta1.Established, + Status: apiextv1beta1.ConditionTrue, + }, + }, + }, + } + crdUnHealthy := apiextv1beta1.CustomResourceDefinition{ + Spec: apiextv1beta1.CustomResourceDefinitionSpec{ + Versions: []apiextv1beta1.CustomResourceDefinitionVersion{ + { + Name: "v1beta2", + Served: true, + }, + }, + }, + Status: apiextv1beta1.CustomResourceDefinitionStatus{ + Conditions: []apiextv1beta1.CustomResourceDefinitionCondition{ + { + Type: apiextv1beta1.Established, + Status: apiextv1beta1.ConditionFalse, + }, + { + Type: apiextv1beta1.Terminating, + Status: apiextv1beta1.ConditionTrue, + }, + }, + }, + } + + tests := []struct { + name string + crdBase *apiextv1beta1.CustomResourceDefinition + err string + }{ + {name: "no crd", err: `CRDs invalid: CRD operators.kudo.dev is not installed +CRD operatorversions.kudo.dev is not installed +CRD instances.kudo.dev is not installed +`}, + {name: "wrong version", crdBase: &crdWrongVersion, err: `CRDs invalid: Expected API version v1beta1 was not found for operators.kudo.dev, api-server only supports [v1beta2]. Please update your KUDO CLI. +Expected API version v1beta1 was not found for operatorversions.kudo.dev, api-server only supports [v1beta2]. Please update your KUDO CLI. +Expected API version v1beta1 was not found for instances.kudo.dev, api-server only supports [v1beta2]. Please update your KUDO CLI. +`}, + {name: "not served", crdBase: &crdNotServed, err: `CRDs invalid: Expected API version v1beta1 for operators.kudo.dev is known to api-server, but is not served. Please update your KUDO CLI. +Expected API version v1beta1 for operatorversions.kudo.dev is known to api-server, but is not served. Please update your KUDO CLI. +Expected API version v1beta1 for instances.kudo.dev is known to api-server, but is not served. Please update your KUDO CLI. +`}, + {name: "unhealthy", crdBase: &crdUnHealthy, err: `CRDs invalid: CRD operators.kudo.dev is not healthy ( Conditions: [{Established False 0001-01-01 00:00:00 +0000 UTC } {Terminating True 0001-01-01 00:00:00 +0000 UTC }] ) +CRD operatorversions.kudo.dev is not healthy ( Conditions: [{Established False 0001-01-01 00:00:00 +0000 UTC } {Terminating True 0001-01-01 00:00:00 +0000 UTC }] ) +CRD instances.kudo.dev is not healthy ( Conditions: [{Established False 0001-01-01 00:00:00 +0000 UTC } {Terminating True 0001-01-01 00:00:00 +0000 UTC }] ) +`}, + } + + crdNames := []string{"instances.kudo.dev", "operators.kudo.dev", "operatorversions.kudo.dev"} + + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + kudoClient := newTestSimpleK2o() + kubeClient := newFakeKubeClient() + + if tt.crdBase != nil { + for _, crdName := range crdNames { + crd := tt.crdBase.DeepCopy() + crd.Name = crdName + _, _ = kubeClient.ExtClient.ApiextensionsV1beta1().CustomResourceDefinitions().Create(context.TODO(), crd, metav1.CreateOptions{}) + } + } + + err := kudoClient.VerifyServedCRDs(kubeClient) + + assert.EqualError(t, err, tt.err) + + }) + } + +} + func TestKudoClient_OperatorExistsInCluster(t *testing.T) { obj := kudoapi.Operator{