From ec8a9376e703c345c27c049ae4a2293608dd74a2 Mon Sep 17 00:00:00 2001 From: Andriy Bulynko Date: Wed, 3 Jun 2020 10:38:11 -0400 Subject: [PATCH] Checking if CRDs are installed before allowing install to proceed (#387) --- go.mod | 1 + pkg/qliksense/config.go | 6 ++- pkg/qliksense/crds.go | 94 +++++++++++++++++++++++++++++++++--- pkg/qliksense/crds_test.go | 98 +++++++++++++++++++++++++++++++++++++- pkg/qliksense/install.go | 16 ++++--- 5 files changed, 198 insertions(+), 17 deletions(-) diff --git a/go.mod b/go.mod index 4bde7732..2ad328b4 100644 --- a/go.mod +++ b/go.mod @@ -52,6 +52,7 @@ require ( google.golang.org/genproto v0.0.0-20200128133413-58ce757ed39b // indirect gopkg.in/yaml.v2 v2.2.8 k8s.io/api v0.17.2 + k8s.io/apiextensions-apiserver v0.17.2 k8s.io/apimachinery v0.17.2 k8s.io/client-go v11.0.0+incompatible sigs.k8s.io/kustomize/api v0.3.2 diff --git a/pkg/qliksense/config.go b/pkg/qliksense/config.go index 016736d6..10302418 100644 --- a/pkg/qliksense/config.go +++ b/pkg/qliksense/config.go @@ -40,7 +40,7 @@ func (q *Qliksense) ConfigApplyQK8s() error { } // create patch dependent resoruces - fmt.Println("Installing resoruces used kuztomize patch") + fmt.Println("Installing resources used by the kuztomize patch") if err := q.createK8sResoruceBeforePatch(qcr); err != nil { return err } @@ -89,11 +89,13 @@ func (q *Qliksense) applyConfigToK8s(qcr *qapi.QliksenseCR) error { cr.GeneratePatches(&qcr.KApiCr, path.Join(userHomeDir, ".kube", "config")) // apply generated manifests profilePath := filepath.Join(qcr.Spec.GetManifestsRoot(), qcr.Spec.GetProfileDir()) + fmt.Printf("Generating manifests for profile: %v\n", profilePath) mByte, err := ExecuteKustomizeBuild(profilePath) if err != nil { - fmt.Println("cannot generate manifests for "+profilePath, err) + fmt.Printf("error generating manifests: %v\n", err) return err } + fmt.Println("Applying manifests to the cluster") if err = qapi.KubectlApply(string(mByte), qcr.GetNamespace()); err != nil { return err } diff --git a/pkg/qliksense/crds.go b/pkg/qliksense/crds.go index 7f5bd091..4fab11c7 100644 --- a/pkg/qliksense/crds.go +++ b/pkg/qliksense/crds.go @@ -5,7 +5,15 @@ import ( "os" "path/filepath" + "github.com/mitchellh/go-homedir" qapi "github.com/qlik-oss/sense-installer/pkg/api" + apixv1beta1client "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/typed/apiextensions/v1beta1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/tools/clientcmd" + "sigs.k8s.io/kustomize/api/k8sdeps/kunstruct" + "sigs.k8s.io/kustomize/api/resmap" + "sigs.k8s.io/kustomize/api/resource" ) type CrdCommandOptions struct { @@ -20,11 +28,11 @@ func (q *Qliksense) ViewCrds(opts *CrdCommandOptions) error { fmt.Println("cannot get the current-context cr", err) return err } - engineCRD, err := getQliksenseInitCrd(qcr) + engineCRD, err := getQliksenseInitCrds(qcr) if err != nil { return err } - customCrd, err := getCustomCrd(qcr) + customCrd, err := getCustomCrds(qcr) if err != nil { return nil } @@ -51,12 +59,12 @@ func (q *Qliksense) InstallCrds(opts *CrdCommandOptions) error { return err } - if engineCRD, err := getQliksenseInitCrd(qcr); err != nil { + if engineCRD, err := getQliksenseInitCrds(qcr); err != nil { return err } else if err = qapi.KubectlApply(engineCRD, ""); err != nil { return err } - if customCrd, err := getCustomCrd(qcr); err != nil { + if customCrd, err := getCustomCrds(qcr); err != nil { return err } else if customCrd != "" { if err = qapi.KubectlApply(customCrd, ""); err != nil { @@ -73,7 +81,7 @@ func (q *Qliksense) InstallCrds(opts *CrdCommandOptions) error { return nil } -func getQliksenseInitCrd(qcr *qapi.QliksenseCR) (string, error) { +func getQliksenseInitCrds(qcr *qapi.QliksenseCR) (string, error) { var repoPath string var err error @@ -98,7 +106,7 @@ func getQliksenseInitCrd(qcr *qapi.QliksenseCR) (string, error) { return string(qInitByte), nil } -func getCustomCrd(qcr *qapi.QliksenseCR) (string, error) { +func getCustomCrds(qcr *qapi.QliksenseCR) (string, error) { crdPath := qcr.GetCustomCrdsPath() if crdPath == "" { return "", nil @@ -110,3 +118,77 @@ func getCustomCrd(qcr *qapi.QliksenseCR) (string, error) { } return string(qInitByte), nil } + +func (q *Qliksense) CheckAllCrdsInstalled() (bool, error) { + qConfig := qapi.NewQConfig(q.QliksenseHome) + qcr, err := qConfig.GetCurrentCR() + if err != nil { + return false, err + } + + customResourceDefinitionInterface, err := getCustomResourceDefinitionInterface() + if err != nil { + return false, err + } + + if engineCRDs, err := getQliksenseInitCrds(qcr); err != nil { + return false, err + } else if allInstalled, err := checkCrdsInstalled(engineCRDs, customResourceDefinitionInterface); err != nil { + return false, err + } else if !allInstalled { + return false, nil + } + + if customCrds, err := getCustomCrds(qcr); err != nil { + return false, err + } else if allInstalled, err := checkCrdsInstalled(customCrds, customResourceDefinitionInterface); err != nil { + return false, err + } else if !allInstalled { + return false, nil + } + + if allInstalled, err := checkCrdsInstalled(q.GetOperatorCRDString(), customResourceDefinitionInterface); err != nil { + return false, err + } else if !allInstalled { + return false, nil + } + + return true, nil +} + +func checkCrdsInstalled(crds string, customResourceDefinitionInterface apixv1beta1client.CustomResourceDefinitionInterface) (bool, error) { + kuzResourceFactory := resmap.NewFactory(resource.NewFactory(kunstruct.NewKunstructuredFactoryImpl()), nil) + if kuzResMap, err := kuzResourceFactory.NewResMapFromBytes([]byte(crds)); err != nil { + return false, err + } else { + for _, kuzRes := range kuzResMap.Resources() { + if customResourceDefinition, err := customResourceDefinitionInterface.Get(kuzRes.GetName(), v1.GetOptions{}); err != nil && apierrors.IsNotFound(err) { + return false, nil + } else if err != nil { + return false, err + } else if customResourceDefinition == nil { + return false, fmt.Errorf("failed looking up crd: %v", kuzRes.GetName()) + } + } + return true, nil + } +} + +func getCustomResourceDefinitionInterface() (apixv1beta1client.CustomResourceDefinitionInterface, error) { + homeDir, err := homedir.Dir() + if err != nil { + return nil, err + } + kubeconfigPath := filepath.Join(homeDir, ".kube", "config") + k8sRestConfig, err := clientcmd.BuildConfigFromFlags("", kubeconfigPath) + if err != nil { + return nil, err + } + + apixClient, err := apixv1beta1client.NewForConfig(k8sRestConfig) + if err != nil { + return nil, err + } + + return apixClient.CustomResourceDefinitions(), nil +} diff --git a/pkg/qliksense/crds_test.go b/pkg/qliksense/crds_test.go index b19aac7b..19fcd8a7 100644 --- a/pkg/qliksense/crds_test.go +++ b/pkg/qliksense/crds_test.go @@ -1,8 +1,18 @@ package qliksense import ( + "io/ioutil" + "os" "testing" + apixv1beta1client "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/typed/apiextensions/v1beta1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/kustomize/api/k8sdeps/kunstruct" + "sigs.k8s.io/kustomize/api/resmap" + "sigs.k8s.io/kustomize/api/resource" + + "github.com/gobuffalo/packr/v2" + kapi_config "github.com/qlik-oss/k-apis/pkg/config" qapi "github.com/qlik-oss/sense-installer/pkg/api" ) @@ -13,7 +23,7 @@ func TestGetQliksenseInitCrd(t *testing.T) { t.Fatalf("unexpected error: %v", err) } - crdFromContextConfig, err := getQliksenseInitCrd(&qapi.QliksenseCR{ + crdFromContextConfig, err := getQliksenseInitCrds(&qapi.QliksenseCR{ KApiCr: kapi_config.KApiCr{ Spec: &kapi_config.CRSpec{ ManifestsRoot: someTmpRepoPath, @@ -24,7 +34,7 @@ func TestGetQliksenseInitCrd(t *testing.T) { t.Fatalf("unexpected error: %v", err) } - crdFromDownloadedConfig, err := getQliksenseInitCrd(&qapi.QliksenseCR{ + crdFromDownloadedConfig, err := getQliksenseInitCrds(&qapi.QliksenseCR{ KApiCr: kapi_config.KApiCr{ Spec: &kapi_config.CRSpec{ ManifestsRoot: "", @@ -39,3 +49,87 @@ func TestGetQliksenseInitCrd(t *testing.T) { t.Fatalf("expected %v to equal %v, but they didn't", crdFromContextConfig, crdFromDownloadedConfig) } } + +func TestCheckAllCrdsInstalled(t *testing.T) { + t.Skip("Skipping this test because it makes kubernetes calls") + + tmpQlikSenseHome, err := ioutil.TempDir("", "tmp-qlik-sense-home-") + if err != nil { + t.Fatalf("unexpected error creating tmp dir: %v", err) + } + defer os.RemoveAll(tmpQlikSenseHome) + + setupQliksenseTestDefaultContext(t, tmpQlikSenseHome, ` +apiVersion: qlik.com/v1 +kind: Qliksense +metadata: + name: qlik-default +spec: + profile: docker-desktop +`) + + q := &Qliksense{ + QliksenseHome: tmpQlikSenseHome, + CrdBox: packr.New("crds", "./crds"), + } + + if err := q.FetchQK8s("v1.50.3"); err != nil { + t.Fatalf("unexpected error: %v", err) + } + + if allInstalled, err := q.CheckAllCrdsInstalled(); err != nil { + t.Fatalf("unexpected error: %v", err) + } else if allInstalled { + t.Fatal("expected crds to NOT be installed at this point") + } + + if err := q.InstallCrds(&CrdCommandOptions{All: true}); err != nil { + t.Fatalf("unexpected error: %v", err) + } else if allInstalled, err := q.CheckAllCrdsInstalled(); err != nil { + t.Fatalf("unexpected error: %v", err) + } else if !allInstalled { + t.Fatal("expected crds to BE installed at this point") + } + + //cleanup: + qConfig := qapi.NewQConfig(q.QliksenseHome) + qcr, err := qConfig.GetCurrentCR() + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + customResourceDefinitionInterface, err := getCustomResourceDefinitionInterface() + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + if engineCRDs, err := getQliksenseInitCrds(qcr); err != nil { + t.Fatalf("unexpected error: %v", err) + } else if err := deleteCrds(engineCRDs, customResourceDefinitionInterface); err != nil { + t.Fatalf("unexpected error: %v", err) + } + + if customCrd, err := getCustomCrds(qcr); err != nil { + t.Fatalf("unexpected error: %v", err) + } else if err := deleteCrds(customCrd, customResourceDefinitionInterface); err != nil { + t.Fatalf("unexpected error: %v", err) + } + + if err := deleteCrds(q.GetOperatorCRDString(), customResourceDefinitionInterface); err != nil { + t.Fatalf("unexpected error: %v", err) + } +} + +func deleteCrds(crds string, customResourceDefinitionInterface apixv1beta1client.CustomResourceDefinitionInterface) error { + kuzResourceFactory := resmap.NewFactory(resource.NewFactory(kunstruct.NewKunstructuredFactoryImpl()), nil) + if kuzResMap, err := kuzResourceFactory.NewResMapFromBytes([]byte(crds)); err != nil { + return err + } else { + for _, kuzRes := range kuzResMap.Resources() { + if err := customResourceDefinitionInterface.Delete(kuzRes.GetName(), &v1.DeleteOptions{}); err != nil { + return err + } + } + return nil + } +} diff --git a/pkg/qliksense/install.go b/pkg/qliksense/install.go index 52ce8d7d..a94b8990 100644 --- a/pkg/qliksense/install.go +++ b/pkg/qliksense/install.go @@ -65,13 +65,15 @@ func (q *Qliksense) InstallQK8s(version string, opts *InstallCommandOptions, kee } qConfig.WriteCurrentContextCR(qcr) - if err := applyImagePullSecret(qConfig); err != nil { + if installed, err := q.CheckAllCrdsInstalled(); err != nil { + fmt.Println("error verifying whether CRDs are installed", err) return err + } else if !installed { + return errors.New(`please install CRDs by executing: $ qliksense crds install --all`) } - // check if acceptEULA is yes or not - if !qcr.IsEULA() { - return errors.New(agreementTempalte + "\n Please do $ qliksense install --acceptEULA=yes\n") + if err := applyImagePullSecret(qConfig); err != nil { + return err } //CRD will be installed outside of operator @@ -86,7 +88,7 @@ func (q *Qliksense) InstallQK8s(version string, opts *InstallCommandOptions, kee } // create patch dependent resoruces - fmt.Println("Installing resoruces used kuztomize patch") + fmt.Println("Installing resources used by the kuztomize patch") if err := q.createK8sResoruceBeforePatch(qcr); err != nil { return err } @@ -115,7 +117,7 @@ func (q *Qliksense) InstallQK8s(version string, opts *InstallCommandOptions, kee } // install generated manifests into cluster - fmt.Println("Installing generated manifests into cluster") + fmt.Println("Installing generated manifests into the cluster") if dcr, err := qConfig.GetDecryptedCr(qcr); err != nil { return err @@ -193,7 +195,7 @@ images: func (q *Qliksense) applyCR(cr *qapi.QliksenseCR) error { // install operator cr into cluster //get the current context cr - fmt.Println("Install operator CR into cluster") + fmt.Println("Installing operator CR into the cluster") r, err := cr.GetString() if err != nil { return err