diff --git a/apis/ossm.plugins.kubeflow.org/v1alpha1/ossm_types.go b/apis/ossm.plugins.kubeflow.org/v1alpha1/ossm_types.go index eb96b2f8ccf..2c2f1bcb938 100644 --- a/apis/ossm.plugins.kubeflow.org/v1alpha1/ossm_types.go +++ b/apis/ossm.plugins.kubeflow.org/v1alpha1/ossm_types.go @@ -17,12 +17,14 @@ type OssmPlugin struct { Status OssmPluginStatus `json:"status,omitempty"` } -// OssmPluginSpec defines the extra data provided by the Openshift Service Mesh Plugin in KfDef spec. +// OssmPluginSpec defines configuration needed for Openshift Service Mesh +// for integration with Opendatahub. type OssmPluginSpec struct { Mesh MeshSpec `json:"mesh,omitempty"` Auth AuthSpec `json:"auth,omitempty"` } +// MeshSpec holds information on how Service Mesh should be configured. type MeshSpec struct { Name string `json:"name,omitempty"` Namespace string `json:"namespace,omitempty"` @@ -30,7 +32,7 @@ type MeshSpec struct { } type CertSpec struct { - Name string `json:"name,omitempty" default:"opendatahub-self-signed-cert"` + Name string `json:"name,omitempty"` Generate bool `json:"generate,omitempty"` } @@ -55,13 +57,49 @@ type OssmPluginStatus struct { //+kubebuilder:object:root=true -// OssmPluginList contains a list of GcpPlugin +// OssmPluginList contains a list of OssmPlugins type OssmPluginList struct { metav1.TypeMeta `json:",inline"` metav1.ListMeta `json:"metadata,omitempty"` Items []OssmPlugin `json:"items"` } +// OssmResourceTracker is a cluster-scoped resource for tracking objects +// created by Ossm plugin. It's primarily used as owner reference +// for resources created across namespaces so that they can be +// garbage collected by Kubernetes when they're not needed anymore. +// +kubebuilder:object:root=true +// +kubebuilder:resource:scope=Cluster +type OssmResourceTracker struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec OssmResourceTrackerSpec `json:"spec,omitempty"` + Status OssmResourceTrackerStatus `json:"status,omitempty"` +} + +// OssmResourceTrackerSpec defines the desired state of OssmResourceTracker +type OssmResourceTrackerSpec struct { +} + +// OssmResourceTrackerStatus defines the observed state of OssmResourceTracker +type OssmResourceTrackerStatus struct { +} + +// +kubebuilder:object:root=true + +// OssmResourceTrackerList contains a list of OssmResourceTracker +type OssmResourceTrackerList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []OssmResourceTracker `json:"items"` +} + func init() { - SchemeBuilder.Register(&OssmPlugin{}, &OssmPluginList{}) + SchemeBuilder.Register( + &OssmPlugin{}, + &OssmPluginList{}, + &OssmResourceTracker{}, + &OssmResourceTrackerList{}, + ) } diff --git a/apis/ossm.plugins.kubeflow.org/v1alpha1/zz_generated.deepcopy.go b/apis/ossm.plugins.kubeflow.org/v1alpha1/zz_generated.deepcopy.go index e57dc39d8fd..efc1769b36a 100644 --- a/apis/ossm.plugins.kubeflow.org/v1alpha1/zz_generated.deepcopy.go +++ b/apis/ossm.plugins.kubeflow.org/v1alpha1/zz_generated.deepcopy.go @@ -177,3 +177,92 @@ func (in *OssmPluginStatus) DeepCopy() *OssmPluginStatus { in.DeepCopyInto(out) return out } + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *OssmResourceTracker) DeepCopyInto(out *OssmResourceTracker) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + out.Spec = in.Spec + out.Status = in.Status +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OssmResourceTracker. +func (in *OssmResourceTracker) DeepCopy() *OssmResourceTracker { + if in == nil { + return nil + } + out := new(OssmResourceTracker) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *OssmResourceTracker) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *OssmResourceTrackerList) DeepCopyInto(out *OssmResourceTrackerList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]OssmResourceTracker, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OssmResourceTrackerList. +func (in *OssmResourceTrackerList) DeepCopy() *OssmResourceTrackerList { + if in == nil { + return nil + } + out := new(OssmResourceTrackerList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *OssmResourceTrackerList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *OssmResourceTrackerSpec) DeepCopyInto(out *OssmResourceTrackerSpec) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OssmResourceTrackerSpec. +func (in *OssmResourceTrackerSpec) DeepCopy() *OssmResourceTrackerSpec { + if in == nil { + return nil + } + out := new(OssmResourceTrackerSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *OssmResourceTrackerStatus) DeepCopyInto(out *OssmResourceTrackerStatus) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OssmResourceTrackerStatus. +func (in *OssmResourceTrackerStatus) DeepCopy() *OssmResourceTrackerStatus { + if in == nil { + return nil + } + out := new(OssmResourceTrackerStatus) + in.DeepCopyInto(out) + return out +} diff --git a/bundle/manifests/ossm.plugins.kubeflow.org_ossmresourcetrackers.yaml b/bundle/manifests/ossm.plugins.kubeflow.org_ossmresourcetrackers.yaml new file mode 100644 index 00000000000..665d0115845 --- /dev/null +++ b/bundle/manifests/ossm.plugins.kubeflow.org_ossmresourcetrackers.yaml @@ -0,0 +1,51 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.9.2 + creationTimestamp: null + name: ossmresourcetrackers.ossm.plugins.kubeflow.org +spec: + group: ossm.plugins.kubeflow.org + names: + kind: OssmResourceTracker + listKind: OssmResourceTrackerList + plural: ossmresourcetrackers + singular: ossmresourcetracker + scope: Cluster + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: OssmResourceTracker is a cluster-scoped resource for tracking + objects created by Ossm plugin. It's primarily used as owner reference for + resources created across namespaces so that they can be garbage collected + by Kubernetes when they're not needed anymore. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: OssmResourceTrackerSpec defines the desired state of OssmResourceTracker + type: object + status: + description: OssmResourceTrackerStatus defines the observed state of OssmResourceTracker + type: object + type: object + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: null + storedVersions: null diff --git a/config/crd/bases/ossm.plugins.kubeflow.org_ossmplugins.yaml b/config/crd/bases/ossm.plugins.kubeflow.org_ossmplugins.yaml index 541dc6d664f..59b16a11552 100644 --- a/config/crd/bases/ossm.plugins.kubeflow.org_ossmplugins.yaml +++ b/config/crd/bases/ossm.plugins.kubeflow.org_ossmplugins.yaml @@ -32,8 +32,8 @@ spec: metadata: type: object spec: - description: OssmPluginSpec defines the extra data provided by the Openshift - Service Mesh Plugin in KfDef spec. + description: OssmPluginSpec defines configuration needed for Openshift + Service Mesh for integration with Opendatahub. properties: auth: properties: @@ -52,6 +52,8 @@ spec: type: string type: object mesh: + description: MeshSpec holds information on how Service Mesh should + be configured. properties: certificate: properties: diff --git a/config/crd/bases/ossm.plugins.kubeflow.org_ossmresourcetrackers.yaml b/config/crd/bases/ossm.plugins.kubeflow.org_ossmresourcetrackers.yaml new file mode 100644 index 00000000000..6ead4e764c6 --- /dev/null +++ b/config/crd/bases/ossm.plugins.kubeflow.org_ossmresourcetrackers.yaml @@ -0,0 +1,46 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.9.2 + creationTimestamp: null + name: ossmresourcetrackers.ossm.plugins.kubeflow.org +spec: + group: ossm.plugins.kubeflow.org + names: + kind: OssmResourceTracker + listKind: OssmResourceTrackerList + plural: ossmresourcetrackers + singular: ossmresourcetracker + scope: Cluster + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: OssmResourceTracker is a cluster-scoped resource for tracking + objects created by Ossm plugin. It's primarily used as owner reference for + resources created across namespaces so that they can be garbage collected + by Kubernetes when they're not needed anymore. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: OssmResourceTrackerSpec defines the desired state of OssmResourceTracker + type: object + status: + description: OssmResourceTrackerStatus defines the observed state of OssmResourceTracker + type: object + type: object + served: true + storage: true diff --git a/config/crd/kustomization.yaml b/config/crd/kustomization.yaml index a2ac69087f0..1994da20f60 100644 --- a/config/crd/kustomization.yaml +++ b/config/crd/kustomization.yaml @@ -4,6 +4,7 @@ resources: - bases/kfdef.apps.kubeflow.org_kfdefs.yaml - bases/ossm.plugins.kubeflow.org_ossmplugins.yaml + - bases/ossm.plugins.kubeflow.org_ossmresourcetrackers.yaml - dashboard-crds/odhapplications.dashboard.opendatahub.io.crd.yaml - dashboard-crds/odhdashboardconfigs.opendatahub.io.crd.yaml - dashboard-crds/odhdocuments.dashboard.opendatahub.io.crd.yaml diff --git a/main.go b/main.go index 258b4dc0464..856f3249181 100644 --- a/main.go +++ b/main.go @@ -156,7 +156,7 @@ func main() { os.Exit(1) } - setupLog.Info("starting manager 123 !!!!") + setupLog.Info("starting manager") if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil { setupLog.Error(err, "problem running manager") os.Exit(1) diff --git a/pkg/kfapp/coordinator/coordinator.go b/pkg/kfapp/coordinator/coordinator.go index f26f45f1e22..e7f9fd9a658 100644 --- a/pkg/kfapp/coordinator/coordinator.go +++ b/pkg/kfapp/coordinator/coordinator.go @@ -471,6 +471,22 @@ func (kfapp *coordinator) Delete(resources kftypesv3.ResourceEnum) error { return nil } + ossmCleanup := func() error { + if kfapp.KfDef.Spec.Platform != kftypesv3.OSSM { + return nil + } + + if p, ok := kfapp.Platforms[kfapp.KfDef.Spec.Platform]; !ok { + return &kfapis.KfError{ + Code: int(kfapis.INTERNAL_ERROR), + Message: "Platform OSSM specified but not loaded.", + } + } else { + ossmInstaller := p.(*ossm.OssmInstaller) + return ossmInstaller.CleanupOwnedResources() + } + } + if err := kfapp.KfDef.SyncCache(); err != nil { return &kfapis.KfError{ Code: int(kfapis.INTERNAL_ERROR), @@ -499,6 +515,7 @@ func (kfapp *coordinator) Delete(resources kftypesv3.ResourceEnum) error { if err := k8s(); err != nil { return err } + return ossmCleanup() } return nil } diff --git a/pkg/kfapp/ossm/cert.go b/pkg/kfapp/ossm/cert.go index f4505c1f64f..1e039918139 100644 --- a/pkg/kfapp/ossm/cert.go +++ b/pkg/kfapp/ossm/cert.go @@ -13,7 +13,6 @@ import ( apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" - "k8s.io/client-go/rest" "math/big" "math/rand" "net" @@ -22,13 +21,22 @@ import ( var seededRand = rand.New(rand.NewSource(time.Now().UnixNano())) -func createSelfSignedCerts(config *rest.Config, addr string, objectMeta metav1.ObjectMeta) error { +func (o *OssmInstaller) createSelfSignedCerts(addr string, objectMeta metav1.ObjectMeta) error { cert, key, err := generateCertificate(addr) if err != nil { return errors.WithStack(err) } + objectMeta.SetOwnerReferences([]metav1.OwnerReference{ + { + APIVersion: o.tracker.APIVersion, + Kind: o.tracker.Kind, + Name: o.tracker.Name, + UID: o.tracker.UID, + }, + }) + secret := &corev1.Secret{ ObjectMeta: objectMeta, Data: map[string][]byte{ @@ -37,7 +45,7 @@ func createSelfSignedCerts(config *rest.Config, addr string, objectMeta metav1.O }, } - clientset, err := kubernetes.NewForConfig(config) + clientset, err := kubernetes.NewForConfig(o.config) if err != nil { return errors.WithStack(err) } diff --git a/pkg/kfapp/ossm/cleanup.go b/pkg/kfapp/ossm/cleanup.go new file mode 100644 index 00000000000..6dcffc29243 --- /dev/null +++ b/pkg/kfapp/ossm/cleanup.go @@ -0,0 +1,250 @@ +package ossm + +import ( + "context" + "fmt" + "github.com/hashicorp/go-multierror" + "github.com/opendatahub-io/opendatahub-operator/apis/ossm.plugins.kubeflow.org/v1alpha1" + k8serrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/dynamic" +) + +type cleanup func() error + +func (o *OssmInstaller) CleanupOwnedResources() error { + var cleanupErrors *multierror.Error + for _, cleanupFunc := range o.cleanupFuncs { + cleanupErrors = multierror.Append(cleanupErrors, cleanupFunc()) + } + + return cleanupErrors.ErrorOrNil() +} + +func (o *OssmInstaller) onCleanup(cleanupFunc ...cleanup) { + o.cleanupFuncs = append(o.cleanupFuncs, cleanupFunc...) +} + +// createResourceTracker instantiates OssmResourceTracker for given KfDef application in a namespce. +// This cluster-scoped resource is used as OwnerReference in all objects OssmInstaller is created across the cluster. +// Once created, there's a cleanup function added which will be invoked on deletion of the KfDef. +func (o *OssmInstaller) createResourceTracker() error { + tracker := &v1alpha1.OssmResourceTracker{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "ossm.plugins.kubeflow.org/v1alpha1", + Kind: "OssmResourceTracker", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: o.KfConfig.Name + "." + o.KfConfig.Namespace, + }, + } + + c, err := dynamic.NewForConfig(o.config) + if err != nil { + return err + } + + gvr := schema.GroupVersionResource{ + Group: "ossm.plugins.kubeflow.org", + Version: "v1alpha1", + Resource: "ossmresourcetrackers", + } + + foundTracker, err := c.Resource(gvr).Get(context.Background(), tracker.Name, metav1.GetOptions{}) + if k8serrors.IsNotFound(err) { + unstructuredTracker, err := runtime.DefaultUnstructuredConverter.ToUnstructured(tracker) + if err != nil { + return err + } + + u := unstructured.Unstructured{Object: unstructuredTracker} + + foundTracker, err = c.Resource(gvr).Create(context.Background(), &u, metav1.CreateOptions{}) + if err != nil { + return err + } + } else if err != nil { + return err + } + + o.tracker = &v1alpha1.OssmResourceTracker{} + if err := runtime.DefaultUnstructuredConverter.FromUnstructured(foundTracker.Object, o.tracker); err != nil { + return err + } + + o.onCleanup(func() error { + err := c.Resource(gvr).Delete(context.Background(), o.tracker.Name, metav1.DeleteOptions{}) + if k8serrors.IsNotFound(err) { + return nil + } + return err + }) + + return nil +} + +func (o *OssmInstaller) ingressVolumesRemoval() cleanup { + + return func() error { + spec, err := o.GetPluginSpec() + if err != nil { + return err + } + + tokenVolume := fmt.Sprintf("%s-oauth2-tokens", o.KfConfig.Namespace) + + dynamicClient, err := dynamic.NewForConfig(o.config) + if err != nil { + return err + } + + gvr := schema.GroupVersionResource{ + Group: "maistra.io", + Version: "v2", + Resource: "servicemeshcontrolplanes", + } + + smcp, err := dynamicClient.Resource(gvr).Namespace(spec.Mesh.Namespace).Get(context.Background(), spec.Mesh.Name, metav1.GetOptions{}) + if err != nil { + return err + } + volumes, found, err := unstructured.NestedSlice(smcp.Object, "spec", "gateways", "ingress", "volumes") + if err != nil { + return err + } + if !found { + log.Info("no volumes found", "smcp", spec.Mesh.Name, "istio-ns", spec.Mesh.Namespace) + return nil + } + + for i, v := range volumes { + volume, ok := v.(map[string]interface{}) + if !ok { + fmt.Println("Unexpected type for volume") + continue + } + + volumeMount, found, err := unstructured.NestedMap(volume, "volumeMount") + if err != nil { + return err + } + if !found { + fmt.Println("No volumeMount found in the volume") + continue + } + + if volumeMount["name"] == tokenVolume { + volumes = append(volumes[:i], volumes[i+1:]...) + err = unstructured.SetNestedSlice(smcp.Object, volumes, "spec", "gateways", "ingress", "volumes") + if err != nil { + return err + } + break + } + } + + _, err = dynamicClient.Resource(gvr).Namespace(spec.Mesh.Namespace).Update(context.Background(), smcp, metav1.UpdateOptions{}) + if err != nil { + return err + } + + return nil + } + +} + +func (o *OssmInstaller) oauthClientRemoval() func() error { + + return func() error { + c, err := dynamic.NewForConfig(o.config) + if err != nil { + return err + } + + oauthClientName := fmt.Sprintf("%s-oauth2-client", o.KfConfig.Namespace) + gvr := schema.GroupVersionResource{ + Group: "oauth.openshift.io", + Version: "v1", + Resource: "oauthclients", + } + + if _, err := c.Resource(gvr).Get(context.Background(), oauthClientName, metav1.GetOptions{}); err != nil { + if k8serrors.IsNotFound(err) { + return nil + } + + return err + } + + if err := c.Resource(gvr).Delete(context.Background(), oauthClientName, metav1.DeleteOptions{}); err != nil { + log.Error(err, "failed deleting OAuthClient", "name", oauthClientName) + return err + } + + return nil + } +} + +func (o *OssmInstaller) externalAuthzProviderRemoval() cleanup { + + return func() error { + spec, err := o.GetPluginSpec() + if err != nil { + return err + } + + ossmAuthzProvider := fmt.Sprintf("%s-odh-auth-provider", o.KfConfig.Namespace) + + dynamicClient, err := dynamic.NewForConfig(o.config) + if err != nil { + return err + } + + gvr := schema.GroupVersionResource{ + Group: "maistra.io", + Version: "v2", + Resource: "servicemeshcontrolplanes", + } + + smcp, err := dynamicClient.Resource(gvr).Namespace(spec.Mesh.Namespace).Get(context.Background(), spec.Mesh.Name, metav1.GetOptions{}) + if err != nil { + return err + } + + extensionProviders, found, err := unstructured.NestedSlice(smcp.Object, "spec", "techPreview", "meshConfig", "extensionProviders") + if err != nil { + return err + } + if !found { + log.Info("no extension providers found", "smcp", spec.Mesh.Name, "istio-ns", spec.Mesh.Namespace) + return nil + } + + for i, v := range extensionProviders { + extensionProvider, ok := v.(map[string]interface{}) + if !ok { + fmt.Println("Unexpected type for extensionProvider") + continue + } + + if extensionProvider["name"] == ossmAuthzProvider { + extensionProviders = append(extensionProviders[:i], extensionProviders[i+1:]...) + err = unstructured.SetNestedSlice(smcp.Object, extensionProviders, "spec", "techPreview", "meshConfig", "extensionProviders") + if err != nil { + return err + } + break + } + } + + _, err = dynamicClient.Resource(gvr).Namespace(spec.Mesh.Namespace).Update(context.Background(), smcp, metav1.UpdateOptions{}) + if err != nil { + return err + } + + return nil + } +} diff --git a/pkg/kfapp/ossm/envoy_secrets.go b/pkg/kfapp/ossm/envoy_secrets.go index afd7d12dad4..dc1cd604531 100644 --- a/pkg/kfapp/ossm/envoy_secrets.go +++ b/pkg/kfapp/ossm/envoy_secrets.go @@ -9,7 +9,6 @@ import ( apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" - "k8s.io/client-go/rest" "text/template" ) @@ -31,7 +30,7 @@ resources: inline_bytes: "{{ .Secret }}" ` -func createEnvoySecret(config *rest.Config, oAuth oAuth, objectMeta metav1.ObjectMeta) error { +func (o *OssmInstaller) createEnvoySecret(oAuth oAuth, objectMeta metav1.ObjectMeta) error { clientSecret, err := processInlineTemplate(tokenSecret, struct{ Secret string }{Secret: oAuth.ClientSecret}) if err != nil { @@ -43,6 +42,15 @@ func createEnvoySecret(config *rest.Config, oAuth oAuth, objectMeta metav1.Objec return errors.WithStack(err) } + objectMeta.SetOwnerReferences([]metav1.OwnerReference{ + { + APIVersion: o.tracker.APIVersion, + Kind: o.tracker.Kind, + Name: o.tracker.Name, + UID: o.tracker.UID, + }, + }) + secret := &corev1.Secret{ ObjectMeta: objectMeta, Data: map[string][]byte{ @@ -51,7 +59,7 @@ func createEnvoySecret(config *rest.Config, oAuth oAuth, objectMeta metav1.Objec }, } - clientset, err := kubernetes.NewForConfig(config) + clientset, err := kubernetes.NewForConfig(o.config) if err != nil { return errors.WithStack(err) } diff --git a/pkg/kfapp/ossm/k8s_resources.go b/pkg/kfapp/ossm/k8s_resources.go new file mode 100644 index 00000000000..dc25cb6e056 --- /dev/null +++ b/pkg/kfapp/ossm/k8s_resources.go @@ -0,0 +1,170 @@ +/* +Copyright (c) 2016-2017 Bitnami +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package ossm + +import ( + "context" + "fmt" + "github.com/ghodss/yaml" + configtypes "github.com/opendatahub-io/opendatahub-operator/apis/config" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" + k8serrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" + k8stypes "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/dynamic" + "os" + "regexp" + "sigs.k8s.io/controller-runtime/pkg/client" + "strings" +) + +const ( + YamlSeparator = "(?m)^---[ \t]*$" +) + +func (o *OssmInstaller) CreateResourceFromFile(filename string, elems ...configtypes.NameValue) error { + elemsMap := make(map[string]configtypes.NameValue) + for _, nv := range elems { + elemsMap[nv.Name] = nv + } + c, err := client.New(o.config, client.Options{}) + if err != nil { + return errors.WithStack(err) + } + + data, err := os.ReadFile(filename) + if err != nil { + return errors.WithStack(err) + } + splitter := regexp.MustCompile(YamlSeparator) + objectStrings := splitter.Split(string(data), -1) + for _, str := range objectStrings { + if strings.TrimSpace(str) == "" { + continue + } + u := &unstructured.Unstructured{} + if err := yaml.Unmarshal([]byte(str), u); err != nil { + return errors.WithStack(err) + } + + name := u.GetName() + namespace := u.GetNamespace() + if namespace == "" { + if val, exists := elemsMap["namespace"]; exists { + u.SetNamespace(val.Value) + } else { + u.SetNamespace("default") + } + } + + u.SetOwnerReferences([]metav1.OwnerReference{ + { + APIVersion: o.tracker.APIVersion, + Kind: o.tracker.Kind, + Name: o.tracker.Name, + UID: o.tracker.UID, + }, + }) + + logrus.Infof("Creating %s", name) + + err := c.Get(context.TODO(), k8stypes.NamespacedName{Name: name, Namespace: namespace}, u.DeepCopy()) + if err == nil { + log.Info("Object already exists...") + continue + } + if !k8serrors.IsNotFound(err) { + return errors.WithStack(err) + } + + err = c.Create(context.TODO(), u) + if err != nil { + return errors.WithStack(err) + } + } + return nil +} + +func (o *OssmInstaller) PatchResourceFromFile(filename string, elems ...configtypes.NameValue) error { + elemsMap := make(map[string]configtypes.NameValue) + for _, nv := range elems { + elemsMap[nv.Name] = nv + } + + dynamicClient, err := dynamic.NewForConfig(o.config) + if err != nil { + return errors.WithStack(err) + } + + data, err := os.ReadFile(filename) + if err != nil { + return errors.WithStack(err) + } + splitter := regexp.MustCompile(YamlSeparator) + objectStrings := splitter.Split(string(data), -1) + for _, str := range objectStrings { + if strings.TrimSpace(str) == "" { + continue + } + p := &unstructured.Unstructured{} + if err := yaml.Unmarshal([]byte(str), p); err != nil { + logrus.Error("error unmarshalling yaml") + return errors.WithStack(err) + } + + // Adding `namespace:` to Namespace resource doesn't make sense + if p.GetKind() != "Namespace" { + namespace := p.GetNamespace() + if namespace == "" { + if val, exists := elemsMap["namespace"]; exists { + p.SetNamespace(val.Value) + } else { + p.SetNamespace("default") + } + } + } + + gvr := schema.GroupVersionResource{ + Group: strings.ToLower(p.GroupVersionKind().Group), + Version: p.GroupVersionKind().Version, + Resource: strings.ToLower(p.GroupVersionKind().Kind) + "s", + } + + // Convert the patch from YAML to JSON + patchAsJson, err := yaml.YAMLToJSON(data) + if err != nil { + logrus.Error("error converting yaml to json") + return errors.WithStack(err) + } + + _, err = dynamicClient.Resource(gvr). + Namespace(p.GetNamespace()). + Patch(context.Background(), p.GetName(), k8stypes.MergePatchType, patchAsJson, metav1.PatchOptions{}) + if err != nil { + logrus.Error("error patching resource\n", + fmt.Sprintf("%+v\n", gvr), + fmt.Sprintf("%+v\n", p), + fmt.Sprintf("%+v\n", patchAsJson)) + return errors.WithStack(err) + } + + if err != nil { + return errors.WithStack(err) + } + } + return nil +} diff --git a/pkg/kfapp/ossm/ossm_installer.go b/pkg/kfapp/ossm/ossm_installer.go index ce4c9e64c26..db627c28436 100644 --- a/pkg/kfapp/ossm/ossm_installer.go +++ b/pkg/kfapp/ossm/ossm_installer.go @@ -6,6 +6,7 @@ import ( multierror "github.com/hashicorp/go-multierror" kfapisv3 "github.com/opendatahub-io/opendatahub-operator/apis" kftypesv3 "github.com/opendatahub-io/opendatahub-operator/apis/apps" + "github.com/opendatahub-io/opendatahub-operator/apis/ossm.plugins.kubeflow.org/v1alpha1" "github.com/opendatahub-io/opendatahub-operator/pkg/kfconfig" "github.com/opendatahub-io/opendatahub-operator/pkg/kfconfig/ossmplugin" "github.com/pkg/errors" @@ -26,15 +27,17 @@ const ( var log = ctrlLog.Log.WithName(PluginName) -type Ossm struct { +type OssmInstaller struct { *kfconfig.KfConfig - pluginSpec *ossmplugin.OssmPluginSpec - config *rest.Config - manifests []manifest + pluginSpec *ossmplugin.OssmPluginSpec + config *rest.Config + manifests []manifest + tracker *v1alpha1.OssmResourceTracker + cleanupFuncs []cleanup } -func NewOssm(kfConfig *kfconfig.KfConfig, restConfig *rest.Config) *Ossm { - return &Ossm{ +func NewOssmInstaller(kfConfig *kfconfig.KfConfig, restConfig *rest.Config) *OssmInstaller { + return &OssmInstaller{ KfConfig: kfConfig, config: restConfig, } @@ -43,28 +46,28 @@ func NewOssm(kfConfig *kfconfig.KfConfig, restConfig *rest.Config) *Ossm { // GetPlatform returns the ossm kfapp. It's called by coordinator.GetPlatform func GetPlatform(kfConfig *kfconfig.KfConfig) (kftypesv3.Platform, error) { - return NewOssm(kfConfig, kftypesv3.GetConfig()), nil + return NewOssmInstaller(kfConfig, kftypesv3.GetConfig()), nil } // GetPluginSpec gets the plugin spec. -func (ossm *Ossm) GetPluginSpec() (*ossmplugin.OssmPluginSpec, error) { - if ossm.pluginSpec != nil { - return ossm.pluginSpec, nil +func (o *OssmInstaller) GetPluginSpec() (*ossmplugin.OssmPluginSpec, error) { + if o.pluginSpec != nil { + return o.pluginSpec, nil } - ossm.pluginSpec = &ossmplugin.OssmPluginSpec{} - err := ossm.KfConfig.GetPluginSpec(PluginName, ossm.pluginSpec) + o.pluginSpec = &ossmplugin.OssmPluginSpec{} + err := o.KfConfig.GetPluginSpec(PluginName, o.pluginSpec) - return ossm.pluginSpec, err + return o.pluginSpec, err } -func (ossm *Ossm) Init(_ kftypesv3.ResourceEnum) error { - if ossm.KfConfig.Spec.SkipInitProject { - log.Info("Skipping init phase") +func (o *OssmInstaller) Init(_ kftypesv3.ResourceEnum) error { + if o.KfConfig.Spec.SkipInitProject { + log.Info("Skipping init phase", "plugin", PluginName) } - log.Info("Initializing " + PluginName) - pluginSpec, err := ossm.GetPluginSpec() + log.Info("Initializing", "plugin", PluginName) + pluginSpec, err := o.GetPluginSpec() if err != nil { return internalError(errors.WithStack(err)) } @@ -77,7 +80,11 @@ func (ossm *Ossm) Init(_ kftypesv3.ResourceEnum) error { // TODO ensure operators are installed - if err := ossm.createConfigMap("service-mesh-refs", + if err := o.createResourceTracker(); err != nil { + return internalError(err) + } + + if err := o.createConfigMap("service-mesh-refs", map[string]string{ "CONTROL_PLANE_NAME": pluginSpec.Mesh.Name, "MESH_NAMESPACE": pluginSpec.Mesh.Namespace, @@ -85,30 +92,36 @@ func (ossm *Ossm) Init(_ kftypesv3.ResourceEnum) error { return internalError(err) } - if err := ossm.createConfigMap("auth-refs", + if err := o.createConfigMap("auth-refs", map[string]string{ "AUTHORINO_LABEL": pluginSpec.Auth.Authorino.Label, }); err != nil { return internalError(err) } - if err := ossm.MigrateDSProjects(); err != nil { + if err := o.MigrateDSProjects(); err != nil { log.Error(err, "failed migrating Data Science Projects") } - if err := ossm.processManifests(); err != nil { + if err := o.processManifests(); err != nil { return internalError(err) } return nil } -func (ossm *Ossm) Generate(resources kftypesv3.ResourceEnum) error { +func (o *OssmInstaller) Generate(resources kftypesv3.ResourceEnum) error { // TODO sort by Kind as .Apply does - if err := ossm.applyManifests(); err != nil { + if err := o.applyManifests(); err != nil { return internalError(errors.WithStack(err)) } + o.onCleanup( + o.oauthClientRemoval(), + o.ingressVolumesRemoval(), + o.externalAuthzProviderRemoval(), + ) + return nil } @@ -128,17 +141,25 @@ func ExtractHostName(s string) string { return withoutProtocol[:index] } -func (ossm *Ossm) createConfigMap(cfgMapName string, data map[string]string) error { +func (o *OssmInstaller) createConfigMap(cfgMapName string, data map[string]string) error { configMap := &corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ Name: cfgMapName, - Namespace: ossm.KfConfig.Namespace, + Namespace: o.KfConfig.Namespace, + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: o.tracker.APIVersion, + Kind: o.tracker.Kind, + Name: o.tracker.Name, + UID: o.tracker.UID, + }, + }, }, Data: data, } - client, err := clientset.NewForConfig(ossm.config) + client, err := clientset.NewForConfig(o.config) if err != nil { return err } @@ -163,9 +184,9 @@ func (ossm *Ossm) createConfigMap(cfgMapName string, data map[string]string) err return nil } -func (ossm *Ossm) MigrateDSProjects() error { +func (o *OssmInstaller) MigrateDSProjects() error { - client, err := clientset.NewForConfig(ossm.config) + client, err := clientset.NewForConfig(o.config) if err != nil { return err } @@ -195,8 +216,6 @@ func (ossm *Ossm) MigrateDSProjects() error { return result.ErrorOrNil() } -// TODO handle delete - func internalError(err error) error { return &kfapisv3.KfError{ Code: int(kfapisv3.INTERNAL_ERROR), diff --git a/pkg/kfapp/ossm/ossm_installer_noop.go b/pkg/kfapp/ossm/ossm_installer_noop.go index d05c98a788d..b4a515e601c 100644 --- a/pkg/kfapp/ossm/ossm_installer_noop.go +++ b/pkg/kfapp/ossm/ossm_installer_noop.go @@ -5,19 +5,19 @@ import kftypesv3 "github.com/opendatahub-io/opendatahub-operator/apis/apps" // Below are the functions which are not used/executed at this point. // They're here to satisfy the Plugin interface. -func (ossm *Ossm) Apply(resources kftypesv3.ResourceEnum) error { +func (o *OssmInstaller) Apply(resources kftypesv3.ResourceEnum) error { // Plugins invoked within k8s (as a platform) won't be participating in Apply // This is responsibility of PackageManagers - in this case kustomize return nil } -func (ossm *Ossm) Delete(resources kftypesv3.ResourceEnum) error { +func (o *OssmInstaller) Delete(resources kftypesv3.ResourceEnum) error { // Plugins invoked within k8s (as a platform) won't be participating in Delete // This is responsibility of PackageManagers - in this case kustomize return nil } -func (ossm *Ossm) Dump(resources kftypesv3.ResourceEnum) error { +func (o *OssmInstaller) Dump(resources kftypesv3.ResourceEnum) error { // Plugins invoked within k8s (as a platform) won't be participating in Dump // This is responsibility of PackageManagers - in this case kustomize return nil diff --git a/pkg/kfapp/ossm/ossm_manifests.go b/pkg/kfapp/ossm/ossm_manifests.go index f8c319d0858..7ebae536809 100644 --- a/pkg/kfapp/ossm/ossm_manifests.go +++ b/pkg/kfapp/ossm/ossm_manifests.go @@ -6,7 +6,6 @@ import ( configtypes "github.com/opendatahub-io/opendatahub-operator/apis/config" "github.com/opendatahub-io/opendatahub-operator/pkg/kfconfig/ossmplugin" "github.com/opendatahub-io/opendatahub-operator/pkg/secret" - "github.com/opendatahub-io/opendatahub-operator/pkg/utils" "github.com/pkg/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/rest" @@ -18,24 +17,24 @@ import ( type applier func(config *rest.Config, filename string, elems ...configtypes.NameValue) error -func (ossm *Ossm) applyManifests() error { +func (o *OssmInstaller) applyManifests() error { var apply applier - for _, m := range ossm.manifests { + for _, m := range o.manifests { if m.patch { apply = func(config *rest.Config, filename string, elems ...configtypes.NameValue) error { log.Info("patching using manifest", "name", m.name, "path", m.targetPath()) - return utils.PatchResourceFromFile(config, filename, elems...) + return o.PatchResourceFromFile(filename, elems...) } } else { apply = func(config *rest.Config, filename string, elems ...configtypes.NameValue) error { log.Info("applying manifest", "name", m.name, "path", m.targetPath()) - return utils.CreateResourceFromFile(config, filename, elems...) + return o.CreateResourceFromFile(filename, elems...) } } err := apply( - ossm.config, + o.config, m.targetPath(), ) if err != nil { @@ -47,14 +46,14 @@ func (ossm *Ossm) applyManifests() error { return nil } -func (ossm *Ossm) processManifests() error { - if err := ossm.SyncCache(); err != nil { +func (o *OssmInstaller) processManifests() error { + if err := o.SyncCache(); err != nil { return internalError(err) } // TODO warn when file is not present instead of throwing an error // IMPORTANT: Order of locations from where we load manifests/templates to process is significant - err := ossm.loadManifestsFrom( + err := o.loadManifestsFrom( path.Join("control-plane", "base"), path.Join("control-plane", "filters"), path.Join("control-plane", "oauth"), @@ -71,12 +70,12 @@ func (ossm *Ossm) processManifests() error { return internalError(errors.WithStack(err)) } - data, err := ossm.prepareTemplateData() + data, err := o.prepareTemplateData() if err != nil { return internalError(errors.WithStack(err)) } - for i, m := range ossm.manifests { + for i, m := range o.manifests { if err := m.processTemplate(data); err != nil { return internalError(errors.WithStack(err)) } @@ -87,8 +86,8 @@ func (ossm *Ossm) processManifests() error { return nil } -func (ossm *Ossm) loadManifestsFrom(paths ...string) error { - manifestRepo, ok := ossm.GetRepoCache(kftypesv3.ManifestsRepoName) +func (o *OssmInstaller) loadManifestsFrom(paths ...string) error { + manifestRepo, ok := o.GetRepoCache(kftypesv3.ManifestsRepoName) if !ok { return internalError(errors.New("manifests repo is not defined.")) } @@ -102,7 +101,7 @@ func (ossm *Ossm) loadManifestsFrom(paths ...string) error { } } - ossm.manifests = manifests + o.manifests = manifests return nil } @@ -134,23 +133,23 @@ func loadManifestsFrom(manifests []manifest, dir string) ([]manifest, error) { // TODO(smell) this is now holding two responsibilities: // - creates data structure to be fed to templates // - creates secrets using k8s API calls -func (ossm *Ossm) prepareTemplateData() (interface{}, error) { +func (o *OssmInstaller) prepareTemplateData() (interface{}, error) { data := struct { *ossmplugin.OssmPluginSpec OAuth oAuth Domain, AppNamespace string }{ - AppNamespace: ossm.KfConfig.Namespace, + AppNamespace: o.KfConfig.Namespace, } - spec, err := ossm.GetPluginSpec() + spec, err := o.GetPluginSpec() if err != nil { return nil, internalError(errors.WithStack(err)) } data.OssmPluginSpec = spec - if domain, err := GetDomain(ossm.config); err == nil { + if domain, err := GetDomain(o.config); err == nil { data.Domain = domain } else { return nil, internalError(errors.WithStack(err)) @@ -178,7 +177,7 @@ func (ossm *Ossm) prepareTemplateData() (interface{}, error) { } if spec.Mesh.Certificate.Generate { - if err := createSelfSignedCerts(ossm.config, data.Domain, metav1.ObjectMeta{ + if err := o.createSelfSignedCerts(data.Domain, metav1.ObjectMeta{ Name: spec.Mesh.Certificate.Name, Namespace: spec.Mesh.Namespace, }); err != nil { @@ -186,7 +185,7 @@ func (ossm *Ossm) prepareTemplateData() (interface{}, error) { } } - if err := createEnvoySecret(ossm.config, data.OAuth, metav1.ObjectMeta{ + if err := o.createEnvoySecret(data.OAuth, metav1.ObjectMeta{ Name: data.AppNamespace + "-oauth2-tokens", Namespace: data.Mesh.Namespace, }); err != nil { diff --git a/pkg/kfapp/ossm/ossm_plugin_test.go b/pkg/kfapp/ossm/ossm_plugin_test.go index 65a41f48140..4be7a616451 100644 --- a/pkg/kfapp/ossm/ossm_plugin_test.go +++ b/pkg/kfapp/ossm/ossm_plugin_test.go @@ -22,24 +22,24 @@ var _ = When("Migrating Data Science Projects", func() { var ( objectCleaner *testenv.Cleaner - ossmPlugin *ossm.Ossm + ossmInstaller *ossm.OssmInstaller ) BeforeEach(func() { - ossmPlugin = ossm.NewOssm(&kfconfig.KfConfig{}, envTest.Config) + ossmInstaller = ossm.NewOssmInstaller(&kfconfig.KfConfig{}, envTest.Config) objectCleaner = testenv.CreateCleaner(cli, envTest.Config, timeout, interval) }) It("should find one namespace to migrate", func() { // given - dataSciencNs := createDSProject("dsp-01") + dataScienceNs := createDSProject("dsp-01") regularNs := createNs("non-dsp") - Expect(cli.Create(context.Background(), dataSciencNs)).To(Succeed()) + Expect(cli.Create(context.Background(), dataScienceNs)).To(Succeed()) Expect(cli.Create(context.Background(), regularNs)).To(Succeed()) - defer objectCleaner.DeleteAll(dataSciencNs, regularNs) + defer objectCleaner.DeleteAll(dataScienceNs, regularNs) // when - Expect(ossmPlugin.MigrateDSProjects()).ToNot(HaveOccurred()) + Expect(ossmInstaller.MigrateDSProjects()).ToNot(HaveOccurred()) // then Eventually(func() []v1.Namespace { diff --git a/pkg/utils/k8utils.go b/pkg/utils/k8utils.go index a2bf99ac3fa..a5ff7dc0b82 100644 --- a/pkg/utils/k8utils.go +++ b/pkg/utils/k8utils.go @@ -34,13 +34,11 @@ import ( "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - "k8s.io/apimachinery/pkg/runtime/schema" k8stypes "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/sets" k8syaml "k8s.io/apimachinery/pkg/util/yaml" "k8s.io/cli-runtime/pkg/genericclioptions" "k8s.io/cli-runtime/pkg/printers" - "k8s.io/client-go/dynamic" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" kubectlapply "k8s.io/kubectl/pkg/cmd/apply" @@ -143,76 +141,6 @@ func CreateResourceFromFile(config *rest.Config, filename string, elems ...confi return nil } -func PatchResourceFromFile(config *rest.Config, filename string, elems ...configtypes.NameValue) error { - elemsMap := make(map[string]configtypes.NameValue) - for _, nv := range elems { - elemsMap[nv.Name] = nv - } - - dynamicClient, err := dynamic.NewForConfig(config) - if err != nil { - return errors.WithStack(err) - } - - data, err := os.ReadFile(filename) - if err != nil { - return errors.WithStack(err) - } - splitter := regexp.MustCompile(YamlSeparator) - objectStrings := splitter.Split(string(data), -1) - for _, str := range objectStrings { - if strings.TrimSpace(str) == "" { - continue - } - p := &unstructured.Unstructured{} - if err := yaml.Unmarshal([]byte(str), p); err != nil { - log.Error("error unmarshalling yaml") - return errors.WithStack(err) - } - - // Adding `namespace:` to Namespace resource doesn't make sense - if p.GetKind() != "Namespace" { - namespace := p.GetNamespace() - if namespace == "" { - if val, exists := elemsMap["namespace"]; exists { - p.SetNamespace(val.Value) - } else { - p.SetNamespace("default") - } - } - } - - gvr := schema.GroupVersionResource{ - Group: strings.ToLower(p.GroupVersionKind().Group), - Version: p.GroupVersionKind().Version, - Resource: strings.ToLower(p.GroupVersionKind().Kind) + "s", - } - - // Convert the patch from YAML to JSON - patchAsJson, err := yaml.YAMLToJSON(data) - if err != nil { - log.Error("error converting yaml to json") - return errors.WithStack(err) - } - - _, err = dynamicClient.Resource(gvr). - Namespace(p.GetNamespace()). - Patch(context.Background(), p.GetName(), k8stypes.MergePatchType, patchAsJson, metav1.PatchOptions{}) - if err != nil { - log.Error("error patching resource\n", - fmt.Sprintf("%+v\n", gvr), - fmt.Sprintf("%+v\n", p), - fmt.Sprintf("%+v\n", patchAsJson)) - return errors.WithStack(err) - } - - if err != nil { - return errors.WithStack(err) - } - } - return nil -} - // Checks if the path configFile is remote (e.g. http://github...) func IsRemoteFile(configFile string) (bool, error) { if configFile == "" {