feat: handles cleanup by using owner ref and cleanup hook

apis/ossm.plugins.kubeflow.org/v1alpha1/ossm_types.go

@@ -17,20 +17,22 @@ type OssmPlugin struct {
 	Status OssmPluginStatus `json:"status,omitempty"`
 }
 
-// OssmPluginSpec defines the extra data provided by the Openshift Service Mesh Plugin in KfDef spec.
+// OssmPluginSpec defines configuration needed for Openshift Service Mesh
+// for integration with Opendatahub.
 type OssmPluginSpec struct {
 	Mesh MeshSpec `json:"mesh,omitempty"`
 	Auth AuthSpec `json:"auth,omitempty"`
 }
 
+// MeshSpec holds information on how Service Mesh should be configured.
 type MeshSpec struct {
 	Name        string   `json:"name,omitempty"`
 	Namespace   string   `json:"namespace,omitempty"`
 	Certificate CertSpec `json:"certificate,omitempty"`
 }
 
 type CertSpec struct {
-	Name     string `json:"name,omitempty" default:"opendatahub-self-signed-cert"`
+	Name     string `json:"name,omitempty"`
 	Generate bool   `json:"generate,omitempty"`
 }
 
@@ -55,13 +57,49 @@ type OssmPluginStatus struct {
 
 //+kubebuilder:object:root=true
 
-// OssmPluginList contains a list of GcpPlugin
+// OssmPluginList contains a list of OssmPlugins
 type OssmPluginList struct {
 	metav1.TypeMeta `json:",inline"`
 	metav1.ListMeta `json:"metadata,omitempty"`
 	Items           []OssmPlugin `json:"items"`
 }
 
+// OssmResourceTracker is a cluster-scoped resource for tracking objects
+// created by Ossm plugin. It's primarily used as owner reference
+// for resources created across namespaces so that they can be
+// garbage collected by Kubernetes when they're not needed anymore.
+// +kubebuilder:object:root=true
+// +kubebuilder:resource:scope=Cluster
+type OssmResourceTracker struct {
+	metav1.TypeMeta   `json:",inline"`
+	metav1.ObjectMeta `json:"metadata,omitempty"`
+
+	Spec   OssmResourceTrackerSpec   `json:"spec,omitempty"`
+	Status OssmResourceTrackerStatus `json:"status,omitempty"`
+}
+
+// OssmResourceTrackerSpec defines the desired state of OssmResourceTracker
+type OssmResourceTrackerSpec struct {
+}
+
+// OssmResourceTrackerStatus defines the observed state of OssmResourceTracker
+type OssmResourceTrackerStatus struct {
+}
+
+// +kubebuilder:object:root=true
+
+// OssmResourceTrackerList contains a list of OssmResourceTracker
+type OssmResourceTrackerList struct {
+	metav1.TypeMeta `json:",inline"`
+	metav1.ListMeta `json:"metadata,omitempty"`
+	Items           []OssmResourceTracker `json:"items"`
+}
+
 func init() {
-	SchemeBuilder.Register(&OssmPlugin{}, &OssmPluginList{})
+	SchemeBuilder.Register(
+		&OssmPlugin{},
+		&OssmPluginList{},
+		&OssmResourceTracker{},
+		&OssmResourceTrackerList{},
+	)
 }

bundle/manifests/ossm.plugins.kubeflow.org_ossmresourcetrackers.yaml

@@ -0,0 +1,51 @@
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: v0.9.2
+  creationTimestamp: null
+  name: ossmresourcetrackers.ossm.plugins.kubeflow.org
+spec:
+  group: ossm.plugins.kubeflow.org
+  names:
+    kind: OssmResourceTracker
+    listKind: OssmResourceTrackerList
+    plural: ossmresourcetrackers
+    singular: ossmresourcetracker
+  scope: Cluster
+  versions:
+  - name: v1alpha1
+    schema:
+      openAPIV3Schema:
+        description: OssmResourceTracker is a cluster-scoped resource for tracking
+          objects created by Ossm plugin. It's primarily used as owner reference for
+          resources created across namespaces so that they can be garbage collected
+          by Kubernetes when they're not needed anymore.
+        properties:
+          apiVersion:
+            description: 'APIVersion defines the versioned schema of this representation
+              of an object. Servers should convert recognized schemas to the latest
+              internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+            type: string
+          kind:
+            description: 'Kind is a string value representing the REST resource this
+              object represents. Servers may infer this from the endpoint the client
+              submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+            type: string
+          metadata:
+            type: object
+          spec:
+            description: OssmResourceTrackerSpec defines the desired state of OssmResourceTracker
+            type: object
+          status:
+            description: OssmResourceTrackerStatus defines the observed state of OssmResourceTracker
+            type: object
+        type: object
+    served: true
+    storage: true
+status:
+  acceptedNames:
+    kind: ""
+    plural: ""
+  conditions: null
+  storedVersions: null

config/crd/bases/ossm.plugins.kubeflow.org_ossmplugins.yaml

@@ -32,8 +32,8 @@ spec:
           metadata:
             type: object
           spec:
-            description: OssmPluginSpec defines the extra data provided by the Openshift
-              Service Mesh Plugin in KfDef spec.
+            description: OssmPluginSpec defines configuration needed for Openshift
+              Service Mesh for integration with Opendatahub.
             properties:
               auth:
                 properties:
@@ -52,6 +52,8 @@ spec:
                     type: string
                 type: object
               mesh:
+                description: MeshSpec holds information on how Service Mesh should
+                  be configured.
                 properties:
                   certificate:
                     properties:

config/crd/bases/ossm.plugins.kubeflow.org_ossmresourcetrackers.yaml

@@ -0,0 +1,46 @@
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: v0.9.2
+  creationTimestamp: null
+  name: ossmresourcetrackers.ossm.plugins.kubeflow.org
+spec:
+  group: ossm.plugins.kubeflow.org
+  names:
+    kind: OssmResourceTracker
+    listKind: OssmResourceTrackerList
+    plural: ossmresourcetrackers
+    singular: ossmresourcetracker
+  scope: Cluster
+  versions:
+  - name: v1alpha1
+    schema:
+      openAPIV3Schema:
+        description: OssmResourceTracker is a cluster-scoped resource for tracking
+          objects created by Ossm plugin. It's primarily used as owner reference for
+          resources created across namespaces so that they can be garbage collected
+          by Kubernetes when they're not needed anymore.
+        properties:
+          apiVersion:
+            description: 'APIVersion defines the versioned schema of this representation
+              of an object. Servers should convert recognized schemas to the latest
+              internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+            type: string
+          kind:
+            description: 'Kind is a string value representing the REST resource this
+              object represents. Servers may infer this from the endpoint the client
+              submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+            type: string
+          metadata:
+            type: object
+          spec:
+            description: OssmResourceTrackerSpec defines the desired state of OssmResourceTracker
+            type: object
+          status:
+            description: OssmResourceTrackerStatus defines the observed state of OssmResourceTracker
+            type: object
+        type: object
+    served: true
+    storage: true

config/crd/kustomization.yaml

@@ -4,6 +4,7 @@
 resources:
   - bases/kfdef.apps.kubeflow.org_kfdefs.yaml
   - bases/ossm.plugins.kubeflow.org_ossmplugins.yaml
+  - bases/ossm.plugins.kubeflow.org_ossmresourcetrackers.yaml
   - dashboard-crds/odhapplications.dashboard.opendatahub.io.crd.yaml
   - dashboard-crds/odhdashboardconfigs.opendatahub.io.crd.yaml
   - dashboard-crds/odhdocuments.dashboard.opendatahub.io.crd.yaml

main.go

@@ -156,7 +156,7 @@ func main() {
 		os.Exit(1)
 	}
 
-	setupLog.Info("starting manager 123 !!!!")
+	setupLog.Info("starting manager")
 	if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil {
 		setupLog.Error(err, "problem running manager")
 		os.Exit(1)

pkg/kfapp/coordinator/coordinator.go

@@ -471,6 +471,22 @@ func (kfapp *coordinator) Delete(resources kftypesv3.ResourceEnum) error {
 		return nil
 	}
 
+	ossmCleanup := func() error {
+		if kfapp.KfDef.Spec.Platform != kftypesv3.OSSM {
+			return nil
+		}
+
+		if p, ok := kfapp.Platforms[kfapp.KfDef.Spec.Platform]; !ok {
+			return &kfapis.KfError{
+				Code:    int(kfapis.INTERNAL_ERROR),
+				Message: "Platform OSSM specified but not loaded.",
+			}
+		} else {
+			ossmInstaller := p.(*ossm.OssmInstaller)
+			return ossmInstaller.CleanupOwnedResources()
+		}
+	}
+
 	if err := kfapp.KfDef.SyncCache(); err != nil {
 		return &kfapis.KfError{
 			Code:    int(kfapis.INTERNAL_ERROR),
@@ -499,6 +515,7 @@ func (kfapp *coordinator) Delete(resources kftypesv3.ResourceEnum) error {
 		if err := k8s(); err != nil {
 			return err
 		}
+		return ossmCleanup()
 	}
 	return nil
 }

pkg/kfapp/ossm/cert.go

@@ -13,7 +13,6 @@ import (
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/kubernetes"
-	"k8s.io/client-go/rest"
 	"math/big"
 	"math/rand"
 	"net"
@@ -22,13 +21,22 @@ import (
 
 var seededRand = rand.New(rand.NewSource(time.Now().UnixNano()))
 
-func createSelfSignedCerts(config *rest.Config, addr string, objectMeta metav1.ObjectMeta) error {
+func (o *OssmInstaller) createSelfSignedCerts(addr string, objectMeta metav1.ObjectMeta) error {
 
 	cert, key, err := generateCertificate(addr)
 	if err != nil {
 		return errors.WithStack(err)
 	}
 
+	objectMeta.SetOwnerReferences([]metav1.OwnerReference{
+		{
+			APIVersion: o.tracker.APIVersion,
+			Kind:       o.tracker.Kind,
+			Name:       o.tracker.Name,
+			UID:        o.tracker.UID,
+		},
+	})
+
 	secret := &corev1.Secret{
 		ObjectMeta: objectMeta,
 		Data: map[string][]byte{
@@ -37,7 +45,7 @@ func createSelfSignedCerts(config *rest.Config, addr string, objectMeta metav1.O
 		},
 	}
 
-	clientset, err := kubernetes.NewForConfig(config)
+	clientset, err := kubernetes.NewForConfig(o.config)
 	if err != nil {
 		return errors.WithStack(err)
 	}