Skip to content

Commit

Permalink
Remove FeatureTracker code from Kserve reconciler
Browse files Browse the repository at this point in the history 
Jira: https://issues.redhat.com/browse/RHOAIENG-18045
  • Loading branch information
@grdryn
grdryn committed Jan 20, 2025
1 parent 72443c5 commit 480d57d
Show file tree
Hide file tree
Showing 4 changed files with 155 additions and 88 deletions.
63 changes: 10 additions & 53 deletions controllers/components/kserve/kserve_controller.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,17 +28,14 @@ import (
rbacv1 "k8s.io/api/rbac/v1"
extv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/handler"

componentApi "github.com/opendatahub-io/opendatahub-operator/v2/apis/components/v1alpha1"
featuresv1 "github.com/opendatahub-io/opendatahub-operator/v2/apis/features/v1"
"github.com/opendatahub-io/opendatahub-operator/v2/pkg/cluster/gvk"
"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/deploy"
"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/gc"
"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/render/kustomize"
"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/updatestatus"
"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/handlers"
"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/predicates"
"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/predicates/clusterrole"
"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/predicates/component"
"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/predicates/hash"
Expand All @@ -49,8 +46,6 @@ import (

// NewComponentReconciler creates a ComponentReconciler for the Dashboard API.
func (s *componentHandler) NewComponentReconciler(ctx context.Context, mgr ctrl.Manager) error {
ownedViaFTMapFunc := ownedViaFT(mgr.GetClient())

_, err := reconciler.ReconcilerFor(mgr, &componentApi.Kserve{}).
// operands - owned
Owns(&corev1.Secret{}).
Expand All @@ -65,22 +60,20 @@ func (s *componentHandler) NewComponentReconciler(ctx context.Context, mgr ctrl.
// changes. The compareHashPredicate ensures that we don't needlessly enqueue
// requests if there are no changes that we don't care about.
Owns(&templatev1.Template{}, reconciler.WithPredicates(hash.Updated())).
// The FeatureTrackers are created slightly differently, and have
// ownerRefs set by controllerutil.SetOwnerReference() rather than
// controllerutil.SetControllerReference(), which means that the default
// eventHandler for Owns won't work, so a slightly modified variant is
// added here
Owns(&featuresv1.FeatureTracker{}, reconciler.WithEventHandler(
handler.EnqueueRequestForOwner(
mgr.GetScheme(),
mgr.GetRESTMapper(),
&componentApi.Kserve{},
))).
Owns(&networkingv1.NetworkPolicy{}).
Owns(&monitoringv1.ServiceMonitor{}).
Owns(&admissionregistrationv1.MutatingWebhookConfiguration{}).
Owns(&admissionregistrationv1.ValidatingWebhookConfiguration{}).
Owns(&appsv1.Deployment{}, reconciler.WithPredicates(resources.NewDeploymentPredicate())).

// operands - dynamically owned
OwnsGVK(gvk.Gateway, reconciler.Dynamic()).
OwnsGVK(gvk.EnvoyFilter, reconciler.Dynamic()).
OwnsGVK(gvk.KnativeServing, reconciler.Dynamic()).
OwnsGVK(gvk.ServiceMeshMember, reconciler.Dynamic()).
OwnsGVK(gvk.AuthorizationPolicy, reconciler.Dynamic()).
OwnsGVK(gvk.AuthorizationPolicyv1beta1, reconciler.Dynamic()).

// operands - watched
//
// By default the Watches functions adds:
Expand All @@ -97,49 +90,13 @@ func (s *componentHandler) NewComponentReconciler(ctx context.Context, mgr ctrl.
component.ForLabel(labels.ODH.Component(LegacyComponentName), labels.True)),
).

// operands - dynamically watched
//
// A watch will be created dynamically for these kinds, if they exist on the cluster
// (they come from ServiceMesh and Serverless operators).
//
// They're owned by FeatureTrackers, which are owned by a Kserve; so there's a
// custom event mapper to enqueue a reconcile request for a Kserve object, if
// applicable.
//
// They also don't have the "partOf" label that Watches expects in the
// implicit predicate, so the simpler "DefaultPredicate" is also added.
WatchesGVK(
gvk.KnativeServing,
reconciler.Dynamic(),
reconciler.WithEventMapper(ownedViaFTMapFunc),
reconciler.WithPredicates(predicates.DefaultPredicate)).
WatchesGVK(
gvk.ServiceMeshMember,
reconciler.Dynamic(),
reconciler.WithEventMapper(ownedViaFTMapFunc),
reconciler.WithPredicates(predicates.DefaultPredicate)).
WatchesGVK(
gvk.EnvoyFilter,
reconciler.Dynamic(),
reconciler.WithEventMapper(ownedViaFTMapFunc),
reconciler.WithPredicates(predicates.DefaultPredicate)).
WatchesGVK(
gvk.AuthorizationPolicy,
reconciler.Dynamic(),
reconciler.WithEventMapper(ownedViaFTMapFunc),
reconciler.WithPredicates(predicates.DefaultPredicate)).
WatchesGVK(
gvk.Gateway,
reconciler.Dynamic(),
reconciler.WithEventMapper(ownedViaFTMapFunc),
reconciler.WithPredicates(predicates.DefaultPredicate)).

// actions
WithAction(checkPreConditions).
WithAction(initialize).
WithAction(devFlags).
WithAction(configureServerless).
WithAction(configureServiceMesh).
WithAction(renderTemplates).
WithAction(kustomize.NewAction(
kustomize.WithCache(),
// These are the default labels added by the legacy deploy method
Expand Down
88 changes: 82 additions & 6 deletions controllers/components/kserve/kserve_controller_actions.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,9 +18,9 @@ import (
"github.com/opendatahub-io/opendatahub-operator/v2/pkg/cluster"
"github.com/opendatahub-io/opendatahub-operator/v2/pkg/cluster/gvk"
odherrors "github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/errors"
"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/render/template"
odhtypes "github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/types"
"github.com/opendatahub-io/opendatahub-operator/v2/pkg/deploy"
"github.com/opendatahub-io/opendatahub-operator/v2/pkg/feature"
"github.com/opendatahub-io/opendatahub-operator/v2/pkg/metadata/labels"
)

Expand Down Expand Up @@ -137,6 +137,21 @@ func devFlags(ctx context.Context, rr *odhtypes.ReconciliationRequest) error {
return nil
}

func renderTemplates(ctx context.Context, rr *odhtypes.ReconciliationRequest) error {
k, ok := rr.Instance.(*componentApi.Kserve)
if !ok {
return fmt.Errorf("resource instance %v is not a componentApi.Kserve)", rr.Instance)
}

data := getTemplateData(ctx, rr.Client, k, rr.DSCI)

actionFn := template.NewAction(
template.WithData(data),
)

return actionFn(ctx, rr)
}

func configureServerless(ctx context.Context, rr *odhtypes.ReconciliationRequest) error {
k, ok := rr.Instance.(*componentApi.Kserve)
if !ok {
Expand Down Expand Up @@ -168,11 +183,48 @@ func configureServerless(ctx context.Context, rr *odhtypes.ReconciliationRequest
"as it is required by the KServe serving field", rr.DSCI.Spec.ServiceMesh.ManagementState)
}

serverlessFeatures := feature.ComponentFeaturesHandler(rr.Instance, componentName, rr.DSCI.Spec.ApplicationsNamespace, configureServerlessFeatures(&rr.DSCI.Spec, k))
err := createServingCertResource(ctx, cli, &rr.DSCI.Spec, k)
if err != nil {
return fmt.Errorf("unable to create serverless serving certificate secret: %w", err)
}

if err := serverlessFeatures.Apply(ctx, cli); err != nil {
return err
templates := []odhtypes.TemplateInfo{
{
FS: resourcesFS,
Path: "resources/serving-install/service-mesh-subscription.tmpl.yaml",
},
{
FS: resourcesFS,
Path: "resources/serving-install/knative-serving.tmpl.yaml",
},
{
FS: resourcesFS,
Path: "resources/serving-net-istio-secret-filtering.patch.tmpl.yaml",
},

{
FS: resourcesFS,
Path: "resources/servicemesh/routing/istio-ingress-gateway.tmpl.yaml",
},
{
FS: resourcesFS,
Path: "resources/servicemesh/routing/istio-kserve-local-gateway.tmpl.yaml",
},
{
FS: resourcesFS,
Path: "resources/servicemesh/routing/istio-local-gateway.yaml",
},
{
FS: resourcesFS,
Path: "resources/servicemesh/routing/kserve-local-gateway-svc.tmpl.yaml",
},
{
FS: resourcesFS,
Path: "resources/servicemesh/routing/local-gateway-svc.tmpl.yaml",
},
}

rr.Templates = append(rr.Templates, templates...)
}
return nil
}
Expand All @@ -187,8 +239,32 @@ func configureServiceMesh(ctx context.Context, rr *odhtypes.ReconciliationReques

if rr.DSCI.Spec.ServiceMesh != nil {
if rr.DSCI.Spec.ServiceMesh.ManagementState == operatorv1.Managed {
serviceMeshInitializer := feature.ComponentFeaturesHandler(k, componentName, rr.DSCI.Spec.ApplicationsNamespace, defineServiceMeshFeatures(ctx, cli, &rr.DSCI.Spec))
return serviceMeshInitializer.Apply(ctx, cli)
templates := []odhtypes.TemplateInfo{
{
FS: resourcesFS,
Path: "resources/servicemesh/activator-envoyfilter.tmpl.yaml",
},
{
FS: resourcesFS,
Path: "resources/servicemesh/envoy-oauth-temp-fix.tmpl.yaml",
},
{
FS: resourcesFS,
Path: "resources/servicemesh/kserve-predictor-authorizationpolicy.tmpl.yaml",
},
{
FS: resourcesFS,
Path: "resources/servicemesh/kserve-inferencegraph-envoyfilter.tmpl.yaml",
},
{
FS: resourcesFS,
Path: "resources/servicemesh/kserve-inferencegraph-authorizationpolicy.tmpl.yaml",
},
}

rr.Templates = append(rr.Templates, templates...)

return nil
}
if rr.DSCI.Spec.ServiceMesh.ManagementState == operatorv1.Unmanaged {
return nil
Expand Down
86 changes: 57 additions & 29 deletions controllers/components/kserve/kserve_support.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package kserve

import (
"context"
"embed"
"encoding/base64"
"encoding/json"
"fmt"
Expand All @@ -12,15 +13,12 @@ import (
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/types"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/handler"
"sigs.k8s.io/controller-runtime/pkg/reconcile"

componentApi "github.com/opendatahub-io/opendatahub-operator/v2/apis/components/v1alpha1"
dsciv1 "github.com/opendatahub-io/opendatahub-operator/v2/apis/dscinitialization/v1"
featuresv1 "github.com/opendatahub-io/opendatahub-operator/v2/apis/features/v1"
infrav1 "github.com/opendatahub-io/opendatahub-operator/v2/apis/infrastructure/v1"
"github.com/opendatahub-io/opendatahub-operator/v2/pkg/cluster"
odhtypes "github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/types"
"github.com/opendatahub-io/opendatahub-operator/v2/pkg/deploy"
Expand All @@ -31,6 +29,9 @@ import (
"github.com/opendatahub-io/opendatahub-operator/v2/pkg/resources"
)

//go:embed resources
var resourcesFS embed.FS

func kserveManifestInfo(sourcePath string) odhtypes.ManifestInfo {
return odhtypes.ManifestInfo{
Path: deploy.DefaultManifestPath,
Expand Down Expand Up @@ -100,6 +101,23 @@ func configureServerlessFeatures(dsciSpec *dsciv1.DSCInitializationSpec, kserve
}
}

func createServingCertResource(ctx context.Context, cli client.Client, dscispec *dsciv1.DSCInitializationSpec, kserve *componentApi.Kserve) error {
domain := getKnativeDomain(ctx, cli, kserve)
secretName := getKnativeCertSecretName(kserve)

switch kserve.Spec.Serving.IngressGateway.Certificate.Type {
case infrav1.SelfSigned:
return cluster.CreateSelfSignedCertificate(ctx, cli, secretName,
domain, dscispec.ServiceMesh.ControlPlane.Namespace,
cluster.OwnedBy(kserve, cli.Scheme()))
case infrav1.Provided:
return nil
default:
return cluster.PropagateDefaultIngressCertificate(ctx, cli,
secretName, dscispec.ServiceMesh.ControlPlane.Namespace)
}
}

func defineServiceMeshFeatures(ctx context.Context, cli client.Client, dscispec *dsciv1.DSCInitializationSpec) feature.FeaturesProvider {
return func(registry feature.FeaturesRegistry) error {
authorinoInstalled, err := cluster.SubscriptionExists(ctx, cli, "authorino-operator")
Expand Down Expand Up @@ -140,6 +158,41 @@ func defineServiceMeshFeatures(ctx context.Context, cli client.Client, dscispec
}
}

func getTemplateData(ctx context.Context, cli client.Client, k *componentApi.Kserve, dsci *dsciv1.DSCInitialization) map[string]any {
knativeIngressDomain := getKnativeDomain(ctx, cli, k)
knativeCertificateSecret := getKnativeCertSecretName(k)

return map[string]any{
"AuthExtensionName": dsci.Spec.ApplicationsNamespace + "-auth-provider",
"ControlPlane": dsci.Spec.ServiceMesh.ControlPlane,
"KnativeCertificateSecret": knativeCertificateSecret,
"KnativeIngressDomain": knativeIngressDomain,
"Serving": k.Spec.Serving,
}
}

func getKnativeDomain(ctx context.Context, cli client.Client, k *componentApi.Kserve) string {
domain := k.Spec.Serving.IngressGateway.Domain
if domain != "" {
return domain
}

domain, err := cluster.GetDomain(ctx, cli)
if err != nil {
return ""
}
return domain
}

func getKnativeCertSecretName(k *componentApi.Kserve) string {
name := k.Spec.Serving.IngressGateway.Certificate.SecretName
if name == "" {
name = "knative-serving-cert"
}

return name
}

func removeServiceMeshConfigurations(ctx context.Context, cli client.Client, owner metav1.Object, dscispec *dsciv1.DSCInitializationSpec) error {
serviceMeshInitializer := feature.ComponentFeaturesHandler(owner, componentName, dscispec.ApplicationsNamespace, defineServiceMeshFeatures(ctx, cli, dscispec))
return serviceMeshInitializer.Delete(ctx, cli)
Expand Down Expand Up @@ -241,28 +294,3 @@ func hashConfigMap(cm *corev1.ConfigMap) (string, error) {

return base64.RawURLEncoding.EncodeToString(h), nil
}

func ownedViaFT(cli client.Client) handler.MapFunc {
return func(ctx context.Context, a client.Object) []reconcile.Request {
for _, or := range a.GetOwnerReferences() {
if or.Kind == "FeatureTracker" {
ft := featuresv1.FeatureTracker{}
if err := cli.Get(ctx, client.ObjectKey{Name: or.Name}, &ft); err != nil {
return []reconcile.Request{}
}

for _, ftor := range ft.GetOwnerReferences() {
if ftor.Kind == componentApi.KserveKind && ftor.Name != "" {
return []reconcile.Request{{
NamespacedName: types.NamespacedName{
Name: ftor.Name,
},
}}
}
}
}
}

return []reconcile.Request{}
}
}
6 changes: 6 additions & 0 deletions pkg/cluster/gvk/gvk.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -215,6 +215,12 @@ var (
Kind: "AuthorizationPolicy",
}

AuthorizationPolicyv1beta1 = schema.GroupVersionKind{
Group: "security.istio.io",
Version: "v1beta1",
Kind: "AuthorizationPolicy",
}

Gateway = schema.GroupVersionKind{
Group: "networking.istio.io",
Version: "v1beta1",
Expand Down

0 comments on commit 480d57d

Please sign in to comment.