From ed731c8b9bd3505173cb6e7a4a0a9a9b844024e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Hern=C3=A1ndez?= Date: Mon, 19 Feb 2024 16:23:43 -0600 Subject: [PATCH] feat(authz): Authorino for Service Mesh (#784) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(authz): Authorino for Service Mesh This first iteration is to cover authentication needs for KServe * Add templates to install Authorino * Add templates to configure Service Mesh to use Authorino to delegate Authorization * Add KServe-specific templates add ability to secure KServe Inference Services * Add relevant fields to DSCInitialization resource * Code for proper cleanup, in case of uninstalling Most (if not all) of this code comes from pull request opendatahub-io/opendatahub-operator#605. Attribution to original authors: @bartoszmajsak, @aslakknutsen, @cam-garrison, et. al. Related opendatahub-io/kserve#128 Signed-off-by: Edgar Hernández <23639005+israel-hdez@users.noreply.github.com> * Fix linter issues Signed-off-by: Edgar Hernández <23639005+israel-hdez@users.noreply.github.com> * Resolve feedback: Bartosz Signed-off-by: Edgar Hernández <23639005+israel-hdez@users.noreply.github.com> * fix: Remove port from the authorization policy Also, add `/metrics` to the ignored paths for auth. Signed-off-by: Edgar Hernández <23639005+israel-hdez@users.noreply.github.com> * Fix feedback: Bartosz Signed-off-by: Edgar Hernández <23639005+israel-hdez@users.noreply.github.com> * More feedback: Bartosz Co-authored-by: Bartosz Majsak * Fix feedback: Reto - Adjust AuthorizationPolicy Signed-off-by: Edgar Hernández <23639005+israel-hdez@users.noreply.github.com> * Fix more feedback: Bartosz - Remove Authorino namespace field from DSCI. - Move around some code in kserve.go to servicemesh_setup.go Signed-off-by: Edgar Hernández <23639005+israel-hdez@users.noreply.github.com> * chore: adds sec. prefix to authorino label selector * fix: adds base dir to manifest sources * chore: uses security instead of sec as a prefix in authorino label * fix: /healthz is called by _something_, skipp * fix: adopt ODH-ADR-0006 for clean up label * fix: uses correct CRD name for authconfigs Co-authored-by: Cameron Garrison * Remove left-over file Signed-off-by: Edgar Hernández <23639005+israel-hdez@users.noreply.github.com> * Feedback: remove auth-refs ConfigMap Signed-off-by: Edgar Hernández <23639005+israel-hdez@users.noreply.github.com> * Add missing role.yaml changes Signed-off-by: Edgar Hernández <23639005+israel-hdez@users.noreply.github.com> * Go back to installing Authorino on its own namespace Signed-off-by: Edgar Hernández <23639005+israel-hdez@users.noreply.github.com> * Feedback: Add clean-up for KServe/OSSM-auth Signed-off-by: Edgar Hernández <23639005+israel-hdez@users.noreply.github.com> * Feedback: Simplify namings Signed-off-by: Edgar Hernández <23639005+israel-hdez@users.noreply.github.com> * fix: add auth-refs cm * Feedback: adjust labels and a log message Signed-off-by: Edgar Hernández <23639005+israel-hdez@users.noreply.github.com> * Bugfix: Extension provider terminating with error when SMCP is gone Signed-off-by: Edgar Hernández <23639005+israel-hdez@users.noreply.github.com> * Fix: add missing RBAC for ConfigMaps func Signed-off-by: Edgar Hernández <23639005+israel-hdez@users.noreply.github.com> * Fix: Run `make bundle` and commit resulting changes Signed-off-by: Edgar Hernández <23639005+israel-hdez@users.noreply.github.com> * Feedback: Wen - Better feature namings Signed-off-by: Edgar Hernández <23639005+israel-hdez@users.noreply.github.com> * Feedback: Bartosz * Use feature logger * Don't trim -applications suffix on ResolveAuthNamespace Signed-off-by: Edgar Hernández <23639005+israel-hdez@users.noreply.github.com> * Feedback: Wen - revert image placeholder was replaced Signed-off-by: Edgar Hernández <23639005+israel-hdez@users.noreply.github.com> --------- Signed-off-by: Edgar Hernández <23639005+israel-hdez@users.noreply.github.com> Co-authored-by: Bartosz Majsak Co-authored-by: Aslak Knutsen Co-authored-by: Cameron Garrison (cherry picked from commit e32a7c2c082b1c616f3f7f290c41e25efc58682e) --- .../v1/zz_generated.deepcopy.go | 2 +- ...ion.opendatahub.io_dscinitializations.yaml | 22 +++++ .../rhods-operator.clusterserviceversion.yaml | 24 ++++++ components/kserve/kserve.go | 9 ++- components/kserve/servicemesh_setup.go | 46 +++++++++++ ...ion.opendatahub.io_dscinitializations.yaml | 22 +++++ config/rbac/role.yaml | 24 ++++++ .../datasciencecluster/kubebuilder_rbac.go | 24 ++++-- .../dscinitialization/servicemesh_setup.go | 41 ++++++++++ infrastructure/v1/servicemesh_types.go | 16 ++++ infrastructure/v1/zz_generated.deepcopy.go | 25 ++++++ pkg/cluster/cluster_config.go | 31 +++++++ pkg/feature/builder.go | 9 ++- pkg/feature/feature.go | 46 +++++++++++ pkg/feature/manifest.go | 3 + pkg/feature/serverless/loaders.go | 3 +- pkg/feature/serverless/resources.go | 27 ------- pkg/feature/servicemesh/cleanup.go | 55 +++++++++++++ pkg/feature/servicemesh/conditions.go | 9 +++ pkg/feature/servicemesh/loaders.go | 32 ++++++++ pkg/feature/servicemesh/resources.go | 33 ++++++++ .../servicemesh/authorino/auth-smm.tmpl | 10 +++ .../base/operator-cluster-wide-no-tls.tmpl | 14 ++++ .../authorino/deployment.injection.patch.tmpl | 10 +++ .../mesh-authz-ext-provider.patch.tmpl | 13 +++ .../kserve/activator-envoyfilter.tmpl | 43 ++++++++++ .../kserve/envoy-oauth-temp-fix.tmpl | 80 +++++++++++++++++++ .../kserve-predictor-authorizationpolicy.tmpl | 23 ++++++ pkg/feature/types.go | 1 + 29 files changed, 654 insertions(+), 43 deletions(-) create mode 100644 components/kserve/servicemesh_setup.go create mode 100644 pkg/cluster/cluster_config.go create mode 100644 pkg/feature/servicemesh/cleanup.go create mode 100644 pkg/feature/servicemesh/loaders.go create mode 100644 pkg/feature/servicemesh/resources.go create mode 100644 pkg/feature/templates/servicemesh/authorino/auth-smm.tmpl create mode 100644 pkg/feature/templates/servicemesh/authorino/base/operator-cluster-wide-no-tls.tmpl create mode 100644 pkg/feature/templates/servicemesh/authorino/deployment.injection.patch.tmpl create mode 100644 pkg/feature/templates/servicemesh/authorino/mesh-authz-ext-provider.patch.tmpl create mode 100644 pkg/feature/templates/servicemesh/kserve/activator-envoyfilter.tmpl create mode 100644 pkg/feature/templates/servicemesh/kserve/envoy-oauth-temp-fix.tmpl create mode 100644 pkg/feature/templates/servicemesh/kserve/kserve-predictor-authorizationpolicy.tmpl diff --git a/apis/dscinitialization/v1/zz_generated.deepcopy.go b/apis/dscinitialization/v1/zz_generated.deepcopy.go index c25082c24e2..3a1af9709b7 100644 --- a/apis/dscinitialization/v1/zz_generated.deepcopy.go +++ b/apis/dscinitialization/v1/zz_generated.deepcopy.go @@ -90,7 +90,7 @@ func (in *DSCInitializationList) DeepCopyObject() runtime.Object { func (in *DSCInitializationSpec) DeepCopyInto(out *DSCInitializationSpec) { *out = *in out.Monitoring = in.Monitoring - out.ServiceMesh = in.ServiceMesh + in.ServiceMesh.DeepCopyInto(&out.ServiceMesh) out.TrustedCABundle = in.TrustedCABundle if in.DevFlags != nil { in, out := &in.DevFlags, &out.DevFlags diff --git a/bundle/manifests/dscinitialization.opendatahub.io_dscinitializations.yaml b/bundle/manifests/dscinitialization.opendatahub.io_dscinitializations.yaml index 6b9514bb1e5..5ee2a501dd3 100644 --- a/bundle/manifests/dscinitialization.opendatahub.io_dscinitializations.yaml +++ b/bundle/manifests/dscinitialization.opendatahub.io_dscinitializations.yaml @@ -88,6 +88,28 @@ spec: user experience; e.g. it provides unified authentication giving a Single Sign On experience. properties: + auth: + description: Auth holds configuration of authentication and authorization + services used by Service Mesh in Opendatahub. + properties: + audiences: + default: + - https://kubernetes.default.svc + description: Audiences is a list of the identifiers that the + resource server presented with the token identifies as. + Audience-aware token authenticators will verify that the + token was intended for at least one of the audiences in + this list. If no audiences are provided, the audience will + default to the audience of the Kubernetes apiserver (kubernetes.default.svc). + items: + type: string + type: array + namespace: + description: Namespace where it is deployed. If not provided, + the default is to use '-auth-provider' suffix on the ApplicationsNamespace + of the DSCI. + type: string + type: object controlPlane: description: ControlPlane holds configuration of Service Mesh used by Opendatahub. diff --git a/bundle/manifests/rhods-operator.clusterserviceversion.yaml b/bundle/manifests/rhods-operator.clusterserviceversion.yaml index 17753c7dfc3..d37fcd56905 100644 --- a/bundle/manifests/rhods-operator.clusterserviceversion.yaml +++ b/bundle/manifests/rhods-operator.clusterserviceversion.yaml @@ -374,6 +374,12 @@ spec: - tokenreviews verbs: - create + - apiGroups: + - authorino.kuadrant.io + resources: + - authconfigs + verbs: + - '*' - apiGroups: - authorization.k8s.io resources: @@ -1201,6 +1207,12 @@ spec: - deletecollection - get - patch + - apiGroups: + - networking.istio.io + resources: + - envoyfilters + verbs: + - '*' - apiGroups: - networking.istio.io resources: @@ -1281,6 +1293,12 @@ spec: - patch - update - watch + - apiGroups: + - operator.authorino.kuadrant.io + resources: + - authorinos + verbs: + - '*' - apiGroups: - operator.knative.dev resources: @@ -1422,6 +1440,12 @@ spec: - patch - update - watch + - apiGroups: + - security.istio.io + resources: + - authorizationpolicies + verbs: + - '*' - apiGroups: - security.openshift.io resources: diff --git a/components/kserve/kserve.go b/components/kserve/kserve.go index 739a9345d28..e11dd62f4fe 100644 --- a/components/kserve/kserve.go +++ b/components/kserve/kserve.go @@ -177,9 +177,14 @@ func (k *Kserve) ReconcileComponent(ctx context.Context, cli client.Client, resC return err } } - return nil + + return k.configureServiceMesh(dscispec) } func (k *Kserve) Cleanup(_ client.Client, instance *dsciv1.DSCInitializationSpec) error { - return k.removeServerlessFeatures(instance) + if removeServerlessErr := k.removeServerlessFeatures(instance); removeServerlessErr != nil { + return removeServerlessErr + } + + return k.removeServiceMeshConfigurations(instance) } diff --git a/components/kserve/servicemesh_setup.go b/components/kserve/servicemesh_setup.go new file mode 100644 index 00000000000..49da2c30d20 --- /dev/null +++ b/components/kserve/servicemesh_setup.go @@ -0,0 +1,46 @@ +package kserve + +import ( + "path" + + operatorv1 "github.com/openshift/api/operator/v1" + + dsciv1 "github.com/opendatahub-io/opendatahub-operator/v2/apis/dscinitialization/v1" + "github.com/opendatahub-io/opendatahub-operator/v2/pkg/feature" + "github.com/opendatahub-io/opendatahub-operator/v2/pkg/feature/servicemesh" +) + +func (k *Kserve) configureServiceMesh(dscispec *dsciv1.DSCInitializationSpec) error { + if dscispec.ServiceMesh.ManagementState == operatorv1.Managed && k.GetManagementState() == operatorv1.Managed { + serviceMeshInitializer := feature.ComponentFeaturesHandler(k, dscispec, k.defineServiceMeshFeatures()) + return serviceMeshInitializer.Apply() + } + if dscispec.ServiceMesh.ManagementState == operatorv1.Unmanaged && k.GetManagementState() == operatorv1.Managed { + return nil + } + + return k.removeServiceMeshConfigurations(dscispec) +} + +func (k *Kserve) removeServiceMeshConfigurations(dscispec *dsciv1.DSCInitializationSpec) error { + serviceMeshInitializer := feature.ComponentFeaturesHandler(k, dscispec, k.defineServiceMeshFeatures()) + return serviceMeshInitializer.Delete() +} + +func (k *Kserve) defineServiceMeshFeatures() feature.FeaturesProvider { + return func(handler *feature.FeaturesHandler) error { + kserveExtAuthzErr := feature.CreateFeature("kserve-external-authz"). + For(handler). + Manifests( + path.Join(feature.KServeDir), + ). + WithData(servicemesh.ClusterDetails). + Load() + + if kserveExtAuthzErr != nil { + return kserveExtAuthzErr + } + + return nil + } +} diff --git a/config/crd/bases/dscinitialization.opendatahub.io_dscinitializations.yaml b/config/crd/bases/dscinitialization.opendatahub.io_dscinitializations.yaml index a821a61ff36..ada718ed5c4 100644 --- a/config/crd/bases/dscinitialization.opendatahub.io_dscinitializations.yaml +++ b/config/crd/bases/dscinitialization.opendatahub.io_dscinitializations.yaml @@ -89,6 +89,28 @@ spec: user experience; e.g. it provides unified authentication giving a Single Sign On experience. properties: + auth: + description: Auth holds configuration of authentication and authorization + services used by Service Mesh in Opendatahub. + properties: + audiences: + default: + - https://kubernetes.default.svc + description: Audiences is a list of the identifiers that the + resource server presented with the token identifies as. + Audience-aware token authenticators will verify that the + token was intended for at least one of the audiences in + this list. If no audiences are provided, the audience will + default to the audience of the Kubernetes apiserver (kubernetes.default.svc). + items: + type: string + type: array + namespace: + description: Namespace where it is deployed. If not provided, + the default is to use '-auth-provider' suffix on the ApplicationsNamespace + of the DSCI. + type: string + type: object controlPlane: description: ControlPlane holds configuration of Service Mesh used by Opendatahub. diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 2f07b94eb07..e41820991f6 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -132,6 +132,12 @@ rules: - tokenreviews verbs: - create +- apiGroups: + - authorino.kuadrant.io + resources: + - authconfigs + verbs: + - '*' - apiGroups: - authorization.k8s.io resources: @@ -959,6 +965,12 @@ rules: - deletecollection - get - patch +- apiGroups: + - networking.istio.io + resources: + - envoyfilters + verbs: + - '*' - apiGroups: - networking.istio.io resources: @@ -1039,6 +1051,12 @@ rules: - patch - update - watch +- apiGroups: + - operator.authorino.kuadrant.io + resources: + - authorinos + verbs: + - '*' - apiGroups: - operator.knative.dev resources: @@ -1180,6 +1198,12 @@ rules: - patch - update - watch +- apiGroups: + - security.istio.io + resources: + - authorizationpolicies + verbs: + - '*' - apiGroups: - security.openshift.io resources: diff --git a/controllers/datasciencecluster/kubebuilder_rbac.go b/controllers/datasciencecluster/kubebuilder_rbac.go index ab7994513fb..8749296cbeb 100644 --- a/controllers/datasciencecluster/kubebuilder_rbac.go +++ b/controllers/datasciencecluster/kubebuilder_rbac.go @@ -4,14 +4,25 @@ package datasciencecluster //+kubebuilder:rbac:groups="datasciencecluster.opendatahub.io",resources=datascienceclusters/finalizers,verbs=update;patch //+kubebuilder:rbac:groups="datasciencecluster.opendatahub.io",resources=datascienceclusters,verbs=get;list;watch;create;update;patch;delete -/* Service Mesh prerequisite */ -// +kubebuilder:rbac:groups="maistra.io",resources=servicemeshcontrolplanes,verbs=create;get;list;patch;update;use;watch - /* Serverless prerequisite */ // +kubebuilder:rbac:groups="networking.istio.io",resources=gateways,verbs=* // +kubebuilder:rbac:groups="operator.knative.dev",resources=knativeservings,verbs=* // +kubebuilder:rbac:groups="config.openshift.io",resources=ingresses,verbs=get +/* Service Mesh Integration */ +// +kubebuilder:rbac:groups="maistra.io",resources=servicemeshcontrolplanes,verbs=create;get;list;patch;update;use;watch +// +kubebuilder:rbac:groups="maistra.io",resources=servicemeshmemberrolls,verbs=create;get;list;patch;update;use;watch +// +kubebuilder:rbac:groups="maistra.io",resources=servicemeshmembers,verbs=create;get;list;patch;update;use;watch +// +kubebuilder:rbac:groups="maistra.io",resources=servicemeshmembers/finalizers,verbs=create;get;list;patch;update;use;watch +// +kubebuilder:rbac:groups="networking.istio.io",resources=virtualservices/status,verbs=update;patch;delete +// +kubebuilder:rbac:groups="networking.istio.io",resources=virtualservices/finalizers,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups="networking.istio.io",resources=virtualservices,verbs=* +// +kubebuilder:rbac:groups="networking.istio.io",resources=gateways,verbs=* +// +kubebuilder:rbac:groups="networking.istio.io",resources=envoyfilters,verbs=* +// +kubebuilder:rbac:groups="security.istio.io",resources=authorizationpolicies,verbs=* +// +kubebuilder:rbac:groups="authorino.kuadrant.io",resources=authconfigs,verbs=* +// +kubebuilder:rbac:groups="operator.authorino.kuadrant.io",resources=authorinos,verbs=* + /* This is for DSP */ //+kubebuilder:rbac:groups="datasciencepipelinesapplications.opendatahub.io",resources=datasciencepipelinesapplications/status,verbs=update;patch;get //+kubebuilder:rbac:groups="datasciencepipelinesapplications.opendatahub.io",resources=datasciencepipelinesapplications/finalizers,verbs=update;patch @@ -93,10 +104,6 @@ package datasciencecluster // +kubebuilder:rbac:groups="networking.k8s.io",resources=networkpolicies,verbs=get;create;list;watch;delete;update;patch // +kubebuilder:rbac:groups="networking.k8s.io",resources=ingresses,verbs=create;delete;list;update;watch;patch;get -// +kubebuilder:rbac:groups="networking.istio.io",resources=virtualservices/status,verbs=update;patch;delete -// +kubebuilder:rbac:groups="networking.istio.io",resources=virtualservices/finalizers,verbs=get;list;watch;create;update;patch;delete -// +kubebuilder:rbac:groups="networking.istio.io",resources=virtualservices,verbs=* - // +kubebuilder:rbac:groups="monitoring.coreos.com",resources=servicemonitors,verbs=get;create;delete;update;watch;list;patch;deletecollection // +kubebuilder:rbac:groups="monitoring.coreos.com",resources=podmonitors,verbs=get;create;delete;update;watch;list;patch // +kubebuilder:rbac:groups="monitoring.coreos.com",resources=prometheusrules,verbs=get;create;patch;delete;deletecollection @@ -185,7 +192,7 @@ package datasciencecluster // +kubebuilder:rbac:groups="core",resources=endpoints,verbs=watch;list;get;create;update;delete // +kubebuilder:rbac:groups="core",resources=configmaps/status,verbs=get;update;patch;delete -// +kubebuilder:rbac:groups="core",resources=configmaps,verbs=get;create;watch;patch;delete;list;update +// +kubebuilder:rbac:groups="core",resources=configmaps,verbs=get;create;update;watch;patch;delete;list // +kubebuilder:rbac:groups="core",resources=clusterversions,verbs=watch;list // +kubebuilder:rbac:groups="config.openshift.io",resources=clusterversions,verbs=watch;list @@ -250,5 +257,6 @@ package datasciencecluster // +kubebuilder:rbac:groups="maistra.io",resources=servicemeshmembers/finalizers,verbs=create;get;list;patch;update;use;watch /* Only for RHOAI */ + // +kubebuilder:rbac:groups="user.openshift.io",resources=groups,verbs=get;create;list;watch;patch;delete // +kubebuilder:rbac:groups="console.openshift.io",resources=consolelinks,verbs=create;get;patch;delete diff --git a/controllers/dscinitialization/servicemesh_setup.go b/controllers/dscinitialization/servicemesh_setup.go index 7d67bf5205e..bfb57dcba8d 100644 --- a/controllers/dscinitialization/servicemesh_setup.go +++ b/controllers/dscinitialization/servicemesh_setup.go @@ -86,6 +86,47 @@ func configureServiceMeshFeatures() feature.FeaturesProvider { } } + cfgMapErr := feature.CreateFeature("mesh-shared-configmap"). + For(handler). + WithResources(servicemesh.ConfigMaps). + Load() + if cfgMapErr != nil { + return cfgMapErr + } + + extAuthzErr := feature.CreateFeature("mesh-control-plane-external-authz"). + For(handler). + Manifests( + path.Join(feature.AuthDir, "auth-smm.tmpl"), + path.Join(feature.AuthDir, "base"), + path.Join(feature.AuthDir, "mesh-authz-ext-provider.patch.tmpl"), + ). + WithData(servicemesh.ClusterDetails). + PreConditions( + feature.EnsureCRDIsInstalled("authconfigs.authorino.kuadrant.io"), + servicemesh.EnsureServiceMeshInstalled, + servicemesh.EnsureAuthNamespaceExists, + ). + PostConditions( + feature.WaitForPodsToBeReady(serviceMeshSpec.ControlPlane.Namespace), + func(f *feature.Feature) error { + return feature.WaitForPodsToBeReady(f.Spec.Auth.Namespace)(f) + }, + func(f *feature.Feature) error { + // We do not have the control over deployment resource creation. + // It is created by Authorino operator using Authorino CR + // + // To make it part of Service Mesh we have to patch it with injection + // enabled instead, otherwise it will not have proxy pod injected. + return f.ApplyManifest(path.Join(feature.AuthDir, "deployment.injection.patch.tmpl")) + }, + ). + OnDelete(servicemesh.RemoveExtensionProvider). + Load() + if extAuthzErr != nil { + return extAuthzErr + } + return nil } } diff --git a/infrastructure/v1/servicemesh_types.go b/infrastructure/v1/servicemesh_types.go index 9c093d0c663..8ae7e743645 100644 --- a/infrastructure/v1/servicemesh_types.go +++ b/infrastructure/v1/servicemesh_types.go @@ -9,6 +9,9 @@ type ServiceMeshSpec struct { ManagementState operatorv1.ManagementState `json:"managementState,omitempty"` // ControlPlane holds configuration of Service Mesh used by Opendatahub. ControlPlane ControlPlaneSpec `json:"controlPlane,omitempty"` + // Auth holds configuration of authentication and authorization services + // used by Service Mesh in Opendatahub. + Auth AuthSpec `json:"auth,omitempty"` } type ControlPlaneSpec struct { @@ -38,3 +41,16 @@ type IngressGatewaySpec struct { // the for Ingress Gateway. Certificate CertificateSpec `json:"certificate,omitempty"` } + +type AuthSpec struct { + // Namespace where it is deployed. If not provided, the default is to + // use '-auth-provider' suffix on the ApplicationsNamespace of the DSCI. + Namespace string `json:"namespace,omitempty"` + // Audiences is a list of the identifiers that the resource server presented + // with the token identifies as. Audience-aware token authenticators will verify + // that the token was intended for at least one of the audiences in this list. + // If no audiences are provided, the audience will default to the audience of the + // Kubernetes apiserver (kubernetes.default.svc). + // +kubebuilder:default={"https://kubernetes.default.svc"} + Audiences *[]string `json:"audiences,omitempty"` +} diff --git a/infrastructure/v1/zz_generated.deepcopy.go b/infrastructure/v1/zz_generated.deepcopy.go index efc062e396b..b84a224094f 100644 --- a/infrastructure/v1/zz_generated.deepcopy.go +++ b/infrastructure/v1/zz_generated.deepcopy.go @@ -23,6 +23,30 @@ package v1 import () +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AuthSpec) DeepCopyInto(out *AuthSpec) { + *out = *in + if in.Audiences != nil { + in, out := &in.Audiences, &out.Audiences + *out = new([]string) + if **in != nil { + in, out := *in, *out + *out = make([]string, len(*in)) + copy(*out, *in) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AuthSpec. +func (in *AuthSpec) DeepCopy() *AuthSpec { + if in == nil { + return nil + } + out := new(AuthSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *CertificateSpec) DeepCopyInto(out *CertificateSpec) { *out = *in @@ -73,6 +97,7 @@ func (in *IngressGatewaySpec) DeepCopy() *IngressGatewaySpec { func (in *ServiceMeshSpec) DeepCopyInto(out *ServiceMeshSpec) { *out = *in out.ControlPlane = in.ControlPlane + in.Auth.DeepCopyInto(&out.Auth) } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ServiceMeshSpec. diff --git a/pkg/cluster/cluster_config.go b/pkg/cluster/cluster_config.go new file mode 100644 index 00000000000..ede5eb6dd6f --- /dev/null +++ b/pkg/cluster/cluster_config.go @@ -0,0 +1,31 @@ +package cluster + +import ( + "context" + "errors" + "fmt" + + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// +kubebuilder:rbac:groups="config.openshift.io",resources=ingresses,verbs=get + +func GetDomain(c client.Client) (string, error) { + ingress := &unstructured.Unstructured{} + ingress.SetGroupVersionKind(OpenshiftIngressGVK) + + if err := c.Get(context.TODO(), client.ObjectKey{ + Namespace: "", + Name: "cluster", + }, ingress); err != nil { + return "", fmt.Errorf("failed fetching cluster's ingress details: %w", err) + } + + domain, found, err := unstructured.NestedString(ingress.Object, "spec", "domain") + if !found { + return "", errors.New("spec.domain not found") + } + + return domain, err +} diff --git a/pkg/feature/builder.go b/pkg/feature/builder.go index 9b952d9f9db..17079ba6777 100644 --- a/pkg/feature/builder.go +++ b/pkg/feature/builder.go @@ -38,10 +38,11 @@ type usingFeaturesHandler struct { func (u *usingFeaturesHandler) For(featuresHandler *FeaturesHandler) *featureBuilder { createSpec := func(f *Feature) error { f.Spec = &Spec{ - ServiceMeshSpec: &featuresHandler.DSCInitializationSpec.ServiceMesh, - Serving: &infrav1.ServingSpec{}, - Source: &featuresHandler.source, - AppNamespace: featuresHandler.DSCInitializationSpec.ApplicationsNamespace, + ServiceMeshSpec: &featuresHandler.DSCInitializationSpec.ServiceMesh, + Serving: &infrav1.ServingSpec{}, + Source: &featuresHandler.source, + AppNamespace: featuresHandler.DSCInitializationSpec.ApplicationsNamespace, + AuthProviderName: "authorino", } return nil diff --git a/pkg/feature/feature.go b/pkg/feature/feature.go index d3bf6b67066..40a400b2ba1 100644 --- a/pkg/feature/feature.go +++ b/pkg/feature/feature.go @@ -9,6 +9,7 @@ import ( conditionsv1 "github.com/openshift/custom-resource-status/conditions/v1" "github.com/pkg/errors" corev1 "k8s.io/api/core/v1" + k8serrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client" ctrlLog "sigs.k8s.io/controller-runtime/pkg/log" @@ -143,10 +144,55 @@ func (f *Feature) applyManifests() error { return applyErrors.ErrorOrNil() } +func (f *Feature) CreateConfigMap(cfgMapName string, data map[string]string) error { + configMap := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: cfgMapName, + Namespace: f.Spec.AppNamespace, + OwnerReferences: []metav1.OwnerReference{ + f.AsOwnerReference(), + }, + }, + Data: data, + } + + err := f.Client.Get(context.TODO(), client.ObjectKey{ + Namespace: configMap.Namespace, + Name: configMap.Name, + }, configMap) + if err != nil { + if k8serrors.IsNotFound(err) { + err = f.Client.Create(context.TODO(), configMap) + if err != nil { + return err + } + } else { + return err + } + } else { + err = f.Client.Update(context.TODO(), configMap) + if err != nil { + return err + } + } + + return nil +} + func (f *Feature) addCleanup(cleanupFuncs ...Action) { f.cleanups = append(f.cleanups, cleanupFuncs...) } +func (f *Feature) ApplyManifest(path string) error { + m := createManifestFrom(embeddedFiles, path) + + if err := m.process(f.Spec); err != nil { + return err + } + + return f.apply(m) +} + type apply func(data string) error func (f *Feature) apply(m manifest) error { diff --git a/pkg/feature/manifest.go b/pkg/feature/manifest.go index db4b111f1dc..3821d3ddc3f 100644 --- a/pkg/feature/manifest.go +++ b/pkg/feature/manifest.go @@ -19,6 +19,8 @@ var ( BaseDir = "templates" ServiceMeshDir = path.Join(BaseDir, "servicemesh") ServerlessDir = path.Join(BaseDir, "serverless") + AuthDir = path.Join(ServiceMeshDir, "authorino") + KServeDir = path.Join(ServiceMeshDir, "kserve") ) type manifest struct { @@ -81,6 +83,7 @@ func (m *manifest) process(data interface{}) error { if err != nil { return err } + defer manifestFile.Close() content, err := io.ReadAll(manifestFile) diff --git a/pkg/feature/serverless/loaders.go b/pkg/feature/serverless/loaders.go index 966021e39da..58867b29f99 100644 --- a/pkg/feature/serverless/loaders.go +++ b/pkg/feature/serverless/loaders.go @@ -4,6 +4,7 @@ import ( "fmt" "strings" + "github.com/opendatahub-io/opendatahub-operator/v2/pkg/cluster" "github.com/opendatahub-io/opendatahub-operator/v2/pkg/feature" ) @@ -23,7 +24,7 @@ func ServingIngressDomain(f *feature.Feature) error { domain := strings.TrimSpace(f.Spec.Serving.IngressGateway.Domain) if len(domain) == 0 { var errDomain error - domain, errDomain = GetDomain(f.Client) + domain, errDomain = cluster.GetDomain(f.Client) if errDomain != nil { return fmt.Errorf("failed to fetch OpenShift domain to generate certificate for Serverless: %w", errDomain) } diff --git a/pkg/feature/serverless/resources.go b/pkg/feature/serverless/resources.go index 5acf5514974..e3360ddef0c 100644 --- a/pkg/feature/serverless/resources.go +++ b/pkg/feature/serverless/resources.go @@ -1,36 +1,9 @@ package serverless import ( - "context" - "fmt" - - "github.com/pkg/errors" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - "sigs.k8s.io/controller-runtime/pkg/client" - - "github.com/opendatahub-io/opendatahub-operator/v2/pkg/cluster" "github.com/opendatahub-io/opendatahub-operator/v2/pkg/feature" ) func ServingCertificateResource(f *feature.Feature) error { return f.CreateSelfSignedCertificate(f.Spec.KnativeCertificateSecret, f.Spec.Serving.IngressGateway.Certificate.Type, f.Spec.KnativeIngressDomain, f.Spec.ControlPlane.Namespace) } - -func GetDomain(c client.Client) (string, error) { - ingress := &unstructured.Unstructured{} - ingress.SetGroupVersionKind(cluster.OpenshiftIngressGVK) - - if err := c.Get(context.TODO(), client.ObjectKey{ - Namespace: "", - Name: "cluster", - }, ingress); err != nil { - return "", fmt.Errorf("failed fetching cluster's ingress details: %w", err) - } - - domain, found, err := unstructured.NestedString(ingress.Object, "spec", "domain") - if !found { - return "", errors.New("spec.domain not found") - } - - return domain, err -} diff --git a/pkg/feature/servicemesh/cleanup.go b/pkg/feature/servicemesh/cleanup.go new file mode 100644 index 00000000000..0956674be24 --- /dev/null +++ b/pkg/feature/servicemesh/cleanup.go @@ -0,0 +1,55 @@ +package servicemesh + +import ( + "context" + "fmt" + + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/opendatahub-io/opendatahub-operator/v2/pkg/cluster" + "github.com/opendatahub-io/opendatahub-operator/v2/pkg/feature" +) + +func RemoveExtensionProvider(f *feature.Feature) error { + ossmAuthzProvider := fmt.Sprintf("%s-odh-auth-provider", f.Spec.AppNamespace) + + mesh := f.Spec.ControlPlane + smcp := &unstructured.Unstructured{} + smcp.SetGroupVersionKind(cluster.ServiceMeshControlPlaneGVK) + + if err := f.Client.Get(context.TODO(), client.ObjectKey{ + Namespace: mesh.Namespace, + Name: mesh.Name, + }, smcp); err != nil { + return client.IgnoreNotFound(err) + } + + extensionProviders, found, err := unstructured.NestedSlice(smcp.Object, "spec", "techPreview", "meshConfig", "extensionProviders") + if err != nil { + return err + } + if !found { + f.Log.Info("no extension providers found", "feature", f.Name, "control-plane", mesh.Name, "namespace", mesh.Namespace) + return nil + } + + for i, v := range extensionProviders { + extensionProvider, ok := v.(map[string]interface{}) + if !ok { + f.Log.Info("WARN: Unexpected type for extensionProvider will not be removed") + continue + } + + if extensionProvider["name"] == ossmAuthzProvider { + extensionProviders = append(extensionProviders[:i], extensionProviders[i+1:]...) + err = unstructured.SetNestedSlice(smcp.Object, extensionProviders, "spec", "techPreview", "meshConfig", "extensionProviders") + if err != nil { + return err + } + break + } + } + + return f.Client.Update(context.TODO(), smcp) +} diff --git a/pkg/feature/servicemesh/conditions.go b/pkg/feature/servicemesh/conditions.go index 947d811da76..17f42d40026 100644 --- a/pkg/feature/servicemesh/conditions.go +++ b/pkg/feature/servicemesh/conditions.go @@ -20,6 +20,15 @@ const ( duration = 5 * time.Minute ) +func EnsureAuthNamespaceExists(f *feature.Feature) error { + if resolveNsErr := ResolveAuthNamespace(f); resolveNsErr != nil { + return resolveNsErr + } + + _, err := cluster.CreateNamespace(f.Client, f.Spec.Auth.Namespace) + return err +} + func EnsureServiceMeshOperatorInstalled(f *feature.Feature) error { if err := feature.EnsureCRDIsInstalled("servicemeshcontrolplanes.maistra.io")(f); err != nil { f.Log.Info("Failed to find the pre-requisite Service Mesh Control Plane CRD, please ensure Service Mesh Operator is installed.") diff --git a/pkg/feature/servicemesh/loaders.go b/pkg/feature/servicemesh/loaders.go new file mode 100644 index 00000000000..8d61dc4aae0 --- /dev/null +++ b/pkg/feature/servicemesh/loaders.go @@ -0,0 +1,32 @@ +package servicemesh + +import ( + "strings" + + "github.com/pkg/errors" + + "github.com/opendatahub-io/opendatahub-operator/v2/pkg/cluster" + "github.com/opendatahub-io/opendatahub-operator/v2/pkg/feature" +) + +func ClusterDetails(f *feature.Feature) error { + data := f.Spec + + if domain, err := cluster.GetDomain(f.Client); err == nil { + data.Domain = domain + } else { + return errors.WithStack(err) + } + + return nil +} + +func ResolveAuthNamespace(f *feature.Feature) error { + dsciAuthNamespace := strings.TrimSpace(f.Spec.Auth.Namespace) + + if len(dsciAuthNamespace) == 0 { + f.Spec.Auth.Namespace = f.Spec.AppNamespace + "-auth-provider" + } + + return nil +} diff --git a/pkg/feature/servicemesh/resources.go b/pkg/feature/servicemesh/resources.go new file mode 100644 index 00000000000..deeed45d681 --- /dev/null +++ b/pkg/feature/servicemesh/resources.go @@ -0,0 +1,33 @@ +package servicemesh + +import ( + "strings" + + "github.com/pkg/errors" + + "github.com/opendatahub-io/opendatahub-operator/v2/pkg/feature" +) + +func ConfigMaps(feature *feature.Feature) error { + meshConfig := feature.Spec.ControlPlane + if err := feature.CreateConfigMap("service-mesh-refs", + map[string]string{ + "CONTROL_PLANE_NAME": meshConfig.Name, + "MESH_NAMESPACE": meshConfig.Namespace, + }); err != nil { + return errors.WithStack(err) + } + + audiences := feature.Spec.Auth.Audiences + audiencesList := "" + if audiences != nil && len(*audiences) > 0 { + audiencesList = strings.Join(*audiences, ",") + } + if err := feature.CreateConfigMap("auth-refs", + map[string]string{ + "AUTH_AUDIENCE": audiencesList, + }); err != nil { + return errors.WithStack(err) + } + return nil +} diff --git a/pkg/feature/templates/servicemesh/authorino/auth-smm.tmpl b/pkg/feature/templates/servicemesh/authorino/auth-smm.tmpl new file mode 100644 index 00000000000..6b0aa06aa82 --- /dev/null +++ b/pkg/feature/templates/servicemesh/authorino/auth-smm.tmpl @@ -0,0 +1,10 @@ +apiVersion: maistra.io/v1 +kind: ServiceMeshMember +metadata: + name: default + namespace: {{ .Auth.Namespace }} +spec: + controlPlaneRef: + namespace: {{ .ControlPlane.Namespace }} + name: {{ .ControlPlane.Name }} + diff --git a/pkg/feature/templates/servicemesh/authorino/base/operator-cluster-wide-no-tls.tmpl b/pkg/feature/templates/servicemesh/authorino/base/operator-cluster-wide-no-tls.tmpl new file mode 100644 index 00000000000..5624128f904 --- /dev/null +++ b/pkg/feature/templates/servicemesh/authorino/base/operator-cluster-wide-no-tls.tmpl @@ -0,0 +1,14 @@ +apiVersion: operator.authorino.kuadrant.io/v1beta1 +kind: Authorino +metadata: + name: {{ .AuthProviderName }} + namespace: {{ .Auth.Namespace }} +spec: + authConfigLabelSelectors: security.opendatahub.io/authorization-group=default + clusterWide: true + listener: + tls: + enabled: false + oidcServer: + tls: + enabled: false diff --git a/pkg/feature/templates/servicemesh/authorino/deployment.injection.patch.tmpl b/pkg/feature/templates/servicemesh/authorino/deployment.injection.patch.tmpl new file mode 100644 index 00000000000..7040f76da79 --- /dev/null +++ b/pkg/feature/templates/servicemesh/authorino/deployment.injection.patch.tmpl @@ -0,0 +1,10 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ .AuthProviderName }} + namespace: {{ .Auth.Namespace }} +spec: + template: + metadata: + annotations: + sidecar.istio.io/inject: "true" diff --git a/pkg/feature/templates/servicemesh/authorino/mesh-authz-ext-provider.patch.tmpl b/pkg/feature/templates/servicemesh/authorino/mesh-authz-ext-provider.patch.tmpl new file mode 100644 index 00000000000..2dea63bf14d --- /dev/null +++ b/pkg/feature/templates/servicemesh/authorino/mesh-authz-ext-provider.patch.tmpl @@ -0,0 +1,13 @@ +apiVersion: maistra.io/v2 +kind: ServiceMeshControlPlane +metadata: + name: {{ .ControlPlane.Name }} + namespace: {{ .ControlPlane.Namespace }} +spec: + techPreview: + meshConfig: + extensionProviders: + - name: {{ .AppNamespace }}-auth-provider + envoyExtAuthzGrpc: + service: {{ .AuthProviderName }}-authorization.{{ .Auth.Namespace }}.svc.cluster.local + port: 50051 diff --git a/pkg/feature/templates/servicemesh/kserve/activator-envoyfilter.tmpl b/pkg/feature/templates/servicemesh/kserve/activator-envoyfilter.tmpl new file mode 100644 index 00000000000..bd47b454b61 --- /dev/null +++ b/pkg/feature/templates/servicemesh/kserve/activator-envoyfilter.tmpl @@ -0,0 +1,43 @@ +apiVersion: networking.istio.io/v1alpha3 +kind: EnvoyFilter +metadata: + name: activator-host-header + namespace: {{ .ControlPlane.Namespace }} + labels: + app.opendatahub.io/kserve: "true" + app.kubernetes.io/part-of: kserve +spec: + priority: 20 + workloadSelector: + labels: + component: predictor + configPatches: + - applyTo: HTTP_FILTER + match: + listener: + filterChain: + filter: + name: envoy.filters.network.http_connection_manager + patch: + operation: INSERT_BEFORE + value: + name: envoy.filters.http.lua + typed_config: + '@type': type.googleapis.com/envoy.extensions.filters.http.lua.v3.Lua + inlineCode: | + function envoy_on_request(request_handle) + local headers = request_handle:headers() + if not headers then + return + end + + local original_host = headers:get("k-original-host") + if original_host then + + port_seperator = string.find(original_host, ":", 7) + if port_seperator then + original_host = string.sub(original_host, 0, port_seperator-1) + end + headers:replace('host', original_host) + end + end diff --git a/pkg/feature/templates/servicemesh/kserve/envoy-oauth-temp-fix.tmpl b/pkg/feature/templates/servicemesh/kserve/envoy-oauth-temp-fix.tmpl new file mode 100644 index 00000000000..b852d2f1e2e --- /dev/null +++ b/pkg/feature/templates/servicemesh/kserve/envoy-oauth-temp-fix.tmpl @@ -0,0 +1,80 @@ +apiVersion: networking.istio.io/v1alpha3 +kind: EnvoyFilter +metadata: + name: envoy-oauth-temp-fix-before + namespace: {{ .ControlPlane.Namespace }} + labels: + opendatahub.io/related-to: OSSM-4873 + app.opendatahub.io/kserve: "true" + app.kubernetes.io/part-of: kserve +spec: + workloadSelector: + labels: + istio: ingressgateway + priority: 20 + configPatches: + - applyTo: HTTP_FILTER + match: + listener: + filterChain: + filter: + name: envoy.filters.network.http_connection_manager + patch: + operation: INSERT_BEFORE + value: + name: envoy.filters.http.lua + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.lua.v3.Lua + inlineCode: | + function envoy_on_request(request_handle) + local headers = request_handle:headers() + if not headers then + return + end + + local auth = headers:get("authorization") + if auth then + headers:replace("x-authorization", auth) + end + end +--- +apiVersion: networking.istio.io/v1alpha3 +kind: EnvoyFilter +metadata: + name: envoy-oauth-temp-fix-after + namespace: {{ .ControlPlane.Namespace }} + labels: + opendatahub.io/related-to: OSSM-4873 + app.opendatahub.io/kserve: "true" + app.kubernetes.io/part-of: kserve +spec: + workloadSelector: + labels: + istio: ingressgateway + priority: 5 + configPatches: + - applyTo: HTTP_FILTER + match: + listener: + filterChain: + filter: + name: envoy.filters.network.http_connection_manager + patch: + operation: INSERT_BEFORE + value: + name: envoy.filters.http.lua + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.lua.v3.Lua + inlineCode: | + function envoy_on_request(request_handle) + local headers = request_handle:headers() + if not headers then + return + end + + local xauth = headers:get("x-authorization") + if xauth then + headers:replace("authorization", xauth) + headers.remove("x-authorization") + end + end diff --git a/pkg/feature/templates/servicemesh/kserve/kserve-predictor-authorizationpolicy.tmpl b/pkg/feature/templates/servicemesh/kserve/kserve-predictor-authorizationpolicy.tmpl new file mode 100644 index 00000000000..a79057f26a9 --- /dev/null +++ b/pkg/feature/templates/servicemesh/kserve/kserve-predictor-authorizationpolicy.tmpl @@ -0,0 +1,23 @@ +apiVersion: security.istio.io/v1beta1 +kind: AuthorizationPolicy +metadata: + name: kserve-predictor + namespace: {{ .ControlPlane.Namespace }} + labels: + app.opendatahub.io/kserve: "true" + app.kubernetes.io/part-of: kserve +spec: + action: CUSTOM + provider: + name: opendatahub-odh-auth-provider + rules: + - to: + - operation: + notPaths: + - /healthz + - /debug/pprof/ + - /metrics + - /wait-for-drain + selector: + matchLabels: + component: predictor diff --git a/pkg/feature/types.go b/pkg/feature/types.go index 9733587b074..77c30d29ece 100644 --- a/pkg/feature/types.go +++ b/pkg/feature/types.go @@ -10,6 +10,7 @@ import ( type Spec struct { *infrav1.ServiceMeshSpec Serving *infrav1.ServingSpec + AuthProviderName string OAuth OAuth AppNamespace string Domain string