From 74a6f8ada12625ad0360dbebf3d1ba56e2ce87d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Hern=C3=A1ndez?= <23639005+israel-hdez@users.noreply.github.com> Date: Mon, 11 Dec 2023 14:13:34 -0600 Subject: [PATCH] feat(authz): Authorino for Service Mesh MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This first iteration is to cover authentication needs for KServe * Add templates to install Authorino * Add templates to configure Service Mesh to use Authorino to delegate Authorization * Add KServe-specific templates add ability to secure KServe Inference Services * Add relevant fields to DSCInitialization resource * Code for proper cleanup, in case of uninstalling Most (if not all) of this code comes from pull request opendatahub-io/opendatahub-operator#605. Attribution to original authors: @bartoszmajsak, @aslakknutsen, @cam-garrison, et. al. Related opendatahub-io/kserve#128 Signed-off-by: Edgar Hernández <23639005+israel-hdez@users.noreply.github.com> --- .../v1/zz_generated.deepcopy.go | 4 +- components/kserve/kserve.go | 54 +++++++++++++ ...ion.opendatahub.io_dscinitializations.yaml | 42 ++++++++++ config/rbac/role.yaml | 67 +++++++++++----- .../datasciencecluster/kubebuilder_rbac.go | 33 ++++---- .../dscinitialization/servicemesh_setup.go | 42 ++++++++++ infrastructure/v1/servicemesh_types.go | 30 +++++++ infrastructure/v1/zz_generated.deepcopy.go | 37 +++++++++ pkg/cluster/cluster_config.go | 27 +++++++ pkg/deploy/setup.go | 14 ++++ pkg/feature/feature.go | 9 +++ pkg/feature/manifest.go | 28 +++++-- pkg/feature/serverless/loaders.go | 3 +- pkg/feature/serverless/resources.go | 21 ----- pkg/feature/servicemesh/cleanup.go | 60 ++++++++++++++ pkg/feature/servicemesh/conditions.go | 5 +- pkg/feature/servicemesh/loaders.go | 19 +++++ pkg/feature/servicemesh/resources.go | 30 +++++++ .../servicemesh/authorino/auth-smm.tmpl | 10 +++ .../base/operator-cluster-wide-no-tls.tmpl | 15 ++++ .../authorino/deployment.injection.patch.tmpl | 10 +++ .../mesh-authz-ext-provider.patch.tmpl | 13 +++ .../kserve/activator-envoyfilter.tmpl | 42 ++++++++++ .../kserve/envoy-oauth-temp-fix.tmpl | 79 +++++++++++++++++++ .../kserve/grpc-authorizationpolicy.tmpl | 19 +++++ 25 files changed, 645 insertions(+), 68 deletions(-) create mode 100644 pkg/cluster/cluster_config.go create mode 100644 pkg/feature/servicemesh/cleanup.go create mode 100644 pkg/feature/servicemesh/loaders.go create mode 100644 pkg/feature/servicemesh/resources.go create mode 100644 pkg/feature/templates/servicemesh/authorino/auth-smm.tmpl create mode 100644 pkg/feature/templates/servicemesh/authorino/base/operator-cluster-wide-no-tls.tmpl create mode 100644 pkg/feature/templates/servicemesh/authorino/deployment.injection.patch.tmpl create mode 100644 pkg/feature/templates/servicemesh/authorino/mesh-authz-ext-provider.patch.tmpl create mode 100644 pkg/feature/templates/servicemesh/kserve/activator-envoyfilter.tmpl create mode 100644 pkg/feature/templates/servicemesh/kserve/envoy-oauth-temp-fix.tmpl create mode 100644 pkg/feature/templates/servicemesh/kserve/grpc-authorizationpolicy.tmpl diff --git a/apis/dscinitialization/v1/zz_generated.deepcopy.go b/apis/dscinitialization/v1/zz_generated.deepcopy.go index d8b29a11839..0f216f26e65 100644 --- a/apis/dscinitialization/v1/zz_generated.deepcopy.go +++ b/apis/dscinitialization/v1/zz_generated.deepcopy.go @@ -32,7 +32,7 @@ func (in *DSCInitialization) DeepCopyInto(out *DSCInitialization) { *out = *in out.TypeMeta = in.TypeMeta in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) - out.Spec = in.Spec + in.Spec.DeepCopyInto(&out.Spec) in.Status.DeepCopyInto(&out.Status) } @@ -90,7 +90,7 @@ func (in *DSCInitializationList) DeepCopyObject() runtime.Object { func (in *DSCInitializationSpec) DeepCopyInto(out *DSCInitializationSpec) { *out = *in out.Monitoring = in.Monitoring - out.ServiceMesh = in.ServiceMesh + in.ServiceMesh.DeepCopyInto(&out.ServiceMesh) out.DevFlags = in.DevFlags } diff --git a/components/kserve/kserve.go b/components/kserve/kserve.go index fd9fd032cf9..7d6af54f482 100644 --- a/components/kserve/kserve.go +++ b/components/kserve/kserve.go @@ -3,6 +3,7 @@ package kserve import ( "fmt" + "path" "path/filepath" "strings" @@ -17,6 +18,7 @@ import ( "github.com/opendatahub-io/opendatahub-operator/v2/pkg/cluster" "github.com/opendatahub-io/opendatahub-operator/v2/pkg/deploy" "github.com/opendatahub-io/opendatahub-operator/v2/pkg/feature" + "github.com/opendatahub-io/opendatahub-operator/v2/pkg/feature/servicemesh" ) var ( @@ -148,6 +150,12 @@ func (k *Kserve) ReconcileComponent(cli client.Client, owner metav1.Object, dsci } } + if enabled { + if err := k.configureServiceMesh(cli, dscispec); err != nil { + return err + } + } + return nil } @@ -206,3 +214,49 @@ func checkRequiredOperatorsInstalled(cli client.Client) error { return multiErr.ErrorOrNil() } + +func (k *Kserve) configureServiceMesh(cli client.Client, dscispec *dsciv1.DSCInitializationSpec) error { + shouldConfigureServiceMesh, err := deploy.ShouldConfigureServiceMesh(cli, dscispec) + if err != nil { + return err + } + + if shouldConfigureServiceMesh { + serviceMeshInitializer := feature.NewFeaturesInitializer(dscispec, k.defineServiceMeshFeatures(dscispec)) + + if err := serviceMeshInitializer.Prepare(); err != nil { + return err + } + + if err := serviceMeshInitializer.Apply(); err != nil { + return err + } + } + + return nil +} + +func (k *Kserve) defineServiceMeshFeatures(dscispec *dsciv1.DSCInitializationSpec) feature.DefinedFeatures { + return func(s *feature.FeaturesInitializer) error { + var rootDir = filepath.Join(feature.BaseOutputDir, dscispec.ApplicationsNamespace) + if err := feature.CopyEmbeddedFiles("templates", rootDir); err != nil { + return err + } + + kserve, err := feature.CreateFeature("configure-kserve-for-external-authz"). + For(dscispec). + Manifests( + path.Join(rootDir, feature.KServeDir), + ). + WithData(servicemesh.ClusterDetails). + Load() + + if err != nil { + return err + } + + s.Features = append(s.Features, kserve) + + return nil + } +} diff --git a/config/crd/bases/dscinitialization.opendatahub.io_dscinitializations.yaml b/config/crd/bases/dscinitialization.opendatahub.io_dscinitializations.yaml index e805f082618..dbd912d8e3c 100644 --- a/config/crd/bases/dscinitialization.opendatahub.io_dscinitializations.yaml +++ b/config/crd/bases/dscinitialization.opendatahub.io_dscinitializations.yaml @@ -89,6 +89,48 @@ spec: user experience; e.g. it provides unified authentication giving a Single Sign On experience. properties: + auth: + description: Auth holds configuration of authentication and authorization + services used by Service Mesh in Opendatahub. + properties: + authorino: + description: Authorino holds configuration of Authorino service + used as external authorization provider. + properties: + audiences: + default: + - https://kubernetes.default.svc + description: Audiences is a list of the identifiers that + the resource server presented with the token identifies + as. Audience-aware token authenticators will verify + that the token was intended for at least one of the + audiences in this list. If no audiences are provided, + the audience will default to the audience of the Kubernetes + apiserver (kubernetes.default.svc). + items: + type: string + type: array + image: + default: quay.io/kuadrant/authorino:v0.16.0 + description: Image allows to define a custom container + image to be used when deploying Authorino's instance. + type: string + label: + default: authorino/topic=odh + description: Label narrows amount of AuthConfigs to process + by Authorino service. + type: string + name: + default: authorino-mesh-authz-provider + description: Name specifies how external authorization + provider should be called. + type: string + type: object + namespace: + default: auth-provider + description: Namespace where it is deployed. + type: string + type: object controlPlane: description: ControlPlane holds configuration of Service Mesh used by Opendatahub. diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 619225b7be3..3b484639fa8 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -115,6 +115,30 @@ rules: - statefulsets verbs: - '*' +- apiGroups: + - apps.openshift.io + resources: + - deploymentconfigs + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - apps.openshift.io + resources: + - deploymentconfigs/instantiate + verbs: + - create + - delete + - get + - list + - patch + - update + - watch - apiGroups: - argoproj.io resources: @@ -127,6 +151,12 @@ rules: - tokenreviews verbs: - create +- apiGroups: + - authorino.kuadrant.io + resources: + - authconfigs + verbs: + - '*' - apiGroups: - authorization.k8s.io resources: @@ -450,24 +480,7 @@ rules: resources: - secrets verbs: - - create - - delete - - list - - patch - - update - - watch -- apiGroups: - - "" - resources: - - secrets/finalizers - verbs: - - create - - delete - - get - - list - - patch - - update - - watch + - '*' - apiGroups: - "" resources: @@ -943,6 +956,12 @@ rules: - deletecollection - get - patch +- apiGroups: + - networking.istio.io + resources: + - envoyfilters + verbs: + - '*' - apiGroups: - networking.istio.io resources: @@ -1023,6 +1042,12 @@ rules: - patch - update - watch +- apiGroups: + - operator.authorino.kuadrant.io + resources: + - authorinos + verbs: + - '*' - apiGroups: - operator.knative.dev resources: @@ -1163,6 +1188,12 @@ rules: - patch - update - watch +- apiGroups: + - security.istio.io + resources: + - authorizationpolicies + verbs: + - '*' - apiGroups: - security.openshift.io resources: diff --git a/controllers/datasciencecluster/kubebuilder_rbac.go b/controllers/datasciencecluster/kubebuilder_rbac.go index 33d0184ec72..422cef469fb 100644 --- a/controllers/datasciencecluster/kubebuilder_rbac.go +++ b/controllers/datasciencecluster/kubebuilder_rbac.go @@ -4,14 +4,25 @@ package datasciencecluster //+kubebuilder:rbac:groups="datasciencecluster.opendatahub.io",resources=datascienceclusters/finalizers,verbs=update;patch //+kubebuilder:rbac:groups="datasciencecluster.opendatahub.io",resources=datascienceclusters,verbs=get;list;watch;create;update;patch;delete -/* Service Mesh prerequisite */ -// +kubebuilder:rbac:groups="maistra.io",resources=servicemeshcontrolplanes,verbs=create;get;list;patch;update;use;watch - /* Serverless prerequisite */ // +kubebuilder:rbac:groups="networking.istio.io",resources=gateways,verbs=* // +kubebuilder:rbac:groups="operator.knative.dev",resources=knativeservings,verbs=* // +kubebuilder:rbac:groups="config.openshift.io",resources=ingresses,verbs=get +/* Service Mesh Integration */ +// +kubebuilder:rbac:groups="maistra.io",resources=servicemeshcontrolplanes,verbs=create;get;list;patch;update;use;watch +// +kubebuilder:rbac:groups="maistra.io",resources=servicemeshmemberrolls,verbs=create;get;list;patch;update;use;watch +// +kubebuilder:rbac:groups="maistra.io",resources=servicemeshmembers,verbs=create;get;list;patch;update;use;watch +// +kubebuilder:rbac:groups="maistra.io",resources=servicemeshmembers/finalizers,verbs=create;get;list;patch;update;use;watch +// +kubebuilder:rbac:groups="networking.istio.io",resources=virtualservices/status,verbs=update;patch;delete +// +kubebuilder:rbac:groups="networking.istio.io",resources=virtualservices/finalizers,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups="networking.istio.io",resources=virtualservices,verbs=* +// +kubebuilder:rbac:groups="networking.istio.io",resources=gateways,verbs=* +// +kubebuilder:rbac:groups="networking.istio.io",resources=envoyfilters,verbs=* +// +kubebuilder:rbac:groups="security.istio.io",resources=authorizationpolicies,verbs=* +// +kubebuilder:rbac:groups="authorino.kuadrant.io",resources=authconfigs,verbs=* +// +kubebuilder:rbac:groups="operator.authorino.kuadrant.io",resources=authorinos,verbs=* + /* This is for DSP */ //+kubebuilder:rbac:groups="datasciencepipelinesapplications.opendatahub.io",resources=datasciencepipelinesapplications/status,verbs=update;patch;get //+kubebuilder:rbac:groups="datasciencepipelinesapplications.opendatahub.io",resources=datasciencepipelinesapplications/finalizers,verbs=update;patch @@ -93,10 +104,6 @@ package datasciencecluster // +kubebuilder:rbac:groups="networking.k8s.io",resources=networkpolicies,verbs=get;create;list;watch;delete;update;patch // +kubebuilder:rbac:groups="networking.k8s.io",resources=ingresses,verbs=create;delete;list;update;watch;patch;get -// +kubebuilder:rbac:groups="networking.istio.io",resources=virtualservices/status,verbs=update;patch;delete -// +kubebuilder:rbac:groups="networking.istio.io",resources=virtualservices/finalizers,verbs=get;list;watch;create;update;patch;delete -// +kubebuilder:rbac:groups="networking.istio.io",resources=virtualservices,verbs=* - // +kubebuilder:rbac:groups="monitoring.coreos.com",resources=servicemonitors,verbs=get;create;delete;update;watch;list;patch;deletecollection // +kubebuilder:rbac:groups="monitoring.coreos.com",resources=podmonitors,verbs=get;create;delete;update;watch;list;patch // +kubebuilder:rbac:groups="monitoring.coreos.com",resources=prometheusrules,verbs=get;create;patch;delete;deletecollection @@ -152,8 +159,7 @@ package datasciencecluster // +kubebuilder:rbac:groups="core",resources=serviceaccounts,verbs=get;list;watch;create;update;patch;delete -// +kubebuilder:rbac:groups="core",resources=secrets,verbs=create;delete;list;update;watch;patch -// +kubebuilder:rbac:groups="core",resources=secrets/finalizers,verbs=get;create;watch;update;patch;list;delete +// +kubebuilder:rbac:groups="core",resources=secrets,verbs=* // +kubebuilder:rbac:groups="core",resources=rhmis,verbs=watch;list @@ -187,7 +193,6 @@ package datasciencecluster // +kubebuilder:rbac:groups="cert-manager.io",resources=certificates;issuers,verbs=create;patch -// OpenVino still need buildconfig // +kubebuilder:rbac:groups="build.openshift.io",resources=builds,verbs=create;patch;delete;list;watch // +kubebuilder:rbac:groups="build.openshift.io",resources=buildconfigs/instantiate,verbs=create;patch;delete;get;list;watch // +kubebuilder:rbac:groups="build.openshift.io",resources=buildconfigs,verbs=list;watch;create;patch;delete @@ -218,6 +223,9 @@ package datasciencecluster // +kubebuilder:rbac:groups="*",resources=deployments,verbs=* // +kubebuilder:rbac:groups="extensions",resources=deployments,verbs=* +// +kubebuilder:rbac:groups="apps.openshift.io",resources=deploymentconfigs,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups="apps.openshift.io",resources=deploymentconfigs/instantiate,verbs=get;list;watch;create;update;patch;delete + // +kubebuilder:rbac:groups="apiextensions.k8s.io",resources=customresourcedefinitions,verbs=get;list;watch;create;patch;delete // +kubebuilder:rbac:groups="admissionregistration.k8s.io",resources=validatingwebhookconfigurations,verbs=get;list;watch;create;update;delete;patch @@ -232,11 +240,6 @@ package datasciencecluster // +kubebuilder:rbac:groups="*",resources=customresourcedefinitions,verbs=get;list;watch -// +kubebuilder:rbac:groups="maistra.io",resources=servicemeshcontrolplanes,verbs=create;get;list;patch;update;use;watch -// +kubebuilder:rbac:groups="maistra.io",resources=servicemeshmemberrolls,verbs=create;get;list;patch;update;use;watch -// +kubebuilder:rbac:groups="maistra.io",resources=servicemeshmembers,verbs=create;get;list;patch;update;use;watch -// +kubebuilder:rbac:groups="maistra.io",resources=servicemeshmembers/finalizers,verbs=create;get;list;patch;update;use;watch - /* Only for RHODS */ // +kubebuilder:rbac:groups="user.openshift.io",resources=groups,verbs=get;create;list;watch;patch;delete // +kubebuilder:rbac:groups="console.openshift.io",resources=consolelinks,verbs=create;get;patch;delete diff --git a/controllers/dscinitialization/servicemesh_setup.go b/controllers/dscinitialization/servicemesh_setup.go index 7bfca96a221..f05828213c8 100644 --- a/controllers/dscinitialization/servicemesh_setup.go +++ b/controllers/dscinitialization/servicemesh_setup.go @@ -100,5 +100,47 @@ func configureServiceMeshFeatures(s *feature.FeaturesInitializer) error { s.Features = append(s.Features, metricsCollection) } + if cfMaps, err := feature.CreateFeature("shared-config-maps"). + For(s.DSCInitializationSpec). + WithResources(servicemesh.ConfigMaps). + Load(); err != nil { + return err + } else { + s.Features = append(s.Features, cfMaps) + } + + if extAuthz, err := feature.CreateFeature("service-mesh-control-plane-setup-external-authorization"). + For(s.DSCInitializationSpec). + Manifests( + path.Join(rootDir, feature.AuthDir, "auth-smm.tmpl"), + path.Join(rootDir, feature.AuthDir, "base"), + //path.Join(rootDir, feature.AuthDir, "rbac"), + path.Join(rootDir, feature.AuthDir, "mesh-authz-ext-provider.patch.tmpl"), + ). + WithData(servicemesh.ClusterDetails). + PreConditions( + feature.EnsureCRDIsInstalled("authconfigs.authorino.kuadrant.io"), + servicemesh.EnsureServiceMeshInstalled, + feature.CreateNamespaceIfNotExists(serviceMeshSpec.Auth.Namespace), + ). + PostConditions( + feature.WaitForPodsToBeReady(serviceMeshSpec.ControlPlane.Namespace), + feature.WaitForPodsToBeReady(serviceMeshSpec.Auth.Namespace), + func(f *feature.Feature) error { + // We do not have the control over deployment resource creation. + // It is created by Authorino operator using Authorino CR + // + // To make it part of Service Mesh we have to patch it with injection + // enabled instead, otherwise it will not have proxy pod injected. + return f.ApplyManifest(path.Join(rootDir, feature.AuthDir, "deployment.injection.patch.tmpl")) + }, + ). + OnDelete(servicemesh.RemoveExtensionProvider). + Load(); err != nil { + return err + } else { + s.Features = append(s.Features, extAuthz) + } + return nil } diff --git a/infrastructure/v1/servicemesh_types.go b/infrastructure/v1/servicemesh_types.go index db4af44d712..b3499302624 100644 --- a/infrastructure/v1/servicemesh_types.go +++ b/infrastructure/v1/servicemesh_types.go @@ -9,6 +9,9 @@ type ServiceMeshSpec struct { ManagementState operatorv1.ManagementState `json:"managementState,omitempty"` // ControlPlane holds configuration of Service Mesh used by Opendatahub. ControlPlane ControlPlaneSpec `json:"controlPlane,omitempty"` + // Auth holds configuration of authentication and authorization services + // used by Service Mesh in Opendatahub. + Auth AuthSpec `json:"auth,omitempty"` } type ControlPlaneSpec struct { @@ -38,3 +41,30 @@ type IngressGatewaySpec struct { // the for Ingress Gateway. Certificate CertificateSpec `json:"certificate,omitempty"` } + +type AuthSpec struct { + // Namespace where it is deployed. + // +kubebuilder:default=auth-provider + Namespace string `json:"namespace,omitempty"` + // Authorino holds configuration of Authorino service used as external authorization provider. + Authorino AuthorinoSpec `json:"authorino,omitempty"` +} + +type AuthorinoSpec struct { + // Name specifies how external authorization provider should be called. + // +kubebuilder:default=authorino-mesh-authz-provider + Name string `json:"name,omitempty"` + // Audiences is a list of the identifiers that the resource server presented + // with the token identifies as. Audience-aware token authenticators will verify + // that the token was intended for at least one of the audiences in this list. + // If no audiences are provided, the audience will default to the audience of the + // Kubernetes apiserver (kubernetes.default.svc). + // +kubebuilder:default={"https://kubernetes.default.svc"} + Audiences []string `json:"audiences,omitempty"` + // Label narrows amount of AuthConfigs to process by Authorino service. + // +kubebuilder:default=authorino/topic=odh + Label string `json:"label,omitempty"` + // Image allows to define a custom container image to be used when deploying Authorino's instance. + // +kubebuilder:default="quay.io/kuadrant/authorino:v0.16.0" + Image string `json:"image,omitempty"` +} diff --git a/infrastructure/v1/zz_generated.deepcopy.go b/infrastructure/v1/zz_generated.deepcopy.go index efc062e396b..aa0c027157d 100644 --- a/infrastructure/v1/zz_generated.deepcopy.go +++ b/infrastructure/v1/zz_generated.deepcopy.go @@ -23,6 +23,42 @@ package v1 import () +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AuthSpec) DeepCopyInto(out *AuthSpec) { + *out = *in + in.Authorino.DeepCopyInto(&out.Authorino) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AuthSpec. +func (in *AuthSpec) DeepCopy() *AuthSpec { + if in == nil { + return nil + } + out := new(AuthSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AuthorinoSpec) DeepCopyInto(out *AuthorinoSpec) { + *out = *in + if in.Audiences != nil { + in, out := &in.Audiences, &out.Audiences + *out = make([]string, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AuthorinoSpec. +func (in *AuthorinoSpec) DeepCopy() *AuthorinoSpec { + if in == nil { + return nil + } + out := new(AuthorinoSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *CertificateSpec) DeepCopyInto(out *CertificateSpec) { *out = *in @@ -73,6 +109,7 @@ func (in *IngressGatewaySpec) DeepCopy() *IngressGatewaySpec { func (in *ServiceMeshSpec) DeepCopyInto(out *ServiceMeshSpec) { *out = *in out.ControlPlane = in.ControlPlane + in.Auth.DeepCopyInto(&out.Auth) } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ServiceMeshSpec. diff --git a/pkg/cluster/cluster_config.go b/pkg/cluster/cluster_config.go new file mode 100644 index 00000000000..9ff2b903e9c --- /dev/null +++ b/pkg/cluster/cluster_config.go @@ -0,0 +1,27 @@ +package cluster + +import ( + "context" + "errors" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/client-go/dynamic" + + "github.com/opendatahub-io/opendatahub-operator/v2/pkg/gvr" +) + +// +kubebuilder:rbac:groups="config.openshift.io",resources=ingresses,verbs=get + +func GetDomain(dynamicClient dynamic.Interface) (string, error) { + cluster, err := dynamicClient.Resource(gvr.OpenshiftIngress).Get(context.TODO(), "cluster", metav1.GetOptions{}) + if err != nil { + return "", err + } + + domain, found, err := unstructured.NestedString(cluster.Object, "spec", "domain") + if !found { + return "", errors.New("spec.domain not found") + } + return domain, err +} diff --git a/pkg/deploy/setup.go b/pkg/deploy/setup.go index dfcb4809164..7cc5e9462ca 100644 --- a/pkg/deploy/setup.go +++ b/pkg/deploy/setup.go @@ -4,10 +4,13 @@ import ( "context" "strings" + operatorv1 "github.com/openshift/api/operator/v1" ofapi "github.com/operator-framework/api/pkg/operators/v1alpha1" apiextv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" apierrs "k8s.io/apimachinery/pkg/api/errors" "sigs.k8s.io/controller-runtime/pkg/client" + + dsci "github.com/opendatahub-io/opendatahub-operator/v2/apis/dscinitialization/v1" ) const ( @@ -91,3 +94,14 @@ func GetPlatform(cli client.Client) (Platform, error) { // check and return whether ODH or self-managed platform return isSelfManaged(cli) } + +// ShouldConfigureServiceMesh determines if the operator should invoke service-mesh specific setup. +func ShouldConfigureServiceMesh(cli client.Client, dscispec *dsci.DSCInitializationSpec) (bool, error) { + platform, err := GetPlatform(cli) + if err != nil { + return false, err + } + + supportedPlatforms := platform == OpenDataHub || platform == Unknown + return dscispec.ServiceMesh.ManagementState == operatorv1.Managed && supportedPlatforms, nil +} diff --git a/pkg/feature/feature.go b/pkg/feature/feature.go index 397fda734a3..dcd95946116 100644 --- a/pkg/feature/feature.go +++ b/pkg/feature/feature.go @@ -157,6 +157,15 @@ func (f *Feature) addCleanup(cleanupFuncs ...Action) { f.cleanups = append(f.cleanups, cleanupFuncs...) } +func (f *Feature) ApplyManifest(filename string) error { + m := loadManifestFrom(filename) + if err := m.processTemplate(f.Spec); err != nil { + return err + } + + return f.apply(m) +} + type apply func(filename string) error func (f *Feature) apply(m manifest) error { diff --git a/pkg/feature/manifest.go b/pkg/feature/manifest.go index e741c65b307..7b97f7d1f14 100644 --- a/pkg/feature/manifest.go +++ b/pkg/feature/manifest.go @@ -10,7 +10,12 @@ import ( "github.com/pkg/errors" ) -const BaseOutputDir = "/tmp/odh-operator" +const ( + BaseDir = "templates/servicemesh/" + AuthDir = BaseDir + "authorino" + KServeDir = BaseDir + "kserve" + BaseOutputDir = "/tmp/opendatahub-manifests/" +) type manifest struct { name, @@ -29,13 +34,8 @@ func loadManifestsFrom(path string) ([]manifest, error) { if info.IsDir() { return nil } - basePath := filepath.Base(path) - manifests = append(manifests, manifest{ - name: basePath, - path: path, - patch: strings.Contains(basePath, ".patch"), - template: filepath.Ext(path) == ".tmpl", - }) + m := loadManifestFrom(path) + manifests = append(manifests, m) return nil }); err != nil { @@ -45,6 +45,18 @@ func loadManifestsFrom(path string) ([]manifest, error) { return manifests, nil } +func loadManifestFrom(path string) manifest { + basePath := filepath.Base(path) + m := manifest{ + name: basePath, + path: path, + patch: strings.Contains(basePath, ".patch"), + template: filepath.Ext(path) == ".tmpl", + } + + return m +} + func (m *manifest) targetPath() string { return fmt.Sprintf("%s%s", m.path[:len(m.path)-len(filepath.Ext(m.path))], ".yaml") } diff --git a/pkg/feature/serverless/loaders.go b/pkg/feature/serverless/loaders.go index 115034dbdcc..6973c2bcf73 100644 --- a/pkg/feature/serverless/loaders.go +++ b/pkg/feature/serverless/loaders.go @@ -2,6 +2,7 @@ package serverless import ( "fmt" + "github.com/opendatahub-io/opendatahub-operator/v2/pkg/cluster" "strings" "github.com/opendatahub-io/opendatahub-operator/v2/pkg/feature" @@ -23,7 +24,7 @@ func ServingIngressDomain(f *feature.Feature) error { domain := strings.TrimSpace(f.Spec.Serving.IngressGateway.Domain) if len(domain) == 0 { var errDomain error - domain, errDomain = GetDomain(f.DynamicClient) + domain, errDomain = cluster.GetDomain(f.DynamicClient) if errDomain != nil { return fmt.Errorf("failed to fetch OpenShift domain to generate certificate for Serverless: %w", errDomain) } diff --git a/pkg/feature/serverless/resources.go b/pkg/feature/serverless/resources.go index d06e4f1ae21..e3360ddef0c 100644 --- a/pkg/feature/serverless/resources.go +++ b/pkg/feature/serverless/resources.go @@ -1,30 +1,9 @@ package serverless import ( - "context" - - "github.com/pkg/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - "k8s.io/client-go/dynamic" - "github.com/opendatahub-io/opendatahub-operator/v2/pkg/feature" - "github.com/opendatahub-io/opendatahub-operator/v2/pkg/gvr" ) func ServingCertificateResource(f *feature.Feature) error { return f.CreateSelfSignedCertificate(f.Spec.KnativeCertificateSecret, f.Spec.Serving.IngressGateway.Certificate.Type, f.Spec.KnativeIngressDomain, f.Spec.ControlPlane.Namespace) } - -func GetDomain(dynamicClient dynamic.Interface) (string, error) { - cluster, err := dynamicClient.Resource(gvr.OpenshiftIngress).Get(context.TODO(), "cluster", metav1.GetOptions{}) - if err != nil { - return "", err - } - - domain, found, err := unstructured.NestedString(cluster.Object, "spec", "domain") - if !found { - return "", errors.New("spec.domain not found") - } - return domain, err -} diff --git a/pkg/feature/servicemesh/cleanup.go b/pkg/feature/servicemesh/cleanup.go new file mode 100644 index 00000000000..5aace52efee --- /dev/null +++ b/pkg/feature/servicemesh/cleanup.go @@ -0,0 +1,60 @@ +package servicemesh + +import ( + "context" + "fmt" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + ctrlLog "sigs.k8s.io/controller-runtime/pkg/log" + + "github.com/opendatahub-io/opendatahub-operator/v2/pkg/feature" + "github.com/opendatahub-io/opendatahub-operator/v2/pkg/gvr" +) + +var log = ctrlLog.Log.WithName("features") + +func RemoveExtensionProvider(f *feature.Feature) error { + ossmAuthzProvider := fmt.Sprintf("%s-odh-auth-provider", f.Spec.AppNamespace) + + mesh := f.Spec.ControlPlane + + smcp, err := f.DynamicClient.Resource(gvr.SMCP). + Namespace(mesh.Namespace). + Get(context.TODO(), mesh.Name, metav1.GetOptions{}) + if err != nil { + return err + } + + extensionProviders, found, err := unstructured.NestedSlice(smcp.Object, "spec", "techPreview", "meshConfig", "extensionProviders") + if err != nil { + return err + } + if !found { + log.Info("no extension providers found", "f", f.Name, "control-plane", mesh.Name, "namespace", mesh.Namespace) + return nil + } + + for i, v := range extensionProviders { + extensionProvider, ok := v.(map[string]interface{}) + if !ok { + fmt.Println("Unexpected type for extensionProvider") + continue + } + + if extensionProvider["name"] == ossmAuthzProvider { + extensionProviders = append(extensionProviders[:i], extensionProviders[i+1:]...) + err = unstructured.SetNestedSlice(smcp.Object, extensionProviders, "spec", "techPreview", "meshConfig", "extensionProviders") + if err != nil { + return err + } + break + } + } + + _, err = f.DynamicClient.Resource(gvr.SMCP). + Namespace(mesh.Namespace). + Update(context.TODO(), smcp, metav1.UpdateOptions{}) + + return err +} diff --git a/pkg/feature/servicemesh/conditions.go b/pkg/feature/servicemesh/conditions.go index da8e17e51e9..f59cda9435e 100644 --- a/pkg/feature/servicemesh/conditions.go +++ b/pkg/feature/servicemesh/conditions.go @@ -10,14 +10,11 @@ import ( "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/dynamic" - ctrlLog "sigs.k8s.io/controller-runtime/pkg/log" "github.com/opendatahub-io/opendatahub-operator/v2/pkg/feature" "github.com/opendatahub-io/opendatahub-operator/v2/pkg/gvr" ) -var log = ctrlLog.Log.WithName("features") - const ( interval = 2 * time.Second duration = 5 * time.Minute @@ -71,12 +68,14 @@ func CheckControlPlaneComponentReadiness(dynamicClient dynamic.Interface, smcp, unstructObj, err := dynamicClient.Resource(gvr.SMCP).Namespace(smcpNs).Get(context.TODO(), smcp, metav1.GetOptions{}) if err != nil { log.Info("failed to find Service Mesh Control Plane", "control-plane", smcp, "namespace", smcpNs) + return false, err } components, found, err := unstructured.NestedMap(unstructObj.Object, "status", "readiness", "components") if err != nil || !found { log.Info("status conditions not found or error in parsing of Service Mesh Control Plane") + return false, err } diff --git a/pkg/feature/servicemesh/loaders.go b/pkg/feature/servicemesh/loaders.go new file mode 100644 index 00000000000..959348836fe --- /dev/null +++ b/pkg/feature/servicemesh/loaders.go @@ -0,0 +1,19 @@ +package servicemesh + +import ( + "github.com/opendatahub-io/opendatahub-operator/v2/pkg/cluster" + "github.com/opendatahub-io/opendatahub-operator/v2/pkg/feature" + "github.com/pkg/errors" +) + +func ClusterDetails(f *feature.Feature) error { + data := f.Spec + + if domain, err := cluster.GetDomain(f.DynamicClient); err == nil { + data.Domain = domain + } else { + return errors.WithStack(err) + } + + return nil +} diff --git a/pkg/feature/servicemesh/resources.go b/pkg/feature/servicemesh/resources.go new file mode 100644 index 00000000000..d5282fb7480 --- /dev/null +++ b/pkg/feature/servicemesh/resources.go @@ -0,0 +1,30 @@ +package servicemesh + +import ( + "strings" + + "github.com/opendatahub-io/opendatahub-operator/v2/pkg/feature" + "github.com/pkg/errors" +) + +func ConfigMaps(feature *feature.Feature) error { + meshConfig := feature.Spec.ControlPlane + if err := feature.CreateConfigMap("service-mesh-refs", + map[string]string{ + "CONTROL_PLANE_NAME": meshConfig.Name, + "MESH_NAMESPACE": meshConfig.Namespace, + }); err != nil { + return errors.WithStack(err) + } + + authorinoConfig := feature.Spec.Auth.Authorino + if err := feature.CreateConfigMap("auth-refs", + map[string]string{ + "AUTHORINO_LABEL": authorinoConfig.Label, + "AUTH_AUDIENCE": strings.Join(authorinoConfig.Audiences, ","), + }); err != nil { + return errors.WithStack(err) + } + + return nil +} diff --git a/pkg/feature/templates/servicemesh/authorino/auth-smm.tmpl b/pkg/feature/templates/servicemesh/authorino/auth-smm.tmpl new file mode 100644 index 00000000000..6b0aa06aa82 --- /dev/null +++ b/pkg/feature/templates/servicemesh/authorino/auth-smm.tmpl @@ -0,0 +1,10 @@ +apiVersion: maistra.io/v1 +kind: ServiceMeshMember +metadata: + name: default + namespace: {{ .Auth.Namespace }} +spec: + controlPlaneRef: + namespace: {{ .ControlPlane.Namespace }} + name: {{ .ControlPlane.Name }} + diff --git a/pkg/feature/templates/servicemesh/authorino/base/operator-cluster-wide-no-tls.tmpl b/pkg/feature/templates/servicemesh/authorino/base/operator-cluster-wide-no-tls.tmpl new file mode 100644 index 00000000000..95b1a8fb63b --- /dev/null +++ b/pkg/feature/templates/servicemesh/authorino/base/operator-cluster-wide-no-tls.tmpl @@ -0,0 +1,15 @@ +apiVersion: operator.authorino.kuadrant.io/v1beta1 +kind: Authorino +metadata: + name: {{ .Auth.Authorino.Name }} + namespace: {{ .Auth.Namespace }} +spec: + image: {{ .Auth.Authorino.Image }} + authConfigLabelSelectors: {{ .Auth.Authorino.Label }} + clusterWide: true + listener: + tls: + enabled: false + oidcServer: + tls: + enabled: false diff --git a/pkg/feature/templates/servicemesh/authorino/deployment.injection.patch.tmpl b/pkg/feature/templates/servicemesh/authorino/deployment.injection.patch.tmpl new file mode 100644 index 00000000000..e15fb31100f --- /dev/null +++ b/pkg/feature/templates/servicemesh/authorino/deployment.injection.patch.tmpl @@ -0,0 +1,10 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ .Auth.Authorino.Name }} + namespace: {{ .Auth.Namespace }} +spec: + template: + metadata: + annotations: + sidecar.istio.io/inject: "true" diff --git a/pkg/feature/templates/servicemesh/authorino/mesh-authz-ext-provider.patch.tmpl b/pkg/feature/templates/servicemesh/authorino/mesh-authz-ext-provider.patch.tmpl new file mode 100644 index 00000000000..706c995dc4c --- /dev/null +++ b/pkg/feature/templates/servicemesh/authorino/mesh-authz-ext-provider.patch.tmpl @@ -0,0 +1,13 @@ +apiVersion: maistra.io/v2 +kind: ServiceMeshControlPlane +metadata: + name: {{ .ControlPlane.Name }} + namespace: {{ .ControlPlane.Namespace }} +spec: + techPreview: + meshConfig: + extensionProviders: + - name: {{ .AppNamespace }}-odh-auth-provider + envoyExtAuthzGrpc: + service: {{ .Auth.Authorino.Name }}-authorino-authorization.{{ .Auth.Namespace }}.svc.cluster.local + port: 50051 diff --git a/pkg/feature/templates/servicemesh/kserve/activator-envoyfilter.tmpl b/pkg/feature/templates/servicemesh/kserve/activator-envoyfilter.tmpl new file mode 100644 index 00000000000..5a6286b3d0a --- /dev/null +++ b/pkg/feature/templates/servicemesh/kserve/activator-envoyfilter.tmpl @@ -0,0 +1,42 @@ +apiVersion: networking.istio.io/v1alpha3 +kind: EnvoyFilter +metadata: + labels: + app: odh + name: activator-host-header + namespace: {{ .ControlPlane.Namespace }} +spec: + priority: 20 + workloadSelector: + labels: + component: predictor + configPatches: + - applyTo: HTTP_FILTER + match: + listener: + filterChain: + filter: + name: envoy.filters.network.http_connection_manager + patch: + operation: INSERT_BEFORE + value: + name: envoy.filters.http.lua + typed_config: + '@type': type.googleapis.com/envoy.extensions.filters.http.lua.v3.Lua + inlineCode: | + function envoy_on_request(request_handle) + local headers = request_handle:headers() + if not headers then + return + end + + local original_host = headers:get("k-original-host") + if original_host then + + port_seperator = string.find(original_host, ":", 7) + if port_seperator then + original_host = string.sub(original_host, 0, port_seperator-1) + end + headers:replace('host', original_host) + end + end diff --git a/pkg/feature/templates/servicemesh/kserve/envoy-oauth-temp-fix.tmpl b/pkg/feature/templates/servicemesh/kserve/envoy-oauth-temp-fix.tmpl new file mode 100644 index 00000000000..ee0678003dd --- /dev/null +++ b/pkg/feature/templates/servicemesh/kserve/envoy-oauth-temp-fix.tmpl @@ -0,0 +1,79 @@ +# https://issues.redhat.com/browse/OSSM-4873 +apiVersion: networking.istio.io/v1alpha3 +kind: EnvoyFilter +metadata: + name: envoy-oauth-temp-fix-before + namespace: {{ .ControlPlane.Namespace }} + labels: + app: odh + temp: hack +spec: + workloadSelector: + labels: + istio: ingressgateway + priority: 20 + configPatches: + - applyTo: HTTP_FILTER + match: + listener: + filterChain: + filter: + name: envoy.filters.network.http_connection_manager + patch: + operation: INSERT_BEFORE + value: + name: envoy.filters.http.lua + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.lua.v3.Lua + inlineCode: | + function envoy_on_request(request_handle) + local headers = request_handle:headers() + if not headers then + return + end + + local auth = headers:get("authorization") + if auth then + headers:replace("x-authorization", auth) + end + end +--- +apiVersion: networking.istio.io/v1alpha3 +kind: EnvoyFilter +metadata: + name: envoy-oauth-temp-fix-after + namespace: {{ .ControlPlane.Namespace }} + labels: + app: odh + temp: hack +spec: + workloadSelector: + labels: + istio: ingressgateway + priority: 5 + configPatches: + - applyTo: HTTP_FILTER + match: + listener: + filterChain: + filter: + name: envoy.filters.network.http_connection_manager + patch: + operation: INSERT_BEFORE + value: + name: envoy.filters.http.lua + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.lua.v3.Lua + inlineCode: | + function envoy_on_request(request_handle) + local headers = request_handle:headers() + if not headers then + return + end + + local xauth = headers:get("x-authorization") + if xauth then + headers:replace("authorization", xauth) + headers.remove("x-authorization") + end + end diff --git a/pkg/feature/templates/servicemesh/kserve/grpc-authorizationpolicy.tmpl b/pkg/feature/templates/servicemesh/kserve/grpc-authorizationpolicy.tmpl new file mode 100644 index 00000000000..e0b57df4a56 --- /dev/null +++ b/pkg/feature/templates/servicemesh/kserve/grpc-authorizationpolicy.tmpl @@ -0,0 +1,19 @@ +apiVersion: security.istio.io/v1beta1 +kind: AuthorizationPolicy +metadata: + name: kserve-predicator + namespace: {{ .ControlPlane.Namespace }} +spec: + action: CUSTOM + provider: + name: opendatahub-odh-auth-provider + rules: + - to: + - operation: + notPaths: + - /healthz* + ports: + - "8013" + selector: + matchLabels: + component: predictor