Skip to content

Commit

Permalink
Add Trusted Bundle Configmap to all namespaces (opendatahub-io#848)
Browse files Browse the repository at this point in the history
* Add odh-trusted-ca-bundle CM to DSCInitialization

(cherry picked from commit 5d415c2)

* Update ca bundle configMap to use dscInit TrustedCABundle for the default value

(cherry picked from commit 9a94ef6)

* Add Trusted Bundle Configmap to all namespaces

* Add ManagementState to TrustedCABundle

Co-authored-by: Wen Zhou <wenzhou@redhat.com>

fix

---------

Co-authored-by: Landon LaSmith <LLaSmith@redhat.com>
(cherry picked from commit d998839)
  • Loading branch information
VaishnaviHire committed Feb 22, 2024
1 parent d93a2dd commit 1935074
Show file tree
Hide file tree
Showing 13 changed files with 531 additions and 12 deletions.
20 changes: 19 additions & 1 deletion apis/dscinitialization/v1/dscinitialization_types.go
Original file line number Diff line number Diff line change
Expand Up @@ -45,9 +45,15 @@ type DSCInitializationSpec struct {
// +operator-sdk:csv:customresourcedefinitions:type=spec,order=3
// +optional
ServiceMesh infrav1.ServiceMeshSpec `json:"serviceMesh,omitempty"`
// When set to `Managed`, adds odh-trusted-ca-bundle Configmap to all namespaces that includes
// cluster-wide Trusted CA Bundle in .data["ca-bundle.crt"].
// Additionally, this fields allows admins to add custom CA bundles to the configmap using the .CustomCABundle field.
// +operator-sdk:csv:customresourcedefinitions:type=spec,order=4
// +optional
TrustedCABundle TrustedCABundleSpec `json:"trustedCABundle,omitempty"`
// Internal development useful field to test customizations.
// This is not recommended to be used in production environment.
// +operator-sdk:csv:customresourcedefinitions:type=spec,order=4
// +operator-sdk:csv:customresourcedefinitions:type=spec,order=5
// +optional
DevFlags *DevFlags `json:"devFlags,omitempty"`
}
Expand All @@ -73,6 +79,18 @@ type DevFlags struct {
ManifestsUri string `json:"manifestsUri,omitempty"`
}

type TrustedCABundleSpec struct {
// managementState indicates whether and how the operator should manage customized CA bundle
// +kubebuilder:validation:Enum=Managed;Removed;Unmanaged
// +kubebuilder:default=Removed
ManagementState operatorv1.ManagementState `json:"managementState"`
// A custom CA bundle that will be available for all components in the
// Data Science Cluster(DSC). This bundle will be stored in odh-trusted-ca-bundle
// ConfigMap .data.odh-ca-bundle.crt .
// +kubebuilder:default=""
CustomCABundle string `json:"customCABundle"`
}

// DSCInitializationStatus defines the observed state of DSCInitialization.
type DSCInitializationStatus struct {
// Phase describes the Phase of DSCInitializationStatus
Expand Down
16 changes: 16 additions & 0 deletions apis/dscinitialization/v1/zz_generated.deepcopy.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Original file line number Diff line number Diff line change
Expand Up @@ -123,6 +123,31 @@ spec:
pattern: ^(Managed|Unmanaged|Force|Removed)$
type: string
type: object
trustedCABundle:
description: When set to `Managed`, adds odh-trusted-ca-bundle Configmap
to all namespaces that includes cluster-wide Trusted CA Bundle in
.data["ca-bundle.crt"]. Additionally, this fields allows admins
to add custom CA bundles to the configmap using the .CustomCABundle
field.
properties:
customCABundle:
description: A custom CA bundle that will be available for all components
in the Data Science Cluster(DSC). This bundle will be stored
in odh-trusted-ca-bundle ConfigMap .data.odh-ca-bundle.crt .
type: string
managementState:
default: Removed
description: managementState indicates whether and how the operator
should manage customized CA bundle
enum:
- Managed
- Removed
- Unmanaged
pattern: ^(Managed|Unmanaged|Force|Removed)$
type: string
required:
- managementState
type: object
required:
- applicationsNamespace
type: object
Expand Down
10 changes: 10 additions & 0 deletions bundle/manifests/rhods-operator.clusterserviceversion.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,10 @@ metadata:
"namespace": "istio-system"
},
"managementState": "Managed"
},
"trustedCABundle": {
"customCABundle": "",
"managementState": "Managed"
}
}
},
Expand Down Expand Up @@ -201,6 +205,12 @@ spec:
e.g. it provides unified authentication giving a Single Sign On experience.
displayName: Service Mesh
path: serviceMesh
- description: When set to `Managed`, adds odh-trusted-ca-bundle Configmap to
all namespaces that includes cluster-wide Trusted CA Bundle in .data["ca-bundle.crt"].
Additionally, this fields allows admins to add custom CA bundles to the
configmap using the .CustomCABundle field.
displayName: Trusted CABundle
path: trustedCABundle
- description: Internal development useful field to test customizations. This
is not recommended to be used in production environment.
displayName: Dev Flags
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -124,6 +124,31 @@ spec:
pattern: ^(Managed|Unmanaged|Force|Removed)$
type: string
type: object
trustedCABundle:
description: When set to `Managed`, adds odh-trusted-ca-bundle Configmap
to all namespaces that includes cluster-wide Trusted CA Bundle in
.data["ca-bundle.crt"]. Additionally, this fields allows admins
to add custom CA bundles to the configmap using the .CustomCABundle
field.
properties:
customCABundle:
description: A custom CA bundle that will be available for all components
in the Data Science Cluster(DSC). This bundle will be stored
in odh-trusted-ca-bundle ConfigMap .data.odh-ca-bundle.crt .
type: string
managementState:
default: Removed
description: managementState indicates whether and how the operator
should manage customized CA bundle
enum:
- Managed
- Removed
- Unmanaged
pattern: ^(Managed|Unmanaged|Force|Removed)$
type: string
required:
- managementState
type: object
required:
- applicationsNamespace
type: object
Expand Down
5 changes: 4 additions & 1 deletion config/samples/dscinitialization_v1_dscinitialization.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -18,5 +18,8 @@ spec:
metricsCollection: Istio
name: data-science-smcp
namespace: istio-system
managementState: Managed
managementState: "Managed"
trustedCABundle:
managementState: "Managed"
customCABundle: ""

Original file line number Diff line number Diff line change
@@ -0,0 +1,158 @@
// Package certconfigmapgenerator contains generator logic of add cert configmap resource in user namespaces
package certconfigmapgenerator

import (
"context"
"reflect"

"github.com/go-logr/logr"
operatorv1 "github.com/openshift/api/operator/v1"
"github.com/pkg/errors"
corev1 "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/types"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/builder"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/event"
"sigs.k8s.io/controller-runtime/pkg/handler"
"sigs.k8s.io/controller-runtime/pkg/log"
"sigs.k8s.io/controller-runtime/pkg/predicate"
"sigs.k8s.io/controller-runtime/pkg/reconcile"
"sigs.k8s.io/controller-runtime/pkg/source"

dsci "github.com/opendatahub-io/opendatahub-operator/v2/apis/dscinitialization/v1"
"github.com/opendatahub-io/opendatahub-operator/v2/pkg/trustedcabundle"
)

var configmapGenLog = log.Log.WithName("cert-configmap-generator")

// CertConfigmapGeneratorReconciler holds the controller configuration.
type CertConfigmapGeneratorReconciler struct { //nolint:golint,revive // Readability
Client client.Client
Scheme *runtime.Scheme
Log logr.Logger
}

// SetupWithManager sets up the controller with the Manager.
func (r *CertConfigmapGeneratorReconciler) SetupWithManager(mgr ctrl.Manager) error {
configmapGenLog.Info("Adding controller for Configmap Generation.")
return ctrl.NewControllerManagedBy(mgr).
Named("cert-configmap-generator-controller").
Watches(&source.Kind{Type: &corev1.ConfigMap{}}, handler.EnqueueRequestsFromMapFunc(r.watchTrustedCABundleConfigMapResource), builder.WithPredicates(ConfigMapChangedPredicate)).
Watches(&source.Kind{Type: &corev1.Namespace{}}, handler.EnqueueRequestsFromMapFunc(r.watchNamespaceResource), builder.WithPredicates(NamespaceCreatedPredicate)).
Complete(r)
}

// Reconcile will generate new configmap, odh-trusted-ca-bundle, that includes cluster-wide trusted-ca bundle and custom
// ca bundle in every new namespace created.
func (r *CertConfigmapGeneratorReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
// Request includes namespace that is newly created or where odh-trusted-ca-bundle configmap is updated.
r.Log.Info("Reconciling certConfigMapGenerator.", " CertConfigMapGenerator Request.Namespace", req.NamespacedName)
// Get namespace instance
userNamespace := &corev1.Namespace{}
err := r.Client.Get(ctx, client.ObjectKey{Name: req.Namespace}, userNamespace)
if err != nil {
return ctrl.Result{}, errors.WithMessage(err, "error getting user namespace to inject trustedCA bundle ")
}

// Get DSCI instance
dsciInstances := &dsci.DSCInitializationList{}
err = r.Client.List(ctx, dsciInstances)
if err != nil {
r.Log.Error(err, "Failed to retrieve DSCInitialization resource for certconfigmapgenerator ", "CertConfigmapGenerator Request.Name", req.Name)
return ctrl.Result{}, err
}

var dsciInstance *dsci.DSCInitialization
switch len(dsciInstances.Items) {
case 0:
return ctrl.Result{}, nil
case 1:
dsciInstance = &dsciInstances.Items[0]
default:
message := "only one instance of DSCInitialization object is allowed"
return ctrl.Result{}, errors.New(message)
}

if dsciInstance.Spec.TrustedCABundle.ManagementState != operatorv1.Managed {
return ctrl.Result{}, nil
}

// Verify if namespace did not opt out of trustedCABundle injection
if trustedcabundle.HasCABundleAnnotationDisabled(userNamespace) {
r.Log.Info("Namespace has opted-out of CA bundle injection using annotation ", "namespace", userNamespace.Name,
"annotation", trustedcabundle.InjectionOfCABundleAnnotatoion)
err := trustedcabundle.DeleteOdhTrustedCABundleConfigMap(ctx, r.Client, req.Namespace)
if err != nil {
if !apierrors.IsNotFound(err) {
r.Log.Error(err, "error deleting existing configmap from namespace", "name", trustedcabundle.CAConfigMapName, "namespace", req.Namespace)
return reconcile.Result{}, err
}
}
return reconcile.Result{}, nil
}

// Verify odh-trusted-ca-bundle Configmap is created for the given namespace
if trustedcabundle.ShouldInjectTrustedBundle(userNamespace) {
r.Log.Info("Adding trusted CA bundle configmap to the new or existing namespace ", "namespace", userNamespace.Name,
"configmap", trustedcabundle.CAConfigMapName)
trustCAData := dsciInstance.Spec.TrustedCABundle.CustomCABundle
err = trustedcabundle.CreateOdhTrustedCABundleConfigMap(ctx, r.Client, req.Namespace, trustCAData)
if err != nil {
r.Log.Error(err, "error adding configmap to namespace", "name", trustedcabundle.CAConfigMapName, "namespace", req.Namespace)
return reconcile.Result{}, err
}
}
return ctrl.Result{}, err
}

func (r *CertConfigmapGeneratorReconciler) watchNamespaceResource(a client.Object) []reconcile.Request {
if trustedcabundle.ShouldInjectTrustedBundle(a) {
return []reconcile.Request{{NamespacedName: types.NamespacedName{Name: trustedcabundle.CAConfigMapName, Namespace: a.GetName()}}}
}
return nil
}

func (r *CertConfigmapGeneratorReconciler) watchTrustedCABundleConfigMapResource(a client.Object) []reconcile.Request {
if a.GetName() == trustedcabundle.CAConfigMapName {
r.Log.Info("Cert configmap has been updated, start reconcile")
return []reconcile.Request{{NamespacedName: types.NamespacedName{Name: a.GetName(), Namespace: a.GetNamespace()}}}
}
return nil
}

var NamespaceCreatedPredicate = predicate.Funcs{
CreateFunc: func(e event.CreateEvent) bool {
return trustedcabundle.ShouldInjectTrustedBundle(e.Object)
},

// If user changes the annotation of namespace to opt out of CABundle injection, reconcile.
UpdateFunc: func(e event.UpdateEvent) bool {
oldNamespace, _ := e.ObjectOld.(*corev1.Namespace)
newNamespace, _ := e.ObjectNew.(*corev1.Namespace)

oldNsAnnValue, oldNsAnnExists := oldNamespace.GetAnnotations()[trustedcabundle.InjectionOfCABundleAnnotatoion]
newNsAnnValue, newNsAnnExists := newNamespace.GetAnnotations()[trustedcabundle.InjectionOfCABundleAnnotatoion]

if newNsAnnExists && !oldNsAnnExists {
return true
} else if newNsAnnExists && oldNsAnnExists && oldNsAnnValue != newNsAnnValue {
return true
}
return false
},
}

var ConfigMapChangedPredicate = predicate.Funcs{
UpdateFunc: func(e event.UpdateEvent) bool {
oldCM, _ := e.ObjectOld.(*corev1.ConfigMap)
newCM, _ := e.ObjectNew.(*corev1.ConfigMap)
return !reflect.DeepEqual(oldCM.Data, newCM.Data)
},

DeleteFunc: func(deleteEvent event.DeleteEvent) bool {
return true
},
}
35 changes: 34 additions & 1 deletion controllers/dscinitialization/dscinitialization_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -49,13 +49,18 @@ import (
dsciv1 "github.com/opendatahub-io/opendatahub-operator/v2/apis/dscinitialization/v1"
"github.com/opendatahub-io/opendatahub-operator/v2/controllers/status"
"github.com/opendatahub-io/opendatahub-operator/v2/pkg/deploy"
"github.com/opendatahub-io/opendatahub-operator/v2/pkg/trustedcabundle"
"github.com/opendatahub-io/opendatahub-operator/v2/pkg/upgrade"
)

const (
finalizerName = "dscinitialization.opendatahub.io/finalizer"
)

// This ar is required by the .spec.TrustedCABundle field. When a user goes from Unmanaged to Managed, update all
// namespaces irrespective of any changes in the configmap.
var managementStateChangeTrustedCA = false

// DSCInitializationReconciler reconciles a DSCInitialization object.
type DSCInitializationReconciler struct { //nolint:golint,revive // Readability
client.Client
Expand Down Expand Up @@ -153,6 +158,21 @@ func (r *DSCInitializationReconciler) Reconcile(ctx context.Context, req ctrl.Re
}
}

// Check namespace
namespace := instance.Spec.ApplicationsNamespace
err = r.createOdhNamespace(ctx, instance, namespace)
if err != nil {
// no need to log error as it was already logged in createOdhNamespace
return reconcile.Result{}, err
}

// Verify odh-trusted-ca-bundle Configmap is configured for the namespaces
err = trustedcabundle.ConfigureTrustedCABundle(ctx, r.Client, r.Log, instance, managementStateChangeTrustedCA)
if err != nil {
return reconcile.Result{}, err
}
managementStateChangeTrustedCA = false

// Get platform
platform, err := deploy.GetPlatform(r.Client)
if err != nil {
Expand Down Expand Up @@ -279,7 +299,8 @@ func (r *DSCInitializationReconciler) SetupWithManager(mgr ctrl.Manager) error {
return ctrl.NewControllerManagedBy(mgr).
// add predicates prevents meaningless reconciliations from being triggered
// not use WithEventFilter() because it conflict with secret and configmap predicate
For(&dsciv1.DSCInitialization{}, builder.WithPredicates(predicate.Or(predicate.GenerationChangedPredicate{}, predicate.LabelChangedPredicate{}))).
For(&dsciv1.DSCInitialization{}, builder.WithPredicates(predicate.Or(predicate.GenerationChangedPredicate{},
predicate.LabelChangedPredicate{}), dsciPredicateStateChangeTrustedCA)).
Owns(&corev1.Namespace{}, builder.WithPredicates(predicate.Or(predicate.GenerationChangedPredicate{}, predicate.LabelChangedPredicate{}))).
Owns(&corev1.Secret{}, builder.WithPredicates(predicate.Or(predicate.GenerationChangedPredicate{}, predicate.LabelChangedPredicate{}))).
Owns(&corev1.ConfigMap{}, builder.WithPredicates(predicate.Or(predicate.GenerationChangedPredicate{}, predicate.LabelChangedPredicate{}))).
Expand Down Expand Up @@ -339,6 +360,18 @@ var DSCDeletionPredicate = predicate.Funcs{
},
}

var dsciPredicateStateChangeTrustedCA = predicate.Funcs{
UpdateFunc: func(e event.UpdateEvent) bool {
oldDSCI, _ := e.ObjectOld.(*dsciv1.DSCInitialization)
newDSCI, _ := e.ObjectNew.(*dsciv1.DSCInitialization)

if oldDSCI.Spec.TrustedCABundle.ManagementState != newDSCI.Spec.TrustedCABundle.ManagementState {
managementStateChangeTrustedCA = true
}
return true
},
}

func (r *DSCInitializationReconciler) watchMonitoringConfigMapResource(a client.Object) []reconcile.Request {
if a.GetName() == "prometheus" && a.GetNamespace() == "redhat-ods-monitoring" {
r.Log.Info("Found monitoring configmap has updated, start reconcile")
Expand Down
Loading

0 comments on commit 1935074

Please sign in to comment.