feat: Clusterrole and Webhook update for RAG Engine (#619)

**Reason for Change**:
Clusterrole and Webhook update for RAG Engine

**Requirements**

- [ ] added unit tests and e2e tests (if applicable).

**Issue Fixed**:
<!-- If this PR fixes GitHub issue 4321, add "Fixes #4321" to the next
line. -->

**Notes for Reviewers**:

Signed-off-by: Bangqi Zhu <bangqizhu@microsoft.com>
Co-authored-by: Bangqi Zhu <bangqizhu@microsoft.com>

api/v1alpha1/ragengine_default.go

@@ -0,0 +1,12 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT license.
+
+package v1alpha1
+
+import (
+	"context"
+)
+
+// SetDefaults for the RAG Engine
+func (w *RAGEngine) SetDefaults(_ context.Context) {
+}

api/v1alpha1/ragengine_validation.go

@@ -0,0 +1,36 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT license.
+
+package v1alpha1
+
+import (
+	"context"
+	"fmt"
+
+	admissionregistrationv1 "k8s.io/api/admissionregistration/v1"
+	"k8s.io/klog/v2"
+	"knative.dev/pkg/apis"
+)
+
+func (w *RAGEngine) SupportedVerbs() []admissionregistrationv1.OperationType {
+	return []admissionregistrationv1.OperationType{
+		admissionregistrationv1.Create,
+		admissionregistrationv1.Update,
+	}
+}
+
+func (w *RAGEngine) Validate(ctx context.Context) (errs *apis.FieldError) {
+	base := apis.GetBaseline(ctx)
+	if base == nil {
+		klog.InfoS("Validate creation", "ragengine", fmt.Sprintf("%s/%s", w.Namespace, w.Name))
+		errs = errs.Also(w.validateCreate().ViaField("spec"))
+	}
+	return errs
+}
+
+func (w *RAGEngine) validateCreate() (errs *apis.FieldError) {
+	if w.Spec.InferenceService == nil {
+		errs = errs.Also(apis.ErrGeneric("InferenceService must be specified", ""))
+	}
+	return errs
+}

charts/kaito/ragengine/templates/clusterrole.yaml

@@ -12,9 +12,15 @@ rules:
   - apiGroups: ["kaito.sh"]
     resources: ["ragengines/status"]
     verbs: ["update", "patch","get","list","watch"]
+  - apiGroups: [""]
+    resources: ["nodes", "namespaces"]
+    verbs: ["get","list","watch","update", "patch"]
   - apiGroups: [""]
     resources: ["services"]
     verbs: ["get","list","watch","create", "delete", "update", "patch"]
+  - apiGroups: [ "" ]
+    resources: [ "pods"]
+    verbs: ["get","list","watch","create", "update", "patch" ]
   - apiGroups: [ "" ]
     resources: [ "configmaps" ]
     verbs: [ "get","list","watch","create", "delete" ]
@@ -24,6 +30,9 @@ rules:
   - apiGroups: [ "apps" ]
     resources: ["controllerrevisions" ]
     verbs: [ "get","list","watch","create", "delete","update", "patch"]
+  - apiGroups: ["karpenter.sh"]
+    resources: ["machines", "machines/status", "nodeclaims", "nodeclaims/status"]
+    verbs: ["get","list","watch","create", "delete", "update", "patch"]
   - apiGroups: ["admissionregistration.k8s.io"]
     resources: ["validatingwebhookconfigurations"]
     verbs: ["get","list","watch"]

cmd/ragengine/main.go

@@ -11,11 +11,15 @@ import (
 	"syscall"
 	"time"
 
+	azurev1alpha2 "github.com/Azure/karpenter-provider-azure/pkg/apis/v1alpha2"
+	"github.com/aws/karpenter-core/pkg/apis/v1alpha5"
+	awsv1beta1 "github.com/aws/karpenter-provider-aws/pkg/apis/v1beta1"
 	"github.com/azure/kaito/pkg/k8sclient"
 	metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
 
 	"github.com/azure/kaito/pkg/controllers"
 	"github.com/azure/kaito/pkg/webhooks"
+	"k8s.io/api/apps/v1beta1"
 	"k8s.io/klog/v2"
 	"knative.dev/pkg/injection/sharedmain"
 	"knative.dev/pkg/webhook"
@@ -53,6 +57,10 @@ var (
 func init() {
 	utilruntime.Must(clientgoscheme.AddToScheme(scheme))
 	utilruntime.Must(kaitov1alpha1.AddToScheme(scheme))
+	utilruntime.Must(v1alpha5.SchemeBuilder.AddToScheme(scheme))
+	utilruntime.Must(v1beta1.SchemeBuilder.AddToScheme(scheme))
+	utilruntime.Must(azurev1alpha2.SchemeBuilder.AddToScheme(scheme))
+	utilruntime.Must(awsv1beta1.SchemeBuilder.AddToScheme(scheme))
 
 	//+kubebuilder:scaffold:scheme
 	klog.InitFlags(nil)
@@ -63,13 +71,15 @@ func main() {
 	var enableLeaderElection bool
 	var enableWebhook bool
 	var probeAddr string
+	var featureGates string
 	flag.StringVar(&metricsAddr, "metrics-bind-address", ":8080", "The address the metric endpoint binds to.")
 	flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.")
 	flag.BoolVar(&enableLeaderElection, "leader-elect", false,
 		"Enable leader election for controller manager. "+
 			"Enabling this will ensure there is only one active controller manager.")
-	flag.BoolVar(&enableWebhook, "webhook", false,
-		"Enable webhook for controller manager. Default is false.")
+	flag.BoolVar(&enableWebhook, "webhook", true,
+		"Enable webhook for controller manager. Default is true.")
+	flag.StringVar(&featureGates, "feature-gates", "Karpenter=false", "Enable Kaito feature gates. Default,	Karpenter=false.")
 	opts := zap.Options{
 		Development: true,
 	}
@@ -116,7 +126,7 @@ func main() {
 	)
 
 	if err = ragengineReconciler.SetupWithManager(mgr); err != nil {
-		klog.ErrorS(err, "unable to create controller", "controller", "Workspace")
+		klog.ErrorS(err, "unable to create controller", "controller", "RAG Eingine")
 		exitWithErrorFunc()
 	}
 	//+kubebuilder:scaffold:builder
@@ -144,7 +154,7 @@ func main() {
 		})
 		ctx = sharedmain.WithHealthProbesDisabled(ctx)
 		ctx = sharedmain.WithHADisabled(ctx)
-		go sharedmain.MainWithConfig(ctx, "webhook", ctrl.GetConfigOrDie(), webhooks.NewWebhooks()...)
+		go sharedmain.MainWithConfig(ctx, "webhook", ctrl.GetConfigOrDie(), webhooks.NewRAGEngineWebhooks()...)
 
 		// wait 2 seconds to allow reconciling webhookconfiguration and service endpoint.
 		time.Sleep(2 * time.Second)

cmd/workspace/main.go

@@ -157,7 +157,7 @@ func main() {
 		})
 		ctx = sharedmain.WithHealthProbesDisabled(ctx)
 		ctx = sharedmain.WithHADisabled(ctx)
-		go sharedmain.MainWithConfig(ctx, "webhook", ctrl.GetConfigOrDie(), webhooks.NewWebhooks()...)
+		go sharedmain.MainWithConfig(ctx, "webhook", ctrl.GetConfigOrDie(), webhooks.NewWorkspaceWebhooks()...)
 
 		// wait 2 seconds to allow reconciling webhookconfiguration and service endpoint.
 		time.Sleep(2 * time.Second)

pkg/webhooks/webhooks.go

@@ -16,23 +16,44 @@ import (
 	kaitov1alpha1 "github.com/azure/kaito/api/v1alpha1"
 )
 
-func NewWebhooks() []knativeinjection.ControllerConstructor {
+func NewWorkspaceWebhooks() []knativeinjection.ControllerConstructor {
 	return []knativeinjection.ControllerConstructor{
 		certificates.NewController,
-		NewCRDValidationWebhook,
+		NewWorkspaceCRDValidationWebhook,
 	}
 }
 
-func NewCRDValidationWebhook(ctx context.Context, _ configmap.Watcher) *controller.Impl {
+func NewWorkspaceCRDValidationWebhook(ctx context.Context, _ configmap.Watcher) *controller.Impl {
 	return validation.NewAdmissionController(ctx,
 		"validation.workspace.kaito.sh",
 		"/validate/workspace.kaito.sh",
-		Resources,
+		WorkspaceResources,
 		func(ctx context.Context) context.Context { return ctx },
 		true,
 	)
 }
 
-var Resources = map[schema.GroupVersionKind]resourcesemantics.GenericCRD{
+var WorkspaceResources = map[schema.GroupVersionKind]resourcesemantics.GenericCRD{
 	kaitov1alpha1.GroupVersion.WithKind("Workspace"): &kaitov1alpha1.Workspace{},
 }
+
+func NewRAGEngineWebhooks() []knativeinjection.ControllerConstructor {
+	return []knativeinjection.ControllerConstructor{
+		certificates.NewController,
+		NewRAGEngineCRDValidationWebhook,
+	}
+}
+
+func NewRAGEngineCRDValidationWebhook(ctx context.Context, _ configmap.Watcher) *controller.Impl {
+	return validation.NewAdmissionController(ctx,
+		"validation.ragengine.kaito.sh",
+		"/validate/ragengine.kaito.sh",
+		WorkspaceResources,
+		func(ctx context.Context) context.Context { return ctx },
+		true,
+	)
+}
+
+var RAGEngineResources = map[schema.GroupVersionKind]resourcesemantics.GenericCRD{
+	kaitov1alpha1.GroupVersion.WithKind("RAGEngine"): &kaitov1alpha1.RAGEngine{},
+}