Allow to run the e2e test suite locally, against an out of the cluste…

…r operator (#1351)

* Allow to run the e2e test suite locally, against an out of the cluster operator

* Update readme

Makefile

@@ -88,7 +88,8 @@ SHELL = /usr/bin/env bash -o pipefail
 .SHELLFLAGS = -ec
 
 # E2E tests additional flags
-E2E_TEST_FLAGS = "--skip-deletion=false" -timeout 25m # See README.md, default go test timeout 10m
+# See README.md, default go test timeout 10m
+E2E_TEST_FLAGS = -timeout 25m
 
 # Default image-build is to not use local odh-manifests folder
 # set to "true" to use local instead

README.md

@@ -393,16 +393,29 @@ make e2e-test
 Additional flags that can be passed to e2e-tests by setting up `E2E_TEST_FLAGS`
 variable. Following table lists all the available flags to run the tests:
 
-| Flag            | Description                                                                                                                                                        | Default value |
-|-----------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------|
-| --skip-deletion | To skip running  of `dsc-deletion` test that includes deleting `DataScienceCluster` resources. Assign this variable to `true` to skip DataScienceCluster deletion. | false         |
+| Flag                       | Description                                                                                                                                                                   | Default value |
+|----------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------|
+| --skip-deletion            | To skip running  of `dsc-deletion` test that includes deleting `DataScienceCluster` resources. Assign this variable to `true` to skip DataScienceCluster deletion.            | false         |
+| --test-operator-controller | To configure the execution of tests related to the Operator POD, this is useful to run e2e tests for an operator running out of the cluster i.e. for debugging purposes       | true          |
+| --test-webhook             | To configure the execution of tests rellated to the Operator WebHooks, this is useful to run e2e tests for an operator running out of the cluster i.e. for debugging purposes | true          |
+| --test-component           | A repeatable flag that control what component should be tested, by default all component specific test are executed                                                           | true          |
 
-Example command to run full test suite skipping the test
-for DataScienceCluster deletion.
+Example command to run full test suite skipping the test for DataScienceCluster deletion.
 
 ```shell
 make e2e-test -e OPERATOR_NAMESPACE=<namespace> -e E2E_TEST_FLAGS="--skip-deletion=true"
 ```
+
+Example commands to run test suite for the dashboard `component` only, with the operator running out of the cluster.
+
+```shell
+make run-nowebhook
+```
+```shell
+make e2e-test -e OPERATOR_NAMESPACE=<namespace> -e E2E_TEST_FLAGS="--test-operator-controller=false --test-webhook=false --test-component=dashboard"
+```
+
+
 ### API Overview
 
 Please refer to [api documentation](docs/api-overview.md)

tests/e2e/controller_test.go

@@ -5,12 +5,15 @@ import (
 	"flag"
 	"fmt"
 	"os"
+	"strings"
 	"testing"
 
 	operatorv1 "github.com/openshift/api/operator/v1"
 	routev1 "github.com/openshift/api/route/v1"
 	ofapi "github.com/operator-framework/api/pkg/operators/v1alpha1"
 	monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1"
+	"golang.org/x/exp/maps"
+	"golang.org/x/exp/slices"
 	autoscalingv1 "k8s.io/api/autoscaling/v1"
 	apiextv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
 	"k8s.io/apimachinery/pkg/runtime"
@@ -29,12 +32,39 @@ import (
 	featurev1 "github.com/opendatahub-io/opendatahub-operator/v2/apis/features/v1"
 )
 
+type TestFn func(t *testing.T)
+
 var (
-	opNamespace  string
-	skipDeletion bool
-	scheme       = runtime.NewScheme()
+	testOpts testContextConfig
+	scheme   = runtime.NewScheme()
+
+	componentsTestSuites = map[string]TestFn{
+		"dashboard":     dashboardTestSuite,
+		"ray":           rayTestSuite,
+		"modelregistry": modelRegistryTestSuite,
+	}
 )
 
+type arrayFlags []string
+
+func (i *arrayFlags) String() string {
+	return fmt.Sprintf("%v", *i)
+}
+
+func (i *arrayFlags) Set(value string) error {
+	*i = append(*i, value)
+	return nil
+}
+
+type testContextConfig struct {
+	operatorNamespace string
+	skipDeletion      bool
+
+	operatorControllerTest bool
+	webhookTest            bool
+	components             arrayFlags
+}
+
 // Holds information specific to individual tests.
 type testContext struct {
 	// Rest config
@@ -54,6 +84,8 @@ type testContext struct {
 	// context for accessing resources
 	//nolint:containedctx //reason: legacy v1 test setup
 	ctx context.Context
+	// test configuration
+	testOpts testContextConfig
 }
 
 func NewTestContext() (*testContext, error) {
@@ -85,11 +117,12 @@ func NewTestContext() (*testContext, error) {
 		cfg:                   config,
 		kubeClient:            kc,
 		customClient:          custClient,
-		operatorNamespace:     opNamespace,
+		operatorNamespace:     testOpts.operatorNamespace,
 		applicationsNamespace: testDSCI.Spec.ApplicationsNamespace,
 		ctx:                   context.TODO(),
 		testDsc:               testDSC,
 		testDSCI:              testDSCI,
+		testOpts:              testOpts,
 	}, nil
 }
 
@@ -109,21 +142,27 @@ func TestOdhOperator(t *testing.T) {
 
 	log.SetLogger(zap.New(zap.UseDevMode(true)))
 
-	// individual test suites after the operator is running
-	if !t.Run("validate operator pod is running", testODHOperatorValidation) {
-		return
+	if testOpts.operatorControllerTest {
+		// individual test suites after the operator is running
+		if !t.Run("validate operator pod is running", testODHOperatorValidation) {
+			return
+		}
 	}
 
 	// Run create and delete tests for all the components
 	t.Run("create DSCI and DSC CRs", creationTestSuite)
 
-	// Validate deployment of each component in separate test suite
-	t.Run("validate installation of Dashboard Component", dashboardTestSuite)
-	t.Run("validate installation of Ray Component", rayTestSuite)
-	t.Run("validate installation of ModelRegistry Component", modelRegistryTestSuite)
+	for k, v := range componentsTestSuites {
+		if len(testOpts.components) != 0 && !slices.Contains(testOpts.components, k) {
+			t.Logf("Skipping tests for component %s", k)
+			continue
+		}
+
+		t.Run("validate installation of "+k+" component", v)
+	}
 
 	// Run deletion if skipDeletion is not set
-	if !skipDeletion {
+	if !testOpts.skipDeletion {
 		// this is a negative test case, since by using the positive CM('true'), even CSV gets deleted which leaves no operator pod in prow
 		t.Run("components should not be removed if labeled is set to 'false' on configmap", cfgMapDeletionTestSuite)
 
@@ -133,10 +172,23 @@ func TestOdhOperator(t *testing.T) {
 
 func TestMain(m *testing.M) {
 	// call flag.Parse() here if TestMain uses flags
-	flag.StringVar(&opNamespace, "operator-namespace",
-		"opendatahub-operator-system", "Namespace where the odh operator is deployed")
-	flag.BoolVar(&skipDeletion, "skip-deletion", false, "skip deletion of the controllers")
+	flag.StringVar(&testOpts.operatorNamespace, "operator-namespace", "opendatahub-operator-system", "Namespace where the odh operator is deployed")
+	flag.BoolVar(&testOpts.skipDeletion, "skip-deletion", false, "skip deletion of the controllers")
+
+	flag.BoolVar(&testOpts.operatorControllerTest, "test-operator-controller", true, "run operator controller tests")
+	flag.BoolVar(&testOpts.webhookTest, "test-webhook", true, "run webhook tests")
+
+	componentNames := strings.Join(maps.Keys(componentsTestSuites), ", ")
+	flag.Var(&testOpts.components, "test-component", "run tests for the specified component. valid components names are: "+componentNames)
 
 	flag.Parse()
+
+	for _, n := range testOpts.components {
+		if _, ok := componentsTestSuites[n]; !ok {
+			fmt.Printf("test-component: unknown component %s, valid values are: %s", n, componentNames)
+			os.Exit(1)
+		}
+	}
+
 	os.Exit(m.Run())
 }

tests/e2e/creation_test.go

@@ -43,14 +43,17 @@ func creationTestSuite(t *testing.T) {
 			err = testCtx.testDSCICreation()
 			require.NoError(t, err, "error creating DSCI CR")
 		})
-		t.Run("Creation of more than one of DSCInitialization instance", func(t *testing.T) {
-			testCtx.testDSCIDuplication(t)
-		})
+		if testCtx.testOpts.webhookTest {
+			t.Run("Creation of more than one of DSCInitialization instance", func(t *testing.T) {
+				testCtx.testDSCIDuplication(t)
+			})
+		}
 		// Validates Servicemesh fields
 		t.Run("Validate DSCInitialization instance", func(t *testing.T) {
 			err = testCtx.validateDSCI()
 			require.NoError(t, err, "error validating DSCInitialization instance")
 		})
+
 		t.Run("Check owned namespaces exist", func(t *testing.T) {
 			err = testCtx.testOwnedNamespacesAllExist()
 			require.NoError(t, err, "error owned namespace is missing")
@@ -61,9 +64,11 @@ func creationTestSuite(t *testing.T) {
 			err = testCtx.testDSCCreation(t)
 			require.NoError(t, err, "error creating DataScienceCluster instance")
 		})
-		t.Run("Creation of more than one of DataScienceCluster instance", func(t *testing.T) {
-			testCtx.testDSCDuplication(t)
-		})
+		if testCtx.testOpts.webhookTest {
+			t.Run("Creation of more than one of DataScienceCluster instance", func(t *testing.T) {
+				testCtx.testDSCDuplication(t)
+			})
+		}
 
 		// // Kserve
 		// t.Run("Validate Knative resoruce", func(t *testing.T) {
@@ -77,10 +82,13 @@ func creationTestSuite(t *testing.T) {
 		// })
 		//
 		// ModelReg
-		t.Run("Validate model registry config", func(t *testing.T) {
-			err = testCtx.validateModelRegistryConfig()
-			require.NoError(t, err, "error validating ModelRegistry config")
-		})
+
+		if testCtx.testOpts.webhookTest {
+			t.Run("Validate model registry config", func(t *testing.T) {
+				err = testCtx.validateModelRegistryConfig()
+				require.NoError(t, err, "error validating ModelRegistry config")
+			})
+		}
 	})
 }
 

tests/e2e/dashboard_test.go

@@ -28,6 +28,8 @@ type DashboardTestCtx struct {
 }
 
 func dashboardTestSuite(t *testing.T) {
+	t.Helper()
+
 	dashboardCtx := DashboardTestCtx{}
 	var err error
 	dashboardCtx.testCtx, err = NewTestContext()

tests/e2e/modelregistry_test.go

@@ -112,6 +112,8 @@ func (mr *ModelRegistryTestCtx) updateComponent(fn func(dsc *dscv1.Components))
 }
 
 func modelRegistryTestSuite(t *testing.T) {
+	t.Helper()
+
 	tc, err := NewTestContext()
 	require.NoError(t, err)
 

tests/e2e/ray_test.go

@@ -28,6 +28,8 @@ type RayTestCtx struct {
 }
 
 func rayTestSuite(t *testing.T) {
+	t.Helper()
+
 	rayCtx := RayTestCtx{}
 	var err error
 	rayCtx.testCtx, err = NewTestContext()