test: update e2e testcases (opendatahub-io#1208)

* test: update e2e testcases

- rename files
- add check on FTer CRD
- use label from labels.ODHAppPrefix
- update misleading log message
- re-order test cases
- rename DSC CR from e2e-test to e2e-test-dsc

Signed-off-by: Wen Zhou <wenzhou@redhat.com>

* fix: missing "/"

Signed-off-by: Wen Zhou <wenzhou@redhat.com>

---------

Signed-off-by: Wen Zhou <wenzhou@redhat.com>

tests/e2e/dsc_cfmap_deletion_test.go → tests/e2e/cfmap_deletion_test.go

@@ -30,7 +30,7 @@ func cfgMapDeletionTestSuite(t *testing.T) {
 
 		t.Run("owned namespaces should be not deleted", func(t *testing.T) {
 			err = testCtx.testOwnedNamespacesAllExist()
-			require.NoError(t, err, "Error while deleting owned namespaces")
+			require.NoError(t, err, "Configmap should not delete owned namespaces")
 		})
 	})
 }
@@ -62,21 +62,21 @@ func (tc *testContext) testOwnedNamespacesAllExist() error {
 	if err != nil {
 		return fmt.Errorf("failed getting owned namespaces %w", err)
 	}
-	if len(namespaces.Items) == 0 {
-		return errors.New("all namespaces are gone")
+	if len(namespaces.Items) < ownedNamespaceNumber {
+		return errors.New("some owned namespaces are missing")
 	}
 
 	return nil
 }
 
 func removeDeletionConfigMap(tc *testContext) {
-	_ = tc.kubeClient.CoreV1().ConfigMaps(tc.operatorNamespace).Delete(tc.ctx, "delete-self-managed", metav1.DeleteOptions{})
+	_ = tc.kubeClient.CoreV1().ConfigMaps(tc.operatorNamespace).Delete(tc.ctx, deleteConfigMap, metav1.DeleteOptions{})
 }
 
 func createDeletionConfigMap(tc *testContext, enableDeletion string) error {
 	configMap := &corev1.ConfigMap{
 		ObjectMeta: metav1.ObjectMeta{
-			Name:      "delete-self-managed",
+			Name:      deleteConfigMap,
 			Namespace: tc.operatorNamespace,
 			Labels: map[string]string{
 				upgrade.DeleteConfigMapLabel: enableDeletion,

tests/e2e/controller_setup_test.go → tests/e2e/controller_test.go

@@ -83,7 +83,7 @@ func NewTestContext() (*testContext, error) {
 	// setup DSCI CR since we do not create automatically by operator
 	testDSCI := setupDSCICR("e2e-test-dsci")
 	// Setup DataScienceCluster CR
-	testDSC := setupDSCInstance("e2e-test")
+	testDSC := setupDSCInstance("e2e-test-dsc")
 
 	return &testContext{
 		cfg:                     config,
@@ -123,7 +123,6 @@ func TestOdhOperator(t *testing.T) {
 
 	// Run deletion if skipDeletion is not set
 	if !skipDeletion {
-		// This test case recreates entire DSC again and deletes afterward
 		// this is a negative test case, since by using the positive CM('true'), even CSV gets deleted which leaves no operator pod in prow
 		t.Run("components should not be removed if labeled is set to 'false' on configmap", cfgMapDeletionTestSuite)
 

tests/e2e/dsc_creation_test.go → tests/e2e/creation_test.go

@@ -30,10 +30,7 @@ import (
 	"github.com/opendatahub-io/opendatahub-operator/v2/components/modelregistry"
 	"github.com/opendatahub-io/opendatahub-operator/v2/pkg/cluster"
 	"github.com/opendatahub-io/opendatahub-operator/v2/pkg/feature/serverless"
-)
-
-const (
-	odhLabelPrefix = "app.opendatahub.io/"
+	"github.com/opendatahub-io/opendatahub-operator/v2/pkg/metadata/labels"
 )
 
 func creationTestSuite(t *testing.T) {
@@ -44,24 +41,31 @@ func creationTestSuite(t *testing.T) {
 	require.NoError(t, err, "error setting up environment")
 
 	t.Run(testCtx.testDsc.Name, func(t *testing.T) {
+		// DSCI
 		t.Run("Creation of DSCI CR", func(t *testing.T) {
 			err = testCtx.testDSCICreation()
 			require.NoError(t, err, "error creating DSCI CR")
 		})
 		t.Run("Creation of more than one of DSCInitialization instance", func(t *testing.T) {
 			testCtx.testDSCIDuplication(t)
 		})
+		t.Run("Validate DSCInitialization instance", func(t *testing.T) {
+			err = testCtx.validateDSCI()
+			require.NoError(t, err, "error validating DSCInitialization instance")
+		})
+		t.Run("Check owned namespaces exist", func(t *testing.T) {
+			err = testCtx.testOwnedNamespacesAllExist()
+			require.NoError(t, err, "error owned namespace is missing")
+		})
+
+		// DSC
 		t.Run("Creation of DataScienceCluster instance", func(t *testing.T) {
 			err = testCtx.testDSCCreation()
 			require.NoError(t, err, "error creating DataScienceCluster instance")
 		})
 		t.Run("Creation of more than one of DataScienceCluster instance", func(t *testing.T) {
 			testCtx.testDSCDuplication(t)
 		})
-		t.Run("Validate DSCInitialization instance", func(t *testing.T) {
-			err = testCtx.validateDSCI()
-			require.NoError(t, err, "error validating DSCInitialization instance")
-		})
 		t.Run("Validate Ownerrefrences exist", func(t *testing.T) {
 			err = testCtx.testOwnerrefrences()
 			require.NoError(t, err, "error getting all DataScienceCluster's Ownerrefrences")
@@ -70,15 +74,19 @@ func creationTestSuite(t *testing.T) {
 			err = testCtx.testDefaultCertsAvailable()
 			require.NoError(t, err, "error getting default cert secrets for Kserve")
 		})
-		t.Run("Validate Knative resoruce", func(t *testing.T) {
-			err = testCtx.validateDSC()
-			require.NoError(t, err, "error getting Knatvie resrouce as part of DataScienceCluster validation")
-		})
 		t.Run("Validate all deployed components", func(t *testing.T) {
-			// this will take about 5-6mins to complete
+			// this will take about 5-6 mins to complete
 			err = testCtx.testAllApplicationCreation(t)
 			require.NoError(t, err, "error testing deployments for DataScienceCluster: "+testCtx.testDsc.Name)
 		})
+
+		// Kserve
+		t.Run("Validate Knative resoruce", func(t *testing.T) {
+			err = testCtx.validateDSC()
+			require.NoError(t, err, "error getting Knatvie resrouce as part of DataScienceCluster validation")
+		})
+
+		// ModelReg
 		t.Run("Validate default model registry cert available", func(t *testing.T) {
 			err = testCtx.testDefaultModelRegistryCertAvailable()
 			require.NoError(t, err, "error getting default cert secret for ModelRegistry")
@@ -87,6 +95,8 @@ func creationTestSuite(t *testing.T) {
 			err = testCtx.testMRServiceMeshMember()
 			require.NoError(t, err, "error getting servicemeshmember for Model Registry")
 		})
+
+		// reconcile
 		t.Run("Validate Controller reconcile", func(t *testing.T) {
 			// only test Dashboard component for now
 			err = testCtx.testUpdateComponentReconcile()
@@ -187,10 +197,10 @@ func (tc *testContext) testDSCCreation() error {
 				return true, nil
 			})
 			if nberr != nil {
-				return fmt.Errorf("error creating e2e-test DSC %s: %w", tc.testDsc.Name, nberr)
+				return fmt.Errorf("error creating e2e-test-dsc DSC %s: %w", tc.testDsc.Name, nberr)
 			}
 		} else {
-			return fmt.Errorf("error getting e2e-test DSC %s: %w", tc.testDsc.Name, err)
+			return fmt.Errorf("error getting e2e-test-dsc DSC %s: %w", tc.testDsc.Name, err)
 		}
 	}
 
@@ -242,7 +252,7 @@ func (tc *testContext) testDSCDuplication(t *testing.T) { //nolint:thelper
 		Version: "v1",
 		Kind:    "DataScienceCluster",
 	}
-	dup := setupDSCInstance("e2e-test-dup")
+	dup := setupDSCInstance("e2e-test-dsc-dup")
 
 	tc.testDuplication(t, gvk, dup)
 }
@@ -295,7 +305,7 @@ func (tc *testContext) testApplicationCreation(component components.ComponentInt
 	err := wait.PollUntilContextTimeout(tc.ctx, tc.resourceRetryInterval, tc.resourceCreationTimeout, false, func(ctx context.Context) (bool, error) {
 		// TODO: see if checking deployment is a good test, CF does not create deployment
 		appList, err := tc.kubeClient.AppsV1().Deployments(tc.applicationsNamespace).List(ctx, metav1.ListOptions{
-			LabelSelector: odhLabelPrefix + component.GetComponentName(),
+			LabelSelector: labels.ODH.Component(component.GetComponentName()),
 		})
 		if err != nil {
 			log.Printf("error listing application deployments :%v. Trying again...", err)
@@ -381,7 +391,7 @@ func (tc *testContext) testOwnerrefrences() error {
 	// Test any one of the apps
 	if tc.testDsc.Spec.Components.Dashboard.ManagementState == operatorv1.Managed {
 		appDeployments, err := tc.kubeClient.AppsV1().Deployments(tc.applicationsNamespace).List(tc.ctx, metav1.ListOptions{
-			LabelSelector: odhLabelPrefix + tc.testDsc.Spec.Components.Dashboard.GetComponentName(),
+			LabelSelector: labels.ODH.Component(tc.testDsc.Spec.Components.Dashboard.GetComponentName()),
 		})
 		if err != nil {
 			return fmt.Errorf("error listing application deployments %w", err)
@@ -497,7 +507,7 @@ func (tc *testContext) testUpdateComponentReconcile() error {
 	// Test Updating Dashboard Replicas
 
 	appDeployments, err := tc.kubeClient.AppsV1().Deployments(tc.applicationsNamespace).List(tc.ctx, metav1.ListOptions{
-		LabelSelector: odhLabelPrefix + tc.testDsc.Spec.Components.Dashboard.GetComponentName(),
+		LabelSelector: labels.ODH.Component(tc.testDsc.Spec.Components.Dashboard.GetComponentName()),
 	})
 	if err != nil {
 		return err
@@ -548,7 +558,7 @@ func (tc *testContext) testUpdateDSCComponentEnabled() error {
 
 	if tc.testDsc.Spec.Components.Dashboard.ManagementState == operatorv1.Managed {
 		appDeployments, err := tc.kubeClient.AppsV1().Deployments(tc.applicationsNamespace).List(tc.ctx, metav1.ListOptions{
-			LabelSelector: odhLabelPrefix + tc.testDsc.Spec.Components.Dashboard.GetComponentName(),
+			LabelSelector: labels.ODH.Component(tc.testDsc.Spec.Components.Dashboard.GetComponentName()),
 		})
 		if err != nil {
 			return fmt.Errorf("error getting enabled component %v", tc.testDsc.Spec.Components.Dashboard.GetComponentName())

tests/e2e/dsc_deletion_test.go → tests/e2e/deletion_test.go

@@ -16,25 +16,27 @@ import (
 	dscv1 "github.com/opendatahub-io/opendatahub-operator/v2/apis/datasciencecluster/v1"
 	dsciv1 "github.com/opendatahub-io/opendatahub-operator/v2/apis/dscinitialization/v1"
 	"github.com/opendatahub-io/opendatahub-operator/v2/components"
+	"github.com/opendatahub-io/opendatahub-operator/v2/pkg/metadata/labels"
 )
 
 func deletionTestSuite(t *testing.T) {
 	testCtx, err := NewTestContext()
 	require.NoError(t, err)
 
+	// pre-check before deletion
 	t.Run("ensure all components created", func(t *testing.T) {
 		err = testCtx.testAllApplicationCreation(t)
-		require.NoError(t, err, "Error to create DSC instance")
+		require.NoError(t, err, "Not all components are created")
 	})
 
 	t.Run(testCtx.testDsc.Name, func(t *testing.T) {
-		t.Run("Deletion: DataScienceCluster instance", func(t *testing.T) {
+		t.Run("Deletion: DSC instance", func(t *testing.T) {
 			err = testCtx.testDeletionExistDSC()
 			require.NoError(t, err, "Error to delete DSC instance")
 		})
-		t.Run("Deletion: Application Resource", func(t *testing.T) {
+		t.Run("Check: all component resource are deleted", func(t *testing.T) {
 			err = testCtx.testAllApplicationDeletion(t)
-			require.NoError(t, err, "Error to delete component")
+			require.NoError(t, err, "Should not found component exist")
 		})
 		t.Run("Deletion: DSCI instance", func(t *testing.T) {
 			err = testCtx.testDeletionExistDSCI()
@@ -57,7 +59,7 @@ func (tc *testContext) testDeletionExistDSC() error {
 		}
 	} else if !errors.IsNotFound(err) {
 		if err != nil {
-			return fmt.Errorf("error getting DSC instance :%w", err)
+			return fmt.Errorf("could not find DSC instance to delete: %w", err)
 		}
 	}
 
@@ -69,17 +71,17 @@ func (tc *testContext) testApplicationDeletion(component components.ComponentInt
 
 	if err := wait.PollUntilContextTimeout(tc.ctx, tc.resourceRetryInterval, tc.resourceCreationTimeout, false, func(ctx context.Context) (bool, error) {
 		appList, err := tc.kubeClient.AppsV1().Deployments(tc.applicationsNamespace).List(ctx, metav1.ListOptions{
-			LabelSelector: odhLabelPrefix + component.GetComponentName(),
+			LabelSelector: labels.ODH.Component(component.GetComponentName()),
 		})
 		if err != nil {
-			log.Printf("error listing component deployments :%v. Trying again...", err)
+			log.Printf("error getting component deployments :%v. Trying again...", err)
 
 			return false, err
 		}
 
 		return len(appList.Items) == 0, nil
 	}); err != nil {
-		return fmt.Errorf("error deleting component: %v", component.GetComponentName())
+		return fmt.Errorf("error to find component still exist: %v", component.GetComponentName())
 	}
 
 	return nil
@@ -121,7 +123,7 @@ func (tc *testContext) testDeletionExistDSCI() error {
 		}
 	} else if !errors.IsNotFound(err) {
 		if err != nil {
-			return fmt.Errorf("error getting DSCI instance :%w", err)
+			return fmt.Errorf("could not find DSCI instance to delete :%w", err)
 		}
 	}
 

tests/e2e/helper_test.go

@@ -41,6 +41,8 @@ const (
 	servicemeshNamespace = "openshift-operators"
 	servicemeshOpName    = "servicemeshoperator"
 	serverlessOpName     = "serverless-operator"
+	ownedNamespaceNumber = 1 // set to 4 for RHOAI
+	deleteConfigMap      = "delete-configmap-name"
 )
 
 func (tc *testContext) waitForControllerDeployment(name string, replicas int32) error {
@@ -361,7 +363,7 @@ func approveInstallPlan(tc *testContext, plan *ofapi.InstallPlan) error {
 	}
 	force := true
 	opt := &client.PatchOptions{
-		FieldManager: "e2e-test",
+		FieldManager: "e2e-test-dsc",
 		Force:        &force,
 	}
 

tests/e2e/odh_manager_test.go

@@ -27,4 +27,7 @@ func (tc *testContext) validateOwnedCRDs(t *testing.T) {
 
 	require.NoErrorf(t, tc.validateCRD("dscinitializations.dscinitialization.opendatahub.io"),
 		"error in validating CRD : dscinitializations.dscinitialization.opendatahub.io")
+
+	require.NoErrorf(t, tc.validateCRD("featuretrackers.features.opendatahub.io"),
+		"error in validating CRD : featuretrackers.features.opendatahub.io")
 }