upgrade: cleanup CreateWithRetry usage

pkg/cluster/resources.go

@@ -206,11 +206,34 @@ func CreateWithRetry(ctx context.Context, cli client.Client, obj client.Object,
 	timeout := time.Duration(timeoutMin) * time.Minute
 
 	return wait.PollUntilContextTimeout(ctx, interval, timeout, true, func(ctx context.Context) (bool, error) {
-		err := cli.Create(ctx, obj)
-		if err != nil {
-			fmt.Printf("Error creating object: %v. Retrying...\n", err)
-			return false, err
+		// Create can return:
+		// If webhook enabled:
+		//   - no error (err == nil)
+		//   - 500 InternalError likely if webhook is not available (yet)
+		//   - 403 Forbidden if webhook blocks creation (check of existence)
+		//   - some problem (real error)
+		// else, if webhook disabled:
+		//   - no error (err == nil)
+		//   - 409 AlreadyExists if object exists
+		//   - some problem (real error)
+		errCreate := cli.Create(ctx, obj)
+		if errCreate == nil {
+			return true, nil
 		}
-		return true, nil
+
+		// check existence, success case for the function, covers 409 and 403 (or newly created)
+		errGet := cli.Get(ctx, client.ObjectKeyFromObject(obj), obj)
+		if errGet == nil {
+			return true, nil
+		}
+
+		// retry if 500, assume webhook is not available
+		if k8serr.IsInternalError(errCreate) {
+			fmt.Printf("Error creating object: %v. Retrying...\n", errCreate)
+			return false, nil
+		}
+
+		// some other error
+		return false, errCreate
 	})
 }

pkg/upgrade/upgrade.go

@@ -7,7 +7,6 @@ import (
 	"errors"
 	"fmt"
 	"reflect"
-	"time"
 
 	"github.com/hashicorp/go-multierror"
 	operatorv1 "github.com/openshift/api/operator/v1"
@@ -104,17 +103,9 @@ func CreateDefaultDSC(ctx context.Context, cli client.Client) error {
 		},
 	}
 	err := cluster.CreateWithRetry(ctx, cli, releaseDataScienceCluster, 1) // 1 min timeout
-	switch {
-	case err == nil:
-		fmt.Printf("created DataScienceCluster resource\n")
-	case k8serr.IsAlreadyExists(err):
-		// Do not update the DSC if it already exists
-		fmt.Printf("DataScienceCluster resource already exists. It will not be updated with default DSC.\n")
-		return nil
-	default:
+	if err != nil {
 		return fmt.Errorf("failed to create DataScienceCluster custom resource: %w", err)
 	}
-
 	return nil
 }
 
@@ -167,9 +158,8 @@ func CreateDefaultDSCI(ctx context.Context, cli client.Client, _ cluster.Platfor
 		return nil
 	case len(instances.Items) == 0:
 		fmt.Println("create default DSCI CR.")
-		time.Sleep(10 * time.Second)                             // put 10 seconds sleep for webhook to fully functional before request first creation
 		err := cluster.CreateWithRetry(ctx, cli, defaultDsci, 1) // 1 min timeout
-		if err != nil && !k8serr.IsAlreadyExists(err) {
+		if err != nil {
 			return err
 		}
 	}