Remove used providers in e2e

EKS cluster suffers from limited number of private IPs, which are
exausted by test suites running in parallel. It affects GitRepo
processing, where a pod created to update content of the repo in order
to apply manifests to the cluster. Since the pod is not able to be
scheduled, e2e tests are failing.

Signed-off-by: Danil-Grigorev <danil.grigorev@suse.com>

test/e2e/const.go

@@ -33,8 +33,14 @@ var (
 	//go:embed data/capi-operator/capv-provider.yaml
 	CapvProvider []byte
 
-	//go:embed data/capi-operator/full-providers.yaml
-	FullProviders []byte
+	//go:embed data/capi-operator/aws-provider.yaml
+	AWSProvider []byte
+
+	//go:embed data/capi-operator/gcp-provider.yaml
+	GCPProvider []byte
+
+	//go:embed data/capi-operator/azure-provider.yaml
+	AzureProvider []byte
 
 	//go:embed data/capi-operator/capa-variables.yaml
 	AWSProviderSecret []byte

test/e2e/data/capi-operator/aws-provider.yaml

@@ -0,0 +1,14 @@
+---
+apiVersion: turtles-capi.cattle.io/v1alpha1
+kind: CAPIProvider
+metadata:
+  name: aws
+  namespace: capa-system
+spec:
+  type: infrastructure
+  variables:
+    EXP_MACHINE_POOL: "true"
+    EXP_EXTERNAL_RESOURCE_GC: "true"
+    CAPA_LOGLEVEL: "4"
+  manager:
+    syncPeriod: "5m"

test/e2e/data/capi-operator/azure-provider.yaml

@@ -0,0 +1,8 @@
+---
+apiVersion: turtles-capi.cattle.io/v1alpha1
+kind: CAPIProvider
+metadata:
+  name: azure
+  namespace: capz-system
+spec:
+  type: infrastructure

test/e2e/data/capi-operator/gcp-provider.yaml

@@ -0,0 +1,8 @@
+---
+apiVersion: turtles-capi.cattle.io/v1alpha1
+kind: CAPIProvider
+metadata:
+  name: gcp
+  namespace: capg-system
+spec:
+  type: infrastructure

test/e2e/specs/import_gitops_mgmtv3.go

@@ -93,7 +93,7 @@ type CreateMgmtV3UsingGitOpsSpecInput struct {
 // automatically imports into Rancher Manager.
 func CreateMgmtV3UsingGitOpsSpec(ctx context.Context, inputGetter func() CreateMgmtV3UsingGitOpsSpecInput) {
 	var (
-		specName              = "creategitops-v3"
+		specName              = "creategitops"
 		input                 CreateMgmtV3UsingGitOpsSpecInput
 		namespace             *corev1.Namespace
 		repoName              string
@@ -172,6 +172,7 @@ func CreateMgmtV3UsingGitOpsSpec(ctx context.Context, inputGetter func() CreateM
 		input = inputGetter()
 		Expect(turtlesframework.Parse(&input)).To(Succeed(), "Failed to parse environment variables")
 
+		Expect(input.GitAddr).ToNot(BeEmpty(), "Invalid argument. input.GitAddr can't be empty when calling %s spec", specName)
 		Expect(input.E2EConfig).ToNot(BeNil(), "Invalid argument. input.E2EConfig can't be nil when calling %s spec", specName)
 		Expect(input.BootstrapClusterProxy).ToNot(BeNil(), "Invalid argument. input.BootstrapClusterProxy can't be nil when calling %s spec", specName)
 		Expect(os.MkdirAll(input.ArtifactFolder, 0750)).To(Succeed(), "Invalid argument. input.ArtifactFolder can't be created for %s spec", specName)

test/e2e/suites/import-gitops-v3/import_gitops_v3_test.go

@@ -27,6 +27,7 @@ import (
 
 	"github.com/rancher/turtles/test/e2e"
 	"github.com/rancher/turtles/test/e2e/specs"
+	"github.com/rancher/turtles/test/testenv"
 )
 
 var _ = Describe("[Docker] [Kubeadm] - [management.cattle.io/v3] Create and delete CAPI cluster functionality should work with namespace auto-import", Label(e2e.ShortTestLabel), func() {
@@ -36,6 +37,14 @@ var _ = Describe("[Docker] [Kubeadm] - [management.cattle.io/v3] Create and dele
 	})
 
 	specs.CreateMgmtV3UsingGitOpsSpec(ctx, func() specs.CreateMgmtV3UsingGitOpsSpecInput {
+		testenv.CAPIOperatorDeployProvider(ctx, testenv.CAPIOperatorDeployProviderInput{
+			BootstrapClusterProxy: bootstrapClusterProxy,
+			CAPIProvidersYAML: [][]byte{
+				e2e.CapiProviders,
+			},
+			WaitForDeployments: testenv.DefaultDeployments,
+		})
+
 		return specs.CreateMgmtV3UsingGitOpsSpecInput{
 			E2EConfig:                      e2e.LoadE2EConfig(),
 			BootstrapClusterProxy:          bootstrapClusterProxy,
@@ -65,6 +74,14 @@ var _ = Describe("[Docker] [RKE2] - [management.cattle.io/v3] Create and delete
 	})
 
 	specs.CreateMgmtV3UsingGitOpsSpec(ctx, func() specs.CreateMgmtV3UsingGitOpsSpecInput {
+		testenv.CAPIOperatorDeployProvider(ctx, testenv.CAPIOperatorDeployProviderInput{
+			BootstrapClusterProxy: bootstrapClusterProxy,
+			CAPIProvidersYAML: [][]byte{
+				e2e.CapiProviders,
+			},
+			WaitForDeployments: testenv.DefaultDeployments,
+		})
+
 		return specs.CreateMgmtV3UsingGitOpsSpecInput{
 			E2EConfig:                      e2e.LoadE2EConfig(),
 			BootstrapClusterProxy:          bootstrapClusterProxy,
@@ -94,11 +111,27 @@ var _ = Describe("[Azure] [AKS] - [management.cattle.io/v3] Create and delete CA
 	})
 
 	specs.CreateMgmtV3UsingGitOpsSpec(ctx, func() specs.CreateMgmtV3UsingGitOpsSpecInput {
+		testenv.CAPIOperatorDeployProvider(ctx, testenv.CAPIOperatorDeployProviderInput{
+			BootstrapClusterProxy: bootstrapClusterProxy,
+			CAPIProvidersSecretsYAML: [][]byte{
+				e2e.AzureIdentitySecret,
+			},
+			CAPIProvidersYAML: [][]byte{
+				e2e.AzureProvider,
+			},
+			WaitForDeployments: []testenv.NamespaceName{
+				{
+					Name:      "capz-controller-manager",
+					Namespace: "capz-system",
+				},
+			},
+		})
+
 		return specs.CreateMgmtV3UsingGitOpsSpecInput{
 			E2EConfig:                      e2e.LoadE2EConfig(),
 			BootstrapClusterProxy:          bootstrapClusterProxy,
 			ClusterTemplate:                e2e.CAPIAzureAKSTopology,
-			ClusterName:                    "highlander-e2e-topology",
+			ClusterName:                    "cluster-aks",
 			ControlPlaneMachineCount:       ptr.To[int](1),
 			WorkerMachineCount:             ptr.To[int](1),
 			GitAddr:                        gitAddress,
@@ -121,11 +154,27 @@ var _ = Describe("[AWS] [EKS] - [management.cattle.io/v3] Create and delete CAPI
 	})
 
 	specs.CreateMgmtV3UsingGitOpsSpec(ctx, func() specs.CreateMgmtV3UsingGitOpsSpecInput {
+		testenv.CAPIOperatorDeployProvider(ctx, testenv.CAPIOperatorDeployProviderInput{
+			BootstrapClusterProxy: bootstrapClusterProxy,
+			CAPIProvidersSecretsYAML: [][]byte{
+				e2e.AWSProviderSecret,
+			},
+			CAPIProvidersYAML: [][]byte{
+				e2e.AWSProvider,
+			},
+			WaitForDeployments: []testenv.NamespaceName{
+				{
+					Name:      "capa-controller-manager",
+					Namespace: "capa-system",
+				},
+			},
+		})
+
 		return specs.CreateMgmtV3UsingGitOpsSpecInput{
 			E2EConfig:                      e2e.LoadE2EConfig(),
 			BootstrapClusterProxy:          bootstrapClusterProxy,
 			ClusterTemplate:                e2e.CAPIAwsEKSMMP,
-			ClusterName:                    "clusterv3-eks",
+			ClusterName:                    "cluster-eks",
 			ControlPlaneMachineCount:       ptr.To[int](1),
 			WorkerMachineCount:             ptr.To[int](1),
 			GitAddr:                        gitAddress,
@@ -148,11 +197,27 @@ var _ = Describe("[GCP] [GKE] - [management.cattle.io/v3] Create and delete CAPI
 	})
 
 	specs.CreateMgmtV3UsingGitOpsSpec(ctx, func() specs.CreateMgmtV3UsingGitOpsSpecInput {
+		testenv.CAPIOperatorDeployProvider(ctx, testenv.CAPIOperatorDeployProviderInput{
+			BootstrapClusterProxy: bootstrapClusterProxy,
+			CAPIProvidersSecretsYAML: [][]byte{
+				e2e.GCPProviderSecret,
+			},
+			CAPIProvidersYAML: [][]byte{
+				e2e.GCPProvider,
+			},
+			WaitForDeployments: []testenv.NamespaceName{
+				{
+					Name:      "capg-controller-manager",
+					Namespace: "capg-system",
+				},
+			},
+		})
+
 		return specs.CreateMgmtV3UsingGitOpsSpecInput{
 			E2EConfig:                      e2e.LoadE2EConfig(),
 			BootstrapClusterProxy:          bootstrapClusterProxy,
 			ClusterTemplate:                e2e.CAPIGCPGKE,
-			ClusterName:                    "clusterv3-gke",
+			ClusterName:                    "cluster-gke",
 			ControlPlaneMachineCount:       ptr.To[int](1),
 			WorkerMachineCount:             ptr.To[int](1),
 			GitAddr:                        gitAddress,
@@ -176,6 +241,22 @@ var _ = Describe("[vSphere] [Kubeadm] Create and delete CAPI cluster functionali
 	})
 
 	specs.CreateMgmtV3UsingGitOpsSpec(ctx, func() specs.CreateMgmtV3UsingGitOpsSpecInput {
+		By("Running local vSphere tests, deploying vSphere infrastructure provider")
+
+		testenv.CAPIOperatorDeployProvider(ctx, testenv.CAPIOperatorDeployProviderInput{
+			BootstrapClusterProxy: setupClusterResult.BootstrapClusterProxy,
+			CAPIProvidersSecretsYAML: [][]byte{
+				e2e.VSphereProviderSecret,
+			},
+			CAPIProvidersYAML: [][]byte{e2e.CapvProvider},
+			WaitForDeployments: []testenv.NamespaceName{
+				{
+					Name:      "capv-controller-manager",
+					Namespace: "capv-system",
+				},
+			},
+		})
+
 		return specs.CreateMgmtV3UsingGitOpsSpecInput{
 			E2EConfig:                 e2e.LoadE2EConfig(),
 			BootstrapClusterProxy:     bootstrapClusterProxy,

test/e2e/suites/import-gitops-v3/suite_test.go

@@ -21,13 +21,13 @@ package import_gitops_v3
 
 import (
 	"context"
-	"encoding/json"
 	"testing"
 
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 	"github.com/rancher/turtles/test/e2e"
 	"github.com/rancher/turtles/test/testenv"
+	"k8s.io/apimachinery/pkg/util/json"
 	capiframework "sigs.k8s.io/cluster-api/test/framework"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/log/zap"
@@ -73,59 +73,8 @@ var _ = SynchronizedBeforeSuite(
 
 		testenv.DeployRancherTurtles(ctx, testenv.DeployRancherTurtlesInput{
 			BootstrapClusterProxy: setupClusterResult.BootstrapClusterProxy,
-			CAPIProvidersYAML:     e2e.CapiProviders,
-			WaitForDeployments: append(
-				testenv.DefaultDeployments,
-				testenv.NamespaceName{
-					Name:      "caapf-controller-manager",
-					Namespace: e2e.RancherTurtlesNamespace,
-				}),
 		})
 
-		if !shortTestOnly() && !localTestOnly() {
-			By("Running full tests, deploying additional infrastructure providers")
-
-			testenv.CAPIOperatorDeployProvider(ctx, testenv.CAPIOperatorDeployProviderInput{
-				BootstrapClusterProxy: setupClusterResult.BootstrapClusterProxy,
-				CAPIProvidersSecretsYAML: [][]byte{
-					e2e.AWSProviderSecret,
-					e2e.AzureIdentitySecret,
-					e2e.GCPProviderSecret,
-				},
-				CAPIProvidersYAML: e2e.FullProviders,
-				WaitForDeployments: []testenv.NamespaceName{
-					{
-						Name:      "capa-controller-manager",
-						Namespace: "capa-system",
-					},
-					{
-						Name:      "capz-controller-manager",
-						Namespace: "capz-system",
-					},
-					{
-						Name:      "capg-controller-manager",
-						Namespace: "capg-system",
-					},
-				},
-			})
-		} else if Label(e2e.LocalTestLabel).MatchesLabelFilter(GinkgoLabelFilter()) {
-			By("Running local vSphere tests, deploying vSphere infrastructure provider")
-
-			testenv.CAPIOperatorDeployProvider(ctx, testenv.CAPIOperatorDeployProviderInput{
-				BootstrapClusterProxy: setupClusterResult.BootstrapClusterProxy,
-				CAPIProvidersSecretsYAML: [][]byte{
-					e2e.VSphereProviderSecret,
-				},
-				CAPIProvidersYAML: e2e.CapvProvider,
-				WaitForDeployments: []testenv.NamespaceName{
-					{
-						Name:      "capv-controller-manager",
-						Namespace: "capv-system",
-					},
-				},
-			})
-		}
-
 		giteaResult := testenv.DeployGitea(ctx, testenv.DeployGiteaInput{
 			BootstrapClusterProxy: setupClusterResult.BootstrapClusterProxy,
 			ValuesFile:            e2e.GiteaValues,
@@ -158,11 +107,11 @@ var _ = SynchronizedAfterSuite(
 	},
 	func() {
 		testenv.UninstallGitea(ctx, testenv.UninstallGiteaInput{
-			BootstrapClusterProxy: setupClusterResult.BootstrapClusterProxy,
+			BootstrapClusterProxy: bootstrapClusterProxy,
 		})
 
 		testenv.UninstallRancherTurtles(ctx, testenv.UninstallRancherTurtlesInput{
-			BootstrapClusterProxy: setupClusterResult.BootstrapClusterProxy,
+			BootstrapClusterProxy: bootstrapClusterProxy,
 		})
 
 		testenv.CleanupTestCluster(ctx, testenv.CleanupTestClusterInput{

test/testenv/operator.go

@@ -44,7 +44,7 @@ type CAPIOperatorDeployProviderInput struct {
 	CAPIProvidersSecretsYAML [][]byte
 
 	// CAPIProvidersYAML is the YAML representation of the CAPI providers.
-	CAPIProvidersYAML []byte
+	CAPIProvidersYAML [][]byte
 
 	// TemplateData is the data used for templating.
 	TemplateData TemplateData
@@ -90,8 +90,12 @@ func CAPIOperatorDeployProvider(ctx context.Context, input CAPIOperatorDeployPro
 		})).To(Succeed(), "Failed to apply secret for capi providers")
 	}
 
-	By("Adding CAPI Operator providers")
-	Expect(turtlesframework.Apply(ctx, input.BootstrapClusterProxy, input.CAPIProvidersYAML)).To(Succeed(), "Failed to add CAPI operator providers")
+	for _, provider := range input.CAPIProvidersYAML {
+		provider := provider
+
+		By("Adding CAPI Operator provider")
+		Expect(turtlesframework.Apply(ctx, input.BootstrapClusterProxy, provider)).To(Succeed(), "Failed to add CAPI operator providers")
+	}
 
 	if len(input.WaitForDeployments) == 0 {
 		By("No deployments to wait for")

test/testenv/turtles.go

@@ -136,7 +136,6 @@ func DeployRancherTurtles(ctx context.Context, input DeployRancherTurtlesInput)
 
 	Expect(ctx).NotTo(BeNil(), "ctx is required for DeployRancherTurtles")
 	Expect(input.BootstrapClusterProxy).ToNot(BeNil(), "BootstrapClusterProxy is required for DeployRancherTurtles")
-	Expect(input.CAPIProvidersYAML).ToNot(BeNil(), "CAPIProvidersYAML is required for DeployRancherTurtles")
 	Expect(input.TurtlesChartPath).ToNot(BeEmpty(), "ChartPath is required for DeployRancherTurtles")
 	Expect(input.HelmBinaryPath).ToNot(BeEmpty(), "HelmBinaryPath is required for DeployRancherTurtles")
 	Expect(input.WaitDeploymentsReadyInterval).ToNot(BeNil(), "WaitDeploymentsReadyInterval is required for DeployRancherTurtles")
@@ -228,8 +227,10 @@ func DeployRancherTurtles(ctx context.Context, input DeployRancherTurtlesInput)
 		Expect(fmt.Errorf("Unable to perform chart upgrade: %w\nOutput: %s, Command: %s", err, out, strings.Join(fullCommand, " "))).ToNot(HaveOccurred())
 	}
 
-	By("Adding CAPI infrastructure providers")
-	Expect(turtlesframework.Apply(ctx, input.BootstrapClusterProxy, input.CAPIProvidersYAML)).To(Succeed())
+	if input.CAPIProvidersYAML != nil {
+		By("Adding CAPI infrastructure providers")
+		Expect(turtlesframework.Apply(ctx, input.BootstrapClusterProxy, input.CAPIProvidersYAML)).To(Succeed())
+	}
 
 	if input.WaitForDeployments != nil {
 		By("Waiting for provider deployments to be ready")