Add first gateway failover scenario

One gateway node configured when the active submariner engine pod fails.

Signed-off-by: Tom Pantelis <tompantelis@gmail.com>

test/e2e/e2e_test.go

@@ -7,6 +7,7 @@ import (
 
 	_ "github.com/submariner-io/submariner/test/e2e/dataplane"
 	_ "github.com/submariner-io/submariner/test/e2e/example"
+	_ "github.com/submariner-io/submariner/test/e2e/redundancy"
 )
 
 func init() {

test/e2e/framework/pods.go

@@ -16,11 +16,17 @@ func (f *Framework) AwaitPodsByAppLabel(cluster ClusterIndex, appName string, na
 		})
 	}, func(result interface{}) (bool, string, error) {
 		pods := result.(*v1.PodList)
-		if expectedCount < 0 || len(pods.Items) == expectedCount {
-			return true, "", nil
+		if expectedCount >= 0 && len(pods.Items) != expectedCount {
+			return false, fmt.Sprintf("Actual pod count %d does not match the expected pod count %d", len(pods.Items), expectedCount), nil
 		}
 
-		return false, fmt.Sprintf("Actual pod count %d does not match the expected pod count %d", len(pods.Items), expectedCount), nil
+		for _, pod := range pods.Items {
+			if pod.Status.Phase != v1.PodRunning {
+				return false, fmt.Sprintf("Status for pod %q is %v", pod.Name, pod.Status.Phase), nil
+			}
+		}
+
+		return true, "", nil
 	}).(*v1.PodList)
 }
 
@@ -32,7 +38,7 @@ func (f *Framework) AwaitSubmarinerEnginePod(cluster ClusterIndex) *v1.Pod {
 
 // DeletePod deletes the pod for the given name and namespace.
 func (f *Framework) DeletePod(cluster ClusterIndex, podName string, namespace string) {
-	AwaitUntil("list pods", func() (interface{}, error) {
+	AwaitUntil("delete pod", func() (interface{}, error) {
 		return nil, f.ClusterClients[cluster].CoreV1().Pods(namespace).Delete(podName, &metav1.DeleteOptions{})
 	}, NoopCheckResult)
 }

test/e2e/redundancy/gateway_failover.go

@@ -0,0 +1,42 @@
+package redundancy
+
+import (
+	"fmt"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	"github.com/submariner-io/submariner/test/e2e/dataplane"
+	"github.com/submariner-io/submariner/test/e2e/framework"
+)
+
+var _ = Describe("[redundancy] Gateway fail-over tests", func() {
+	f := framework.NewDefaultFramework("gateway-redundancy")
+
+	When("one gateway node is configured and the submariner engine pod fails", func() {
+		It("should start a new submariner engine pod and be able to connect from another cluster", func() {
+			testOneGatewayNode(f)
+		})
+	})
+})
+
+func testOneGatewayNode(f *framework.Framework) {
+	clusterAName := framework.TestContext.KubeContexts[framework.ClusterA]
+	clusterBName := framework.TestContext.KubeContexts[framework.ClusterB]
+
+	By(fmt.Sprintf("Sanity check - ensuring there's only one gateway node on %q", clusterAName))
+	gatewayNodes := f.FindNodesByGatewayLabel(framework.ClusterA, true)
+	Expect(gatewayNodes).To(HaveLen(1), fmt.Sprintf("Expected only one gateway node on %q", clusterAName))
+
+	enginePod := f.AwaitSubmarinerEnginePod(framework.ClusterA)
+	By(fmt.Sprintf("Found submariner engine pod %q on %q", enginePod.Name, clusterAName))
+
+	By(fmt.Sprintf("Deleting submariner engine pod %q", enginePod.Name))
+	f.DeletePod(framework.ClusterA, enginePod.Name, framework.TestContext.SubmarinerNamespace)
+
+	newEnginePod := f.AwaitSubmarinerEnginePod(framework.ClusterA)
+	By(fmt.Sprintf("Found new submariner engine pod %q", newEnginePod.Name))
+
+	By(fmt.Sprintf("Verifying TCP connectivity from %q to %q", clusterBName, clusterAName))
+	dataplane.RunConnectivityTest(f, false, framework.GatewayNode, framework.GatewayNode, framework.ClusterA, framework.ClusterB)
+	dataplane.RunConnectivityTest(f, false, framework.NonGatewayNode, framework.NonGatewayNode, framework.ClusterA, framework.ClusterB)
+}