Skip to content

Commit

Permalink
Move TCP connectivity test functions to new module
Browse files Browse the repository at this point in the history
Moved RunConnectivityTest, RunNoConnectivityTest, et al from dataplane
to a new "tcp" module/package. This separates the actual connectivity code
from the dataplane Ginko specs so one can use the connectivity test code
from other specs w/o also causing the dataplane specs to run.

Signed-off-by: Tom Pantelis <tompantelis@gmail.com>
  • Loading branch information
tpantelis authored and mangelajo committed Feb 10, 2020
1 parent ce8db19 commit 6cca728
Show file tree
Hide file tree
Showing 4 changed files with 109 additions and 102 deletions.
14 changes: 7 additions & 7 deletions test/e2e/cluster/add_remove_cluster.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,8 @@ import (
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"

"github.com/submariner-io/submariner/test/e2e/dataplane"
"github.com/submariner-io/submariner/test/e2e/framework"
"github.com/submariner-io/submariner/test/e2e/tcp"
)

var _ = PDescribe("[expansion] Test expanding/shrinking an existing cluster fleet", func() {
Expand All @@ -23,10 +23,10 @@ var _ = PDescribe("[expansion] Test expanding/shrinking an existing cluster flee
Expect(gatewayNode).To(HaveLen(0), fmt.Sprintf("Expected no gateway node on %q", framework.ClusterC))

By(fmt.Sprintf("Verifying that a pod in cluster %q cannot connect to a pod in cluster %q", clusterAName, clusterCName))
dataplane.RunNoConnectivityTest(f, false, framework.NonGatewayNode, framework.GatewayNode, framework.ClusterC, framework.ClusterA)
tcp.RunNoConnectivityTest(f, false, framework.NonGatewayNode, framework.GatewayNode, framework.ClusterC, framework.ClusterA)

By(fmt.Sprintf("Verifying that a pod in cluster %q cannot connect to a service in cluster %q", clusterBName, clusterCName))
dataplane.RunNoConnectivityTest(f, true, framework.NonGatewayNode, framework.NonGatewayNode, framework.ClusterC, framework.ClusterB)
tcp.RunNoConnectivityTest(f, true, framework.NonGatewayNode, framework.NonGatewayNode, framework.ClusterC, framework.ClusterB)

nonGatewayNodes := f.FindNodesByGatewayLabel(framework.ClusterC, false)
Expect(nonGatewayNodes).ToNot(HaveLen(0), fmt.Sprintf("No non-gateway nodes found on %q", clusterCName))
Expand All @@ -38,17 +38,17 @@ var _ = PDescribe("[expansion] Test expanding/shrinking an existing cluster flee
By(fmt.Sprintf("Found submariner engine pod %q on %q", enginePod.Name, clusterCName))

By(fmt.Sprintf("Checking connectivity between clusters"))
dataplane.RunConnectivityTest(f, false, framework.PodNetworking, framework.GatewayNode, framework.GatewayNode, framework.ClusterC, framework.ClusterB)
dataplane.RunConnectivityTest(f, true, framework.PodNetworking, framework.NonGatewayNode, framework.NonGatewayNode, framework.ClusterC, framework.ClusterA)
tcp.RunConnectivityTest(f, false, framework.PodNetworking, framework.GatewayNode, framework.GatewayNode, framework.ClusterC, framework.ClusterB)
tcp.RunConnectivityTest(f, true, framework.PodNetworking, framework.NonGatewayNode, framework.NonGatewayNode, framework.ClusterC, framework.ClusterA)

By(fmt.Sprintf("Removing cluster %q by unsetting the gateway label and deleting submariner engine pod %q", clusterCName, enginePod.Name))
f.SetGatewayLabelOnNode(framework.ClusterC, nonGatewayNode, false)
f.DeletePod(framework.ClusterC, enginePod.Name, framework.TestContext.SubmarinerNamespace)

By(fmt.Sprintf("Verifying that a pod in cluster %q cannot connect to a service in cluster %q", clusterAName, clusterCName))
dataplane.RunNoConnectivityTest(f, false, framework.NonGatewayNode, framework.GatewayNode, framework.ClusterC, framework.ClusterA)
tcp.RunNoConnectivityTest(f, false, framework.NonGatewayNode, framework.GatewayNode, framework.ClusterC, framework.ClusterA)

By(fmt.Sprintf("Verifying that a pod in cluster %q cannot connect to a pod in cluster %q", clusterBName, clusterCName))
dataplane.RunNoConnectivityTest(f, true, framework.NonGatewayNode, framework.NonGatewayNode, framework.ClusterC, framework.ClusterB)
tcp.RunNoConnectivityTest(f, true, framework.NonGatewayNode, framework.NonGatewayNode, framework.ClusterC, framework.ClusterB)
})
})
91 changes: 3 additions & 88 deletions test/e2e/dataplane/tcp_pod_connectivity.go
Original file line number Diff line number Diff line change
@@ -1,12 +1,9 @@
package dataplane

import (
"fmt"

"github.com/submariner-io/submariner/test/e2e/framework"

. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/submariner-io/submariner/test/e2e/framework"
"github.com/submariner-io/submariner/test/e2e/tcp"
)

var _ = Describe("[dataplane] Basic TCP connectivity tests across clusters without discovery", func() {
Expand All @@ -16,7 +13,7 @@ var _ = Describe("[dataplane] Basic TCP connectivity tests across clusters witho

verifyInteraction := func(listenerScheduling, connectorScheduling framework.NetworkPodScheduling) {
It("should have sent the expected data from the pod to the other pod", func() {
RunConnectivityTest(f, useService, networkType, listenerScheduling, connectorScheduling, framework.ClusterB, framework.ClusterA)
tcp.RunConnectivityTest(f, useService, networkType, listenerScheduling, connectorScheduling, framework.ClusterB, framework.ClusterA)
})
}

Expand Down Expand Up @@ -81,85 +78,3 @@ var _ = Describe("[dataplane] Basic TCP connectivity tests across clusters witho
})
})
})

func RunConnectivityTest(f *framework.Framework, useService bool, networkType bool, listenerScheduling framework.NetworkPodScheduling, connectorScheduling framework.NetworkPodScheduling, listenerCluster framework.ClusterIndex, connectorCluster framework.ClusterIndex) (*framework.NetworkPod, *framework.NetworkPod) {

listenerPod, connectorPod := createPods(f, useService, networkType, listenerScheduling, connectorScheduling, listenerCluster, connectorCluster,
framework.TestContext.ConnectionTimeout, framework.TestContext.ConnectionAttempts)
listenerPod.CheckSuccessfulFinish()
connectorPod.CheckSuccessfulFinish()

By("Verifying that the listener got the connector's data and the connector got the listener's data")
Expect(listenerPod.TerminationMessage).To(ContainSubstring(connectorPod.Config.Data))
Expect(connectorPod.TerminationMessage).To(ContainSubstring(listenerPod.Config.Data))

// Normally, Submariner preserves the sourceIP for inter-cluster traffic. However, when a POD is using the
// HostNetwork, it does not get an IPAddress from the podCIDR and it uses the HostIP. Submariner, for such PODs,
// would MASQUERADE the external traffic from that POD to the corresponding CNI interface ip-address on that
// Host. So, we skip source-ip validation for PODs using HostNetworking.
if networkType == framework.PodNetworking {
By("Verifying the output of listener pod which must contain the source IP")
Expect(listenerPod.TerminationMessage).To(ContainSubstring(connectorPod.Pod.Status.PodIP))
}

// Return the pods in case further verification is needed
return listenerPod, connectorPod
}

func RunNoConnectivityTest(f *framework.Framework, useService bool, listenerScheduling framework.NetworkPodScheduling, connectorScheduling framework.NetworkPodScheduling, listenerCluster framework.ClusterIndex, connectorCluster framework.ClusterIndex) (*framework.NetworkPod, *framework.NetworkPod) {
listenerPod, connectorPod := createPods(f, useService, framework.PodNetworking, listenerScheduling, connectorScheduling, listenerCluster, connectorCluster, 5, 1)

By("Verifying that listener pod exits with non-zero code and timed out message")
Expect(listenerPod.TerminationMessage).To(ContainSubstring("nc: timeout"))
Expect(listenerPod.TerminationCode).To(Equal(int32(1)))

By("Verifying that connector pod exists with zero code but times out")
Expect(connectorPod.TerminationMessage).To(ContainSubstring("Connection timed out"))
Expect(connectorPod.TerminationCode).To(Equal(int32(0)))

// Return the pods in case further verification is needed
return listenerPod, connectorPod
}

func createPods(f *framework.Framework, useService bool, networkType bool, listenerScheduling framework.NetworkPodScheduling, connectorScheduling framework.NetworkPodScheduling, listenerCluster framework.ClusterIndex,
connectorCluster framework.ClusterIndex, connectionTimeout uint, connectionAttempts uint) (*framework.NetworkPod, *framework.NetworkPod) {

By(fmt.Sprintf("Creating a listener pod in cluster %q, which will wait for a handshake over TCP", framework.TestContext.ClusterIDs[listenerCluster]))
listenerPod := f.NewNetworkPod(&framework.NetworkPodConfig{
Type: framework.ListenerPod,
Cluster: listenerCluster,
Scheduling: listenerScheduling,
ConnectionTimeout: connectionTimeout,
ConnectionAttempts: connectionAttempts,
})

remoteIP := listenerPod.Pod.Status.PodIP
if useService {
By(fmt.Sprintf("Pointing a service ClusterIP to the listener pod in cluster %q", framework.TestContext.ClusterIDs[listenerCluster]))
service := listenerPod.CreateService()
remoteIP = service.Spec.ClusterIP
}

framework.Logf("Will send traffic to IP: %v", remoteIP)

By(fmt.Sprintf("Creating a connector pod in cluster %q, which will attempt the specific UUID handshake over TCP", framework.TestContext.ClusterIDs[connectorCluster]))
connectorPod := f.NewNetworkPod(&framework.NetworkPodConfig{
Type: framework.ConnectorPod,
Cluster: connectorCluster,
Scheduling: connectorScheduling,
RemoteIP: remoteIP,
ConnectionTimeout: connectionTimeout,
ConnectionAttempts: connectionAttempts,
NetworkType: networkType,
})

By(fmt.Sprintf("Waiting for the listener pod %q to exit, returning what listener sent", listenerPod.Pod.Name))
listenerPod.AwaitFinish()

By(fmt.Sprintf("Waiting for the connector pod %q to exit, returning what connector sent", connectorPod.Pod.Name))
connectorPod.AwaitFinish()

framework.Logf("Connector pod has IP: %s", connectorPod.Pod.Status.PodIP)

return listenerPod, connectorPod
}
14 changes: 7 additions & 7 deletions test/e2e/redundancy/gateway_failover.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,8 @@ import (

. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/submariner-io/submariner/test/e2e/dataplane"
"github.com/submariner-io/submariner/test/e2e/framework"
"github.com/submariner-io/submariner/test/e2e/tcp"
appsv1 "k8s.io/api/apps/v1"
)

Expand Down Expand Up @@ -60,10 +60,10 @@ func testOneGatewayNode(f *framework.Framework) {
By(fmt.Sprintf("Found new submariner engine pod %q", newEnginePod.Name))

By(fmt.Sprintf("Verifying TCP connectivity from gateway node on %q to gateway node on %q", clusterBName, clusterAName))
dataplane.RunConnectivityTest(f, false, framework.PodNetworking, framework.GatewayNode, framework.GatewayNode, framework.ClusterA, framework.ClusterB)
tcp.RunConnectivityTest(f, false, framework.PodNetworking, framework.GatewayNode, framework.GatewayNode, framework.ClusterA, framework.ClusterB)

By(fmt.Sprintf("Verifying TCP connectivity from non-gateway node on %q to non-gateway node on %q", clusterBName, clusterAName))
dataplane.RunConnectivityTest(f, false, framework.PodNetworking, framework.NonGatewayNode, framework.NonGatewayNode, framework.ClusterA, framework.ClusterB)
tcp.RunConnectivityTest(f, false, framework.PodNetworking, framework.NonGatewayNode, framework.NonGatewayNode, framework.ClusterA, framework.ClusterB)
}

func testTwoGatewayNodesWithOneReplica(f *framework.Framework) {
Expand Down Expand Up @@ -113,10 +113,10 @@ func testTwoGatewayNodesWithOneReplica(f *framework.Framework) {
By(fmt.Sprintf("Found new submariner endpoint for %q: %#v", clusterAName, newSubmEndpoint))

By(fmt.Sprintf("Verifying TCP connectivity from gateway node on %q to gateway node on %q", clusterBName, clusterAName))
dataplane.RunConnectivityTest(f, false, framework.PodNetworking, framework.GatewayNode, framework.GatewayNode, framework.ClusterA, framework.ClusterB)
tcp.RunConnectivityTest(f, false, framework.PodNetworking, framework.GatewayNode, framework.GatewayNode, framework.ClusterA, framework.ClusterB)

By(fmt.Sprintf("Verifying TCP connectivity from non-gateway node on %q to non-gateway node on %q", clusterBName, clusterAName))
dataplane.RunConnectivityTest(f, false, framework.PodNetworking, framework.NonGatewayNode, framework.NonGatewayNode, framework.ClusterA, framework.ClusterB)
tcp.RunConnectivityTest(f, false, framework.PodNetworking, framework.NonGatewayNode, framework.NonGatewayNode, framework.ClusterA, framework.ClusterB)
}

func testTwoGatewayNodesWithTwoReplicas(f *framework.Framework, deployment *appsv1.Deployment) {
Expand Down Expand Up @@ -160,8 +160,8 @@ func testTwoGatewayNodesWithTwoReplicas(f *framework.Framework, deployment *apps
By(fmt.Sprintf("Found new submariner endpoint for %q: %#v", clusterAName, newSubmEndpoint))

By(fmt.Sprintf("Verifying TCP connectivity from gateway node on %q to gateway node on %q", clusterBName, clusterAName))
dataplane.RunConnectivityTest(f, false, framework.PodNetworking, framework.GatewayNode, framework.GatewayNode, framework.ClusterA, framework.ClusterB)
tcp.RunConnectivityTest(f, false, framework.PodNetworking, framework.GatewayNode, framework.GatewayNode, framework.ClusterA, framework.ClusterB)

By(fmt.Sprintf("Verifying TCP connectivity from non-gateway node on %q to non-gateway node on %q", clusterBName, clusterAName))
dataplane.RunConnectivityTest(f, false, framework.PodNetworking, framework.NonGatewayNode, framework.NonGatewayNode, framework.ClusterA, framework.ClusterB)
tcp.RunConnectivityTest(f, false, framework.PodNetworking, framework.NonGatewayNode, framework.NonGatewayNode, framework.ClusterA, framework.ClusterB)
}
92 changes: 92 additions & 0 deletions test/e2e/tcp/connectivity.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,92 @@
package tcp

import (
"fmt"

"github.com/submariner-io/submariner/test/e2e/framework"

. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
)

func RunConnectivityTest(f *framework.Framework, useService bool, networkType bool, listenerScheduling framework.NetworkPodScheduling, connectorScheduling framework.NetworkPodScheduling, listenerCluster framework.ClusterIndex, connectorCluster framework.ClusterIndex) (*framework.NetworkPod, *framework.NetworkPod) {

listenerPod, connectorPod := createPods(f, useService, networkType, listenerScheduling, connectorScheduling, listenerCluster, connectorCluster,
framework.TestContext.ConnectionTimeout, framework.TestContext.ConnectionAttempts)
listenerPod.CheckSuccessfulFinish()
connectorPod.CheckSuccessfulFinish()

By("Verifying that the listener got the connector's data and the connector got the listener's data")
Expect(listenerPod.TerminationMessage).To(ContainSubstring(connectorPod.Config.Data))
Expect(connectorPod.TerminationMessage).To(ContainSubstring(listenerPod.Config.Data))

// Normally, Submariner preserves the sourceIP for inter-cluster traffic. However, when a POD is using the
// HostNetwork, it does not get an IPAddress from the podCIDR and it uses the HostIP. Submariner, for such PODs,
// would MASQUERADE the external traffic from that POD to the corresponding CNI interface ip-address on that
// Host. So, we skip source-ip validation for PODs using HostNetworking.
if networkType == framework.PodNetworking {
By("Verifying the output of listener pod which must contain the source IP")
Expect(listenerPod.TerminationMessage).To(ContainSubstring(connectorPod.Pod.Status.PodIP))
}

// Return the pods in case further verification is needed
return listenerPod, connectorPod
}

func RunNoConnectivityTest(f *framework.Framework, useService bool, listenerScheduling framework.NetworkPodScheduling, connectorScheduling framework.NetworkPodScheduling, listenerCluster framework.ClusterIndex, connectorCluster framework.ClusterIndex) (*framework.NetworkPod, *framework.NetworkPod) {
listenerPod, connectorPod := createPods(f, useService, framework.PodNetworking, listenerScheduling, connectorScheduling, listenerCluster, connectorCluster, 5, 1)

By("Verifying that listener pod exits with non-zero code and timed out message")
Expect(listenerPod.TerminationMessage).To(ContainSubstring("nc: timeout"))
Expect(listenerPod.TerminationCode).To(Equal(int32(1)))

By("Verifying that connector pod exists with zero code but times out")
Expect(connectorPod.TerminationMessage).To(ContainSubstring("Connection timed out"))
Expect(connectorPod.TerminationCode).To(Equal(int32(0)))

// Return the pods in case further verification is needed
return listenerPod, connectorPod
}

func createPods(f *framework.Framework, useService bool, networkType bool, listenerScheduling framework.NetworkPodScheduling, connectorScheduling framework.NetworkPodScheduling, listenerCluster framework.ClusterIndex,
connectorCluster framework.ClusterIndex, connectionTimeout uint, connectionAttempts uint) (*framework.NetworkPod, *framework.NetworkPod) {

By(fmt.Sprintf("Creating a listener pod in cluster %q, which will wait for a handshake over TCP", framework.TestContext.ClusterIDs[listenerCluster]))
listenerPod := f.NewNetworkPod(&framework.NetworkPodConfig{
Type: framework.ListenerPod,
Cluster: listenerCluster,
Scheduling: listenerScheduling,
ConnectionTimeout: connectionTimeout,
ConnectionAttempts: connectionAttempts,
})

remoteIP := listenerPod.Pod.Status.PodIP
if useService {
By(fmt.Sprintf("Pointing a service ClusterIP to the listener pod in cluster %q", framework.TestContext.ClusterIDs[listenerCluster]))
service := listenerPod.CreateService()
remoteIP = service.Spec.ClusterIP
}

framework.Logf("Will send traffic to IP: %v", remoteIP)

By(fmt.Sprintf("Creating a connector pod in cluster %q, which will attempt the specific UUID handshake over TCP", framework.TestContext.ClusterIDs[connectorCluster]))
connectorPod := f.NewNetworkPod(&framework.NetworkPodConfig{
Type: framework.ConnectorPod,
Cluster: connectorCluster,
Scheduling: connectorScheduling,
RemoteIP: remoteIP,
ConnectionTimeout: connectionTimeout,
ConnectionAttempts: connectionAttempts,
NetworkType: networkType,
})

By(fmt.Sprintf("Waiting for the listener pod %q to exit, returning what listener sent", listenerPod.Pod.Name))
listenerPod.AwaitFinish()

By(fmt.Sprintf("Waiting for the connector pod %q to exit, returning what connector sent", connectorPod.Pod.Name))
connectorPod.AwaitFinish()

framework.Logf("Connector pod has IP: %s", connectorPod.Pod.Status.PodIP)

return listenerPod, connectorPod
}

0 comments on commit 6cca728

Please sign in to comment.