Add --extracontext arg to subctl verify

...to run lighthouse tests that require a third cluster.

Added a system test using the new arg that is run when lighthouse is
deployed. This requires the cluster settings to specify a third cluster
which can be problematic wrt CI env resources. So the System Test
globalnet and lighthouse options were split into 2 jobs with lighthouse
using a separate cluster settings containing the third cluster and only
the control-plane node.

Fixes #276

Signed-off-by: Tom Pantelis <tompantelis@gmail.com>

.github/workflows/system.yml

@@ -14,7 +14,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        using: ['', 'globalnet lighthouse']
+        using: ['', 'globalnet', 'lighthouse']
     steps:
       - name: Check out the repository
         uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11

.shipyard.lighthouse.yml

@@ -0,0 +1,7 @@
+---
+submariner: true
+nodes: control-plane
+clusters:
+  cluster1:
+  cluster2:
+  cluster3:

cmd/subctl/verify.go

@@ -62,18 +62,21 @@ var (
 
 var verifyRestConfigProducer = restconfig.NewProducer().
 	WithPrefixedContext("to").
+	WithPrefixedContext("extra").
 	WithDefaultNamespace(constants.OperatorNamespace)
 
 var verifyCmd = &cobra.Command{
-	Use:   "verify --context <kubeContext1> --tocontext <kubeContext2>",
+	Use:   "verify --context <kubeContext1> --tocontext <kubeContext2> [--extracontext <kubeContext3>]",
 	Short: "Run verifications between two clusters",
-	Long: `This command performs various tests to verify that a Submariner deployment between two clusters
-is functioning properly. The verifications performed are controlled by the --only and --enable-disruptive
-flags. All verifications listed in --only are performed with special handling for those deemed as disruptive.
-A disruptive verification is one that changes the state of the clusters as a side effect. If running the
-command interactively, you will be prompted for confirmation to perform disruptive verifications unless
-the --enable-disruptive flag is also specified. If running non-interactively (that is with no stdin),
---enable-disruptive must be specified otherwise disruptive verifications are skipped.
+	Long: `This command performs various tests to verify that a Submariner deployment between two clusters,
+specified via the --context and --tocontext args, is functioning properly. Some Service Discovery tests require a third cluster,
+specified via the --extracontext arg, to verify additional functionality. If the third cluster is not specified,
+those tests are skipped. The verifications performed are controlled by the --only and --enable-disruptive flags.
+All verifications listed in --only are performed with special handling for those deemed as disruptive. A disruptive
+verification is one that changes the state of the clusters as a side effect. If running the command interactively,
+you will be prompted for confirmation to perform disruptive verifications unless the --enable-disruptive flag is
+also specified. If running non-interactively (that is with no stdin), --enable-disruptive must be specified otherwise
+disruptive verifications are skipped.
 
 The following verifications are deemed disruptive:
 
@@ -86,7 +89,16 @@ The following verifications are deemed disruptive:
 				toContextPresent, err := verifyRestConfigProducer.RunOnSelectedPrefixedContext(
 					"to",
 					func(toClusterInfo *cluster.Info, _ string, status reporter.Interface) error {
-						return runVerify(fromClusterInfo, toClusterInfo, namespace, determineSpecLabelsToVerify())
+						extraContextPresent, err := verifyRestConfigProducer.RunOnSelectedPrefixedContext(
+							"extra",
+							func(extraClusterInfo *cluster.Info, _ string, status reporter.Interface) error {
+								return runVerify(fromClusterInfo, toClusterInfo, extraClusterInfo, namespace, determineSpecLabelsToVerify())
+							}, status)
+						if extraContextPresent {
+							return err //nolint:wrapcheck // No need to wrap errors here.
+						}
+
+						return runVerify(fromClusterInfo, toClusterInfo, nil, namespace, determineSpecLabelsToVerify())
 					}, status)
 
 				if toContextPresent {
@@ -283,11 +295,17 @@ prompt for confirmation therefore you must specify --enable-disruptive to run th
 	return labels
 }
 
-func runVerify(fromClusterInfo, toClusterInfo *cluster.Info, namespace string, specLabels []string) error {
+func runVerify(fromClusterInfo, toClusterInfo, extraClusterInfo *cluster.Info, namespace string, specLabels []string) error {
 	framework.RestConfigs = []*rest.Config{fromClusterInfo.RestConfig, toClusterInfo.RestConfig}
 	framework.TestContext.ClusterIDs = []string{fromClusterInfo.Name, toClusterInfo.Name}
 	framework.TestContext.KubeContexts = []string{fromClusterInfo.Name, toClusterInfo.Name}
 
+	if extraClusterInfo != nil {
+		framework.RestConfigs = append(framework.RestConfigs, extraClusterInfo.RestConfig)
+		framework.TestContext.ClusterIDs = append(framework.TestContext.ClusterIDs, extraClusterInfo.Name)
+		framework.TestContext.KubeContexts = append(framework.TestContext.KubeContexts, extraClusterInfo.Name)
+	}
+
 	framework.TestContext.OperationTimeout = operationTimeout
 	framework.TestContext.ConnectionTimeout = connectionTimeout
 	framework.TestContext.ConnectionAttempts = connectionAttempts

scripts/test/system.sh

@@ -148,6 +148,13 @@ function validate_and_clean_broker_info() {
 
 subm_ns=submariner-operator
 submariner_broker_ns=submariner-k8s-broker
+
+if [[ "${LIGHTHOUSE}" != true ]]; then
+  SETTINGS="$DAPPER_SOURCE"/.shipyard.system.yml
+else
+  SETTINGS="$DAPPER_SOURCE"/.shipyard.lighthouse.yml
+fi
+
 load_settings
 declare_kubeconfig
 deploy_env_once
@@ -156,7 +163,8 @@ deploy_env_once
 
 # Test subctl show invocations
 
-_subctl show all | tee /dev/stderr | (sponge ||:) | grep -q 'Cluster "cluster2"'
+# Some clusters may not be connected yet so retry
+with_retries 30 sleep_on_fail 1s _subctl show all | tee /dev/stderr | (sponge ||:) | grep -q 'Cluster "cluster2"'
 # Single-context variants don't say 'Cluster "foo"', check what cluster is considered local
 _subctl show all --context cluster1 | tee /dev/stderr | (sponge ||:) | grep -qv 'cluster2.*local'
 _subctl show all --context cluster2 | tee /dev/stderr | (sponge ||:) | grep -q 'cluster2.*local'
@@ -170,7 +178,8 @@ test_subctl_gather
 
 # Test subctl diagnose invocations
 
-_subctl diagnose all --validation-timeout 20
+# Some clusters may not be connected yet so retry
+with_retries 30 sleep_on_fail 1s _subctl diagnose all --validation-timeout 20
 _subctl diagnose firewall inter-cluster --validation-timeout 20 --kubeconfig "${KUBECONFIGS_DIR}"/kind-config-cluster1 --remoteconfig "${KUBECONFIGS_DIR}"/kind-config-cluster2
 _subctl diagnose firewall inter-cluster --validation-timeout 20 --context cluster1 --remotecontext cluster2
 _subctl diagnose firewall nat-discovery --validation-timeout 20 --kubeconfig "${KUBECONFIGS_DIR}"/kind-config-cluster1 --remoteconfig "${KUBECONFIGS_DIR}"/kind-config-cluster2
@@ -185,28 +194,32 @@ _subctl diagnose firewall nat-discovery --validation-timeout 20 "${KUBECONFIGS_D
 with_context "${clusters[0]}" test_subctl_diagnose_in_cluster
 
 # Test subctl benchmark invocations
+# Note that we skip these tests for lighthouse b/c clusters are deployed with only one node
+if [[ "${LIGHTHOUSE}" != true ]]; then
+  _subctl benchmark latency --context cluster1 | tee /dev/stderr | (sponge ||:) | grep -q 'Performing latency tests from Non-Gateway pod to Gateway pod on cluster "cluster1"'
+  _subctl benchmark latency --context cluster1 --tocontext cluster2 | tee /dev/stderr | (sponge ||:) | grep -qE '(Performing latency tests from Gateway pod on cluster "cluster1" to Gateway pod on cluster "cluster2"|Latency test is not supported with Globalnet enabled, skipping the test)'
 
-_subctl benchmark latency --context cluster1 | tee /dev/stderr | (sponge ||:) | grep -q 'Performing latency tests from Non-Gateway pod to Gateway pod on cluster "cluster1"'
-_subctl benchmark latency --context cluster1 --tocontext cluster2 | tee /dev/stderr | (sponge ||:) | grep -qE '(Performing latency tests from Gateway pod on cluster "cluster1" to Gateway pod on cluster "cluster2"|Latency test is not supported with Globalnet enabled, skipping the test)'
-
-_subctl benchmark throughput --context cluster1 | tee /dev/stderr | (sponge ||:) | grep -q 'Performing throughput tests from Non-Gateway pod to Gateway pod on cluster "cluster1"'
-_subctl benchmark throughput --context cluster1 --tocontext cluster2
+  _subctl benchmark throughput --context cluster1 | tee /dev/stderr | (sponge ||:) | grep -q 'Performing throughput tests from Non-Gateway pod to Gateway pod on cluster "cluster1"'
+  _subctl benchmark throughput --context cluster1 --tocontext cluster2
 
-# Obsolete variant with contexts
-_subctl benchmark latency --intra-cluster --kubecontexts cluster1 && exit 1
-_subctl benchmark latency --kubecontexts cluster1,cluster2 && exit 1
+  # Obsolete variant with contexts
+  _subctl benchmark latency --intra-cluster --kubecontexts cluster1 && exit 1
+  _subctl benchmark latency --kubecontexts cluster1,cluster2 && exit 1
 
-_subctl benchmark throughput --intra-cluster --kubecontexts cluster1 && exit 1
-_subctl benchmark throughput --kubecontexts cluster1,cluster2 && exit 1
+  _subctl benchmark throughput --intra-cluster --kubecontexts cluster1 && exit 1
+  _subctl benchmark throughput --kubecontexts cluster1,cluster2 && exit 1
 
-# Obsolete variant with kubeconfigs
-_subctl benchmark latency "${KUBECONFIGS_DIR}"/kind-config-cluster1 "${KUBECONFIGS_DIR}"/kind-config-cluster2 && exit 1
+  # Obsolete variant with kubeconfigs
+  _subctl benchmark latency "${KUBECONFIGS_DIR}"/kind-config-cluster1 "${KUBECONFIGS_DIR}"/kind-config-cluster2 && exit 1
 
-_subctl benchmark throughput "${KUBECONFIGS_DIR}"/kind-config-cluster1 "${KUBECONFIGS_DIR}"/kind-config-cluster2 && exit 1
+  _subctl benchmark throughput "${KUBECONFIGS_DIR}"/kind-config-cluster1 "${KUBECONFIGS_DIR}"/kind-config-cluster2 && exit 1
 
-# Test subctl verify basic
+  # Test subctl verify basic
 
-_subctl verify --context cluster1 --tocontext cluster2 --only basic-connectivity --verbose
+  _subctl verify --context cluster1 --tocontext cluster2 --only basic-connectivity --verbose
+else
+  _subctl verify --context cluster1 --tocontext cluster2 --extracontext cluster3 --only service-discovery --verbose
+fi
 
 # Test subctl cloud prepare invocations