Fixing tests using consul for dns

examples/vault-consul-ami/vault-consul.json

@@ -1,10 +1,10 @@
 {
-  "min_packer_version": "0.12.0",
+  "min_packer_version": "1.5.4",
   "variables": {
     "aws_region": "us-east-1",
-    "vault_version": "1.5.4",
+    "vault_version": "1.6.1",
     "consul_module_version": "v0.8.0",
-    "consul_version": "1.5.3",
+    "consul_version": "1.9.2",
     "consul_download_url": "{{env `CONSUL_DOWNLOAD_URL`}}",
     "vault_download_url": "{{env `VAULT_DOWNLOAD_URL`}}",
     "install_auth_signing_script": "true",

modules/run-vault/run-vault

@@ -288,6 +288,7 @@ EOF
   local dynamodb_storage_type="storage"
   local s3_config=""
   local vault_storage_backend=""
+  local service_registration=""
   if [[ "$enable_s3_backend" == "true" ]]; then
     s3_config=$(cat <<EOF
 storage "s3" {
@@ -299,9 +300,14 @@ EOF
 )
     consul_storage_type="ha_storage"
     dynamodb_storage_type="ha_storage"
+    service_registration=$(cat <<EOF
+service_registration "consul" {
+  address = "127.0.0.1:8500"
+}\n
+EOF
+)
   fi
 
-
   if [[ "$enable_dynamo_backend" == "true" ]]; then
     vault_storage_backend=$(cat <<EOF
 $dynamodb_storage_type "dynamodb" {
@@ -335,6 +341,7 @@ EOF
   echo -e "$listener_config" >> "$config_path"
   echo -e "$s3_config" >> "$config_path"
   echo -e "$vault_storage_backend" >> "$config_path"
+  echo -e "$service_registration" >> "$config_path"
 
   chown "$user:$user" "$config_path"
 }
@@ -368,6 +375,8 @@ Documentation=https://www.vaultproject.io/docs/
 Requires=network-online.target
 After=network-online.target
 ConditionFileNotEmpty=$config_path
+StartLimitIntervalSec=60
+StartLimitBurst=3
 
 EOF
 )
@@ -392,9 +401,12 @@ KillSignal=SIGINT
 Restart=on-failure
 RestartSec=5
 TimeoutStopSec=30
+StartLimitInterval=60
 StartLimitIntervalSec=60
 StartLimitBurst=3
 LimitNOFILE=65536
+LimitMEMLOCK=infinity
+
 EOF
 )
 

test/vault_cluster_private_test.go

@@ -39,14 +39,21 @@ func runVaultPrivateClusterTest(t *testing.T, amiId string, awsRegion string, ss
 			VAR_CONSUL_CLUSTER_NAME:    fmt.Sprintf("consul-test-%s", uniqueId),
 			VAR_CONSUL_CLUSTER_TAG_KEY: fmt.Sprintf("consul-test-%s", uniqueId),
 		}
-		deployCluster(t, amiId, awsRegion, examplesDir, random.UniqueId(), terraformVars)
+		deployCluster(t, amiId, awsRegion, examplesDir, uniqueId, terraformVars)
+	})
+
+	test_structure.RunTestStage(t, "initialize_unseal", func() {
+		terraformOptions := test_structure.LoadTerraformOptions(t, examplesDir)
+		keyPair := test_structure.LoadEc2KeyPair(t, examplesDir)
+
+		initializeAndUnsealVaultCluster(t, OUTPUT_VAULT_CLUSTER_ASG_NAME, sshUserName, terraformOptions, awsRegion, keyPair)
 	})
 
 	test_structure.RunTestStage(t, "validate", func() {
 		terraformOptions := test_structure.LoadTerraformOptions(t, examplesDir)
 		keyPair := test_structure.LoadEc2KeyPair(t, examplesDir)
 
-		cluster := initializeAndUnsealVaultCluster(t, OUTPUT_VAULT_CLUSTER_ASG_NAME, sshUserName, terraformOptions, awsRegion, keyPair)
+		cluster := getInitializedAndUnsealedVaultCluster(t, OUTPUT_VAULT_CLUSTER_ASG_NAME, sshUserName, terraformOptions, awsRegion, keyPair)
 		testVaultUsesConsulForDns(t, cluster)
 	})
 }

test/vault_cluster_public_test.go

@@ -46,14 +46,22 @@ func runVaultPublicClusterTest(t *testing.T, amiId string, awsRegion string, ssh
 			VAR_CONSUL_CLUSTER_NAME:                          fmt.Sprintf("consul-test-%s", uniqueId),
 			VAR_CONSUL_CLUSTER_TAG_KEY:                       fmt.Sprintf("consul-test-%s", uniqueId),
 		}
-		deployCluster(t, amiId, awsRegion, examplesDir, random.UniqueId(), terraformVars)
+		deployCluster(t, amiId, awsRegion, examplesDir, uniqueId, terraformVars)
 	})
 
-	test_structure.RunTestStage(t, "validate", func() {
+	test_structure.RunTestStage(t, "initialize_unseal", func() {
 		terraformOptions := test_structure.LoadTerraformOptions(t, examplesDir)
 		keyPair := test_structure.LoadEc2KeyPair(t, examplesDir)
 
 		initializeAndUnsealVaultCluster(t, OUTPUT_VAULT_CLUSTER_ASG_NAME, sshUserName, terraformOptions, awsRegion, keyPair)
+	})
+
+	test_structure.RunTestStage(t, "validate", func() {
+		terraformOptions := test_structure.LoadTerraformOptions(t, examplesDir)
+		keyPair := test_structure.LoadEc2KeyPair(t, examplesDir)
+
+		cluster := getInitializedAndUnsealedVaultCluster(t, OUTPUT_VAULT_CLUSTER_ASG_NAME, sshUserName, terraformOptions, awsRegion, keyPair)
 		testVaultViaElb(t, terraformOptions)
+		testVaultUsesConsulForDns(t, cluster)
 	})
 }

test/vault_cluster_s3_backend_test.go

@@ -47,16 +47,18 @@ func runVaultWithS3BackendClusterTest(t *testing.T, amiId string, awsRegion, ssh
 		deployCluster(t, amiId, awsRegion, examplesDir, uniqueId, terraformVars)
 	})
 
-	test_structure.RunTestStage(t, "validate", func() {
+	test_structure.RunTestStage(t, "initialize_unseal", func() {
 		terraformOptions := test_structure.LoadTerraformOptions(t, examplesDir)
 		keyPair := test_structure.LoadEc2KeyPair(t, examplesDir)
 
 		initializeAndUnsealVaultCluster(t, OUTPUT_VAULT_CLUSTER_ASG_NAME, sshUserName, terraformOptions, awsRegion, keyPair)
+	})
+
+	test_structure.RunTestStage(t, "validate", func() {
+		terraformOptions := test_structure.LoadTerraformOptions(t, examplesDir)
+		keyPair := test_structure.LoadEc2KeyPair(t, examplesDir)
 
-		// TODO: temporarily disable DNS check until https://github.com/hashicorp/terraform-aws-consul/issues/155 is
-		// fixed. See https://github.com/hashicorp/terraform-aws-vault/pull/222 for details.
-		//
-		// cluster := initializeAndUnsealVaultCluster(t, OUTPUT_VAULT_CLUSTER_ASG_NAME, sshUserName, terraformOptions, awsRegion, nil)
-		// testVaultUsesConsulForDns(t, cluster)
+		cluster := getInitializedAndUnsealedVaultCluster(t, OUTPUT_VAULT_CLUSTER_ASG_NAME, sshUserName, terraformOptions, awsRegion, keyPair)
+		testVaultUsesConsulForDns(t, cluster)
 	})
 }

test/vault_helpers.go

@@ -191,6 +191,24 @@ func initializeAndUnsealVaultCluster(t *testing.T, asgNameOutputVar string, sshU
 	return cluster
 }
 
+// Find the initialized and unsealed Vault cluster,
+// exit if cluster is not initialized and unsealed
+func getInitializedAndUnsealedVaultCluster(t *testing.T, asgNameOutputVar string, sshUserName string, terraformOptions *terraform.Options, awsRegion string, keyPair *aws.Ec2Keypair) VaultCluster {
+	cluster := findVaultClusterNodes(t, asgNameOutputVar, sshUserName, terraformOptions, awsRegion, keyPair)
+
+	establishConnectionToCluster(t, cluster)
+	isInitializedAndUnsealed, err := isVaultClusterInitializedAndUnsealed(t, cluster)
+	if err != nil {
+		t.Logf("Failed to check is vault cluster is already initialized and unsealed: %v", err)
+	}
+	// exit if cluster is not initialized and unsealed
+	if !isInitializedAndUnsealed {
+		t.Fatalf("Expected to find an initialized and unsealed cluster but it wasn't: %v", cluster)
+	}
+
+	return cluster
+}
+
 // Find the nodes in the given Vault ASG and return them in a VaultCluster struct
 func findVaultClusterNodes(t *testing.T, asgNameOutputVar string, sshUserName string, terraformOptions *terraform.Options, awsRegion string, keyPair *aws.Ec2Keypair) VaultCluster {
 	asgName := terraform.Output(t, terraformOptions, asgNameOutputVar)
@@ -401,7 +419,7 @@ func boolToTerraformVar(val bool) int {
 	}
 }
 
-// Check that the Vault node at the given host has the given
+// Check that the Vault node at the given host has the given status
 func assertStatus(t *testing.T, host ssh.Host, expectedStatus VaultStatus) {
 	description := fmt.Sprintf("Check that the Vault node %s has status %d", host.Hostname, int(expectedStatus))
 	logger.Logf(t, description)
@@ -444,3 +462,45 @@ func checkStatus(t *testing.T, host ssh.Host, expectedStatus VaultStatus) (strin
 		return "", fmt.Errorf("Expected status code %d for host %s, but got %d", int(expectedStatus), host.Hostname, status)
 	}
 }
+
+// Check if the given Vault cluster has been initialized and unsealed.
+func isVaultClusterInitializedAndUnsealed(t *testing.T, cluster VaultCluster) (bool, error) {
+	leader, err := hasExpectedStatus(t, cluster.Leader, Leader)
+	if err != nil {
+		return false, err
+	}
+	standby1, err := hasExpectedStatus(t, cluster.Standby1, Standby)
+	if err != nil {
+		return false, err
+	}
+	standby2, err := hasExpectedStatus(t, cluster.Standby2, Standby)
+	if err != nil {
+		return false, err
+	}
+	if leader && standby1 && standby2 {
+		return true, nil
+	} else {
+		return false, nil
+	}
+}
+
+// Check the status of the given Vault node and ensure it matches the expected status.
+// Returns true if the status match, false otherwise.
+func hasExpectedStatus(t *testing.T, host ssh.Host, expectedStatus VaultStatus) (bool, error) {
+	curlCommand := "curl -s -o /dev/null -w '%{http_code}' https://127.0.0.1:8200/v1/sys/health"
+	logger.Logf(t, "Using curl to check status of Vault server %s: %s", host.Hostname, curlCommand)
+
+	output, err := ssh.CheckSshCommandE(t, host, curlCommand)
+	if err != nil {
+		return false, err
+	}
+	status, err := strconv.Atoi(output)
+	if err != nil {
+		return false, err
+	}
+	if status == int(expectedStatus) {
+		return true, nil
+	} else {
+		return false, nil
+	}
+}

test/vault_main_test.go

@@ -89,6 +89,7 @@ func TestMainVaultCluster(t *testing.T) {
 
 	// os.Setenv("SKIP_setup_amis", "true")
 	// os.Setenv("SKIP_deploy", "true")
+	// os.Setenv("SKIP_initialize_unseal", "true")
 	// os.Setenv("SKIP_validate", "true")
 	// os.Setenv("SKIP_log", "true")
 	// os.Setenv("SKIP_teardown", "true")