diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 81355c81..5db53b5f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/gruntwork-io/pre-commit - rev: v0.1.10 + rev: v0.1.12 hooks: - id: terraform-fmt - id: gofmt \ No newline at end of file diff --git a/examples/vault-consul-ami/vault-consul.json b/examples/vault-consul-ami/vault-consul.json index 8a350c36..0ebbac94 100644 --- a/examples/vault-consul-ami/vault-consul.json +++ b/examples/vault-consul-ami/vault-consul.json @@ -1,10 +1,10 @@ { - "min_packer_version": "0.12.0", + "min_packer_version": "1.5.4", "variables": { "aws_region": "us-east-1", - "vault_version": "1.5.4", + "vault_version": "1.6.1", "consul_module_version": "v0.8.0", - "consul_version": "1.5.3", + "consul_version": "1.9.2", "consul_download_url": "{{env `CONSUL_DOWNLOAD_URL`}}", "vault_download_url": "{{env `VAULT_DOWNLOAD_URL`}}", "install_auth_signing_script": "true", diff --git a/examples/vault-dynamodb-backend/main.tf b/examples/vault-dynamodb-backend/main.tf index 169027c5..3b402a53 100644 --- a/examples/vault-dynamodb-backend/main.tf +++ b/examples/vault-dynamodb-backend/main.tf @@ -9,7 +9,7 @@ terraform { } # --------------------------------------------------------------------------------------------------------------------- -# DEPLOY THE VAULT SERVER CLUSTER +# DEPLOY THE DYNAMODB STORAGE BACKEND # --------------------------------------------------------------------------------------------------------------------- module "backend" { @@ -19,6 +19,10 @@ module "backend" { write_capacity = var.dynamo_write_capacity } +# --------------------------------------------------------------------------------------------------------------------- +# DEPLOY THE VAULT SERVER CLUSTER +# --------------------------------------------------------------------------------------------------------------------- + module "vault_cluster" { # When using these modules in your own templates, you will need to use a Git URL with a ref attribute that pins you # to a specific version of the modules, such as the following example: @@ -32,6 +36,11 @@ module "vault_cluster" { ami_id = var.ami_id user_data = data.template_file.user_data_vault_cluster.rendered + # Enable S3 storage backend + enable_s3_backend = true + s3_bucket_name = var.s3_bucket_name + force_destroy_s3_bucket = var.force_destroy_s3_bucket + vpc_id = data.aws_vpc.default.id subnet_ids = data.aws_subnet_ids.default.ids @@ -44,16 +53,23 @@ module "vault_cluster" { allowed_inbound_security_group_count = 0 ssh_key_name = var.ssh_key_name + # Enable DynamoDB high availability storage backend enable_dynamo_backend = true dynamo_table_name = var.dynamo_table_name } +# --------------------------------------------------------------------------------------------------------------------- +# THE USER DATA SCRIPT THAT WILL RUN ON EACH VAULT SERVER WHEN IT'S BOOTING +# This script will configure and start Vault +# --------------------------------------------------------------------------------------------------------------------- + data "template_file" "user_data_vault_cluster" { template = file("${path.module}/user-data-vault.sh") vars = { aws_region = data.aws_region.current.name dynamo_table_name = var.dynamo_table_name + s3_bucket_name = var.s3_bucket_name } } diff --git a/examples/vault-dynamodb-backend/user-data-vault.sh b/examples/vault-dynamodb-backend/user-data-vault.sh index 6ff712c8..86b8047e 100644 --- a/examples/vault-dynamodb-backend/user-data-vault.sh +++ b/examples/vault-dynamodb-backend/user-data-vault.sh @@ -20,4 +20,7 @@ readonly VAULT_TLS_KEY_FILE="/opt/vault/tls/vault.key.pem" --dynamo-table "${dynamo_table_name}" \ --dynamo-region "${aws_region}" \ --tls-cert-file "$VAULT_TLS_CERT_FILE" \ - --tls-key-file "$VAULT_TLS_KEY_FILE" \ No newline at end of file + --tls-key-file "$VAULT_TLS_KEY_FILE" \ + --enable-s3-backend \ + --s3-bucket "${s3_bucket_name}" \ + --s3-bucket-region "${aws_region}" diff --git a/examples/vault-dynamodb-backend/variables.tf b/examples/vault-dynamodb-backend/variables.tf index 928f9b9e..b615bda3 100644 --- a/examples/vault-dynamodb-backend/variables.tf +++ b/examples/vault-dynamodb-backend/variables.tf @@ -65,3 +65,15 @@ variable "dynamo_write_capacity" { description = "Sets the DynamoDB write capacity for storage backend" default = 5 } + +variable "s3_bucket_name" { + description = "The name of an S3 bucket to create and use as a storage backend (if configured). Note: S3 bucket names must be *globally* unique." + type = string + default = "my-vault-bucket" +} + +variable "force_destroy_s3_bucket" { + description = "If you set this to true, when you run terraform destroy, this tells Terraform to delete all the objects in the S3 bucket used for backend storage (if configured). You should NOT set this to true in production or you risk losing all your data! This property is only here so automated tests of this module can clean up after themselves." + type = bool + default = false +} diff --git a/examples/vault-s3-backend/variables.tf b/examples/vault-s3-backend/variables.tf index f526eaaa..d73a2590 100644 --- a/examples/vault-s3-backend/variables.tf +++ b/examples/vault-s3-backend/variables.tf @@ -86,4 +86,3 @@ variable "force_destroy_s3_bucket" { type = bool default = false } - diff --git a/modules/run-vault/run-vault b/modules/run-vault/run-vault index c7982409..edc880df 100755 --- a/modules/run-vault/run-vault +++ b/modules/run-vault/run-vault @@ -14,6 +14,8 @@ readonly DEFAULT_AGENT_AUTH_MOUNT_PATH="auth/aws" readonly DEFAULT_PORT=8200 readonly DEFAULT_LOG_LEVEL="info" +readonly DEFAULT_CONSUL_AGENT_SERVICE_REGISTRATION_ADDRESS="localhost:8500" + readonly EC2_INSTANCE_METADATA_URL="http://169.254.169.254/latest/meta-data" readonly SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" @@ -44,6 +46,7 @@ function print_usage { echo -e " --s3-bucket\tSpecifies the S3 bucket to use to store Vault data. Only used if '--enable-s3-backend' is set." echo -e " --s3-bucket-path\tSpecifies the S3 bucket path to use to store Vault data. Only used if '--enable-s3-backend' is set." echo -e " --s3-bucket-region\tSpecifies the AWS region where '--s3-bucket' lives. Only used if '--enable-s3-backend' is set." + echo -e " --consul-agent-service-registration-address\tSpecifies the address of the Consul agent to communicate with when using a different storage backend, in this case an S3 backend. Only used if '--enable-s3-backend' is set. Default is ${DEFAULT_CONSUL_AGENT_SERVICE_REGISTRATION_ADDRESS}." echo -e " --enable-dynamo-backend\tIf this flag is set, DynamoDB will be enabled as the backend storage (HA)" echo -e " --dynamo-region\tSpecifies the AWS region where --dynamo-table lives. Only used if '--enable-dynamo-backend is on'" echo -e " --dynamo--table\tSpecifies the DynamoDB table to use for HA Storage. Only used if '--enable-dynamo-backend is on'" @@ -237,13 +240,14 @@ function generate_vault_config { local -r s3_bucket="$9" local -r s3_bucket_path="${10}" local -r s3_bucket_region="${11}" - local -r enable_dynamo_backend="${12}" - local -r dynamo_region="${13}" - local -r dynamo_table="${14}" - local -r enable_auto_unseal="${15}" - local -r auto_unseal_kms_key_id="${16}" - local -r auto_unseal_kms_key_region="${17}" - local -r auto_unseal_endpoint="${18}" + local -r consul_agent_service_registration_address="${12}" + local -r enable_dynamo_backend="${13}" + local -r dynamo_region="${14}" + local -r dynamo_table="${15}" + local -r enable_auto_unseal="${16}" + local -r auto_unseal_kms_key_id="${17}" + local -r auto_unseal_kms_key_region="${18}" + local -r auto_unseal_endpoint="${19}" local -r config_path="$config_dir/$VAULT_CONFIG_FILE" local instance_ip_address @@ -288,6 +292,7 @@ EOF local dynamodb_storage_type="storage" local s3_config="" local vault_storage_backend="" + local service_registration="" if [[ "$enable_s3_backend" == "true" ]]; then s3_config=$(cat <> "$config_path" echo -e "$s3_config" >> "$config_path" echo -e "$vault_storage_backend" >> "$config_path" + echo -e "$service_registration" >> "$config_path" chown "$user:$user" "$config_path" } @@ -368,6 +379,8 @@ Documentation=https://www.vaultproject.io/docs/ Requires=network-online.target After=network-online.target ConditionFileNotEmpty=$config_path +StartLimitIntervalSec=60 +StartLimitBurst=3 EOF ) @@ -392,9 +405,12 @@ KillSignal=SIGINT Restart=on-failure RestartSec=5 TimeoutStopSec=30 +StartLimitInterval=60 StartLimitIntervalSec=60 StartLimitBurst=3 LimitNOFILE=65536 +LimitMEMLOCK=infinity + EOF ) @@ -449,6 +465,7 @@ function run { local s3_bucket="" local s3_bucket_path="" local s3_bucket_region="" + local consul_agent_service_registration_address="${DEFAULT_CONSUL_AGENT_SERVICE_REGISTRATION_ADDRESS}" local enable_dynamo_backend="false" local dynamo_region="" local dynamo_table="" @@ -547,6 +564,11 @@ function run { s3_bucket_region="$2" shift ;; + --consul-agent-service-registration-address) + assert_not_empty "$key" "$2" + consul_agent_service_registration_address="$2" + shift + ;; --enable-dynamo-backend) enable_dynamo_backend="true" ;; @@ -639,6 +661,7 @@ function run { if [[ "$enable_s3_backend" == "true" ]]; then assert_not_empty "--s3-bucket" "$s3_bucket" assert_not_empty "--s3-bucket-region" "$s3_bucket_region" + assert_not_empty "--consul-agent-service-registration-address" "${consul_agent_service_registration_address}" fi fi @@ -714,6 +737,7 @@ function run { "$s3_bucket" \ "$s3_bucket_path" \ "$s3_bucket_region" \ + "${consul_agent_service_registration_address}" \ "$enable_dynamo_backend" \ "$dynamo_region" \ "$dynamo_table" \ diff --git a/test/go.sum b/test/go.sum index f42d2427..d89b9655 100644 --- a/test/go.sum +++ b/test/go.sum @@ -197,6 +197,8 @@ github.com/gruntwork-io/gruntwork-cli v0.5.1 h1:mVmVsFubUSLSCO8bGigI63HXzvzkC0uW github.com/gruntwork-io/gruntwork-cli v0.5.1/go.mod h1:IBX21bESC1/LGoV7jhXKUnTQTZgQ6dYRsoj/VqxUSZQ= github.com/gruntwork-io/terratest v0.28.15 h1:in1DRBq8/RjxMyb6Amr1SRrczOK/hGnPi+gQXOOtbZI= github.com/gruntwork-io/terratest v0.28.15/go.mod h1:PkVylPuUNmItkfOTwSiFreYA4FkanK8AluBuNeGxQOw= +github.com/gruntwork-io/terratest v0.32.1 h1:Uho3H7VWD4tEulWov7pWW90V3XATLKxSh88AtrxTYvU= +github.com/gruntwork-io/terratest v0.32.3 h1:GSe/mkSQe0rD7Z92NKTUjDKg2FBuy0w82Ttd5gcK7kU= github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= @@ -362,6 +364,7 @@ github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.2.0 h1:Hbg2NidpLE8veEBkEZTL3CvlkUIVzuU9jDplZO54c48= github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/testify v0.0.0-20151208002404-e3a8ff8ce365/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= diff --git a/test/vault_cluster_dynamodb_backend_test.go b/test/vault_cluster_dynamodb_backend_test.go index c2914c1b..a9c821c3 100644 --- a/test/vault_cluster_dynamodb_backend_test.go +++ b/test/vault_cluster_dynamodb_backend_test.go @@ -20,7 +20,6 @@ const VAR_DYNAMO_TABLE_NAME = "dynamo_table_name" // 3. Deploy that AMI using the example Terraform code // 4. SSH to a Vault node and initialize the Vault cluster // 5. SSH to each Vault node and unseal it -// 6. Connect to the Vault cluster via the ELB func runVaultWithDynamoBackendClusterTest(t *testing.T, amiId string, awsRegion, sshUserName string) { examplesDir := test_structure.CopyTerraformFolderToTemp(t, REPO_ROOT, VAULT_CLUSTER_DYNAMODB_BACKEND_PATH) @@ -36,10 +35,13 @@ func runVaultWithDynamoBackendClusterTest(t *testing.T, amiId string, awsRegion, }) test_structure.RunTestStage(t, "deploy", func() { + uniqueId := random.UniqueId() terraformVars := map[string]interface{}{ - VAR_DYNAMO_TABLE_NAME: fmt.Sprintf("vault-dynamo-test-%s", random.UniqueId()), + VAR_DYNAMO_TABLE_NAME: fmt.Sprintf("vault-dynamo-test-%s", uniqueId), + VAR_S3_BUCKET_NAME: s3BucketName(uniqueId), + VAR_FORCE_DESTROY_S3_BUCKET: true, } - deployCluster(t, amiId, awsRegion, examplesDir, random.UniqueId(), terraformVars) + deployCluster(t, amiId, awsRegion, examplesDir, uniqueId, terraformVars) }) test_structure.RunTestStage(t, "validate", func() { diff --git a/test/vault_cluster_enterprise_test.go b/test/vault_cluster_enterprise_test.go index 4e4aad4f..ace11d85 100644 --- a/test/vault_cluster_enterprise_test.go +++ b/test/vault_cluster_enterprise_test.go @@ -59,11 +59,18 @@ func runVaultEnterpriseClusterTest(t *testing.T, amiId string, awsRegion string, deployCluster(t, amiId, awsRegion, examplesDir, uniqueId, terraformVars) }) + test_structure.RunTestStage(t, "initialize_unseal", func() { + terraformOptions := test_structure.LoadTerraformOptions(t, examplesDir) + keyPair := test_structure.LoadEc2KeyPair(t, examplesDir) + + initializeAndUnsealVaultCluster(t, OUTPUT_VAULT_CLUSTER_ASG_NAME, sshUserName, terraformOptions, awsRegion, keyPair) + }) + test_structure.RunTestStage(t, "validate", func() { terraformOptions := test_structure.LoadTerraformOptions(t, examplesDir) keyPair := test_structure.LoadEc2KeyPair(t, examplesDir) - cluster := initializeAndUnsealVaultCluster(t, OUTPUT_VAULT_CLUSTER_ASG_NAME, sshUserName, terraformOptions, awsRegion, keyPair) + cluster := getInitializedAndUnsealedVaultCluster(t, OUTPUT_VAULT_CLUSTER_ASG_NAME, sshUserName, terraformOptions, awsRegion, keyPair) testVaultUsesConsulForDns(t, cluster) checkEnterpriseInstall(t, OUTPUT_VAULT_CLUSTER_ASG_NAME, sshUserName, terraformOptions, awsRegion, keyPair) }) diff --git a/test/vault_cluster_private_test.go b/test/vault_cluster_private_test.go index f115b336..2d5b40c1 100644 --- a/test/vault_cluster_private_test.go +++ b/test/vault_cluster_private_test.go @@ -39,14 +39,21 @@ func runVaultPrivateClusterTest(t *testing.T, amiId string, awsRegion string, ss VAR_CONSUL_CLUSTER_NAME: fmt.Sprintf("consul-test-%s", uniqueId), VAR_CONSUL_CLUSTER_TAG_KEY: fmt.Sprintf("consul-test-%s", uniqueId), } - deployCluster(t, amiId, awsRegion, examplesDir, random.UniqueId(), terraformVars) + deployCluster(t, amiId, awsRegion, examplesDir, uniqueId, terraformVars) + }) + + test_structure.RunTestStage(t, "initialize_unseal", func() { + terraformOptions := test_structure.LoadTerraformOptions(t, examplesDir) + keyPair := test_structure.LoadEc2KeyPair(t, examplesDir) + + initializeAndUnsealVaultCluster(t, OUTPUT_VAULT_CLUSTER_ASG_NAME, sshUserName, terraformOptions, awsRegion, keyPair) }) test_structure.RunTestStage(t, "validate", func() { terraformOptions := test_structure.LoadTerraformOptions(t, examplesDir) keyPair := test_structure.LoadEc2KeyPair(t, examplesDir) - cluster := initializeAndUnsealVaultCluster(t, OUTPUT_VAULT_CLUSTER_ASG_NAME, sshUserName, terraformOptions, awsRegion, keyPair) + cluster := getInitializedAndUnsealedVaultCluster(t, OUTPUT_VAULT_CLUSTER_ASG_NAME, sshUserName, terraformOptions, awsRegion, keyPair) testVaultUsesConsulForDns(t, cluster) }) } diff --git a/test/vault_cluster_public_test.go b/test/vault_cluster_public_test.go index 54f9497b..6d3511e0 100644 --- a/test/vault_cluster_public_test.go +++ b/test/vault_cluster_public_test.go @@ -23,6 +23,7 @@ const VAULT_CLUSTER_PUBLIC_VAR_VAULT_DOMAIN_NAME = "vault_domain_name" // 4. SSH to a Vault node and initialize the Vault cluster // 5. SSH to each Vault node and unseal it // 6. Connect to the Vault cluster via the ELB +// 7. SSH to a Vault node and make sure you can communicate with the nodes via Consul-managed DNS func runVaultPublicClusterTest(t *testing.T, amiId string, awsRegion string, sshUserName string) { examplesDir := test_structure.CopyTerraformFolderToTemp(t, REPO_ROOT, ".") @@ -46,14 +47,22 @@ func runVaultPublicClusterTest(t *testing.T, amiId string, awsRegion string, ssh VAR_CONSUL_CLUSTER_NAME: fmt.Sprintf("consul-test-%s", uniqueId), VAR_CONSUL_CLUSTER_TAG_KEY: fmt.Sprintf("consul-test-%s", uniqueId), } - deployCluster(t, amiId, awsRegion, examplesDir, random.UniqueId(), terraformVars) + deployCluster(t, amiId, awsRegion, examplesDir, uniqueId, terraformVars) }) - test_structure.RunTestStage(t, "validate", func() { + test_structure.RunTestStage(t, "initialize_unseal", func() { terraformOptions := test_structure.LoadTerraformOptions(t, examplesDir) keyPair := test_structure.LoadEc2KeyPair(t, examplesDir) initializeAndUnsealVaultCluster(t, OUTPUT_VAULT_CLUSTER_ASG_NAME, sshUserName, terraformOptions, awsRegion, keyPair) + }) + + test_structure.RunTestStage(t, "validate", func() { + terraformOptions := test_structure.LoadTerraformOptions(t, examplesDir) + keyPair := test_structure.LoadEc2KeyPair(t, examplesDir) + + cluster := getInitializedAndUnsealedVaultCluster(t, OUTPUT_VAULT_CLUSTER_ASG_NAME, sshUserName, terraformOptions, awsRegion, keyPair) testVaultViaElb(t, terraformOptions) + testVaultUsesConsulForDns(t, cluster) }) } diff --git a/test/vault_cluster_s3_backend_test.go b/test/vault_cluster_s3_backend_test.go index 16ac3c34..f9887f7f 100644 --- a/test/vault_cluster_s3_backend_test.go +++ b/test/vault_cluster_s3_backend_test.go @@ -21,7 +21,7 @@ const VAR_FORCE_DESTROY_S3_BUCKET = "force_destroy_s3_bucket" // 3. Deploy that AMI using the example Terraform code // 4. SSH to a Vault node and initialize the Vault cluster // 5. SSH to each Vault node and unseal it -// 6. Connect to the Vault cluster via the ELB +// 6. SSH to a Vault node and make sure you can communicate with the nodes via Consul-managed DNS func runVaultWithS3BackendClusterTest(t *testing.T, amiId string, awsRegion, sshUserName string) { examplesDir := test_structure.CopyTerraformFolderToTemp(t, REPO_ROOT, VAULT_CLUSTER_S3_BACKEND_PATH) @@ -47,16 +47,18 @@ func runVaultWithS3BackendClusterTest(t *testing.T, amiId string, awsRegion, ssh deployCluster(t, amiId, awsRegion, examplesDir, uniqueId, terraformVars) }) - test_structure.RunTestStage(t, "validate", func() { + test_structure.RunTestStage(t, "initialize_unseal", func() { terraformOptions := test_structure.LoadTerraformOptions(t, examplesDir) keyPair := test_structure.LoadEc2KeyPair(t, examplesDir) initializeAndUnsealVaultCluster(t, OUTPUT_VAULT_CLUSTER_ASG_NAME, sshUserName, terraformOptions, awsRegion, keyPair) + }) + + test_structure.RunTestStage(t, "validate", func() { + terraformOptions := test_structure.LoadTerraformOptions(t, examplesDir) + keyPair := test_structure.LoadEc2KeyPair(t, examplesDir) - // TODO: temporarily disable DNS check until https://github.com/hashicorp/terraform-aws-consul/issues/155 is - // fixed. See https://github.com/hashicorp/terraform-aws-vault/pull/222 for details. - // - // cluster := initializeAndUnsealVaultCluster(t, OUTPUT_VAULT_CLUSTER_ASG_NAME, sshUserName, terraformOptions, awsRegion, nil) - // testVaultUsesConsulForDns(t, cluster) + cluster := getInitializedAndUnsealedVaultCluster(t, OUTPUT_VAULT_CLUSTER_ASG_NAME, sshUserName, terraformOptions, awsRegion, keyPair) + testVaultUsesConsulForDns(t, cluster) }) } diff --git a/test/vault_helpers.go b/test/vault_helpers.go index ef041cc1..0f5c1d02 100644 --- a/test/vault_helpers.go +++ b/test/vault_helpers.go @@ -61,10 +61,11 @@ func (cluster VaultCluster) Nodes() []ssh.Host { type VaultStatus int const ( - Leader VaultStatus = 200 - Standby = 429 - Uninitialized = 501 - Sealed = 503 + Leader VaultStatus = 200 + Standby = 429 + PerformanceStandby = 473 + Uninitialized = 501 + Sealed = 503 ) func teardownResources(t *testing.T, examplesDir string) { @@ -188,6 +189,100 @@ func initializeAndUnsealVaultCluster(t *testing.T, asgNameOutputVar string, sshU unsealVaultNode(t, cluster.Standby2, cluster.UnsealKeys) assertStatus(t, cluster.Standby2, Standby) + logger.Logf(t, "Successfully initialized and unsealed Vault Cluster: [Leader: %s, Standby1: %s , Standby2: %s]", cluster.Leader.Hostname, cluster.Standby1.Hostname, cluster.Standby2.Hostname) + + return cluster +} + +// Find and return the initialized and unsealed Vault cluster, or exit if cluster is not initialized and unsealed: +// has not any of the expected staus codes: 200 for Leader node, 429 for Standby node or 473 for PerformanceStandby node (enterprise version) +// +// 1. Get the public IP addresses of the EC2 Instances in an Auto Scaling Group of the given name in the given region +// 2. SSH to each Vault nodes and get node status +// 3. Set Vault node status (Leader or Standby|PerformanceStandby) based on returned node status +// 4. Double check cluster node status +func getInitializedAndUnsealedVaultCluster(t *testing.T, asgNameOutputVar string, sshUserName string, terraformOptions *terraform.Options, awsRegion string, keyPair *aws.Ec2Keypair) VaultCluster { + asgName := terraform.Output(t, terraformOptions, asgNameOutputVar) + nodeIpAddresses := getIpAddressesOfAsgInstances(t, asgName, awsRegion) + if len(nodeIpAddresses) != 3 { + t.Fatalf("Expected to get three IP addresses for Vault cluster, but got %d: %v", len(nodeIpAddresses), nodeIpAddresses) + } + + // build vault cluster variable + // because findVaultClusterNodes method does not guarantee + // that node0 is Leader and node1 and node2 are Standby + cluster := VaultCluster{ + Leader: ssh.Host{ + Hostname: "", + SshUserName: sshUserName, + SshKeyPair: keyPair.KeyPair, + }, + + Standby1: ssh.Host{ + Hostname: "", + SshUserName: sshUserName, + SshKeyPair: keyPair.KeyPair, + }, + + Standby2: ssh.Host{ + Hostname: "", + SshUserName: sshUserName, + SshKeyPair: keyPair.KeyPair, + }, + } + + // ssh each node and get its status + for _, node := range nodeIpAddresses { + if node != "" { + description := fmt.Sprintf("Trying to establish SSH connection to %s", node) + logger.Logf(t, description) + // connection to each of the Vault cluster nodes must be already working + // retrying only 3 times + maxRetries := 3 + sleepBetweenRetries := 10 * time.Second + host := ssh.Host{ + Hostname: node, + SshUserName: sshUserName, + SshKeyPair: keyPair.KeyPair, + } + retry.DoWithRetry(t, description, maxRetries, sleepBetweenRetries, func() (string, error) { + return "", ssh.CheckSshConnectionE(t, host) + }) + // update leader and standby nodes in vault cluster variable + status, err := getNodeStatus(t, host) + if err != nil { + require.NoError(t, err, "Failed to check if vault cluster is already initialized and unsealed") + } + switch status { + case int(Leader): + cluster.Leader.Hostname = node + assertStatus(t, cluster.Leader, Leader) + case int(Standby): + if cluster.Standby1.Hostname == "" { + cluster.Standby1.Hostname = node + assertStatus(t, cluster.Standby1, Standby) + } else if cluster.Standby2.Hostname == "" { + cluster.Standby2.Hostname = node + } + // Managing Performance Standby Nodes status + // https://www.vaultproject.io/docs/enterprise/performance-standby#performance-standby-nodes + case int(PerformanceStandby): + if cluster.Standby1.Hostname == "" { + cluster.Standby1.Hostname = node + assertStatus(t, cluster.Standby1, PerformanceStandby) + } else if cluster.Standby2.Hostname == "" { + cluster.Standby2.Hostname = node + assertStatus(t, cluster.Standby2, PerformanceStandby) + } + default: + errMsg := fmt.Sprintf("error: Unexpected vault cluster node status %d", status) + require.NoError(t, errors.New(errMsg), "Failed to check if vault cluster is already initialized and unsealed") + } + } + } + + logger.Logf(t, "Retrieved Vault Cluster: [Leader: %s, Standby1: %s , Standby2: %s]", cluster.Leader.Hostname, cluster.Standby1.Hostname, cluster.Standby2.Hostname) + return cluster } @@ -401,7 +496,7 @@ func boolToTerraformVar(val bool) int { } } -// Check that the Vault node at the given host has the given +// Check that the Vault node at the given host has the given status func assertStatus(t *testing.T, host ssh.Host, expectedStatus VaultStatus) { description := fmt.Sprintf("Check that the Vault node %s has status %d", host.Hostname, int(expectedStatus)) logger.Logf(t, description) @@ -423,17 +518,9 @@ func cleanupTlsCertFiles(tlsCert TlsCert) { os.Remove(tlsCert.PublicKeyPath) } -// Check the status of the given Vault node and ensure it matches the expected status. Note that we use curl to do the -// status check so we can ensure that TLS certificates work for curl (and not just the Vault client). +// Check the status of the given Vault node and ensure it matches the expected status. func checkStatus(t *testing.T, host ssh.Host, expectedStatus VaultStatus) (string, error) { - curlCommand := "curl -s -o /dev/null -w '%{http_code}' https://127.0.0.1:8200/v1/sys/health" - logger.Logf(t, "Using curl to check status of Vault server %s: %s", host.Hostname, curlCommand) - - output, err := ssh.CheckSshCommandE(t, host, curlCommand) - if err != nil { - return "", err - } - status, err := strconv.Atoi(output) + status, err := getNodeStatus(t, host) if err != nil { return "", err } @@ -444,3 +531,20 @@ func checkStatus(t *testing.T, host ssh.Host, expectedStatus VaultStatus) (strin return "", fmt.Errorf("Expected status code %d for host %s, but got %d", int(expectedStatus), host.Hostname, status) } } + +// Get the status of the given Vault node. Note that we use curl to do the status check so we can ensure that +// TLS certificates work for curl (and not just the Vault client). +func getNodeStatus(t *testing.T, host ssh.Host) (int, error) { + curlCommand := "curl -s -o /dev/null -w '%{http_code}' https://127.0.0.1:8200/v1/sys/health" + logger.Logf(t, "Using curl to check status of Vault server %s: %s", host.Hostname, curlCommand) + + output, err := ssh.CheckSshCommandE(t, host, curlCommand) + if err != nil { + return 0, err + } + status, err := strconv.Atoi(output) + if err != nil { + return 0, err + } + return status, nil +} diff --git a/test/vault_main_test.go b/test/vault_main_test.go index 80d1a88b..220d5580 100644 --- a/test/vault_main_test.go +++ b/test/vault_main_test.go @@ -89,6 +89,7 @@ func TestMainVaultCluster(t *testing.T) { // os.Setenv("SKIP_setup_amis", "true") // os.Setenv("SKIP_deploy", "true") + // os.Setenv("SKIP_initialize_unseal", "true") // os.Setenv("SKIP_validate", "true") // os.Setenv("SKIP_log", "true") // os.Setenv("SKIP_teardown", "true")