Attaching tags to virtual machines fails intermittently

[ryancurrah]

Attaching vSphere tags to virtual machines fails intermittently when deploying more than one virtual machine at the same time

Terraform Version

0.10.8

vSphere Provider Version

1.3.0

Affected Resource(s)

Please list the resources as a list, for example:

• vsphere_virtual_machine

Terraform Configuration Files

Variables.tfvars file

vsphere_vcenter    = "vsphere03.acme.com"
vsphere_user       = "acme.com\ryancurrah"
datacenter         = "TO-Prod-01"
resource_pool      = "Non-Production"
template           = "Templates/core-rhel7/core-rhel7-latest"
cpu_count          = "2"
memory             = "4"
domain             = "acme.com"
datastore          = "DS1"
data_disk_size     = "10"
environment        = "D"
network            = "1001_NonProd"
ipv4_gateway       = "10.0.0.1"
ipv4_prefix_length = "22"
linux_nodes = [
    {hostname = "d9lcwpcurrah04" ipv4_address = "10.0.0.121"},
    {hostname = "d9lcwpcurrah05" ipv4_address = "10.0.0.122"},
    {hostname = "d9lcwpcurrah02" ipv4_address = "10.0.0.119"},
    {hostname = "d9lcwpcurrah03" ipv4_address = "10.0.0.120"}
]

Variables.tf

# Provided by cli command
variable "vsphere_vcenter" {}
variable "vsphere_user" {}
variable "vsphere_password" {}
variable "template" {}
variable "memory" {}
variable "cpu_count" {}
variable "domain" {}
variable "datastore" {}
variable "datacenter" {}
variable "resource_pool" {}
variable "environment" {}
variable "data_disk_size" {}

# Provided by network.tfvars
variable "network" {}
variable "ipv4_netmask" {}
variable "ipv4_gateway" {}
variable "linux_nodes" {
  type    = "list"
  default = []
}
variable "windows_nodes" {
  type    = "list"
  default = []
}

# Mappings
variable "environment_mappings" {
  default = {
    P = "prod"
    Q = "qa"
    D = "dev"
  }
}

variable "win_dns_mappings" {
  default = {
    "BP-Prod-01" = [
      "10.0.1.43",
      "10.0.1.44"
    ]
    "TO-Prod-01" = [
      "10.0.0.51",
      "10.0.0.52"
    ]
  }
}

Main.tf file

provider "vsphere" {
    vsphere_server       = "${var.vsphere_vcenter}"
    user                 = "${var.vsphere_user}"
    password             = "${var.vsphere_password}"
    version              = "1.3.0"
    allow_unverified_ssl = true
}

data "vsphere_datacenter" "dc" {
  name = "${var.datacenter}"
}

data "vsphere_datastore" "datastore" {
  name          = "${var.datastore}"
  datacenter_id = "${data.vsphere_datacenter.dc.id}"
}

data "vsphere_resource_pool" "pool" {
  name          = "${var.resource_pool}"
  datacenter_id = "${data.vsphere_datacenter.dc.id}"
}

data "vsphere_network" "network" {
  name          = "${var.network}"
  datacenter_id = "${data.vsphere_datacenter.dc.id}"
}

data "vsphere_virtual_machine" "template" {
  name          = "${var.template}"
  datacenter_id = "${data.vsphere_datacenter.dc.id}"
}

data "vsphere_tag_category" "devops" {
  name = "DevOps"
}

data "vsphere_tag" "environment" {
  name        = "${var.environment_mappings[var.environment]}"
  category_id = "${data.vsphere_tag_category.devops.id}"
}

data "vsphere_tag" "domain" {
  name        = "${var.domain}"
  category_id = "${data.vsphere_tag_category.devops.id}"
}

data "vsphere_tag" "template_name" {
  name        = "${element(split("/", var.template), 1)}"
  category_id = "${data.vsphere_tag_category.devops.id}"
}

resource "vsphere_virtual_machine" "linux_vm" {
  count            = "${length(var.linux_nodes)}"
  name             = "${lookup(var.linux_nodes[count.index], "hostname")}"
  resource_pool_id = "${data.vsphere_resource_pool.pool.id}"
  datastore_id     = "${data.vsphere_datastore.datastore.id}"
  num_cpus         = "${var.cpu_count}"
  memory           = "${var.memory * 1024}"
  guest_id         = "${data.vsphere_virtual_machine.template.guest_id}"
  scsi_type        = "${data.vsphere_virtual_machine.template.scsi_type}"
  tags             = [
                       "${data.vsphere_tag.environment.id}",
                       "${data.vsphere_tag.domain.id}",
                       "${data.vsphere_tag.template_name.id}"
                     ]

  network_interface {
    network_id       = "${data.vsphere_network.network.id}"
    adapter_type     = "${data.vsphere_virtual_machine.template.network_interface_types[0]}"
  }

  disk {
    label            = "${lookup(var.linux_nodes[count.index], "hostname")}0"
    size             = "${data.vsphere_virtual_machine.template.disks.0.size}"
    unit_number      = 0
    eagerly_scrub    = false
    thin_provisioned = true
  }

  disk {
    label            = "${lookup(var.linux_nodes[count.index], "hostname")}1"
    size             = "${var.data_disk_size}"
    unit_number      = 1
    eagerly_scrub    = false
    thin_provisioned = true
  }

  clone {
    template_uuid = "${data.vsphere_virtual_machine.template.id}"

    customize {
      linux_options {
        host_name = "${lookup(var.linux_nodes[count.index], "hostname")}"
        domain    = "${var.domain}"
      }

      network_interface {
        ipv4_address = "${lookup(var.linux_nodes[count.index], "ipv4_address")}"
        ipv4_netmask = "${var.ipv4_netmask}"
      }

      ipv4_gateway    = "${var.ipv4_gateway}"
      dns_server_list = ["127.0.0.1"]
    }
  }
}

resource "vsphere_virtual_machine" "windows_vm" {
  count            = "${length(var.windows_nodes)}"
  name             = "${lookup(var.windows_nodes[count.index], "hostname")}"
  resource_pool_id = "${data.vsphere_resource_pool.pool.id}"
  datastore_id     = "${data.vsphere_datastore.datastore.id}"
  num_cpus         = "${var.cpu_count}"
  memory           = "${var.memory * 1024}"
  guest_id         = "${data.vsphere_virtual_machine.template.guest_id}"
  scsi_type        = "${data.vsphere_virtual_machine.template.scsi_type}"
  tags             = [
                       "${data.vsphere_tag.environment.id}",
                       "${data.vsphere_tag.domain.id}",
                       "${data.vsphere_tag.template_name.id}"
                     ]

  network_interface {
    network_id       = "${data.vsphere_network.network.id}"
    adapter_type     = "${data.vsphere_virtual_machine.template.network_interface_types[0]}"
  }

  disk {
    label            = "${lookup(var.windows_nodes[count.index], "hostname")}0"
    size             = "${data.vsphere_virtual_machine.template.disks.0.size}"
    unit_number      = 0
    eagerly_scrub    = false
    thin_provisioned = true
  }

  disk {
    label            = "${lookup(var.windows_nodes[count.index], "hostname")}1"
    size             = "${var.data_disk_size}"
    unit_number      = 1
    eagerly_scrub    = false
    thin_provisioned = true
  }

  clone {
    template_uuid = "${data.vsphere_virtual_machine.template.id}"

    customize {
      windows_options {
        computer_name  = "${lookup(var.windows_nodes[count.index], "hostname")}"
        workgroup      = "${var.domain}"
      }

      network_interface {
        ipv4_address    = "${lookup(var.windows_nodes[count.index], "ipv4_address")}"
        ipv4_netmask    = "${var.ipv4_netmask}"
        dns_server_list = "${var.win_dns_mappings[var.datacenter]}"
      }

      ipv4_gateway     = "${var.ipv4_gateway}"
    }
  }
}

Debug Output

Four virtual machines were spun up with this tf apply, 1 failed to get the tags applied and caused terraform to halt

terraform-vsphere-tag-debug.log

Expected Behavior

vSphere tags should have been attached to the virtual machine succesfully

Actual Behavior

HTTP call to attach the vSphere tag fails on a virtual machine and halts the terraform apply

Steps to Reproduce

Please list the steps required to reproduce the issue, for example:

1. Add 3 or more hosts to the linux_nodes variable
2. terraform apply

Repeat until failure occurs

Important Factoids

• Running terraform in a RHEL7.2 container on a RHEL7.2 host

References

Below are similar issues encountered by other golang users

• operation was canceled during DNS lookup with remote_write prometheus/prometheus#3510
• net/http: transport MaxIdleConnsPerHost does not work as expected golang/go#13801

[vancluever]

Hey @ryancurrah, sorry for the delay in responding to this.

This is actually a long-standing issue in all of TF, as we, much like the other issues you have linked, work off of the standard library's default resolver.

Unfortunately this is not an issue that we have an easy answer. We are tracking this in more detail in issues such as hashicorp/terraform#3536. Once a better answer is available, I would imagine we will be implementing it across the entire provider ecosystem.

For more updates, I'd follow the referenced issue - there is also more detail in there as to what some people in the community are doing to mitigate the problem.

Thanks and sorry for the trouble!

[ryancurrah]

No problem I appreciate the response. And will follow the linked issue.