Adding support for launch templates & userdata parameters

README.md

@@ -177,25 +177,28 @@ Available targets:
 
 | Name | Version |
 |------|---------|
-| terraform | >= 0.12.0, < 0.14.0 |
-| aws | >= 2.0, < 4.0 |
+| terraform | >= 0.13.0, < 0.14.0 |
+| aws | ~> 3.0 |
 | local | ~> 1.3 |
 | template | ~> 2.0 |
 
 ## Providers
 
 | Name | Version |
 |------|---------|
-| aws | >= 2.0, < 4.0 |
+| aws | ~> 3.0 |
 
 ## Inputs
 
 | Name | Description | Type | Default | Required |
 |------|-------------|------|---------|:--------:|
 | additional\_tag\_map | Additional tags for appending to tags\_as\_list\_of\_maps. Not added to `tags`. | `map(string)` | `{}` | no |
+| after\_cluster\_joining\_userdata | Additional commands to execute on each worker node after joining the EKS cluster (after executing the `bootstrap.sh` script). For more info, see https://kubedex.com/90-days-of-aws-eks-in-production | `string` | `""` | no |
 | ami\_release\_version | AMI version of the EKS Node Group. Defaults to latest version for Kubernetes version | `string` | `null` | no |
 | ami\_type | Type of Amazon Machine Image (AMI) associated with the EKS Node Group. Defaults to `AL2_x86_64`. Valid values: `AL2_x86_64`, `AL2_x86_64_GPU`. Terraform will only perform drift detection if a configuration value is provided | `string` | `"AL2_x86_64"` | no |
 | attributes | Additional attributes (e.g. `1`) | `list(string)` | `[]` | no |
+| before\_cluster\_joining\_userdata | Additional commands to execute on each worker node before joining the EKS cluster (before executing the `bootstrap.sh` script). For more info, see https://kubedex.com/90-days-of-aws-eks-in-production | `string` | `""` | no |
+| bootstrap\_extra\_args | Extra arguments to the `bootstrap.sh` script to enable `--enable-docker-bridge` or `--use-max-pods` | `string` | `""` | no |
 | cluster\_name | The name of the EKS cluster | `string` | n/a | yes |
 | context | Single object for setting entire context at once.<br>See description of individual variables for details.<br>Leave string and numeric variables as `null` to use default value.<br>Individual variable settings (non-null) override settings in context object,<br>except for attributes, tags, and additional\_tag\_map, which are merged. | <pre>object({<br>    enabled             = bool<br>    namespace           = string<br>    environment         = string<br>    stage               = string<br>    name                = string<br>    delimiter           = string<br>    attributes          = list(string)<br>    tags                = map(string)<br>    additional_tag_map  = map(string)<br>    regex_replace_chars = string<br>    label_order         = list(string)<br>    id_length_limit     = number<br>  })</pre> | <pre>{<br>  "additional_tag_map": {},<br>  "attributes": [],<br>  "delimiter": null,<br>  "enabled": true,<br>  "environment": null,<br>  "id_length_limit": null,<br>  "label_order": [],<br>  "name": null,<br>  "namespace": null,<br>  "regex_replace_chars": null,<br>  "stage": null,<br>  "tags": {}<br>}</pre> | no |
 | delimiter | Delimiter to be used between `namespace`, `environment`, `stage`, `name` and `attributes`.<br>Defaults to `-` (hyphen). Set to `""` to use no delimiter at all. | `string` | `null` | no |
@@ -208,10 +211,13 @@ Available targets:
 | existing\_workers\_role\_policy\_arns | List of existing policy ARNs that will be attached to the workers default role on creation | `list(string)` | `[]` | no |
 | existing\_workers\_role\_policy\_arns\_count | Count of existing policy ARNs that will be attached to the workers default role on creation. Needed to prevent Terraform error `count can't be computed` | `number` | `0` | no |
 | id\_length\_limit | Limit `id` to this many characters.<br>Set to `0` for unlimited length.<br>Set to `null` for default, which is `0`.<br>Does not affect `id_full`. | `number` | `null` | no |
-| instance\_types | Set of instance types associated with the EKS Node Group. Defaults to ["t3.medium"]. Terraform will only perform drift detection if a configuration value is provided | `list(string)` | n/a | yes |
+| instance\_types | Set of instance types associated with the EKS Node Group. Defaults to ["t3.medium"]. Terraform will only perform drift detection if a configuration value is provided | `list(string)` | <pre>[<br>  "t3.medium"<br>]</pre> | no |
+| kubelet\_extra\_args | Extra arguments to pass to kubelet, like "--register-with-taints=dedicated=ci-cd:NoSchedule --node-labels=purpose=ci-worker" | `string` | `""` | no |
 | kubernetes\_labels | Key-value mapping of Kubernetes labels. Only labels that are applied with the EKS API are managed by this argument. Other Kubernetes labels applied to the EKS Node Group will not be managed | `map(string)` | `{}` | no |
 | kubernetes\_version | Kubernetes version. Defaults to EKS Cluster Kubernetes version. Terraform will only perform drift detection if a configuration value is provided | `string` | `null` | no |
 | label\_order | The naming order of the id output and Name tag.<br>Defaults to ["namespace", "environment", "stage", "name", "attributes"].<br>You can omit any of the 5 elements, but at least one must be present. | `list(string)` | `null` | no |
+| launch\_template\_id | The ID of a custom launch template to use for the EKS node group. | `string` | `null` | no |
+| launch\_template\_version | A specific version of the above specific launch template | `string` | `null` | no |
 | max\_size | Maximum number of worker nodes | `number` | n/a | yes |
 | min\_size | Minimum number of worker nodes | `number` | n/a | yes |
 | module\_depends\_on | Can be any value desired. Module will wait for this value to be computed before creating node group. | `any` | `null` | no |

docs/terraform.md

@@ -3,25 +3,28 @@
 
 | Name | Version |
 |------|---------|
-| terraform | >= 0.12.0, < 0.14.0 |
-| aws | >= 2.0, < 4.0 |
+| terraform | >= 0.13.0, < 0.14.0 |
+| aws | ~> 3.0 |
 | local | ~> 1.3 |
 | template | ~> 2.0 |
 
 ## Providers
 
 | Name | Version |
 |------|---------|
-| aws | >= 2.0, < 4.0 |
+| aws | ~> 3.0 |
 
 ## Inputs
 
 | Name | Description | Type | Default | Required |
 |------|-------------|------|---------|:--------:|
 | additional\_tag\_map | Additional tags for appending to tags\_as\_list\_of\_maps. Not added to `tags`. | `map(string)` | `{}` | no |
+| after\_cluster\_joining\_userdata | Additional commands to execute on each worker node after joining the EKS cluster (after executing the `bootstrap.sh` script). For more info, see https://kubedex.com/90-days-of-aws-eks-in-production | `string` | `""` | no |
 | ami\_release\_version | AMI version of the EKS Node Group. Defaults to latest version for Kubernetes version | `string` | `null` | no |
 | ami\_type | Type of Amazon Machine Image (AMI) associated with the EKS Node Group. Defaults to `AL2_x86_64`. Valid values: `AL2_x86_64`, `AL2_x86_64_GPU`. Terraform will only perform drift detection if a configuration value is provided | `string` | `"AL2_x86_64"` | no |
 | attributes | Additional attributes (e.g. `1`) | `list(string)` | `[]` | no |
+| before\_cluster\_joining\_userdata | Additional commands to execute on each worker node before joining the EKS cluster (before executing the `bootstrap.sh` script). For more info, see https://kubedex.com/90-days-of-aws-eks-in-production | `string` | `""` | no |
+| bootstrap\_extra\_args | Extra arguments to the `bootstrap.sh` script to enable `--enable-docker-bridge` or `--use-max-pods` | `string` | `""` | no |
 | cluster\_name | The name of the EKS cluster | `string` | n/a | yes |
 | context | Single object for setting entire context at once.<br>See description of individual variables for details.<br>Leave string and numeric variables as `null` to use default value.<br>Individual variable settings (non-null) override settings in context object,<br>except for attributes, tags, and additional\_tag\_map, which are merged. | <pre>object({<br>    enabled             = bool<br>    namespace           = string<br>    environment         = string<br>    stage               = string<br>    name                = string<br>    delimiter           = string<br>    attributes          = list(string)<br>    tags                = map(string)<br>    additional_tag_map  = map(string)<br>    regex_replace_chars = string<br>    label_order         = list(string)<br>    id_length_limit     = number<br>  })</pre> | <pre>{<br>  "additional_tag_map": {},<br>  "attributes": [],<br>  "delimiter": null,<br>  "enabled": true,<br>  "environment": null,<br>  "id_length_limit": null,<br>  "label_order": [],<br>  "name": null,<br>  "namespace": null,<br>  "regex_replace_chars": null,<br>  "stage": null,<br>  "tags": {}<br>}</pre> | no |
 | delimiter | Delimiter to be used between `namespace`, `environment`, `stage`, `name` and `attributes`.<br>Defaults to `-` (hyphen). Set to `""` to use no delimiter at all. | `string` | `null` | no |
@@ -34,10 +37,13 @@
 | existing\_workers\_role\_policy\_arns | List of existing policy ARNs that will be attached to the workers default role on creation | `list(string)` | `[]` | no |
 | existing\_workers\_role\_policy\_arns\_count | Count of existing policy ARNs that will be attached to the workers default role on creation. Needed to prevent Terraform error `count can't be computed` | `number` | `0` | no |
 | id\_length\_limit | Limit `id` to this many characters.<br>Set to `0` for unlimited length.<br>Set to `null` for default, which is `0`.<br>Does not affect `id_full`. | `number` | `null` | no |
-| instance\_types | Set of instance types associated with the EKS Node Group. Defaults to ["t3.medium"]. Terraform will only perform drift detection if a configuration value is provided | `list(string)` | n/a | yes |
+| instance\_types | Set of instance types associated with the EKS Node Group. Defaults to ["t3.medium"]. Terraform will only perform drift detection if a configuration value is provided | `list(string)` | <pre>[<br>  "t3.medium"<br>]</pre> | no |
+| kubelet\_extra\_args | Extra arguments to pass to kubelet, like "--register-with-taints=dedicated=ci-cd:NoSchedule --node-labels=purpose=ci-worker" | `string` | `""` | no |
 | kubernetes\_labels | Key-value mapping of Kubernetes labels. Only labels that are applied with the EKS API are managed by this argument. Other Kubernetes labels applied to the EKS Node Group will not be managed | `map(string)` | `{}` | no |
 | kubernetes\_version | Kubernetes version. Defaults to EKS Cluster Kubernetes version. Terraform will only perform drift detection if a configuration value is provided | `string` | `null` | no |
 | label\_order | The naming order of the id output and Name tag.<br>Defaults to ["namespace", "environment", "stage", "name", "attributes"].<br>You can omit any of the 5 elements, but at least one must be present. | `list(string)` | `null` | no |
+| launch\_template\_id | The ID of a custom launch template to use for the EKS node group. | `string` | `null` | no |
+| launch\_template\_version | A specific version of the above specific launch template | `string` | `null` | no |
 | max\_size | Maximum number of worker nodes | `number` | n/a | yes |
 | min\_size | Minimum number of worker nodes | `number` | n/a | yes |
 | module\_depends\_on | Can be any value desired. Module will wait for this value to be computed before creating node group. | `any` | `null` | no |

examples/complete/fixtures.us-east-2.tfvars

@@ -29,3 +29,11 @@ min_size = 2
 disk_size = 20
 
 kubernetes_labels = {}
+
+bootstrap_extra_args = "--use-max-pods=5"
+
+kubelet_extra_args = "--node-labels=purpose=ci-worker"
+
+before_cluster_joining_userdata = "echo foo"
+
+after_cluster_joining_userdata = "echo bar"

examples/complete/main.tf

@@ -52,14 +52,8 @@ module "subnets" {
 }
 
 module "eks_cluster" {
-  source = "git::https://github.com/cloudposse/terraform-aws-eks-cluster.git?ref=tags/0.27.0"
-
-  # Temporarily retain old styel, due to circular reference
-  namespace                    = var.namespace
-  stage                        = var.stage
-  name                         = var.name
-  attributes                   = var.attributes
-  tags                         = var.tags
+  source = "git::https://github.com/cloudposse/terraform-aws-eks-cluster.git?ref=tags/0.28.0"
+
   region                       = var.region
   vpc_id                       = module.vpc.vpc_id
   subnet_ids                   = module.subnets.public_subnet_ids
@@ -68,6 +62,8 @@ module "eks_cluster" {
   oidc_provider_enabled        = var.oidc_provider_enabled
   enabled_cluster_log_types    = var.enabled_cluster_log_types
   cluster_log_retention_period = var.cluster_log_retention_period
+
+  context = module.this.context
 }
 
 # Ensure ordering of resource creation to eliminate the race conditions when applying the Kubernetes Auth ConfigMap.
@@ -95,5 +91,11 @@ module "eks_node_group" {
   kubernetes_labels  = var.kubernetes_labels
   disk_size          = var.disk_size
 
+  bootstrap_extra_args = var.bootstrap_extra_args
+  kubelet_extra_args   = var.kubelet_extra_args
+
+  before_cluster_joining_userdata = var.before_cluster_joining_userdata
+  after_cluster_joining_userdata  = var.after_cluster_joining_userdata
+
   context = module.this.context
 }

examples/complete/variables.tf

@@ -98,3 +98,39 @@ variable "min_size" {
   type        = number
   description = "The minimum size of the AutoScaling Group"
 }
+
+variable "launch_template_id" {
+  type        = string
+  description = "The ID of a custom launch template to use for the EKS node group."
+  default     = null
+}
+
+variable "launch_template_version" {
+  type        = string
+  description = "A specific version of the above specific launch template"
+  default     = null
+}
+
+variable "bootstrap_extra_args" {
+  type        = string
+  default     = ""
+  description = "Extra arguments to the `bootstrap.sh` script to enable `--enable-docker-bridge` or `--use-max-pods`"
+}
+
+variable "kubelet_extra_args" {
+  type        = string
+  default     = ""
+  description = "Extra arguments to pass to kubelet, like \"--register-with-taints=dedicated=ci-cd:NoSchedule --node-labels=purpose=ci-worker\""
+}
+
+variable "before_cluster_joining_userdata" {
+  type        = string
+  default     = ""
+  description = "Additional commands to execute on each worker node before joining the EKS cluster (before executing the `bootstrap.sh` script). For more info, see https://kubedex.com/90-days-of-aws-eks-in-production"
+}
+
+variable "after_cluster_joining_userdata" {
+  type        = string
+  default     = ""
+  description = "Additional commands to execute on each worker node after joining the EKS cluster (after executing the `bootstrap.sh` script). For more info, see https://kubedex.com/90-days-of-aws-eks-in-production"
+}

examples/complete/versions.tf

@@ -1,8 +1,8 @@
 terraform {
-  required_version = ">= 0.12.0, < 0.14.0"
+  required_version = ">= 0.13.0, < 0.14.0"
 
   required_providers {
-    aws        = ">= 2.0, < 4.0"
+    aws        = "~> 3.0"
     template   = "~> 2.0"
     null       = "~> 2.0"
     local      = "~> 1.3"

main.tf

@@ -1,8 +1,8 @@
 locals {
   enabled = module.this.enabled
 
-  tags = merge(
-    var.tags,
+  node_group_tags = merge(
+    module.label.tags,
     {
       "kubernetes.io/cluster/${var.cluster_name}" = "owned"
     },
@@ -14,6 +14,21 @@ locals {
     }
   )
   aws_policy_prefix = format("arn:%s:iam::aws:policy", join("", data.aws_partition.current.*.partition))
+
+  userdata_vars = {
+    cluster_name                    = var.cluster_name
+    bootstrap_extra_args            = var.bootstrap_extra_args
+    kubelet_extra_args              = var.kubelet_extra_args
+    before_cluster_joining_userdata = var.before_cluster_joining_userdata
+    after_cluster_joining_userdata  = var.after_cluster_joining_userdata
+  }
+
+  # Use a custom launch_template if one was passed as an input
+  # Otherwise, use the default in this project
+  launch_template = {
+    id             = coalesce(var.launch_template_id, aws_launch_template.default[0].id)
+    latest_version = coalesce(var.launch_template_version, aws_launch_template.default[0].latest_version)
+  }
 }
 
 module "label" {
@@ -24,12 +39,10 @@ module "label" {
   # ...name-workers-blue instead of ...name-blue-workers), historically we forced "workers"
   # to the end of the attribute list, so we do it again here to maintain compatibility.
   attributes = compact(concat(module.this.attributes, ["workers"]))
-  tags       = local.tags
 
   context = module.this.context
 }
 
-
 data "aws_partition" "current" {
   count = local.enabled ? 1 : 0
 }
@@ -113,27 +126,54 @@ resource "aws_iam_role_policy_attachment" "existing_policies_for_eks_workers_rol
   role       = join("", aws_iam_role.default.*.name)
 }
 
+resource "aws_launch_template" "default" {
+  # We'll use this default if we aren't provided with a launch template during invocation
+  count = (local.enabled && (var.launch_template_id == null)) ? 1 : 0
+  block_device_mappings {
+    device_name = "/dev/xvda"
+
+    ebs {
+      volume_size = var.disk_size
+    }
+  }
+
+  instance_type = var.instance_types[0]
+
+  dynamic "tag_specifications" {
+    for_each = ["instance", "volume", "elastic-gpu"]
+    content {
+      resource_type = tag_specifications.value
+      tags          = local.node_group_tags
+    }
+  }
+
+  user_data = base64encode(templatefile("${path.module}/userdata.tpl", local.userdata_vars))
+}
+
 resource "aws_eks_node_group" "default" {
   count           = local.enabled ? 1 : 0
   cluster_name    = var.cluster_name
   node_group_name = module.label.id
   node_role_arn   = join("", aws_iam_role.default.*.arn)
   subnet_ids      = var.subnet_ids
   ami_type        = var.ami_type
-  disk_size       = var.disk_size
-  instance_types  = var.instance_types
   labels          = var.kubernetes_labels
   release_version = var.ami_release_version
   version         = var.kubernetes_version
 
-  tags = module.label.tags
+  tags = local.node_group_tags
 
   scaling_config {
     desired_size = var.desired_size
     max_size     = var.max_size
     min_size     = var.min_size
   }
 
+  launch_template {
+    id      = local.launch_template.id
+    version = local.launch_template.latest_version
+  }
+
   dynamic "remote_access" {
     for_each = var.ec2_ssh_key != null && var.ec2_ssh_key != "" ? ["true"] : []
     content {

userdata.tpl

@@ -0,0 +1,21 @@
+MIME-Version: 1.0
+Content-Type: multipart/mixed; boundary="//"
+
+--//
+Content-Type: text/x-shellscript; charset="us-ascii"
+#!/bin/bash
+
+# userdata for EKS worker nodes to properly configure Kubernetes applications on EC2 instances
+# https://docs.aws.amazon.com/eks/latest/userguide/launch-workers.html
+# https://aws.amazon.com/blogs/opensource/improvements-eks-worker-node-provisioning/
+# https://github.com/awslabs/amazon-eks-ami/blob/master/files/bootstrap.sh#L97
+
+${before_cluster_joining_userdata}
+
+export KUBELET_EXTRA_ARGS="${kubelet_extra_args}"
+
+/etc/eks/bootstrap.sh ${bootstrap_extra_args} '${cluster_name}'
+
+${after_cluster_joining_userdata}
+
+--//--