From b417ff0d4465e5d95d9ca7994509e1b8fd06ba41 Mon Sep 17 00:00:00 2001 From: Chris McKee Date: Tue, 3 Jan 2023 16:29:58 +0000 Subject: [PATCH 01/14] Add initial changes for windows managed node support Add taint example to windows doc --- README.md | 34 +++++++++++++++++++++++++++++++++- README.yaml | 1 + ami.tf | 26 ++++++++++++++++++++++---- docs/terraform.md | 2 +- docs/windows.md | 32 ++++++++++++++++++++++++++++++++ iam.tf | 4 ++-- userdata.tf | 2 +- userdata_nt.tpl | 10 ++++++++++ variables.tf | 6 +++--- 9 files changed, 105 insertions(+), 12 deletions(-) create mode 100644 docs/windows.md create mode 100644 userdata_nt.tpl diff --git a/README.md b/README.md index 2fc518b..ece7c33 100644 --- a/README.md +++ b/README.md @@ -243,6 +243,38 @@ Available targets: ``` +## Windows Managed Node groups + Windows managed node-groups have a few pre-requisites. + + * Your cluster must contain at least one linux based worker node + * Your EKS Cluster must have the `AmazonEKSVPCResourceController` and `AmazonEKSClusterPolicy` policies attached + * Your cluster must have a config-map called amazon-vpc-cni with the following content + ```yaml + apiVersion: v1 + kind: ConfigMap + metadata: + name: amazon-vpc-cni + namespace: kube-system + data: + enable-windows-ipam: "true" + ``` + * It's advisable to taint your Windows nodes + ```yaml + kubernetes_taints = [{ + key = "WINDOWS" + value = "true" + effect = "NO_SCHEDULE" + }] + ``` + * Any pods that target Windows will need to have the following attributes set in their manifest + ```yaml + nodeSelector: + kubernetes.io/os: windows + kubernetes.io/arch: amd64 + ``` + +https://docs.aws.amazon.com/eks/latest/userguide/windows-support.html + ## Requirements | Name | Version | @@ -295,7 +327,7 @@ Available targets: | [after\_cluster\_joining\_userdata](#input\_after\_cluster\_joining\_userdata) | Additional `bash` commands to execute on each worker node after joining the EKS cluster (after executing the `bootstrap.sh` script). For more info, see https://kubedex.com/90-days-of-aws-eks-in-production | `list(string)` | `[]` | no | | [ami\_image\_id](#input\_ami\_image\_id) | AMI to use. Ignored if `launch_template_id` is supplied. | `list(string)` | `[]` | no | | [ami\_release\_version](#input\_ami\_release\_version) | EKS AMI version to use, e.g. For AL2 "1.16.13-20200821" or for bottlerocket "1.2.0-ccf1b754" (no "v"). For AL2 and bottlerocket, it defaults to latest version for Kubernetes version. | `list(string)` | `[]` | no | -| [ami\_type](#input\_ami\_type) | Type of Amazon Machine Image (AMI) associated with the EKS Node Group.
Defaults to `AL2_x86_64`. Valid values: `AL2_x86_64`, `AL2_x86_64_GPU`, `AL2_ARM_64`, `BOTTLEROCKET_x86_64`, and `BOTTLEROCKET_ARM_64`. | `string` | `"AL2_x86_64"` | no | +| [ami\_type](#input\_ami\_type) | Type of Amazon Machine Image (AMI) associated with the EKS Node Group.
Defaults to `AL2_x86_64`. Valid values: `AL2_x86_64, AL2_x86_64_GPU, AL2_ARM_64,CUSTOM, BOTTLEROCKET_ARM_64, BOTTLEROCKET_x86_64, BOTTLEROCKET_ARM_64_NVIDIA, BOTTLEROCKET_x86_64_NVIDIA, WINDOWS_CORE_2019_x86_64, WINDOWS_FULL_2019_x86_64, WINDOWS_CORE_2022_x86_64, WINDOWS_FULL_2022_x86_64`. | `string` | `"AL2_x86_64"` | no | | [associated\_security\_group\_ids](#input\_associated\_security\_group\_ids) | A list of IDs of Security Groups to associate the node group with, in addition to the EKS' created security group.
These security groups will not be modified. | `list(string)` | `[]` | no | | [attributes](#input\_attributes) | ID element. Additional attributes (e.g. `workers` or `cluster`) to add to `id`,
in the order they appear in the list. New attributes are appended to the
end of the list. The elements of the list are joined by the `delimiter`
and treated as a single ID element. | `list(string)` | `[]` | no | | [before\_cluster\_joining\_userdata](#input\_before\_cluster\_joining\_userdata) | Additional `bash` commands to execute on each worker node before joining the EKS cluster (before executing the `bootstrap.sh` script). For more info, see https://kubedex.com/90-days-of-aws-eks-in-production | `list(string)` | `[]` | no | diff --git a/README.yaml b/README.yaml index 5ffe1e4..7fb2255 100644 --- a/README.yaml +++ b/README.yaml @@ -191,6 +191,7 @@ usage: |2- include: - "docs/targets.md" + - "docs/windows.md" - "docs/terraform.md" # Contributors to this project diff --git a/ami.tf b/ami.tf index e349760..0e41107 100644 --- a/ami.tf +++ b/ami.tf @@ -7,6 +7,12 @@ locals { "AL2_ARM_64" : "-arm64", "BOTTLEROCKET_x86_64" : "x86_64", "BOTTLEROCKET_ARM_64" : "aarch64" + "BOTTLEROCKET_ARM_64_NVIDIA" : "-gpu" + "BOTTLEROCKET_x86_64_NVIDIA" : "-gpu" + "WINDOWS_CORE_2019_x86_64" : "" + "WINDOWS_FULL_2019_x86_64" : "" + "WINDOWS_CORE_2022_x86_64" : "" + "WINDOWS_FULL_2022_x86_64" : "" } ami_kind = split("_", var.ami_type)[0] @@ -18,6 +24,12 @@ locals { # bottlerocket-aws-k8s-{ami_kubernetes_version}-{arch_label}-v{ami_version} # e.g. bottlerocket-aws-k8s-1.21-x86_64-v1.2.0-ccf1b754 "BOTTLEROCKET" : "bottlerocket-aws-k8s-%s-%s-%s" + # Windows_Server-2019-English-Core-EKS_Optimized-{ami_kubernetes_version}-{ami_version} + # e.g. Windows_Server-2019-English-Core-EKS_Optimized-1.23-2022.11.08 + "WINDOWS_CORE_2019" : "Windows_Server-2019-English-Core-EKS_Optimized-%s-%s" + "WINDOWS_FULL_2019" : "Windows_Server-2019-English-Full-EKS_Optimized-%s-%s" + "WINDOWS_CORE_2022" : "Windows_Server-2022-English-Core-EKS_Optimized-%s-%s" + "WINDOWS_FULL_2022" : "Windows_Server-2022-English-Full-EKS_Optimized-%s-%s" } # Kubernetes version priority (first one to be set wins) @@ -37,19 +49,25 @@ locals { # if ami_release_version = "1.21-20211013" # insert the letter v prior to the ami_version so it becomes 1.21-v20211013 # if not, use the kubernetes version - "AL2" : (length(var.ami_release_version) == 1 ? - replace(var.ami_release_version[0], "/^(\\d+\\.\\d+)\\.\\d+-(\\d+)$/", "$1-v$2") : + "AL2" : (length(var.ami_release_version) == 1 ? replace(var.ami_release_version[0], "/^(\\d+\\.\\d+)\\.\\d+-(\\d+)$/", "$1-v$2") : "${local.ami_kubernetes_version}-*"), # if ami_release_version = "1.2.0-ccf1b754" # prefex the ami release version with the letter v # if not, use an asterisk - "BOTTLEROCKET" : (length(var.ami_release_version) == 1 ? - format("v%s", var.ami_release_version[0]) : "*"), + "BOTTLEROCKET" : (length(var.ami_release_version) == 1 ? format("v%s", var.ami_release_version[0]) : "*"), + "WINDOWS_CORE_2019" : (length(var.ami_release_version) == 1 ? format("%s", var.ami_release_version[0]) : "*"), + "WINDOWS_FULL_2019" : (length(var.ami_release_version) == 1 ? format("%s", var.ami_release_version[0]) : "*"), + "WINDOWS_CORE_2022" : (length(var.ami_release_version) == 1 ? format("%s", var.ami_release_version[0]) : "*"), + "WINDOWS_FULL_2022" : (length(var.ami_release_version) == 1 ? format("%s", var.ami_release_version[0]) : "*"), } : {} ami_regex = local.need_ami_id ? { "AL2" : format(local.ami_format["AL2"], local.arch_label_map[var.ami_type], local.ami_version_regex[local.ami_kind]), "BOTTLEROCKET" : format(local.ami_format["BOTTLEROCKET"], local.ami_kubernetes_version, local.arch_label_map[var.ami_type], local.ami_version_regex[local.ami_kind]), + "WINDOWS_CORE_2019" : format(local.ami_format["WINDOWS_CORE_2019"], local.ami_kubernetes_version, local.ami_version_regex[local.ami_kind]), + "WINDOWS_FULL_2019" : format(local.ami_format["WINDOWS_FULL_2019"], local.ami_kubernetes_version, local.ami_version_regex[local.ami_kind]), + "WINDOWS_CORE_2022" : format(local.ami_format["WINDOWS_CORE_2022"], local.ami_kubernetes_version, local.ami_version_regex[local.ami_kind]), + "WINDOWS_FULL_2022" : format(local.ami_format["WINDOWS_FULL_2022"], local.ami_kubernetes_version, local.ami_version_regex[local.ami_kind]), } : {} } diff --git a/docs/terraform.md b/docs/terraform.md index 5396860..e749f50 100644 --- a/docs/terraform.md +++ b/docs/terraform.md @@ -51,7 +51,7 @@ | [after\_cluster\_joining\_userdata](#input\_after\_cluster\_joining\_userdata) | Additional `bash` commands to execute on each worker node after joining the EKS cluster (after executing the `bootstrap.sh` script). For more info, see https://kubedex.com/90-days-of-aws-eks-in-production | `list(string)` | `[]` | no | | [ami\_image\_id](#input\_ami\_image\_id) | AMI to use. Ignored if `launch_template_id` is supplied. | `list(string)` | `[]` | no | | [ami\_release\_version](#input\_ami\_release\_version) | EKS AMI version to use, e.g. For AL2 "1.16.13-20200821" or for bottlerocket "1.2.0-ccf1b754" (no "v"). For AL2 and bottlerocket, it defaults to latest version for Kubernetes version. | `list(string)` | `[]` | no | -| [ami\_type](#input\_ami\_type) | Type of Amazon Machine Image (AMI) associated with the EKS Node Group.
Defaults to `AL2_x86_64`. Valid values: `AL2_x86_64`, `AL2_x86_64_GPU`, `AL2_ARM_64`, `BOTTLEROCKET_x86_64`, and `BOTTLEROCKET_ARM_64`. | `string` | `"AL2_x86_64"` | no | +| [ami\_type](#input\_ami\_type) | Type of Amazon Machine Image (AMI) associated with the EKS Node Group.
Defaults to `AL2_x86_64`. Valid values: `AL2_x86_64, AL2_x86_64_GPU, AL2_ARM_64,CUSTOM, BOTTLEROCKET_ARM_64, BOTTLEROCKET_x86_64, BOTTLEROCKET_ARM_64_NVIDIA, BOTTLEROCKET_x86_64_NVIDIA, WINDOWS_CORE_2019_x86_64, WINDOWS_FULL_2019_x86_64, WINDOWS_CORE_2022_x86_64, WINDOWS_FULL_2022_x86_64`. | `string` | `"AL2_x86_64"` | no | | [associated\_security\_group\_ids](#input\_associated\_security\_group\_ids) | A list of IDs of Security Groups to associate the node group with, in addition to the EKS' created security group.
These security groups will not be modified. | `list(string)` | `[]` | no | | [attributes](#input\_attributes) | ID element. Additional attributes (e.g. `workers` or `cluster`) to add to `id`,
in the order they appear in the list. New attributes are appended to the
end of the list. The elements of the list are joined by the `delimiter`
and treated as a single ID element. | `list(string)` | `[]` | no | | [before\_cluster\_joining\_userdata](#input\_before\_cluster\_joining\_userdata) | Additional `bash` commands to execute on each worker node before joining the EKS cluster (before executing the `bootstrap.sh` script). For more info, see https://kubedex.com/90-days-of-aws-eks-in-production | `list(string)` | `[]` | no | diff --git a/docs/windows.md b/docs/windows.md new file mode 100644 index 0000000..93e0ce8 --- /dev/null +++ b/docs/windows.md @@ -0,0 +1,32 @@ + +## Windows Managed Node groups + Windows managed node-groups have a few pre-requisites. + + * Your cluster must contain at least one linux based worker node + * Your EKS Cluster must have the `AmazonEKSVPCResourceController` and `AmazonEKSClusterPolicy` policies attached + * Your cluster must have a config-map called amazon-vpc-cni with the following content + ```yaml + apiVersion: v1 + kind: ConfigMap + metadata: + name: amazon-vpc-cni + namespace: kube-system + data: + enable-windows-ipam: "true" + ``` + * It's advisable to taint your Windows nodes + ```yaml + kubernetes_taints = [{ + key = "WINDOWS" + value = "true" + effect = "NO_SCHEDULE" + }] + ``` + * Any pods that target Windows will need to have the following attributes set in their manifest + ```yaml + nodeSelector: + kubernetes.io/os: windows + kubernetes.io/arch: amd64 + ``` + +https://docs.aws.amazon.com/eks/latest/userguide/windows-support.html diff --git a/iam.tf b/iam.tf index 01addf8..4f3e547 100644 --- a/iam.tf +++ b/iam.tf @@ -51,9 +51,9 @@ resource "aws_iam_role_policy_attachment" "existing_policies_for_eks_workers_rol # Create a CNI policy that is a merger of AmazonEKS_CNI_Policy and required IPv6 permissions # https://github.com/SummitRoute/aws_managed_policies/blob/master/policies/AmazonEKS_CNI_Policy # https://docs.aws.amazon.com/eks/latest/userguide/cni-iam-role.html#cni-iam-role-create-ipv6-policy - +# https://docs.aws.amazon.com/eks/latest/userguide/windows-support.html data "aws_iam_policy_document" "ipv6_eks_cni_policy" { - count = local.create_role && var.node_role_cni_policy_enabled ? 1 : 0 + count = local.create_role && var.node_role_cni_policy_enabled || local.create_role && can(regex("WINDOWS", var.ami_type)) ? 1 : 0 statement { effect = "Allow" diff --git a/userdata.tf b/userdata.tf index 2296a7a..36887ee 100644 --- a/userdata.tf +++ b/userdata.tf @@ -41,7 +41,7 @@ locals { (length(var.before_cluster_joining_userdata) > 0) || local.need_bootstrap) : false userdata = local.need_userdata ? ( - base64encode(templatefile("${path.module}/userdata.tpl", merge(local.userdata_vars, local.cluster_data)))) : ( + base64encode(templatefile(can(regex("WINDOWS", var.ami_type)) ? "${path.module}/userdata_nt.tpl" : "${path.module}/userdata.tpl", merge(local.userdata_vars, local.cluster_data)))) : ( try(var.userdata_override_base64[0], null) ) } diff --git a/userdata_nt.tpl b/userdata_nt.tpl new file mode 100644 index 0000000..7ef7c8c --- /dev/null +++ b/userdata_nt.tpl @@ -0,0 +1,10 @@ + +[string]$EKSBootstrapScriptFile = "$env:ProgramFiles\Amazon\EKS\Start-EKSBootstrap.ps1" + +${before_cluster_joining_userdata} + +& $EKSBootstrapScriptFile -EKSClusterName "${cluster_name}" -APIServerEndpoint "${cluster_endpoint}" -Base64ClusterCA "${certificate_authority_data}" -KubeletExtraArgs "${bootstrap_extra_args}" 3>&1 4>&1 5>&1 6>&1 +$LastError = if ($?) { 0 } else { $Error[0].Exception.HResult } + +${after_cluster_joining_userdata} + \ No newline at end of file diff --git a/variables.tf b/variables.tf index 8fba47a..cf4d6fb 100644 --- a/variables.tf +++ b/variables.tf @@ -111,14 +111,14 @@ variable "ami_type" { type = string description = <<-EOT Type of Amazon Machine Image (AMI) associated with the EKS Node Group. - Defaults to `AL2_x86_64`. Valid values: `AL2_x86_64`, `AL2_x86_64_GPU`, `AL2_ARM_64`, `BOTTLEROCKET_x86_64`, and `BOTTLEROCKET_ARM_64`. + Defaults to `AL2_x86_64`. Valid values: `AL2_x86_64, AL2_x86_64_GPU, AL2_ARM_64,CUSTOM, BOTTLEROCKET_ARM_64, BOTTLEROCKET_x86_64, BOTTLEROCKET_ARM_64_NVIDIA, BOTTLEROCKET_x86_64_NVIDIA, WINDOWS_CORE_2019_x86_64, WINDOWS_FULL_2019_x86_64, WINDOWS_CORE_2022_x86_64, WINDOWS_FULL_2022_x86_64`. EOT default = "AL2_x86_64" validation { condition = ( - contains(["AL2_x86_64", "AL2_x86_64_GPU", "AL2_ARM_64", "BOTTLEROCKET_x86_64", "BOTTLEROCKET_ARM_64", "CUSTOM"], var.ami_type) + contains(["AL2_x86_64", "AL2_x86_64_GPU", "AL2_ARM_64", "CUSTOM", "BOTTLEROCKET_ARM_64", "BOTTLEROCKET_x86_64", "BOTTLEROCKET_ARM_64_NVIDIA", "BOTTLEROCKET_x86_64_NVIDIA", "WINDOWS_CORE_2019_x86_64", "WINDOWS_FULL_2019_x86_64", "WINDOWS_CORE_2022_x86_64", "WINDOWS_FULL_2022_x86_64"], var.ami_type) ) - error_message = "Var ami_type must be one of \"AL2_x86_64\", \"AL2_x86_64_GPU\", \"AL2_ARM_64\", \"BOTTLEROCKET_x86_64\", \"BOTTLEROCKET_ARM_64\", or \"CUSTOM\"." + error_message = "Var ami_type must be one of \"AL2_x86_64\",\"AL2_x86_64_GPU\",\"AL2_ARM_64\",\"BOTTLEROCKET_ARM_64\",\"BOTTLEROCKET_x86_64\",\"BOTTLEROCKET_ARM_64_NVIDIA\",\"BOTTLEROCKET_x86_64_NVIDIA\",\"WINDOWS_CORE_2019_x86_64\",\"WINDOWS_FULL_2019_x86_64\",\"WINDOWS_CORE_2022_x86_64\",\"WINDOWS_FULL_2022_x86_64\", or \"CUSTOM\"." } } From 9e0b0f973b53b92e188d2bcdaa592e9797d6d3c7 Mon Sep 17 00:00:00 2001 From: Chris McKee Date: Thu, 23 Feb 2023 18:54:19 +0000 Subject: [PATCH 02/14] Update iam.tf Co-authored-by: Andriy Knysh --- iam.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/iam.tf b/iam.tf index 4f3e547..a0e0713 100644 --- a/iam.tf +++ b/iam.tf @@ -53,7 +53,7 @@ resource "aws_iam_role_policy_attachment" "existing_policies_for_eks_workers_rol # https://docs.aws.amazon.com/eks/latest/userguide/cni-iam-role.html#cni-iam-role-create-ipv6-policy # https://docs.aws.amazon.com/eks/latest/userguide/windows-support.html data "aws_iam_policy_document" "ipv6_eks_cni_policy" { - count = local.create_role && var.node_role_cni_policy_enabled || local.create_role && can(regex("WINDOWS", var.ami_type)) ? 1 : 0 + count = local.create_role && (var.node_role_cni_policy_enabled || can(regex("WINDOWS", var.ami_type))) ? 1 : 0 statement { effect = "Allow" From feec10b8dc07192928b22adcf70864ab92d19c11 Mon Sep 17 00:00:00 2001 From: Chris McKee Date: Thu, 23 Feb 2023 18:54:50 +0000 Subject: [PATCH 03/14] Update Changes to allow windows autotaint label, fixes * Add the windows taint by default to windows nodes * Correct ami_kind filter * Alter userdata nt to match current userscript * Guard userdata against errors from userscript stopping node-join. * Remove disk_size * Add windnows_coredns_service_address because EKS Windows networking isn't fab. --- .gitignore | 4 + README.md | 12 +- ami.tf | 7 +- docs/terraform.md | 10 +- docs/windows.md | 2 +- examples/complete/fixtures.eu-west-2.tfvars | 46 ++++++ examples/complete/fixtures.us-east-2.tfvars | 2 - examples/complete/main.tf | 169 +++++++++++++++++++- examples/complete/variables.tf | 35 +++- iam.tf | 2 +- launch-template.tf | 1 - main.tf | 18 ++- outputs.tf | 5 + userdata.tf | 2 +- userdata_nt.tpl | 16 +- variables.tf | 12 +- versions.tf | 8 +- 17 files changed, 305 insertions(+), 46 deletions(-) create mode 100644 examples/complete/fixtures.eu-west-2.tfvars diff --git a/.gitignore b/.gitignore index 0c6f3c5..fbbeaaa 100644 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,7 @@ # vim editor *.swp + +/.terraform.lock.hcl +/.vscode +/examples/complete/.terraform.lock.hcl diff --git a/README.md b/README.md index ece7c33..4920adb 100644 --- a/README.md +++ b/README.md @@ -258,7 +258,7 @@ Available targets: data: enable-windows-ipam: "true" ``` - * It's advisable to taint your Windows nodes + * Windows nodes will automatically be tainted ```yaml kubernetes_taints = [{ key = "WINDOWS" @@ -279,15 +279,15 @@ https://docs.aws.amazon.com/eks/latest/userguide/windows-support.html | Name | Version | |------|---------| -| [terraform](#requirement\_terraform) | >= 0.14.11 | -| [aws](#requirement\_aws) | >= 3.56 | +| [terraform](#requirement\_terraform) | >= 0.15.00 | +| [aws](#requirement\_aws) | >= 4.0 | | [random](#requirement\_random) | >= 2.0 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 3.56 | +| [aws](#provider\_aws) | >= 4.0 | | [random](#provider\_random) | >= 2.0 | ## Modules @@ -327,7 +327,7 @@ https://docs.aws.amazon.com/eks/latest/userguide/windows-support.html | [after\_cluster\_joining\_userdata](#input\_after\_cluster\_joining\_userdata) | Additional `bash` commands to execute on each worker node after joining the EKS cluster (after executing the `bootstrap.sh` script). For more info, see https://kubedex.com/90-days-of-aws-eks-in-production | `list(string)` | `[]` | no | | [ami\_image\_id](#input\_ami\_image\_id) | AMI to use. Ignored if `launch_template_id` is supplied. | `list(string)` | `[]` | no | | [ami\_release\_version](#input\_ami\_release\_version) | EKS AMI version to use, e.g. For AL2 "1.16.13-20200821" or for bottlerocket "1.2.0-ccf1b754" (no "v"). For AL2 and bottlerocket, it defaults to latest version for Kubernetes version. | `list(string)` | `[]` | no | -| [ami\_type](#input\_ami\_type) | Type of Amazon Machine Image (AMI) associated with the EKS Node Group.
Defaults to `AL2_x86_64`. Valid values: `AL2_x86_64, AL2_x86_64_GPU, AL2_ARM_64,CUSTOM, BOTTLEROCKET_ARM_64, BOTTLEROCKET_x86_64, BOTTLEROCKET_ARM_64_NVIDIA, BOTTLEROCKET_x86_64_NVIDIA, WINDOWS_CORE_2019_x86_64, WINDOWS_FULL_2019_x86_64, WINDOWS_CORE_2022_x86_64, WINDOWS_FULL_2022_x86_64`. | `string` | `"AL2_x86_64"` | no | +| [ami\_type](#input\_ami\_type) | Type of Amazon Machine Image (AMI) associated with the EKS Node Group.
Defaults to `AL2_x86_64`. Valid values: `AL2_x86_64, AL2_x86_64_GPU, AL2_ARM_64, CUSTOM, BOTTLEROCKET_ARM_64, BOTTLEROCKET_x86_64, BOTTLEROCKET_ARM_64_NVIDIA, BOTTLEROCKET_x86_64_NVIDIA, WINDOWS_CORE_2019_x86_64, WINDOWS_FULL_2019_x86_64, WINDOWS_CORE_2022_x86_64, WINDOWS_FULL_2022_x86_64`. | `string` | `"AL2_x86_64"` | no | | [associated\_security\_group\_ids](#input\_associated\_security\_group\_ids) | A list of IDs of Security Groups to associate the node group with, in addition to the EKS' created security group.
These security groups will not be modified. | `list(string)` | `[]` | no | | [attributes](#input\_attributes) | ID element. Additional attributes (e.g. `workers` or `cluster`) to add to `id`,
in the order they appear in the list. New attributes are appended to the
end of the list. The elements of the list are joined by the `delimiter`
and treated as a single ID element. | `list(string)` | `[]` | no | | [before\_cluster\_joining\_userdata](#input\_before\_cluster\_joining\_userdata) | Additional `bash` commands to execute on each worker node before joining the EKS cluster (before executing the `bootstrap.sh` script). For more info, see https://kubedex.com/90-days-of-aws-eks-in-production | `list(string)` | `[]` | no | @@ -382,6 +382,7 @@ https://docs.aws.amazon.com/eks/latest/userguide/windows-support.html | [tenant](#input\_tenant) | ID element \_(Rarely used, not included by default)\_. A customer identifier, indicating who this instance of a resource is for | `string` | `null` | no | | [update\_config](#input\_update\_config) | Configuration for the `eks_node_group` [`update_config` Configuration Block](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/eks_node_group#update_config-configuration-block).
Specify exactly one of `max_unavailable` (node count) or `max_unavailable_percentage` (percentage of nodes). | `list(map(number))` | `[]` | no | | [userdata\_override\_base64](#input\_userdata\_override\_base64) | Many features of this module rely on the `bootstrap.sh` provided with Amazon Linux, and this module
may generate "user data" that expects to find that script. If you want to use an AMI that is not
compatible with the Amazon Linux `bootstrap.sh` initialization, then use `userdata_override_base64` to provide
your own (Base64 encoded) user data. Use "" to prevent any user data from being set.

Setting `userdata_override_base64` disables `kubernetes_taints`, `kubelet_additional_options`,
`before_cluster_joining_userdata`, `after_cluster_joining_userdata`, and `bootstrap_additional_options`. | `list(string)` | `[]` | no | +| [windnows\_coredns\_service\_address](#input\_windnows\_coredns\_service\_address) | The value passed to the Powershell setup script on Windows Node start
This is required to be set to the service address of core-dns or pod-networking will fail
the default behaviour (without a value) will select the DNS of the first network interface | `string` | `"172.20.0.10"` | no | ## Outputs @@ -398,6 +399,7 @@ https://docs.aws.amazon.com/eks/latest/userguide/windows-support.html | [eks\_node\_group\_role\_name](#output\_eks\_node\_group\_role\_name) | Name of the worker nodes IAM role | | [eks\_node\_group\_status](#output\_eks\_node\_group\_status) | Status of the EKS Node Group | | [eks\_node\_group\_tags\_all](#output\_eks\_node\_group\_tags\_all) | A map of tags assigned to the resource, including those inherited from the provider default\_tags configuration block. | +| [eks\_node\_group\_windows\_note](#output\_eks\_node\_group\_windows\_note) | Instructions on changes a user needs to follow or script for a windows node group in the event of a custom ami | diff --git a/ami.tf b/ami.tf index 0e41107..f338bc4 100644 --- a/ami.tf +++ b/ami.tf @@ -15,7 +15,7 @@ locals { "WINDOWS_FULL_2022_x86_64" : "" } - ami_kind = split("_", var.ami_type)[0] + ami_kind = split("_", var.ami_type)[0] != "WINDOWS" ? split("_", var.ami_type)[0] : format("WINDOWS_%s_%s", split("_", var.ami_type)[1], split("_", var.ami_type)[2]) ami_format = { # amazon-eks{arch_label}-node-{ami_kubernetes_version}-v{ami_version} @@ -49,10 +49,9 @@ locals { # if ami_release_version = "1.21-20211013" # insert the letter v prior to the ami_version so it becomes 1.21-v20211013 # if not, use the kubernetes version - "AL2" : (length(var.ami_release_version) == 1 ? replace(var.ami_release_version[0], "/^(\\d+\\.\\d+)\\.\\d+-(\\d+)$/", "$1-v$2") : - "${local.ami_kubernetes_version}-*"), + "AL2" : (length(var.ami_release_version) == 1 ? replace(var.ami_release_version[0], "/^(\\d+\\.\\d+)\\.\\d+-(\\d+)$/", "$1-v$2") : "${local.ami_kubernetes_version}-*"), # if ami_release_version = "1.2.0-ccf1b754" - # prefex the ami release version with the letter v + # prefix the ami release version with the letter v # if not, use an asterisk "BOTTLEROCKET" : (length(var.ami_release_version) == 1 ? format("v%s", var.ami_release_version[0]) : "*"), "WINDOWS_CORE_2019" : (length(var.ami_release_version) == 1 ? format("%s", var.ami_release_version[0]) : "*"), diff --git a/docs/terraform.md b/docs/terraform.md index e749f50..787d260 100644 --- a/docs/terraform.md +++ b/docs/terraform.md @@ -3,15 +3,15 @@ | Name | Version | |------|---------| -| [terraform](#requirement\_terraform) | >= 0.14.11 | -| [aws](#requirement\_aws) | >= 3.56 | +| [terraform](#requirement\_terraform) | >= 0.15.00 | +| [aws](#requirement\_aws) | >= 4.0 | | [random](#requirement\_random) | >= 2.0 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 3.56 | +| [aws](#provider\_aws) | >= 4.0 | | [random](#provider\_random) | >= 2.0 | ## Modules @@ -51,7 +51,7 @@ | [after\_cluster\_joining\_userdata](#input\_after\_cluster\_joining\_userdata) | Additional `bash` commands to execute on each worker node after joining the EKS cluster (after executing the `bootstrap.sh` script). For more info, see https://kubedex.com/90-days-of-aws-eks-in-production | `list(string)` | `[]` | no | | [ami\_image\_id](#input\_ami\_image\_id) | AMI to use. Ignored if `launch_template_id` is supplied. | `list(string)` | `[]` | no | | [ami\_release\_version](#input\_ami\_release\_version) | EKS AMI version to use, e.g. For AL2 "1.16.13-20200821" or for bottlerocket "1.2.0-ccf1b754" (no "v"). For AL2 and bottlerocket, it defaults to latest version for Kubernetes version. | `list(string)` | `[]` | no | -| [ami\_type](#input\_ami\_type) | Type of Amazon Machine Image (AMI) associated with the EKS Node Group.
Defaults to `AL2_x86_64`. Valid values: `AL2_x86_64, AL2_x86_64_GPU, AL2_ARM_64,CUSTOM, BOTTLEROCKET_ARM_64, BOTTLEROCKET_x86_64, BOTTLEROCKET_ARM_64_NVIDIA, BOTTLEROCKET_x86_64_NVIDIA, WINDOWS_CORE_2019_x86_64, WINDOWS_FULL_2019_x86_64, WINDOWS_CORE_2022_x86_64, WINDOWS_FULL_2022_x86_64`. | `string` | `"AL2_x86_64"` | no | +| [ami\_type](#input\_ami\_type) | Type of Amazon Machine Image (AMI) associated with the EKS Node Group.
Defaults to `AL2_x86_64`. Valid values: `AL2_x86_64, AL2_x86_64_GPU, AL2_ARM_64, CUSTOM, BOTTLEROCKET_ARM_64, BOTTLEROCKET_x86_64, BOTTLEROCKET_ARM_64_NVIDIA, BOTTLEROCKET_x86_64_NVIDIA, WINDOWS_CORE_2019_x86_64, WINDOWS_FULL_2019_x86_64, WINDOWS_CORE_2022_x86_64, WINDOWS_FULL_2022_x86_64`. | `string` | `"AL2_x86_64"` | no | | [associated\_security\_group\_ids](#input\_associated\_security\_group\_ids) | A list of IDs of Security Groups to associate the node group with, in addition to the EKS' created security group.
These security groups will not be modified. | `list(string)` | `[]` | no | | [attributes](#input\_attributes) | ID element. Additional attributes (e.g. `workers` or `cluster`) to add to `id`,
in the order they appear in the list. New attributes are appended to the
end of the list. The elements of the list are joined by the `delimiter`
and treated as a single ID element. | `list(string)` | `[]` | no | | [before\_cluster\_joining\_userdata](#input\_before\_cluster\_joining\_userdata) | Additional `bash` commands to execute on each worker node before joining the EKS cluster (before executing the `bootstrap.sh` script). For more info, see https://kubedex.com/90-days-of-aws-eks-in-production | `list(string)` | `[]` | no | @@ -106,6 +106,7 @@ | [tenant](#input\_tenant) | ID element \_(Rarely used, not included by default)\_. A customer identifier, indicating who this instance of a resource is for | `string` | `null` | no | | [update\_config](#input\_update\_config) | Configuration for the `eks_node_group` [`update_config` Configuration Block](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/eks_node_group#update_config-configuration-block).
Specify exactly one of `max_unavailable` (node count) or `max_unavailable_percentage` (percentage of nodes). | `list(map(number))` | `[]` | no | | [userdata\_override\_base64](#input\_userdata\_override\_base64) | Many features of this module rely on the `bootstrap.sh` provided with Amazon Linux, and this module
may generate "user data" that expects to find that script. If you want to use an AMI that is not
compatible with the Amazon Linux `bootstrap.sh` initialization, then use `userdata_override_base64` to provide
your own (Base64 encoded) user data. Use "" to prevent any user data from being set.

Setting `userdata_override_base64` disables `kubernetes_taints`, `kubelet_additional_options`,
`before_cluster_joining_userdata`, `after_cluster_joining_userdata`, and `bootstrap_additional_options`. | `list(string)` | `[]` | no | +| [windnows\_coredns\_service\_address](#input\_windnows\_coredns\_service\_address) | The value passed to the Powershell setup script on Windows Node start
This is required to be set to the service address of core-dns or pod-networking will fail
the default behaviour (without a value) will select the DNS of the first network interface | `string` | `"172.20.0.10"` | no | ## Outputs @@ -122,4 +123,5 @@ | [eks\_node\_group\_role\_name](#output\_eks\_node\_group\_role\_name) | Name of the worker nodes IAM role | | [eks\_node\_group\_status](#output\_eks\_node\_group\_status) | Status of the EKS Node Group | | [eks\_node\_group\_tags\_all](#output\_eks\_node\_group\_tags\_all) | A map of tags assigned to the resource, including those inherited from the provider default\_tags configuration block. | +| [eks\_node\_group\_windows\_note](#output\_eks\_node\_group\_windows\_note) | Instructions on changes a user needs to follow or script for a windows node group in the event of a custom ami | diff --git a/docs/windows.md b/docs/windows.md index 93e0ce8..c322a2a 100644 --- a/docs/windows.md +++ b/docs/windows.md @@ -14,7 +14,7 @@ data: enable-windows-ipam: "true" ``` - * It's advisable to taint your Windows nodes + * Windows nodes will automatically be tainted ```yaml kubernetes_taints = [{ key = "WINDOWS" diff --git a/examples/complete/fixtures.eu-west-2.tfvars b/examples/complete/fixtures.eu-west-2.tfvars new file mode 100644 index 0000000..0221106 --- /dev/null +++ b/examples/complete/fixtures.eu-west-2.tfvars @@ -0,0 +1,46 @@ +region = "eu-west-2" + +availability_zones = ["eu-west-2a", "eu-west-2b"] + +vpc_cidr_block = "172.16.0.0/16" + +namespace = "eg" + +stage = "test" + +name = "eks-node-group" + +kubernetes_version = "1.25" + +oidc_provider_enabled = true + +enabled_cluster_log_types = ["audit"] + +cluster_log_retention_period = 7 + +instance_types = ["t3.small"] + +desired_size = 1 + +max_size = 1 + +min_size = 1 + +disk_size = 90 + +kubernetes_labels = { + terratest = "true" +} + +before_cluster_joining_userdata = <<-EOT + printf "\n\n###\nExample output from before_cluster_joining_userdata\n###\n\n" + EOT + +update_config = [{ max_unavailable = 1 }] + +kubernetes_taints = [ + { + key = "test" + value = null + effect = "PREFER_NO_SCHEDULE" +}] diff --git a/examples/complete/fixtures.us-east-2.tfvars b/examples/complete/fixtures.us-east-2.tfvars index 8f20c7f..e4f02dc 100644 --- a/examples/complete/fixtures.us-east-2.tfvars +++ b/examples/complete/fixtures.us-east-2.tfvars @@ -26,8 +26,6 @@ max_size = 3 min_size = 2 -disk_size = 20 - kubernetes_labels = { terratest = "true" } diff --git a/examples/complete/main.tf b/examples/complete/main.tf index 286f0ff..19881f2 100644 --- a/examples/complete/main.tf +++ b/examples/complete/main.tf @@ -4,7 +4,7 @@ provider "aws" { module "label" { source = "cloudposse/label/null" - version = "0.25.0" + #version = "0.25.0" # This is the preferred way to add attributes. It will put "cluster" last # after any attributes set in `var.attributes` or `context.attributes`. @@ -116,9 +116,9 @@ module "https_sg" { module "eks_cluster" { - source = "cloudposse/eks-cluster/aws" - version = "2.4.0" - + # source = "cloudposse/eks-cluster/aws" + version = "2.2.0" + source = "github.com/ChrisMcKee/terraform-aws-eks-cluster" region = var.region vpc_id = module.vpc.vpc_id subnet_ids = module.subnets.public_subnet_ids @@ -132,10 +132,12 @@ module "eks_cluster" { kube_data_auth_enabled = false kube_exec_auth_enabled = true + windows_support = true + context = module.this.context } -module "eks_node_group" { +module "eks_nix_node_group" { source = "../../" subnet_ids = module.this.enabled ? module.subnets.public_subnet_ids : ["filler_string_for_enabled_is_false"] @@ -147,19 +149,19 @@ module "eks_node_group" { kubernetes_version = [var.kubernetes_version] kubernetes_labels = merge(var.kubernetes_labels, { attributes = coalesce(join(module.this.delimiter, module.this.attributes), "none") }) kubernetes_taints = var.kubernetes_taints - # disk_size = var.disk_size + ec2_ssh_key_name = var.ec2_ssh_key_name ssh_access_security_group_ids = [module.ssh_source_access.id] associated_security_group_ids = [module.ssh_source_access.id, module.https_sg.id] node_role_policy_arns = [local.extra_policy_arn] update_config = var.update_config - after_cluster_joining_userdata = var.after_cluster_joining_userdata + //after_cluster_joining_userdata = var.after_cluster_joining_userdata ami_type = var.ami_type ami_release_version = var.ami_release_version - before_cluster_joining_userdata = [var.before_cluster_joining_userdata] + //before_cluster_joining_userdata = [var.before_cluster_joining_userdata] context = module.this.context @@ -175,3 +177,154 @@ module "eks_node_group" { delete = "20m" }] } + +module "windowslabel" { + source = "cloudposse/label/null" + + enabled = var.include_windows_node + + context = module.this.context + attributes = ["nt", "workers"] + tags = { + nodeclass = "windows2019" + } +} + +module "eks_nt_node_group" { + source = "../../" + + enabled = var.include_windows_node + + context = module.windowslabel.context + + subnet_ids = module.this.enabled ? module.subnets.public_subnet_ids : ["filler_string_for_enabled_is_false"] + cluster_name = module.eks_cluster.eks_cluster_id + instance_types = var.instance_types + desired_size = 1 + min_size = 1 + max_size = 1 + kubernetes_version = [var.kubernetes_version] + kubernetes_labels = merge(var.kubernetes_labels, { attributes = coalesce(join(module.this.delimiter, module.this.attributes), "none") }) + + ec2_ssh_key_name = var.ec2_ssh_key_name + ssh_access_security_group_ids = [module.ssh_source_access.id] + associated_security_group_ids = [module.ssh_source_access.id, module.https_sg.id] + node_role_policy_arns = [local.extra_policy_arn] + update_config = var.update_config + + ami_type = var.windows_node_ami_type + + # after_cluster_joining_userdata = [ + # "Write-Host Hello" + # ] + + # Ensure ordering of resource creation to eliminate the race conditions when applying the Kubernetes Auth ConfigMap. + # Do not create Node Group before the EKS cluster is created and the `aws-auth` Kubernetes ConfigMap is applied. + depends_on = [module.eks_cluster, module.eks_cluster.kubernetes_config_map_id] + + create_before_destroy = true + + node_group_terraform_timeouts = [{ + create = "40m" + update = null + delete = "20m" + }] + + resources_to_tag = ["instance", "volume", "spot-instances-request", "network-interface"] +} + +data "aws_eks_cluster_auth" "this" { + name = module.eks_cluster.cluster_id +} + +locals { + kubeconfig = yamlencode({ + apiVersion = "v1" + kind = "Config" + current-context = "terraform" + clusters = [{ + name = module.eks_cluster.cluster_id + cluster = { + certificate-authority-data = module.eks_cluster.cluster_certificate_authority_data + server = module.eks_cluster.cluster_endpoint + } + }] + contexts = [{ + name = "terraform" + context = { + cluster = module.eks_cluster.cluster_id + user = "terraform" + } + }] + users = [{ + name = "terraform" + user = { + token = data.aws_eks_cluster_auth.this.token + } + }] + }) +} + +locals { + aws_auth_configmap_data = { + mapRoles = yamlencode(concat( + [{ + rolearn = module.eks_nix_node_group.eks_node_group_role_arn + username = "system:node:{{EC2PrivateDNSName}}" + groups = [ + "system:bootstrappers", + "system:nodes", + ] + } + ], + [{ + rolearn = module.eks_nt_node_group.eks_node_group_role_arn + username = "system:node:{{EC2PrivateDNSName}}" + groups = [ + "eks:kube-proxy-windows", + "system:bootstrappers", + "system:nodes", + ] + } + ], + var.aws_auth_roles + )) + mapUsers = yamlencode(var.aws_auth_users) + mapAccounts = yamlencode(var.aws_auth_accounts) + } +} + +resource "kubernetes_config_map" "aws_auth" { + count = var.include_windows_node && var.create_aws_auth_configmap ? 1 : 0 + + metadata { + name = "aws-auth" + namespace = "kube-system" + } + + data = local.aws_auth_configmap_data + + lifecycle { + # We are ignoring the data here since we will manage it with the resource below + # This is only intended to be used in scenarios where the configmap does not exist + ignore_changes = [data, metadata[0].labels, metadata[0].annotations] + } +} + +resource "null_resource" "apply" { + count = var.include_windows_node ? 1 : 0 + triggers = { + kubeconfig = base64encode(local.kubeconfig) + cmd_patch = <<-EOT + kubectl create configmap aws-auth -n kube-system --kubeconfig <(echo $KUBECONFIG | base64 --decode) + kubectl patch configmap/aws-auth --patch "${kubernetes_config_map.aws_auth_configmap_yaml}" -n kube-system --kubeconfig <(echo $KUBECONFIG | base64 --decode) + EOT + } + provisioner "local-exec" { + interpreter = ["/bin/bash", "-c"] + environment = { + KUBECONFIG = self.triggers.kubeconfig + } + command = self.triggers.cmd_patch + } +} diff --git a/examples/complete/variables.tf b/examples/complete/variables.tf index eb2a6d8..fc64bf5 100644 --- a/examples/complete/variables.tf +++ b/examples/complete/variables.tf @@ -73,11 +73,6 @@ variable "local_exec_interpreter" { description = "shell to use for local_exec" } -variable "disk_size" { - type = number - description = "Disk size in GiB for worker nodes. Defaults to 20. Terraform will only perform drift detection if a configuration value is provided" -} - variable "instance_types" { type = list(string) description = "Set of instance types associated with the EKS Node Group. Defaults to [\"t3.medium\"]. Terraform will only perform drift detection if a configuration value is provided" @@ -198,3 +193,33 @@ variable "after_cluster_joining_userdata" { error_message = "You may not specify more than one `after_cluster_joining_userdata`." } } + +variable "include_windows_node" { + type = bool + description = "Will include a single windows node" + +} + +variable "windows_node_ami_type" { + type = string + description = "windows node ami-type (i.e. WINDOWS_CORE_2019_x86_64)" + default = "WINDOWS_CORE_2019_x86_64" +} + +variable "aws_auth_roles" { + description = "List of role maps to add to the aws-auth configmap" + type = list(any) + default = [] +} + +variable "aws_auth_users" { + description = "List of user maps to add to the aws-auth configmap" + type = list(any) + default = [] +} + +variable "aws_auth_accounts" { + description = "List of account maps to add to the aws-auth configmap" + type = list(any) + default = [] +} diff --git a/iam.tf b/iam.tf index a0e0713..1ff160b 100644 --- a/iam.tf +++ b/iam.tf @@ -53,7 +53,7 @@ resource "aws_iam_role_policy_attachment" "existing_policies_for_eks_workers_rol # https://docs.aws.amazon.com/eks/latest/userguide/cni-iam-role.html#cni-iam-role-create-ipv6-policy # https://docs.aws.amazon.com/eks/latest/userguide/windows-support.html data "aws_iam_policy_document" "ipv6_eks_cni_policy" { - count = local.create_role && (var.node_role_cni_policy_enabled || can(regex("WINDOWS", var.ami_type))) ? 1 : 0 + count = local.create_role && (var.node_role_cni_policy_enabled) ? 1 : 0 statement { effect = "Allow" diff --git a/launch-template.tf b/launch-template.tf index 3670248..bcfc26d 100644 --- a/launch-template.tf +++ b/launch-template.tf @@ -59,7 +59,6 @@ resource "aws_launch_template" "default" { device_name = block_device_mappings.value.device_name ebs { - delete_on_termination = lookup(block_device_mappings.value, "delete_on_termination", null) encrypted = lookup(block_device_mappings.value, "encrypted", null) iops = lookup(block_device_mappings.value, "iops", null) diff --git a/main.tf b/main.tf index 36dd6fd..ed5d18f 100644 --- a/main.tf +++ b/main.tf @@ -44,6 +44,12 @@ locals { } ) node_group_tags = merge(local.node_tags, local.autoscaler_enabled ? local.autoscaler_tags : null) + + windows_taint = [{ + key = "OS" + value = "Windows" + effect = "NO_SCHEDULE" + }] } module "label" { @@ -62,6 +68,7 @@ data "aws_eks_cluster" "this" { # Support keeping 2 node groups in sync by extracting common variable settings locals { + is_windows = can(regex("WINDOWS", var.ami_type)) ng = { cluster_name = var.cluster_name node_role_arn = local.create_role ? join("", aws_iam_role.default.*.arn) : try(var.node_role_arn[0], null) @@ -71,11 +78,12 @@ locals { # because node group supports up to 20 types but launch template does not. # See https://docs.aws.amazon.com/eks/latest/APIReference/API_CreateNodegroup.html#API_CreateNodegroup_RequestSyntax # Keep sorted so that change in order does not trigger replacement via random_pet - instance_types = sort(var.instance_types) - ami_type = local.launch_template_ami == "" ? var.ami_type : null - capacity_type = var.capacity_type - labels = var.kubernetes_labels == null ? {} : var.kubernetes_labels - taints = var.kubernetes_taints + instance_types = sort(var.instance_types) + ami_type = local.launch_template_ami == "" ? var.ami_type : null + capacity_type = var.capacity_type + labels = var.kubernetes_labels == null ? {} : var.kubernetes_labels + + taints = local.is_windows ? concat(local.windows_taint, var.kubernetes_taints) : var.kubernetes_taints release_version = local.launch_template_ami == "" ? try(var.ami_release_version[0], null) : null version = length(compact(concat([local.launch_template_ami], var.ami_release_version))) == 0 ? try(var.kubernetes_version[0], null) : null diff --git a/outputs.tf b/outputs.tf index 8883079..fea2990 100644 --- a/outputs.tf +++ b/outputs.tf @@ -52,3 +52,8 @@ output "eks_node_group_tags_all" { description = "A map of tags assigned to the resource, including those inherited from the provider default_tags configuration block." value = local.enabled ? (var.create_before_destroy ? aws_eks_node_group.cbd[0].tags_all : aws_eks_node_group.default[0].tags_all) : {} } + +output "eks_node_group_windows_note" { + description = "Instructions on changes a user needs to follow or script for a windows node group in the event of a custom ami" + value = local.is_windows && local.need_bootstrap && local.ami ? "When specifying a custom AMI ID for Windows managed node groups, add eks:kube-proxy-windows to your AWS IAM Authenticator configuration map. For more information, see Limits and conditions when specifying an AMI ID. https://docs.aws.amazon.com/eks/latest/userguide/windows-support.html" : "" +} diff --git a/userdata.tf b/userdata.tf index 36887ee..0a3a076 100644 --- a/userdata.tf +++ b/userdata.tf @@ -41,7 +41,7 @@ locals { (length(var.before_cluster_joining_userdata) > 0) || local.need_bootstrap) : false userdata = local.need_userdata ? ( - base64encode(templatefile(can(regex("WINDOWS", var.ami_type)) ? "${path.module}/userdata_nt.tpl" : "${path.module}/userdata.tpl", merge(local.userdata_vars, local.cluster_data)))) : ( + base64encode(templatefile(local.is_windows ? "${path.module}/userdata_nt.tpl" : "${path.module}/userdata.tpl", merge(local.userdata_vars, local.cluster_data)))) : ( try(var.userdata_override_base64[0], null) ) } diff --git a/userdata_nt.tpl b/userdata_nt.tpl index 7ef7c8c..023309d 100644 --- a/userdata_nt.tpl +++ b/userdata_nt.tpl @@ -1,10 +1,20 @@ [string]$EKSBootstrapScriptFile = "$env:ProgramFiles\Amazon\EKS\Start-EKSBootstrap.ps1" +try{ ${before_cluster_joining_userdata} +}catch{ + Write-Host "An error occurred in pre-script" -ForegroundColor Red + Write-Host $_.ScriptStackTrace +} +Write-Host -Foreground Red -Background Black ($formatstring -f $fields) -& $EKSBootstrapScriptFile -EKSClusterName "${cluster_name}" -APIServerEndpoint "${cluster_endpoint}" -Base64ClusterCA "${certificate_authority_data}" -KubeletExtraArgs "${bootstrap_extra_args}" 3>&1 4>&1 5>&1 6>&1 -$LastError = if ($?) { 0 } else { $Error[0].Exception.HResult } +& $EKSBootstrapScriptFile -EKSClusterName "${cluster_name}" -APIServerEndpoint "${cluster_endpoint}" -Base64ClusterCA "${certificate_authority_data}" --register-with-taints="OS=Windows:NoSchedule" -DNSClusterIP "${dns_address}" -KubeletExtraArgs "${bootstrap_extra_args}" +try{ ${after_cluster_joining_userdata} - \ No newline at end of file +}catch{ + Write-Host "An error occurred in post-script" -ForegroundColor Red + Write-Host $_.ScriptStackTrace +} + diff --git a/variables.tf b/variables.tf index cf4d6fb..af277d0 100644 --- a/variables.tf +++ b/variables.tf @@ -111,7 +111,7 @@ variable "ami_type" { type = string description = <<-EOT Type of Amazon Machine Image (AMI) associated with the EKS Node Group. - Defaults to `AL2_x86_64`. Valid values: `AL2_x86_64, AL2_x86_64_GPU, AL2_ARM_64,CUSTOM, BOTTLEROCKET_ARM_64, BOTTLEROCKET_x86_64, BOTTLEROCKET_ARM_64_NVIDIA, BOTTLEROCKET_x86_64_NVIDIA, WINDOWS_CORE_2019_x86_64, WINDOWS_FULL_2019_x86_64, WINDOWS_CORE_2022_x86_64, WINDOWS_FULL_2022_x86_64`. + Defaults to `AL2_x86_64`. Valid values: `AL2_x86_64, AL2_x86_64_GPU, AL2_ARM_64, CUSTOM, BOTTLEROCKET_ARM_64, BOTTLEROCKET_x86_64, BOTTLEROCKET_ARM_64_NVIDIA, BOTTLEROCKET_x86_64_NVIDIA, WINDOWS_CORE_2019_x86_64, WINDOWS_FULL_2019_x86_64, WINDOWS_CORE_2022_x86_64, WINDOWS_FULL_2022_x86_64`. EOT default = "AL2_x86_64" validation { @@ -199,6 +199,16 @@ variable "kubernetes_taints" { default = [] } +variable "windnows_coredns_service_address" { + type = string + description = <<-EOT + The value passed to the Powershell setup script on Windows Node start + This is required to be set to the service address of core-dns or pod-networking will fail + the default behaviour (without a value) will select the DNS of the first network interface + EOT + default = "172.20.0.10" +} + variable "kubelet_additional_options" { type = list(string) description = <<-EOT diff --git a/versions.tf b/versions.tf index 6995caf..2bf5734 100644 --- a/versions.tf +++ b/versions.tf @@ -1,12 +1,10 @@ terraform { - required_version = ">= 0.14.11" + required_version = ">= 0.15.00" required_providers { aws = { - source = "hashicorp/aws" - # retrieve launch template by ID starts at 3.21.0 - # update_config starts at 3.56 - version = ">= 3.56" + source = "hashicorp/aws" + version = ">= 4.0" } random = { source = "hashicorp/random" From 1ce0310e5b6ab6b394b84ab2aa6b450667dbb7ad Mon Sep 17 00:00:00 2001 From: Chris McKee Date: Wed, 1 Mar 2023 22:37:39 +0000 Subject: [PATCH 04/14] Remove extraneous statement, fix output --- README.md | 1 - docs/terraform.md | 1 - examples/complete/fixtures.eu-west-2.tfvars | 46 ------ examples/complete/main.tf | 161 +------------------- examples/complete/outputs.tf | 17 +-- examples/complete/variables.tf | 36 +---- outputs.tf | 2 +- userdata_nt.tpl | 2 +- variables.tf | 10 -- 9 files changed, 16 insertions(+), 260 deletions(-) delete mode 100644 examples/complete/fixtures.eu-west-2.tfvars diff --git a/README.md b/README.md index 4920adb..b6ca5ea 100644 --- a/README.md +++ b/README.md @@ -382,7 +382,6 @@ https://docs.aws.amazon.com/eks/latest/userguide/windows-support.html | [tenant](#input\_tenant) | ID element \_(Rarely used, not included by default)\_. A customer identifier, indicating who this instance of a resource is for | `string` | `null` | no | | [update\_config](#input\_update\_config) | Configuration for the `eks_node_group` [`update_config` Configuration Block](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/eks_node_group#update_config-configuration-block).
Specify exactly one of `max_unavailable` (node count) or `max_unavailable_percentage` (percentage of nodes). | `list(map(number))` | `[]` | no | | [userdata\_override\_base64](#input\_userdata\_override\_base64) | Many features of this module rely on the `bootstrap.sh` provided with Amazon Linux, and this module
may generate "user data" that expects to find that script. If you want to use an AMI that is not
compatible with the Amazon Linux `bootstrap.sh` initialization, then use `userdata_override_base64` to provide
your own (Base64 encoded) user data. Use "" to prevent any user data from being set.

Setting `userdata_override_base64` disables `kubernetes_taints`, `kubelet_additional_options`,
`before_cluster_joining_userdata`, `after_cluster_joining_userdata`, and `bootstrap_additional_options`. | `list(string)` | `[]` | no | -| [windnows\_coredns\_service\_address](#input\_windnows\_coredns\_service\_address) | The value passed to the Powershell setup script on Windows Node start
This is required to be set to the service address of core-dns or pod-networking will fail
the default behaviour (without a value) will select the DNS of the first network interface | `string` | `"172.20.0.10"` | no | ## Outputs diff --git a/docs/terraform.md b/docs/terraform.md index 787d260..12ea817 100644 --- a/docs/terraform.md +++ b/docs/terraform.md @@ -106,7 +106,6 @@ | [tenant](#input\_tenant) | ID element \_(Rarely used, not included by default)\_. A customer identifier, indicating who this instance of a resource is for | `string` | `null` | no | | [update\_config](#input\_update\_config) | Configuration for the `eks_node_group` [`update_config` Configuration Block](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/eks_node_group#update_config-configuration-block).
Specify exactly one of `max_unavailable` (node count) or `max_unavailable_percentage` (percentage of nodes). | `list(map(number))` | `[]` | no | | [userdata\_override\_base64](#input\_userdata\_override\_base64) | Many features of this module rely on the `bootstrap.sh` provided with Amazon Linux, and this module
may generate "user data" that expects to find that script. If you want to use an AMI that is not
compatible with the Amazon Linux `bootstrap.sh` initialization, then use `userdata_override_base64` to provide
your own (Base64 encoded) user data. Use "" to prevent any user data from being set.

Setting `userdata_override_base64` disables `kubernetes_taints`, `kubelet_additional_options`,
`before_cluster_joining_userdata`, `after_cluster_joining_userdata`, and `bootstrap_additional_options`. | `list(string)` | `[]` | no | -| [windnows\_coredns\_service\_address](#input\_windnows\_coredns\_service\_address) | The value passed to the Powershell setup script on Windows Node start
This is required to be set to the service address of core-dns or pod-networking will fail
the default behaviour (without a value) will select the DNS of the first network interface | `string` | `"172.20.0.10"` | no | ## Outputs diff --git a/examples/complete/fixtures.eu-west-2.tfvars b/examples/complete/fixtures.eu-west-2.tfvars deleted file mode 100644 index 0221106..0000000 --- a/examples/complete/fixtures.eu-west-2.tfvars +++ /dev/null @@ -1,46 +0,0 @@ -region = "eu-west-2" - -availability_zones = ["eu-west-2a", "eu-west-2b"] - -vpc_cidr_block = "172.16.0.0/16" - -namespace = "eg" - -stage = "test" - -name = "eks-node-group" - -kubernetes_version = "1.25" - -oidc_provider_enabled = true - -enabled_cluster_log_types = ["audit"] - -cluster_log_retention_period = 7 - -instance_types = ["t3.small"] - -desired_size = 1 - -max_size = 1 - -min_size = 1 - -disk_size = 90 - -kubernetes_labels = { - terratest = "true" -} - -before_cluster_joining_userdata = <<-EOT - printf "\n\n###\nExample output from before_cluster_joining_userdata\n###\n\n" - EOT - -update_config = [{ max_unavailable = 1 }] - -kubernetes_taints = [ - { - key = "test" - value = null - effect = "PREFER_NO_SCHEDULE" -}] diff --git a/examples/complete/main.tf b/examples/complete/main.tf index 19881f2..21af5e1 100644 --- a/examples/complete/main.tf +++ b/examples/complete/main.tf @@ -3,7 +3,7 @@ provider "aws" { } module "label" { - source = "cloudposse/label/null" + source = "cloudposse/label/null" #version = "0.25.0" # This is the preferred way to add attributes. It will put "cluster" last @@ -114,11 +114,9 @@ module "https_sg" { context = module.label.context } - module "eks_cluster" { - # source = "cloudposse/eks-cluster/aws" - version = "2.2.0" - source = "github.com/ChrisMcKee/terraform-aws-eks-cluster" + source = "cloudposse/eks-cluster/aws" + #version = "2.4.0" region = var.region vpc_id = module.vpc.vpc_id subnet_ids = module.subnets.public_subnet_ids @@ -132,8 +130,6 @@ module "eks_cluster" { kube_data_auth_enabled = false kube_exec_auth_enabled = true - windows_support = true - context = module.this.context } @@ -177,154 +173,3 @@ module "eks_nix_node_group" { delete = "20m" }] } - -module "windowslabel" { - source = "cloudposse/label/null" - - enabled = var.include_windows_node - - context = module.this.context - attributes = ["nt", "workers"] - tags = { - nodeclass = "windows2019" - } -} - -module "eks_nt_node_group" { - source = "../../" - - enabled = var.include_windows_node - - context = module.windowslabel.context - - subnet_ids = module.this.enabled ? module.subnets.public_subnet_ids : ["filler_string_for_enabled_is_false"] - cluster_name = module.eks_cluster.eks_cluster_id - instance_types = var.instance_types - desired_size = 1 - min_size = 1 - max_size = 1 - kubernetes_version = [var.kubernetes_version] - kubernetes_labels = merge(var.kubernetes_labels, { attributes = coalesce(join(module.this.delimiter, module.this.attributes), "none") }) - - ec2_ssh_key_name = var.ec2_ssh_key_name - ssh_access_security_group_ids = [module.ssh_source_access.id] - associated_security_group_ids = [module.ssh_source_access.id, module.https_sg.id] - node_role_policy_arns = [local.extra_policy_arn] - update_config = var.update_config - - ami_type = var.windows_node_ami_type - - # after_cluster_joining_userdata = [ - # "Write-Host Hello" - # ] - - # Ensure ordering of resource creation to eliminate the race conditions when applying the Kubernetes Auth ConfigMap. - # Do not create Node Group before the EKS cluster is created and the `aws-auth` Kubernetes ConfigMap is applied. - depends_on = [module.eks_cluster, module.eks_cluster.kubernetes_config_map_id] - - create_before_destroy = true - - node_group_terraform_timeouts = [{ - create = "40m" - update = null - delete = "20m" - }] - - resources_to_tag = ["instance", "volume", "spot-instances-request", "network-interface"] -} - -data "aws_eks_cluster_auth" "this" { - name = module.eks_cluster.cluster_id -} - -locals { - kubeconfig = yamlencode({ - apiVersion = "v1" - kind = "Config" - current-context = "terraform" - clusters = [{ - name = module.eks_cluster.cluster_id - cluster = { - certificate-authority-data = module.eks_cluster.cluster_certificate_authority_data - server = module.eks_cluster.cluster_endpoint - } - }] - contexts = [{ - name = "terraform" - context = { - cluster = module.eks_cluster.cluster_id - user = "terraform" - } - }] - users = [{ - name = "terraform" - user = { - token = data.aws_eks_cluster_auth.this.token - } - }] - }) -} - -locals { - aws_auth_configmap_data = { - mapRoles = yamlencode(concat( - [{ - rolearn = module.eks_nix_node_group.eks_node_group_role_arn - username = "system:node:{{EC2PrivateDNSName}}" - groups = [ - "system:bootstrappers", - "system:nodes", - ] - } - ], - [{ - rolearn = module.eks_nt_node_group.eks_node_group_role_arn - username = "system:node:{{EC2PrivateDNSName}}" - groups = [ - "eks:kube-proxy-windows", - "system:bootstrappers", - "system:nodes", - ] - } - ], - var.aws_auth_roles - )) - mapUsers = yamlencode(var.aws_auth_users) - mapAccounts = yamlencode(var.aws_auth_accounts) - } -} - -resource "kubernetes_config_map" "aws_auth" { - count = var.include_windows_node && var.create_aws_auth_configmap ? 1 : 0 - - metadata { - name = "aws-auth" - namespace = "kube-system" - } - - data = local.aws_auth_configmap_data - - lifecycle { - # We are ignoring the data here since we will manage it with the resource below - # This is only intended to be used in scenarios where the configmap does not exist - ignore_changes = [data, metadata[0].labels, metadata[0].annotations] - } -} - -resource "null_resource" "apply" { - count = var.include_windows_node ? 1 : 0 - triggers = { - kubeconfig = base64encode(local.kubeconfig) - cmd_patch = <<-EOT - kubectl create configmap aws-auth -n kube-system --kubeconfig <(echo $KUBECONFIG | base64 --decode) - kubectl patch configmap/aws-auth --patch "${kubernetes_config_map.aws_auth_configmap_yaml}" -n kube-system --kubeconfig <(echo $KUBECONFIG | base64 --decode) - EOT - } - provisioner "local-exec" { - interpreter = ["/bin/bash", "-c"] - environment = { - KUBECONFIG = self.triggers.kubeconfig - } - command = self.triggers.cmd_patch - } -} diff --git a/examples/complete/outputs.tf b/examples/complete/outputs.tf index cbff747..160c8f7 100644 --- a/examples/complete/outputs.tf +++ b/examples/complete/outputs.tf @@ -60,40 +60,39 @@ output "eks_cluster_managed_security_group_id" { output "eks_node_group_role_arn" { description = "ARN of the worker nodes IAM role" - value = module.eks_node_group.eks_node_group_role_arn + value = module.eks_nix_node_group.eks_node_group_role_arn } output "eks_node_group_role_name" { description = "Name of the worker nodes IAM role" - value = module.eks_node_group.eks_node_group_role_name + value = module.eks_nix_node_group.eks_node_group_role_name } output "eks_node_group_id" { description = "EKS Cluster name and EKS Node Group name separated by a colon" - value = module.eks_node_group.eks_node_group_id + value = module.eks_nix_node_group.eks_node_group_id } output "eks_node_group_arn" { description = "Amazon Resource Name (ARN) of the EKS Node Group" - value = module.eks_node_group.eks_node_group_arn + value = module.eks_nix_node_group.eks_node_group_arn } output "eks_node_group_resources" { description = "List of objects containing information about underlying resources of the EKS Node Group" - value = module.eks_node_group.eks_node_group_resources + value = module.eks_nix_node_group.eks_node_group_resources } output "eks_node_group_status" { description = "Status of the EKS Node Group" - value = module.eks_node_group.eks_node_group_status + value = module.eks_nix_node_group.eks_node_group_status } output "eks_node_group_cbd_pet_name" { description = "The pet name of this node group, if this module generated one" - value = module.eks_node_group.eks_node_group_cbd_pet_name + value = module.eks_nix_node_group.eks_node_group_cbd_pet_name } - output "eks_node_group_launch_template_id" { - value = module.eks_node_group.eks_node_group_launch_template_id + value = module.eks_nix_node_group.eks_node_group_launch_template_id } diff --git a/examples/complete/variables.tf b/examples/complete/variables.tf index fc64bf5..208f203 100644 --- a/examples/complete/variables.tf +++ b/examples/complete/variables.tf @@ -159,14 +159,14 @@ variable "ami_type" { type = string description = <<-EOT Type of Amazon Machine Image (AMI) associated with the EKS Node Group. - Defaults to `AL2_x86_64`. Valid values: `AL2_x86_64`, `AL2_x86_64_GPU`, `AL2_ARM_64`, `BOTTLEROCKET_x86_64`, and `BOTTLEROCKET_ARM_64`. + Defaults to `AL2_x86_64`. Valid values: `AL2_x86_64, AL2_x86_64_GPU, AL2_ARM_64, CUSTOM, BOTTLEROCKET_ARM_64, BOTTLEROCKET_x86_64, BOTTLEROCKET_ARM_64_NVIDIA, BOTTLEROCKET_x86_64_NVIDIA, WINDOWS_CORE_2019_x86_64, WINDOWS_FULL_2019_x86_64, WINDOWS_CORE_2022_x86_64, WINDOWS_FULL_2022_x86_64`. EOT default = "AL2_x86_64" validation { condition = ( - contains(["AL2_x86_64", "AL2_x86_64_GPU", "AL2_ARM_64", "BOTTLEROCKET_x86_64", "BOTTLEROCKET_ARM_64"], var.ami_type) + contains(["AL2_x86_64", "AL2_x86_64_GPU", "AL2_ARM_64", "CUSTOM", "BOTTLEROCKET_ARM_64", "BOTTLEROCKET_x86_64", "BOTTLEROCKET_ARM_64_NVIDIA", "BOTTLEROCKET_x86_64_NVIDIA", "WINDOWS_CORE_2019_x86_64", "WINDOWS_FULL_2019_x86_64", "WINDOWS_CORE_2022_x86_64", "WINDOWS_FULL_2022_x86_64"], var.ami_type) ) - error_message = "Var ami_type must be one of \"AL2_x86_64\", \"AL2_x86_64_GPU\", \"AL2_ARM_64\", \"BOTTLEROCKET_x86_64\", and \"BOTTLEROCKET_ARM_64\"." + error_message = "Var ami_type must be one of \"AL2_x86_64\",\"AL2_x86_64_GPU\",\"AL2_ARM_64\",\"BOTTLEROCKET_ARM_64\",\"BOTTLEROCKET_x86_64\",\"BOTTLEROCKET_ARM_64_NVIDIA\",\"BOTTLEROCKET_x86_64_NVIDIA\",\"WINDOWS_CORE_2019_x86_64\",\"WINDOWS_FULL_2019_x86_64\",\"WINDOWS_CORE_2022_x86_64\",\"WINDOWS_FULL_2022_x86_64\", or \"CUSTOM\"." } } @@ -193,33 +193,3 @@ variable "after_cluster_joining_userdata" { error_message = "You may not specify more than one `after_cluster_joining_userdata`." } } - -variable "include_windows_node" { - type = bool - description = "Will include a single windows node" - -} - -variable "windows_node_ami_type" { - type = string - description = "windows node ami-type (i.e. WINDOWS_CORE_2019_x86_64)" - default = "WINDOWS_CORE_2019_x86_64" -} - -variable "aws_auth_roles" { - description = "List of role maps to add to the aws-auth configmap" - type = list(any) - default = [] -} - -variable "aws_auth_users" { - description = "List of user maps to add to the aws-auth configmap" - type = list(any) - default = [] -} - -variable "aws_auth_accounts" { - description = "List of account maps to add to the aws-auth configmap" - type = list(any) - default = [] -} diff --git a/outputs.tf b/outputs.tf index fea2990..d9701bf 100644 --- a/outputs.tf +++ b/outputs.tf @@ -55,5 +55,5 @@ output "eks_node_group_tags_all" { output "eks_node_group_windows_note" { description = "Instructions on changes a user needs to follow or script for a windows node group in the event of a custom ami" - value = local.is_windows && local.need_bootstrap && local.ami ? "When specifying a custom AMI ID for Windows managed node groups, add eks:kube-proxy-windows to your AWS IAM Authenticator configuration map. For more information, see Limits and conditions when specifying an AMI ID. https://docs.aws.amazon.com/eks/latest/userguide/windows-support.html" : "" + value = local.enabled && local.is_windows && local.need_bootstrap ? "When specifying a custom AMI ID for Windows managed node groups, add eks:kube-proxy-windows to your AWS IAM Authenticator configuration map. For more information, see Limits and conditions when specifying an AMI ID. https://docs.aws.amazon.com/eks/latest/userguide/windows-support.html" : "" } diff --git a/userdata_nt.tpl b/userdata_nt.tpl index 023309d..6bd8727 100644 --- a/userdata_nt.tpl +++ b/userdata_nt.tpl @@ -9,7 +9,7 @@ ${before_cluster_joining_userdata} } Write-Host -Foreground Red -Background Black ($formatstring -f $fields) -& $EKSBootstrapScriptFile -EKSClusterName "${cluster_name}" -APIServerEndpoint "${cluster_endpoint}" -Base64ClusterCA "${certificate_authority_data}" --register-with-taints="OS=Windows:NoSchedule" -DNSClusterIP "${dns_address}" -KubeletExtraArgs "${bootstrap_extra_args}" +& $EKSBootstrapScriptFile -EKSClusterName "${cluster_name}" -APIServerEndpoint "${cluster_endpoint}" -Base64ClusterCA "${certificate_authority_data}" -ContainerRuntime "containerd" -KubeletExtraArgs "${bootstrap_extra_args}" 3>&1 4>&1 5>&1 6>&1 try{ ${after_cluster_joining_userdata} diff --git a/variables.tf b/variables.tf index af277d0..cbff89e 100644 --- a/variables.tf +++ b/variables.tf @@ -199,16 +199,6 @@ variable "kubernetes_taints" { default = [] } -variable "windnows_coredns_service_address" { - type = string - description = <<-EOT - The value passed to the Powershell setup script on Windows Node start - This is required to be set to the service address of core-dns or pod-networking will fail - the default behaviour (without a value) will select the DNS of the first network interface - EOT - default = "172.20.0.10" -} - variable "kubelet_additional_options" { type = list(string) description = <<-EOT From 10f7401b4138458d7131cc5b00eee3b075c5b036 Mon Sep 17 00:00:00 2001 From: Chris McKee Date: Wed, 8 Mar 2023 12:29:03 +0000 Subject: [PATCH 05/14] Add drive mapping to windows userdata; put bootstrap extra args in correct place and remove extraneous params (based on eksctl code base and aws generated script) --- userdata.tpl | 1 + userdata_nt.tpl | 23 ++++++++++++++++++++--- 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/userdata.tpl b/userdata.tpl index 181eabb..f11a015 100644 --- a/userdata.tpl +++ b/userdata.tpl @@ -4,6 +4,7 @@ Content-Type: multipart/mixed; boundary="/:/+++" --/:/+++ Content-Type: text/x-shellscript; charset="us-ascii" #!/bin/bash +set -ex # In multipart MIME format to support EKS appending to it diff --git a/userdata_nt.tpl b/userdata_nt.tpl index 6bd8727..5507816 100644 --- a/userdata_nt.tpl +++ b/userdata_nt.tpl @@ -1,6 +1,4 @@ -[string]$EKSBootstrapScriptFile = "$env:ProgramFiles\Amazon\EKS\Start-EKSBootstrap.ps1" - try{ ${before_cluster_joining_userdata} }catch{ @@ -9,7 +7,26 @@ ${before_cluster_joining_userdata} } Write-Host -Foreground Red -Background Black ($formatstring -f $fields) -& $EKSBootstrapScriptFile -EKSClusterName "${cluster_name}" -APIServerEndpoint "${cluster_endpoint}" -Base64ClusterCA "${certificate_authority_data}" -ContainerRuntime "containerd" -KubeletExtraArgs "${bootstrap_extra_args}" 3>&1 4>&1 5>&1 6>&1 +# Deal with extra new disks +$disks_to_adjust = Get-Disk | Select-Object Number,Size,PartitionStyle | Where-Object PartitionStyle -Match RAW +if ($disks_to_adjust -ne $null) { + [int64] $partition_mbr_max_size = 2199023255552 + $partition_style = "MBR" + foreach ($disk in $disks_to_adjust) { + if ($disk.Size -gt $partition_mbr_max_size) { + $partition_style = "GPT" + } + + Initialize-Disk -Number $disk.Number -PartitionStyle $partition_style + New-Partition -DiskNumber $disk.Number -UseMaximumSize -AssignDriveLetter | Format-Volume -FileSystem NTFS + } +} + +[string]$EKSBinDir = "$env:ProgramFiles\Amazon\EKS" +[string]$EKSBootstrapScriptName = 'Start-EKSBootstrap.ps1' +[string]$EKSBootstrapScriptFile = "$EKSBinDir\$EKSBootstrapScriptName" + +& $EKSBootstrapScriptFile -EKSClusterName "${cluster_name}" -APIServerEndpoint "${cluster_endpoint}" -Base64ClusterCA "${certificate_authority_data}" ${bootstrap_extra_args} -KubeletExtraArgs "${kubelet_extra_args}" 3>&1 4>&1 5>&1 6>&1 try{ ${after_cluster_joining_userdata} From 93f2ada3d638fde8cb0a4795937900e06ac90ab3 Mon Sep 17 00:00:00 2001 From: Chris McKee Date: Thu, 9 Mar 2023 10:30:48 +0000 Subject: [PATCH 06/14] Convert ami.tf maps to map format ("":"" to key = "value") --- ami.tf | 58 +++++++++++++++++++++++++++++----------------------------- 1 file changed, 29 insertions(+), 29 deletions(-) diff --git a/ami.tf b/ami.tf index f338bc4..1560b81 100644 --- a/ami.tf +++ b/ami.tf @@ -2,17 +2,17 @@ locals { # "amazon-eks-gpu-node-", arch_label_map = { - "AL2_x86_64" : "", - "AL2_x86_64_GPU" : "-gpu", - "AL2_ARM_64" : "-arm64", - "BOTTLEROCKET_x86_64" : "x86_64", - "BOTTLEROCKET_ARM_64" : "aarch64" - "BOTTLEROCKET_ARM_64_NVIDIA" : "-gpu" - "BOTTLEROCKET_x86_64_NVIDIA" : "-gpu" - "WINDOWS_CORE_2019_x86_64" : "" - "WINDOWS_FULL_2019_x86_64" : "" - "WINDOWS_CORE_2022_x86_64" : "" - "WINDOWS_FULL_2022_x86_64" : "" + AL2_x86_64 = "", + AL2_x86_64_GPU = "-gpu", + AL2_ARM_64 = "-arm64", + BOTTLEROCKET_x86_64 = "x86_64", + BOTTLEROCKET_ARM_64 = "aarch64" + BOTTLEROCKET_ARM_64_NVIDIA = "-gpu" + BOTTLEROCKET_x86_64_NVIDIA = "-gpu" + WINDOWS_CORE_2019_x86_64 = "" + WINDOWS_FULL_2019_x86_64 = "" + WINDOWS_CORE_2022_x86_64 = "" + WINDOWS_FULL_2022_x86_64 = "" } ami_kind = split("_", var.ami_type)[0] != "WINDOWS" ? split("_", var.ami_type)[0] : format("WINDOWS_%s_%s", split("_", var.ami_type)[1], split("_", var.ami_type)[2]) @@ -20,16 +20,16 @@ locals { ami_format = { # amazon-eks{arch_label}-node-{ami_kubernetes_version}-v{ami_version} # e.g. amazon-eks-arm64-node-1.21-v20211013 - "AL2" : "amazon-eks%s-node-%s" + AL2 = "amazon-eks%s-node-%s" # bottlerocket-aws-k8s-{ami_kubernetes_version}-{arch_label}-v{ami_version} # e.g. bottlerocket-aws-k8s-1.21-x86_64-v1.2.0-ccf1b754 - "BOTTLEROCKET" : "bottlerocket-aws-k8s-%s-%s-%s" + BOTTLEROCKET = "bottlerocket-aws-k8s-%s-%s-%s" # Windows_Server-2019-English-Core-EKS_Optimized-{ami_kubernetes_version}-{ami_version} # e.g. Windows_Server-2019-English-Core-EKS_Optimized-1.23-2022.11.08 - "WINDOWS_CORE_2019" : "Windows_Server-2019-English-Core-EKS_Optimized-%s-%s" - "WINDOWS_FULL_2019" : "Windows_Server-2019-English-Full-EKS_Optimized-%s-%s" - "WINDOWS_CORE_2022" : "Windows_Server-2022-English-Core-EKS_Optimized-%s-%s" - "WINDOWS_FULL_2022" : "Windows_Server-2022-English-Full-EKS_Optimized-%s-%s" + WINDOWS_CORE_2019 = "Windows_Server-2019-English-Core-EKS_Optimized-%s-%s" + WINDOWS_FULL_2019 = "Windows_Server-2019-English-Full-EKS_Optimized-%s-%s" + WINDOWS_CORE_2022 = "Windows_Server-2022-English-Core-EKS_Optimized-%s-%s" + WINDOWS_FULL_2022 = "Windows_Server-2022-English-Full-EKS_Optimized-%s-%s" } # Kubernetes version priority (first one to be set wins) @@ -49,24 +49,24 @@ locals { # if ami_release_version = "1.21-20211013" # insert the letter v prior to the ami_version so it becomes 1.21-v20211013 # if not, use the kubernetes version - "AL2" : (length(var.ami_release_version) == 1 ? replace(var.ami_release_version[0], "/^(\\d+\\.\\d+)\\.\\d+-(\\d+)$/", "$1-v$2") : "${local.ami_kubernetes_version}-*"), + AL2 = (length(var.ami_release_version) == 1 ? replace(var.ami_release_version[0], "/^(\\d+\\.\\d+)\\.\\d+-(\\d+)$/", "$1-v$2") : "${local.ami_kubernetes_version}-*"), # if ami_release_version = "1.2.0-ccf1b754" # prefix the ami release version with the letter v # if not, use an asterisk - "BOTTLEROCKET" : (length(var.ami_release_version) == 1 ? format("v%s", var.ami_release_version[0]) : "*"), - "WINDOWS_CORE_2019" : (length(var.ami_release_version) == 1 ? format("%s", var.ami_release_version[0]) : "*"), - "WINDOWS_FULL_2019" : (length(var.ami_release_version) == 1 ? format("%s", var.ami_release_version[0]) : "*"), - "WINDOWS_CORE_2022" : (length(var.ami_release_version) == 1 ? format("%s", var.ami_release_version[0]) : "*"), - "WINDOWS_FULL_2022" : (length(var.ami_release_version) == 1 ? format("%s", var.ami_release_version[0]) : "*"), + BOTTLEROCKET = (length(var.ami_release_version) == 1 ? format("v%s", var.ami_release_version[0]) : "*"), + WINDOWS_CORE_2019 = (length(var.ami_release_version) == 1 ? format("%s", var.ami_release_version[0]) : "*"), + WINDOWS_FULL_2019 = (length(var.ami_release_version) == 1 ? format("%s", var.ami_release_version[0]) : "*"), + WINDOWS_CORE_2022 = (length(var.ami_release_version) == 1 ? format("%s", var.ami_release_version[0]) : "*"), + WINDOWS_FULL_2022 = (length(var.ami_release_version) == 1 ? format("%s", var.ami_release_version[0]) : "*"), } : {} ami_regex = local.need_ami_id ? { - "AL2" : format(local.ami_format["AL2"], local.arch_label_map[var.ami_type], local.ami_version_regex[local.ami_kind]), - "BOTTLEROCKET" : format(local.ami_format["BOTTLEROCKET"], local.ami_kubernetes_version, local.arch_label_map[var.ami_type], local.ami_version_regex[local.ami_kind]), - "WINDOWS_CORE_2019" : format(local.ami_format["WINDOWS_CORE_2019"], local.ami_kubernetes_version, local.ami_version_regex[local.ami_kind]), - "WINDOWS_FULL_2019" : format(local.ami_format["WINDOWS_FULL_2019"], local.ami_kubernetes_version, local.ami_version_regex[local.ami_kind]), - "WINDOWS_CORE_2022" : format(local.ami_format["WINDOWS_CORE_2022"], local.ami_kubernetes_version, local.ami_version_regex[local.ami_kind]), - "WINDOWS_FULL_2022" : format(local.ami_format["WINDOWS_FULL_2022"], local.ami_kubernetes_version, local.ami_version_regex[local.ami_kind]), + AL2 = format(local.ami_format["AL2"], local.arch_label_map[var.ami_type], local.ami_version_regex[local.ami_kind]), + BOTTLEROCKET = format(local.ami_format["BOTTLEROCKET"], local.ami_kubernetes_version, local.arch_label_map[var.ami_type], local.ami_version_regex[local.ami_kind]), + WINDOWS_CORE_2019 = format(local.ami_format["WINDOWS_CORE_2019"], local.ami_kubernetes_version, local.ami_version_regex[local.ami_kind]), + WINDOWS_FULL_2019 = format(local.ami_format["WINDOWS_FULL_2019"], local.ami_kubernetes_version, local.ami_version_regex[local.ami_kind]), + WINDOWS_CORE_2022 = format(local.ami_format["WINDOWS_CORE_2022"], local.ami_kubernetes_version, local.ami_version_regex[local.ami_kind]), + WINDOWS_FULL_2022 = format(local.ami_format["WINDOWS_FULL_2022"], local.ami_kubernetes_version, local.ami_version_regex[local.ami_kind]), } : {} } From 50619871dfd50e8b4e15de3f3dffbcb26e99f402 Mon Sep 17 00:00:00 2001 From: Chris McKee Date: Thu, 9 Mar 2023 10:31:23 +0000 Subject: [PATCH 07/14] Revert min provider; remove enclosure on ipv6 policy --- iam.tf | 2 +- versions.tf | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/iam.tf b/iam.tf index 1ff160b..cd3241f 100644 --- a/iam.tf +++ b/iam.tf @@ -53,7 +53,7 @@ resource "aws_iam_role_policy_attachment" "existing_policies_for_eks_workers_rol # https://docs.aws.amazon.com/eks/latest/userguide/cni-iam-role.html#cni-iam-role-create-ipv6-policy # https://docs.aws.amazon.com/eks/latest/userguide/windows-support.html data "aws_iam_policy_document" "ipv6_eks_cni_policy" { - count = local.create_role && (var.node_role_cni_policy_enabled) ? 1 : 0 + count = local.create_role && var.node_role_cni_policy_enabled ? 1 : 0 statement { effect = "Allow" diff --git a/versions.tf b/versions.tf index 2bf5734..dfca973 100644 --- a/versions.tf +++ b/versions.tf @@ -1,10 +1,10 @@ terraform { - required_version = ">= 0.15.00" + required_version = ">= 0.14.11" required_providers { aws = { source = "hashicorp/aws" - version = ">= 4.0" + version = ">= 4.48" } random = { source = "hashicorp/random" From 0633a9c404d68bef2f8121adfe060981cd57a82f Mon Sep 17 00:00:00 2001 From: Chris McKee Date: Thu, 9 Mar 2023 10:32:15 +0000 Subject: [PATCH 08/14] Uncomment userdata/label versions in example --- examples/complete/main.tf | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/examples/complete/main.tf b/examples/complete/main.tf index 21af5e1..6092ebf 100644 --- a/examples/complete/main.tf +++ b/examples/complete/main.tf @@ -3,8 +3,8 @@ provider "aws" { } module "label" { - source = "cloudposse/label/null" - #version = "0.25.0" + source = "cloudposse/label/null" + version = "0.25.0" # This is the preferred way to add attributes. It will put "cluster" last # after any attributes set in `var.attributes` or `context.attributes`. @@ -115,8 +115,8 @@ module "https_sg" { } module "eks_cluster" { - source = "cloudposse/eks-cluster/aws" - #version = "2.4.0" + source = "cloudposse/eks-cluster/aws" + version = "2.4.0" region = var.region vpc_id = module.vpc.vpc_id subnet_ids = module.subnets.public_subnet_ids @@ -152,12 +152,12 @@ module "eks_nix_node_group" { node_role_policy_arns = [local.extra_policy_arn] update_config = var.update_config - //after_cluster_joining_userdata = var.after_cluster_joining_userdata + after_cluster_joining_userdata = var.after_cluster_joining_userdata ami_type = var.ami_type ami_release_version = var.ami_release_version - //before_cluster_joining_userdata = [var.before_cluster_joining_userdata] + before_cluster_joining_userdata = [var.before_cluster_joining_userdata] context = module.this.context From 23ae1020a85968500ac46667e4dd58f00938ef71 Mon Sep 17 00:00:00 2001 From: Chris McKee Date: Thu, 9 Mar 2023 10:32:29 +0000 Subject: [PATCH 09/14] Remove leading whitespace in markdown --- README.md | 67 +++++++++++++++++++++++++---------------------- docs/terraform.md | 6 ++--- docs/windows.md | 59 ++++++++++++++++++++++------------------- 3 files changed, 71 insertions(+), 61 deletions(-) diff --git a/README.md b/README.md index b6ca5ea..8bfc8ba 100644 --- a/README.md +++ b/README.md @@ -244,34 +244,39 @@ Available targets: ## Windows Managed Node groups - Windows managed node-groups have a few pre-requisites. - - * Your cluster must contain at least one linux based worker node - * Your EKS Cluster must have the `AmazonEKSVPCResourceController` and `AmazonEKSClusterPolicy` policies attached - * Your cluster must have a config-map called amazon-vpc-cni with the following content - ```yaml - apiVersion: v1 - kind: ConfigMap - metadata: - name: amazon-vpc-cni - namespace: kube-system - data: - enable-windows-ipam: "true" - ``` - * Windows nodes will automatically be tainted - ```yaml - kubernetes_taints = [{ - key = "WINDOWS" - value = "true" - effect = "NO_SCHEDULE" - }] - ``` - * Any pods that target Windows will need to have the following attributes set in their manifest - ```yaml - nodeSelector: - kubernetes.io/os: windows - kubernetes.io/arch: amd64 - ``` + +Windows managed node-groups have a few pre-requisites. + +* Your cluster must contain at least one linux based worker node +* Your EKS Cluster must have the `AmazonEKSVPCResourceController` and `AmazonEKSClusterPolicy` policies attached +* Your cluster must have a config-map called amazon-vpc-cni with the following content + +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: +name: amazon-vpc-cni +namespace: kube-system +data: +enable-windows-ipam: "true" +``` + +* Windows nodes will automatically be tainted + +```yaml +kubernetes_taints = [{ + key = "WINDOWS" + value = "true" + effect = "NO_SCHEDULE" +}] +``` +* Any pods that target Windows will need to have the following attributes set in their manifest + +```yaml + nodeSelector: + kubernetes.io/os: windows + kubernetes.io/arch: amd64 +``` https://docs.aws.amazon.com/eks/latest/userguide/windows-support.html @@ -279,15 +284,15 @@ https://docs.aws.amazon.com/eks/latest/userguide/windows-support.html | Name | Version | |------|---------| -| [terraform](#requirement\_terraform) | >= 0.15.00 | -| [aws](#requirement\_aws) | >= 4.0 | +| [terraform](#requirement\_terraform) | >= 0.14.11 | +| [aws](#requirement\_aws) | >= 4.48 | | [random](#requirement\_random) | >= 2.0 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 4.0 | +| [aws](#provider\_aws) | >= 4.48 | | [random](#provider\_random) | >= 2.0 | ## Modules diff --git a/docs/terraform.md b/docs/terraform.md index 12ea817..df47349 100644 --- a/docs/terraform.md +++ b/docs/terraform.md @@ -3,15 +3,15 @@ | Name | Version | |------|---------| -| [terraform](#requirement\_terraform) | >= 0.15.00 | -| [aws](#requirement\_aws) | >= 4.0 | +| [terraform](#requirement\_terraform) | >= 0.14.11 | +| [aws](#requirement\_aws) | >= 4.48 | | [random](#requirement\_random) | >= 2.0 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 4.0 | +| [aws](#provider\_aws) | >= 4.48 | | [random](#provider\_random) | >= 2.0 | ## Modules diff --git a/docs/windows.md b/docs/windows.md index c322a2a..1dd4a22 100644 --- a/docs/windows.md +++ b/docs/windows.md @@ -1,32 +1,37 @@ ## Windows Managed Node groups - Windows managed node-groups have a few pre-requisites. - * Your cluster must contain at least one linux based worker node - * Your EKS Cluster must have the `AmazonEKSVPCResourceController` and `AmazonEKSClusterPolicy` policies attached - * Your cluster must have a config-map called amazon-vpc-cni with the following content - ```yaml - apiVersion: v1 - kind: ConfigMap - metadata: - name: amazon-vpc-cni - namespace: kube-system - data: - enable-windows-ipam: "true" - ``` - * Windows nodes will automatically be tainted - ```yaml - kubernetes_taints = [{ - key = "WINDOWS" - value = "true" - effect = "NO_SCHEDULE" - }] - ``` - * Any pods that target Windows will need to have the following attributes set in their manifest - ```yaml - nodeSelector: - kubernetes.io/os: windows - kubernetes.io/arch: amd64 - ``` +Windows managed node-groups have a few pre-requisites. + +* Your cluster must contain at least one linux based worker node +* Your EKS Cluster must have the `AmazonEKSVPCResourceController` and `AmazonEKSClusterPolicy` policies attached +* Your cluster must have a config-map called amazon-vpc-cni with the following content + +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: +name: amazon-vpc-cni +namespace: kube-system +data: +enable-windows-ipam: "true" +``` + +* Windows nodes will automatically be tainted + +```yaml +kubernetes_taints = [{ + key = "WINDOWS" + value = "true" + effect = "NO_SCHEDULE" +}] +``` +* Any pods that target Windows will need to have the following attributes set in their manifest + +```yaml + nodeSelector: + kubernetes.io/os: windows + kubernetes.io/arch: amd64 +``` https://docs.aws.amazon.com/eks/latest/userguide/windows-support.html From a908961970c3e9e29ba7be35cae4fc622176ebfc Mon Sep 17 00:00:00 2001 From: Chris McKee Date: Thu, 9 Mar 2023 10:57:15 +0000 Subject: [PATCH 10/14] Change `ami_release_version` regex to support windows ami format Tested here: https://regex101.com/r/xb7q2f/2 --- README.md | 6 +++--- docs/terraform.md | 6 +++--- variables.tf | 6 +++--- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 8bfc8ba..5a80082 100644 --- a/README.md +++ b/README.md @@ -292,8 +292,8 @@ https://docs.aws.amazon.com/eks/latest/userguide/windows-support.html | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 4.48 | -| [random](#provider\_random) | >= 2.0 | +| [aws](#provider\_aws) | 4.55.0 | +| [random](#provider\_random) | 3.4.3 | ## Modules @@ -331,7 +331,7 @@ https://docs.aws.amazon.com/eks/latest/userguide/windows-support.html | [additional\_tag\_map](#input\_additional\_tag\_map) | Additional key-value pairs to add to each map in `tags_as_list_of_maps`. Not added to `tags` or `id`.
This is for some rare cases where resources want additional configuration of tags
and therefore take a list of maps with tag key, value, and additional configuration. | `map(string)` | `{}` | no | | [after\_cluster\_joining\_userdata](#input\_after\_cluster\_joining\_userdata) | Additional `bash` commands to execute on each worker node after joining the EKS cluster (after executing the `bootstrap.sh` script). For more info, see https://kubedex.com/90-days-of-aws-eks-in-production | `list(string)` | `[]` | no | | [ami\_image\_id](#input\_ami\_image\_id) | AMI to use. Ignored if `launch_template_id` is supplied. | `list(string)` | `[]` | no | -| [ami\_release\_version](#input\_ami\_release\_version) | EKS AMI version to use, e.g. For AL2 "1.16.13-20200821" or for bottlerocket "1.2.0-ccf1b754" (no "v"). For AL2 and bottlerocket, it defaults to latest version for Kubernetes version. | `list(string)` | `[]` | no | +| [ami\_release\_version](#input\_ami\_release\_version) | EKS AMI version to use, e.g. For AL2 "1.16.13-20200821" or for bottlerocket "1.2.0-ccf1b754" (no "v") or for Windows "2023.02.14". For AL2, bottlerocket and Windows, it defaults to latest version for Kubernetes version. | `list(string)` | `[]` | no | | [ami\_type](#input\_ami\_type) | Type of Amazon Machine Image (AMI) associated with the EKS Node Group.
Defaults to `AL2_x86_64`. Valid values: `AL2_x86_64, AL2_x86_64_GPU, AL2_ARM_64, CUSTOM, BOTTLEROCKET_ARM_64, BOTTLEROCKET_x86_64, BOTTLEROCKET_ARM_64_NVIDIA, BOTTLEROCKET_x86_64_NVIDIA, WINDOWS_CORE_2019_x86_64, WINDOWS_FULL_2019_x86_64, WINDOWS_CORE_2022_x86_64, WINDOWS_FULL_2022_x86_64`. | `string` | `"AL2_x86_64"` | no | | [associated\_security\_group\_ids](#input\_associated\_security\_group\_ids) | A list of IDs of Security Groups to associate the node group with, in addition to the EKS' created security group.
These security groups will not be modified. | `list(string)` | `[]` | no | | [attributes](#input\_attributes) | ID element. Additional attributes (e.g. `workers` or `cluster`) to add to `id`,
in the order they appear in the list. New attributes are appended to the
end of the list. The elements of the list are joined by the `delimiter`
and treated as a single ID element. | `list(string)` | `[]` | no | diff --git a/docs/terraform.md b/docs/terraform.md index df47349..ee39b79 100644 --- a/docs/terraform.md +++ b/docs/terraform.md @@ -11,8 +11,8 @@ | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 4.48 | -| [random](#provider\_random) | >= 2.0 | +| [aws](#provider\_aws) | 4.55.0 | +| [random](#provider\_random) | 3.4.3 | ## Modules @@ -50,7 +50,7 @@ | [additional\_tag\_map](#input\_additional\_tag\_map) | Additional key-value pairs to add to each map in `tags_as_list_of_maps`. Not added to `tags` or `id`.
This is for some rare cases where resources want additional configuration of tags
and therefore take a list of maps with tag key, value, and additional configuration. | `map(string)` | `{}` | no | | [after\_cluster\_joining\_userdata](#input\_after\_cluster\_joining\_userdata) | Additional `bash` commands to execute on each worker node after joining the EKS cluster (after executing the `bootstrap.sh` script). For more info, see https://kubedex.com/90-days-of-aws-eks-in-production | `list(string)` | `[]` | no | | [ami\_image\_id](#input\_ami\_image\_id) | AMI to use. Ignored if `launch_template_id` is supplied. | `list(string)` | `[]` | no | -| [ami\_release\_version](#input\_ami\_release\_version) | EKS AMI version to use, e.g. For AL2 "1.16.13-20200821" or for bottlerocket "1.2.0-ccf1b754" (no "v"). For AL2 and bottlerocket, it defaults to latest version for Kubernetes version. | `list(string)` | `[]` | no | +| [ami\_release\_version](#input\_ami\_release\_version) | EKS AMI version to use, e.g. For AL2 "1.16.13-20200821" or for bottlerocket "1.2.0-ccf1b754" (no "v") or for Windows "2023.02.14". For AL2, bottlerocket and Windows, it defaults to latest version for Kubernetes version. | `list(string)` | `[]` | no | | [ami\_type](#input\_ami\_type) | Type of Amazon Machine Image (AMI) associated with the EKS Node Group.
Defaults to `AL2_x86_64`. Valid values: `AL2_x86_64, AL2_x86_64_GPU, AL2_ARM_64, CUSTOM, BOTTLEROCKET_ARM_64, BOTTLEROCKET_x86_64, BOTTLEROCKET_ARM_64_NVIDIA, BOTTLEROCKET_x86_64_NVIDIA, WINDOWS_CORE_2019_x86_64, WINDOWS_FULL_2019_x86_64, WINDOWS_CORE_2022_x86_64, WINDOWS_FULL_2022_x86_64`. | `string` | `"AL2_x86_64"` | no | | [associated\_security\_group\_ids](#input\_associated\_security\_group\_ids) | A list of IDs of Security Groups to associate the node group with, in addition to the EKS' created security group.
These security groups will not be modified. | `list(string)` | `[]` | no | | [attributes](#input\_attributes) | ID element. Additional attributes (e.g. `workers` or `cluster`) to add to `id`,
in the order they appear in the list. New attributes are appended to the
end of the list. The elements of the list are joined by the `delimiter`
and treated as a single ID element. | `list(string)` | `[]` | no | diff --git a/variables.tf b/variables.tf index cbff89e..5597a73 100644 --- a/variables.tf +++ b/variables.tf @@ -231,12 +231,12 @@ variable "ami_image_id" { variable "ami_release_version" { type = list(string) default = [] - description = "EKS AMI version to use, e.g. For AL2 \"1.16.13-20200821\" or for bottlerocket \"1.2.0-ccf1b754\" (no \"v\"). For AL2 and bottlerocket, it defaults to latest version for Kubernetes version." + description = "EKS AMI version to use, e.g. For AL2 \"1.16.13-20200821\" or for bottlerocket \"1.2.0-ccf1b754\" (no \"v\") or for Windows \"2023.02.14\". For AL2, bottlerocket and Windows, it defaults to latest version for Kubernetes version." validation { condition = ( - length(var.ami_release_version) == 0 ? true : length(regexall("^\\d+\\.\\d+\\.\\d+-[\\da-z]+$", var.ami_release_version[0])) == 1 + length(var.ami_release_version) == 0 ? true : length(regexall("(^\\d+\\.\\d+\\.\\d+-[\\da-z]+$)|(^\\d+\\.\\d+\\.\\d+$)", var.ami_release_version[0])) == 1 ) - error_message = "Var ami_release_version, if supplied, must be like for AL2 \"1.16.13-20200821\" or for bottlerocket \"1.2.0-ccf1b754\" (no \"v\")." + error_message = "Var ami_release_version, if supplied, must be like for AL2 \"1.16.13-20200821\" or for bottlerocket \"1.2.0-ccf1b754\" (no \"v\") or for Windows \"2023.02.14\"." } } From 6d93e0f4c4b9ca8edcc79ff5d8c492d6db09a21d Mon Sep 17 00:00:00 2001 From: cloudpossebot <11232728+cloudpossebot@users.noreply.github.com> Date: Thu, 9 Mar 2023 11:59:20 +0000 Subject: [PATCH 11/14] Auto Format --- README.md | 4 ++-- docs/terraform.md | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 5a80082..674ac7d 100644 --- a/README.md +++ b/README.md @@ -292,8 +292,8 @@ https://docs.aws.amazon.com/eks/latest/userguide/windows-support.html | Name | Version | |------|---------| -| [aws](#provider\_aws) | 4.55.0 | -| [random](#provider\_random) | 3.4.3 | +| [aws](#provider\_aws) | >= 4.48 | +| [random](#provider\_random) | >= 2.0 | ## Modules diff --git a/docs/terraform.md b/docs/terraform.md index ee39b79..4d17233 100644 --- a/docs/terraform.md +++ b/docs/terraform.md @@ -11,8 +11,8 @@ | Name | Version | |------|---------| -| [aws](#provider\_aws) | 4.55.0 | -| [random](#provider\_random) | 3.4.3 | +| [aws](#provider\_aws) | >= 4.48 | +| [random](#provider\_random) | >= 2.0 | ## Modules From 0cbcccdea3fea8577c13fe6eaade25a4cd834dbf Mon Sep 17 00:00:00 2001 From: Chris McKee Date: Wed, 15 Mar 2023 11:59:23 +0000 Subject: [PATCH 12/14] Update versions.tf Ref version change related to change log https://github.com/hashicorp/terraform-provider-aws/blob/main/CHANGELOG.md#4480-december-19-2022 --- versions.tf | 3 +++ 1 file changed, 3 insertions(+) diff --git a/versions.tf b/versions.tf index dfca973..41afb1a 100644 --- a/versions.tf +++ b/versions.tf @@ -3,6 +3,9 @@ terraform { required_providers { aws = { + # retrieve launch template by ID starts at 3.21.0 + # update_config starts at 3.56 + # Windows support starts at 4.48 https://github.com/hashicorp/terraform-provider-aws/blob/main/CHANGELOG.md#4480-december-19-2022 source = "hashicorp/aws" version = ">= 4.48" } From 49dd7cedaea7c3b246b173a8ac9ac953b063eb3b Mon Sep 17 00:00:00 2001 From: Chris McKee Date: Wed, 15 Mar 2023 12:02:23 +0000 Subject: [PATCH 13/14] Revert renaming of linux worker node group in example/complete --- examples/complete/main.tf | 2 +- examples/complete/outputs.tf | 16 ++++++++-------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/examples/complete/main.tf b/examples/complete/main.tf index 6092ebf..77d22c0 100644 --- a/examples/complete/main.tf +++ b/examples/complete/main.tf @@ -133,7 +133,7 @@ module "eks_cluster" { context = module.this.context } -module "eks_nix_node_group" { +module "eks_node_group" { source = "../../" subnet_ids = module.this.enabled ? module.subnets.public_subnet_ids : ["filler_string_for_enabled_is_false"] diff --git a/examples/complete/outputs.tf b/examples/complete/outputs.tf index 160c8f7..022b0db 100644 --- a/examples/complete/outputs.tf +++ b/examples/complete/outputs.tf @@ -60,39 +60,39 @@ output "eks_cluster_managed_security_group_id" { output "eks_node_group_role_arn" { description = "ARN of the worker nodes IAM role" - value = module.eks_nix_node_group.eks_node_group_role_arn + value = module.eks_node_group.eks_node_group_role_arn } output "eks_node_group_role_name" { description = "Name of the worker nodes IAM role" - value = module.eks_nix_node_group.eks_node_group_role_name + value = module.eks_node_group.eks_node_group_role_name } output "eks_node_group_id" { description = "EKS Cluster name and EKS Node Group name separated by a colon" - value = module.eks_nix_node_group.eks_node_group_id + value = module.eks_node_group.eks_node_group_id } output "eks_node_group_arn" { description = "Amazon Resource Name (ARN) of the EKS Node Group" - value = module.eks_nix_node_group.eks_node_group_arn + value = module.eks_node_group.eks_node_group_arn } output "eks_node_group_resources" { description = "List of objects containing information about underlying resources of the EKS Node Group" - value = module.eks_nix_node_group.eks_node_group_resources + value = module.eks_node_group.eks_node_group_resources } output "eks_node_group_status" { description = "Status of the EKS Node Group" - value = module.eks_nix_node_group.eks_node_group_status + value = module.eks_node_group.eks_node_group_status } output "eks_node_group_cbd_pet_name" { description = "The pet name of this node group, if this module generated one" - value = module.eks_nix_node_group.eks_node_group_cbd_pet_name + value = module.eks_node_group.eks_node_group_cbd_pet_name } output "eks_node_group_launch_template_id" { - value = module.eks_nix_node_group.eks_node_group_launch_template_id + value = module.eks_node_group.eks_node_group_launch_template_id } From 16aae50bb16aaa86363233528e0ce0ea64e3e5ab Mon Sep 17 00:00:00 2001 From: cloudpossebot <11232728+cloudpossebot@users.noreply.github.com> Date: Wed, 15 Mar 2023 12:07:05 +0000 Subject: [PATCH 14/14] Auto Format --- README.md | 1 + docs/terraform.md | 1 + 2 files changed, 2 insertions(+) diff --git a/README.md b/README.md index 674ac7d..69203dd 100644 --- a/README.md +++ b/README.md @@ -333,6 +333,7 @@ https://docs.aws.amazon.com/eks/latest/userguide/windows-support.html | [ami\_image\_id](#input\_ami\_image\_id) | AMI to use. Ignored if `launch_template_id` is supplied. | `list(string)` | `[]` | no | | [ami\_release\_version](#input\_ami\_release\_version) | EKS AMI version to use, e.g. For AL2 "1.16.13-20200821" or for bottlerocket "1.2.0-ccf1b754" (no "v") or for Windows "2023.02.14". For AL2, bottlerocket and Windows, it defaults to latest version for Kubernetes version. | `list(string)` | `[]` | no | | [ami\_type](#input\_ami\_type) | Type of Amazon Machine Image (AMI) associated with the EKS Node Group.
Defaults to `AL2_x86_64`. Valid values: `AL2_x86_64, AL2_x86_64_GPU, AL2_ARM_64, CUSTOM, BOTTLEROCKET_ARM_64, BOTTLEROCKET_x86_64, BOTTLEROCKET_ARM_64_NVIDIA, BOTTLEROCKET_x86_64_NVIDIA, WINDOWS_CORE_2019_x86_64, WINDOWS_FULL_2019_x86_64, WINDOWS_CORE_2022_x86_64, WINDOWS_FULL_2022_x86_64`. | `string` | `"AL2_x86_64"` | no | +| [associate\_cluster\_security\_group](#input\_associate\_cluster\_security\_group) | When true, associate the default cluster security group to the nodes. If disabled the EKS managed security group will not
be associated to the nodes, therefore the communications between pods and nodes will not work. Be aware that if no `associated_security_group_ids`
nor `ssh_access_security_group_ids` are provided then the nodes will have no inbound or outbound rules. | `bool` | `true` | no | | [associated\_security\_group\_ids](#input\_associated\_security\_group\_ids) | A list of IDs of Security Groups to associate the node group with, in addition to the EKS' created security group.
These security groups will not be modified. | `list(string)` | `[]` | no | | [attributes](#input\_attributes) | ID element. Additional attributes (e.g. `workers` or `cluster`) to add to `id`,
in the order they appear in the list. New attributes are appended to the
end of the list. The elements of the list are joined by the `delimiter`
and treated as a single ID element. | `list(string)` | `[]` | no | | [before\_cluster\_joining\_userdata](#input\_before\_cluster\_joining\_userdata) | Additional `bash` commands to execute on each worker node before joining the EKS cluster (before executing the `bootstrap.sh` script). For more info, see https://kubedex.com/90-days-of-aws-eks-in-production | `list(string)` | `[]` | no | diff --git a/docs/terraform.md b/docs/terraform.md index 4d17233..dac4ed3 100644 --- a/docs/terraform.md +++ b/docs/terraform.md @@ -52,6 +52,7 @@ | [ami\_image\_id](#input\_ami\_image\_id) | AMI to use. Ignored if `launch_template_id` is supplied. | `list(string)` | `[]` | no | | [ami\_release\_version](#input\_ami\_release\_version) | EKS AMI version to use, e.g. For AL2 "1.16.13-20200821" or for bottlerocket "1.2.0-ccf1b754" (no "v") or for Windows "2023.02.14". For AL2, bottlerocket and Windows, it defaults to latest version for Kubernetes version. | `list(string)` | `[]` | no | | [ami\_type](#input\_ami\_type) | Type of Amazon Machine Image (AMI) associated with the EKS Node Group.
Defaults to `AL2_x86_64`. Valid values: `AL2_x86_64, AL2_x86_64_GPU, AL2_ARM_64, CUSTOM, BOTTLEROCKET_ARM_64, BOTTLEROCKET_x86_64, BOTTLEROCKET_ARM_64_NVIDIA, BOTTLEROCKET_x86_64_NVIDIA, WINDOWS_CORE_2019_x86_64, WINDOWS_FULL_2019_x86_64, WINDOWS_CORE_2022_x86_64, WINDOWS_FULL_2022_x86_64`. | `string` | `"AL2_x86_64"` | no | +| [associate\_cluster\_security\_group](#input\_associate\_cluster\_security\_group) | When true, associate the default cluster security group to the nodes. If disabled the EKS managed security group will not
be associated to the nodes, therefore the communications between pods and nodes will not work. Be aware that if no `associated_security_group_ids`
nor `ssh_access_security_group_ids` are provided then the nodes will have no inbound or outbound rules. | `bool` | `true` | no | | [associated\_security\_group\_ids](#input\_associated\_security\_group\_ids) | A list of IDs of Security Groups to associate the node group with, in addition to the EKS' created security group.
These security groups will not be modified. | `list(string)` | `[]` | no | | [attributes](#input\_attributes) | ID element. Additional attributes (e.g. `workers` or `cluster`) to add to `id`,
in the order they appear in the list. New attributes are appended to the
end of the list. The elements of the list are joined by the `delimiter`
and treated as a single ID element. | `list(string)` | `[]` | no | | [before\_cluster\_joining\_userdata](#input\_before\_cluster\_joining\_userdata) | Additional `bash` commands to execute on each worker node before joining the EKS cluster (before executing the `bootstrap.sh` script). For more info, see https://kubedex.com/90-days-of-aws-eks-in-production | `list(string)` | `[]` | no |