Add support for AMIs based on AmazonLinux2023 (#7684)

* add support for AL2023 for EKS-managed and self-managed nodes * ensure AL2023 only supports containerd * add GPU related validations + small nits * add support for upgrades * add support for EFA * improve validations * fix lint and unit tests * update docs * add validation error for maxpods limitation * add integration tests for al2023 * improve validation message
eksctl-io · Apr 4, 2024 · f9475f8 · f9475f8
1 parent 2ca7b65
commit f9475f8
Show file tree

Hide file tree

Showing 29 changed files with 729 additions and 57 deletions.
diff --git a/integration/tests/custom_ami/custom_ami_test.go b/integration/tests/custom_ami/custom_ami_test.go
@@ -38,6 +38,7 @@ func TestOverrideBootstrap(t *testing.T) {
 
 var (
 	customAMIAL2          string
+	customAMIAL2023       string
 	customAMIBottlerocket string
 )
 
@@ -47,15 +48,23 @@ var _ = BeforeSuite(func() {
 
 	// retrieve AL2 AMI
 	input := &awsssm.GetParameterInput{
-		Name: aws.String("/aws/service/eks/optimized-ami/1.22/amazon-linux-2/recommended/image_id"),
+		Name: aws.String(fmt.Sprintf("/aws/service/eks/optimized-ami/%s/amazon-linux-2/recommended/image_id", params.Version)),
 	}
 	output, err := ssm.GetParameter(context.Background(), input)
 	Expect(err).NotTo(HaveOccurred())
 	customAMIAL2 = *output.Parameter.Value
 
+	// retrieve AL2023 AMI
+	input = &awsssm.GetParameterInput{
+		Name: aws.String(fmt.Sprintf("/aws/service/eks/optimized-ami/%s/amazon-linux-2023/x86_64/standard/recommended/image_id", params.Version)),
+	}
+	output, err = ssm.GetParameter(context.Background(), input)
+	Expect(err).NotTo(HaveOccurred())
+	customAMIAL2023 = *output.Parameter.Value
+
 	// retrieve Bottlerocket AMI
 	input = &awsssm.GetParameterInput{
-		Name: aws.String("/aws/service/bottlerocket/aws-k8s-1.25/x86_64/latest/image_id"),
+		Name: aws.String(fmt.Sprintf("/aws/service/bottlerocket/aws-k8s-%s/x86_64/latest/image_id", params.Version)),
 	}
 	output, err = ssm.GetParameter(context.Background(), input)
 	Expect(err).NotTo(HaveOccurred())
@@ -75,6 +84,26 @@ var _ = BeforeSuite(func() {
 var _ = Describe("(Integration) [Test Custom AMI]", func() {
 	params.LogStacksEventsOnFailure()
 
+	Context("al2023 managed and un-managed nodegroups", func() {
+		It("can create working nodegroups which can join the cluster", func() {
+			By(fmt.Sprintf("using the following EKS optimised AMI: %s", customAMIAL2023))
+			content, err := os.ReadFile(filepath.Join("testdata/al2023.yaml"))
+			Expect(err).NotTo(HaveOccurred())
+			content = bytes.ReplaceAll(content, []byte("<generated>"), []byte(params.ClusterName))
+			content = bytes.ReplaceAll(content, []byte("<generated-region>"), []byte(params.Region))
+			content = bytes.ReplaceAll(content, []byte("<generated-ami>"), []byte(customAMIAL2023))
+			cmd := params.EksctlCreateCmd.
+				WithArgs(
+					"nodegroup",
+					"--config-file", "-",
+					"--verbose", "4",
+				).
+				WithoutArg("--region", params.Region).
+				WithStdin(bytes.NewReader(content))
+			Expect(cmd).To(RunSuccessfully())
+		})
+	})
+
 	Context("override bootstrap command for managed and un-managed nodegroups", func() {
 
 		It("can create a working nodegroup which can join the cluster", func() {

diff --git a/integration/tests/custom_ami/testdata/al2023.yaml b/integration/tests/custom_ami/testdata/al2023.yaml
@@ -0,0 +1,19 @@
+apiVersion: eksctl.io/v1alpha5
+kind: ClusterConfig
+
+# name is generated
+metadata:
+  name: <generated>
+  region: <generated-region>
+
+nodeGroups:
+  - name: unm-al2023
+    ami: <generated-ami>
+    amiFamily: AmazonLinux2023
+    desiredCapacity: 1
+
+managedNodeGroups:
+  - name: mng-al2023
+    ami: <generated-ami>
+    amiFamily: AmazonLinux2023
+    desiredCapacity: 1
diff --git a/integration/tests/managed/managed_nodegroup_test.go b/integration/tests/managed/managed_nodegroup_test.go
@@ -70,6 +70,7 @@ var _ = Describe("(Integration) Create Managed Nodegroups", func() {
 
 	const (
 		updateConfigNodegroup    = "ng-update-config"
+		al2023Nodegroup          = "ng-al2023"
 		bottlerocketNodegroup    = "ng-bottlerocket"
 		bottlerocketGPUNodegroup = "ng-bottlerocket-gpu"
 		ubuntuNodegroup          = "ng-ubuntu"
@@ -302,6 +303,32 @@ var _ = Describe("(Integration) Create Managed Nodegroups", func() {
 			checkNg(bottlerocketGPUNodegroup)
 		})
 
+		It("supports AmazonLinux2023 nodegroups", func() {
+			clusterConfig := makeClusterConfig()
+			clusterConfig.ManagedNodeGroups = []*api.ManagedNodeGroup{
+				{
+					NodeGroupBase: &api.NodeGroupBase{
+						Name:      al2023Nodegroup,
+						AMIFamily: "AmazonLinux2023",
+					},
+				},
+			}
+
+			By("creating it")
+			Expect(params.EksctlCreateCmd.
+				WithArgs(
+					"nodegroup",
+					"--config-file", "-",
+					"--verbose", "4",
+				).
+				WithoutArg("--region", params.Region).
+				WithStdin(clusterutils.Reader(clusterConfig))).
+				To(RunSuccessfully())
+
+			By("ensuring it is healthy")
+			checkNg(al2023Nodegroup)
+		})
+
 		It("supports bottlerocket and ubuntu nodegroups with additional volumes", func() {
 			clusterConfig := makeClusterConfig()
 			clusterConfig.ManagedNodeGroups = []*api.ManagedNodeGroup{

diff --git a/pkg/ami/auto_resolver.go b/pkg/ami/auto_resolver.go
@@ -23,6 +23,10 @@ const (
 // MakeImageSearchPatterns creates a map of image search patterns by image OS family and class
 func MakeImageSearchPatterns(version string) map[string]map[int]string {
 	return map[string]map[int]string{
+		api.NodeImageFamilyAmazonLinux2023: {
+			ImageClassGeneral: fmt.Sprintf("amazon-eks-node-al2023-x86_64-standard-%s-v*", version),
+			ImageClassARM:     fmt.Sprintf("amazon-eks-node-al2023-arm64-standard-%s-v*", version),
+		},
 		api.NodeImageFamilyAmazonLinux2: {
 			ImageClassGeneral: fmt.Sprintf("amazon-eks-node-%s-v*", version),
 			ImageClassGPU:     fmt.Sprintf("amazon-eks-gpu-node-%s-*", version),
@@ -59,7 +63,7 @@ func OwnerAccountID(imageFamily, region string) (string, error) {
 	switch imageFamily {
 	case api.NodeImageFamilyUbuntu2204, api.NodeImageFamilyUbuntu2004, api.NodeImageFamilyUbuntu1804:
 		return ownerIDUbuntuFamily, nil
-	case api.NodeImageFamilyAmazonLinux2:
+	case api.NodeImageFamilyAmazonLinux2023, api.NodeImageFamilyAmazonLinux2:
 		return api.EKSResourceAccountID(region), nil
 	default:
 		if api.IsWindowsImage(imageFamily) {

diff --git a/pkg/ami/ssm_resolver.go b/pkg/ami/ssm_resolver.go
@@ -54,6 +54,9 @@ func MakeSSMParameterName(version, instanceType, imageFamily string) (string, er
 	const fieldName = "image_id"
 
 	switch imageFamily {
+	case api.NodeImageFamilyAmazonLinux2023:
+		return fmt.Sprintf("/aws/service/eks/optimized-ami/%s/%s/%s/standard/recommended/%s",
+			version, utils.ToKebabCase(imageFamily), instanceEC2ArchName(instanceType), fieldName), nil
 	case api.NodeImageFamilyAmazonLinux2:
 		return fmt.Sprintf("/aws/service/eks/optimized-ami/%s/%s/recommended/%s", version, imageType(imageFamily, instanceType, version), fieldName), nil
 	case api.NodeImageFamilyWindowsServer2019CoreContainer,
@@ -83,6 +86,10 @@ func MakeSSMParameterName(version, instanceType, imageFamily string) (string, er
 // MakeManagedSSMParameterName creates an SSM parameter name for a managed nodegroup
 func MakeManagedSSMParameterName(version string, amiType ekstypes.AMITypes) (string, error) {
 	switch amiType {
+	case ekstypes.AMITypesAl2023X8664Standard:
+		return fmt.Sprintf("/aws/service/eks/optimized-ami/%s/%s/x86_64/standard/recommended/release_version", version, utils.ToKebabCase(api.NodeImageFamilyAmazonLinux2023)), nil
+	case ekstypes.AMITypesAl2023Arm64Standard:
+		return fmt.Sprintf("/aws/service/eks/optimized-ami/%s/%s/arm64/standard/recommended/release_version", version, utils.ToKebabCase(api.NodeImageFamilyAmazonLinux2023)), nil
 	case ekstypes.AMITypesAl2X8664:
 		imageType := utils.ToKebabCase(api.NodeImageFamilyAmazonLinux2)
 		return fmt.Sprintf("/aws/service/eks/optimized-ami/%s/%s/recommended/release_version", version, imageType), nil

diff --git a/pkg/apis/eksctl.io/v1alpha5/assets/schema.json b/pkg/apis/eksctl.io/v1alpha5/assets/schema.json
@@ -1256,11 +1256,12 @@
         },
         "amiFamily": {
           "type": "string",
-          "description": "Valid variants are: `\"AmazonLinux2\"` (default), `\"Ubuntu2204\"`, `\"Ubuntu2004\"`, `\"Ubuntu1804\"`, `\"Bottlerocket\"`, `\"WindowsServer2019CoreContainer\"`, `\"WindowsServer2019FullContainer\"`, `\"WindowsServer2022CoreContainer\"`, `\"WindowsServer2022FullContainer\"`.",
-          "x-intellij-html-description": "Valid variants are: <code>&quot;AmazonLinux2&quot;</code> (default), <code>&quot;Ubuntu2204&quot;</code>, <code>&quot;Ubuntu2004&quot;</code>, <code>&quot;Ubuntu1804&quot;</code>, <code>&quot;Bottlerocket&quot;</code>, <code>&quot;WindowsServer2019CoreContainer&quot;</code>, <code>&quot;WindowsServer2019FullContainer&quot;</code>, <code>&quot;WindowsServer2022CoreContainer&quot;</code>, <code>&quot;WindowsServer2022FullContainer&quot;</code>.",
+          "description": "Valid variants are: `\"AmazonLinux2\"` (default), `\"AmazonLinux2023\"`, `\"Ubuntu2204\"`, `\"Ubuntu2004\"`, `\"Ubuntu1804\"`, `\"Bottlerocket\"`, `\"WindowsServer2019CoreContainer\"`, `\"WindowsServer2019FullContainer\"`, `\"WindowsServer2022CoreContainer\"`, `\"WindowsServer2022FullContainer\"`.",
+          "x-intellij-html-description": "Valid variants are: <code>&quot;AmazonLinux2&quot;</code> (default), <code>&quot;AmazonLinux2023&quot;</code>, <code>&quot;Ubuntu2204&quot;</code>, <code>&quot;Ubuntu2004&quot;</code>, <code>&quot;Ubuntu1804&quot;</code>, <code>&quot;Bottlerocket&quot;</code>, <code>&quot;WindowsServer2019CoreContainer&quot;</code>, <code>&quot;WindowsServer2019FullContainer&quot;</code>, <code>&quot;WindowsServer2022CoreContainer&quot;</code>, <code>&quot;WindowsServer2022FullContainer&quot;</code>.",
           "default": "AmazonLinux2",
           "enum": [
             "AmazonLinux2",
+            "AmazonLinux2023",
             "Ubuntu2204",
             "Ubuntu2004",
             "Ubuntu1804",
@@ -1589,11 +1590,12 @@
         },
         "amiFamily": {
           "type": "string",
-          "description": "Valid variants are: `\"AmazonLinux2\"` (default), `\"Ubuntu2204\"`, `\"Ubuntu2004\"`, `\"Ubuntu1804\"`, `\"Bottlerocket\"`, `\"WindowsServer2019CoreContainer\"`, `\"WindowsServer2019FullContainer\"`, `\"WindowsServer2022CoreContainer\"`, `\"WindowsServer2022FullContainer\"`.",
-          "x-intellij-html-description": "Valid variants are: <code>&quot;AmazonLinux2&quot;</code> (default), <code>&quot;Ubuntu2204&quot;</code>, <code>&quot;Ubuntu2004&quot;</code>, <code>&quot;Ubuntu1804&quot;</code>, <code>&quot;Bottlerocket&quot;</code>, <code>&quot;WindowsServer2019CoreContainer&quot;</code>, <code>&quot;WindowsServer2019FullContainer&quot;</code>, <code>&quot;WindowsServer2022CoreContainer&quot;</code>, <code>&quot;WindowsServer2022FullContainer&quot;</code>.",
+          "description": "Valid variants are: `\"AmazonLinux2\"` (default), `\"AmazonLinux2023\"`, `\"Ubuntu2204\"`, `\"Ubuntu2004\"`, `\"Ubuntu1804\"`, `\"Bottlerocket\"`, `\"WindowsServer2019CoreContainer\"`, `\"WindowsServer2019FullContainer\"`, `\"WindowsServer2022CoreContainer\"`, `\"WindowsServer2022FullContainer\"`.",
+          "x-intellij-html-description": "Valid variants are: <code>&quot;AmazonLinux2&quot;</code> (default), <code>&quot;AmazonLinux2023&quot;</code>, <code>&quot;Ubuntu2204&quot;</code>, <code>&quot;Ubuntu2004&quot;</code>, <code>&quot;Ubuntu1804&quot;</code>, <code>&quot;Bottlerocket&quot;</code>, <code>&quot;WindowsServer2019CoreContainer&quot;</code>, <code>&quot;WindowsServer2019FullContainer&quot;</code>, <code>&quot;WindowsServer2022CoreContainer&quot;</code>, <code>&quot;WindowsServer2022FullContainer&quot;</code>.",
           "default": "AmazonLinux2",
           "enum": [
             "AmazonLinux2",
+            "AmazonLinux2023",
             "Ubuntu2204",
             "Ubuntu2004",
             "Ubuntu1804",

diff --git a/pkg/apis/eksctl.io/v1alpha5/defaults.go b/pkg/apis/eksctl.io/v1alpha5/defaults.go
@@ -249,17 +249,23 @@ func setContainerRuntimeDefault(ng *NodeGroup, clusterVersion string) {
 		return
 	}
 
-	// since clusterVersion is standardised beforehand, we can safely ignore the error
-	isDockershimDeprecated, _ := utils.IsMinVersion(DockershimDeprecationVersion, clusterVersion)
+	if ng.AMIFamily == NodeImageFamilyAmazonLinux2023 {
+		ng.ContainerRuntime = aws.String(ContainerRuntimeContainerD)
+		return
+	}
 
-	if isDockershimDeprecated {
+	// since clusterVersion is standardised beforehand, we can safely ignore the error
+	if isDockershimDeprecated, _ := utils.IsMinVersion(DockershimDeprecationVersion, clusterVersion); isDockershimDeprecated {
 		ng.ContainerRuntime = aws.String(ContainerRuntimeContainerD)
-	} else {
-		ng.ContainerRuntime = aws.String(ContainerRuntimeDockerD)
-		if IsWindowsImage(ng.AMIFamily) {
-			ng.ContainerRuntime = aws.String(ContainerRuntimeDockerForWindows)
-		}
+		return
 	}
+
+	if IsWindowsImage(ng.AMIFamily) {
+		ng.ContainerRuntime = aws.String(ContainerRuntimeDockerForWindows)
+		return
+	}
+
+	ng.ContainerRuntime = aws.String(ContainerRuntimeDockerD)
 }
 
 func setIAMDefaults(iamConfig *NodeGroupIAM) {

diff --git a/pkg/apis/eksctl.io/v1alpha5/defaults_test.go b/pkg/apis/eksctl.io/v1alpha5/defaults_test.go
@@ -282,6 +282,17 @@ var _ = Describe("ClusterConfig validation", func() {
 					Expect(*testNodeGroup.ContainerRuntime).To(Equal(ContainerRuntimeDockerForWindows))
 				})
 			})
+			When("ami family is AmazonLinux2023", func() {
+				It("defaults to containerd as a container runtime", func() {
+					testNodeGroup := NodeGroup{
+						NodeGroupBase: &NodeGroupBase{
+							AMIFamily: NodeImageFamilyAmazonLinux2023,
+						},
+					}
+					SetNodeGroupDefaults(&testNodeGroup, &ClusterMeta{Version: Version1_23}, false)
+					Expect(*testNodeGroup.ContainerRuntime).To(Equal(ContainerRuntimeContainerD))
+				})
+			})
 		})
 
 		Context("Kubernetes version 1.24 or greater", func() {

diff --git a/pkg/apis/eksctl.io/v1alpha5/gpu_validation_test.go b/pkg/apis/eksctl.io/v1alpha5/gpu_validation_test.go
@@ -39,6 +39,24 @@ var _ = Describe("GPU instance support", func() {
 		mng.InstanceSelector = &api.InstanceSelector{}
 		assertValidationError(e, api.ValidateManagedNodeGroup(0, mng))
 	},
+		Entry("AL2023 INF", gpuInstanceEntry{
+			amiFamily:            api.NodeImageFamilyAmazonLinux2023,
+			gpuInstanceType:      "inf1.xlarge",
+			expectUnsupportedErr: true,
+			instanceTypeName:     "Inferentia",
+		}),
+		Entry("AL2023 TRN", gpuInstanceEntry{
+			amiFamily:            api.NodeImageFamilyAmazonLinux2023,
+			gpuInstanceType:      "trn1.2xlarge",
+			expectUnsupportedErr: true,
+			instanceTypeName:     "Trainium",
+		}),
+		Entry("AL2023 NVIDIA", gpuInstanceEntry{
+			amiFamily:            api.NodeImageFamilyAmazonLinux2023,
+			gpuInstanceType:      "g4dn.xlarge",
+			expectUnsupportedErr: true,
+			instanceTypeName:     "GPU",
+		}),
 		Entry("AL2", gpuInstanceEntry{
 			gpuInstanceType: "asdf",
 			amiFamily:       api.NodeImageFamilyAmazonLinux2,
@@ -72,6 +90,24 @@ var _ = Describe("GPU instance support", func() {
 		assertValidationError(e, api.ValidateNodeGroup(0, ng, api.NewClusterConfig()))
 
 	},
+		Entry("AL2023 INF", gpuInstanceEntry{
+			amiFamily:            api.NodeImageFamilyAmazonLinux2023,
+			gpuInstanceType:      "inf1.xlarge",
+			expectUnsupportedErr: true,
+			instanceTypeName:     "Inferentia",
+		}),
+		Entry("AL2023 TRN", gpuInstanceEntry{
+			amiFamily:            api.NodeImageFamilyAmazonLinux2023,
+			gpuInstanceType:      "trn1.2xlarge",
+			expectUnsupportedErr: true,
+			instanceTypeName:     "Trainium",
+		}),
+		Entry("AL2023 NVIDIA", gpuInstanceEntry{
+			amiFamily:            api.NodeImageFamilyAmazonLinux2023,
+			gpuInstanceType:      "g4dn.xlarge",
+			expectUnsupportedErr: true,
+			instanceTypeName:     "GPU",
+		}),
 		Entry("AL2", gpuInstanceEntry{
 			gpuInstanceType: "g4dn.xlarge",
 			amiFamily:       api.NodeImageFamilyAmazonLinux2,

diff --git a/pkg/apis/eksctl.io/v1alpha5/types.go b/pkg/apis/eksctl.io/v1alpha5/types.go
@@ -222,12 +222,13 @@ const (
 // All valid values of supported families should go in this block
 const (
 	// DefaultNodeImageFamily (default)
-	DefaultNodeImageFamily      = NodeImageFamilyAmazonLinux2
-	NodeImageFamilyAmazonLinux2 = "AmazonLinux2"
-	NodeImageFamilyUbuntu2204   = "Ubuntu2204"
-	NodeImageFamilyUbuntu2004   = "Ubuntu2004"
-	NodeImageFamilyUbuntu1804   = "Ubuntu1804"
-	NodeImageFamilyBottlerocket = "Bottlerocket"
+	DefaultNodeImageFamily         = NodeImageFamilyAmazonLinux2
+	NodeImageFamilyAmazonLinux2023 = "AmazonLinux2023"
+	NodeImageFamilyAmazonLinux2    = "AmazonLinux2"
+	NodeImageFamilyUbuntu2204      = "Ubuntu2204"
+	NodeImageFamilyUbuntu2004      = "Ubuntu2004"
+	NodeImageFamilyUbuntu1804      = "Ubuntu1804"
+	NodeImageFamilyBottlerocket    = "Bottlerocket"
 
 	NodeImageFamilyWindowsServer2019CoreContainer = "WindowsServer2019CoreContainer"
 	NodeImageFamilyWindowsServer2019FullContainer = "WindowsServer2019FullContainer"
@@ -603,9 +604,10 @@ func SupportedNodeVolumeTypes() []string {
 	}
 }
 
-// supportedAMIFamilies are the AMI families supported by EKS
-func supportedAMIFamilies() []string {
+// SupportedAMIFamilies are the AMI families supported by EKS
+func SupportedAMIFamilies() []string {
 	return []string{
+		NodeImageFamilyAmazonLinux2023,
 		NodeImageFamilyAmazonLinux2,
 		NodeImageFamilyUbuntu2204,
 		NodeImageFamilyUbuntu2004,