feat: on-demand capacity reservation support

aws · Feb 21, 2025 · 24c8e2d · 24c8e2d
1 parent 9e48f2f
commit 24c8e2d
Show file tree

Hide file tree

Showing 42 changed files with 1,535 additions and 421 deletions.
diff --git a/charts/karpenter-crd/templates/karpenter.k8s.aws_ec2nodeclasses.yaml b/charts/karpenter-crd/templates/karpenter.k8s.aws_ec2nodeclasses.yaml
@@ -239,6 +239,39 @@ spec:
                   x-kubernetes-validations:
                     - message: must have only one blockDeviceMappings with rootVolume
                       rule: self.filter(x, has(x.rootVolume)?x.rootVolume==true:false).size() <= 1
+                capacityReservationSelectorTerms:
+                  description: |-
+                    CapacityReservationSelectorTerms is a list of capacity reservation selector terms. Each term is ORed together to
+                    determine the set of eligible capacity reservations.
+                  items:
+                    properties:
+                      id:
+                        description: ID is the capacity reservation id in EC2
+                        pattern: ^cr-[0-9a-z]+$
+                        type: string
+                      ownerID:
+                        description: Owner is the owner id for the ami.
+                        pattern: ^[0-9]{12}$
+                        type: string
+                      tags:
+                        additionalProperties:
+                          type: string
+                        description: |-
+                          Tags is a map of key/value tags used to select capacity reservations.
+                          Specifying '*' for a value selects all values for a given tag key.
+                        maxProperties: 20
+                        type: object
+                        x-kubernetes-validations:
+                          - message: empty tag keys or values aren't supported
+                            rule: self.all(k, k != '' && self[k] != '')
+                    type: object
+                  maxItems: 30
+                  type: array
+                  x-kubernetes-validations:
+                    - message: expected at least one, got none, ['tags', 'id']
+                      rule: self.all(x, has(x.tags) || has(x.id))
+                    - message: '''id'' is mutually exclusive, cannot be set along with tags in a capacity reservation selector term'
+                      rule: '!self.all(x, has(x.id) && (has(x.tags) || has(x.ownerID)))'
                 context:
                   description: |-
                     Context is a Reserved field in EC2 APIs
@@ -469,7 +502,7 @@ spec:
                     - message: immutable field changed
                       rule: self == oldSelf
                 securityGroupSelectorTerms:
-                  description: SecurityGroupSelectorTerms is a list of or security group selector terms. The terms are ORed.
+                  description: SecurityGroupSelectorTerms is a list of security group selector terms. The terms are ORed.
                   items:
                     description: |-
                       SecurityGroupSelectorTerm defines selection logic for a security group used by Karpenter to launch nodes.
@@ -503,12 +536,12 @@ spec:
                       rule: self.size() != 0
                     - message: expected at least one, got none, ['tags', 'id', 'name']
                       rule: self.all(x, has(x.tags) || has(x.id) || has(x.name))
-                    - message: '''id'' is mutually exclusive, cannot be set with a combination of other fields in securityGroupSelectorTerms'
+                    - message: '''id'' is mutually exclusive, cannot be set with a combination of other fields in a security group selector term'
                       rule: '!self.all(x, has(x.id) && (has(x.tags) || has(x.name)))'
-                    - message: '''name'' is mutually exclusive, cannot be set with a combination of other fields in securityGroupSelectorTerms'
+                    - message: '''name'' is mutually exclusive, cannot be set with a combination of other fields in a security group selector term'
                       rule: '!self.all(x, has(x.name) && (has(x.tags) || has(x.id)))'
                 subnetSelectorTerms:
-                  description: SubnetSelectorTerms is a list of or subnet selector terms. The terms are ORed.
+                  description: SubnetSelectorTerms is a list of subnet selector terms. The terms are ORed.
                   items:
                     description: |-
                       SubnetSelectorTerm defines selection logic for a subnet used by Karpenter to launch nodes.
@@ -537,7 +570,7 @@ spec:
                       rule: self.size() != 0
                     - message: expected at least one, got none, ['tags', 'id']
                       rule: self.all(x, has(x.tags) || has(x.id))
-                    - message: '''id'' is mutually exclusive, cannot be set with a combination of other fields in subnetSelectorTerms'
+                    - message: '''id'' is mutually exclusive, cannot be set with a combination of other fields in a subnet selector term'
                       rule: '!self.all(x, has(x.id) && has(x.tags))'
                 tags:
                   additionalProperties:
@@ -640,6 +673,51 @@ spec:
                       - requirements
                     type: object
                   type: array
+                capacityReservations:
+                  items:
+                    properties:
+                      availabilityZone:
+                        description: The availability zone the capacity reservation is available in.
+                        type: string
+                      availableInstanceCount:
+                        description: The last known available instance count for the capacity reservation.
+                        type: integer
+                      endTime:
+                        description: |-
+                          The time at which the capacity reservation expires. Once expired, the reserved capacity is released and Karpenter
+                          will no longer be able to launch instances into that reservation.
+                        format: date-time
+                        type: string
+                      id:
+                        description: The id for the capacity reservation.
+                        pattern: ^cr-[0-9a-z]+$
+                        type: string
+                      instanceMatchCriteria:
+                        description: Indicates the type of instance launches the capacity reservation accepts.
+                        enum:
+                          - open
+                          - targeted
+                        type: string
+                      instanceType:
+                        description: The instance type for the capacity reservation.
+                        type: string
+                      ownerID:
+                        description: The ID of the AWS account that owns the capacity reservation.
+                        pattern: ^[0-9]{12}$
+                        type: string
+                      totalInstanceCount:
+                        description: The total instance count for the capacity reservation.
+                        type: integer
+                    required:
+                      - availabilityZone
+                      - availableInstanceCount
+                      - id
+                      - instanceMatchCriteria
+                      - instanceType
+                      - ownerID
+                      - totalInstanceCount
+                    type: object
+                  type: array
                 conditions:
                   description: Conditions contains signals for health and readiness
                   items:

diff --git a/charts/karpenter-crd/templates/karpenter.sh_nodeclaims.yaml b/charts/karpenter-crd/templates/karpenter.sh_nodeclaims.yaml
@@ -137,7 +137,7 @@ spec:
                           - message: label "kubernetes.io/hostname" is restricted
                             rule: self != "kubernetes.io/hostname"
                           - message: label domain "karpenter.k8s.aws" is restricted
-                            rule: self in ["karpenter.k8s.aws/ec2nodeclass", "karpenter.k8s.aws/instance-encryption-in-transit-supported", "karpenter.k8s.aws/instance-category", "karpenter.k8s.aws/instance-hypervisor", "karpenter.k8s.aws/instance-family", "karpenter.k8s.aws/instance-generation", "karpenter.k8s.aws/instance-local-nvme", "karpenter.k8s.aws/instance-size", "karpenter.k8s.aws/instance-cpu", "karpenter.k8s.aws/instance-cpu-manufacturer", "karpenter.k8s.aws/instance-cpu-sustained-clock-speed-mhz", "karpenter.k8s.aws/instance-memory", "karpenter.k8s.aws/instance-ebs-bandwidth", "karpenter.k8s.aws/instance-network-bandwidth", "karpenter.k8s.aws/instance-gpu-name", "karpenter.k8s.aws/instance-gpu-manufacturer", "karpenter.k8s.aws/instance-gpu-count", "karpenter.k8s.aws/instance-gpu-memory", "karpenter.k8s.aws/instance-accelerator-name", "karpenter.k8s.aws/instance-accelerator-manufacturer", "karpenter.k8s.aws/instance-accelerator-count"] || !self.find("^([^/]+)").endsWith("karpenter.k8s.aws")
+                            rule: self in ["karpenter.k8s.aws/capacity-reservation-id", "karpenter.k8s.aws/ec2nodeclass", "karpenter.k8s.aws/instance-encryption-in-transit-supported", "karpenter.k8s.aws/instance-category", "karpenter.k8s.aws/instance-hypervisor", "karpenter.k8s.aws/instance-family", "karpenter.k8s.aws/instance-generation", "karpenter.k8s.aws/instance-local-nvme", "karpenter.k8s.aws/instance-size", "karpenter.k8s.aws/instance-cpu", "karpenter.k8s.aws/instance-cpu-manufacturer", "karpenter.k8s.aws/instance-cpu-sustained-clock-speed-mhz", "karpenter.k8s.aws/instance-memory", "karpenter.k8s.aws/instance-ebs-bandwidth", "karpenter.k8s.aws/instance-network-bandwidth", "karpenter.k8s.aws/instance-gpu-name", "karpenter.k8s.aws/instance-gpu-manufacturer", "karpenter.k8s.aws/instance-gpu-count", "karpenter.k8s.aws/instance-gpu-memory", "karpenter.k8s.aws/instance-accelerator-name", "karpenter.k8s.aws/instance-accelerator-manufacturer", "karpenter.k8s.aws/instance-accelerator-count"] || !self.find("^([^/]+)").endsWith("karpenter.k8s.aws")
                       minValues:
                         description: |-
                           This field is ALPHA and can be dropped or replaced at any time

diff --git a/charts/karpenter-crd/templates/karpenter.sh_nodepools.yaml b/charts/karpenter-crd/templates/karpenter.sh_nodepools.yaml
@@ -210,7 +210,7 @@ spec:
                             - message: label "kubernetes.io/hostname" is restricted
                               rule: self.all(x, x != "kubernetes.io/hostname")
                             - message: label domain "karpenter.k8s.aws" is restricted
-                              rule: self.all(x, x in ["karpenter.k8s.aws/ec2nodeclass", "karpenter.k8s.aws/instance-encryption-in-transit-supported", "karpenter.k8s.aws/instance-category", "karpenter.k8s.aws/instance-hypervisor", "karpenter.k8s.aws/instance-family", "karpenter.k8s.aws/instance-generation", "karpenter.k8s.aws/instance-local-nvme", "karpenter.k8s.aws/instance-size", "karpenter.k8s.aws/instance-cpu", "karpenter.k8s.aws/instance-cpu-manufacturer", "karpenter.k8s.aws/instance-cpu-sustained-clock-speed-mhz", "karpenter.k8s.aws/instance-memory", "karpenter.k8s.aws/instance-ebs-bandwidth", "karpenter.k8s.aws/instance-network-bandwidth", "karpenter.k8s.aws/instance-gpu-name", "karpenter.k8s.aws/instance-gpu-manufacturer", "karpenter.k8s.aws/instance-gpu-count", "karpenter.k8s.aws/instance-gpu-memory", "karpenter.k8s.aws/instance-accelerator-name", "karpenter.k8s.aws/instance-accelerator-manufacturer", "karpenter.k8s.aws/instance-accelerator-count"] || !x.find("^([^/]+)").endsWith("karpenter.k8s.aws"))
+                              rule: self.all(x, x in ["karpenter.k8s.aws/capacity-reservation-id", "karpenter.k8s.aws/ec2nodeclass", "karpenter.k8s.aws/instance-encryption-in-transit-supported", "karpenter.k8s.aws/instance-category", "karpenter.k8s.aws/instance-hypervisor", "karpenter.k8s.aws/instance-family", "karpenter.k8s.aws/instance-generation", "karpenter.k8s.aws/instance-local-nvme", "karpenter.k8s.aws/instance-size", "karpenter.k8s.aws/instance-cpu", "karpenter.k8s.aws/instance-cpu-manufacturer", "karpenter.k8s.aws/instance-cpu-sustained-clock-speed-mhz", "karpenter.k8s.aws/instance-memory", "karpenter.k8s.aws/instance-ebs-bandwidth", "karpenter.k8s.aws/instance-network-bandwidth", "karpenter.k8s.aws/instance-gpu-name", "karpenter.k8s.aws/instance-gpu-manufacturer", "karpenter.k8s.aws/instance-gpu-count", "karpenter.k8s.aws/instance-gpu-memory", "karpenter.k8s.aws/instance-accelerator-name", "karpenter.k8s.aws/instance-accelerator-manufacturer", "karpenter.k8s.aws/instance-accelerator-count"] || !x.find("^([^/]+)").endsWith("karpenter.k8s.aws"))
                       type: object
                     spec:
                       description: |-
@@ -283,7 +283,7 @@ spec:
                                   - message: label "kubernetes.io/hostname" is restricted
                                     rule: self != "kubernetes.io/hostname"
                                   - message: label domain "karpenter.k8s.aws" is restricted
-                                    rule: self in ["karpenter.k8s.aws/ec2nodeclass", "karpenter.k8s.aws/instance-encryption-in-transit-supported", "karpenter.k8s.aws/instance-category", "karpenter.k8s.aws/instance-hypervisor", "karpenter.k8s.aws/instance-family", "karpenter.k8s.aws/instance-generation", "karpenter.k8s.aws/instance-local-nvme", "karpenter.k8s.aws/instance-size", "karpenter.k8s.aws/instance-cpu", "karpenter.k8s.aws/instance-cpu-manufacturer", "karpenter.k8s.aws/instance-cpu-sustained-clock-speed-mhz", "karpenter.k8s.aws/instance-memory", "karpenter.k8s.aws/instance-ebs-bandwidth", "karpenter.k8s.aws/instance-network-bandwidth", "karpenter.k8s.aws/instance-gpu-name", "karpenter.k8s.aws/instance-gpu-manufacturer", "karpenter.k8s.aws/instance-gpu-count", "karpenter.k8s.aws/instance-gpu-memory", "karpenter.k8s.aws/instance-accelerator-name", "karpenter.k8s.aws/instance-accelerator-manufacturer", "karpenter.k8s.aws/instance-accelerator-count"] || !self.find("^([^/]+)").endsWith("karpenter.k8s.aws")
+                                    rule: self in ["karpenter.k8s.aws/capacity-reservation-id", "karpenter.k8s.aws/ec2nodeclass", "karpenter.k8s.aws/instance-encryption-in-transit-supported", "karpenter.k8s.aws/instance-category", "karpenter.k8s.aws/instance-hypervisor", "karpenter.k8s.aws/instance-family", "karpenter.k8s.aws/instance-generation", "karpenter.k8s.aws/instance-local-nvme", "karpenter.k8s.aws/instance-size", "karpenter.k8s.aws/instance-cpu", "karpenter.k8s.aws/instance-cpu-manufacturer", "karpenter.k8s.aws/instance-cpu-sustained-clock-speed-mhz", "karpenter.k8s.aws/instance-memory", "karpenter.k8s.aws/instance-ebs-bandwidth", "karpenter.k8s.aws/instance-network-bandwidth", "karpenter.k8s.aws/instance-gpu-name", "karpenter.k8s.aws/instance-gpu-manufacturer", "karpenter.k8s.aws/instance-gpu-count", "karpenter.k8s.aws/instance-gpu-memory", "karpenter.k8s.aws/instance-accelerator-name", "karpenter.k8s.aws/instance-accelerator-manufacturer", "karpenter.k8s.aws/instance-accelerator-count"] || !self.find("^([^/]+)").endsWith("karpenter.k8s.aws")
                               minValues:
                                 description: |-
                                   This field is ALPHA and can be dropped or replaced at any time

diff --git a/cmd/controller/main.go b/cmd/controller/main.go
@@ -69,6 +69,7 @@ func main() {
 			op.LaunchTemplateProvider,
 			op.VersionProvider,
 			op.InstanceTypesProvider,
+			op.CapacityReservationProvider,
 		)...).
 		Start(ctx)
 }
diff --git a/hack/docs/instancetypes_gen/main.go b/hack/docs/instancetypes_gen/main.go
@@ -134,15 +134,15 @@ below are the resources available with some assumptions and after the instance o
 			cache.New(awscache.DiscoveredCapacityCacheTTL, awscache.DefaultCleanupInterval),
 			ec2api,
 			subnetProvider,
+			pricing.NewDefaultProvider(
+				ctx,
+				pricing.NewAPI(cfg),
+				ec2api,
+				cfg.Region,
+			),
+			awscache.NewUnavailableOfferings(),
 			instancetype.NewDefaultResolver(
 				region,
-				pricing.NewDefaultProvider(
-					ctx,
-					pricing.NewAPI(cfg),
-					ec2api,
-					cfg.Region,
-				),
-				awscache.NewUnavailableOfferings(),
 			),
 		)
 		if err = instanceTypeProvider.UpdateInstanceTypes(ctx); err != nil {

diff --git a/hack/tools/launchtemplate_counter/main.go b/hack/tools/launchtemplate_counter/main.go
@@ -61,15 +61,15 @@ func main() {
 		cache.New(awscache.DiscoveredCapacityCacheTTL, awscache.DefaultCleanupInterval),
 		ec2api,
 		subnetProvider,
+		pricing.NewDefaultProvider(
+			ctx,
+			pricing.NewAPI(cfg),
+			ec2api,
+			cfg.Region,
+		),
+		awscache.NewUnavailableOfferings(),
 		instancetype.NewDefaultResolver(
 			region,
-			pricing.NewDefaultProvider(
-				ctx,
-				pricing.NewAPI(cfg),
-				ec2api,
-				cfg.Region,
-			),
-			awscache.NewUnavailableOfferings(),
 		),
 	)
 	if err := instanceTypeProvider.UpdateInstanceTypes(ctx); err != nil {

diff --git a/hack/validation/labels.sh b/hack/validation/labels.sh
@@ -2,7 +2,7 @@
 
 function injectDomainLabelRestrictions() {
     domain=$1
-	rule="self.all(x, x in [\"${domain}/ec2nodeclass\", \"${domain}/instance-encryption-in-transit-supported\", \"${domain}/instance-category\", \"${domain}/instance-hypervisor\", \"${domain}/instance-family\", \"${domain}/instance-generation\", \"${domain}/instance-local-nvme\", \"${domain}/instance-size\", \"${domain}/instance-cpu\", \"${domain}/instance-cpu-manufacturer\", \"${domain}/instance-cpu-sustained-clock-speed-mhz\", \"${domain}/instance-memory\", \"${domain}/instance-ebs-bandwidth\", \"${domain}/instance-network-bandwidth\", \"${domain}/instance-gpu-name\", \"${domain}/instance-gpu-manufacturer\", \"${domain}/instance-gpu-count\", \"${domain}/instance-gpu-memory\", \"${domain}/instance-accelerator-name\", \"${domain}/instance-accelerator-manufacturer\", \"${domain}/instance-accelerator-count\"] || !x.find(\"^([^/]+)\").endsWith(\"${domain}\"))"
+	rule="self.all(x, x in [\"${domain}/capacity-reservation-id\", \"${domain}/ec2nodeclass\", \"${domain}/instance-encryption-in-transit-supported\", \"${domain}/instance-category\", \"${domain}/instance-hypervisor\", \"${domain}/instance-family\", \"${domain}/instance-generation\", \"${domain}/instance-local-nvme\", \"${domain}/instance-size\", \"${domain}/instance-cpu\", \"${domain}/instance-cpu-manufacturer\", \"${domain}/instance-cpu-sustained-clock-speed-mhz\", \"${domain}/instance-memory\", \"${domain}/instance-ebs-bandwidth\", \"${domain}/instance-network-bandwidth\", \"${domain}/instance-gpu-name\", \"${domain}/instance-gpu-manufacturer\", \"${domain}/instance-gpu-count\", \"${domain}/instance-gpu-memory\", \"${domain}/instance-accelerator-name\", \"${domain}/instance-accelerator-manufacturer\", \"${domain}/instance-accelerator-count\"] || !x.find(\"^([^/]+)\").endsWith(\"${domain}\"))"
     message="label domain \"${domain}\" is restricted"
     MSG="${message}" RULE="${rule}" yq eval '.spec.versions[0].schema.openAPIV3Schema.properties.spec.properties.template.properties.metadata.properties.labels.x-kubernetes-validations += [{"message": strenv(MSG), "rule": strenv(RULE)}]' -i pkg/apis/crds/karpenter.sh_nodepools.yaml
 }