From e624f03e869efc52668e119357c889449998a036 Mon Sep 17 00:00:00 2001 From: Mikkel Oscar Lyderik Larsen Date: Wed, 12 Feb 2025 15:58:15 +0100 Subject: [PATCH] Update to Karpenter v1.1.1 Signed-off-by: Mikkel Oscar Lyderik Larsen --- cluster/manifests/deletions.yaml | 13 +- cluster/manifests/z-karpenter/02-role.yaml | 27 - .../manifests/z-karpenter/03-rolebinding.yaml | 18 - .../z-karpenter/05-clusterrole-core.yaml | 14 +- .../07-karpenter.k8s.aws_ec2nodeclasses.yaml | 578 +---------------- .../08-karpenter.sh_nodeclaims.yaml | 458 +------------- .../09-karpenter.sh_nodepools.yaml | 595 +----------------- cluster/manifests/z-karpenter/deployment.yaml | 16 +- .../z-karpenter/secret-webhook-cert.yaml | 11 - .../worker-karpenter/provisioners.yaml | 4 - 10 files changed, 48 insertions(+), 1686 deletions(-) delete mode 100644 cluster/manifests/z-karpenter/secret-webhook-cert.yaml diff --git a/cluster/manifests/deletions.yaml b/cluster/manifests/deletions.yaml index 2aaade5a26..9e152ac594 100644 --- a/cluster/manifests/deletions.yaml +++ b/cluster/manifests/deletions.yaml @@ -132,6 +132,16 @@ post_apply: namespace: kubenurse kind: Service {{- end }} +# Remove karpenter webhook related resources which are not used since v1.1.1 +- name: karpenter-lease + kind: Role + namespace: kube-node-lease +- name: karpenter-lease + kind: RoleBinding + namespace: kube-node-lease +- name: karpenter-cert + kind: Secret + namespace: kube-system {{ if eq .Cluster.ConfigItems.karpenter_pools_enabled "false" }} - name: provisioners.karpenter.sh kind: CustomResourceDefinition @@ -391,6 +401,3 @@ post_apply: kind: ServiceAccount namespace: kube-system {{- end}} -- name: karpenter-nodeclaim-migrator - kind: CronJob - namespace: kube-system diff --git a/cluster/manifests/z-karpenter/02-role.yaml b/cluster/manifests/z-karpenter/02-role.yaml index 7199ad1a19..c82eef9f2d 100644 --- a/cluster/manifests/z-karpenter/02-role.yaml +++ b/cluster/manifests/z-karpenter/02-role.yaml @@ -15,15 +15,7 @@ rules: - apiGroups: ["coordination.k8s.io"] resources: ["leases"] verbs: ["get", "watch"] - - apiGroups: [""] - resources: ["configmaps", "secrets"] - verbs: ["get", "list", "watch"] # Write - - apiGroups: [""] - resources: ["secrets"] - verbs: ["update"] - resourceNames: - - "karpenter-cert" - apiGroups: ["coordination.k8s.io"] resources: ["leases"] verbs: ["patch", "update"] @@ -50,23 +42,4 @@ rules: resources: ["services"] resourceNames: ["kube-dns"] verbs: ["get"] ---- -# Source: karpenter/templates/role.yaml -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: karpenter-lease - namespace: kube-node-lease - labels: - application: kubernetes - component: karpenter -rules: - # Read - - apiGroups: ["coordination.k8s.io"] - resources: ["leases"] - verbs: ["get", "list", "watch"] - # Write - - apiGroups: ["coordination.k8s.io"] - resources: ["leases"] - verbs: ["delete"] {{end}} diff --git a/cluster/manifests/z-karpenter/03-rolebinding.yaml b/cluster/manifests/z-karpenter/03-rolebinding.yaml index 5904a04019..e5d818dc46 100644 --- a/cluster/manifests/z-karpenter/03-rolebinding.yaml +++ b/cluster/manifests/z-karpenter/03-rolebinding.yaml @@ -35,22 +35,4 @@ subjects: - kind: ServiceAccount name: karpenter namespace: kube-system ---- -# Source: karpenter/templates/rolebinding.yaml -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: karpenter-lease - namespace: kube-node-lease - labels: - application: kubernetes - component: karpenter -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: karpenter-lease -subjects: - - kind: ServiceAccount - name: karpenter - namespace: kube-system {{end}} diff --git a/cluster/manifests/z-karpenter/05-clusterrole-core.yaml b/cluster/manifests/z-karpenter/05-clusterrole-core.yaml index b5ec4e4d7b..367d897f30 100644 --- a/cluster/manifests/z-karpenter/05-clusterrole-core.yaml +++ b/cluster/manifests/z-karpenter/05-clusterrole-core.yaml @@ -22,12 +22,12 @@ rules: - apiGroups: ["apps"] resources: ["daemonsets", "deployments", "replicasets", "statefulsets"] verbs: ["list", "watch"] - - apiGroups: ["apiextensions.k8s.io"] - resources: ["customresourcedefinitions"] - verbs: ["get", "watch", "list"] - apiGroups: ["policy"] resources: ["poddisruptionbudgets"] verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["get", list, "watch"] # Write - apiGroups: ["karpenter.sh"] resources: ["nodeclaims", "nodeclaims/status"] @@ -47,14 +47,6 @@ rules: - apiGroups: [""] resources: ["pods"] verbs: ["delete"] - - apiGroups: ["apiextensions.k8s.io"] - resources: ["customresourcedefinitions/status"] - resourceNames: ["ec2nodeclasses.karpenter.k8s.aws", "nodepools.karpenter.sh", "nodeclaims.karpenter.sh"] - verbs: ["patch"] - - apiGroups: ["apiextensions.k8s.io"] - resources: ["customresourcedefinitions"] - resourceNames: ["ec2nodeclasses.karpenter.k8s.aws", "nodepools.karpenter.sh", "nodeclaims.karpenter.sh"] - verbs: ["update"] --- # Source: karpenter/templates/clusterrole-core.yaml apiVersion: rbac.authorization.k8s.io/v1 diff --git a/cluster/manifests/z-karpenter/07-karpenter.k8s.aws_ec2nodeclasses.yaml b/cluster/manifests/z-karpenter/07-karpenter.k8s.aws_ec2nodeclasses.yaml index 779ec8b9d2..c1ac454841 100644 --- a/cluster/manifests/z-karpenter/07-karpenter.k8s.aws_ec2nodeclasses.yaml +++ b/cluster/manifests/z-karpenter/07-karpenter.k8s.aws_ec2nodeclasses.yaml @@ -116,7 +116,7 @@ spec: additionalProperties: type: string description: |- - Tags is a map of key/value tags used to select subnets + Tags is a map of key/value tags used to select amis. Specifying '*' for a value selects all values for a given tag key. maxProperties: 20 type: object @@ -486,7 +486,7 @@ spec: additionalProperties: type: string description: |- - Tags is a map of key/value tags used to select subnets + Tags is a map of key/value tags used to select security groups. Specifying '*' for a value selects all values for a given tag key. maxProperties: 20 type: object @@ -593,6 +593,9 @@ spec: items: description: AMI contains resolved AMI selector values utilized for node launch properties: + deprecated: + description: Deprecation status of the AMI + type: boolean id: description: ID of the AMI type: string @@ -696,7 +699,7 @@ spec: type: string securityGroups: description: |- - SecurityGroups contains the current Security Groups values that are available to the + SecurityGroups contains the current security group values that are available to the cluster under the SecurityGroups selectors. items: description: SecurityGroup contains resolved SecurityGroup selector values utilized for node launch @@ -713,7 +716,7 @@ spec: type: array subnets: description: |- - Subnets contains the current Subnet values that are available to the + Subnets contains the current subnet values that are available to the cluster under the subnet selectors. items: description: Subnet contains resolved Subnet selector values utilized for node launch @@ -738,571 +741,4 @@ spec: storage: true subresources: status: {} - - name: v1beta1 - schema: - openAPIV3Schema: - description: EC2NodeClass is the Schema for the EC2NodeClass API - properties: - apiVersion: - description: |- - APIVersion defines the versioned schema of this representation of an object. - Servers should convert recognized schemas to the latest internal value, and - may reject unrecognized values. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources - type: string - kind: - description: |- - Kind is a string value representing the REST resource this object represents. - Servers may infer this from the endpoint the client submits requests to. - Cannot be updated. - In CamelCase. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds - type: string - metadata: - type: object - spec: - description: |- - EC2NodeClassSpec is the top level specification for the AWS Karpenter Provider. - This will contain configuration necessary to launch instances in AWS. - properties: - amiFamily: - description: AMIFamily is the AMI family that instances use. - enum: - - AL2 - - AL2023 - - Bottlerocket - - Ubuntu - - Custom - - Windows2019 - - Windows2022 - type: string - amiSelectorTerms: - description: AMISelectorTerms is a list of or ami selector terms. The terms are ORed. - items: - description: |- - AMISelectorTerm defines selection logic for an ami used by Karpenter to launch nodes. - If multiple fields are used for selection, the requirements are ANDed. - properties: - id: - description: ID is the ami id in EC2 - pattern: ami-[0-9a-z]+ - type: string - name: - description: |- - Name is the ami name in EC2. - This value is the name field, which is different from the name tag. - type: string - owner: - description: |- - Owner is the owner for the ami. - You can specify a combination of AWS account IDs, "self", "amazon", and "aws-marketplace" - type: string - tags: - additionalProperties: - type: string - description: |- - Tags is a map of key/value tags used to select subnets - Specifying '*' for a value selects all values for a given tag key. - maxProperties: 20 - type: object - x-kubernetes-validations: - - message: empty tag keys or values aren't supported - rule: self.all(k, k != '' && self[k] != '') - type: object - maxItems: 30 - type: array - x-kubernetes-validations: - - message: expected at least one, got none, ['tags', 'id', 'name'] - rule: self.all(x, has(x.tags) || has(x.id) || has(x.name)) - - message: '''id'' is mutually exclusive, cannot be set with a combination of other fields in amiSelectorTerms' - rule: '!self.all(x, has(x.id) && (has(x.tags) || has(x.name) || has(x.owner)))' - associatePublicIPAddress: - description: AssociatePublicIPAddress controls if public IP addresses are assigned to instances that are launched with the nodeclass. - type: boolean - blockDeviceMappings: - description: BlockDeviceMappings to be applied to provisioned nodes. - items: - properties: - deviceName: - description: The device name (for example, /dev/sdh or xvdh). - type: string - ebs: - description: EBS contains parameters used to automatically set up EBS volumes when an instance is launched. - properties: - deleteOnTermination: - description: DeleteOnTermination indicates whether the EBS volume is deleted on instance termination. - type: boolean - encrypted: - description: |- - Encrypted indicates whether the EBS volume is encrypted. Encrypted volumes can only - be attached to instances that support Amazon EBS encryption. If you are creating - a volume from a snapshot, you can't specify an encryption value. - type: boolean - iops: - description: |- - IOPS is the number of I/O operations per second (IOPS). For gp3, io1, and io2 volumes, - this represents the number of IOPS that are provisioned for the volume. For - gp2 volumes, this represents the baseline performance of the volume and the - rate at which the volume accumulates I/O credits for bursting. - - The following are the supported values for each volume type: - - * gp3: 3,000-16,000 IOPS - - * io1: 100-64,000 IOPS - - * io2: 100-64,000 IOPS - - For io1 and io2 volumes, we guarantee 64,000 IOPS only for Instances built - on the Nitro System (https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/instance-types.html#ec2-nitro-instances). - Other instance families guarantee performance up to 32,000 IOPS. - - This parameter is supported for io1, io2, and gp3 volumes only. This parameter - is not supported for gp2, st1, sc1, or standard volumes. - format: int64 - type: integer - kmsKeyID: - description: KMSKeyID (ARN) of the symmetric Key Management Service (KMS) CMK used for encryption. - type: string - snapshotID: - description: SnapshotID is the ID of an EBS snapshot - type: string - throughput: - description: |- - Throughput to provision for a gp3 volume, with a maximum of 1,000 MiB/s. - Valid Range: Minimum value of 125. Maximum value of 1000. - format: int64 - type: integer - volumeSize: - description: |- - VolumeSize in `Gi`, `G`, `Ti`, or `T`. You must specify either a snapshot ID or - a volume size. The following are the supported volumes sizes for each volume - type: - - * gp2 and gp3: 1-16,384 - - * io1 and io2: 4-16,384 - - * st1 and sc1: 125-16,384 - - * standard: 1-1,024 - pattern: ^((?:[1-9][0-9]{0,3}|[1-4][0-9]{4}|[5][0-8][0-9]{3}|59000)Gi|(?:[1-9][0-9]{0,3}|[1-5][0-9]{4}|[6][0-3][0-9]{3}|64000)G|([1-9]||[1-5][0-7]|58)Ti|([1-9]||[1-5][0-9]|6[0-3]|64)T)$ - type: string - volumeType: - description: |- - VolumeType of the block device. - For more information, see Amazon EBS volume types (https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/EBSVolumeTypes.html) - in the Amazon Elastic Compute Cloud User Guide. - enum: - - standard - - io1 - - io2 - - gp2 - - sc1 - - st1 - - gp3 - type: string - type: object - x-kubernetes-validations: - - message: snapshotID or volumeSize must be defined - rule: has(self.snapshotID) || has(self.volumeSize) - rootVolume: - description: |- - RootVolume is a flag indicating if this device is mounted as kubelet root dir. You can - configure at most one root volume in BlockDeviceMappings. - type: boolean - type: object - maxItems: 50 - type: array - x-kubernetes-validations: - - message: must have only one blockDeviceMappings with rootVolume - rule: self.filter(x, has(x.rootVolume)?x.rootVolume==true:false).size() <= 1 - context: - description: |- - Context is a Reserved field in EC2 APIs - https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_CreateFleet.html - type: string - detailedMonitoring: - description: DetailedMonitoring controls if detailed monitoring is enabled for instances that are launched - type: boolean - instanceProfile: - description: |- - InstanceProfile is the AWS entity that instances use. - This field is mutually exclusive from role. - The instance profile should already have a role assigned to it that Karpenter - has PassRole permission on for instance launch using this instanceProfile to succeed. - type: string - x-kubernetes-validations: - - message: instanceProfile cannot be empty - rule: self != '' - instanceStorePolicy: - description: InstanceStorePolicy specifies how to handle instance-store disks. - enum: - - RAID0 - type: string - metadataOptions: - default: - httpEndpoint: enabled - httpProtocolIPv6: disabled - httpPutResponseHopLimit: 1 - httpTokens: required - description: |- - MetadataOptions for the generated launch template of provisioned nodes. - - This specifies the exposure of the Instance Metadata Service to - provisioned EC2 nodes. For more information, - see Instance Metadata and User Data - (https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-metadata.html) - in the Amazon Elastic Compute Cloud User Guide. - - Refer to recommended, security best practices - (https://aws.github.io/aws-eks-best-practices/security/docs/iam/#restrict-access-to-the-instance-profile-assigned-to-the-worker-node) - for limiting exposure of Instance Metadata and User Data to pods. - If omitted, defaults to httpEndpoint enabled, with httpProtocolIPv6 - disabled, with httpPutResponseLimit of 1, and with httpTokens - required. - properties: - httpEndpoint: - default: enabled - description: |- - HTTPEndpoint enables or disables the HTTP metadata endpoint on provisioned - nodes. If metadata options is non-nil, but this parameter is not specified, - the default state is "enabled". - - If you specify a value of "disabled", instance metadata will not be accessible - on the node. - enum: - - enabled - - disabled - type: string - httpProtocolIPv6: - default: disabled - description: |- - HTTPProtocolIPv6 enables or disables the IPv6 endpoint for the instance metadata - service on provisioned nodes. If metadata options is non-nil, but this parameter - is not specified, the default state is "disabled". - enum: - - enabled - - disabled - type: string - httpPutResponseHopLimit: - default: 2 - description: |- - HTTPPutResponseHopLimit is the desired HTTP PUT response hop limit for - instance metadata requests. The larger the number, the further instance - metadata requests can travel. Possible values are integers from 1 to 64. - If metadata options is non-nil, but this parameter is not specified, the - default value is 2. - format: int64 - maximum: 64 - minimum: 1 - type: integer - httpTokens: - default: required - description: |- - HTTPTokens determines the state of token usage for instance metadata - requests. If metadata options is non-nil, but this parameter is not - specified, the default state is "required". - - If the state is optional, one can choose to retrieve instance metadata with - or without a signed token header on the request. If one retrieves the IAM - role credentials without a token, the version 1.0 role credentials are - returned. If one retrieves the IAM role credentials using a valid signed - token, the version 2.0 role credentials are returned. - - If the state is "required", one must send a signed token header with any - instance metadata retrieval requests. In this state, retrieving the IAM - role credentials always returns the version 2.0 credentials; the version - 1.0 credentials are not available. - enum: - - required - - optional - type: string - type: object - role: - description: |- - Role is the AWS identity that nodes use. This field is immutable. - This field is mutually exclusive from instanceProfile. - Marking this field as immutable avoids concerns around terminating managed instance profiles from running instances. - This field may be made mutable in the future, assuming the correct garbage collection and drift handling is implemented - for the old instance profiles on an update. - type: string - x-kubernetes-validations: - - message: role cannot be empty - rule: self != '' - - message: immutable field changed - rule: self == oldSelf - securityGroupSelectorTerms: - description: SecurityGroupSelectorTerms is a list of or security group selector terms. The terms are ORed. - items: - description: |- - SecurityGroupSelectorTerm defines selection logic for a security group used by Karpenter to launch nodes. - If multiple fields are used for selection, the requirements are ANDed. - properties: - id: - description: ID is the security group id in EC2 - pattern: sg-[0-9a-z]+ - type: string - name: - description: |- - Name is the security group name in EC2. - This value is the name field, which is different from the name tag. - type: string - tags: - additionalProperties: - type: string - description: |- - Tags is a map of key/value tags used to select subnets - Specifying '*' for a value selects all values for a given tag key. - maxProperties: 20 - type: object - x-kubernetes-validations: - - message: empty tag keys or values aren't supported - rule: self.all(k, k != '' && self[k] != '') - type: object - maxItems: 30 - type: array - x-kubernetes-validations: - - message: securityGroupSelectorTerms cannot be empty - rule: self.size() != 0 - - message: expected at least one, got none, ['tags', 'id', 'name'] - rule: self.all(x, has(x.tags) || has(x.id) || has(x.name)) - - message: '''id'' is mutually exclusive, cannot be set with a combination of other fields in securityGroupSelectorTerms' - rule: '!self.all(x, has(x.id) && (has(x.tags) || has(x.name)))' - - message: '''name'' is mutually exclusive, cannot be set with a combination of other fields in securityGroupSelectorTerms' - rule: '!self.all(x, has(x.name) && (has(x.tags) || has(x.id)))' - subnetSelectorTerms: - description: SubnetSelectorTerms is a list of or subnet selector terms. The terms are ORed. - items: - description: |- - SubnetSelectorTerm defines selection logic for a subnet used by Karpenter to launch nodes. - If multiple fields are used for selection, the requirements are ANDed. - properties: - id: - description: ID is the subnet id in EC2 - pattern: subnet-[0-9a-z]+ - type: string - tags: - additionalProperties: - type: string - description: |- - Tags is a map of key/value tags used to select subnets - Specifying '*' for a value selects all values for a given tag key. - maxProperties: 20 - type: object - x-kubernetes-validations: - - message: empty tag keys or values aren't supported - rule: self.all(k, k != '' && self[k] != '') - type: object - maxItems: 30 - type: array - x-kubernetes-validations: - - message: subnetSelectorTerms cannot be empty - rule: self.size() != 0 - - message: expected at least one, got none, ['tags', 'id'] - rule: self.all(x, has(x.tags) || has(x.id)) - - message: '''id'' is mutually exclusive, cannot be set with a combination of other fields in subnetSelectorTerms' - rule: '!self.all(x, has(x.id) && has(x.tags))' - tags: - additionalProperties: - type: string - description: Tags to be applied on ec2 resources like instances and launch templates. - type: object - x-kubernetes-validations: - - message: empty tag keys aren't supported - rule: self.all(k, k != '') - - message: tag contains a restricted tag matching kubernetes.io/cluster/ - rule: self.all(k, !k.startsWith('kubernetes.io/cluster') ) - - message: tag contains a restricted tag matching karpenter.sh/nodepool - rule: self.all(k, k != 'karpenter.sh/nodepool') - - message: tag contains a restricted tag matching karpenter.sh/managed-by - rule: self.all(k, k !='karpenter.sh/managed-by') - - message: tag contains a restricted tag matching karpenter.sh/nodeclaim - rule: self.all(k, k !='karpenter.sh/nodeclaim') - - message: tag contains a restricted tag matching karpenter.k8s.aws/ec2nodeclass - rule: self.all(k, k !='karpenter.k8s.aws/ec2nodeclass') - userData: - description: |- - UserData to be applied to the provisioned nodes. - It must be in the appropriate format based on the AMIFamily in use. Karpenter will merge certain fields into - this UserData to ensure nodes are being provisioned with the correct configuration. - type: string - required: - - amiFamily - - securityGroupSelectorTerms - - subnetSelectorTerms - type: object - x-kubernetes-validations: - - message: amiSelectorTerms is required when amiFamily == 'Custom' - rule: 'self.amiFamily == ''Custom'' ? self.amiSelectorTerms.size() != 0 : true' - - message: must specify exactly one of ['role', 'instanceProfile'] - rule: (has(self.role) && !has(self.instanceProfile)) || (!has(self.role) && has(self.instanceProfile)) - - message: changing from 'instanceProfile' to 'role' is not supported. You must delete and recreate this node class if you want to change this. - rule: (has(oldSelf.role) && has(self.role)) || (has(oldSelf.instanceProfile) && has(self.instanceProfile)) - status: - description: EC2NodeClassStatus contains the resolved state of the EC2NodeClass - properties: - amis: - description: |- - AMI contains the current AMI values that are available to the - cluster under the AMI selectors. - items: - description: AMI contains resolved AMI selector values utilized for node launch - properties: - id: - description: ID of the AMI - type: string - name: - description: Name of the AMI - type: string - requirements: - description: Requirements of the AMI to be utilized on an instance type - items: - description: |- - A node selector requirement is a selector that contains values, a key, and an operator - that relates the key and values. - properties: - key: - description: The label key that the selector applies to. - type: string - operator: - description: |- - Represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. - type: string - values: - description: |- - An array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. If the operator is Gt or Lt, the values - array must have a single element, which will be interpreted as an integer. - This array is replaced during a strategic merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - required: - - id - - requirements - type: object - type: array - conditions: - description: Conditions contains signals for health and readiness - items: - description: Condition aliases the upstream type and adds additional helper methods - properties: - lastTransitionTime: - description: |- - lastTransitionTime is the last time the condition transitioned from one status to another. - This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. - format: date-time - type: string - message: - description: |- - message is a human readable message indicating details about the transition. - This may be an empty string. - maxLength: 32768 - type: string - observedGeneration: - description: |- - observedGeneration represents the .metadata.generation that the condition was set based upon. - For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date - with respect to the current state of the instance. - format: int64 - minimum: 0 - type: integer - reason: - description: |- - reason contains a programmatic identifier indicating the reason for the condition's last transition. - Producers of specific condition types may define expected values and meanings for this field, - and whether the values are considered a guaranteed API. - The value should be a CamelCase string. - This field may not be empty. - maxLength: 1024 - minLength: 1 - pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ - type: string - status: - description: status of the condition, one of True, False, Unknown. - enum: - - "True" - - "False" - - Unknown - type: string - type: - description: type of condition in CamelCase or in foo.example.com/CamelCase. - maxLength: 316 - pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ - type: string - required: - - lastTransitionTime - - message - - reason - - status - - type - type: object - type: array - instanceProfile: - description: InstanceProfile contains the resolved instance profile for the role - type: string - securityGroups: - description: |- - SecurityGroups contains the current Security Groups values that are available to the - cluster under the SecurityGroups selectors. - items: - description: SecurityGroup contains resolved SecurityGroup selector values utilized for node launch - properties: - id: - description: ID of the security group - type: string - name: - description: Name of the security group - type: string - required: - - id - type: object - type: array - subnets: - description: |- - Subnets contains the current Subnet values that are available to the - cluster under the subnet selectors. - items: - description: Subnet contains resolved Subnet selector values utilized for node launch - properties: - id: - description: ID of the subnet - type: string - zone: - description: The associated availability zone - type: string - zoneID: - description: The associated availability zone ID - type: string - required: - - id - - zone - type: object - type: array - type: object - type: object - served: true - storage: false - subresources: - status: {} - conversion: - strategy: Webhook - webhook: - conversionReviewVersions: - - v1beta1 - - v1 - clientConfig: - service: - name: karpenter - namespace: kube-system - port: 8443 {{ end }} diff --git a/cluster/manifests/z-karpenter/08-karpenter.sh_nodeclaims.yaml b/cluster/manifests/z-karpenter/08-karpenter.sh_nodeclaims.yaml index c2291738e5..cb9b86f92a 100644 --- a/cluster/manifests/z-karpenter/08-karpenter.sh_nodeclaims.yaml +++ b/cluster/manifests/z-karpenter/08-karpenter.sh_nodeclaims.yaml @@ -89,12 +89,21 @@ spec: description: API version of the referent pattern: ^[^/]*$ type: string + x-kubernetes-validations: + - message: group may not be empty + rule: self != '' kind: description: 'Kind of the referent; More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds"' type: string + x-kubernetes-validations: + - message: kind may not be empty + rule: self != '' name: description: 'Name of the referent; More info: http://kubernetes.io/docs/user-guide/identifiers#names' type: string + x-kubernetes-validations: + - message: name may not be empty + rule: self != '' required: - group - kind @@ -122,7 +131,7 @@ spec: - message: label "kubernetes.io/hostname" is restricted rule: self != "kubernetes.io/hostname" - message: label domain "karpenter.k8s.aws" is restricted - rule: self in ["karpenter.k8s.aws/instance-encryption-in-transit-supported", "karpenter.k8s.aws/instance-category", "karpenter.k8s.aws/instance-hypervisor", "karpenter.k8s.aws/instance-family", "karpenter.k8s.aws/instance-generation", "karpenter.k8s.aws/instance-local-nvme", "karpenter.k8s.aws/instance-size", "karpenter.k8s.aws/instance-cpu","karpenter.k8s.aws/instance-cpu-manufacturer","karpenter.k8s.aws/instance-memory", "karpenter.k8s.aws/instance-ebs-bandwidth", "karpenter.k8s.aws/instance-network-bandwidth", "karpenter.k8s.aws/instance-gpu-name", "karpenter.k8s.aws/instance-gpu-manufacturer", "karpenter.k8s.aws/instance-gpu-count", "karpenter.k8s.aws/instance-gpu-memory", "karpenter.k8s.aws/instance-accelerator-name", "karpenter.k8s.aws/instance-accelerator-manufacturer", "karpenter.k8s.aws/instance-accelerator-count"] || !self.find("^([^/]+)").endsWith("karpenter.k8s.aws") + rule: self in ["karpenter.k8s.aws/ec2nodeclass", "karpenter.k8s.aws/instance-encryption-in-transit-supported", "karpenter.k8s.aws/instance-category", "karpenter.k8s.aws/instance-hypervisor", "karpenter.k8s.aws/instance-family", "karpenter.k8s.aws/instance-generation", "karpenter.k8s.aws/instance-local-nvme", "karpenter.k8s.aws/instance-size", "karpenter.k8s.aws/instance-cpu", "karpenter.k8s.aws/instance-cpu-manufacturer", "karpenter.k8s.aws/instance-cpu-sustained-clock-speed-mhz", "karpenter.k8s.aws/instance-memory", "karpenter.k8s.aws/instance-ebs-bandwidth", "karpenter.k8s.aws/instance-network-bandwidth", "karpenter.k8s.aws/instance-gpu-name", "karpenter.k8s.aws/instance-gpu-manufacturer", "karpenter.k8s.aws/instance-gpu-count", "karpenter.k8s.aws/instance-gpu-memory", "karpenter.k8s.aws/instance-accelerator-name", "karpenter.k8s.aws/instance-accelerator-manufacturer", "karpenter.k8s.aws/instance-accelerator-count"] || !self.find("^([^/]+)").endsWith("karpenter.k8s.aws") minValues: description: |- This field is ALPHA and can be dropped or replaced at any time @@ -381,451 +390,4 @@ spec: storage: true subresources: status: {} - - additionalPrinterColumns: - - jsonPath: .metadata.labels.node\.kubernetes\.io/instance-type - name: Type - type: string - - jsonPath: .metadata.labels.topology\.kubernetes\.io/zone - name: Zone - type: string - - jsonPath: .status.nodeName - name: Node - type: string - - jsonPath: .status.conditions[?(@.type=="Ready")].status - name: Ready - type: string - - jsonPath: .metadata.creationTimestamp - name: Age - type: date - - jsonPath: .metadata.labels.karpenter\.sh/capacity-type - name: Capacity - priority: 1 - type: string - - jsonPath: .metadata.labels.karpenter\.sh/nodepool - name: NodePool - priority: 1 - type: string - - jsonPath: .spec.nodeClassRef.name - name: NodeClass - priority: 1 - type: string - name: v1beta1 - schema: - openAPIV3Schema: - description: NodeClaim is the Schema for the NodeClaims API - properties: - apiVersion: - description: |- - APIVersion defines the versioned schema of this representation of an object. - Servers should convert recognized schemas to the latest internal value, and - may reject unrecognized values. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources - type: string - kind: - description: |- - Kind is a string value representing the REST resource this object represents. - Servers may infer this from the endpoint the client submits requests to. - Cannot be updated. - In CamelCase. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds - type: string - metadata: - type: object - spec: - description: NodeClaimSpec describes the desired state of the NodeClaim - properties: - kubelet: - description: |- - Kubelet defines args to be used when configuring kubelet on provisioned nodes. - They are a subset of the upstream types, recognizing not all options may be supported. - Wherever possible, the types and names should reflect the upstream kubelet types. - properties: - clusterDNS: - description: |- - clusterDNS is a list of IP addresses for the cluster DNS server. - Note that not all providers may use all addresses. - items: - type: string - type: array - cpuCFSQuota: - description: CPUCFSQuota enables CPU CFS quota enforcement for containers that specify CPU limits. - type: boolean - evictionHard: - additionalProperties: - type: string - pattern: ^((\d{1,2}(\.\d{1,2})?|100(\.0{1,2})?)%||(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?)$ - description: EvictionHard is the map of signal names to quantities that define hard eviction thresholds - type: object - x-kubernetes-validations: - - message: valid keys for evictionHard are ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available'] - rule: self.all(x, x in ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available']) - evictionMaxPodGracePeriod: - description: |- - EvictionMaxPodGracePeriod is the maximum allowed grace period (in seconds) to use when terminating pods in - response to soft eviction thresholds being met. - format: int32 - type: integer - evictionSoft: - additionalProperties: - type: string - pattern: ^((\d{1,2}(\.\d{1,2})?|100(\.0{1,2})?)%||(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?)$ - description: EvictionSoft is the map of signal names to quantities that define soft eviction thresholds - type: object - x-kubernetes-validations: - - message: valid keys for evictionSoft are ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available'] - rule: self.all(x, x in ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available']) - evictionSoftGracePeriod: - additionalProperties: - type: string - description: EvictionSoftGracePeriod is the map of signal names to quantities that define grace periods for each eviction signal - type: object - x-kubernetes-validations: - - message: valid keys for evictionSoftGracePeriod are ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available'] - rule: self.all(x, x in ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available']) - imageGCHighThresholdPercent: - description: |- - ImageGCHighThresholdPercent is the percent of disk usage after which image - garbage collection is always run. The percent is calculated by dividing this - field value by 100, so this field must be between 0 and 100, inclusive. - When specified, the value must be greater than ImageGCLowThresholdPercent. - format: int32 - maximum: 100 - minimum: 0 - type: integer - imageGCLowThresholdPercent: - description: |- - ImageGCLowThresholdPercent is the percent of disk usage before which image - garbage collection is never run. Lowest disk usage to garbage collect to. - The percent is calculated by dividing this field value by 100, - so the field value must be between 0 and 100, inclusive. - When specified, the value must be less than imageGCHighThresholdPercent - format: int32 - maximum: 100 - minimum: 0 - type: integer - kubeReserved: - additionalProperties: - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - description: KubeReserved contains resources reserved for Kubernetes system components. - type: object - x-kubernetes-validations: - - message: valid keys for kubeReserved are ['cpu','memory','ephemeral-storage','pid'] - rule: self.all(x, x=='cpu' || x=='memory' || x=='ephemeral-storage' || x=='pid') - - message: kubeReserved value cannot be a negative resource quantity - rule: self.all(x, !self[x].startsWith('-')) - maxPods: - description: |- - MaxPods is an override for the maximum number of pods that can run on - a worker node instance. - format: int32 - minimum: 0 - type: integer - podsPerCore: - description: |- - PodsPerCore is an override for the number of pods that can run on a worker node - instance based on the number of cpu cores. This value cannot exceed MaxPods, so, if - MaxPods is a lower value, that value will be used. - format: int32 - minimum: 0 - type: integer - systemReserved: - additionalProperties: - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - description: SystemReserved contains resources reserved for OS system daemons and kernel memory. - type: object - x-kubernetes-validations: - - message: valid keys for systemReserved are ['cpu','memory','ephemeral-storage','pid'] - rule: self.all(x, x=='cpu' || x=='memory' || x=='ephemeral-storage' || x=='pid') - - message: systemReserved value cannot be a negative resource quantity - rule: self.all(x, !self[x].startsWith('-')) - type: object - x-kubernetes-validations: - - message: imageGCHighThresholdPercent must be greater than imageGCLowThresholdPercent - rule: 'has(self.imageGCHighThresholdPercent) && has(self.imageGCLowThresholdPercent) ? self.imageGCHighThresholdPercent > self.imageGCLowThresholdPercent : true' - - message: evictionSoft OwnerKey does not have a matching evictionSoftGracePeriod - rule: has(self.evictionSoft) ? self.evictionSoft.all(e, (e in self.evictionSoftGracePeriod)):true - - message: evictionSoftGracePeriod OwnerKey does not have a matching evictionSoft - rule: has(self.evictionSoftGracePeriod) ? self.evictionSoftGracePeriod.all(e, (e in self.evictionSoft)):true - nodeClassRef: - description: NodeClassRef is a reference to an object that defines provider specific configuration - properties: - apiVersion: - description: API version of the referent - type: string - kind: - description: 'Kind of the referent; More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds"' - type: string - name: - description: 'Name of the referent; More info: http://kubernetes.io/docs/user-guide/identifiers#names' - type: string - required: - - name - type: object - requirements: - description: Requirements are layered with GetLabels and applied to every node. - items: - description: |- - A node selector requirement with min values is a selector that contains values, a key, an operator that relates the key and values - and minValues that represent the requirement to have at least that many values. - properties: - key: - description: The label key that the selector applies to. - type: string - maxLength: 316 - pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ - x-kubernetes-validations: - - message: label domain "kubernetes.io" is restricted - rule: self in ["beta.kubernetes.io/instance-type", "failure-domain.beta.kubernetes.io/region", "beta.kubernetes.io/os", "beta.kubernetes.io/arch", "failure-domain.beta.kubernetes.io/zone", "topology.kubernetes.io/zone", "topology.kubernetes.io/region", "node.kubernetes.io/instance-type", "kubernetes.io/arch", "kubernetes.io/os", "node.kubernetes.io/windows-build"] || self.find("^([^/]+)").endsWith("node.kubernetes.io") || self.find("^([^/]+)").endsWith("node-restriction.kubernetes.io") || !self.find("^([^/]+)").endsWith("kubernetes.io") - - message: label domain "k8s.io" is restricted - rule: self.find("^([^/]+)").endsWith("kops.k8s.io") || !self.find("^([^/]+)").endsWith("k8s.io") - - message: label domain "karpenter.sh" is restricted - rule: self in ["karpenter.sh/capacity-type", "karpenter.sh/nodepool"] || !self.find("^([^/]+)").endsWith("karpenter.sh") - - message: label "kubernetes.io/hostname" is restricted - rule: self != "kubernetes.io/hostname" - - message: label domain "karpenter.k8s.aws" is restricted - rule: self in ["karpenter.k8s.aws/instance-encryption-in-transit-supported", "karpenter.k8s.aws/instance-category", "karpenter.k8s.aws/instance-hypervisor", "karpenter.k8s.aws/instance-family", "karpenter.k8s.aws/instance-generation", "karpenter.k8s.aws/instance-local-nvme", "karpenter.k8s.aws/instance-size", "karpenter.k8s.aws/instance-cpu","karpenter.k8s.aws/instance-cpu-manufacturer","karpenter.k8s.aws/instance-memory", "karpenter.k8s.aws/instance-ebs-bandwidth", "karpenter.k8s.aws/instance-network-bandwidth", "karpenter.k8s.aws/instance-gpu-name", "karpenter.k8s.aws/instance-gpu-manufacturer", "karpenter.k8s.aws/instance-gpu-count", "karpenter.k8s.aws/instance-gpu-memory", "karpenter.k8s.aws/instance-accelerator-name", "karpenter.k8s.aws/instance-accelerator-manufacturer", "karpenter.k8s.aws/instance-accelerator-count"] || !self.find("^([^/]+)").endsWith("karpenter.k8s.aws") - minValues: - description: |- - This field is ALPHA and can be dropped or replaced at any time - MinValues is the minimum number of unique values required to define the flexibility of the specific requirement. - maximum: 50 - minimum: 1 - type: integer - operator: - description: |- - Represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. - type: string - enum: - - In - - NotIn - - Exists - - DoesNotExist - - Gt - - Lt - values: - description: |- - An array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. If the operator is Gt or Lt, the values - array must have a single element, which will be interpreted as an integer. - This array is replaced during a strategic merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - maxLength: 63 - pattern: ^(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])?$ - required: - - key - - operator - type: object - maxItems: 100 - type: array - x-kubernetes-validations: - - message: requirements with operator 'In' must have a value defined - rule: 'self.all(x, x.operator == ''In'' ? x.values.size() != 0 : true)' - - message: requirements operator 'Gt' or 'Lt' must have a single positive integer value - rule: 'self.all(x, (x.operator == ''Gt'' || x.operator == ''Lt'') ? (x.values.size() == 1 && int(x.values[0]) >= 0) : true)' - - message: requirements with 'minValues' must have at least that many values specified in the 'values' field - rule: 'self.all(x, (x.operator == ''In'' && has(x.minValues)) ? x.values.size() >= x.minValues : true)' - resources: - description: Resources models the resource requirements for the NodeClaim to launch - properties: - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: Requests describes the minimum required resources for the NodeClaim to launch - type: object - type: object - startupTaints: - description: |- - StartupTaints are taints that are applied to nodes upon startup which are expected to be removed automatically - within a short period of time, typically by a DaemonSet that tolerates the taint. These are commonly used by - daemonsets to allow initialization and enforce startup ordering. StartupTaints are ignored for provisioning - purposes in that pods are not required to tolerate a StartupTaint in order to have nodes provisioned for them. - items: - description: |- - The node this Taint is attached to has the "effect" on - any pod that does not tolerate the Taint. - properties: - effect: - description: |- - Required. The effect of the taint on pods - that do not tolerate the taint. - Valid effects are NoSchedule, PreferNoSchedule and NoExecute. - type: string - enum: - - NoSchedule - - PreferNoSchedule - - NoExecute - key: - description: Required. The taint key to be applied to a node. - type: string - minLength: 1 - pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ - timeAdded: - description: |- - TimeAdded represents the time at which the taint was added. - It is only written for NoExecute taints. - format: date-time - type: string - value: - description: The taint value corresponding to the taint key. - type: string - pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ - required: - - effect - - key - type: object - type: array - taints: - description: Taints will be applied to the NodeClaim's node. - items: - description: |- - The node this Taint is attached to has the "effect" on - any pod that does not tolerate the Taint. - properties: - effect: - description: |- - Required. The effect of the taint on pods - that do not tolerate the taint. - Valid effects are NoSchedule, PreferNoSchedule and NoExecute. - type: string - enum: - - NoSchedule - - PreferNoSchedule - - NoExecute - key: - description: Required. The taint key to be applied to a node. - type: string - minLength: 1 - pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ - timeAdded: - description: |- - TimeAdded represents the time at which the taint was added. - It is only written for NoExecute taints. - format: date-time - type: string - value: - description: The taint value corresponding to the taint key. - type: string - pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ - required: - - effect - - key - type: object - type: array - required: - - nodeClassRef - - requirements - type: object - status: - description: NodeClaimStatus defines the observed state of NodeClaim - properties: - allocatable: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: Allocatable is the estimated allocatable capacity of the node - type: object - capacity: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: Capacity is the estimated full capacity of the node - type: object - conditions: - description: Conditions contains signals for health and readiness - items: - description: Condition aliases the upstream type and adds additional helper methods - properties: - lastTransitionTime: - description: |- - lastTransitionTime is the last time the condition transitioned from one status to another. - This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. - format: date-time - type: string - message: - description: |- - message is a human readable message indicating details about the transition. - This may be an empty string. - maxLength: 32768 - type: string - observedGeneration: - description: |- - observedGeneration represents the .metadata.generation that the condition was set based upon. - For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date - with respect to the current state of the instance. - format: int64 - minimum: 0 - type: integer - reason: - description: |- - reason contains a programmatic identifier indicating the reason for the condition's last transition. - Producers of specific condition types may define expected values and meanings for this field, - and whether the values are considered a guaranteed API. - The value should be a CamelCase string. - This field may not be empty. - maxLength: 1024 - pattern: ^([A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?|)$ - type: string - status: - description: status of the condition, one of True, False, Unknown. - enum: - - "True" - - "False" - - Unknown - type: string - type: - description: type of condition in CamelCase or in foo.example.com/CamelCase. - maxLength: 316 - pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ - type: string - required: - - lastTransitionTime - - status - - type - type: object - type: array - imageID: - description: ImageID is an identifier for the image that runs on the node - type: string - nodeName: - description: NodeName is the name of the corresponding node object - type: string - providerID: - description: ProviderID of the corresponding node object - type: string - type: object - required: - - spec - type: object - served: true - storage: false - subresources: - status: {} - conversion: - strategy: Webhook - webhook: - conversionReviewVersions: - - v1beta1 - - v1 - clientConfig: - service: - name: karpenter - namespace: kube-system - port: 8443 {{ end }} diff --git a/cluster/manifests/z-karpenter/09-karpenter.sh_nodepools.yaml b/cluster/manifests/z-karpenter/09-karpenter.sh_nodepools.yaml index 96f676dcf9..5c2f72da09 100644 --- a/cluster/manifests/z-karpenter/09-karpenter.sh_nodepools.yaml +++ b/cluster/manifests/z-karpenter/09-karpenter.sh_nodepools.yaml @@ -208,7 +208,7 @@ spec: - message: label "kubernetes.io/hostname" is restricted rule: self.all(x, x != "kubernetes.io/hostname") - message: label domain "karpenter.k8s.aws" is restricted - rule: self.all(x, x in ["karpenter.k8s.aws/instance-encryption-in-transit-supported", "karpenter.k8s.aws/instance-category", "karpenter.k8s.aws/instance-hypervisor", "karpenter.k8s.aws/instance-family", "karpenter.k8s.aws/instance-generation", "karpenter.k8s.aws/instance-local-nvme", "karpenter.k8s.aws/instance-size", "karpenter.k8s.aws/instance-cpu","karpenter.k8s.aws/instance-cpu-manufacturer","karpenter.k8s.aws/instance-memory", "karpenter.k8s.aws/instance-ebs-bandwidth", "karpenter.k8s.aws/instance-network-bandwidth", "karpenter.k8s.aws/instance-gpu-name", "karpenter.k8s.aws/instance-gpu-manufacturer", "karpenter.k8s.aws/instance-gpu-count", "karpenter.k8s.aws/instance-gpu-memory", "karpenter.k8s.aws/instance-accelerator-name", "karpenter.k8s.aws/instance-accelerator-manufacturer", "karpenter.k8s.aws/instance-accelerator-count"] || !x.find("^([^/]+)").endsWith("karpenter.k8s.aws")) + rule: self.all(x, x in ["karpenter.k8s.aws/ec2nodeclass", "karpenter.k8s.aws/instance-encryption-in-transit-supported", "karpenter.k8s.aws/instance-category", "karpenter.k8s.aws/instance-hypervisor", "karpenter.k8s.aws/instance-family", "karpenter.k8s.aws/instance-generation", "karpenter.k8s.aws/instance-local-nvme", "karpenter.k8s.aws/instance-size", "karpenter.k8s.aws/instance-cpu", "karpenter.k8s.aws/instance-cpu-manufacturer", "karpenter.k8s.aws/instance-cpu-sustained-clock-speed-mhz", "karpenter.k8s.aws/instance-memory", "karpenter.k8s.aws/instance-ebs-bandwidth", "karpenter.k8s.aws/instance-network-bandwidth", "karpenter.k8s.aws/instance-gpu-name", "karpenter.k8s.aws/instance-gpu-manufacturer", "karpenter.k8s.aws/instance-gpu-count", "karpenter.k8s.aws/instance-gpu-memory", "karpenter.k8s.aws/instance-accelerator-name", "karpenter.k8s.aws/instance-accelerator-manufacturer", "karpenter.k8s.aws/instance-accelerator-count"] || !x.find("^([^/]+)").endsWith("karpenter.k8s.aws")) type: object spec: description: |- @@ -232,17 +232,31 @@ spec: description: API version of the referent pattern: ^[^/]*$ type: string + x-kubernetes-validations: + - message: group may not be empty + rule: self != '' kind: description: 'Kind of the referent; More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds"' type: string + x-kubernetes-validations: + - message: kind may not be empty + rule: self != '' name: description: 'Name of the referent; More info: http://kubernetes.io/docs/user-guide/identifiers#names' type: string + x-kubernetes-validations: + - message: name may not be empty + rule: self != '' required: - group - kind - name type: object + x-kubernetes-validations: + - message: nodeClassRef.group is immutable + rule: self.group == oldSelf.group + - message: nodeClassRef.kind is immutable + rule: self.kind == oldSelf.kind requirements: description: Requirements are layered with GetLabels and applied to every node. items: @@ -267,7 +281,7 @@ spec: - message: label "kubernetes.io/hostname" is restricted rule: self != "kubernetes.io/hostname" - message: label domain "karpenter.k8s.aws" is restricted - rule: self in ["karpenter.k8s.aws/instance-encryption-in-transit-supported", "karpenter.k8s.aws/instance-category", "karpenter.k8s.aws/instance-hypervisor", "karpenter.k8s.aws/instance-family", "karpenter.k8s.aws/instance-generation", "karpenter.k8s.aws/instance-local-nvme", "karpenter.k8s.aws/instance-size", "karpenter.k8s.aws/instance-cpu","karpenter.k8s.aws/instance-cpu-manufacturer","karpenter.k8s.aws/instance-memory", "karpenter.k8s.aws/instance-ebs-bandwidth", "karpenter.k8s.aws/instance-network-bandwidth", "karpenter.k8s.aws/instance-gpu-name", "karpenter.k8s.aws/instance-gpu-manufacturer", "karpenter.k8s.aws/instance-gpu-count", "karpenter.k8s.aws/instance-gpu-memory", "karpenter.k8s.aws/instance-accelerator-name", "karpenter.k8s.aws/instance-accelerator-manufacturer", "karpenter.k8s.aws/instance-accelerator-count"] || !self.find("^([^/]+)").endsWith("karpenter.k8s.aws") + rule: self in ["karpenter.k8s.aws/ec2nodeclass", "karpenter.k8s.aws/instance-encryption-in-transit-supported", "karpenter.k8s.aws/instance-category", "karpenter.k8s.aws/instance-hypervisor", "karpenter.k8s.aws/instance-family", "karpenter.k8s.aws/instance-generation", "karpenter.k8s.aws/instance-local-nvme", "karpenter.k8s.aws/instance-size", "karpenter.k8s.aws/instance-cpu", "karpenter.k8s.aws/instance-cpu-manufacturer", "karpenter.k8s.aws/instance-cpu-sustained-clock-speed-mhz", "karpenter.k8s.aws/instance-memory", "karpenter.k8s.aws/instance-ebs-bandwidth", "karpenter.k8s.aws/instance-network-bandwidth", "karpenter.k8s.aws/instance-gpu-name", "karpenter.k8s.aws/instance-gpu-manufacturer", "karpenter.k8s.aws/instance-gpu-count", "karpenter.k8s.aws/instance-gpu-memory", "karpenter.k8s.aws/instance-accelerator-name", "karpenter.k8s.aws/instance-accelerator-manufacturer", "karpenter.k8s.aws/instance-accelerator-count"] || !self.find("^([^/]+)").endsWith("karpenter.k8s.aws") minValues: description: |- This field is ALPHA and can be dropped or replaced at any time @@ -504,581 +518,4 @@ spec: storage: true subresources: status: {} - - additionalPrinterColumns: - - jsonPath: .spec.template.spec.nodeClassRef.name - name: NodeClass - type: string - - jsonPath: .spec.weight - name: Weight - priority: 1 - type: string - name: v1beta1 - schema: - openAPIV3Schema: - description: NodePool is the Schema for the NodePools API - properties: - apiVersion: - description: |- - APIVersion defines the versioned schema of this representation of an object. - Servers should convert recognized schemas to the latest internal value, and - may reject unrecognized values. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources - type: string - kind: - description: |- - Kind is a string value representing the REST resource this object represents. - Servers may infer this from the endpoint the client submits requests to. - Cannot be updated. - In CamelCase. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds - type: string - metadata: - type: object - spec: - description: |- - NodePoolSpec is the top level nodepool specification. Nodepools - launch nodes in response to pods that are unschedulable. A single nodepool - is capable of managing a diverse set of nodes. Node properties are determined - from a combination of nodepool and pod scheduling constraints. - properties: - disruption: - default: - consolidationPolicy: WhenUnderutilized - expireAfter: 720h - description: Disruption contains the parameters that relate to Karpenter's disruption logic - properties: - budgets: - default: - - nodes: 10% - description: |- - Budgets is a list of Budgets. - If there are multiple active budgets, Karpenter uses - the most restrictive value. If left undefined, - this will default to one budget with a value to 10%. - items: - description: |- - Budget defines when Karpenter will restrict the - number of Node Claims that can be terminating simultaneously. - properties: - duration: - description: |- - Duration determines how long a Budget is active since each Schedule hit. - Only minutes and hours are accepted, as cron does not work in seconds. - If omitted, the budget is always active. - This is required if Schedule is set. - This regex has an optional 0s at the end since the duration.String() always adds - a 0s at the end. - pattern: ^((([0-9]+(h|m))|([0-9]+h[0-9]+m))(0s)?)$ - type: string - nodes: - default: 10% - description: |- - Nodes dictates the maximum number of NodeClaims owned by this NodePool - that can be terminating at once. This is calculated by counting nodes that - have a deletion timestamp set, or are actively being deleted by Karpenter. - This field is required when specifying a budget. - This cannot be of type intstr.IntOrString since kubebuilder doesn't support pattern - checking for int nodes for IntOrString nodes. - Ref: https://github.com/kubernetes-sigs/controller-tools/blob/55efe4be40394a288216dab63156b0a64fb82929/pkg/crd/markers/validation.go#L379-L388 - pattern: ^((100|[0-9]{1,2})%|[0-9]+)$ - type: string - schedule: - description: |- - Schedule specifies when a budget begins being active, following - the upstream cronjob syntax. If omitted, the budget is always active. - Timezones are not supported. - This field is required if Duration is set. - pattern: ^(@(annually|yearly|monthly|weekly|daily|midnight|hourly))|((.+)\s(.+)\s(.+)\s(.+)\s(.+))$ - type: string - required: - - nodes - type: object - maxItems: 50 - type: array - x-kubernetes-validations: - - message: '''schedule'' must be set with ''duration''' - rule: self.all(x, has(x.schedule) == has(x.duration)) - consolidateAfter: - description: |- - ConsolidateAfter is the duration the controller will wait - before attempting to terminate nodes that are underutilized. - Refer to ConsolidationPolicy for how underutilization is considered. - pattern: ^(([0-9]+(s|m|h))+)|(Never)$ - type: string - consolidationPolicy: - default: WhenUnderutilized - description: |- - ConsolidationPolicy describes which nodes Karpenter can disrupt through its consolidation - algorithm. This policy defaults to "WhenUnderutilized" if not specified - enum: - - WhenEmpty - - WhenUnderutilized - type: string - expireAfter: - default: 720h - description: |- - ExpireAfter is the duration the controller will wait - before terminating a node, measured from when the node is created. This - is useful to implement features like eventually consistent node upgrade, - memory leak protection, and disruption testing. - pattern: ^(([0-9]+(s|m|h))+)|(Never)$ - type: string - type: object - x-kubernetes-validations: - - message: consolidateAfter cannot be combined with consolidationPolicy=WhenUnderutilized - rule: 'has(self.consolidateAfter) ? self.consolidationPolicy != ''WhenUnderutilized'' || self.consolidateAfter == ''Never'' : true' - - message: consolidateAfter must be specified with consolidationPolicy=WhenEmpty - rule: 'self.consolidationPolicy == ''WhenEmpty'' ? has(self.consolidateAfter) : true' - limits: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: Limits define a set of bounds for provisioning capacity. - type: object - template: - description: |- - Template contains the template of possibilities for the provisioning logic to launch a NodeClaim with. - NodeClaims launched from this NodePool will often be further constrained than the template specifies. - properties: - metadata: - properties: - annotations: - additionalProperties: - type: string - description: |- - Annotations is an unstructured key value map stored with a resource that may be - set by external tools to store and retrieve arbitrary metadata. They are not - queryable and should be preserved when modifying objects. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations - type: object - labels: - additionalProperties: - type: string - maxLength: 63 - pattern: ^(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])?$ - description: |- - Map of string keys and values that can be used to organize and categorize - (scope and select) objects. May match selectors of replication controllers - and services. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels - type: object - maxProperties: 100 - x-kubernetes-validations: - - message: label domain "kubernetes.io" is restricted - rule: self.all(x, x in ["beta.kubernetes.io/instance-type", "failure-domain.beta.kubernetes.io/region", "beta.kubernetes.io/os", "beta.kubernetes.io/arch", "failure-domain.beta.kubernetes.io/zone", "topology.kubernetes.io/zone", "topology.kubernetes.io/region", "kubernetes.io/arch", "kubernetes.io/os", "node.kubernetes.io/windows-build"] || x.find("^([^/]+)").endsWith("node.kubernetes.io") || x.find("^([^/]+)").endsWith("node-restriction.kubernetes.io") || !x.find("^([^/]+)").endsWith("kubernetes.io")) - - message: label domain "k8s.io" is restricted - rule: self.all(x, x.find("^([^/]+)").endsWith("kops.k8s.io") || !x.find("^([^/]+)").endsWith("k8s.io")) - - message: label domain "karpenter.sh" is restricted - rule: self.all(x, x in ["karpenter.sh/capacity-type", "karpenter.sh/nodepool"] || !x.find("^([^/]+)").endsWith("karpenter.sh")) - - message: label "karpenter.sh/nodepool" is restricted - rule: self.all(x, x != "karpenter.sh/nodepool") - - message: label "kubernetes.io/hostname" is restricted - rule: self.all(x, x != "kubernetes.io/hostname") - - message: label domain "karpenter.k8s.aws" is restricted - rule: self.all(x, x in ["karpenter.k8s.aws/instance-encryption-in-transit-supported", "karpenter.k8s.aws/instance-category", "karpenter.k8s.aws/instance-hypervisor", "karpenter.k8s.aws/instance-family", "karpenter.k8s.aws/instance-generation", "karpenter.k8s.aws/instance-local-nvme", "karpenter.k8s.aws/instance-size", "karpenter.k8s.aws/instance-cpu","karpenter.k8s.aws/instance-cpu-manufacturer","karpenter.k8s.aws/instance-memory", "karpenter.k8s.aws/instance-ebs-bandwidth", "karpenter.k8s.aws/instance-network-bandwidth", "karpenter.k8s.aws/instance-gpu-name", "karpenter.k8s.aws/instance-gpu-manufacturer", "karpenter.k8s.aws/instance-gpu-count", "karpenter.k8s.aws/instance-gpu-memory", "karpenter.k8s.aws/instance-accelerator-name", "karpenter.k8s.aws/instance-accelerator-manufacturer", "karpenter.k8s.aws/instance-accelerator-count"] || !x.find("^([^/]+)").endsWith("karpenter.k8s.aws")) - type: object - spec: - description: NodeClaimSpec describes the desired state of the NodeClaim - properties: - kubelet: - description: |- - Kubelet defines args to be used when configuring kubelet on provisioned nodes. - They are a subset of the upstream types, recognizing not all options may be supported. - Wherever possible, the types and names should reflect the upstream kubelet types. - properties: - clusterDNS: - description: |- - clusterDNS is a list of IP addresses for the cluster DNS server. - Note that not all providers may use all addresses. - items: - type: string - type: array - cpuCFSQuota: - description: CPUCFSQuota enables CPU CFS quota enforcement for containers that specify CPU limits. - type: boolean - evictionHard: - additionalProperties: - type: string - pattern: ^((\d{1,2}(\.\d{1,2})?|100(\.0{1,2})?)%||(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?)$ - description: EvictionHard is the map of signal names to quantities that define hard eviction thresholds - type: object - x-kubernetes-validations: - - message: valid keys for evictionHard are ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available'] - rule: self.all(x, x in ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available']) - evictionMaxPodGracePeriod: - description: |- - EvictionMaxPodGracePeriod is the maximum allowed grace period (in seconds) to use when terminating pods in - response to soft eviction thresholds being met. - format: int32 - type: integer - evictionSoft: - additionalProperties: - type: string - pattern: ^((\d{1,2}(\.\d{1,2})?|100(\.0{1,2})?)%||(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?)$ - description: EvictionSoft is the map of signal names to quantities that define soft eviction thresholds - type: object - x-kubernetes-validations: - - message: valid keys for evictionSoft are ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available'] - rule: self.all(x, x in ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available']) - evictionSoftGracePeriod: - additionalProperties: - type: string - description: EvictionSoftGracePeriod is the map of signal names to quantities that define grace periods for each eviction signal - type: object - x-kubernetes-validations: - - message: valid keys for evictionSoftGracePeriod are ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available'] - rule: self.all(x, x in ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available']) - imageGCHighThresholdPercent: - description: |- - ImageGCHighThresholdPercent is the percent of disk usage after which image - garbage collection is always run. The percent is calculated by dividing this - field value by 100, so this field must be between 0 and 100, inclusive. - When specified, the value must be greater than ImageGCLowThresholdPercent. - format: int32 - maximum: 100 - minimum: 0 - type: integer - imageGCLowThresholdPercent: - description: |- - ImageGCLowThresholdPercent is the percent of disk usage before which image - garbage collection is never run. Lowest disk usage to garbage collect to. - The percent is calculated by dividing this field value by 100, - so the field value must be between 0 and 100, inclusive. - When specified, the value must be less than imageGCHighThresholdPercent - format: int32 - maximum: 100 - minimum: 0 - type: integer - kubeReserved: - additionalProperties: - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - description: KubeReserved contains resources reserved for Kubernetes system components. - type: object - x-kubernetes-validations: - - message: valid keys for kubeReserved are ['cpu','memory','ephemeral-storage','pid'] - rule: self.all(x, x=='cpu' || x=='memory' || x=='ephemeral-storage' || x=='pid') - - message: kubeReserved value cannot be a negative resource quantity - rule: self.all(x, !self[x].startsWith('-')) - maxPods: - description: |- - MaxPods is an override for the maximum number of pods that can run on - a worker node instance. - format: int32 - minimum: 0 - type: integer - podsPerCore: - description: |- - PodsPerCore is an override for the number of pods that can run on a worker node - instance based on the number of cpu cores. This value cannot exceed MaxPods, so, if - MaxPods is a lower value, that value will be used. - format: int32 - minimum: 0 - type: integer - systemReserved: - additionalProperties: - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - description: SystemReserved contains resources reserved for OS system daemons and kernel memory. - type: object - x-kubernetes-validations: - - message: valid keys for systemReserved are ['cpu','memory','ephemeral-storage','pid'] - rule: self.all(x, x=='cpu' || x=='memory' || x=='ephemeral-storage' || x=='pid') - - message: systemReserved value cannot be a negative resource quantity - rule: self.all(x, !self[x].startsWith('-')) - type: object - x-kubernetes-validations: - - message: imageGCHighThresholdPercent must be greater than imageGCLowThresholdPercent - rule: 'has(self.imageGCHighThresholdPercent) && has(self.imageGCLowThresholdPercent) ? self.imageGCHighThresholdPercent > self.imageGCLowThresholdPercent : true' - - message: evictionSoft OwnerKey does not have a matching evictionSoftGracePeriod - rule: has(self.evictionSoft) ? self.evictionSoft.all(e, (e in self.evictionSoftGracePeriod)):true - - message: evictionSoftGracePeriod OwnerKey does not have a matching evictionSoft - rule: has(self.evictionSoftGracePeriod) ? self.evictionSoftGracePeriod.all(e, (e in self.evictionSoft)):true - nodeClassRef: - description: NodeClassRef is a reference to an object that defines provider specific configuration - properties: - apiVersion: - description: API version of the referent - type: string - kind: - description: 'Kind of the referent; More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds"' - type: string - name: - description: 'Name of the referent; More info: http://kubernetes.io/docs/user-guide/identifiers#names' - type: string - required: - - name - type: object - requirements: - description: Requirements are layered with GetLabels and applied to every node. - items: - description: |- - A node selector requirement with min values is a selector that contains values, a key, an operator that relates the key and values - and minValues that represent the requirement to have at least that many values. - properties: - key: - description: The label key that the selector applies to. - type: string - maxLength: 316 - pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ - x-kubernetes-validations: - - message: label domain "kubernetes.io" is restricted - rule: self in ["beta.kubernetes.io/instance-type", "failure-domain.beta.kubernetes.io/region", "beta.kubernetes.io/os", "beta.kubernetes.io/arch", "failure-domain.beta.kubernetes.io/zone", "topology.kubernetes.io/zone", "topology.kubernetes.io/region", "node.kubernetes.io/instance-type", "kubernetes.io/arch", "kubernetes.io/os", "node.kubernetes.io/windows-build"] || self.find("^([^/]+)").endsWith("node.kubernetes.io") || self.find("^([^/]+)").endsWith("node-restriction.kubernetes.io") || !self.find("^([^/]+)").endsWith("kubernetes.io") - - message: label domain "k8s.io" is restricted - rule: self.find("^([^/]+)").endsWith("kops.k8s.io") || !self.find("^([^/]+)").endsWith("k8s.io") - - message: label domain "karpenter.sh" is restricted - rule: self in ["karpenter.sh/capacity-type", "karpenter.sh/nodepool"] || !self.find("^([^/]+)").endsWith("karpenter.sh") - - message: label "karpenter.sh/nodepool" is restricted - rule: self != "karpenter.sh/nodepool" - - message: label "kubernetes.io/hostname" is restricted - rule: self != "kubernetes.io/hostname" - - message: label domain "karpenter.k8s.aws" is restricted - rule: self in ["karpenter.k8s.aws/instance-encryption-in-transit-supported", "karpenter.k8s.aws/instance-category", "karpenter.k8s.aws/instance-hypervisor", "karpenter.k8s.aws/instance-family", "karpenter.k8s.aws/instance-generation", "karpenter.k8s.aws/instance-local-nvme", "karpenter.k8s.aws/instance-size", "karpenter.k8s.aws/instance-cpu","karpenter.k8s.aws/instance-cpu-manufacturer","karpenter.k8s.aws/instance-memory", "karpenter.k8s.aws/instance-ebs-bandwidth", "karpenter.k8s.aws/instance-network-bandwidth", "karpenter.k8s.aws/instance-gpu-name", "karpenter.k8s.aws/instance-gpu-manufacturer", "karpenter.k8s.aws/instance-gpu-count", "karpenter.k8s.aws/instance-gpu-memory", "karpenter.k8s.aws/instance-accelerator-name", "karpenter.k8s.aws/instance-accelerator-manufacturer", "karpenter.k8s.aws/instance-accelerator-count"] || !self.find("^([^/]+)").endsWith("karpenter.k8s.aws") - minValues: - description: |- - This field is ALPHA and can be dropped or replaced at any time - MinValues is the minimum number of unique values required to define the flexibility of the specific requirement. - maximum: 50 - minimum: 1 - type: integer - operator: - description: |- - Represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. - type: string - enum: - - In - - NotIn - - Exists - - DoesNotExist - - Gt - - Lt - values: - description: |- - An array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. If the operator is Gt or Lt, the values - array must have a single element, which will be interpreted as an integer. - This array is replaced during a strategic merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - maxLength: 63 - pattern: ^(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])?$ - required: - - key - - operator - type: object - maxItems: 100 - type: array - x-kubernetes-validations: - - message: requirements with operator 'In' must have a value defined - rule: 'self.all(x, x.operator == ''In'' ? x.values.size() != 0 : true)' - - message: requirements operator 'Gt' or 'Lt' must have a single positive integer value - rule: 'self.all(x, (x.operator == ''Gt'' || x.operator == ''Lt'') ? (x.values.size() == 1 && int(x.values[0]) >= 0) : true)' - - message: requirements with 'minValues' must have at least that many values specified in the 'values' field - rule: 'self.all(x, (x.operator == ''In'' && has(x.minValues)) ? x.values.size() >= x.minValues : true)' - resources: - description: Resources models the resource requirements for the NodeClaim to launch - properties: - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: Requests describes the minimum required resources for the NodeClaim to launch - type: object - type: object - maxProperties: 0 - startupTaints: - description: |- - StartupTaints are taints that are applied to nodes upon startup which are expected to be removed automatically - within a short period of time, typically by a DaemonSet that tolerates the taint. These are commonly used by - daemonsets to allow initialization and enforce startup ordering. StartupTaints are ignored for provisioning - purposes in that pods are not required to tolerate a StartupTaint in order to have nodes provisioned for them. - items: - description: |- - The node this Taint is attached to has the "effect" on - any pod that does not tolerate the Taint. - properties: - effect: - description: |- - Required. The effect of the taint on pods - that do not tolerate the taint. - Valid effects are NoSchedule, PreferNoSchedule and NoExecute. - type: string - enum: - - NoSchedule - - PreferNoSchedule - - NoExecute - key: - description: Required. The taint key to be applied to a node. - type: string - minLength: 1 - pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ - timeAdded: - description: |- - TimeAdded represents the time at which the taint was added. - It is only written for NoExecute taints. - format: date-time - type: string - value: - description: The taint value corresponding to the taint key. - type: string - pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ - required: - - effect - - key - type: object - type: array - taints: - description: Taints will be applied to the NodeClaim's node. - items: - description: |- - The node this Taint is attached to has the "effect" on - any pod that does not tolerate the Taint. - properties: - effect: - description: |- - Required. The effect of the taint on pods - that do not tolerate the taint. - Valid effects are NoSchedule, PreferNoSchedule and NoExecute. - type: string - enum: - - NoSchedule - - PreferNoSchedule - - NoExecute - key: - description: Required. The taint key to be applied to a node. - type: string - minLength: 1 - pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ - timeAdded: - description: |- - TimeAdded represents the time at which the taint was added. - It is only written for NoExecute taints. - format: date-time - type: string - value: - description: The taint value corresponding to the taint key. - type: string - pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ - required: - - effect - - key - type: object - type: array - required: - - nodeClassRef - - requirements - type: object - required: - - spec - type: object - weight: - description: |- - Weight is the priority given to the nodepool during scheduling. A higher - numerical weight indicates that this nodepool will be ordered - ahead of other nodepools with lower weights. A nodepool with no weight - will be treated as if it is a nodepool with a weight of 0. - format: int32 - maximum: 100 - minimum: 1 - type: integer - required: - - template - type: object - status: - description: NodePoolStatus defines the observed state of NodePool - properties: - conditions: - description: Conditions contains signals for health and readiness - items: - description: Condition aliases the upstream type and adds additional helper methods - properties: - lastTransitionTime: - description: |- - lastTransitionTime is the last time the condition transitioned from one status to another. - This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. - format: date-time - type: string - message: - description: |- - message is a human readable message indicating details about the transition. - This may be an empty string. - maxLength: 32768 - type: string - observedGeneration: - description: |- - observedGeneration represents the .metadata.generation that the condition was set based upon. - For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date - with respect to the current state of the instance. - format: int64 - minimum: 0 - type: integer - reason: - description: |- - reason contains a programmatic identifier indicating the reason for the condition's last transition. - Producers of specific condition types may define expected values and meanings for this field, - and whether the values are considered a guaranteed API. - The value should be a CamelCase string. - This field may not be empty. - maxLength: 1024 - minLength: 1 - pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ - type: string - status: - description: status of the condition, one of True, False, Unknown. - enum: - - "True" - - "False" - - Unknown - type: string - type: - description: type of condition in CamelCase or in foo.example.com/CamelCase. - maxLength: 316 - pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ - type: string - required: - - lastTransitionTime - - message - - reason - - status - - type - type: object - type: array - resources: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: Resources is the list of resources that have been provisioned. - type: object - type: object - required: - - spec - type: object - served: true - storage: false - subresources: - status: {} - conversion: - strategy: Webhook - webhook: - conversionReviewVersions: - - v1beta1 - - v1 - clientConfig: - service: - name: karpenter - namespace: kube-system - port: 8443 {{ end }} diff --git a/cluster/manifests/z-karpenter/deployment.yaml b/cluster/manifests/z-karpenter/deployment.yaml index 5547df7616..98e556474d 100644 --- a/cluster/manifests/z-karpenter/deployment.yaml +++ b/cluster/manifests/z-karpenter/deployment.yaml @@ -51,9 +51,9 @@ spec: - ALL readOnlyRootFilesystem: true {{if eq .Cluster.ConfigItems.karpenter_version "current"}} - image: "container-registry-test.zalando.net/teapot/karpenter:1.0.5-pr-27-6.patched" + image: "container-registry-test.zalando.net/teapot/karpenter:1.1.1-pr-30-1.patched {{else if eq .Cluster.ConfigItems.karpenter_version "legacy"}} - image: "container-registry.zalando.net/teapot/karpenter:0.37.0-main-26.patched" + image: "container-registry-test.zalando.net/teapot/karpenter:1.0.5-pr-27-6.patched" {{end}} imagePullPolicy: IfNotPresent env: @@ -71,12 +71,6 @@ spec: value: "false" - name: KARPENTER_SERVICE value: karpenter - - name: WEBHOOK_PORT - value: "8443" - - name: WEBHOOK_METRICS_PORT - value: "8001" - - name: DISABLE_WEBHOOK - value: "false" - name: LOG_LEVEL value: {{ .Cluster.ConfigItems.karpenter_log_level }} - name: LOG_OUTPUT_PATHS @@ -118,12 +112,6 @@ spec: - name: http-metrics containerPort: 8000 protocol: TCP - - name: webhook-metrics - containerPort: 8001 - protocol: TCP - - name: https-webhook - containerPort: 8443 - protocol: TCP - name: http containerPort: 8081 protocol: TCP diff --git a/cluster/manifests/z-karpenter/secret-webhook-cert.yaml b/cluster/manifests/z-karpenter/secret-webhook-cert.yaml deleted file mode 100644 index c1f436abb7..0000000000 --- a/cluster/manifests/z-karpenter/secret-webhook-cert.yaml +++ /dev/null @@ -1,11 +0,0 @@ ---- -# Source: karpenter/templates/secret-webhook-cert.yaml -apiVersion: v1 -kind: Secret -metadata: - name: karpenter-cert - namespace: kube-system - labels: - application: kubernetes - component: karpenter -# data: {} # Injected by karpenter-webhook diff --git a/cluster/node-pools/worker-karpenter/provisioners.yaml b/cluster/node-pools/worker-karpenter/provisioners.yaml index 0bea5caecb..73296818d2 100644 --- a/cluster/node-pools/worker-karpenter/provisioners.yaml +++ b/cluster/node-pools/worker-karpenter/provisioners.yaml @@ -90,10 +90,6 @@ apiVersion: karpenter.sh/v1 kind: NodePool metadata: name: "{{.NodePool.Name}}" - annotations: - # reset the annotations as required by karpenter v1.1.1 - compatibility.karpenter.sh/v1beta1-kubelet-conversion: "" - compatibility.karpenter.sh/v1beta1-nodeclass-reference: "" spec: # Template section that describes how to template out NodeClaim resources that Karpenter will provision # Karpenter will consider this template to be the minimum requirements needed to provision a Node using this NodePool