From 9bf5c719af694f425014d8c80c558344effd98dc Mon Sep 17 00:00:00 2001 From: Erik Sundell Date: Wed, 15 May 2024 16:08:27 +0200 Subject: [PATCH 1/3] Update aws-node from 1.12.6 to 1.18.1 1.18.1 is recommended for EKS clusters, where its documented that "For all Kubernetes releases, we recommend installing the latest VPC CNI release." as read at https://github.com/aws/amazon-vpc-cni-k8s?tab=readme-ov-file#recommended-version. The latest available addon for various k8s minor versions are listed at https://docs.aws.amazon.com/eks/latest/userguide/managing-vpc-cni.html#updating-vpc-cni-add-on, and it currently sais 1.18.1 for k8s 1.23 to 1.29. --- pkg/addons/default/assets/aws-node.yaml | 338 +++++++++++++++++++++++- pkg/addons/default/aws_node_generate.go | 2 +- pkg/addons/default/aws_node_test.go | 8 +- 3 files changed, 329 insertions(+), 19 deletions(-) diff --git a/pkg/addons/default/assets/aws-node.yaml b/pkg/addons/default/assets/aws-node.yaml index 8dba481b47..a251eb4951 100644 --- a/pkg/addons/default/assets/aws-node.yaml +++ b/pkg/addons/default/assets/aws-node.yaml @@ -1,5 +1,5 @@ --- -# Source: crds/customresourcedefinition.yaml +# Source: aws-vpc-cni/crds/customresourcedefinition.yaml apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: @@ -20,7 +20,241 @@ spec: plural: eniconfigs singular: eniconfig kind: ENIConfig - +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.11.3 + creationTimestamp: null + labels: + app.kubernetes.io/name: amazon-network-policy-controller-k8s + name: policyendpoints.networking.k8s.aws +spec: + group: networking.k8s.aws + names: + kind: PolicyEndpoint + listKind: PolicyEndpointList + plural: policyendpoints + singular: policyendpoint + scope: Namespaced + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: PolicyEndpoint is the Schema for the policyendpoints API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: PolicyEndpointSpec defines the desired state of PolicyEndpoint + properties: + egress: + description: Egress is the list of egress rules containing resolved + network addresses + items: + description: EndpointInfo defines the network endpoint information + for the policy ingress/egress + properties: + cidr: + description: CIDR is the network address(s) of the endpoint + type: string + except: + description: Except is the exceptions to the CIDR ranges mentioned + above. + items: + type: string + type: array + ports: + description: Ports is the list of ports + items: + description: Port contains information about the transport + port/protocol + properties: + endPort: + description: Endport specifies the port range port to + endPort port must be defined and an integer, endPort + > port + format: int32 + type: integer + port: + description: Port specifies the numerical port for the + protocol. If empty applies to all ports + format: int32 + type: integer + protocol: + default: TCP + description: Protocol specifies the transport protocol, + default TCP + type: string + type: object + type: array + required: + - cidr + type: object + type: array + ingress: + description: Ingress is the list of ingress rules containing resolved + network addresses + items: + description: EndpointInfo defines the network endpoint information + for the policy ingress/egress + properties: + cidr: + description: CIDR is the network address(s) of the endpoint + type: string + except: + description: Except is the exceptions to the CIDR ranges mentioned + above. + items: + type: string + type: array + ports: + description: Ports is the list of ports + items: + description: Port contains information about the transport + port/protocol + properties: + endPort: + description: Endport specifies the port range port to + endPort port must be defined and an integer, endPort + > port + format: int32 + type: integer + port: + description: Port specifies the numerical port for the + protocol. If empty applies to all ports + format: int32 + type: integer + protocol: + default: TCP + description: Protocol specifies the transport protocol, + default TCP + type: string + type: object + type: array + required: + - cidr + type: object + type: array + podIsolation: + description: PodIsolation specifies whether the pod needs to be isolated + for a particular traffic direction Ingress or Egress, or both. If + default isolation is not specified, and there are no ingress/egress + rules, then the pod is not isolated from the point of view of this + policy. This follows the NetworkPolicy spec.PolicyTypes. + items: + description: PolicyType string describes the NetworkPolicy type + This type is beta-level in 1.8 + type: string + type: array + podSelector: + description: PodSelector is the podSelector from the policy resource + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. + The requirements are ANDed. + items: + description: A label selector requirement is a selector that + contains values, a key, and an operator that relates the key + and values. + properties: + key: + description: key is the label key that the selector applies + to. + type: string + operator: + description: operator represents a key's relationship to + a set of values. Valid operators are In, NotIn, Exists + and DoesNotExist. + type: string + values: + description: values is an array of string values. If the + operator is In or NotIn, the values array must be non-empty. + If the operator is Exists or DoesNotExist, the values + array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. A single + {key,value} in the matchLabels map is equivalent to an element + of matchExpressions, whose key field is "key", the operator + is "In", and the values array contains only "value". The requirements + are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + podSelectorEndpoints: + description: PodSelectorEndpoints contains information about the pods + matching the podSelector + items: + description: PodEndpoint defines the summary information for the + pods + properties: + hostIP: + description: HostIP is the IP address of the host the pod is + currently running on + type: string + name: + description: Name is the pod name + type: string + namespace: + description: Namespace is the pod namespace + type: string + podIP: + description: PodIP is the IP address of the pod + type: string + required: + - hostIP + - name + - namespace + - podIP + type: object + type: array + policyRef: + description: PolicyRef is a reference to the Kubernetes NetworkPolicy + resource. + properties: + name: + description: Name is the name of the Policy + type: string + namespace: + description: Namespace is the namespace of the Policy + type: string + required: + - name + - namespace + type: object + required: + - policyRef + type: object + status: + description: PolicyEndpointStatus defines the observed state of PolicyEndpoint + type: object + type: object + served: true + storage: true + subresources: + status: {} --- # Source: aws-vpc-cni/templates/serviceaccount.yaml apiVersion: v1 @@ -32,7 +266,27 @@ metadata: app.kubernetes.io/name: aws-node app.kubernetes.io/instance: aws-vpc-cni k8s-app: aws-node - app.kubernetes.io/version: "v1.12.6" + app.kubernetes.io/version: "v1.18.1" +--- +# Source: aws-vpc-cni/templates/configmap.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: amazon-vpc-cni + namespace: kube-system + labels: + app.kubernetes.io/name: aws-node + app.kubernetes.io/instance: aws-vpc-cni + k8s-app: aws-node + app.kubernetes.io/version: "v1.18.1" +data: + enable-windows-ipam: "false" + enable-network-policy-controller: "false" + enable-windows-prefix-delegation: "false" + warm-prefix-target: "0" + warm-ip-target: "1" + minimum-ip-target: "3" + branch-eni-cooldown: "60" --- # Source: aws-vpc-cni/templates/clusterrole.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -43,7 +297,7 @@ metadata: app.kubernetes.io/name: aws-node app.kubernetes.io/instance: aws-vpc-cni k8s-app: aws-node - app.kubernetes.io/version: "v1.12.6" + app.kubernetes.io/version: "v1.18.1" rules: - apiGroups: - crd.k8s.amazonaws.com @@ -57,19 +311,28 @@ rules: - apiGroups: [""] resources: - pods - verbs: ["list", "watch", "get"] + verbs: ["list", "watch", "get"] - apiGroups: [""] resources: - nodes - verbs: ["list", "watch", "get", "update"] - - apiGroups: ["extensions"] - resources: - - '*' - verbs: ["list", "watch"] + verbs: ["list", "watch", "get"] - apiGroups: ["", "events.k8s.io"] resources: - events verbs: ["create", "patch", "list"] + - apiGroups: ["networking.k8s.aws"] + resources: + - policyendpoints + verbs: ["get", "list", "watch"] + - apiGroups: ["networking.k8s.aws"] + resources: + - policyendpoints/status + verbs: ["get"] + - apiGroups: + - vpcresources.k8s.aws + resources: + - cninodes + verbs: ["get", "list", "watch", "patch"] --- # Source: aws-vpc-cni/templates/clusterrolebinding.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -80,7 +343,7 @@ metadata: app.kubernetes.io/name: aws-node app.kubernetes.io/instance: aws-vpc-cni k8s-app: aws-node - app.kubernetes.io/version: "v1.12.6" + app.kubernetes.io/version: "v1.18.1" roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole @@ -100,7 +363,7 @@ metadata: app.kubernetes.io/name: aws-node app.kubernetes.io/instance: aws-vpc-cni k8s-app: aws-node - app.kubernetes.io/version: "v1.12.6" + app.kubernetes.io/version: "v1.18.1" spec: updateStrategy: rollingUpdate: @@ -121,7 +384,7 @@ spec: hostNetwork: true initContainers: - name: aws-vpc-cni-init - image: "602401143452.dkr.ecr.us-west-2.amazonaws.com/amazon-k8s-cni-init:v1.12.6" + image: 602401143452.dkr.ecr.us-west-2.amazonaws.com/amazon-k8s-cni-init:v1.18.1 env: - name: DISABLE_TCP_EARLY_DEMUX value: "false" @@ -129,6 +392,9 @@ spec: value: "false" securityContext: privileged: true + resources: + requests: + cpu: 25m volumeMounts: - mountPath: /host/opt/cni/bin name: cni-bin-dir @@ -139,7 +405,7 @@ spec: {} containers: - name: aws-node - image: "602401143452.dkr.ecr.us-west-2.amazonaws.com/amazon-k8s-cni:v1.12.6" + image: 602401143452.dkr.ecr.us-west-2.amazonaws.com/amazon-k8s-cni:v1.18.1 ports: - containerPort: 61678 name: metrics @@ -198,6 +464,12 @@ spec: value: "false" - name: ENABLE_PREFIX_DELEGATION value: "false" + - name: ENABLE_SUBNET_DISCOVERY + value: "true" + - name: NETWORK_POLICY_ENFORCING_MODE + value: "standard" + - name: VPC_CNI_VERSION + value: "v1.18.1" - name: WARM_ENI_TARGET value: "1" - name: WARM_PREFIX_TARGET @@ -205,10 +477,12 @@ spec: - name: MY_NODE_NAME valueFrom: fieldRef: + apiVersion: v1 fieldPath: spec.nodeName - name: MY_POD_NAME valueFrom: fieldRef: + apiVersion: v1 fieldPath: metadata.name resources: requests: @@ -229,7 +503,43 @@ spec: name: run-dir - mountPath: /run/xtables.lock name: xtables-lock + - name: aws-eks-nodeagent + image: 602401143452.dkr.ecr.us-west-2.amazonaws.com/amazon/aws-network-policy-agent:v1.1.1 + env: + - name: MY_NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + args: + - --enable-ipv6=false + - --enable-network-policy=false + - --enable-cloudwatch-logs=false + - --enable-policy-event-logs=false + - --metrics-bind-addr=:8162 + - --health-probe-bind-addr=:8163 + - --conntrack-cache-cleanup-period=300 + resources: + requests: + cpu: 25m + securityContext: + capabilities: + add: + - NET_ADMIN + privileged: true + volumeMounts: + - mountPath: /host/opt/cni/bin + name: cni-bin-dir + - mountPath: /sys/fs/bpf + name: bpf-pin-path + - mountPath: /var/log/aws-routed-eni + name: log-dir + - mountPath: /var/run/aws-node + name: run-dir volumes: + - name: bpf-pin-path + hostPath: + path: /sys/fs/bpf - name: cni-bin-dir hostPath: path: /opt/cni/bin diff --git a/pkg/addons/default/aws_node_generate.go b/pkg/addons/default/aws_node_generate.go index f908e58459..446b912e4d 100644 --- a/pkg/addons/default/aws_node_generate.go +++ b/pkg/addons/default/aws_node_generate.go @@ -1,4 +1,4 @@ package defaultaddons // Please refer to https://docs.aws.amazon.com/eks/latest/userguide/managing-vpc-cni.html -//go:generate curl --silent --location https://raw.githubusercontent.com/aws/amazon-vpc-cni-k8s/v1.12.6/config/master/aws-k8s-cni.yaml?raw=1 --output assets/aws-node.yaml +//go:generate curl --silent --location https://raw.githubusercontent.com/aws/amazon-vpc-cni-k8s/v1.18.1/config/master/aws-k8s-cni.yaml?raw=1 --output assets/aws-node.yaml diff --git a/pkg/addons/default/aws_node_test.go b/pkg/addons/default/aws_node_test.go index 272a47f080..2a26e69b4e 100644 --- a/pkg/addons/default/aws_node_test.go +++ b/pkg/addons/default/aws_node_test.go @@ -79,11 +79,11 @@ var _ = Describe("AWS Node", func() { Expect(err).NotTo(HaveOccurred()) Expect(awsNode.Spec.Template.Spec.Containers).To(HaveLen(1)) Expect(awsNode.Spec.Template.Spec.Containers[0].Image).To( - Equal("602401143452.dkr.ecr.us-east-1.amazonaws.com/amazon-k8s-cni:v1.12.6"), + Equal("602401143452.dkr.ecr.us-east-1.amazonaws.com/amazon-k8s-cni:v1.18.1"), ) Expect(awsNode.Spec.Template.Spec.InitContainers).To(HaveLen(1)) Expect(awsNode.Spec.Template.Spec.InitContainers[0].Image).To( - Equal("602401143452.dkr.ecr.us-east-1.amazonaws.com/amazon-k8s-cni-init:v1.12.6"), + Equal("602401143452.dkr.ecr.us-east-1.amazonaws.com/amazon-k8s-cni-init:v1.18.1"), ) }) }) @@ -99,11 +99,11 @@ var _ = Describe("AWS Node", func() { Expect(err).NotTo(HaveOccurred()) Expect(awsNode.Spec.Template.Spec.Containers).To(HaveLen(1)) Expect(awsNode.Spec.Template.Spec.Containers[0].Image).To( - Equal("961992271922.dkr.ecr.cn-northwest-1.amazonaws.com.cn/amazon-k8s-cni:v1.12.6"), + Equal("961992271922.dkr.ecr.cn-northwest-1.amazonaws.com.cn/amazon-k8s-cni:v1.18.1"), ) Expect(awsNode.Spec.Template.Spec.InitContainers).To(HaveLen(1)) Expect(awsNode.Spec.Template.Spec.InitContainers[0].Image).To( - Equal("961992271922.dkr.ecr.cn-northwest-1.amazonaws.com.cn/amazon-k8s-cni-init:v1.12.6"), + Equal("961992271922.dkr.ecr.cn-northwest-1.amazonaws.com.cn/amazon-k8s-cni-init:v1.18.1"), ) }) }) From 73dbe7aecd20d89fcc98014e7e3169125a7c1234 Mon Sep 17 00:00:00 2001 From: Erik Sundell Date: Thu, 16 May 2024 15:27:38 +0200 Subject: [PATCH 2/3] Update tests for aws-node 1.18.1 --- pkg/addons/default/aws_node_test.go | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/pkg/addons/default/aws_node_test.go b/pkg/addons/default/aws_node_test.go index 2a26e69b4e..33296af305 100644 --- a/pkg/addons/default/aws_node_test.go +++ b/pkg/addons/default/aws_node_test.go @@ -77,10 +77,13 @@ var _ = Describe("AWS Node", func() { awsNode, err := rawClient.ClientSet().AppsV1().DaemonSets(metav1.NamespaceSystem).Get(context.Background(), da.AWSNode, metav1.GetOptions{}) Expect(err).NotTo(HaveOccurred()) - Expect(awsNode.Spec.Template.Spec.Containers).To(HaveLen(1)) + Expect(awsNode.Spec.Template.Spec.Containers).To(HaveLen(2)) Expect(awsNode.Spec.Template.Spec.Containers[0].Image).To( Equal("602401143452.dkr.ecr.us-east-1.amazonaws.com/amazon-k8s-cni:v1.18.1"), ) + Expect(awsNode.Spec.Template.Spec.Containers[1].Image).To( + Equal("602401143452.dkr.ecr.us-east-1.amazonaws.com/amazon/aws-network-policy-agent:v1.18.1"), + ) Expect(awsNode.Spec.Template.Spec.InitContainers).To(HaveLen(1)) Expect(awsNode.Spec.Template.Spec.InitContainers[0].Image).To( Equal("602401143452.dkr.ecr.us-east-1.amazonaws.com/amazon-k8s-cni-init:v1.18.1"), @@ -97,10 +100,13 @@ var _ = Describe("AWS Node", func() { awsNode, err := rawClient.ClientSet().AppsV1().DaemonSets(metav1.NamespaceSystem).Get(context.Background(), da.AWSNode, metav1.GetOptions{}) Expect(err).NotTo(HaveOccurred()) - Expect(awsNode.Spec.Template.Spec.Containers).To(HaveLen(1)) + Expect(awsNode.Spec.Template.Spec.Containers).To(HaveLen(2)) Expect(awsNode.Spec.Template.Spec.Containers[0].Image).To( Equal("961992271922.dkr.ecr.cn-northwest-1.amazonaws.com.cn/amazon-k8s-cni:v1.18.1"), ) + Expect(awsNode.Spec.Template.Spec.Containers[1].Image).To( + Equal("602401143452.dkr.ecr.cn-northwest-1.amazonaws.com/amazon/aws-network-policy-agent:v1.18.1"), + ) Expect(awsNode.Spec.Template.Spec.InitContainers).To(HaveLen(1)) Expect(awsNode.Spec.Template.Spec.InitContainers[0].Image).To( Equal("961992271922.dkr.ecr.cn-northwest-1.amazonaws.com.cn/amazon-k8s-cni-init:v1.18.1"), From ebd4d590948548c30179faf1d99a3ecfee785022 Mon Sep 17 00:00:00 2001 From: Erik Sundell Date: Thu, 16 May 2024 16:49:20 +0200 Subject: [PATCH 3/3] Reduce complexity of aws-node test --- pkg/addons/default/aws_node_test.go | 6 ------ 1 file changed, 6 deletions(-) diff --git a/pkg/addons/default/aws_node_test.go b/pkg/addons/default/aws_node_test.go index 33296af305..3175b4cb4b 100644 --- a/pkg/addons/default/aws_node_test.go +++ b/pkg/addons/default/aws_node_test.go @@ -81,9 +81,6 @@ var _ = Describe("AWS Node", func() { Expect(awsNode.Spec.Template.Spec.Containers[0].Image).To( Equal("602401143452.dkr.ecr.us-east-1.amazonaws.com/amazon-k8s-cni:v1.18.1"), ) - Expect(awsNode.Spec.Template.Spec.Containers[1].Image).To( - Equal("602401143452.dkr.ecr.us-east-1.amazonaws.com/amazon/aws-network-policy-agent:v1.18.1"), - ) Expect(awsNode.Spec.Template.Spec.InitContainers).To(HaveLen(1)) Expect(awsNode.Spec.Template.Spec.InitContainers[0].Image).To( Equal("602401143452.dkr.ecr.us-east-1.amazonaws.com/amazon-k8s-cni-init:v1.18.1"), @@ -104,9 +101,6 @@ var _ = Describe("AWS Node", func() { Expect(awsNode.Spec.Template.Spec.Containers[0].Image).To( Equal("961992271922.dkr.ecr.cn-northwest-1.amazonaws.com.cn/amazon-k8s-cni:v1.18.1"), ) - Expect(awsNode.Spec.Template.Spec.Containers[1].Image).To( - Equal("602401143452.dkr.ecr.cn-northwest-1.amazonaws.com/amazon/aws-network-policy-agent:v1.18.1"), - ) Expect(awsNode.Spec.Template.Spec.InitContainers).To(HaveLen(1)) Expect(awsNode.Spec.Template.Spec.InitContainers[0].Image).To( Equal("961992271922.dkr.ecr.cn-northwest-1.amazonaws.com.cn/amazon-k8s-cni-init:v1.18.1"),