From 1d83e232b267744db2cd2d44a4544ec4e2fd6a3d Mon Sep 17 00:00:00 2001 From: Chinmay Gadgil Date: Wed, 2 Feb 2022 18:00:48 -0800 Subject: [PATCH] Cherry pick of workflow changes from master branch (#1793) * Change workflow to use git install (#1785) - Change workflow to use git install as the go get command was altering go.mod file without updating go.sum file * Workflow file updates (#1787) * Change workflow to use git install - Change workflow to use git install as the go get command was altering go.mod file without updating go.sum file * Update workflow files to use go 1.16 - Updated workflow files to use go 1.16 for consistency. - Changed integ test script to use git commit id and deleted unused code * Specify go version at 1.16 for workflow files Cherry pick multi-arch changes in release branch 1.10.2 changes (#1830) * VlanID changes PPSG Test agent changes CNI metrics helper changes minor fix Move logging for CLUSTER_ID and Region inside publisher.go from main.go revert manifest changes as the image is not released yet * Cherry pick multi-arch changes in release branch * cni-metrics-helper changes vlanID changes disable network provisioning fix ipamd error code fix * Minor change * remove redundant changes * Go version changes * Switch to public ECR for AL2 (#1804) * Switch to public ecr for al2 image - Removed docker_arch variable which is redundant with this change * Fix makefile and dockerfile entries * Merge changes to auto-sync manifests * minor change: add ContainerID to dummyVlanInterface * Remove sudo from workflow files (#1818) * remove set -x from bash, add -Ss to curl (#1802) * remove weekly-cron-test.yml as its not needed for this branch Co-authored-by: Vikas Basavaraj <5373156+vikasmb@users.noreply.github.com> Co-authored-by: Jayanth Varavani <1111446+jayanthvn@users.noreply.github.com> Co-authored-by: Scott Merrill fixed conflicts with routed-eni-cni-plugin/cni.go --- .github/workflows/build-multi-arch.yaml | 10 +- .github/workflows/cron-test.yml | 4 +- .github/workflows/forked-pr-tests.yml | 4 +- .github/workflows/integration-tests.yml | 6 +- .github/workflows/release.yaml | 2 +- .github/workflows/unit-tests.yml | 4 +- Makefile | 8 +- charts/cni-metrics-helper/values.yaml | 1 + cmd/cni-metrics-helper/README.md | 96 ++++++++++++ cmd/routed-eni-cni-plugin/cni.go | 53 +++++-- config/master/cni-metrics-helper-cn.yaml | 30 +--- .../cni-metrics-helper-us-gov-east-1.yaml | 30 +--- .../cni-metrics-helper-us-gov-west-1.yaml | 30 +--- config/master/cni-metrics-helper.yaml | 33 +--- config/master/manifests.jsonnet | 32 +--- go.mod | 60 ++++++- go.sum | 42 +---- misc/10-aws.conflist | 1 + .../crd/v1alpha1/zz_generated.deepcopy.go | 1 + pkg/ipamd/ipamd.go | 21 ++- pkg/ipamd/rpc_handler.go | 2 +- pkg/publisher/publisher.go | 1 - scripts/dockerfiles/Dockerfile.init | 3 +- scripts/dockerfiles/Dockerfile.metrics | 3 +- scripts/dockerfiles/Dockerfile.release | 3 +- scripts/dockerfiles/Dockerfile.test | 1 - scripts/generate-cni-yaml.sh | 5 +- scripts/init.sh | 6 +- scripts/upload-resources-to-github.sh | 148 ++++++++++++++++-- .../resources/k8s/utils/daemonset.go | 2 +- test/framework/utils/const.go | 2 - .../metrics_helper_suite_test.go | 35 ++++- 32 files changed, 414 insertions(+), 265 deletions(-) diff --git a/.github/workflows/build-multi-arch.yaml b/.github/workflows/build-multi-arch.yaml index d912cf9658..1ce2ff872e 100644 --- a/.github/workflows/build-multi-arch.yaml +++ b/.github/workflows/build-multi-arch.yaml @@ -1,4 +1,4 @@ -name: Build x86 and ARM64 images +name: Build x64 and ARM64 images on: pull_request: @@ -12,10 +12,10 @@ jobs: runs-on: ubuntu-latest steps: - - name: Set up Go 1.16 + - name: Set up Go 1.17 uses: actions/setup-go@v2 with: - go-version: '1.16' + go-version: '1.17' id: go - name: Check out code into the Go module directory @@ -32,10 +32,10 @@ jobs: runs-on: [self-hosted, linux, arm64] steps: - - name: Set up Go 1.16 + - name: Set up Go 1.17 uses: actions/setup-go@v2 with: - go-version: '1.16' + go-version: '1.17' id: go - name: Check out code into the Go module directory diff --git a/.github/workflows/cron-test.yml b/.github/workflows/cron-test.yml index db2320eab1..d92ca9f4ae 100644 --- a/.github/workflows/cron-test.yml +++ b/.github/workflows/cron-test.yml @@ -12,10 +12,10 @@ jobs: integration-cron: runs-on: [self-hosted, linux, x64] steps: - - name: Set up Go 1.16 + - name: Set up Go 1.17 uses: actions/setup-go@v2 with: - go-version: '1.16' + go-version: '1.17' id: go - name: Check out code into the Go module directory diff --git a/.github/workflows/forked-pr-tests.yml b/.github/workflows/forked-pr-tests.yml index 30fa3947b8..6f91878651 100644 --- a/.github/workflows/forked-pr-tests.yml +++ b/.github/workflows/forked-pr-tests.yml @@ -18,10 +18,10 @@ jobs: if: github.event.inputs.pull_request_number != '' steps: - - name: Set up Go 1.16 + - name: Set up Go 1.17 uses: actions/setup-go@v2 with: - go-version: '1.16' + go-version: '1.17' id: go # Check out a merge commit diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 4717a0066e..6d47817bbb 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -14,10 +14,10 @@ jobs: integration-trusted: runs-on: [self-hosted, linux, x64] steps: - - name: Set up Go 1.16 + - name: Set up Go 1.17 uses: actions/setup-go@v2 with: - go-version: '1.16' + go-version: '1.17' id: go - name: Check out code into the Go module directory @@ -29,7 +29,7 @@ jobs: go install golang.org/x/tools/cmd/goimports@latest - name: Clean up stale docker images - run: sudo docker image prune -f + run: docker image prune -f - name: Run e2e tests env: diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 81b6af6bd2..4b7d802642 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -3,7 +3,7 @@ name: VPC CNI Release on: [push, workflow_dispatch] env: - DEFAULT_GO_VERSION: ^1.15 + DEFAULT_GO_VERSION: ^1.17 GITHUB_USERNAME: ${{ secrets.EKS_BOT_GITHUB_USERNAME }} GITHUB_TOKEN: ${{ secrets.EKS_BOT_GITHUB_TOKEN }} diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 434ab6d8d7..772aeaf2f7 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -12,10 +12,10 @@ jobs: runs-on: ubuntu-latest steps: - - name: Set up Go 1.16 + - name: Set up Go 1.17 uses: actions/setup-go@v2 with: - go-version: '1.16' + go-version: '1.17' id: go - name: Check out code into the Go module directory diff --git a/Makefile b/Makefile index ab2006c724..41b079a1c3 100644 --- a/Makefile +++ b/Makefile @@ -43,7 +43,7 @@ HELM_CHART_NAME ?= "aws-vpc-cni" # TEST_IMAGE is the testing environment container image. TEST_IMAGE = amazon-k8s-cni-test TEST_IMAGE_NAME = $(TEST_IMAGE)$(IMAGE_ARCH_SUFFIX):$(VERSION) -# These values derive ARCH and DOCKER_ARCH which are needed by dependencies in +# These values derive ARCH which is needed by dependencies in # image build defaulting to system's architecture when not specified. # # UNAME_ARCH is the runtime architecture of the building host. @@ -52,16 +52,13 @@ UNAME_ARCH = $(shell uname -m) # # These are pairs of input_arch to derived_arch separated by colons: ARCH = $(lastword $(subst :, ,$(filter $(UNAME_ARCH):%,x86_64:amd64 aarch64:arm64))) -# DOCKER_ARCH is the docker specific architecture specifier used for building on -# multiarch container images. -DOCKER_ARCH = $(lastword $(subst :, ,$(filter $(ARCH):%,amd64:amd64 arm64:arm64v8))) # IMAGE_ARCH_SUFFIX is the `-arch` suffix included in the container image name. # # This is only applied to the arm64 container image by default. Override to # provide an alternate suffix or to omit. IMAGE_ARCH_SUFFIX = $(addprefix -,$(filter $(ARCH),arm64)) # GOLANG_IMAGE is the building golang container image used. -GOLANG_IMAGE = golang:1.16-stretch +GOLANG_IMAGE = public.ecr.aws/docker/library/golang:1.17-stretch # For the requested build, these are the set of Go specific build environment variables. export GOARCH ?= $(ARCH) export GOOS = linux @@ -96,7 +93,6 @@ DOCKER_RUN_FLAGS = --rm -ti $(DOCKER_ARGS) # DOCKER_BUILD_FLAGS is the set of flags passed during container image builds # based on the requested build. DOCKER_BUILD_FLAGS = --build-arg GOARCH="$(ARCH)" \ - --build-arg docker_arch="$(DOCKER_ARCH)" \ --build-arg golang_image="$(GOLANG_IMAGE)" \ --network=host \ $(DOCKER_ARGS) diff --git a/charts/cni-metrics-helper/values.yaml b/charts/cni-metrics-helper/values.yaml index ab89878633..fe90724ae1 100644 --- a/charts/cni-metrics-helper/values.yaml +++ b/charts/cni-metrics-helper/values.yaml @@ -12,6 +12,7 @@ image: env: USE_CLOUDWATCH: "true" + AWS_CLUSTER_ID: "" fullnameOverride: "cni-metrics-helper" diff --git a/cmd/cni-metrics-helper/README.md b/cmd/cni-metrics-helper/README.md index 85cd93ce89..634f1d01b1 100644 --- a/cmd/cni-metrics-helper/README.md +++ b/cmd/cni-metrics-helper/README.md @@ -13,6 +13,102 @@ The following diagram shows how `cni-metrics-helper` works in a cluster: ![](../../docs/images/cni-metrics-helper.png) +### Using IRSA +As per [AWS EKS Security Best Practice](https://docs.aws.amazon.com/eks/latest/userguide/best-practices-security.html), if you are using IRSA for pods then following requirements must be satisfied to succesfully publish metrics to CloudWatch + +1. The IAM Role for your SA [(IRSA)](https://docs.aws.amazon.com/eks/latest/userguide/iam-roles-for-service-accounts.html) must have following policy attached + +``` +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "cloudwatch:PutMetricData" + ], + "Resource": "*" + } + ] +} +``` + +2. You should have similar ClusterRole and ClusterRoleBinding for the IRSA + +``` +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: cni-metrics-helper +rules: + - apiGroups: [""] + resources: + - pods + - pods/proxy + verbs: ["get", "watch", "list"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: cni-metrics-helper + labels: + app.kubernetes.io/name: cni-metrics-helper + app.kubernetes.io/instance: cni-metrics-helper + app.kubernetes.io/version: "v1.10.2" +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cni-metrics-helper +subjects: + - kind: ServiceAccount + name: + namespace: kube-system +``` + +3. Specify the IRSA name in the cni-metrics-helper deployment spec alongwith the AWS_CLUSTER_ID (as described below). The value that you specify here will show up under the dimension 'CLUSTER_ID' for your published metrics. Specifying value for this field is mandatory only if you are blocking IMDS access + +#### `AWS_CLUSTER_ID` + +Type: String + +Default: `""` + +An Identifier for your Cluster which will be used as the dimension for published metrics. Ideally it should be ClusterName or ClusterID. + +``` +kind: Deployment +apiVersion: apps/v1 +metadata: + name: cni-metrics-helper + namespace: kube-system + labels: + k8s-app: cni-metrics-helper +spec: + selector: + matchLabels: + k8s-app: cni-metrics-helper + template: + metadata: + labels: + k8s-app: cni-metrics-helper + spec: + containers: + - env: + - name: USE_CLOUDWATCH + value: "true" + - name: AWS_CLUSTER_ID + value: "" + name: cni-metrics-helper + image: + serviceAccountName: +``` +With IRSA, the above deployment spec will be auto-injected with AWS_REGION parameter and it will be used to fetch Region information when we publish metrics. +Possible Scenarios for above configuration +1. If you are not using IRSA, then Region and CLUSTER_ID information will be fetched using IMDS (should have access) +2. If you are using IRSA but have not specified AWS_CLUSTER_ID, we will fetch the value for CLUSTER_ID if IMDS access is not blocked +3. If you have blocked IMDS access, then you must specify a value for AWS_CLUSTER_ID in the deployment spec +4. If you have not blocked IMDS access but have specified AWS_CLUSTER_ID value, then this value will be used. + ### Installing the cni-metrics-helper ``` kubectl apply -f v1.6/cni-metrics-helper.yaml diff --git a/cmd/routed-eni-cni-plugin/cni.go b/cmd/routed-eni-cni-plugin/cni.go index 1f373ebc21..553cba11af 100644 --- a/cmd/routed-eni-cni-plugin/cni.go +++ b/cmd/routed-eni-cni-plugin/cni.go @@ -44,7 +44,9 @@ import ( ) const ipamdAddress = "127.0.0.1:50051" -const vlanInterfaceName = "vlanId" + +const vlanInterfacePrefix = "vlan" +const dummyVlanInterfacePrefix = "dummy" var version string @@ -129,8 +131,7 @@ func add(args *skel.CmdArgs, cniTypes typeswrapper.CNITYPES, grpcClient grpcwrap log.Infof("Received CNI add request: ContainerID(%s) Netns(%s) IfName(%s) Args(%s) Path(%s) argsStdinData(%s)", args.ContainerID, args.Netns, args.IfName, args.Args, args.Path, args.StdinData) - - log.Infof("Prev Result: %v\n", conf.PrevResult) + log.Debugf("Prev Result: %v\n", conf.PrevResult) var k8sArgs K8sArgs if err := cniTypes.LoadArgs(args.Args, &k8sArgs); err != nil { @@ -202,10 +203,24 @@ func add(args *skel.CmdArgs, cniTypes typeswrapper.CNITYPES, grpcClient grpcwrap } var hostVethName string + var dummyVlanInterface *current.Interface + + // Non-zero value means pods are using branch ENI if r.PodVlanId != 0 { - hostVethName = generateHostVethName("vlan", string(k8sArgs.K8S_POD_NAMESPACE), string(k8sArgs.K8S_POD_NAME)) + hostVethName = generateHostVethName(vlanInterfacePrefix, string(k8sArgs.K8S_POD_NAMESPACE), string(k8sArgs.K8S_POD_NAME)) err = driverClient.SetupPodENINetwork(hostVethName, args.IfName, args.Netns, v4Addr, v6Addr, int(r.PodVlanId), r.PodENIMAC, r.PodENISubnetGW, int(r.ParentIfIndex), mtu, log) + + // This is a dummyVlanInterfaceName generated to identify dummyVlanInterface + // which will be created for PPSG scenario to pass along the vlanId information + // as a part of the ADD cmd Result struct + // The podVlanId is used by DEL cmd, fetched from the prevResult struct to cleanup the pod network + dummyVlanInterfaceName := generateHostVethName(dummyVlanInterfacePrefix, string(k8sArgs.K8S_POD_NAMESPACE), string(k8sArgs.K8S_POD_NAME)) + + // The dummyVlanInterface is purely virtual and relevent only for ppsg, so we decided to keep it separate + // and not overload the already available hostVethInterface + dummyVlanInterface = ¤t.Interface{Name: dummyVlanInterfaceName, Mac: fmt.Sprint(r.PodVlanId), Sandbox: args.ContainerID} + log.Debugf("Using dummy vlanInterface: %v", dummyVlanInterface) } else { // build hostVethName // Note: the maximum length for linux interface name is 15 @@ -249,18 +264,20 @@ func add(args *skel.CmdArgs, cniTypes typeswrapper.CNITYPES, grpcClient grpcwrap hostInterface := ¤t.Interface{Name: hostVethName} containerInterface := ¤t.Interface{Name: args.IfName, Sandbox: args.Netns} - vlanInterface := ¤t.Interface{Name: vlanInterfaceName, Mac: fmt.Sprint(r.PodVlanId)} - log.Infof("Using vlanInterface: %v", vlanInterface) result := ¤t.Result{ IPs: ips, Interfaces: []*current.Interface{ hostInterface, containerInterface, - vlanInterface, }, } + // We append dummyVlanInterface only for pods using branch ENI + if dummyVlanInterface != nil { + result.Interfaces = append(result.Interfaces, dummyVlanInterface) + } + return cniTypes.PrintResult(result, conf.CNIVersion) } @@ -281,7 +298,7 @@ func del(args *skel.CmdArgs, cniTypes typeswrapper.CNITYPES, grpcClient grpcwrap driverClient driver.NetworkAPIs) error { conf, log, err := LoadNetConf(args.StdinData) - log.Infof("Prev Result: %v\n", conf.PrevResult) + log.Debugf("Prev Result: %v\n", conf.PrevResult) if err != nil { return errors.Wrap(err, "add cmd: error loading config from args") @@ -296,6 +313,10 @@ func del(args *skel.CmdArgs, cniTypes typeswrapper.CNITYPES, grpcClient grpcwrap return errors.Wrap(err, "del cmd: failed to load k8s config from args") } + // With containerd as the runtime, it was observed that sometimes spurious delete requests + // are triggered from kubelet with an empty Netns. This check safeguards against such + // scenarios and we just return + // ref: https://github.com/kubernetes/kubernetes/issues/44100#issuecomment-329780382 if args.Netns == "" { log.Info("Netns() is empty, so network already cleanedup. Nothing to do") return nil @@ -306,18 +327,22 @@ func del(args *skel.CmdArgs, cniTypes typeswrapper.CNITYPES, grpcClient grpcwrap // prevResult might not be availabe, if we are still using older cni spec < 0.4.0. // So we should fallback to the old clean up method if ok { + dummyVlanInterfaceName := generateHostVethName(dummyVlanInterfacePrefix, string(k8sArgs.K8S_POD_NAMESPACE), string(k8sArgs.K8S_POD_NAME)) for _, iface := range prevResult.Interfaces { - if iface.Name == vlanInterfaceName { + if iface.Name == dummyVlanInterfaceName { podVlanId, err := strconv.Atoi(iface.Mac) if err != nil { - return errors.Wrap(err, "Failed to parse vlanId from prevResult") + log.Errorf("Failed to parse vlanId from prevResult: %v", err) + return errors.Wrap(err, "del cmd: failed to parse vlanId from prevResult") } - // podVlanId == 0 means pod is not using branch ENI - // then fallback to existing cleanup + + // podVlanID can not be 0 as we add dummyVlanInterface only for ppsg + // if it is 0 then we should return an error if podVlanId == 0 { - break + log.Errorf("Found SG pod:%s namespace:%s with 0 vlanID", k8sArgs.K8S_POD_NAME, k8sArgs.K8S_POD_NAMESPACE) + return errors.Wrap(err, "del cmd: found Incorrect 0 vlandId for ppsg") } - // if podVlanId != 0 means pod is using branch ENI + err = cleanUpPodENI(podVlanId, log, args.ContainerID, driverClient) if err != nil { return err diff --git a/config/master/cni-metrics-helper-cn.yaml b/config/master/cni-metrics-helper-cn.yaml index 0c07079ef3..a1e57bb7f4 100644 --- a/config/master/cni-metrics-helper-cn.yaml +++ b/config/master/cni-metrics-helper-cn.yaml @@ -18,37 +18,9 @@ metadata: rules: - apiGroups: [""] resources: - - nodes - pods - pods/proxy - - services - - resourcequotas - - replicationcontrollers - - limitranges - - persistentvolumeclaims - - persistentvolumes - - namespaces - - endpoints - verbs: ["list", "watch", "get"] - - apiGroups: ["extensions"] - resources: - - daemonsets - - deployments - - replicasets - verbs: ["list", "watch"] - - apiGroups: ["apps"] - resources: - - statefulsets - verbs: ["list", "watch"] - - apiGroups: ["batch"] - resources: - - cronjobs - - jobs - verbs: ["list", "watch"] - - apiGroups: ["autoscaling"] - resources: - - horizontalpodautoscalers - verbs: ["list", "watch"] + verbs: ["get", "watch", "list"] --- # Source: cni-metrics-helper/templates/clusterrolebinding.yaml apiVersion: rbac.authorization.k8s.io/v1 diff --git a/config/master/cni-metrics-helper-us-gov-east-1.yaml b/config/master/cni-metrics-helper-us-gov-east-1.yaml index 91a40ed6ca..4276fd3392 100644 --- a/config/master/cni-metrics-helper-us-gov-east-1.yaml +++ b/config/master/cni-metrics-helper-us-gov-east-1.yaml @@ -18,37 +18,9 @@ metadata: rules: - apiGroups: [""] resources: - - nodes - pods - pods/proxy - - services - - resourcequotas - - replicationcontrollers - - limitranges - - persistentvolumeclaims - - persistentvolumes - - namespaces - - endpoints - verbs: ["list", "watch", "get"] - - apiGroups: ["extensions"] - resources: - - daemonsets - - deployments - - replicasets - verbs: ["list", "watch"] - - apiGroups: ["apps"] - resources: - - statefulsets - verbs: ["list", "watch"] - - apiGroups: ["batch"] - resources: - - cronjobs - - jobs - verbs: ["list", "watch"] - - apiGroups: ["autoscaling"] - resources: - - horizontalpodautoscalers - verbs: ["list", "watch"] + verbs: ["get", "watch", "list"] --- # Source: cni-metrics-helper/templates/clusterrolebinding.yaml apiVersion: rbac.authorization.k8s.io/v1 diff --git a/config/master/cni-metrics-helper-us-gov-west-1.yaml b/config/master/cni-metrics-helper-us-gov-west-1.yaml index 6f592e93ff..3adb81ec1d 100644 --- a/config/master/cni-metrics-helper-us-gov-west-1.yaml +++ b/config/master/cni-metrics-helper-us-gov-west-1.yaml @@ -18,37 +18,9 @@ metadata: rules: - apiGroups: [""] resources: - - nodes - pods - pods/proxy - - services - - resourcequotas - - replicationcontrollers - - limitranges - - persistentvolumeclaims - - persistentvolumes - - namespaces - - endpoints - verbs: ["list", "watch", "get"] - - apiGroups: ["extensions"] - resources: - - daemonsets - - deployments - - replicasets - verbs: ["list", "watch"] - - apiGroups: ["apps"] - resources: - - statefulsets - verbs: ["list", "watch"] - - apiGroups: ["batch"] - resources: - - cronjobs - - jobs - verbs: ["list", "watch"] - - apiGroups: ["autoscaling"] - resources: - - horizontalpodautoscalers - verbs: ["list", "watch"] + verbs: ["get", "watch", "list"] --- # Source: cni-metrics-helper/templates/clusterrolebinding.yaml apiVersion: rbac.authorization.k8s.io/v1 diff --git a/config/master/cni-metrics-helper.yaml b/config/master/cni-metrics-helper.yaml index 18404d7f8d..dc1d8f4916 100644 --- a/config/master/cni-metrics-helper.yaml +++ b/config/master/cni-metrics-helper.yaml @@ -18,37 +18,9 @@ metadata: rules: - apiGroups: [""] resources: - - nodes - pods - pods/proxy - - services - - resourcequotas - - replicationcontrollers - - limitranges - - persistentvolumeclaims - - persistentvolumes - - namespaces - - endpoints - verbs: ["list", "watch", "get"] - - apiGroups: ["extensions"] - resources: - - daemonsets - - deployments - - replicasets - verbs: ["list", "watch"] - - apiGroups: ["apps"] - resources: - - statefulsets - verbs: ["list", "watch"] - - apiGroups: ["batch"] - resources: - - cronjobs - - jobs - verbs: ["list", "watch"] - - apiGroups: ["autoscaling"] - resources: - - horizontalpodautoscalers - verbs: ["list", "watch"] + verbs: ["get", "watch", "list"] --- # Source: cni-metrics-helper/templates/clusterrolebinding.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -89,6 +61,9 @@ spec: - env: - name: USE_CLOUDWATCH value: "true" + # Optional: Should be ClusterName/ClusterIdentifier used as the metric dimension + - name: AWS_CLUSTER_ID + value: "" name: cni-metrics-helper image: "602401143452.dkr.ecr.us-west-2.amazonaws.com/cni-metrics-helper:v1.10.1" serviceAccountName: cni-metrics-helper diff --git a/config/master/manifests.jsonnet b/config/master/manifests.jsonnet index 44e666f8a7..5d6edf4563 100644 --- a/config/master/manifests.jsonnet +++ b/config/master/manifests.jsonnet @@ -297,40 +297,11 @@ local metricsHelper = { { apiGroups: [""], resources: [ - "nodes", "pods", - "pods/proxy", - "services", - "resourcequotas", - "replicationcontrollers", - "limitranges", - "persistentvolumeclaims", - "persistentvolumes", - "namespaces", - "endpoints", + "pods/proxy" ], verbs: ["list", "watch", "get"], }, - { - apiGroups: ["extensions"], - resources: ["daemonsets", "deployments", "replicasets"], - verbs: ["list", "watch"], - }, - { - apiGroups: ["apps"], - resources: ["statefulsets"], - verbs: ["list", "watch"], - }, - { - apiGroups: ["batch"], - resources: ["cronjobs", "jobs"], - verbs: ["list", "watch"], - }, - { - apiGroups: ["autoscaling"], - resources: ["horizontalpodautoscalers"], - verbs: ["list", "watch"], - }, ], }, @@ -390,6 +361,7 @@ local metricsHelper = { name: "cni-metrics-helper", env_:: { USE_CLOUDWATCH: "true", + AWS_CLUSTER_ID: "", }, env: [ {name: kv[0]} + if std.isObject(kv[1]) then kv[1] else {value: kv[1]} diff --git a/go.mod b/go.mod index 85570dd336..e2aa4dc95a 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/aws/amazon-vpc-cni-k8s -go 1.14 +go 1.17 require ( github.com/aws/aws-sdk-go v1.40.6 @@ -18,13 +18,10 @@ require ( github.com/stretchr/testify v1.5.1 github.com/vishvananda/netlink v1.1.1-0.20201029203352-d40f9887b852 go.uber.org/zap v1.15.0 - golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 // indirect - golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d - golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 // indirect + golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d golang.org/x/sys v0.0.0-20210616094352-59db8d763f22 - golang.org/x/tools v0.1.5 // indirect google.golang.org/grpc v1.29.0 - google.golang.org/protobuf v1.23.0 // indirect + google.golang.org/protobuf v1.23.0 gopkg.in/natefinch/lumberjack.v2 v2.0.0 k8s.io/api v0.18.6 k8s.io/apimachinery v0.18.6 @@ -32,3 +29,54 @@ require ( k8s.io/cri-api v0.0.0-20191107035106-03d130a7dc28 sigs.k8s.io/controller-runtime v0.6.3 ) + +require ( + github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc // indirect + github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf // indirect + github.com/beorn7/perks v1.0.0 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/evanphx/json-patch v4.9.0+incompatible // indirect + github.com/fsnotify/fsnotify v1.4.9 // indirect + github.com/go-logr/logr v0.1.0 // indirect + github.com/gogo/protobuf v1.3.1 // indirect + github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef // indirect + github.com/google/go-cmp v0.4.0 // indirect + github.com/google/gofuzz v1.1.0 // indirect + github.com/google/uuid v1.1.1 // indirect + github.com/googleapis/gnostic v0.3.1 // indirect + github.com/hashicorp/golang-lru v0.5.4 // indirect + github.com/imdario/mergo v0.3.9 // indirect + github.com/jmespath/go-jmespath v0.4.0 // indirect + github.com/json-iterator/go v1.1.10 // indirect + github.com/konsorten/go-windows-terminal-sequences v1.0.1 // indirect + github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/prometheus/procfs v0.0.11 // indirect + github.com/safchain/ethtool v0.0.0-20190326074333-42ed695e3de8 // indirect + github.com/sirupsen/logrus v1.4.2 // indirect + github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae // indirect + go.uber.org/atomic v1.6.0 // indirect + go.uber.org/multierr v1.5.0 // indirect + golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 // indirect + golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 // indirect + golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 // indirect + golang.org/x/text v0.3.6 // indirect + golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 // indirect + golang.org/x/tools v0.1.5 // indirect + golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect + gomodules.xyz/jsonpatch/v2 v2.0.1 // indirect + google.golang.org/appengine v1.5.0 // indirect + google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55 // indirect + gopkg.in/alecthomas/kingpin.v2 v2.2.6 // indirect + gopkg.in/inf.v0 v0.9.1 // indirect + gopkg.in/yaml.v2 v2.3.0 // indirect + k8s.io/apiextensions-apiserver v0.18.6 // indirect + k8s.io/klog v1.0.0 // indirect + k8s.io/klog/v2 v2.0.0 // indirect + k8s.io/kube-openapi v0.0.0-20200410145947-61e04a5be9a6 // indirect + k8s.io/utils v0.0.0-20200603063816-c1c6865ac451 // indirect + sigs.k8s.io/structured-merge-diff/v3 v3.0.0 // indirect + sigs.k8s.io/yaml v1.2.0 // indirect +) diff --git a/go.sum b/go.sum index 425aebf2b8..3350a6f682 100644 --- a/go.sum +++ b/go.sum @@ -31,7 +31,6 @@ github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:l github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/aws/aws-sdk-go v1.40.6 h1:JCQfi5MD8cW0PCAzr88hj9tj4BdEJkAy8EyAJ6c8I/k= github.com/aws/aws-sdk-go v1.40.6/go.mod h1:585smgzpB/KqRA+K3y/NL/oYRqQvpNJYvLm+LY1U59Q= -github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLMYoU8P317H5OQ+Via4RmuPwCS0= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0 h1:HWo1m869IqiPhD389kmkxeTalrjNbbJTC8LXupb+sl0= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= @@ -86,14 +85,11 @@ github.com/evanphx/json-patch v4.5.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLi github.com/evanphx/json-patch v4.9.0+incompatible h1:kLcOMZeuLAJvL2BPWLMIj5oaZQobrkAqrL+WFZwQses= github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= -github.com/fatih/color v1.9.0 h1:8xPHl4/q1VyqGIPif1F+1V3Y3lSmrq01EabUW3CoW5s= github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= -github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= -github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/globalsign/mgo v0.0.0-20180905125535-1ca0a4f7cbcb/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q= github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q= @@ -150,11 +146,9 @@ github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/me github.com/godbus/dbus v0.0.0-20180201030542-885f9cc04c9c/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= -github.com/gogo/protobuf v1.2.2-0.20190723190241-65acae22fc9d h1:3PaI8p3seN09VjbTYC/QWlUZdZ1qS1zGjy7LH2Wt07I= github.com/gogo/protobuf v1.2.2-0.20190723190241-65acae22fc9d/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls= github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= -github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef h1:veQD95Isof8w9/WXiA+pa3tz3fJXkt5B7QaRBrM62gk= @@ -176,9 +170,7 @@ github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvq github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0= github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -208,12 +200,10 @@ github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc= github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= -github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/imdario/mergo v0.3.9 h1:UauaLniWCFHWd+Jp9oCEkTBj8VO/9DKg3PV3VCNMDIg= @@ -252,11 +242,9 @@ github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= -github.com/mattn/go-colorable v0.1.4 h1:snbPLB8fVfU9iwbbo30TPtbLRzwWu6aJS6Xh4eaaviA= github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= -github.com/mattn/go-isatty v0.0.11 h1:FxPOTFNqGkuDUGi3H/qkUbQO4ZiBa2brKq5r0l8TGeM= github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-shellwords v1.0.3/go.mod h1:3xCvwCdWdlDJUrvuMn7Wuy9eWs4pE8vqg+NOMyg4B2o= @@ -267,7 +255,6 @@ github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 h1:Esafd1046DLDQ0W1YjYsBW+p8U2u7vzgW2SQVmlNazg= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= @@ -292,7 +279,6 @@ github.com/onsi/gomega v1.10.3 h1:gph6h/qe9GSUw1NhH1gp+qb+h8rXD8Cy60Z32Qw3ELA= github.com/onsi/gomega v1.10.3/go.mod h1:V9xEwhxec5O8UDM77eCW8vLymOMltsqPVYWrpDsH8xc= github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= -github.com/peterbourgon/diskv v2.0.1+incompatible h1:UBdAOUP5p4RWqPBg048CAvpKN+vxiaj6gdUUzhl4XmI= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -306,7 +292,6 @@ github.com/prometheus/client_golang v1.0.0 h1:vrDKnkGzuGvhNAL56c7DBz29ZL+KxnoR0x github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4 h1:gQz4mCbXsO+nc9n1hCxHcGA3Zx3Eo+UHZoInFGUIXNM= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= @@ -387,7 +372,6 @@ golang.org/x/crypto v0.0.0-20190320223903-b7391e95e576/go.mod h1:djNgcEr1/C05ACk golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190617133340-57b3e21c3d56/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 h1:ObdrDkeb4kJdCP557AjRjq69pTHfNouLtWZG7j9rPN8= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200220183623-bac4c82f6975/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI= @@ -397,7 +381,6 @@ golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTk golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190930215403-16217165b5de h1:5hukYrvBGR8/eNkX5mdUezrA6JiaEZDtJb9Ei+1LlBs= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 h1:VLliZ0d+/avPrXXH+OakdXhpJuEoBZuwh1m2j7U6Iug= golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= @@ -424,11 +407,7 @@ golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20191004110552-13f9640d40b9/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20201006153459-a7d1128ccaa0/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201110031124-69a78807bb2b h1:uwuIcX0g4Yl1NC5XAz37xsr2lTtcqevgzYNVt49waME= -golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4 h1:4nGaVu0QrbjT/AK2PRLuQfQuh6DJve+pELhqTdAj3x0= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.0.0-20210614182718-04defd469f4e h1:XpT3nA5TvE525Ne3hInMh6+GETgn27Zfm9dxsThnX2Q= golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d h1:20cMwl2fHAzkJMEA+8J4JgqBQcQGzbisXo31MIeenXI= golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= @@ -440,7 +419,6 @@ golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -459,28 +437,22 @@ golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456 h1:ng0gs1AKnRRuEMZoTLLlbOd+C17zUDepwGQBb/n+JVg= golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191022100944-742c48ecaeb7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191026070338-33540a1f6037 h1:YyJpGZS1sBuBCzLAR1VEpK193GlqGZbnPFnPV/5Rsb4= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200217220822-9197077df867/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200728102440-3e129f6d46b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f h1:+Nyd8tzPX9R7BWHguqsrbFdRx3WQ/1ib8I44HXV5yTA= -golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f h1:+Nyd8tzPX9R7BWHguqsrbFdRx3WQ/1ib8I44HXV5yTA= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201117170446-d9b008d0a637 h1:O5hKNaGxIT4A8OTMnuh6UpmBdI3SAPxlZ3g0olDrJVM= golang.org/x/sys v0.0.0-20201117170446-d9b008d0a637/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210510120138-977fb7262007 h1:gG67DSER+11cZvqIMb8S8bt0vZtiN6xWYARwirrOSfE= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210616094352-59db8d763f22 h1:RqytpXGR1iVNX7psjB3ff8y7sNFinVFvkx1c8SjBkio= golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -489,13 +461,10 @@ golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fq golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= -golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2 h1:+DCIGbF/swA92ohVg0//6X2IVY3KZs6p9mix0ziNYJM= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 h1:SvFZT6jyqRaOeXpc5h/JSfZenJ2O330aBsf7JfSUXmQ= @@ -516,14 +485,12 @@ golang.org/x/tools v0.0.0-20190617190820-da514acc4774/go.mod h1:/rFqwRUd4F7ZHNgw golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190920225731-5eefd052ad72/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5 h1:hKsoRgsbwY1NafxrwTs+k64bikrLBkAgPir1TNCj3Zs= golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.1.3 h1:L69ShwSZEyCsLKoAxDKeMvLDZkumEe8gXUZAjab0tX8= -golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.5 h1:ouewzE6p+/VEB31YYnTbEJdi8pFqKp4P4n85vwo3DHA= +golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7 h1:9zdDQZ7Thm29KFXgAX/+yaf3eVbP7djjWp/dXAppNCc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -560,13 +527,11 @@ gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4 gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= -gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= @@ -579,11 +544,8 @@ gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkep gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/misc/10-aws.conflist b/misc/10-aws.conflist index ff2c531919..cfbc3e97e4 100644 --- a/misc/10-aws.conflist +++ b/misc/10-aws.conflist @@ -1,6 +1,7 @@ { "cniVersion": "0.4.0", "name": "aws-cni", + "disableCheck": true, "plugins": [ { "name": "aws-cni", diff --git a/pkg/apis/crd/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/crd/v1alpha1/zz_generated.deepcopy.go index 1a2497a83f..0a81a090fe 100644 --- a/pkg/apis/crd/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/crd/v1alpha1/zz_generated.deepcopy.go @@ -1,3 +1,4 @@ +//go:build !ignore_autogenerated // +build !ignore_autogenerated /* diff --git a/pkg/ipamd/ipamd.go b/pkg/ipamd/ipamd.go index f76c55b201..da217be445 100644 --- a/pkg/ipamd/ipamd.go +++ b/pkg/ipamd/ipamd.go @@ -165,6 +165,10 @@ const ( // Present and set to the empty string, which we use to mean "CNI DEL had occurred; networking has been removed from this pod" // The empty string one helps close a trace at pod shutdown where it looks like the pod still has its IP when the IP has been released envAnnotatePodIP = "ANNOTATE_POD_IP" + + // aws error codes for insufficient IP address scenario + INSUFFICIENT_CIDR_BLOCKS = "InsufficientCidrBlocks" + INSUFFICIENT_FREE_IP_SUBNET = "InsufficientFreeAddressesInSubnet" ) var log = logger.Get() @@ -340,11 +344,14 @@ func prometheusRegister() { } } -// containsInsufficientCidrBlocksError returns whether exceeds ENI's IP address limit -func containsInsufficientCidrBlocksError(err error) bool { +// containsInsufficientCIDRsOrSubnetIPs returns whether a CIDR cannot be carved in the subnet or subnet is running out of IP addresses +func containsInsufficientCIDRsOrSubnetIPs(err error) bool { var awsErr awserr.Error + // IP exhaustion can be due to Insufficient Cidr blocks or Insufficient Free Address in a Subnet + // In these 2 cases we will back off for 2 minutes before retrying if errors.As(err, &awsErr) { - return awsErr.Code() == "InsufficientCidrBlocks" + log.Debugf("Insufficient IP Addresses due to: %v\n", awsErr.Code()) + return awsErr.Code() == INSUFFICIENT_CIDR_BLOCKS || awsErr.Code() == INSUFFICIENT_FREE_IP_SUBNET } return false } @@ -413,7 +420,7 @@ func New(rawK8SClient client.Client, cachedK8SClient client.Client) (*IPAMContex mac := c.awsClient.GetPrimaryENImac() // retrieve security groups - if c.enableIPv4 || !c.disableENIProvisioning { + if c.enableIPv4 && !c.disableENIProvisioning { err = c.awsClient.RefreshSGIDs(mac) if err != nil { return nil, err @@ -570,7 +577,7 @@ func (c *IPAMContext) nodeInit() error { if err == nil && increasedPool { c.updateLastNodeIPPoolAction() } else if err != nil { - if containsInsufficientCidrBlocksError(err) { + if containsInsufficientCIDRsOrSubnetIPs(err) { log.Errorf("Unable to attach IPs/Prefixes for the ENI, subnet doesn't seem to have enough IPs/Prefixes. Consider using new subnet or carve a reserved range using create-subnet-cidr-reservation") c.lastInsufficientCidrError = time.Now() return nil @@ -786,7 +793,7 @@ func (c *IPAMContext) increaseDatastorePool(ctx context.Context) { increasedPool, err := c.tryAssignCidrs() if err != nil { log.Errorf(err.Error()) - if containsInsufficientCidrBlocksError(err) { + if containsInsufficientCIDRsOrSubnetIPs(err) { log.Errorf("Unable to attach IPs/Prefixes for the ENI, subnet doesn't seem to have enough IPs/Prefixes. Consider using new subnet or carve a reserved range using create-subnet-cidr-reservation") c.lastInsufficientCidrError = time.Now() return @@ -857,7 +864,7 @@ func (c *IPAMContext) tryAllocateENI(ctx context.Context) error { log.Warnf("Failed to allocate %d IP addresses on an ENI: %v", resourcesToAllocate, err) // Continue to process the allocated IP addresses ipamdErrInc("increaseIPPoolAllocIPAddressesFailed") - if containsInsufficientCidrBlocksError(err) { + if containsInsufficientCIDRsOrSubnetIPs(err) { log.Errorf("Unable to attach IPs/Prefixes for the ENI, subnet doesn't seem to have enough IPs/Prefixes. Consider using new subnet or carve a reserved range using create-subnet-cidr-reservation") c.lastInsufficientCidrError = time.Now() return err diff --git a/pkg/ipamd/rpc_handler.go b/pkg/ipamd/rpc_handler.go index 35898fe2f6..cb6b8503b9 100644 --- a/pkg/ipamd/rpc_handler.go +++ b/pkg/ipamd/rpc_handler.go @@ -110,7 +110,7 @@ func (s *server) AddNetwork(ctx context.Context, in *rpc.AddNetworkRequest) (*rp ipv4Addr = firstENI.PrivateIP branchENIMAC = firstENI.IfAddress vlanID = firstENI.VlanID - log.Infof("Pod vlandId: %d", vlanID) + log.Debugf("Pod vlandId: %d", vlanID) if ipv4Addr == "" || branchENIMAC == "" || vlanID == 0 { log.Errorf("Failed to parse pod-ENI annotation: %s", val) diff --git a/pkg/publisher/publisher.go b/pkg/publisher/publisher.go index a658d2b164..6e4fbca106 100644 --- a/pkg/publisher/publisher.go +++ b/pkg/publisher/publisher.go @@ -99,7 +99,6 @@ func New(ctx context.Context, region string, clusterID string, log logger.Logger if err != nil { return nil, errors.Wrap(err, "publisher: unable to obtain EC2 service client") } - clusterID = getClusterID(ec2Client) } diff --git a/scripts/dockerfiles/Dockerfile.init b/scripts/dockerfiles/Dockerfile.init index dcb3cfed6f..44b208554e 100644 --- a/scripts/dockerfiles/Dockerfile.init +++ b/scripts/dockerfiles/Dockerfile.init @@ -1,4 +1,3 @@ -ARG docker_arch ARG golang_image FROM $golang_image as builder @@ -14,7 +13,7 @@ RUN make plugins && make debug-script COPY . ./ # Build the architecture specific container image: -FROM $docker_arch/amazonlinux:2 +FROM public.ecr.aws/amazonlinux/amazonlinux:2 RUN yum update -y && \ yum install -y iproute procps-ng && \ yum clean all diff --git a/scripts/dockerfiles/Dockerfile.metrics b/scripts/dockerfiles/Dockerfile.metrics index f334797b44..694bd3972c 100644 --- a/scripts/dockerfiles/Dockerfile.metrics +++ b/scripts/dockerfiles/Dockerfile.metrics @@ -1,4 +1,3 @@ -ARG docker_arch ARG golang_image FROM $golang_image as builder @@ -16,7 +15,7 @@ RUN go mod download COPY . ./ RUN make build-metrics -FROM $docker_arch/amazonlinux:2 +FROM public.ecr.aws/amazonlinux/amazonlinux:2 RUN yum update -y && \ yum clean all diff --git a/scripts/dockerfiles/Dockerfile.release b/scripts/dockerfiles/Dockerfile.release index 48329e017d..67a77b8e9e 100644 --- a/scripts/dockerfiles/Dockerfile.release +++ b/scripts/dockerfiles/Dockerfile.release @@ -1,4 +1,3 @@ -ARG docker_arch ARG golang_image FROM $golang_image as builder @@ -19,7 +18,7 @@ COPY . ./ RUN make build-linux # Build the architecture specific container image: -FROM $docker_arch/amazonlinux:2 +FROM public.ecr.aws/amazonlinux/amazonlinux:2 RUN yum update -y && \ yum install -y iptables iproute jq && \ yum clean all diff --git a/scripts/dockerfiles/Dockerfile.test b/scripts/dockerfiles/Dockerfile.test index 2f518d60a1..6ac450862c 100644 --- a/scripts/dockerfiles/Dockerfile.test +++ b/scripts/dockerfiles/Dockerfile.test @@ -1,4 +1,3 @@ -ARG docker_arch ARG golang_image FROM $golang_image WORKDIR /go/src/github.com/aws/amazon-vpc-cni-k8s diff --git a/scripts/generate-cni-yaml.sh b/scripts/generate-cni-yaml.sh index 0a0ed4b61f..b8da0483e7 100755 --- a/scripts/generate-cni-yaml.sh +++ b/scripts/generate-cni-yaml.sh @@ -75,6 +75,8 @@ jq -c '.[]' $REGIONS_FILE | while read i; do --set init.image.region=$ecrRegion,\ --set init.image.account=$ecrAccount,\ --set init.image.domain=$ecrDomain,\ + --set init.image.tag=$VERSION,\ + --set image.tag=$VERSION,\ --set image.region=$ecrRegion,\ --set image.account=$ecrAccount,\ --set image.domain=$ecrDomain \ @@ -87,6 +89,7 @@ jq -c '.[]' $REGIONS_FILE | while read i; do --set image.region=$ecrRegion,\ --set image.account=$ecrAccount,\ --set image.domain=$ecrDomain \ + --set image.tag=$VERSION,\ --namespace $NAMESPACE \ $SCRIPTPATH/../charts/cni-metrics-helper > $NEW_METRICS_RESOURCES_YAML cat $NEW_METRICS_RESOURCES_YAML | grep -v 'helm.sh\|app.kubernetes.io/managed-by: Helm' > $BUILD_DIR/helm_annotations_removed.yaml @@ -149,4 +152,4 @@ echo " - $CALICO_OPERATOR_RESOURCES_YAML" echo " - $CALICO_CRS_RESOURCES_YAML" echo " - $CNI_TAR_RESOURCES_FILE" echo " - $METRICS_TAR_RESOURCES_FILE" -echo " - $CALICO_TAR_RESOURCES_FILE" +echo " - $CALICO_TAR_RESOURCES_FILE" \ No newline at end of file diff --git a/scripts/init.sh b/scripts/init.sh index 6951e09cf5..ae734a1778 100755 --- a/scripts/init.sh +++ b/scripts/init.sh @@ -1,10 +1,10 @@ #!/usr/bin/env bash -set -euxo pipefail +set -euo pipefail get_metadata() { - TOKEN=$(curl -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 60") + TOKEN=$(curl -Ss -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 60") attempts=60 false while [ "${?}" -gt 0 ]; do @@ -12,7 +12,7 @@ get_metadata() echo "Failed to get metdata" exit 1 fi - meta=$(curl -H "X-aws-ec2-metadata-token: $TOKEN" http://169.254.169.254/latest/meta-data/${1}) + meta=$(curl -Ss -H "X-aws-ec2-metadata-token: $TOKEN" http://169.254.169.254/latest/meta-data/${1}) if [ "${?}" -gt 0 ]; then let attempts-- sleep 0.5 diff --git a/scripts/upload-resources-to-github.sh b/scripts/upload-resources-to-github.sh index 0b890c8e3b..899d3192ee 100755 --- a/scripts/upload-resources-to-github.sh +++ b/scripts/upload-resources-to-github.sh @@ -7,6 +7,7 @@ SCRIPTPATH="$( cd "$(dirname "$0")" ; pwd -P )" VERSION=$(make -s -f $SCRIPTPATH/../Makefile version) BUILD_DIR=$SCRIPTPATH/../build/cni-rel-yamls/$VERSION BINARY_DIR=$SCRIPTPATH/../build/bin + CNI_TAR_RESOURCES_FILE=$BUILD_DIR/cni_individual-resources.tar METRICS_TAR_RESOURCES_FILE=$BUILD_DIR/cni_metrics_individual-resources.tar CALICO_TAR_RESOURCES_FILE=$BUILD_DIR/calico_individual-resources.tar @@ -18,6 +19,17 @@ CALICO_CRS_RESOURCES_YAML=$BUILD_DIR/calico-crs.yaml REGIONS_FILE=$SCRIPTPATH/../charts/regions.json BINARIES_ONLY="false" +PR_ID=$(uuidgen | cut -d '-' -f1) +BINARY_BASE="aws-vpc-cni-k8s" + +REPO="aws/amazon-vpc-cni-k8s" +GH_CLI_VERSION="0.10.1" +GH_CLI_CONFIG_PATH="${HOME}/.config/gh/config.yml" +KERNEL=$(uname -s | tr '[:upper:]' '[:lower:]') +OS="${KERNEL}" +if [[ "${KERNEL}" == "darwin" ]]; then + OS="macOS" +fi USAGE=$(cat << 'EOM' Usage: upload-resources-to-github [-b] @@ -81,14 +93,25 @@ upload_asset() { if [[ $response_code -eq 201 ]]; then asset_id=$(echo $response_content | jq '.id') ASSET_IDS_UPLOADED+=("$asset_id") - echo "Created asset ID $asset_id successfully" + echo "✅ Created asset ID $asset_id successfully" else echo -e "❌ Upload failed with response code $response_code and message \n$response_content ❌" exit 1 fi } -jq -c '.[]' $REGIONS_FILE | while read i; do +RESOURCES_TO_UPLOAD=("$CALICO_OPERATOR_RESOURCES_YAML" "$CALICO_CRS_RESOURCES_YAML" "$CNI_TAR_RESOURCES_FILE" "$METRICS_TAR_RESOURCES_FILE" "$CALICO_TAR_RESOURCES_FILE") +RESOURCES_TO_COPY=("$CALICO_OPERATOR_RESOURCES_YAML" "$CALICO_CRS_RESOURCES_YAML") + +COUNT=1 +echo -e "\nUploading release assets for release id '$RELEASE_ID' to Github" +for asset in ${RESOURCES_TO_UPLOAD[@]}; do + name=$(echo $asset | tr '/' '\n' | tail -1) + echo -e "\n $((COUNT++)). $name" + upload_asset $asset +done + +while read i; do ecrRegion=`echo $i | jq '.ecrRegion' -r` ecrAccount=`echo $i | jq '.ecrAccount' -r` ecrDomain=`echo $i | jq '.ecrDomain' -r` @@ -104,7 +127,7 @@ jq -c '.[]' $REGIONS_FILE | while read i; do NEW_METRICS_RESOURCES_YAML="${METRICS_RESOURCES_YAML}-${ecrRegion}.yaml" fi RESOURCES_TO_UPLOAD=("$NEW_CNI_RESOURCES_YAML" "$NEW_METRICS_RESOURCES_YAML") - + RESOURCES_TO_COPY=(${RESOURCES_TO_COPY[@]} "$NEW_CNI_RESOURCES_YAML" "$NEW_METRICS_RESOURCES_YAML") COUNT=1 echo -e "\nUploading release assets for release id '$RELEASE_ID' to Github" for asset in ${RESOURCES_TO_UPLOAD[@]}; do @@ -112,14 +135,115 @@ jq -c '.[]' $REGIONS_FILE | while read i; do echo -e "\n $((COUNT++)). $name" upload_asset $asset done -done +done < <(jq -c '.[]' $REGIONS_FILE) + +echo "✅ Attach artifacts to release page done" + +echo $REPO + +if [[ -z $(command -v gh) ]] || [[ ! $(gh --version) =~ $GH_CLI_VERSION ]]; then + mkdir -p "${BUILD_DIR}"/gh + curl -Lo "${BUILD_DIR}"/gh/gh.tar.gz "https://github.com/cli/cli/releases/download/v${GH_CLI_VERSION}/gh_${GH_CLI_VERSION}_${OS}_amd64.tar.gz" + tar -C "${BUILD_DIR}"/gh -xvf "${BUILD_DIR}/gh/gh.tar.gz" + export PATH="${BUILD_DIR}/gh/gh_${GH_CLI_VERSION}_${OS}_amd64/bin:$PATH" + if [[ ! $(gh --version) =~ $GH_CLI_VERSION ]]; then + echo "❌ Failed install of github cli" + exit 4 + fi +fi + +function fail() { + echo "❌ Create PR failed" + exit 5 +} -RESOURCES_TO_UPLOAD=("$CALICO_OPERATOR_RESOURCES_YAML" "CALICO_CRS_RESOURCES_YAML" "$CNI_TAR_RESOURCES_FILE" "$METRICS_TAR_RESOURCES_FILE" "$CALICO_TAR_RESOURCES_FILE") +CLONE_DIR="${BUILD_DIR}/config-sync" +SYNC_DIR="$CLONE_DIR" +echo $SYNC_DIR +rm -rf "${SYNC_DIR}" +mkdir -p "${SYNC_DIR}" +cd "${SYNC_DIR}" +gh repo clone aws/amazon-vpc-cni-k8s +DEFAULT_BRANCH=$(git rev-parse --abbrev-ref HEAD | tr -d '\n') + CONFIG_DIR=amazon-vpc-cni-k8s/config/master + cd $CONFIG_DIR + REPO_NAME=$(echo ${REPO} | cut -d'/' -f2) + git remote set-url origin https://"${GITHUB_USERNAME}":"${GITHUB_TOKEN}"@github.com/"${GITHUB_USERNAME}"/"${REPO_NAME}".git -COUNT=1 -echo -e "\nUploading release assets for release id '$RELEASE_ID' to Github" -for asset in ${RESOURCES_TO_UPLOAD[@]}; do - name=$(echo $asset | tr '/' '\n' | tail -1) - echo -e "\n $((COUNT++)). $name" - upload_asset $asset -done + git config user.name "eks-bot" + git config user.email "eks-bot@users.noreply.github.com" + + FORK_RELEASE_BRANCH="${BINARY_BASE}-${VERSION}-${PR_ID}" + git checkout -b "${FORK_RELEASE_BRANCH}" origin + + COUNT=1 + for asset in ${RESOURCES_TO_COPY[@]}; do + name=$(echo $asset | tr '/' '\n' | tail -1) + echo -e "\n $((COUNT++)). $name" + cp "$asset" . + done + + git add --all + git commit -m "${BINARY_BASE}: ${VERSION}" + +PR_BODY=$(cat << EOM +## ${BINARY_BASE} ${VERSION} Automated manifest folder Sync! 🤖🤖 + +### Description 📝 + +Updating all the generated release artifacts in master/config for master branch. + +EOM +) + + git push -u origin "${FORK_RELEASE_BRANCH}" + gh pr create --title "🥳 ${BINARY_BASE} ${VERSION} Automated manifest sync! 🥑" \ + --body "${PR_BODY}" --repo ${REPO} + + echo "✅ Manifest folder PR created for master" + +CLONE_DIR="${BUILD_DIR}/config-sync-release" +SYNC_DIR="$CLONE_DIR" +echo $SYNC_DIR +rm -rf "${SYNC_DIR}" +mkdir -p "${SYNC_DIR}" +cd "${SYNC_DIR}" +gh repo clone aws/amazon-vpc-cni-k8s +RELEASE_BRANCH=$(git branch -a --contains $VERSION | grep "upstream" | cut -d '/' -f3) +echo "Release branch $RELEASE_BRANCH" + CONFIG_DIR=amazon-vpc-cni-k8s/config/master + cd $CONFIG_DIR + REPO_NAME=$(echo ${REPO} | cut -d'/' -f2) + git remote set-url origin https://"${GITHUB_USERNAME}":"${GITHUB_TOKEN}"@github.com/"${GITHUB_USERNAME}"/"${REPO_NAME}".git + + git config user.name "eks-bot" + git config user.email "eks-bot@users.noreply.github.com" + + FORK_RELEASE_BRANCH="${BINARY_BASE}-${VERSION}-${PR_ID}" + git checkout -b "${FORK_RELEASE_BRANCH}" origin/$RELEASE_BRANCH + + COUNT=1 + for asset in ${RESOURCES_TO_COPY[@]}; do + name=$(echo $asset | tr '/' '\n' | tail -1) + echo -e "\n $((COUNT++)). $name" + cp "$asset" . + done + + git add --all + git commit -m "${BINARY_BASE}: ${VERSION}" + +PR_BODY=$(cat << EOM +## ${BINARY_BASE} ${VERSION} Automated manifest folder Sync! 🤖🤖 + +### Description 📝 + +Updating all the generated release artifacts in master/config for $RELEASE_BRANCH branch. + +EOM +) + + git push -u origin "${FORK_RELEASE_BRANCH}":$RELEASE_BRANCH + gh pr create --title "🥳 ${BINARY_BASE} ${VERSION} Automated manifest sync! 🥑" \ + --body "${PR_BODY}" --repo ${REPO} --base ${RELEASE_BRANCH} + + echo "✅ Manifest folder PR created for $RELEASE_BRANCH" diff --git a/test/framework/resources/k8s/utils/daemonset.go b/test/framework/resources/k8s/utils/daemonset.go index 94888bc818..c30dcb7bda 100644 --- a/test/framework/resources/k8s/utils/daemonset.go +++ b/test/framework/resources/k8s/utils/daemonset.go @@ -76,7 +76,7 @@ func updateDaemonsetEnvVarsAndWait(f *framework.Framework, dsName string, dsName if dsName != utils.MultusNodeName { _, err := f.K8sResourceManagers.DaemonSetManager().GetDaemonSet(dsNamespace, utils.MultusNodeName) if err == nil { - By("Restarting Multus daemonset to use the update aws-node changes") + By("Restarting Multus daemonset to use the updated aws-node changes") td := time.Now() updateDaemonsetEnvVarsAndWait(f, utils.MultusNodeName, dsNamespace, utils.MultusContainerName, map[string]string{ "forceUpdatedAt": td.String(), diff --git a/test/framework/utils/const.go b/test/framework/utils/const.go index 6ac8b4c613..1ca131f31e 100644 --- a/test/framework/utils/const.go +++ b/test/framework/utils/const.go @@ -25,9 +25,7 @@ const ( // See https://gallery.ecr.aws/r3i6j7b0/aws-vpc-cni-test-helper TestAgentImage = "public.ecr.aws/r3i6j7b0/aws-vpc-cni-test-helper:86ece934" -) -const ( PollIntervalShort = time.Second * 2 PollIntervalMedium = time.Second * 5 PollIntervalLong = time.Second * 20 diff --git a/test/integration-new/metrics-helper/metrics_helper_suite_test.go b/test/integration-new/metrics-helper/metrics_helper_suite_test.go index 81a0832280..67cc7376e7 100644 --- a/test/integration-new/metrics-helper/metrics_helper_suite_test.go +++ b/test/integration-new/metrics-helper/metrics_helper_suite_test.go @@ -43,12 +43,25 @@ var ( ngName string // node name which has CW publish metric privileges nodeName string + + clusterIDKeys []string +) + +const ( + DEFAULT_CLUSTER_ID = "k8s-cluster" ) // Parse optional flags for setting the cni metrics helper image func init() { flag.StringVar(&imageRepository, "cni-metrics-helper-image-repo", "602401143452.dkr.ecr.us-west-2.amazonaws.com/cni-metrics-helper", "CNI Metrics Helper Image Repository") flag.StringVar(&imageTag, "cni-metrics-helper-image-tag", "v1.7.10", "CNI Metrics Helper Image Tag") + + // Order in which we try fetch the keys and use it as CLUSTER_ID dimension + clusterIDKeys = []string{ + "eks:cluster-name", + "CLUSTER_ID", + "Name", + } } func TestCNIMetricsHelper(t *testing.T) { @@ -92,12 +105,28 @@ var _ = BeforeSuite(func() { instance, err := f.CloudServices.EC2().DescribeInstance(instanceID) Expect(err).ToNot(HaveOccurred()) + instanceTagKeyValuePair := map[string]string{ + "eks:cluster-name": "", + "CLUSTER_ID": "", + "Name": "", + } + for _, instanceTag := range instance.Tags { - if *instanceTag.Key == "Name" { - ngName = *instanceTag.Value + if _, ok := instanceTagKeyValuePair[*instanceTag.Key]; ok { + instanceTagKeyValuePair[*instanceTag.Key] = *instanceTag.Value + } + } + + for _, k := range clusterIDKeys { + if tagVal, ok := instanceTagKeyValuePair[k]; ok && tagVal != "" { + ngName = tagVal + break } } - Expect(ngName).ToNot(BeEmpty()) + + if ngName == "" { + ngName = DEFAULT_CLUSTER_ID + } By("getting the node instance role") instanceProfileRoleName := strings.Split(*instance.IamInstanceProfile.Arn, "instance-profile/")[1]