diff --git a/Makefile b/Makefile index dcfc98d279..a25bd60134 100644 --- a/Makefile +++ b/Makefile @@ -64,6 +64,12 @@ unit-test: GOOS=linux CGO_ENABLED=1 go test -v -cover -race -timeout 10s ./pkg/eniconfig/... GOOS=linux CGO_ENABLED=1 go test -v -cover -race -timeout 10s ./ipamd/... +docker-unit-test: + docker run -v $(shell pwd):/usr/src/app/src/github.com/aws/amazon-vpc-cni-k8s \ + --workdir=/usr/src/app/src/github.com/aws/amazon-vpc-cni-k8s \ + --env GOPATH=/usr/src/app \ + golang:1.10 make unit-test + # golint # To install: go get -u golang.org/x/lint/golint lint: diff --git a/README.md b/README.md index 4d65b9d358..d38ff9d6c1 100644 --- a/README.md +++ b/README.md @@ -136,6 +136,11 @@ Default: None Specifies the number of free IP addresses that the `ipamD` daemon should attempt to keep available for pod assignment on the node\. For example, if `WARM_IP_TARGET` is set to 10, then `ipamD` attempts to keep 10 free IP addresses available at all times\. If the elastic network interfaces on the node are unable to provide these free addresses, `ipamD` attempts to allocate more interfaces until `WARM_IP_TARGET` free IP addresses are available\. This environment variable overrides `WARM_ENI_TARGET` behavior\. +`MAX_ENI` +Type: Integer +Default: None +Specifies the maximum number of ENIs that will be attached to the node. When MAX_ENI is unset or 0 (or lower), the setting is not used, and the maximum number of ENIs is always equal to the maximum number for the instance type in question. Even when MAX_ENI is a positive number, it is limited by the maximum number for the instance type. + ### Notes `L-IPAMD`(aws-node daemonSet) running on every worker node requires access to kubernetes API server. If it can **not** reach kubernetes API server, ipamD will exit and CNI will not be able to get any IP address for Pods. Here is a way to confirm if `L-IPAMD` has access to the kubernetes API server. diff --git a/ipamd/ipamd.go b/ipamd/ipamd.go index e83373f7bd..a5b24845e1 100644 --- a/ipamd/ipamd.go +++ b/ipamd/ipamd.go @@ -76,6 +76,13 @@ const ( envWarmENITarget = "WARM_ENI_TARGET" defaultWarmENITarget = 1 + // This environment variable is used to specify the maximum number of ENIs that will be allocated. + // When it is not set or less than 1, the default is to use the maximum available for the instance type. + // + // The maximum number of ENIs is in any case limited to the amount allowed for the instance type. + envMaxENI = "MAX_ENI" + defaultMaxENI = -1 + // This environment is used to specify whether Pods need to use securitygroup and subnet defined in ENIConfig CRD // When it is NOT set or set to false, ipamD will use primary interface security group and subnet for Pod network. envCustomNetworkCfg = "AWS_VPC_K8S_CNI_CUSTOM_NETWORK_CFG" @@ -194,10 +201,13 @@ func New(k8sapiClient k8sapi.K8SAPIs, eniConfig *eniconfig.ENIConfigController) func (c *IPAMContext) nodeInit() error { ipamdActionsInprogress.WithLabelValues("nodeInit").Add(float64(1)) defer ipamdActionsInprogress.WithLabelValues("nodeInit").Sub(float64(1)) - maxENIs, err := c.awsClient.GetENILimit() - if err == nil { + + instanceMaxENIs, _ := c.awsClient.GetENILimit() + maxENIs := getMaxENI(instanceMaxENIs) + if maxENIs >= 1 { enisMax.Set(float64(maxENIs)) } + maxIPs, err := c.awsClient.GetENIipLimit() if err == nil { ipMax.Set(float64(maxIPs * int64(maxENIs))) @@ -427,8 +437,11 @@ func (c *IPAMContext) increaseIPPool() { return } - maxENIs, err := c.awsClient.GetENILimit() - enisMax.Set(float64(maxENIs)) + instanceMaxENIs, err := c.awsClient.GetENILimit() + maxENIs := getMaxENI(instanceMaxENIs) + if maxENIs >= 1 { + enisMax.Set(float64(maxENIs)) + } if err == nil && maxENIs == c.dataStore.GetENIs() { log.Debugf("Skipping increase IPPOOL due to max ENI already attached to the instance : %d", maxENIs) @@ -626,6 +639,32 @@ func (c *IPAMContext) waitENIAttached(eni string) (awsutils.ENIMetadata, error) } } +// getMaxENI returns the maximum number of ENIs for this instance, which is +// the lesser of the given lower bound (for example, the limit for the instance +// type) and a value configured via the MAX_ENI environment variable. +// +// If the value configured via environment variable is 0 or less, it is +// ignored, and the lowerBound is returned. +func getMaxENI(lowerBound int) int { + inputStr, found := os.LookupEnv(envMaxENI) + + envMax := defaultMaxENI + if found { + if input, err := strconv.Atoi(inputStr); err == nil && input >= 1 { + log.Debugf("Using MAX_ENI %v", input) + envMax = input + } + } + + // If envMax is defined (>=1) and is less than the input lower bound, return + // envMax. + if envMax >= 1 && envMax < lowerBound { + return envMax + } + + return lowerBound +} + func getWarmENITarget() int { inputStr, found := os.LookupEnv(envWarmENITarget) diff --git a/ipamd/ipamd_test.go b/ipamd/ipamd_test.go index 148d8acf71..01738b0644 100644 --- a/ipamd/ipamd_test.go +++ b/ipamd/ipamd_test.go @@ -361,6 +361,41 @@ func TestGetWarmENITarget(t *testing.T) { assert.Equal(t, warmIPTarget, noWarmIPTarget) } +func TestGetMaxENI(t *testing.T) { + ctrl, _, _, _, _, _ := setup(t) + defer ctrl.Finish() + + // MaxENI 5 is less than lower bound of 10, so 5 + os.Setenv("MAX_ENI", "5") + maxENI := getMaxENI(10) + assert.Equal(t, maxENI, 5) + + // MaxENI 5 is greater than lower bound of 4, so 4 + os.Setenv("MAX_ENI", "5") + maxENI = getMaxENI(4) + assert.Equal(t, maxENI, 4) + + // MaxENI 0 is 0, which means disabled; so use lower bound + os.Setenv("MAX_ENI", "0") + maxENI = getMaxENI(4) + assert.Equal(t, maxENI, 4) + + // MaxENI 1 is less than lower bound of 4, so 1. + os.Setenv("MAX_ENI", "1") + maxENI = getMaxENI(4) + assert.Equal(t, maxENI, 1) + + // Empty MaxENI means disabled, so use lower bound + os.Unsetenv("MAX_ENI") + maxENI = getMaxENI(10) + assert.Equal(t, maxENI, 10) + + // Invalid MaxENI means disabled, so use lower bound + os.Setenv("MAX_ENI", "non-integer-string") + maxENI = getMaxENI(10) + assert.Equal(t, maxENI, 10) +} + func TestGetCurWarmIPTarget(t *testing.T) { ctrl, mockAWS, mockK8S, _, mockNetwork, _ := setup(t) defer ctrl.Finish()