Skip to content

Commit

Permalink
Merge pull request #251 from pdbogen/pdbogen-max-eni
Browse files Browse the repository at this point in the history
add an environment variable to limit the number of ENIs
  • Loading branch information
mattlandis authored Jan 22, 2019
2 parents ced0fba + 9f471e2 commit 2cb11cb
Show file tree
Hide file tree
Showing 4 changed files with 89 additions and 4 deletions.
6 changes: 6 additions & 0 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,12 @@ unit-test:
GOOS=linux CGO_ENABLED=1 go test -v -cover -race -timeout 10s ./pkg/eniconfig/...
GOOS=linux CGO_ENABLED=1 go test -v -cover -race -timeout 10s ./ipamd/...

docker-unit-test:
docker run -v $(shell pwd):/usr/src/app/src/github.com/aws/amazon-vpc-cni-k8s \
--workdir=/usr/src/app/src/github.com/aws/amazon-vpc-cni-k8s \
--env GOPATH=/usr/src/app \
golang:1.10 make unit-test

# golint
# To install: go get -u golang.org/x/lint/golint
lint:
Expand Down
5 changes: 5 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -136,6 +136,11 @@ Default: None
Specifies the number of free IP addresses that the `ipamD` daemon should attempt to keep available for pod assignment on the node\. For example, if `WARM_IP_TARGET` is set to 10, then `ipamD` attempts to keep 10 free IP addresses available at all times\. If the elastic network interfaces on the node are unable to provide these free addresses, `ipamD` attempts to allocate more interfaces until `WARM_IP_TARGET` free IP addresses are available\.
This environment variable overrides `WARM_ENI_TARGET` behavior\.

`MAX_ENI`
Type: Integer
Default: None
Specifies the maximum number of ENIs that will be attached to the node. When MAX_ENI is unset or 0 (or lower), the setting is not used, and the maximum number of ENIs is always equal to the maximum number for the instance type in question. Even when MAX_ENI is a positive number, it is limited by the maximum number for the instance type.

### Notes

`L-IPAMD`(aws-node daemonSet) running on every worker node requires access to kubernetes API server. If it can **not** reach kubernetes API server, ipamD will exit and CNI will not be able to get any IP address for Pods. Here is a way to confirm if `L-IPAMD` has access to the kubernetes API server.
Expand Down
47 changes: 43 additions & 4 deletions ipamd/ipamd.go
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,13 @@ const (
envWarmENITarget = "WARM_ENI_TARGET"
defaultWarmENITarget = 1

// This environment variable is used to specify the maximum number of ENIs that will be allocated.
// When it is not set or less than 1, the default is to use the maximum available for the instance type.
//
// The maximum number of ENIs is in any case limited to the amount allowed for the instance type.
envMaxENI = "MAX_ENI"
defaultMaxENI = -1

// This environment is used to specify whether Pods need to use securitygroup and subnet defined in ENIConfig CRD
// When it is NOT set or set to false, ipamD will use primary interface security group and subnet for Pod network.
envCustomNetworkCfg = "AWS_VPC_K8S_CNI_CUSTOM_NETWORK_CFG"
Expand Down Expand Up @@ -194,10 +201,13 @@ func New(k8sapiClient k8sapi.K8SAPIs, eniConfig *eniconfig.ENIConfigController)
func (c *IPAMContext) nodeInit() error {
ipamdActionsInprogress.WithLabelValues("nodeInit").Add(float64(1))
defer ipamdActionsInprogress.WithLabelValues("nodeInit").Sub(float64(1))
maxENIs, err := c.awsClient.GetENILimit()
if err == nil {

instanceMaxENIs, _ := c.awsClient.GetENILimit()
maxENIs := getMaxENI(instanceMaxENIs)
if maxENIs >= 1 {
enisMax.Set(float64(maxENIs))
}

maxIPs, err := c.awsClient.GetENIipLimit()
if err == nil {
ipMax.Set(float64(maxIPs * int64(maxENIs)))
Expand Down Expand Up @@ -427,8 +437,11 @@ func (c *IPAMContext) increaseIPPool() {
return
}

maxENIs, err := c.awsClient.GetENILimit()
enisMax.Set(float64(maxENIs))
instanceMaxENIs, err := c.awsClient.GetENILimit()
maxENIs := getMaxENI(instanceMaxENIs)
if maxENIs >= 1 {
enisMax.Set(float64(maxENIs))
}

if err == nil && maxENIs == c.dataStore.GetENIs() {
log.Debugf("Skipping increase IPPOOL due to max ENI already attached to the instance : %d", maxENIs)
Expand Down Expand Up @@ -626,6 +639,32 @@ func (c *IPAMContext) waitENIAttached(eni string) (awsutils.ENIMetadata, error)
}
}

// getMaxENI returns the maximum number of ENIs for this instance, which is
// the lesser of the given lower bound (for example, the limit for the instance
// type) and a value configured via the MAX_ENI environment variable.
//
// If the value configured via environment variable is 0 or less, it is
// ignored, and the lowerBound is returned.
func getMaxENI(lowerBound int) int {
inputStr, found := os.LookupEnv(envMaxENI)

envMax := defaultMaxENI
if found {
if input, err := strconv.Atoi(inputStr); err == nil && input >= 1 {
log.Debugf("Using MAX_ENI %v", input)
envMax = input
}
}

// If envMax is defined (>=1) and is less than the input lower bound, return
// envMax.
if envMax >= 1 && envMax < lowerBound {
return envMax
}

return lowerBound
}

func getWarmENITarget() int {
inputStr, found := os.LookupEnv(envWarmENITarget)

Expand Down
35 changes: 35 additions & 0 deletions ipamd/ipamd_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -361,6 +361,41 @@ func TestGetWarmENITarget(t *testing.T) {
assert.Equal(t, warmIPTarget, noWarmIPTarget)
}

func TestGetMaxENI(t *testing.T) {
ctrl, _, _, _, _, _ := setup(t)
defer ctrl.Finish()

// MaxENI 5 is less than lower bound of 10, so 5
os.Setenv("MAX_ENI", "5")
maxENI := getMaxENI(10)
assert.Equal(t, maxENI, 5)

// MaxENI 5 is greater than lower bound of 4, so 4
os.Setenv("MAX_ENI", "5")
maxENI = getMaxENI(4)
assert.Equal(t, maxENI, 4)

// MaxENI 0 is 0, which means disabled; so use lower bound
os.Setenv("MAX_ENI", "0")
maxENI = getMaxENI(4)
assert.Equal(t, maxENI, 4)

// MaxENI 1 is less than lower bound of 4, so 1.
os.Setenv("MAX_ENI", "1")
maxENI = getMaxENI(4)
assert.Equal(t, maxENI, 1)

// Empty MaxENI means disabled, so use lower bound
os.Unsetenv("MAX_ENI")
maxENI = getMaxENI(10)
assert.Equal(t, maxENI, 10)

// Invalid MaxENI means disabled, so use lower bound
os.Setenv("MAX_ENI", "non-integer-string")
maxENI = getMaxENI(10)
assert.Equal(t, maxENI, 10)
}

func TestGetCurWarmIPTarget(t *testing.T) {
ctrl, mockAWS, mockK8S, _, mockNetwork, _ := setup(t)
defer ctrl.Finish()
Expand Down

0 comments on commit 2cb11cb

Please sign in to comment.