diff --git a/README.md b/README.md index 970b60daab..1278baac4f 100644 --- a/README.md +++ b/README.md @@ -93,6 +93,18 @@ Type: Boolean Default: `false` Specifies that your pods may use subnets and security groups that are independent of your worker node's VPC configuration\. By default, pods share the same subnet and security groups as the worker node's primary interface\. Setting this variable to `true` causes `ipamD` to use the security groups and VPC subnet in a worker node's `ENIConfig` for elastic network interface allocation\. You must create an `ENIConfig` custom resource definition for each subnet that your pods will reside in, and then annotate each worker node to use a specific `ENIConfig` \(multiple worker nodes can be annotated with the same `ENIConfig`\)\. Worker nodes can only be annotated with a single `ENIConfig` at a time, and the subnet in the `ENIConfig` must belong to the same Availability Zone that the worker node resides in\. For more information, see [https://github.com/aws/amazon-vpc-cni-k8s/pull/165](https://github.com/aws/amazon-vpc-cni-k8s/pull/165)\. +`ENI_CONFIG_ANNOTATION_DEF` +Type: String +Default: k8s.amazonaws.com/eniConfig +Specifies node annotation key name. This should be used when AWS_VPC_K8S_CNI_CUSTOM_NETWORK_CFG=true. Custom annotation value will be used to set eniConfig name. See ENI_CONFIG_LABEL_DEF for examples. + +`ENI_CONFIG_LABEL_DEF` +Type: String +Default: k8s.amazonaws.com/eniConfig +Specifies node label key name. This should be used when AWS_VPC_K8S_CNI_CUSTOM_NETWORK_CFG=true. Custom label value will be used to set eniConfig name. Note that annotations will take precedence over labels. To use labels, ensure default annotation k8s.amazonaws.com/eniConfig is not set on node and ENI_CONFIG_ANNOTATION_DEF is not used. +For example, you can use custom node label key _example.com/eniConfig_ by setting ENI_CONFIG_LABEL_DEF=example.com/eniConfig. Then you can set that label on node with value of your custom eniConfig name like `eniConfig-us-east-1a`. +In other example if your node has label _failure-domain.beta.kubernetes.io/zone_ and its value is set to availability zone `us-east-1a`, you can set ENI_CONFIG_LABEL_DEF=failure-domain.beta.kubernetes.io/zone. In such case eniConfig would be set to availability zone name `us-east-1a`, and you could use it to differentiate configs between zones. + `AWS_VPC_K8S_CNI_EXTERNALSNAT` Type: Boolean Default: `false` diff --git a/pkg/eniconfig/eniconfig.go b/pkg/eniconfig/eniconfig.go index 135b0fd9f5..b229652e03 100644 --- a/pkg/eniconfig/eniconfig.go +++ b/pkg/eniconfig/eniconfig.go @@ -35,8 +35,19 @@ import ( ) const ( - eniConfigAnnotationDef = "k8s.amazonaws.com/eniConfig" - eniConfigDefault = "default" + defaultEniConfigAnnotationDef = "k8s.amazonaws.com/eniConfig" + defaultEniConfigLabelDef = "k8s.amazonaws.com/eniConfig" + eniConfigDefault = "default" + + // when "ENI_CONFIG_LABEL_DEF is defined, ENIConfigController will use that label key to + // search if is setting value for eniConfigLabelDef + // Example: + // Node has set label k8s.amazonaws.com/eniConfigCustom=customConfig + // We can get that value in controller by setting environmental variable ENI_CONFIG_LABEL_DEF + // ENI_CONFIG_LABEL_DEF=k8s.amazonaws.com/eniConfigOverride + // This will set eniConfigLabelDef to eniConfigOverride + envEniConfigAnnotationDef = "ENI_CONFIG_ANNOTATION_DEF" + envEniConfigLabelDef = "ENI_CONFIG_LABEL_DEF" ) type ENIConfig interface { @@ -48,10 +59,12 @@ var ErrNoENIConfig = errors.New("eniconfig: eniconfig is not available") // ENIConfigController defines global context for ENIConfig controller type ENIConfigController struct { - eni map[string]*v1alpha1.ENIConfigSpec - myENI string - eniLock sync.RWMutex - myNodeName string + eni map[string]*v1alpha1.ENIConfigSpec + myENI string + eniLock sync.RWMutex + myNodeName string + eniConfigAnnotationDef string + eniConfigLabelDef string } // ENIConfigInfo returns locally cached ENIConfigs @@ -66,6 +79,8 @@ func NewENIConfigController() *ENIConfigController { myNodeName: os.Getenv("MY_NODE_NAME"), eni: make(map[string]*v1alpha1.ENIConfigSpec), myENI: eniConfigDefault, + eniConfigAnnotationDef: getEniConfigAnnotationDef(), + eniConfigLabelDef: getEniConfigLabelDef(), } } @@ -104,22 +119,22 @@ func (h *Handler) Handle(ctx context.Context, event sdk.Event) error { case *corev1.Node: - log.Infof("Handle corev1.Node: %s, %v", o.GetName(), o.GetAnnotations()) + log.Infof("Handle corev1.Node: %s, %v, %v", o.GetName(), o.GetAnnotations(), o.GetLabels()) + // Get annotations if not found get labels if not found fallback use default if h.controller.myNodeName == o.GetName() { - annotation := o.GetAnnotations() - - val, ok := annotation[eniConfigAnnotationDef] - if ok { - h.controller.eniLock.Lock() - defer h.controller.eniLock.Unlock() - h.controller.myENI = val - log.Infof(" Setting myENI to: %s", val) - } else { - h.controller.eniLock.Lock() - defer h.controller.eniLock.Unlock() - h.controller.myENI = eniConfigDefault - log.Infof(" Setting myENI to: %s", eniConfigDefault) + + val, ok := o.GetAnnotations()[h.controller.eniConfigAnnotationDef] + if !ok { + val, ok = o.GetLabels()[h.controller.eniConfigLabelDef] + if !ok { + val = eniConfigDefault + } } + + h.controller.eniLock.Lock() + defer h.controller.eniLock.Unlock() + h.controller.myENI = val + log.Infof(" Setting myENI to: %s", val) } } return nil @@ -182,3 +197,33 @@ func (eniCfg *ENIConfigController) MyENIConfig() (*v1alpha1.ENIConfigSpec, error } return nil, ErrNoENIConfig } + +// getEniConfigAnnotationDef returns eniConfigAnnotation +func getEniConfigAnnotationDef() string { + inputStr, found := os.LookupEnv(envEniConfigAnnotationDef) + + if !found { + return defaultEniConfigAnnotationDef + } + if len(inputStr) > 0 { + log.Debugf("Using ENI_CONFIG_ANNOTATION_DEF %v", inputStr) + return inputStr + } + + return defaultEniConfigAnnotationDef +} + +// getEniConfigLabelDef returns eniConfigLabel name +func getEniConfigLabelDef() string { + inputStr, found := os.LookupEnv(envEniConfigLabelDef) + + if !found { + return defaultEniConfigLabelDef + } + if len(inputStr) > 0 { + log.Debugf("Using ENI_CONFIG_LABEL_DEF %v", inputStr) + return inputStr + } + + return defaultEniConfigLabelDef +} diff --git a/pkg/eniconfig/eniconfig_test.go b/pkg/eniconfig/eniconfig_test.go index a87810953a..f5afdc0e55 100644 --- a/pkg/eniconfig/eniconfig_test.go +++ b/pkg/eniconfig/eniconfig_test.go @@ -60,6 +60,7 @@ func updateNodeAnnotation(hdlr sdk.Handler, nodeName string, configName string, Deleted: toDelete, } eniAnnotations := make(map[string]string) + eniConfigAnnotationDef := getEniConfigAnnotationDef() if !toDelete { eniAnnotations[eniConfigAnnotationDef] = configName @@ -68,6 +69,34 @@ func updateNodeAnnotation(hdlr sdk.Handler, nodeName string, configName string, hdlr.Handle(nil, event) } +func updateNodeLabel(hdlr sdk.Handler, nodeName string, configName string, toDelete bool) { + + node := corev1.Node{ + TypeMeta: metav1.TypeMeta{APIVersion: corev1.SchemeGroupVersion.String()}, + ObjectMeta: metav1.ObjectMeta{ + Name: nodeName, + }, + } + accessor, err := meta.Accessor(&node) + + if err != nil { + fmt.Printf("Failed to call meta.Access %v", err) + } + + event := sdk.Event{ + Object: &node, + Deleted: toDelete, + } + eniLabels := make(map[string]string) + eniConfigLabelDef := getEniConfigLabelDef() + + if !toDelete { + eniLabels[eniConfigLabelDef] = configName + } + accessor.SetLabels(eniLabels) + hdlr.Handle(nil, event) +} + func TestENIConfig(t *testing.T) { testENIConfigController := NewENIConfigController() @@ -105,7 +134,7 @@ func TestENIConfig(t *testing.T) { } func TestNodeENIConfig(t *testing.T) { - myNodeName := "testMyNode" + myNodeName := "testMyNodeWithAnnotation" myENIConfig := "testMyENIConfig" os.Setenv("MY_NODE_NAME", myNodeName) testENIConfigController := NewENIConfigController() @@ -144,3 +173,68 @@ func TestNodeENIConfig(t *testing.T) { assert.Equal(t, defaultCfg, *outputCfg) } + +func TestNodeENIConfigLabel(t *testing.T) { + myNodeName := "testMyNodeWithLabel" + myENIConfig := "testMyENIConfig" + os.Setenv("MY_NODE_NAME", myNodeName) + testENIConfigController := NewENIConfigController() + + testHandler := NewHandler(testENIConfigController) + updateNodeLabel(testHandler, myNodeName, myENIConfig, false) + + // If there is no ENI config + _, err := testENIConfigController.MyENIConfig() + assert.Error(t, err) + + // Add eniconfig for myENIConfig + group1Cfg := v1alpha1.ENIConfigSpec{ + SecurityGroups: []string{"sg21-id", "sg22-id"}, + Subnet: "subnet21"} + updateENIConfig(testHandler, myENIConfig, group1Cfg, false) + outputCfg, err := testENIConfigController.MyENIConfig() + assert.NoError(t, err) + assert.Equal(t, group1Cfg, *outputCfg) + + // Add default config + defaultSGs := []string{"sg1-id", "sg2-id"} + defaultSubnet := "subnet1" + defaultCfg := v1alpha1.ENIConfigSpec{ + SecurityGroups: defaultSGs, + Subnet: defaultSubnet} + updateENIConfig(testHandler, eniConfigDefault, defaultCfg, false) + outputCfg, err = testENIConfigController.MyENIConfig() + assert.NoError(t, err) + assert.Equal(t, group1Cfg, *outputCfg) + + // Delete node's myENIConfig annotation, then the value should fallback to default + updateNodeLabel(testHandler, myNodeName, myENIConfig, true) + outputCfg, err = testENIConfigController.MyENIConfig() + assert.NoError(t, err) + assert.Equal(t, defaultCfg, *outputCfg) + +} + +func TestGetEniConfigAnnotationDefDefault(t *testing.T) { + os.Unsetenv(envEniConfigAnnotationDef) + eniConfigAnnotationDef := getEniConfigAnnotationDef() + assert.Equal(t, eniConfigAnnotationDef, defaultEniConfigAnnotationDef) +} + +func TestGetEniConfigAnnotationlDefCustom(t *testing.T) { + os.Setenv(envEniConfigAnnotationDef, "k8s.amazonaws.com/eniConfigCustom") + eniConfigAnnotationDef := getEniConfigAnnotationDef() + assert.Equal(t, eniConfigAnnotationDef, "k8s.amazonaws.com/eniConfigCustom") +} + +func TestGetEniConfigLabelDefDefault(t *testing.T) { + os.Unsetenv(envEniConfigLabelDef) + eniConfigLabelDef := getEniConfigLabelDef() + assert.Equal(t, eniConfigLabelDef, defaultEniConfigLabelDef) +} + +func TestGetEniConfigLabelDefCustom(t *testing.T) { + os.Setenv(envEniConfigLabelDef, "k8s.amazonaws.com/eniConfigCustom") + eniConfigLabelDef := getEniConfigLabelDef() + assert.Equal(t, eniConfigLabelDef, "k8s.amazonaws.com/eniConfigCustom") +}