From ae9a9bdca208c9905c28176c7645e3b0c3044e06 Mon Sep 17 00:00:00 2001 From: Abhinav Pathak Date: Tue, 27 Apr 2021 17:15:28 +0530 Subject: [PATCH 1/4] add custom networking e2e test suite --- test/e2e/README.md | 33 +++ .../custom_networking_suite_test.go | 201 ++++++++++++++ .../custom_networking_test.go | 130 +++++++++ test/framework/framework.go | 3 +- test/framework/options.go | 2 + test/framework/resources/aws/cloud.go | 31 ++- .../resources/aws/services/cloudformation.go | 102 ++++++++ test/framework/resources/aws/services/ec2.go | 108 ++++++++ test/framework/resources/aws/services/eks.go | 46 ++++ .../resources/aws/utils/nodegroup.go | 246 ++++++++++++++++++ test/framework/resources/k8s/manager.go | 7 + .../resources/k8s/manifest/deployment.go | 8 + .../resources/k8s/resources/configmap.go | 46 ++++ .../resources/k8s/resources/eniconfig.go | 15 ++ .../framework/resources/k8s/resources/node.go | 26 ++ test/go.mod | 2 + test/go.sum | 6 + 17 files changed, 1003 insertions(+), 9 deletions(-) create mode 100644 test/e2e/README.md create mode 100644 test/e2e/custom-networking/custom_networking_suite_test.go create mode 100644 test/e2e/custom-networking/custom_networking_test.go create mode 100644 test/framework/resources/aws/services/cloudformation.go create mode 100644 test/framework/resources/aws/services/eks.go create mode 100644 test/framework/resources/aws/utils/nodegroup.go create mode 100644 test/framework/resources/k8s/resources/configmap.go diff --git a/test/e2e/README.md b/test/e2e/README.md new file mode 100644 index 0000000000..3457d8d989 --- /dev/null +++ b/test/e2e/README.md @@ -0,0 +1,33 @@ +##CNI E2E Test Suites + +The package contains e2e tests suites for `amazon-vpc-cni-k8s` . + +###Prerequisites +- Custom Networking Test + - No existing node group should be present the test creates new self managed node group with the reduced MAX_POD value. + +####Testing +Set the environment variables that will be passed to Ginkgo script. If you want to directly pass the arguments you can skip to next step. +``` +CLUSTER_NAME= +VPC_ID= +KUBECONFIG= +AWS_REGION= +# Optional endpooint variable +EKS_ENDPOINT= +``` + +To run the test switch to the integration folder. For instance running the custom-networking test from root of the project. +```bash +cd test/e2e/custom-networking +``` + +Run Ginkgo test suite +```bash +ginkgo -v --failOnPending -- \ + --cluster-kubeconfig=$KUBECONFIG \ + --cluster-name=$CLUSTER_NAME \ + --aws-region=$AWS_REGION \ + --aws-vpc-id=$VPC_ID \ + --eks-endpoint=$EKS_ENDPOINT +``` \ No newline at end of file diff --git a/test/e2e/custom-networking/custom_networking_suite_test.go b/test/e2e/custom-networking/custom_networking_suite_test.go new file mode 100644 index 0000000000..a5f93850c6 --- /dev/null +++ b/test/e2e/custom-networking/custom_networking_suite_test.go @@ -0,0 +1,201 @@ +// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"). You may +// not use this file except in compliance with the License. A copy of the +// License is located at +// +// http://aws.amazon.com/apache2.0/ +// +// or in the "license" file accompanying this file. This file is distributed +// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +// express or implied. See the License for the specific language governing +// permissions and limitations under the License. + +package custom_networking + +import ( + "flag" + "fmt" + "net" + "testing" + "time" + + "github.com/aws/amazon-vpc-cni-k8s/pkg/apis/crd/v1alpha1" + "github.com/aws/amazon-vpc-cni-k8s/test/framework" + awsUtils "github.com/aws/amazon-vpc-cni-k8s/test/framework/resources/aws/utils" + "github.com/aws/amazon-vpc-cni-k8s/test/framework/resources/k8s/manifest" + k8sUtils "github.com/aws/amazon-vpc-cni-k8s/test/framework/resources/k8s/utils" + "github.com/aws/amazon-vpc-cni-k8s/test/framework/utils" + + "github.com/apparentlymart/go-cidr/cidr" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +func TestCustomNetworking(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "CNI Custom Networking e2e Test Suite") +} + +var ( + f *framework.Framework + // VPC Configuration with the details of public subnet and availability + // zone present in the cluster's subnets + clusterVPCConfig *awsUtils.ClusterVPCConfig + // The CIDR Range that will be associated with the VPC to create new + // subnet for custom networking + cidrRangeString string + cidrRange *net.IPNet + cidrBlockAssociationID string + // Key Pair is required for creating a self managed node group + keyPairName = "custom-networking-key" + // Security Group that will be used in ENIConfig + customNetworkingSGID string + customNetworkingSGOpenPort = 8080 + customNetworkingSubnetIDList []string + // List of ENIConfig per Availability Zone + eniConfigList []*v1alpha1.ENIConfig + // Properties of the self managed node group created using CFN template + nodeGroupProperties awsUtils.NodeGroupProperties + err error +) + +// Parse test specific variable from flag +func init() { + flag.StringVar(&cidrRangeString, "custom-networking-cidr-range", "10.10.0.0/16", "custom networking cidr range to be associated with the VPC") +} + +var _ = BeforeSuite(func() { + f = framework.New(framework.GlobalOptions) + + _, cidrRange, err = net.ParseCIDR(cidrRangeString) + Expect(err).ToNot(HaveOccurred()) + + By("creating test namespace") + f.K8sResourceManagers.NamespaceManager(). + CreateNamespace(utils.DefaultTestNamespace) + + By("getting the cluster VPC Config") + clusterVPCConfig, err = awsUtils.GetClusterVPCConfig(f) + Expect(err).ToNot(HaveOccurred()) + + By("creating ec2 key-pair for the new node group") + _, err := f.CloudServices.EC2().CreateKey(keyPairName) + Expect(err).ToNot(HaveOccurred()) + + By("creating security group to be used by custom networking") + createSecurityGroupOutput, err := f.CloudServices.EC2(). + CreateSecurityGroup("custom-networking-test", "custom networking", f.Options.AWSVPCID) + Expect(err).ToNot(HaveOccurred()) + customNetworkingSGID = *createSecurityGroupOutput.GroupId + + By("authorizing egress and ingress on security group for single port") + f.CloudServices.EC2(). + AuthorizeSecurityGroupEgress(customNetworkingSGID, "TCP", customNetworkingSGOpenPort, customNetworkingSGOpenPort, "0.0.0.0/0") + f.CloudServices.EC2(). + AuthorizeSecurityGroupIngress(customNetworkingSGID, "TCP", customNetworkingSGOpenPort, customNetworkingSGOpenPort, "0.0.0.0/0") + + By("associating cidr range to the VPC") + association, err := f.CloudServices.EC2().AssociateVPCCIDRBlock(f.Options.AWSVPCID, cidrRange.String()) + Expect(err).ToNot(HaveOccurred()) + cidrBlockAssociationID = *association.CidrBlockAssociation.AssociationId + + for i, az := range clusterVPCConfig.AvailZones { + By(fmt.Sprintf("creating the subnet in %s", az)) + + subnetCidr, err := cidr.Subnet(cidrRange, 8, 5*i) + Expect(err).ToNot(HaveOccurred()) + + createSubnetOutput, err := f.CloudServices.EC2(). + CreateSubnet(subnetCidr.String(), f.Options.AWSVPCID, az) + Expect(err).ToNot(HaveOccurred()) + + subnetID := *createSubnetOutput.Subnet.SubnetId + + By("associating the route table with the newly created subnet") + err = f.CloudServices.EC2(). + AssociateRouteTableToSubnet(clusterVPCConfig.PublicRouteTableID, subnetID) + Expect(err).ToNot(HaveOccurred()) + + eniConfig, err := manifest.NewENIConfigBuilder(). + Name(az). + SubnetID(subnetID). + SecurityGroup([]string{customNetworkingSGID}). + Build() + Expect(err).ToNot(HaveOccurred()) + + By("creating the ENIConfig with az name") + err = f.K8sResourceManagers.CustomResourceManager().CreateResource(eniConfig) + Expect(err).ToNot(HaveOccurred()) + + // For deleting later + customNetworkingSubnetIDList = append(customNetworkingSubnetIDList, subnetID) + eniConfigList = append(eniConfigList, eniConfig) + } + + By("enabling custom networking on aws-node DaemonSet") + k8sUtils.AddEnvVarToDaemonSetAndWaitTillUpdated(f, "aws-node", + "kube-system", "aws-node", map[string]string{ + "AWS_VPC_K8S_CNI_CUSTOM_NETWORK_CFG": "true", + "ENI_CONFIG_LABEL_DEF": "failure-domain.beta.kubernetes.io/zone", + "WARM_ENI_TARGET": "0", + }) + + nodeGroupProperties = awsUtils.NodeGroupProperties{ + NgLabelKey: "node-type", + NgLabelVal: "custom-networking-node", + AsgSize: 2, + NodeGroupName: "custom-networking-node", + IsCustomNetworkingEnabled: true, + Subnet: clusterVPCConfig.PublicSubnetList, + InstanceType: "c5.xlarge", + KeyPairName: keyPairName, + } + + By("creating a new self managed node group") + err = awsUtils.CreateAndWaitTillSelfManagedNGReady(f, nodeGroupProperties) + Expect(err).ToNot(HaveOccurred()) +}) + +var _ = AfterSuite(func() { + By("deleting test namespace") + f.K8sResourceManagers.NamespaceManager(). + DeleteAndWaitTillNamespaceDeleted(utils.DefaultTestNamespace) + + By("waiting for some time to allow CNI to delete ENI for IP being cooled down") + time.Sleep(time.Second * 60) + + By("deleting the self managed node group") + err = awsUtils.DeleteAndWaitTillSelfManagedNGStackDeleted(f, nodeGroupProperties) + Expect(err).ToNot(HaveOccurred()) + + By("deleting the key pair") + f.CloudServices.EC2().DeleteKey(keyPairName) + + err = f.CloudServices.EC2().DeleteSecurityGroup(customNetworkingSGID) + Expect(err).ToNot(HaveOccurred()) + + for _, subnet := range customNetworkingSubnetIDList { + By(fmt.Sprintf("deleting the subnet %s", subnet)) + err = f.CloudServices.EC2().DeleteSubnet(subnet) + Expect(err).ToNot(HaveOccurred()) + } + + By("disassociating the CIDR range to the VPC") + err = f.CloudServices.EC2().DisAssociateVPCCIDRBlock(cidrBlockAssociationID) + Expect(err).ToNot(HaveOccurred()) + + By("disabling custom networking on aws-node DaemonSet") + k8sUtils.RemoveVarFromDaemonSetAndWaitTillUpdated(f, "aws-node", + "kube-system", "aws-node", map[string]struct{}{ + "AWS_VPC_K8S_CNI_CUSTOM_NETWORK_CFG": {}, + "ENI_CONFIG_LABEL_DEF": {}, + "WARM_ENI_TARGET": {}, + }) + + for _, eniConfig := range eniConfigList { + By("deleting ENIConfig") + err = f.K8sResourceManagers.CustomResourceManager().DeleteResource(eniConfig) + Expect(err).ToNot(HaveOccurred()) + } +}) diff --git a/test/e2e/custom-networking/custom_networking_test.go b/test/e2e/custom-networking/custom_networking_test.go new file mode 100644 index 0000000000..59cb16de45 --- /dev/null +++ b/test/e2e/custom-networking/custom_networking_test.go @@ -0,0 +1,130 @@ +// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"). You may +// not use this file except in compliance with the License. A copy of the +// License is located at +// +// http://aws.amazon.com/apache2.0/ +// +// or in the "license" file accompanying this file. This file is distributed +// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +// express or implied. See the License for the specific language governing +// permissions and limitations under the License. + +package custom_networking + +import ( + "fmt" + "net" + "strconv" + + "github.com/aws/amazon-vpc-cni-k8s/test/framework/resources/k8s/manifest" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + v1 "k8s.io/api/apps/v1" + coreV1 "k8s.io/api/core/v1" +) + +var _ = Describe("Custom Networking Test", func() { + + var ( + deployment *v1.Deployment + podList coreV1.PodList + podLabelKey string + podLabelVal string + port int + replicaCount int + shouldConnect bool + ) + + Context("when creating deployment targeted using ENIConfig", func() { + + BeforeEach(func() { + podLabelKey = "role" + podLabelVal = "custom-networking-test" + }) + + JustBeforeEach(func() { + container := manifest.NewNetCatAlpineContainer(). + Command([]string{"nc"}). + Args([]string{"-k", "-l", strconv.Itoa(port)}). + Build() + + deployment = manifest.NewBusyBoxDeploymentBuilder(). + Container(container). + Replicas(replicaCount). + NodeSelector(nodeGroupProperties.NgLabelKey, nodeGroupProperties.NgLabelVal). + PodLabel(podLabelKey, podLabelVal). + Build() + + deployment, err = f.K8sResourceManagers.DeploymentManager(). + CreateAndWaitTillDeploymentIsReady(deployment) + Expect(err).ToNot(HaveOccurred()) + + podList, err = f.K8sResourceManagers.PodManager(). + GetPodsWithLabelSelector(podLabelKey, podLabelVal) + Expect(err).ToNot(HaveOccurred()) + + // TODO: Parallelize the validation + for _, pod := range podList.Items { + By(fmt.Sprintf("verifying pod's IP %s address belong to the CIDR range %s", + pod.Status.PodIP, cidrRange.String())) + + ip := net.ParseIP(pod.Status.PodIP) + Expect(cidrRange.Contains(ip)).To(BeTrue()) + + testContainer := manifest.NewNetCatAlpineContainer(). + Command([]string{"nc"}). + Args([]string{"-v", "-w2", pod.Status.PodIP, strconv.Itoa(port)}). + Build() + + testJob := manifest.NewDefaultJobBuilder(). + Container(testContainer). + Name("test-pod"). + Parallelism(1). + Build() + + _, err := f.K8sResourceManagers.JobManager(). + CreateAndWaitTillJobCompleted(testJob) + if shouldConnect { + By("verifying connection to pod succeeds on port " + strconv.Itoa(port)) + Expect(err).ToNot(HaveOccurred()) + } else { + By("verifying connection to pod fails on port " + strconv.Itoa(port)) + Expect(err).To(HaveOccurred()) + } + + err = f.K8sResourceManagers.JobManager(). + DeleteAndWaitTillJobIsDeleted(testJob) + Expect(err).ToNot(HaveOccurred()) + } + }) + + JustAfterEach(func() { + err = f.K8sResourceManagers.DeploymentManager(). + DeleteAndWaitTillDeploymentIsDeleted(deployment) + Expect(err).ToNot(HaveOccurred()) + }) + + Context("when connecting to reachable port", func() { + BeforeEach(func() { + port = customNetworkingSGOpenPort + replicaCount = 30 + shouldConnect = true + }) + + It("should connect", func() {}) + }) + + Context("when connecting to unreachable port", func() { + BeforeEach(func() { + port = 8081 + replicaCount = 1 + shouldConnect = false + }) + + It("should fail to connect", func() {}) + }) + }) +}) diff --git a/test/framework/framework.go b/test/framework/framework.go index 34f1d08b19..24ec41e5ee 100644 --- a/test/framework/framework.go +++ b/test/framework/framework.go @@ -68,7 +68,8 @@ func New(options Options) *Framework { StatusClient: realClient, } - cloudConfig := aws.CloudConfig{Region: options.AWSRegion, VpcID: options.AWSVPCID} + cloudConfig := aws.CloudConfig{Region: options.AWSRegion, VpcID: options.AWSVPCID, + EKSEndpoint: options.EKSEndpoint} return &Framework{ Options: options, diff --git a/test/framework/options.go b/test/framework/options.go index 065c19f20d..5b04ef2088 100644 --- a/test/framework/options.go +++ b/test/framework/options.go @@ -33,6 +33,7 @@ type Options struct { AWSVPCID string NgNameLabelKey string NgNameLabelVal string + EKSEndpoint string } func (options *Options) BindFlags() { @@ -42,6 +43,7 @@ func (options *Options) BindFlags() { flag.StringVar(&options.AWSVPCID, "aws-vpc-id", "", `AWS VPC ID for the kubernetes cluster`) flag.StringVar(&options.NgNameLabelKey, "ng-name-label-key", "eks.amazonaws.com/nodegroup", "label key used to identify nodegroup name") flag.StringVar(&options.NgNameLabelVal, "ng-name-label-val", "", "label value with the nodegroup name") + flag.StringVar(&options.EKSEndpoint, "eks-endpoint", "", "optional eks api server endpoint") } func (options *Options) Validate() error { diff --git a/test/framework/resources/aws/cloud.go b/test/framework/resources/aws/cloud.go index 1c07e07300..9bbe5f5d71 100644 --- a/test/framework/resources/aws/cloud.go +++ b/test/framework/resources/aws/cloud.go @@ -21,19 +21,24 @@ import ( ) type CloudConfig struct { - VpcID string - Region string + VpcID string + Region string + EKSEndpoint string } type Cloud interface { + EKS() services.EKS EC2() services.EC2 AutoScaling() services.AutoScaling + CloudFormation() services.CloudFormation } type defaultCloud struct { - cfg CloudConfig - ec2 services.EC2 - autoScaling services.AutoScaling + cfg CloudConfig + ec2 services.EC2 + eks services.EKS + autoScaling services.AutoScaling + cloudFormation services.CloudFormation } func NewCloud(config CloudConfig) Cloud { @@ -41,9 +46,11 @@ func NewCloud(config CloudConfig) Cloud { Region: aws.String(config.Region)})) return &defaultCloud{ - cfg: config, - ec2: services.NewEC2(session), - autoScaling: services.NewAutoScaling(session), + cfg: config, + ec2: services.NewEC2(session), + eks: services.NewEKS(session, config.EKSEndpoint), + autoScaling: services.NewAutoScaling(session), + cloudFormation: services.NewCloudFormation(session), } } @@ -54,3 +61,11 @@ func (c *defaultCloud) EC2() services.EC2 { func (c *defaultCloud) AutoScaling() services.AutoScaling { return c.autoScaling } + +func (c *defaultCloud) CloudFormation() services.CloudFormation { + return c.cloudFormation +} + +func (c *defaultCloud) EKS() services.EKS { + return c.eks +} diff --git a/test/framework/resources/aws/services/cloudformation.go b/test/framework/resources/aws/services/cloudformation.go new file mode 100644 index 0000000000..45d4351c69 --- /dev/null +++ b/test/framework/resources/aws/services/cloudformation.go @@ -0,0 +1,102 @@ +// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"). You may +// not use this file except in compliance with the License. A copy of the +// License is located at +// +// http://aws.amazon.com/apache2.0/ +// +// or in the "license" file accompanying this file. This file is distributed +// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +// express or implied. See the License for the specific language governing +// permissions and limitations under the License. + +package services + +import ( + "context" + "fmt" + + "github.com/aws/amazon-vpc-cni-k8s/test/framework/utils" + "github.com/aws/aws-sdk-go/service/cloudformation/cloudformationiface" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/cloudformation" + "k8s.io/apimachinery/pkg/util/wait" +) + +type CloudFormation interface { + WaitTillStackCreated(stackName string, stackParams []*cloudformation.Parameter, templateBody string) (*cloudformation.DescribeStacksOutput, error) + WaitTillStackDeleted(stackName string) error +} + +type defaultCloudFormation struct { + cloudformationiface.CloudFormationAPI +} + +func NewCloudFormation(session *session.Session) CloudFormation { + return &defaultCloudFormation{ + CloudFormationAPI: cloudformation.New(session), + } +} + +func (d *defaultCloudFormation) WaitTillStackCreated(stackName string, stackParams []*cloudformation.Parameter, templateBody string) (*cloudformation.DescribeStacksOutput, error) { + createStackInput := &cloudformation.CreateStackInput{ + Parameters: stackParams, + StackName: aws.String(stackName), + TemplateBody: aws.String(templateBody), + Capabilities: aws.StringSlice([]string{cloudformation.CapabilityCapabilityIam}), + } + + _, err := d.CloudFormationAPI.CreateStack(createStackInput) + if err != nil { + return nil, err + } + + describeStackInput := &cloudformation.DescribeStacksInput{ + StackName: aws.String(stackName), + } + + var describeStackOutput *cloudformation.DescribeStacksOutput + err = wait.PollImmediateUntil(utils.PollIntervalLong, func() (done bool, err error) { + describeStackOutput, err = d.CloudFormationAPI.DescribeStacks(describeStackInput) + if err != nil { + return true, err + } + if *describeStackOutput.Stacks[0].StackStatus == "CREATE_COMPLETE" { + return true, nil + } + return false, nil + }, context.Background().Done()) + + return describeStackOutput, err +} + +func (d *defaultCloudFormation) WaitTillStackDeleted(stackName string) error { + deleteStackInput := &cloudformation.DeleteStackInput{ + StackName: aws.String(stackName), + } + _, err := d.CloudFormationAPI.DeleteStack(deleteStackInput) + if err != nil { + return fmt.Errorf("faield to delete stack %s: %v", stackName, err) + } + + describeStackInput := &cloudformation.DescribeStacksInput{ + StackName: aws.String(stackName), + } + + var describeStackOutput *cloudformation.DescribeStacksOutput + err = wait.PollImmediateUntil(utils.PollIntervalLong, func() (done bool, err error) { + describeStackOutput, err = d.CloudFormationAPI.DescribeStacks(describeStackInput) + if err != nil { + return true, err + } + if *describeStackOutput.Stacks[0].StackStatus == "DELETE_COMPLETE" { + return true, nil + } + return false, nil + }, context.Background().Done()) + + return nil +} diff --git a/test/framework/resources/aws/services/ec2.go b/test/framework/resources/aws/services/ec2.go index 2349375704..83de977ed2 100644 --- a/test/framework/resources/aws/services/ec2.go +++ b/test/framework/resources/aws/services/ec2.go @@ -30,6 +30,17 @@ type EC2 interface { RevokeSecurityGroupIngress(groupID string, protocol string, fromPort int, toPort int, cidrIP string) error AuthorizeSecurityGroupEgress(groupID string, protocol string, fromPort int, toPort int, cidrIP string) error RevokeSecurityGroupEgress(groupID string, protocol string, fromPort int, toPort int, cidrIP string) error + AssociateVPCCIDRBlock(vpcId string, cidrBlock string) (*ec2.AssociateVpcCidrBlockOutput, error) + DisAssociateVPCCIDRBlock(associationID string) error + DescribeSubnet(subnetID string) (*ec2.DescribeSubnetsOutput, error) + CreateSubnet(cidrBlock string, vpcID string, az string) (*ec2.CreateSubnetOutput, error) + DeleteSubnet(subnetID string) error + DescribeRouteTables(subnetID string) (*ec2.DescribeRouteTablesOutput, error) + CreateSecurityGroup(groupName string, description string, vpcID string) (*ec2.CreateSecurityGroupOutput, error) + DeleteSecurityGroup(groupID string) error + AssociateRouteTableToSubnet(routeTableId string, subnetID string) error + CreateKey(keyName string) (*ec2.CreateKeyPairOutput, error) + DeleteKey(keyName string) error } type defaultEC2 struct { @@ -150,6 +161,103 @@ func (d *defaultEC2) DescribeNetworkInterface(interfaceIDs []string) (*ec2.Descr return d.EC2API.DescribeNetworkInterfaces(describeNetworkInterfaceInput) } +func (d *defaultEC2) AssociateVPCCIDRBlock(vpcId string, cidrBlock string) (*ec2.AssociateVpcCidrBlockOutput, error) { + associateVPCCidrBlockInput := &ec2.AssociateVpcCidrBlockInput{ + CidrBlock: aws.String(cidrBlock), + VpcId: aws.String(vpcId), + } + + return d.EC2API.AssociateVpcCidrBlock(associateVPCCidrBlockInput) +} + +func (d *defaultEC2) DisAssociateVPCCIDRBlock(associationID string) error { + disassociateVPCCidrBlockInput := &ec2.DisassociateVpcCidrBlockInput{ + AssociationId: aws.String(associationID), + } + + _, err := d.EC2API.DisassociateVpcCidrBlock(disassociateVPCCidrBlockInput) + return err +} + +func (d *defaultEC2) CreateSubnet(cidrBlock string, vpcID string, az string) (*ec2.CreateSubnetOutput, error) { + createSubnetInput := &ec2.CreateSubnetInput{ + AvailabilityZone: aws.String(az), + CidrBlock: aws.String(cidrBlock), + VpcId: aws.String(vpcID), + } + return d.EC2API.CreateSubnet(createSubnetInput) +} + +func (d *defaultEC2) DescribeSubnet(subnetID string) (*ec2.DescribeSubnetsOutput, error) { + describeSubnetInput := &ec2.DescribeSubnetsInput{ + SubnetIds: aws.StringSlice([]string{subnetID}), + } + return d.EC2API.DescribeSubnets(describeSubnetInput) +} + +func (d *defaultEC2) DeleteSubnet(subnetID string) error { + deleteSubnetInput := &ec2.DeleteSubnetInput{ + SubnetId: aws.String(subnetID), + } + _, err := d.EC2API.DeleteSubnet(deleteSubnetInput) + return err +} + +func (d *defaultEC2) DescribeRouteTables(subnetID string) (*ec2.DescribeRouteTablesOutput, error) { + describeRouteTableInput := &ec2.DescribeRouteTablesInput{ + Filters: []*ec2.Filter{ + { + Name: aws.String("association.subnet-id"), + Values: aws.StringSlice([]string{subnetID}), + }, + }, + } + return d.EC2API.DescribeRouteTables(describeRouteTableInput) +} + +func (d *defaultEC2) AssociateRouteTableToSubnet(routeTableId string, subnetID string) error { + associateRouteTableInput := &ec2.AssociateRouteTableInput{ + RouteTableId: aws.String(routeTableId), + SubnetId: aws.String(subnetID), + } + _, err := d.EC2API.AssociateRouteTable(associateRouteTableInput) + return err +} + +func (d *defaultEC2) DeleteSecurityGroup(groupID string) error { + deleteSecurityGroupInput := &ec2.DeleteSecurityGroupInput{ + GroupId: aws.String(groupID), + } + + _, err := d.EC2API.DeleteSecurityGroup(deleteSecurityGroupInput) + return err +} + +func (d *defaultEC2) CreateSecurityGroup(groupName string, description string, vpcID string) (*ec2.CreateSecurityGroupOutput, error) { + createSecurityGroupInput := &ec2.CreateSecurityGroupInput{ + Description: aws.String(description), + GroupName: aws.String(groupName), + VpcId: aws.String(vpcID), + } + + return d.EC2API.CreateSecurityGroup(createSecurityGroupInput) +} + +func (d *defaultEC2) CreateKey(keyName string) (*ec2.CreateKeyPairOutput, error) { + createKeyInput := &ec2.CreateKeyPairInput{ + KeyName: aws.String(keyName), + } + return d.EC2API.CreateKeyPair(createKeyInput) +} + +func (d *defaultEC2) DeleteKey(keyName string) error { + deleteKeyPairInput := &ec2.DeleteKeyPairInput{ + KeyName: aws.String(keyName), + } + _, err := d.EC2API.DeleteKeyPair(deleteKeyPairInput) + return err +} + func NewEC2(session *session.Session) EC2 { return &defaultEC2{ EC2API: ec2.New(session), diff --git a/test/framework/resources/aws/services/eks.go b/test/framework/resources/aws/services/eks.go new file mode 100644 index 0000000000..7f9d79bbe6 --- /dev/null +++ b/test/framework/resources/aws/services/eks.go @@ -0,0 +1,46 @@ +// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"). You may +// not use this file except in compliance with the License. A copy of the +// License is located at +// +// http://aws.amazon.com/apache2.0/ +// +// or in the "license" file accompanying this file. This file is distributed +// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +// express or implied. See the License for the specific language governing +// permissions and limitations under the License. + +package services + +import ( + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/eks" + "github.com/aws/aws-sdk-go/service/eks/eksiface" +) + +type EKS interface { + DescribeCluster(clusterName string) (*eks.DescribeClusterOutput, error) +} + +type defaultEKS struct { + eksiface.EKSAPI +} + +func NewEKS(session *session.Session, endpoint string) EKS { + return &defaultEKS{ + EKSAPI: eks.New(session, &aws.Config{ + Endpoint: aws.String(endpoint), + Region: session.Config.Region, + }), + } +} + +func (d defaultEKS) DescribeCluster(clusterName string) (*eks.DescribeClusterOutput, error) { + describeClusterInput := &eks.DescribeClusterInput{ + Name: aws.String(clusterName), + } + + return d.EKSAPI.DescribeCluster(describeClusterInput) +} diff --git a/test/framework/resources/aws/utils/nodegroup.go b/test/framework/resources/aws/utils/nodegroup.go new file mode 100644 index 0000000000..8ad23b9765 --- /dev/null +++ b/test/framework/resources/aws/utils/nodegroup.go @@ -0,0 +1,246 @@ +// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"). You may +// not use this file except in compliance with the License. A copy of the +// License is located at +// +// http://aws.amazon.com/apache2.0/ +// +// or in the "license" file accompanying this file. This file is distributed +// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +// express or implied. See the License for the specific language governing +// permissions and limitations under the License. + +package utils + +import ( + "fmt" + "io/ioutil" + "net/http" + "strconv" + "strings" + + "gopkg.in/yaml.v2" + + "github.com/aws/amazon-vpc-cni-k8s/pkg/awsutils" + "github.com/aws/amazon-vpc-cni-k8s/test/framework" + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/cloudformation" +) + +const CreateNodeGroupCFNTemplateURL = "https://raw.githubusercontent.com/awslabs/amazon-eks-ami/master/amazon-eks-nodegroup.yaml" + +type NodeGroupProperties struct { + // Required to verify the node is up and ready + NgLabelKey string + NgLabelVal string + // ASG Size + AsgSize int + NodeGroupName string + // If custom networking is set then max pod + // will be set on Kubelet extra arguments + IsCustomNetworkingEnabled bool + // Subnet where the node group will be created + Subnet []string + InstanceType string + KeyPairName string +} + +type ClusterVPCConfig struct { + PublicSubnetList []string + AvailZones []string + PublicRouteTableID string +} + +type AWSAuthMapRole struct { + Groups []string `yaml:"groups"` + RoleArn string `yaml:"rolearn"` + UserName string `yaml:"username"` +} + +func CreateAndWaitTillSelfManagedNGReady(f *framework.Framework, properties NodeGroupProperties) error { + // Create self managed node group stack + resp, err := http.Get(CreateNodeGroupCFNTemplateURL) + if err != nil { + return fmt.Errorf("failed to load template from URL %s: %v", + CreateNodeGroupCFNTemplateURL, err) + } + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("non OK status code on getting node group URL %s: %d", + CreateNodeGroupCFNTemplateURL, resp.StatusCode) + } + defer resp.Body.Close() + + templateBytes, err := ioutil.ReadAll(resp.Body) + if err != nil { + return fmt.Errorf("failed to read response body: %v", err) + } + template := string(templateBytes) + + describeClusterOutput, err := f.CloudServices.EKS().DescribeCluster(f.Options.ClusterName) + if err != nil { + return fmt.Errorf("failed to describe cluster %s: %v", f.Options.ClusterName, err) + } + + var bootstrapArgs = fmt.Sprintf("--apiserver-endpoint %s --b64-cluster-ca %s", + *describeClusterOutput.Cluster.Endpoint, *describeClusterOutput.Cluster.CertificateAuthority.Data) + var kubeletExtraArgs = fmt.Sprintf("--node-labels=%s=%s", properties.NgLabelKey, properties.NgLabelVal) + + if properties.IsCustomNetworkingEnabled { + limit := awsutils.InstanceNetworkingLimits[properties.InstanceType] + maxPods := (limit.ENILimit-1)*(limit.IPv4Limit-1) + 2 + + bootstrapArgs += " --use-max-pods false" + kubeletExtraArgs += fmt.Sprintf(" --max-pods=%d", maxPods) + } + + asgSizeString := strconv.Itoa(properties.AsgSize) + + createNgStackParams := []*cloudformation.Parameter{ + { + ParameterKey: aws.String("ClusterName"), + ParameterValue: aws.String(f.Options.ClusterName), + }, + { + ParameterKey: aws.String("VpcId"), + ParameterValue: aws.String(f.Options.AWSVPCID), + }, + { + ParameterKey: aws.String("Subnets"), + ParameterValue: aws.String(strings.Join(properties.Subnet, ",")), + }, + { + ParameterKey: aws.String("ClusterControlPlaneSecurityGroup"), + ParameterValue: describeClusterOutput.Cluster.ResourcesVpcConfig.SecurityGroupIds[0], + }, + { + ParameterKey: aws.String("NodeGroupName"), + ParameterValue: aws.String(properties.NodeGroupName), + }, + { + ParameterKey: aws.String("NodeAutoScalingGroupMinSize"), + ParameterValue: aws.String(asgSizeString), + }, + { + ParameterKey: aws.String("NodeAutoScalingGroupDesiredCapacity"), + ParameterValue: aws.String(asgSizeString), + }, + { + ParameterKey: aws.String("NodeAutoScalingGroupMaxSize"), + ParameterValue: aws.String(asgSizeString), + }, + { + ParameterKey: aws.String("NodeInstanceType"), + ParameterValue: aws.String(properties.InstanceType), + }, + { + ParameterKey: aws.String("BootstrapArguments"), + ParameterValue: aws.String(fmt.Sprintf("%s --kubelet-extra-args '%s'", bootstrapArgs, kubeletExtraArgs)), + }, + { + ParameterKey: aws.String("KeyName"), + ParameterValue: aws.String(properties.KeyPairName), + }, + } + + describeStackOutput, err := f.CloudServices.CloudFormation(). + WaitTillStackCreated(properties.NodeGroupName, createNgStackParams, template) + if err != nil { + return fmt.Errorf("failed to create node group cfn stack: %v", err) + } + + var nodeInstanceRole string + for _, stackOutput := range describeStackOutput.Stacks[0].Outputs { + if *stackOutput.OutputKey == "NodeInstanceRole" { + nodeInstanceRole = *stackOutput.OutputValue + } + } + + if nodeInstanceRole == "" { + return fmt.Errorf("failed to find node instance role in stack %+v", describeStackOutput) + } + + // Update the AWS Auth Config with the Node Instance Role + awsAuth, err := f.K8sResourceManagers.ConfigMapManager(). + GetConfigMap("kube-system", "aws-auth") + if err != nil { + return fmt.Errorf("failed to find aws-auth configmap: %v", err) + } + + updatedAWSAuth := awsAuth.DeepCopy() + authMapRole := []AWSAuthMapRole{ + { + Groups: []string{"system:bootstrappers", "system:nodes"}, + RoleArn: nodeInstanceRole, + UserName: "system:node:{{EC2PrivateDNSName}}", + }, + } + yamlBytes, err := yaml.Marshal(authMapRole) + + updatedAWSAuth.Data["mapRoles"] = updatedAWSAuth.Data["mapRoles"] + string(yamlBytes) + + err = f.K8sResourceManagers.ConfigMapManager().UpdateConfigMap(awsAuth, updatedAWSAuth) + if err != nil { + return fmt.Errorf("failed to update the auth config with new node's instance role: %v", err) + } + + // Wait till the node group have joined the cluster and are ready + err = f.K8sResourceManagers.NodeManager(). + WaitTillNodesReady(properties.NgLabelKey, properties.NgLabelVal, properties.AsgSize) + if err != nil { + return fmt.Errorf("faield to list nodegroup with label key %s:%v: %v", + properties.NgLabelKey, properties.NgLabelVal, err) + } + + return nil +} + +func DeleteAndWaitTillSelfManagedNGStackDeleted(f *framework.Framework, properties NodeGroupProperties) error { + err := f.CloudServices.CloudFormation(). + WaitTillStackDeleted(properties.NodeGroupName) + if err != nil { + return fmt.Errorf("failed to delete node group cfn stack: %v", err) + } + + return nil +} + +func GetClusterVPCConfig(f *framework.Framework) (*ClusterVPCConfig, error) { + clusterConfig := &ClusterVPCConfig{ + PublicSubnetList: []string{}, + AvailZones: []string{}, + } + + describeClusterOutput, err := f.CloudServices.EKS().DescribeCluster(f.Options.ClusterName) + if err != nil { + return nil, fmt.Errorf("failed to describe cluster %s: %v", f.Options.ClusterName, err) + } + + for _, subnet := range describeClusterOutput.Cluster.ResourcesVpcConfig.SubnetIds { + describeRouteOutput, err := f.CloudServices.EC2().DescribeRouteTables(*subnet) + if err != nil { + return nil, fmt.Errorf("failed to describe subnet %s: %v", *subnet, err) + } + for _, route := range describeRouteOutput.RouteTables[0].Routes { + if route.GatewayId != nil && strings.Contains(*route.GatewayId, "igw-") { + clusterConfig.PublicSubnetList = append(clusterConfig.PublicSubnetList, *subnet) + clusterConfig.PublicRouteTableID = *describeRouteOutput.RouteTables[0].RouteTableId + } + } + } + + uniqueAZ := map[string]bool{} + for _, subnet := range clusterConfig.PublicSubnetList { + describeSubnet, err := f.CloudServices.EC2().DescribeSubnet(subnet) + if err != nil { + return nil, fmt.Errorf("failed to descrieb the subnet %s: %v", subnet, err) + } + if ok := uniqueAZ[*describeSubnet.Subnets[0].AvailabilityZone]; !ok { + uniqueAZ[*describeSubnet.Subnets[0].AvailabilityZone] = true + clusterConfig.AvailZones = + append(clusterConfig.AvailZones, *describeSubnet.Subnets[0].AvailabilityZone) + } + } + + return clusterConfig, nil +} diff --git a/test/framework/resources/k8s/manager.go b/test/framework/resources/k8s/manager.go index b6d749ee39..386d690ea3 100644 --- a/test/framework/resources/k8s/manager.go +++ b/test/framework/resources/k8s/manager.go @@ -30,6 +30,7 @@ type ResourceManagers interface { NodeManager() resources.NodeManager PodManager() resources.PodManager DaemonSetManager() resources.DaemonSetManager + ConfigMapManager() resources.ConfigMapManager } type defaultManager struct { @@ -41,6 +42,7 @@ type defaultManager struct { nodeManager resources.NodeManager podManager resources.PodManager daemonSetManager resources.DaemonSetManager + configMapManager resources.ConfigMapManager } func NewResourceManager(k8sClient client.DelegatingClient, @@ -54,6 +56,7 @@ func NewResourceManager(k8sClient client.DelegatingClient, nodeManager: resources.NewDefaultNodeManager(k8sClient), podManager: resources.NewDefaultPodManager(k8sClient, scheme, config), daemonSetManager: resources.NewDefaultDaemonSetManager(k8sClient), + configMapManager: resources.NewConfigMapManager(k8sClient), } } @@ -88,3 +91,7 @@ func (m *defaultManager) PodManager() resources.PodManager { func (m *defaultManager) DaemonSetManager() resources.DaemonSetManager { return m.daemonSetManager } + +func (m *defaultManager) ConfigMapManager() resources.ConfigMapManager { + return m.configMapManager +} diff --git a/test/framework/resources/k8s/manifest/deployment.go b/test/framework/resources/k8s/manifest/deployment.go index a835f5f0dd..6ef89ef8ea 100644 --- a/test/framework/resources/k8s/manifest/deployment.go +++ b/test/framework/resources/k8s/manifest/deployment.go @@ -28,6 +28,7 @@ type DeploymentBuilder struct { replicas int container corev1.Container labels map[string]string + nodeSelector map[string]string terminationGracePeriod int nodeName string } @@ -39,6 +40,7 @@ func NewBusyBoxDeploymentBuilder() *DeploymentBuilder { replicas: 10, container: NewBusyBoxContainerBuilder().Build(), labels: map[string]string{"role": "test"}, + nodeSelector: map[string]string{}, terminationGracePeriod: 0, } } @@ -51,6 +53,11 @@ func NewDefaultDeploymentBuilder() *DeploymentBuilder { } } +func (d *DeploymentBuilder) NodeSelector(labelKey string, labelVal string) *DeploymentBuilder { + d.nodeSelector[labelKey] = labelVal + return d +} + func (d *DeploymentBuilder) Namespace(namespace string) *DeploymentBuilder { d.namespace = namespace return d @@ -103,6 +110,7 @@ func (d *DeploymentBuilder) Build() *v1.Deployment { Labels: d.labels, }, Spec: corev1.PodSpec{ + NodeSelector: d.nodeSelector, Containers: []corev1.Container{d.container}, TerminationGracePeriodSeconds: aws.Int64(int64(d.terminationGracePeriod)), NodeName: d.nodeName, diff --git a/test/framework/resources/k8s/resources/configmap.go b/test/framework/resources/k8s/resources/configmap.go new file mode 100644 index 0000000000..f8095a5f9f --- /dev/null +++ b/test/framework/resources/k8s/resources/configmap.go @@ -0,0 +1,46 @@ +// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"). You may +// not use this file except in compliance with the License. A copy of the +// License is located at +// +// http://aws.amazon.com/apache2.0/ +// +// or in the "license" file accompanying this file. This file is distributed +// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +// express or implied. See the License for the specific language governing +// permissions and limitations under the License. + +package resources + +import ( + "context" + + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +type ConfigMapManager interface { + GetConfigMap(namespace string, name string) (*v1.ConfigMap, error) + UpdateConfigMap(oldConfigMap *v1.ConfigMap, newConfigMap *v1.ConfigMap) error +} + +type defaultConfigMapManager struct { + k8sClient client.DelegatingClient +} + +func (d defaultConfigMapManager) GetConfigMap(namespace string, name string) (*v1.ConfigMap, error) { + configMap := v1.ConfigMap{} + return &configMap, d.k8sClient.Get(context.Background(), types. + NamespacedName{Name: name, Namespace: namespace}, &configMap) +} + +func (d defaultConfigMapManager) UpdateConfigMap(oldConfigMap *v1.ConfigMap, newConfigMap *v1.ConfigMap) error { + ctx := context.Background() + return d.k8sClient.Patch(ctx, newConfigMap, client.MergeFrom(oldConfigMap)) +} + +func NewConfigMapManager(k8sClient client.DelegatingClient) ConfigMapManager { + return &defaultConfigMapManager{k8sClient: k8sClient} +} diff --git a/test/framework/resources/k8s/resources/eniconfig.go b/test/framework/resources/k8s/resources/eniconfig.go index e542fe4cc4..0460997c48 100644 --- a/test/framework/resources/k8s/resources/eniconfig.go +++ b/test/framework/resources/k8s/resources/eniconfig.go @@ -14,10 +14,15 @@ package resources import ( + "context" + + "k8s.io/apimachinery/pkg/runtime" "sigs.k8s.io/controller-runtime/pkg/client" ) type CustomResourceManager interface { + CreateResource(resource runtime.Object) error + DeleteResource(resource runtime.Object) error } type defaultCustomResourceManager struct { @@ -27,3 +32,13 @@ type defaultCustomResourceManager struct { func NewCustomResourceManager(k8sClient client.DelegatingClient) CustomResourceManager { return &defaultCustomResourceManager{k8sClient: k8sClient} } + +func (d *defaultCustomResourceManager) CreateResource(resource runtime.Object) error { + ctx := context.Background() + return d.k8sClient.Create(ctx, resource) +} + +func (d *defaultCustomResourceManager) DeleteResource(resource runtime.Object) error { + ctx := context.Background() + return d.k8sClient.Delete(ctx, resource) +} diff --git a/test/framework/resources/k8s/resources/node.go b/test/framework/resources/k8s/resources/node.go index d72eeac18b..178ab1ce7a 100644 --- a/test/framework/resources/k8s/resources/node.go +++ b/test/framework/resources/k8s/resources/node.go @@ -16,12 +16,16 @@ package resources import ( "context" + "github.com/aws/amazon-vpc-cni-k8s/test/framework/utils" + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/util/wait" "sigs.k8s.io/controller-runtime/pkg/client" ) type NodeManager interface { GetNodes(nodeLabelKey string, nodeLabelVal string) (v1.NodeList, error) + WaitTillNodesReady(nodeLabelKey string, nodeLabelVal string, asgSize int) error } type defaultNodeManager struct { @@ -40,3 +44,25 @@ func (d *defaultNodeManager) GetNodes(nodeLabelKey string, nodeLabelVal string) }) return nodeList, err } + +func (d *defaultNodeManager) WaitTillNodesReady(nodeLabelKey string, nodeLabelVal string, asgSize int) error { + return wait.PollImmediateUntil(utils.PollIntervalLong, func() (done bool, err error) { + nodeList, err := d.GetNodes(nodeLabelKey, nodeLabelVal) + if err != nil { + return false, err + } + if len(nodeList.Items) != asgSize { + return false, nil + } + for _, node := range nodeList.Items { + for _, condition := range node.Status.Conditions { + if condition.Type == v1.NodeReady && condition.Status != v1.ConditionTrue { + return false, nil + } + } + } + + return true, nil + + }, context.Background().Done()) +} diff --git a/test/go.mod b/test/go.mod index 555e3cc00d..fcd7f385dd 100644 --- a/test/go.mod +++ b/test/go.mod @@ -4,12 +4,14 @@ go 1.14 require ( github.com/aws/amazon-vpc-cni-k8s v1.7.10 + github.com/apparentlymart/go-cidr v1.0.1 github.com/aws/aws-sdk-go v1.37.23 github.com/google/gopacket v1.1.19 // indirect github.com/gophercloud/gophercloud v0.1.0 // indirect github.com/onsi/ginkgo v1.12.1 github.com/onsi/gomega v1.11.0 github.com/pkg/errors v0.9.1 + gopkg.in/yaml.v2 v2.4.0 k8s.io/api v0.18.6 k8s.io/apimachinery v0.18.6 k8s.io/client-go v0.18.6 diff --git a/test/go.sum b/test/go.sum index ee29f01cc4..bd5862ee57 100644 --- a/test/go.sum +++ b/test/go.sum @@ -58,6 +58,10 @@ github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRF github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alexflint/go-filemutex v0.0.0-20171022225611-72bdc8eae2ae/go.mod h1:CgnQgUtFrFz9mxFNtED3jI5tLDjKlOM+oUF/sTk6ps0= github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= +github.com/apparentlymart/go-cidr v1.0.0/go.mod h1:EBcsNrHc3zQeuaeCeCtQruQm+n9/YjEn/vI25Lg7Gwc= +github.com/apparentlymart/go-cidr v1.0.1 h1:NmIwLZ/KdsjIUlhf+/Np40atNXm/+lZ5txfTJ/SpF+U= +github.com/apparentlymart/go-cidr v1.0.1/go.mod h1:EBcsNrHc3zQeuaeCeCtQruQm+n9/YjEn/vI25Lg7Gwc= +github.com/apparentlymart/go-cidr v1.1.0/go.mod h1:EBcsNrHc3zQeuaeCeCtQruQm+n9/YjEn/vI25Lg7Gwc= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= @@ -146,6 +150,7 @@ github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMo github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/globalsign/mgo v0.0.0-20180905125535-1ca0a4f7cbcb/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q= github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q= @@ -857,6 +862,7 @@ gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKW gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8= gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/square/go-jose.v2 v2.2.2/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= From 8d733738abe014f4aee49d450a22b9764476cdc9 Mon Sep 17 00:00:00 2001 From: Abhinav Pathak Date: Tue, 4 May 2021 01:13:05 +0530 Subject: [PATCH 2/4] fix formatting --- .../custom-networking/custom_networking_test.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/test/e2e/custom-networking/custom_networking_test.go b/test/e2e/custom-networking/custom_networking_test.go index 59cb16de45..33ece38d16 100644 --- a/test/e2e/custom-networking/custom_networking_test.go +++ b/test/e2e/custom-networking/custom_networking_test.go @@ -29,12 +29,12 @@ import ( var _ = Describe("Custom Networking Test", func() { var ( - deployment *v1.Deployment - podList coreV1.PodList - podLabelKey string - podLabelVal string - port int - replicaCount int + deployment *v1.Deployment + podList coreV1.PodList + podLabelKey string + podLabelVal string + port int + replicaCount int shouldConnect bool ) @@ -87,7 +87,7 @@ var _ = Describe("Custom Networking Test", func() { _, err := f.K8sResourceManagers.JobManager(). CreateAndWaitTillJobCompleted(testJob) - if shouldConnect { + if shouldConnect { By("verifying connection to pod succeeds on port " + strconv.Itoa(port)) Expect(err).ToNot(HaveOccurred()) } else { From 1e5dfd758e1591ac2b500ccfe3cef2acd04c1b73 Mon Sep 17 00:00:00 2001 From: Abhinav Pathak Date: Tue, 4 May 2021 17:32:14 +0530 Subject: [PATCH 3/4] add negative test case --- .../custom_networking_suite_test.go | 16 ++--- .../custom_networking_test.go | 66 ++++++++++++++++++- test/framework/resources/aws/services/ec2.go | 10 +++ .../resources/k8s/resources/deployment.go | 11 ++-- .../framework/resources/k8s/resources/node.go | 5 ++ test/framework/utils/const.go | 2 + .../cni/pod_networking_test.go | 6 +- .../cni/service_connectivity_test.go | 2 +- 8 files changed, 101 insertions(+), 17 deletions(-) diff --git a/test/e2e/custom-networking/custom_networking_suite_test.go b/test/e2e/custom-networking/custom_networking_suite_test.go index a5f93850c6..e53fd26521 100644 --- a/test/e2e/custom-networking/custom_networking_suite_test.go +++ b/test/e2e/custom-networking/custom_networking_suite_test.go @@ -124,18 +124,18 @@ var _ = BeforeSuite(func() { Build() Expect(err).ToNot(HaveOccurred()) + // For deleting later + customNetworkingSubnetIDList = append(customNetworkingSubnetIDList, subnetID) + eniConfigList = append(eniConfigList, eniConfig.DeepCopy()) + By("creating the ENIConfig with az name") err = f.K8sResourceManagers.CustomResourceManager().CreateResource(eniConfig) Expect(err).ToNot(HaveOccurred()) - - // For deleting later - customNetworkingSubnetIDList = append(customNetworkingSubnetIDList, subnetID) - eniConfigList = append(eniConfigList, eniConfig) } By("enabling custom networking on aws-node DaemonSet") - k8sUtils.AddEnvVarToDaemonSetAndWaitTillUpdated(f, "aws-node", - "kube-system", "aws-node", map[string]string{ + k8sUtils.AddEnvVarToDaemonSetAndWaitTillUpdated(f, utils.AwsNodeName, + utils.AwsNodeNamespace, utils.AwsNodeName, map[string]string{ "AWS_VPC_K8S_CNI_CUSTOM_NETWORK_CFG": "true", "ENI_CONFIG_LABEL_DEF": "failure-domain.beta.kubernetes.io/zone", "WARM_ENI_TARGET": "0", @@ -186,8 +186,8 @@ var _ = AfterSuite(func() { Expect(err).ToNot(HaveOccurred()) By("disabling custom networking on aws-node DaemonSet") - k8sUtils.RemoveVarFromDaemonSetAndWaitTillUpdated(f, "aws-node", - "kube-system", "aws-node", map[string]struct{}{ + k8sUtils.RemoveVarFromDaemonSetAndWaitTillUpdated(f, utils.AwsNodeName, + utils.AwsNodeNamespace, utils.AwsNodeName, map[string]struct{}{ "AWS_VPC_K8S_CNI_CUSTOM_NETWORK_CFG": {}, "ENI_CONFIG_LABEL_DEF": {}, "WARM_ENI_TARGET": {}, diff --git a/test/e2e/custom-networking/custom_networking_test.go b/test/e2e/custom-networking/custom_networking_test.go index 33ece38d16..89731b547e 100644 --- a/test/e2e/custom-networking/custom_networking_test.go +++ b/test/e2e/custom-networking/custom_networking_test.go @@ -17,8 +17,11 @@ import ( "fmt" "net" "strconv" + "time" "github.com/aws/amazon-vpc-cni-k8s/test/framework/resources/k8s/manifest" + k8sUtils "github.com/aws/amazon-vpc-cni-k8s/test/framework/resources/k8s/utils" + "github.com/aws/amazon-vpc-cni-k8s/test/framework/utils" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" @@ -39,7 +42,6 @@ var _ = Describe("Custom Networking Test", func() { ) Context("when creating deployment targeted using ENIConfig", func() { - BeforeEach(func() { podLabelKey = "role" podLabelVal = "custom-networking-test" @@ -59,7 +61,7 @@ var _ = Describe("Custom Networking Test", func() { Build() deployment, err = f.K8sResourceManagers.DeploymentManager(). - CreateAndWaitTillDeploymentIsReady(deployment) + CreateAndWaitTillDeploymentIsReady(deployment, utils.DefaultDeploymentReadyTimeout) Expect(err).ToNot(HaveOccurred()) podList, err = f.K8sResourceManagers.PodManager(). @@ -127,4 +129,64 @@ var _ = Describe("Custom Networking Test", func() { It("should fail to connect", func() {}) }) }) + + Context("when creating deployment on nodes that don't have ENIConfig", func() { + JustBeforeEach(func() { + By("deleting all existing ENIConfigs") + for _, eniConfig := range eniConfigList { + err = f.K8sResourceManagers.CustomResourceManager(). + DeleteResource(eniConfig) + Expect(err).ToNot(HaveOccurred()) + } + }) + + JustAfterEach(func() { + By("creating the deleted ENIConfigs") + for _, eniConfig := range eniConfigList { + err = f.K8sResourceManagers.CustomResourceManager(). + CreateResource(eniConfig) + Expect(err).ToNot(HaveOccurred()) + } + }) + + It("deployment should not become ready", func() { + By("getting the list of nodes created") + nodeList, err := f.K8sResourceManagers.NodeManager(). + GetNodes(nodeGroupProperties.NgLabelKey, nodeGroupProperties.NgLabelVal) + Expect(err).ToNot(HaveOccurred()) + + var instanceIDs []string + for _, node := range nodeList.Items { + instanceIDs = append(instanceIDs, k8sUtils.GetInstanceIDFromNode(node)) + } + + By("terminating all the nodes") + err = f.CloudServices.EC2().TerminateInstance(instanceIDs) + Expect(err).ToNot(HaveOccurred()) + + By("waiting for the node to be removed") + time.Sleep(time.Second * 120) + + By("waiting for all nodes to become ready") + err = f.K8sResourceManagers.NodeManager(). + WaitTillNodesReady(nodeGroupProperties.NgLabelKey, nodeGroupProperties.NgLabelVal, + nodeGroupProperties.AsgSize) + Expect(err).ToNot(HaveOccurred()) + + deployment := manifest.NewBusyBoxDeploymentBuilder(). + Replicas(2). + NodeSelector(nodeGroupProperties.NgLabelKey, nodeGroupProperties.NgLabelVal). + Build() + + By("verifying deployment should not succeed") + deployment, err = f.K8sResourceManagers.DeploymentManager(). + CreateAndWaitTillDeploymentIsReady(deployment, utils.DefaultDeploymentReadyTimeout) + Expect(err).To(HaveOccurred()) + + By("deleting the failed deployment") + err = f.K8sResourceManagers.DeploymentManager(). + DeleteAndWaitTillDeploymentIsDeleted(deployment) + Expect(err).ToNot(HaveOccurred()) + }) + }) }) diff --git a/test/framework/resources/aws/services/ec2.go b/test/framework/resources/aws/services/ec2.go index 83de977ed2..2085ace57b 100644 --- a/test/framework/resources/aws/services/ec2.go +++ b/test/framework/resources/aws/services/ec2.go @@ -31,6 +31,7 @@ type EC2 interface { AuthorizeSecurityGroupEgress(groupID string, protocol string, fromPort int, toPort int, cidrIP string) error RevokeSecurityGroupEgress(groupID string, protocol string, fromPort int, toPort int, cidrIP string) error AssociateVPCCIDRBlock(vpcId string, cidrBlock string) (*ec2.AssociateVpcCidrBlockOutput, error) + TerminateInstance(instanceIDs []string) error DisAssociateVPCCIDRBlock(associationID string) error DescribeSubnet(subnetID string) (*ec2.DescribeSubnetsOutput, error) CreateSubnet(cidrBlock string, vpcID string, az string) (*ec2.CreateSubnetOutput, error) @@ -258,6 +259,15 @@ func (d *defaultEC2) DeleteKey(keyName string) error { return err } +func (d *defaultEC2) TerminateInstance(instanceIDs []string) error { + terminateInstanceInput := &ec2.TerminateInstancesInput{ + DryRun: nil, + InstanceIds: aws.StringSlice(instanceIDs), + } + _, err := d.EC2API.TerminateInstances(terminateInstanceInput) + return err +} + func NewEC2(session *session.Session) EC2 { return &defaultEC2{ EC2API: ec2.New(session), diff --git a/test/framework/resources/k8s/resources/deployment.go b/test/framework/resources/k8s/resources/deployment.go index 2f8d33867c..bab77b13b4 100644 --- a/test/framework/resources/k8s/resources/deployment.go +++ b/test/framework/resources/k8s/resources/deployment.go @@ -26,7 +26,7 @@ import ( ) type DeploymentManager interface { - CreateAndWaitTillDeploymentIsReady(deployment *v1.Deployment) (*v1.Deployment, error) + CreateAndWaitTillDeploymentIsReady(deployment *v1.Deployment, timeout time.Duration) (*v1.Deployment, error) DeleteAndWaitTillDeploymentIsDeleted(deployment *v1.Deployment) error } @@ -34,7 +34,9 @@ type defaultDeploymentManager struct { k8sClient client.DelegatingClient } -func (d defaultDeploymentManager) CreateAndWaitTillDeploymentIsReady(deployment *v1.Deployment) (*v1.Deployment, error) { +// CreateAndWaitTillDeploymentIsReady creates and waits for deployment to become ready or timeout +// with error if deployment doesn't become ready. +func (d defaultDeploymentManager) CreateAndWaitTillDeploymentIsReady(deployment *v1.Deployment, timeout time.Duration) (*v1.Deployment, error) { ctx := context.Background() err := d.k8sClient.Create(ctx, deployment) if err != nil { @@ -45,7 +47,7 @@ func (d defaultDeploymentManager) CreateAndWaitTillDeploymentIsReady(deployment time.Sleep(utils.PollIntervalShort) observed := &v1.Deployment{} - return observed, wait.PollImmediateUntil(utils.PollIntervalShort, func() (bool, error) { + return observed, wait.PollImmediate(utils.PollIntervalShort, timeout, func() (bool, error) { if err := d.k8sClient.Get(ctx, utils.NamespacedName(deployment), observed); err != nil { return false, err } @@ -56,9 +58,10 @@ func (d defaultDeploymentManager) CreateAndWaitTillDeploymentIsReady(deployment return true, nil } return false, nil - }, ctx.Done()) + }) } +// func (d defaultDeploymentManager) DeleteAndWaitTillDeploymentIsDeleted(deployment *v1.Deployment) error { ctx := context.Background() err := d.k8sClient.Delete(ctx, deployment) diff --git a/test/framework/resources/k8s/resources/node.go b/test/framework/resources/k8s/resources/node.go index 178ab1ce7a..bfd901ffac 100644 --- a/test/framework/resources/k8s/resources/node.go +++ b/test/framework/resources/k8s/resources/node.go @@ -25,6 +25,7 @@ import ( type NodeManager interface { GetNodes(nodeLabelKey string, nodeLabelVal string) (v1.NodeList, error) + UpdateNode(oldNode *v1.Node, newNode *v1.Node) error WaitTillNodesReady(nodeLabelKey string, nodeLabelVal string, asgSize int) error } @@ -45,6 +46,10 @@ func (d *defaultNodeManager) GetNodes(nodeLabelKey string, nodeLabelVal string) return nodeList, err } +func (d *defaultNodeManager) UpdateNode(oldNode *v1.Node, newNode *v1.Node) error { + return d.k8sClient.Patch(context.Background(), newNode, client.MergeFrom(oldNode)) +} + func (d *defaultNodeManager) WaitTillNodesReady(nodeLabelKey string, nodeLabelVal string, asgSize int) error { return wait.PollImmediateUntil(utils.PollIntervalLong, func() (done bool, err error) { nodeList, err := d.GetNodes(nodeLabelKey, nodeLabelVal) diff --git a/test/framework/utils/const.go b/test/framework/utils/const.go index f829759659..5ca5b21bb3 100644 --- a/test/framework/utils/const.go +++ b/test/framework/utils/const.go @@ -25,4 +25,6 @@ const ( PollIntervalShort = time.Second * 2 PollIntervalMedium = time.Second * 5 PollIntervalLong = time.Second * 20 + + DefaultDeploymentReadyTimeout = time.Second * 120 ) diff --git a/test/integration-new/cni/pod_networking_test.go b/test/integration-new/cni/pod_networking_test.go index 0377b55d90..1d27c6689e 100644 --- a/test/integration-new/cni/pod_networking_test.go +++ b/test/integration-new/cni/pod_networking_test.go @@ -17,6 +17,8 @@ import ( "fmt" "strconv" + "github.com/aws/amazon-vpc-cni-k8s/test/framework/utils" + "github.com/aws/amazon-vpc-cni-k8s/test/framework/resources/k8s/manifest" k8sUtils "github.com/aws/amazon-vpc-cni-k8s/test/framework/resources/k8s/utils" @@ -96,7 +98,7 @@ var _ = Describe("test pod networking", func() { primaryNodeDeployment, err = f.K8sResourceManagers. DeploymentManager(). - CreateAndWaitTillDeploymentIsReady(primaryNodeDeployment) + CreateAndWaitTillDeploymentIsReady(primaryNodeDeployment, utils.DefaultDeploymentReadyTimeout) Expect(err).ToNot(HaveOccurred()) interfaceToPodListOnPrimaryNode = @@ -121,7 +123,7 @@ var _ = Describe("test pod networking", func() { secondaryNodeDeployment, err = f.K8sResourceManagers. DeploymentManager(). - CreateAndWaitTillDeploymentIsReady(secondaryNodeDeployment) + CreateAndWaitTillDeploymentIsReady(secondaryNodeDeployment, utils.DefaultDeploymentReadyTimeout) Expect(err).ToNot(HaveOccurred()) interfaceToPodListOnSecondaryNode = diff --git a/test/integration-new/cni/service_connectivity_test.go b/test/integration-new/cni/service_connectivity_test.go index b78b55ffce..610b6fe20c 100644 --- a/test/integration-new/cni/service_connectivity_test.go +++ b/test/integration-new/cni/service_connectivity_test.go @@ -71,7 +71,7 @@ var _ = Describe("test service connectivity", func() { By("creating and waiting for deployment to be ready") deployment, err = f.K8sResourceManagers.DeploymentManager(). - CreateAndWaitTillDeploymentIsReady(deployment) + CreateAndWaitTillDeploymentIsReady(deployment, utils.DefaultDeploymentReadyTimeout) Expect(err).ToNot(HaveOccurred()) service = manifest.NewHTTPService(). From aa231327a0b71092497bafd59af02143c781288d Mon Sep 17 00:00:00 2001 From: Abhinav Pathak Date: Tue, 4 May 2021 21:40:34 +0530 Subject: [PATCH 4/4] re-word By statement --- test/e2e/custom-networking/custom_networking_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/e2e/custom-networking/custom_networking_test.go b/test/e2e/custom-networking/custom_networking_test.go index 89731b547e..80acf4a535 100644 --- a/test/e2e/custom-networking/custom_networking_test.go +++ b/test/e2e/custom-networking/custom_networking_test.go @@ -132,7 +132,7 @@ var _ = Describe("Custom Networking Test", func() { Context("when creating deployment on nodes that don't have ENIConfig", func() { JustBeforeEach(func() { - By("deleting all existing ENIConfigs") + By("deleting ENIConfig for all availability zones") for _, eniConfig := range eniConfigList { err = f.K8sResourceManagers.CustomResourceManager(). DeleteResource(eniConfig) @@ -141,7 +141,7 @@ var _ = Describe("Custom Networking Test", func() { }) JustAfterEach(func() { - By("creating the deleted ENIConfigs") + By("re-creating ENIConfig for all availability zones") for _, eniConfig := range eniConfigList { err = f.K8sResourceManagers.CustomResourceManager(). CreateResource(eniConfig)