diff --git a/CHANGES.rst b/CHANGES.rst index eb7bd49d..126f5ace 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -6,7 +6,16 @@ Changelog 10.0.0 (2020-12-07) ------------------- -**Important:** This release makes significant changes to how Trusted Advisor is used. +IMPORTANT - Breaking Changes +++++++++++++++++++++++++++++ + +* This release makes significant changes to how Trusted Advisor is used; see below. +* This release requires the following new IAM permissions: ``eks:ListClusters``, ``eks:DescribeCluster``, ``eks:ListNodegroups``, ``eks:ListFargateProfiles``, ``eks:DescribeFargateProfile``, ``kinesis:DescribeLimits``. +* This release introduces a number of new limits, as well as new services. Please see below for details. +* This release **removes** the ``EC2/Security groups per VPC`` limit, which no longer exists, and adds the new ``EC2/VPC security groups per Region`` limit. + +All Changes ++++++++++++ * `Issue #466 `__ - **Significant** changes to Trusted Advisor support. @@ -18,7 +27,17 @@ Changelog * My current intent is to leave Trusted Advisor support in this state until Service Quotas is available in China and GovCloud, at which point I plan on completely removing all Trusted Advisor support. * Migrate CI builds from travis-ci.org to travis-ci.com. * `Issue #503 `__ - Fix ``Units set to "None"`` error when retrieving load balancer data from Service Quotas. We now allow the (A|E)LB per Region quota with a unit of either "Count" (prior to November 2020) or "None" (November 2020 on). -* `PR #490 `__ - Add missing RDS limits: ``Manual Cluster Snapshots``, ``Custom Endpoints Per DB Cluster``, ``DB Instance Roles``, and ``DB Cluster Roles``. Thanks to `sebasrp `__ for this contribution! +* `Issue #489 `__ / `PR #490 `__ - Add missing RDS limits: ``Manual Cluster Snapshots``, ``Custom Endpoints Per DB Cluster``, ``DB Instance Roles``, and ``DB Cluster Roles``. Thanks to `sebasrp `__ for this contribution! +* `Issue #472 `__ / `PR #494 `__ - Add support for the ``EKS`` service, and 8 new limits for it. Thanks to `sebasrp `__ for this contribution! +* `Issue #495 `__ / `PR #496 `__ - Add support for the ``Kinesis`` service, and one new limit for it. Thanks to `sebasrp `__ for this contribution! +* `PR #499 `__ - Set quota_name for VPC "Entries per route table" limit, so that the current limit will be automatically retrieved from Service Quotas. Thanks to `patuck `__ for this contribution! +* `Issue #498 `__ - Fix multiple issues relating to VPC limits: + + * Update the EC2 / ``Rules per VPC security group`` limit to support retrieving the current limit value from Service Quotas. + * Remove the ``EC2/Security groups per VPC`` limit, which no longer exists. + * Add the new ``EC2/VPC security groups per Region`` limit. + +* `Issue #501 `__ - Update ``VPC/Network interfaces per Region`` limit for new calculation method. .. _changelog.9_0_0: diff --git a/awslimitchecker/services/__init__.py b/awslimitchecker/services/__init__.py index 4b6c5d4b..d5031892 100644 --- a/awslimitchecker/services/__init__.py +++ b/awslimitchecker/services/__init__.py @@ -51,8 +51,10 @@ from awslimitchecker.services.elasticache import _ElastiCacheService from awslimitchecker.services.elasticbeanstalk import _ElasticBeanstalkService from awslimitchecker.services.elb import _ElbService +from awslimitchecker.services.eks import _EksService from awslimitchecker.services.firehose import _FirehoseService from awslimitchecker.services.iam import _IamService +from awslimitchecker.services.kinesis import _KinesisService from awslimitchecker.services.lambdafunc import _LambdaService from awslimitchecker.services.rds import _RDSService from awslimitchecker.services.redshift import _RedshiftService diff --git a/awslimitchecker/services/ec2.py b/awslimitchecker/services/ec2.py index 6d68a746..fbf2216b 100644 --- a/awslimitchecker/services/ec2.py +++ b/awslimitchecker/services/ec2.py @@ -640,12 +640,12 @@ def _get_limits_spot(self): def _find_usage_networking_sgs(self): """calculate usage for VPC-related things""" logger.debug("Getting usage for EC2 VPC resources") - sgs_per_vpc = defaultdict(int) + sg_count = 0 rules_per_sg = defaultdict(int) for sg in self.resource_conn.security_groups.all(): if sg.vpc_id is None: continue - sgs_per_vpc[sg.vpc_id] += 1 + sg_count += 1 """ see: https://github.com/jantman/awslimitchecker/issues/431 @@ -676,12 +676,10 @@ def _find_usage_networking_sgs(self): ) rules_per_sg[sg.id] = max(counts) # set usage - for vpc_id, count in sgs_per_vpc.items(): - self.limits['Security groups per VPC']._add_current_usage( - count, - aws_type='AWS::EC2::VPC', - resource_id=vpc_id, - ) + self.limits['VPC security groups per Region']._add_current_usage( + sg_count, + aws_type='AWS::EC2::SecurityGroup', + ) for sg_id, count in rules_per_sg.items(): self.limits['Rules per VPC security group']._add_current_usage( count, @@ -727,14 +725,16 @@ def _get_limits_networking(self): :rtype: dict """ limits = {} - limits['Security groups per VPC'] = AwsLimit( - 'Security groups per VPC', + limits['VPC security groups per Region'] = AwsLimit( + 'VPC security groups per Region', self, - 500, + 2500, self.warning_threshold, self.critical_threshold, limit_type='AWS::EC2::SecurityGroup', limit_subtype='AWS::EC2::VPC', + quotas_name='VPC security groups per Region', + quotas_service_code='vpc' ) limits['Rules per VPC security group'] = AwsLimit( 'Rules per VPC security group', @@ -744,6 +744,8 @@ def _get_limits_networking(self): self.critical_threshold, limit_type='AWS::EC2::SecurityGroup', limit_subtype='AWS::EC2::VPC', + quotas_name='Inbound or outbound rules per security group', + quotas_service_code='vpc' ) limits['VPC Elastic IP addresses (EIPs)'] = AwsLimit( 'VPC Elastic IP addresses (EIPs)', diff --git a/awslimitchecker/services/eks.py b/awslimitchecker/services/eks.py new file mode 100644 index 00000000..5e28c267 --- /dev/null +++ b/awslimitchecker/services/eks.py @@ -0,0 +1,254 @@ +""" +awslimitchecker/services/eks.py + +The latest version of this package is available at: + + +################################################################################ +Copyright 2015-2018 Jason Antman + + This file is part of awslimitchecker, also known as awslimitchecker. + + awslimitchecker is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + awslimitchecker is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with awslimitchecker. If not, see . + +The Copyright and Authors attributions contained herein may not be removed or +otherwise altered, except to add the Author attribution of a contributor to +this work. (Additional Terms pursuant to Section 7b of the AGPL v3) +################################################################################ +While not legally required, I sincerely request that anyone who finds +bugs please submit them at or +to me via email, and that you send any contributions or improvements +either as a pull request on GitHub, or to me via email. +################################################################################ + +AUTHORS: +Jason Antman +################################################################################ +""" + +import abc # noqa +import logging + +from .base import _AwsService +from ..limit import AwsLimit +from ..utils import paginate_dict + +logger = logging.getLogger(__name__) + + +class _EksService(_AwsService): + + service_name = 'EKS' + api_name = 'eks' # AWS API name to connect to (boto3.client) + quotas_service_code = 'eks' + + def find_usage(self): + """ + Determine the current usage for each limit of this service, + and update corresponding Limit via + :py:meth:`~.AwsLimit._add_current_usage`. + """ + logger.debug("Checking usage for service %s", self.service_name) + self.connect() + for lim in self.limits.values(): + lim._reset_usage() + self._find_clusters_usage() + self._have_usage = True + logger.debug("Done checking usage.") + + def _find_clusters_usage(self): + clusters_info = paginate_dict( + self.conn.list_clusters, + alc_marker_path=['nextToken'], + alc_data_path=['clusters'], + alc_marker_param='nextToken' + ) + + cluster_list = clusters_info['clusters'] + + for cluster in cluster_list: + describe_cluster_response = self.conn.describe_cluster( + name=cluster + ) + security_group_id_list = describe_cluster_response['cluster'][ + 'resourcesVpcConfig']['securityGroupIds'] + self.limits[ + 'Control plane security groups per cluster']._add_current_usage( + len(security_group_id_list), + resource_id=cluster, + aws_type='AWS::EKS::Cluster' + ) + public_access_cidrs_list = describe_cluster_response['cluster'][ + 'resourcesVpcConfig']['publicAccessCidrs'] + self.limits[ + 'Public endpoint access CIDR ranges per cluster' + ]._add_current_usage( + len(public_access_cidrs_list), + resource_id=cluster, + aws_type='AWS::EKS::Cluster' + ) + + list_nodegroup_response = paginate_dict( + self.conn.list_nodegroups, + clusterName=cluster, + alc_marker_path=['nextToken'], + alc_data_path=['nodegroups'], + alc_marker_param='nextToken' + ) + nodegroup_list = list_nodegroup_response['nodegroups'] + self.limits['Managed node groups per cluster']._add_current_usage( + len(nodegroup_list), + resource_id=cluster, + aws_type='AWS::EKS::Cluster') + + list_fargate_profiles_response = paginate_dict( + self.conn.list_fargate_profiles, + clusterName=cluster, + alc_marker_path=['nextToken'], + alc_data_path=['fargateProfileNames'], + alc_marker_param='nextToken' + ) + fargate_profiles_list = list_fargate_profiles_response[ + 'fargateProfileNames' + ] + self.limits['Fargate profiles per cluster']._add_current_usage( + len(fargate_profiles_list), + resource_id=cluster, + aws_type='AWS::EKS::FargateProfile') + + for fargate_profile_name in fargate_profiles_list: + fargate_info = self.conn.describe_fargate_profile( + clusterName=cluster, + fargateProfileName=fargate_profile_name + ) + profile_selectors = fargate_info['fargateProfile']['selectors'] + self.limits['Selectors per Fargate profile']._add_current_usage( + len(profile_selectors), + resource_id="{}.{}".format(cluster, fargate_profile_name), + aws_type='AWS::EKS::FargateProfile') + + for selector in profile_selectors: + label_pairs = selector['labels'] + self.limits[ + 'Label pairs per Fargate profile selector' + ]._add_current_usage( + len(label_pairs), + resource_id=( + "{}.{}.{}".format( + cluster, + fargate_profile_name, + selector + ) + ), + aws_type='AWS::EKS::FargateProfile') + + self.limits['Clusters']._add_current_usage( + len(cluster_list), + resource_id=self._boto3_connection_kwargs['region_name'], + aws_type='AWS::EKS::Cluster') + + def get_limits(self): + """ + Return all known limits for this service, as a dict of their names + to :py:class:`~.AwsLimit` objects. + + :returns: dict of limit names to :py:class:`~.AwsLimit` objects + :rtype: dict + """ + if self.limits != {}: + return self.limits + limits = {} + limits['Clusters'] = AwsLimit( + 'Clusters', + self, + 100, + self.warning_threshold, + self.critical_threshold, + limit_type='AWS::EKS::Cluster', + ) + limits['Control plane security groups per cluster'] = AwsLimit( + 'Control plane security groups per cluster', + self, + 4, + self.warning_threshold, + self.critical_threshold, + limit_type='AWS::EKS::Cluster', + ) + limits['Managed node groups per cluster'] = AwsLimit( + 'Managed node groups per cluster', + self, + 30, + self.warning_threshold, + self.critical_threshold, + limit_type='AWS::EKS::Nodegroup', + ) + limits['Nodes per managed node group'] = AwsLimit( + 'Nodes per managed node group', + self, + 100, + self.warning_threshold, + self.critical_threshold, + limit_type='AWS::EKS::Cluster', + ) + limits['Public endpoint access CIDR ranges per cluster'] = AwsLimit( + 'Public endpoint access CIDR ranges per cluster', + self, + 40, + self.warning_threshold, + self.critical_threshold, + limit_type='AWS::EKS::Cluster', + ) + limits['Fargate profiles per cluster'] = AwsLimit( + 'Fargate profiles per cluster', + self, + 10, + self.warning_threshold, + self.critical_threshold, + limit_type='AWS::EKS::FargateProfile', + ) + limits['Selectors per Fargate profile'] = AwsLimit( + 'Selectors per Fargate profile', + self, + 5, + self.warning_threshold, + self.critical_threshold, + limit_type='AWS::EKS::FargateProfile', + ) + limits['Label pairs per Fargate profile selector'] = AwsLimit( + 'Label pairs per Fargate profile selector', + self, + 5, + self.warning_threshold, + self.critical_threshold, + limit_type='AWS::EKS::FargateProfile', + ) + self.limits = limits + return limits + + def required_iam_permissions(self): + """ + Return a list of IAM Actions required for this Service to function + properly. All Actions will be shown with an Effect of "Allow" + and a Resource of "*". + + :returns: list of IAM Action strings + :rtype: list + """ + return [ + "eks:ListClusters", + "eks:DescribeCluster", + "eks:ListNodegroups", + "eks:ListFargateProfiles", + "eks:DescribeFargateProfile", + ] diff --git a/awslimitchecker/services/kinesis.py b/awslimitchecker/services/kinesis.py new file mode 100644 index 00000000..e3c38258 --- /dev/null +++ b/awslimitchecker/services/kinesis.py @@ -0,0 +1,128 @@ +""" +awslimitchecker/services/kinesis.py + +The latest version of this package is available at: + + +################################################################################ +Copyright 2015-2018 Jason Antman + + This file is part of awslimitchecker, also known as awslimitchecker. + + awslimitchecker is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + awslimitchecker is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with awslimitchecker. If not, see . + +The Copyright and Authors attributions contained herein may not be removed or +otherwise altered, except to add the Author attribution of a contributor to +this work. (Additional Terms pursuant to Section 7b of the AGPL v3) +################################################################################ +While not legally required, I sincerely request that anyone who finds +bugs please submit them at or +to me via email, and that you send any contributions or improvements +either as a pull request on GitHub, or to me via email. +################################################################################ + +AUTHORS: +Jason Antman +################################################################################ +""" + +import abc # noqa +import logging + +from .base import _AwsService +from ..limit import AwsLimit + +logger = logging.getLogger(__name__) + + +class _KinesisService(_AwsService): + + service_name = 'Kinesis' + api_name = 'kinesis' + quotas_service_code = 'kinesis' + + def find_usage(self): + """ + Determine the current usage for each limit of this service, + and update corresponding Limit via + :py:meth:`~.AwsLimit._add_current_usage`. + """ + logger.debug("Checking usage for service %s", self.service_name) + self.connect() + for lim in self.limits.values(): + lim._reset_usage() + self._find_shards() + self._have_usage = True + logger.debug("Done checking usage.") + + def _find_shards(self): + describe_limits_response = self.conn.describe_limits() + self.limits['Shards per Region']._add_current_usage( + describe_limits_response['OpenShardCount'], + resource_id=self._boto3_connection_kwargs['region_name'], + aws_type='AWS::Kinesis::Stream' + ) + + def get_limits(self): + """ + Return all known limits for this service, as a dict of their names + to :py:class:`~.AwsLimit` objects. + + :returns: dict of limit names to :py:class:`~.AwsLimit` objects + :rtype: dict + """ + if self.limits != {}: + return self.limits + + self.connect() + region_name = self.conn._client_config.region_name + regions_500_shards = ['us-east-1', 'us-west-2', 'eu-west-1'] + + limits = {} + + limits['Shards per Region'] = AwsLimit( + 'Shards per Region', + self, + 500 if region_name in regions_500_shards else 200, + self.warning_threshold, + self.critical_threshold, + limit_type='AWS::Kinesis::Stream', + ) + self.limits = limits + return limits + + def _update_limits_from_api(self): + """ + Call the service's API action to retrieve limit/quota information, and + update AwsLimit objects in ``self.limits`` with this information. + """ + logger.debug("Updating limits for Kinesis from the AWS API") + self.connect() + describe_limits_response = self.conn.describe_limits() + self.limits['Shards per Region']._set_api_limit( + describe_limits_response['ShardLimit'] + ) + + def required_iam_permissions(self): + """ + Return a list of IAM Actions required for this Service to function + properly. All Actions will be shown with an Effect of "Allow" + and a Resource of "*". + + :returns: list of IAM Action strings + :rtype: list + """ + return [ + 'kinesis:DescribeLimits', + ] diff --git a/awslimitchecker/services/vpc.py b/awslimitchecker/services/vpc.py index b3b0c181..83feb37b 100644 --- a/awslimitchecker/services/vpc.py +++ b/awslimitchecker/services/vpc.py @@ -48,8 +48,6 @@ logger = logging.getLogger(__name__) -DEFAULT_ENI_LIMIT = 350 - class _VpcService(_AwsService): @@ -309,6 +307,7 @@ def get_limits(self): self.critical_threshold, limit_type='AWS::EC2::Route', limit_subtype='AWS::EC2::RouteTable', + quotas_name='Routes per route table' ) limits['Internet gateways'] = AwsLimit( @@ -343,7 +342,7 @@ def get_limits(self): limits['Network interfaces per Region'] = AwsLimit( 'Network interfaces per Region', self, - DEFAULT_ENI_LIMIT, + 5000, self.warning_threshold, self.critical_threshold, limit_type='AWS::EC2::NetworkInterface' @@ -351,28 +350,6 @@ def get_limits(self): self.limits = limits return limits - def _update_limits_from_api(self): - """ - Query EC2's DescribeAccountAttributes API action and - update the network interface limit, as needed. Updates ``self.limits``. - - More info on the network interface limit, from the docs: - 'This limit is the greater of either the default limit (350) or your - On-Demand Instance limit multiplied by 5. - The default limit for On-Demand Instances is 20.' - """ - self.connect() - self.connect_resource() - logger.info("Querying EC2 DescribeAccountAttributes for limits") - attribs = self.conn.describe_account_attributes() - for attrib in attribs['AccountAttributes']: - if attrib['AttributeName'] == 'max-instances': - val = attrib['AttributeValues'][0]['AttributeValue'] - if int(val) * 5 > DEFAULT_ENI_LIMIT: - limit_name = 'Network interfaces per Region' - self.limits[limit_name]._set_api_limit(int(val) * 5) - logger.debug("Done setting limits from API") - def required_iam_permissions(self): """ Return a list of IAM Actions required for this Service to function diff --git a/awslimitchecker/tests/services/result_fixtures.py b/awslimitchecker/tests/services/result_fixtures.py index 8c9d3ca2..32dcc3f8 100644 --- a/awslimitchecker/tests/services/result_fixtures.py +++ b/awslimitchecker/tests/services/result_fixtures.py @@ -221,6 +221,7 @@ class EBS(object): class VPC(object): + test_find_usage_vpcs = { 'Vpcs': [ { @@ -620,48 +621,6 @@ class VPC(object): } } - test_update_limits_from_api_high_max_instances = { - 'ResponseMetadata': { - 'HTTPStatusCode': 200, - 'RequestId': '16b85906-ab0d-4134-b8bb-df3e6120c6c7' - }, - 'AccountAttributes': [ - { - 'AttributeName': 'max-instances', - 'AttributeValues': [ - { - 'AttributeValue': '400' - } - ] - } - ] - } - - test_update_limits_from_api_low_max_instances = { - 'ResponseMetadata': { - 'HTTPStatusCode': 200, - 'RequestId': '16b85906-ab0d-4134-b8bb-df3e6120c6c7' - }, - 'AccountAttributes': [ - { - 'AttributeName': 'max-instances', - 'AttributeValues': [ - { - 'AttributeValue': '50' - } - ] - }, - { - 'AttributeName': 'something-else', - 'AttributeValues': [ - { - 'AttributeValue': '1' - } - ] - } - ] - } - class RDS(object): test_find_usage_instances = [] @@ -4584,3 +4543,145 @@ class CloudTrail(object): } ] } + + +class Kinesis(object): + mock_describe_limits = { + 'ShardLimit': 700, + 'OpenShardCount': 555 + } + + +class EKS(object): + + test_find_clusters_usage_list = { + 'clusters': [ + 'devel', + 'prod', + ] + } + + test_find_clusters_usage_describe = [ + { + 'cluster': { + 'name': 'devel', + 'resourcesVpcConfig': { + 'securityGroupIds': [ + 'abc-1234', + + ], + 'publicAccessCidrs': [ + '1.1.1.1/32', + '2.2.2.0/24', + '203.0.113.5/32' + ] + }, + } + }, + { + 'cluster': { + 'name': 'prod', + 'resourcesVpcConfig': { + 'securityGroupIds': [ + 'foo-1234', + 'bar-1234' + ], + 'publicAccessCidrs': [ + '1.1.1.1/32', + ] + }, + } + } + ] + + test_find_clusters_usage_nodegrps = [ + { + 'nodegroups': [ + 'managed-ng-1', + 'managed-ng-2' + ] + }, + { + 'nodegroups': [ + 'managed-ng-3', + ] + } + ] + + test_find_clusters_usage_fargates = [ + { + 'fargateProfileNames': [ + 'foo', + ] + }, + { + 'fargateProfileNames': [ + 'bar', + 'baz', + ] + } + ] + + test_find_clusters_usage_fargate_prof = [ + { + 'fargateProfile': { + 'fargateProfileName': 'foo', + 'selectors': [ + { + 'namespace': "test_namespace1", + 'labels': { + 'key1': 'value1', + } + }, + ], + } + }, + { + 'fargateProfile': { + 'fargateProfileName': 'bar', + 'selectors': [ + { + 'namespace': "test_namespace1", + 'labels': { + 'key1': 'value1', + } + }, + { + 'namespace': "test_namespace2", + 'labels': { + 'key1': 'value1', + 'key2': 'value2', + } + }, + ], + } + }, + { + 'fargateProfile': { + 'fargateProfileName': 'baz', + 'selectors': [ + { + 'namespace': "test_namespace1", + 'labels': { + 'key1': 'value1', + } + }, + { + 'namespace': "test_namespace2", + 'labels': { + 'key1': 'value1', + 'key2': 'value2', + } + }, + { + 'namespace': "test_namespace3", + 'labels': { + 'key1': 'value1', + 'key2': 'value2', + 'key3': 'value3', + } + }, + ], + } + } + ] diff --git a/awslimitchecker/tests/services/test_ec2.py b/awslimitchecker/tests/services/test_ec2.py index 533d8756..a79e1517 100644 --- a/awslimitchecker/tests/services/test_ec2.py +++ b/awslimitchecker/tests/services/test_ec2.py @@ -777,18 +777,13 @@ def test_simple(self): assert mock_logger.mock_calls == [ call.debug("Getting usage for EC2 VPC resources"), ] - limit = cls.limits['Security groups per VPC'] + limit = cls.limits['VPC security groups per Region'] # relies on AwsLimitUsage sorting by numeric usage value sorted_usage = sorted(limit.get_current_usage()) - assert len(sorted_usage) == 2 + assert len(sorted_usage) == 1 assert sorted_usage[0].limit == limit - assert sorted_usage[0].get_value() == 1 - assert sorted_usage[0].resource_id == 'vpc-bbb' - assert sorted_usage[0].aws_type == 'AWS::EC2::VPC' - assert sorted_usage[1].limit == limit - assert sorted_usage[1].get_value() == 2 - assert sorted_usage[1].resource_id == 'vpc-aaa' - assert sorted_usage[1].aws_type == 'AWS::EC2::VPC' + assert sorted_usage[0].get_value() == 3 + assert sorted_usage[0].aws_type == 'AWS::EC2::SecurityGroup' limit = cls.limits['Rules per VPC security group'] sorted_usage = sorted(limit.get_current_usage()) @@ -886,7 +881,7 @@ def test_simple(self): cls = _Ec2Service(21, 43, {}, None) limits = cls._get_limits_networking() expected = [ - 'Security groups per VPC', + 'VPC security groups per Region', 'Rules per VPC security group', 'VPC Elastic IP addresses (EIPs)', 'Elastic IP addresses (EIPs)', @@ -913,6 +908,10 @@ def test_simple(self): assert limits[ 'Elastic IP addresses (EIPs)' ].quotas_unit == 'None' + vpcsg = limits['VPC security groups per Region'] + assert vpcsg.quota_name == 'VPC security groups per Region' + assert vpcsg.quotas_service_code == 'vpc' + assert vpcsg.default_limit == 2500 class TestGetLimitsSpot(object): diff --git a/awslimitchecker/tests/services/test_eks.py b/awslimitchecker/tests/services/test_eks.py new file mode 100644 index 00000000..34c518de --- /dev/null +++ b/awslimitchecker/tests/services/test_eks.py @@ -0,0 +1,228 @@ +""" +awslimitchecker/tests/services/test_eks.py + +The latest version of this package is available at: + + +################################################################################ +Copyright 2015-2018 Jason Antman + + This file is part of awslimitchecker, also known as awslimitchecker. + + awslimitchecker is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + awslimitchecker is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with awslimitchecker. If not, see . + +The Copyright and Authors attributions contained herein may not be removed or +otherwise altered, except to add the Author attribution of a contributor to +this work. (Additional Terms pursuant to Section 7b of the AGPL v3) +################################################################################ +While not legally required, I sincerely request that anyone who finds +bugs please submit them at or +to me via email, and that you send any contributions or improvements +either as a pull request on GitHub, or to me via email. +################################################################################ + +AUTHORS: +Jason Antman +################################################################################ +""" + +import sys +from awslimitchecker.tests.services import result_fixtures +from awslimitchecker.services.eks import _EksService + +# https://code.google.com/p/mock/issues/detail?id=249 +# py>=3.4 should use unittest.mock not the mock package on pypi +if ( + sys.version_info[0] < 3 or + sys.version_info[0] == 3 and sys.version_info[1] < 4 +): + from mock import patch, call, Mock, DEFAULT, ANY +else: + from unittest.mock import patch, call, Mock, DEFAULT, ANY + + +pbm = 'awslimitchecker.services.eks' # module patch base +pb = '%s._EksService' % pbm # class patch pase + + +class Test_EksService(object): + + def test_init(self): + """test __init__()""" + cls = _EksService(21, 43, {}, None) + assert cls.service_name == 'EKS' + assert cls.api_name == 'eks' + assert cls.conn is None + assert cls.warning_threshold == 21 + assert cls.critical_threshold == 43 + + def test_get_limits(self): + cls = _EksService(21, 43, {}, None) + cls.limits = {} + res = cls.get_limits() + assert sorted(res.keys()) == sorted([ + 'Clusters', + 'Control plane security groups per cluster', + 'Managed node groups per cluster', + 'Nodes per managed node group', + 'Public endpoint access CIDR ranges per cluster', + 'Fargate profiles per cluster', + 'Selectors per Fargate profile', + 'Label pairs per Fargate profile selector', + ]) + for name, limit in res.items(): + assert limit.service == cls + assert limit.def_warning_threshold == 21 + assert limit.def_critical_threshold == 43 + + def test_get_limits_again(self): + """test that existing limits dict is returned on subsequent calls""" + mock_limits = Mock() + cls = _EksService(21, 43, {}, None) + cls.limits = mock_limits + res = cls.get_limits() + assert res == mock_limits + + def test_find_usage(self): + """test find usage method calls other methods""" + mock_conn = Mock() + with patch('%s.connect' % pb) as mock_connect: + with patch.multiple( + pb, + _find_clusters_usage=DEFAULT, + ) as mocks: + cls = _EksService(21, 43, {}, None) + cls.conn = mock_conn + assert cls._have_usage is False + cls.find_usage() + assert mock_connect.mock_calls == [call()] + assert cls._have_usage is True + assert mock_conn.mock_calls == [] + for x in [ + '_find_clusters_usage', + ]: + assert mocks[x].mock_calls == [call()] + + def test_find_clusters_usage(self): + list_clusters = result_fixtures.EKS.test_find_clusters_usage_list + describe_cluster = result_fixtures.EKS.test_find_clusters_usage_describe + list_nodegroups = result_fixtures.EKS.test_find_clusters_usage_nodegrps + list_fargates = result_fixtures.EKS.test_find_clusters_usage_fargates + dsc_fargate = result_fixtures.EKS.test_find_clusters_usage_fargate_prof + + clusters_limit_key = 'Clusters' + security_group_limit_key = 'Control plane security groups per cluster' + nodegroup_limit_key = 'Managed node groups per cluster' + public_cidr_limit_key = 'Public endpoint access CIDR ranges per cluster' + fargate_profiles_limit_key = 'Fargate profiles per cluster' + selectors_limit_key = 'Selectors per Fargate profile' + label_pairs_limit_key = 'Label pairs per Fargate profile selector' + + mock_conn = Mock() + mock_conn.list_clusters.return_value = list_clusters + mock_conn.describe_cluster.side_effect = describe_cluster + mock_conn.list_nodegroups.side_effect = list_nodegroups + mock_conn.list_fargate_profiles.side_effect = list_fargates + mock_conn.describe_fargate_profile.side_effect = dsc_fargate + + cls = _EksService(21, 43, {'region_name': 'us-west-2'}, None) + cls.conn = mock_conn + cls._find_clusters_usage() + + assert mock_conn.mock_calls == [ + call.list_clusters(), + call.describe_cluster(name=ANY), + call.list_nodegroups(clusterName=ANY), + call.list_fargate_profiles(clusterName=ANY), + call.describe_fargate_profile( + clusterName=ANY, + fargateProfileName=ANY + ), + call.describe_cluster(name=ANY), + call.list_nodegroups(clusterName=ANY), + call.list_fargate_profiles(clusterName=ANY), + call.describe_fargate_profile( + clusterName=ANY, + fargateProfileName=ANY + ), + call.describe_fargate_profile( + clusterName=ANY, + fargateProfileName=ANY + ), + ] + assert len(cls.limits[clusters_limit_key].get_current_usage()) == 1 + assert cls.limits[clusters_limit_key].get_current_usage()[ + 0].get_value() == 2 + + assert len(cls.limits[ + security_group_limit_key].get_current_usage()) == 2 + assert cls.limits[security_group_limit_key].get_current_usage()[ + 0].get_value() == 1 + assert cls.limits[security_group_limit_key].get_current_usage()[ + 1].get_value() == 2 + + assert len(cls.limits[ + nodegroup_limit_key].get_current_usage()) == 2 + assert cls.limits[nodegroup_limit_key].get_current_usage()[ + 0].get_value() == 2 + assert cls.limits[nodegroup_limit_key].get_current_usage()[ + 1].get_value() == 1 + + assert len(cls.limits[ + public_cidr_limit_key].get_current_usage()) == 2 + assert cls.limits[public_cidr_limit_key].get_current_usage()[ + 0].get_value() == 3 + assert cls.limits[public_cidr_limit_key].get_current_usage()[ + 1].get_value() == 1 + + assert len(cls.limits[ + fargate_profiles_limit_key].get_current_usage()) == 2 + assert cls.limits[fargate_profiles_limit_key].get_current_usage()[ + 0].get_value() == 1 + assert cls.limits[fargate_profiles_limit_key].get_current_usage()[ + 1].get_value() == 2 + + assert len(cls.limits[ + selectors_limit_key].get_current_usage()) == 3 + assert cls.limits[selectors_limit_key].get_current_usage()[ + 0].get_value() == 1 + assert cls.limits[selectors_limit_key].get_current_usage()[ + 1].get_value() == 2 + assert cls.limits[selectors_limit_key].get_current_usage()[ + 2].get_value() == 3 + + assert len(cls.limits[ + label_pairs_limit_key].get_current_usage()) == 6 + assert cls.limits[label_pairs_limit_key].get_current_usage()[ + 0].get_value() == 1 + assert cls.limits[label_pairs_limit_key].get_current_usage()[ + 1].get_value() == 1 + assert cls.limits[label_pairs_limit_key].get_current_usage()[ + 2].get_value() == 2 + assert cls.limits[label_pairs_limit_key].get_current_usage()[ + 3].get_value() == 1 + assert cls.limits[label_pairs_limit_key].get_current_usage()[ + 4].get_value() == 2 + assert cls.limits[label_pairs_limit_key].get_current_usage()[ + 5].get_value() == 3 + + def test_required_iam_permissions(self): + cls = _EksService(21, 43, {}, None) + assert cls.required_iam_permissions() == [ + "eks:ListClusters", + "eks:DescribeCluster", + "eks:ListNodegroups", + "eks:ListFargateProfiles", + "eks:DescribeFargateProfile" + ] diff --git a/awslimitchecker/tests/services/test_kinesis.py b/awslimitchecker/tests/services/test_kinesis.py new file mode 100644 index 00000000..81cf58d0 --- /dev/null +++ b/awslimitchecker/tests/services/test_kinesis.py @@ -0,0 +1,196 @@ +""" +awslimitchecker/tests/services/test_kinesis.py + +The latest version of this package is available at: + + +################################################################################ +Copyright 2015-2018 Jason Antman + + This file is part of awslimitchecker, also known as awslimitchecker. + + awslimitchecker is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + awslimitchecker is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with awslimitchecker. If not, see . + +The Copyright and Authors attributions contained herein may not be removed or +otherwise altered, except to add the Author attribution of a contributor to +this work. (Additional Terms pursuant to Section 7b of the AGPL v3) +################################################################################ +While not legally required, I sincerely request that anyone who finds +bugs please submit them at or +to me via email, and that you send any contributions or improvements +either as a pull request on GitHub, or to me via email. +################################################################################ + +AUTHORS: +Jason Antman +################################################################################ +""" + +import sys +from awslimitchecker.tests.services import result_fixtures +from awslimitchecker.limit import AwsLimit +from awslimitchecker.services.kinesis import _KinesisService + +# https://code.google.com/p/mock/issues/detail?id=249 +# py>=3.4 should use unittest.mock not the mock package on pypi +if ( + sys.version_info[0] < 3 or + sys.version_info[0] == 3 and sys.version_info[1] < 4 +): + from mock import patch, call, Mock, DEFAULT +else: + from unittest.mock import patch, call, Mock, DEFAULT + + +pbm = 'awslimitchecker.services.kinesis' # module patch base +pb = '%s._KinesisService' % pbm # class patch pase + + +class Test_KinesisService(object): + + def test_init(self): + """test __init__()""" + with patch('%s.get_limits' % pb): + cls = _KinesisService(21, 43, {}, None) + assert cls.service_name == 'Kinesis' + assert cls.api_name == 'kinesis' + assert cls.conn is None + assert cls.warning_threshold == 21 + assert cls.critical_threshold == 43 + + def test_get_limits(self): + mock_conn = Mock() + m_client = Mock() + type(m_client).region_name = 'ap-southeast-2' + type(mock_conn)._client_config = m_client + + def se_conn(cls): + cls.conn = mock_conn + + with patch('%s.connect' % pb, autospec=True) as mock_connect: + mock_connect.side_effect = se_conn + cls = _KinesisService(21, 43, {}, None) + + cls.limits = {} + res = cls.get_limits() + assert sorted(res.keys()) == sorted([ + 'Shards per Region', + ]) + for name, limit in res.items(): + assert limit.service == cls + assert limit.def_warning_threshold == 21 + assert limit.def_critical_threshold == 43 + + limits = cls.limits + assert len(limits) == 1 + assert limits['Shards per Region'].default_limit == 200 + + def test_get_limits_us_east_1(self): + mock_conn = Mock() + m_client = Mock() + type(m_client).region_name = 'us-east-1' + type(mock_conn)._client_config = m_client + + def se_conn(cls): + cls.conn = mock_conn + + with patch('%s.connect' % pb, autospec=True) as mock_connect: + mock_connect.side_effect = se_conn + cls = _KinesisService(21, 43, {}, None) + + limits = cls.limits + for x in limits: + assert isinstance(limits[x], AwsLimit) + assert x == limits[x].name + assert limits[x].service == cls + + assert len(limits) == 1 + assert limits['Shards per Region'].default_limit == 500 + + def test_get_limits_again(self): + """test that existing limits dict is returned on subsequent calls""" + mock_limits = Mock() + cls = _KinesisService(21, 43, {}, None) + cls.limits = mock_limits + res = cls.get_limits() + assert res == mock_limits + + def test_find_usage(self): + mock_conn = Mock() + + def se_conn(cls): + cls.conn = mock_conn + + with patch('%s.connect' % pb, autospec=True) as mock_connect: + mock_connect.side_effect = se_conn + with patch.multiple( + pb, + _find_shards=DEFAULT, + ) as mocks: + cls = _KinesisService(21, 43, {}, None) + cls.conn = mock_conn + assert cls._have_usage is False + cls.find_usage() + assert mock_connect.mock_calls == [call(cls), call(cls)] + assert cls._have_usage is True + assert mock_conn.mock_calls == [] + for x in [ + '_find_shards', + ]: + assert mocks[x].mock_calls == [call()] + + def test_find_shards(self): + response = result_fixtures.Kinesis.mock_describe_limits + limit_key = 'Shards per Region' + + mock_conn = Mock() + mock_conn.describe_limits.return_value = response + + cls = _KinesisService(21, 43, {'region_name': 'us-west-2'}, None) + cls.conn = mock_conn + cls._find_shards() + + assert mock_conn.mock_calls == [ + call.describe_limits() + ] + assert len(cls.limits[limit_key].get_current_usage()) == 1 + assert cls.limits[limit_key].get_current_usage()[ + 0].get_value() == 555 + + def test_update_limits_from_api(self): + response = result_fixtures.Kinesis.mock_describe_limits + mock_conn = Mock() + mock_conn.describe_limits.return_value = response + + def se_conn(cls): + cls.conn = mock_conn + + with patch('%s.connect' % pb, autospec=True) as mock_connect: + mock_connect.side_effect = se_conn + cls = _KinesisService(21, 43, {'region_name': 'us-west-2'}, None) + assert len(cls.limits) == 1 + cls.conn = mock_conn + cls._update_limits_from_api() + + assert mock_connect.mock_calls == [call(cls), call(cls)] + assert mock_conn.mock_calls == [call.describe_limits()] + assert len(cls.limits) == 1 + lim = cls.limits['Shards per Region'].get_limit() + assert lim == 700 + + def test_required_iam_permissions(self): + cls = _KinesisService(21, 43, {}, None) + assert cls.required_iam_permissions() == [ + 'kinesis:DescribeLimits', + ] diff --git a/awslimitchecker/tests/services/test_vpc.py b/awslimitchecker/tests/services/test_vpc.py index 0744d5e2..e968935a 100644 --- a/awslimitchecker/tests/services/test_vpc.py +++ b/awslimitchecker/tests/services/test_vpc.py @@ -39,7 +39,7 @@ import sys from awslimitchecker.tests.services import result_fixtures -from awslimitchecker.services.vpc import _VpcService, DEFAULT_ENI_LIMIT +from awslimitchecker.services.vpc import _VpcService from botocore.exceptions import ClientError @@ -382,56 +382,6 @@ def test_find_usage_network_interfaces(self): }]), ] - def test_update_limits_from_api_high_max_instances(self): - fixtures = result_fixtures.VPC() - response = fixtures.test_update_limits_from_api_high_max_instances - - mock_conn = Mock() - mock_client_conn = Mock() - mock_client_conn.describe_account_attributes.return_value = response - - cls = _VpcService(21, 43, {}, None) - cls.resource_conn = mock_conn - cls.conn = mock_client_conn - with patch('awslimitchecker.services.vpc.logger') as mock_logger: - cls._update_limits_from_api() - assert mock_conn.mock_calls == [] - assert mock_client_conn.mock_calls == [ - call.describe_account_attributes() - ] - assert mock_logger.mock_calls == [ - call.info("Querying EC2 DescribeAccountAttributes for limits"), - call.debug('Done setting limits from API') - ] - assert cls.limits['Network interfaces per Region'].api_limit == 2000 - assert cls.limits['Network interfaces per Region'].get_limit() == 2000 - - def test_update_limits_from_api_low_max_instances(self): - fixtures = result_fixtures.VPC() - response = fixtures.test_update_limits_from_api_low_max_instances - - mock_conn = Mock() - mock_client_conn = Mock() - mock_client_conn.describe_account_attributes.return_value = response - - cls = _VpcService(21, 43, {}, None) - cls.resource_conn = mock_conn - cls.conn = mock_client_conn - with patch('awslimitchecker.services.vpc.logger') as mock_logger: - cls._update_limits_from_api() - assert mock_conn.mock_calls == [] - assert mock_client_conn.mock_calls == [ - call.describe_account_attributes() - ] - assert mock_logger.mock_calls == [ - call.info("Querying EC2 DescribeAccountAttributes for limits"), - call.debug('Done setting limits from API') - ] - - limit_name = 'Network interfaces per Region' - assert cls.limits[limit_name].api_limit is None - assert cls.limits[limit_name].get_limit() == DEFAULT_ENI_LIMIT - def test_required_iam_permissions(self): cls = _VpcService(21, 43, {}, None) assert cls.required_iam_permissions() == [