From 207d5fc5d39aefcd0cb5cedfae21b0367a92164e Mon Sep 17 00:00:00 2001 From: Jason Antman Date: Thu, 29 Aug 2019 13:35:44 -0400 Subject: [PATCH 1/5] Issue #418 - initial implementation of metrics support --- awslimitchecker/checker.py | 12 ++ awslimitchecker/metrics/__init__.py | 41 ++++++ awslimitchecker/metrics/base.py | 121 +++++++++++++++ awslimitchecker/metrics/dummy.py | 71 +++++++++ awslimitchecker/runner.py | 46 +++++- awslimitchecker/tests/metrics/__init__.py | 38 +++++ awslimitchecker/tests/metrics/test_base.py | 86 +++++++++++ awslimitchecker/tests/metrics/test_dummy.py | 85 +++++++++++ awslimitchecker/tests/test_checker.py | 17 ++- awslimitchecker/tests/test_runner.py | 138 +++++++++++++++++- docs/source/awslimitchecker.metrics.base.rst | 7 + docs/source/awslimitchecker.metrics.dummy.rst | 7 + docs/source/awslimitchecker.metrics.rst | 16 ++ docs/source/awslimitchecker.rst | 1 + docs/source/development.rst | 22 +++ pytest.ini | 1 + 16 files changed, 698 insertions(+), 11 deletions(-) create mode 100644 awslimitchecker/metrics/__init__.py create mode 100644 awslimitchecker/metrics/base.py create mode 100644 awslimitchecker/metrics/dummy.py create mode 100644 awslimitchecker/tests/metrics/__init__.py create mode 100644 awslimitchecker/tests/metrics/test_base.py create mode 100644 awslimitchecker/tests/metrics/test_dummy.py create mode 100644 docs/source/awslimitchecker.metrics.base.rst create mode 100644 docs/source/awslimitchecker.metrics.dummy.rst create mode 100644 docs/source/awslimitchecker.metrics.rst diff --git a/awslimitchecker/checker.py b/awslimitchecker/checker.py index 81bdb610..057acb16 100644 --- a/awslimitchecker/checker.py +++ b/awslimitchecker/checker.py @@ -584,3 +584,15 @@ def get_required_iam_policy(self): }], } return policy + + @property + def region_name(self): + """ + Return the name of the AWS region that we're checking. + + :return: AWS region name + :rtype: str + """ + kwargs = self._boto_conn_kwargs + conn = boto3.client('ec2', **kwargs) + return conn._client_config.region_name diff --git a/awslimitchecker/metrics/__init__.py b/awslimitchecker/metrics/__init__.py new file mode 100644 index 00000000..6fda0bd2 --- /dev/null +++ b/awslimitchecker/metrics/__init__.py @@ -0,0 +1,41 @@ +""" +awslimitchecker/metrics/__init__.py + +The latest version of this package is available at: + + +################################################################################ +Copyright 2015-2019 Jason Antman + + This file is part of awslimitchecker, also known as awslimitchecker. + + awslimitchecker is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + awslimitchecker is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with awslimitchecker. If not, see . + +The Copyright and Authors attributions contained herein may not be removed or +otherwise altered, except to add the Author attribution of a contributor to +this work. (Additional Terms pursuant to Section 7b of the AGPL v3) +################################################################################ +While not legally required, I sincerely request that anyone who finds +bugs please submit them at or +to me via email, and that you send any contributions or improvements +either as a pull request on GitHub, or to me via email. +################################################################################ + +AUTHORS: +Jason Antman +################################################################################ +""" + +from .base import MetricsProvider +from .dummy import Dummy diff --git a/awslimitchecker/metrics/base.py b/awslimitchecker/metrics/base.py new file mode 100644 index 00000000..f95247f2 --- /dev/null +++ b/awslimitchecker/metrics/base.py @@ -0,0 +1,121 @@ +""" +awslimitchecker/metrics/base.py + +The latest version of this package is available at: + + +################################################################################ +Copyright 2015-2019 Jason Antman + + This file is part of awslimitchecker, also known as awslimitchecker. + + awslimitchecker is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + awslimitchecker is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with awslimitchecker. If not, see . + +The Copyright and Authors attributions contained herein may not be removed or +otherwise altered, except to add the Author attribution of a contributor to +this work. (Additional Terms pursuant to Section 7b of the AGPL v3) +################################################################################ +While not legally required, I sincerely request that anyone who finds +bugs please submit them at or +to me via email, and that you send any contributions or improvements +either as a pull request on GitHub, or to me via email. +################################################################################ + +AUTHORS: +Jason Antman +################################################################################ +""" + +import logging +from abc import ABCMeta, abstractmethod + +logger = logging.getLogger(__name__) + + +class MetricsProvider(object): + + __metaclass__ = ABCMeta + + def __init__(self, region_name): + """ + Initialize a MetricsProvider class. This MUST be overridden by + subclasses. All configuration must be passed as keyword arguments + to the class constructor (these come from ``--metrics-config`` CLI + arguments). Any dependency imports must be made in the constructor. + The constructor should do as much as possible to validate configuration. + + :param region_name: the name of the region we're connected to + :type region_name: str + """ + self._region_name = region_name + self._duration = 0.0 + self._limits = [] + + def set_run_duration(self, duration): + """ + Set the duration for the awslimitchecker run (the time taken to check + usage against limits). + + :param duration: time taken to check limits + :type duration: float + """ + self._duration = duration + + def add_limit(self, limit): + """ + Cache a given limit for later sending to the metrics store. + + :param limit: a limit to cache + :type limit: AwsLimit + """ + self._limits.append(limit) + + @abstractmethod + def flush(self): + """ + Flush all metrics to the provider. This is the method that actually + sends data to your metrics provider/store. It should iterate over + ``self._limits`` and send metrics for them, as well as for + ``self._duration``. + """ + raise NotImplementedError() + + @staticmethod + def providers_by_name(): + """ + Return a dict of available MetricsProvider subclass names to the class + objects. + + :return: MetricsProvider class names to classes + :rtype: dict + """ + return {x.__name__: x for x in MetricsProvider.__subclasses__()} + + @staticmethod + def get_provider_by_name(name): + """ + Get a reference to the provider class with the specified name. + + :param name: name of the MetricsProvider subclass + :type name: str + :return: MetricsProvider subclass + :rtype: ``class`` + :raises: RuntimeError + """ + try: + return MetricsProvider.providers_by_name()[name] + except KeyError: + raise RuntimeError( + 'ERROR: "%s" is not a valid MetricsProvider class name' % name + ) diff --git a/awslimitchecker/metrics/dummy.py b/awslimitchecker/metrics/dummy.py new file mode 100644 index 00000000..983d2999 --- /dev/null +++ b/awslimitchecker/metrics/dummy.py @@ -0,0 +1,71 @@ +""" +awslimitchecker/metrics/dummy.py + +The latest version of this package is available at: + + +################################################################################ +Copyright 2015-2018 Jason Antman + + This file is part of awslimitchecker, also known as awslimitchecker. + + awslimitchecker is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + awslimitchecker is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with awslimitchecker. If not, see . + +The Copyright and Authors attributions contained herein may not be removed or +otherwise altered, except to add the Author attribution of a contributor to +this work. (Additional Terms pursuant to Section 7b of the AGPL v3) +################################################################################ +While not legally required, I sincerely request that anyone who finds +bugs please submit them at or +to me via email, and that you send any contributions or improvements +either as a pull request on GitHub, or to me via email. +################################################################################ + +AUTHORS: +Jason Antman +################################################################################ +""" + +import logging +from awslimitchecker.metrics.base import MetricsProvider + +logger = logging.getLogger(__name__) + + +class Dummy(MetricsProvider): + """Just writes metrics to STDOUT; mainly used for testing.""" + + def __init__(self, region_name, **_): + super(Dummy, self).__init__(region_name) + + def flush(self): + print('DummyMetrics Provider flush for region=%s' % self._region_name) + print('Duration: %s' % self._duration) + lines = [] + for lim in self._limits: + u = lim.get_current_usage() + if len(u) == 0: + max_usage = 0 + else: + max_usage = max(u).get_value() + limit = lim.get_limit() + if limit is None: + limit = 'unknown' + lines.append( + '%s / %s: limit=%s max_usage=%s' % ( + lim.service.service_name, lim.name, limit, max_usage + ) + ) + for l in sorted(lines): + print(l) diff --git a/awslimitchecker/runner.py b/awslimitchecker/runner.py index 6bb3e830..6d8aa8fb 100644 --- a/awslimitchecker/runner.py +++ b/awslimitchecker/runner.py @@ -43,10 +43,12 @@ import json import termcolor import boto3 +import time from .checker import AwsLimitChecker from .utils import StoreKeyValuePair, dict2cols from .limit import SOURCE_TA, SOURCE_API +from .metrics import MetricsProvider try: from urllib.parse import urlparse @@ -222,6 +224,19 @@ def parse_args(self, argv): p.add_argument('-V', '--version', dest='version', action='store_true', default=False, help='print version number and exit.') + p.add_argument('--list-metrics-providers', + dest='list_metrics_providers', + action='store_true', default=False, + help='List available metrics providers and exit') + p.add_argument('--metrics-provider', dest='metrics_provider', type=str, + action='store', default=None, + help='Metrics provider class name, to enable sending ' + 'metrics') + p.add_argument('--metrics-config', action=StoreKeyValuePair, + dest='metrics_config', + help='Specify key/value parameters for the metrics ' + 'provider constructor. See documentation for ' + 'further information.') args = p.parse_args(argv) args.ta_refresh_mode = None if args.ta_refresh_wait: @@ -323,12 +338,19 @@ def print_issue(self, service_name, limit, crits, warns): ) return (k, v) - def check_thresholds(self): + def check_thresholds(self, metrics=None): have_warn = False have_crit = False problems = self.checker.check_thresholds( use_ta=(not self.skip_ta), - service=self.service_name) + service=self.service_name + ) + if metrics: + for svc, svc_limits in sorted(self.checker.get_limits().items()): + if self.service_name and svc not in self.service_name: + continue + for _, limit in sorted(svc_limits.items()): + metrics.add_limit(limit) columns = {} for svc in sorted(problems.keys()): for lim_name in sorted(problems[svc].keys()): @@ -338,7 +360,6 @@ def check_thresholds(self): ) if check_name in self.skip_check: continue - limit = problems[svc][lim_name] warns = limit.get_warnings() crits = limit.get_criticals() @@ -477,8 +498,25 @@ def console_entry_point(self): self.show_usage() raise SystemExit(0) + if args.list_metrics_providers: + print('Available metrics providers:') + for p in sorted(MetricsProvider.providers_by_name().keys()): + print(p) + raise SystemExit(0) + # else check - res = self.check_thresholds() + metrics = None + if args.metrics_provider: + metrics = MetricsProvider.get_provider_by_name( + args.metrics_provider + )(self.checker.region_name, **args.metrics_config) + start_time = time.time() + res = self.check_thresholds(metrics) + duration = time.time() - start_time + logger.info('Finished checking limits in %s seconds', duration) + if metrics: + metrics.set_run_duration(duration) + metrics.flush() raise SystemExit(res) diff --git a/awslimitchecker/tests/metrics/__init__.py b/awslimitchecker/tests/metrics/__init__.py new file mode 100644 index 00000000..47909a8f --- /dev/null +++ b/awslimitchecker/tests/metrics/__init__.py @@ -0,0 +1,38 @@ +""" +awslimitchecker/tests/metrics/__init__.py + +The latest version of this package is available at: + + +################################################################################ +Copyright 2015-2019 Jason Antman + + This file is part of awslimitchecker, also known as awslimitchecker. + + awslimitchecker is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + awslimitchecker is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with awslimitchecker. If not, see . + +The Copyright and Authors attributions contained herein may not be removed or +otherwise altered, except to add the Author attribution of a contributor to +this work. (Additional Terms pursuant to Section 7b of the AGPL v3) +################################################################################ +While not legally required, I sincerely request that anyone who finds +bugs please submit them at or +to me via email, and that you send any contributions or improvements +either as a pull request on GitHub, or to me via email. +################################################################################ + +AUTHORS: +Jason Antman +################################################################################ +""" diff --git a/awslimitchecker/tests/metrics/test_base.py b/awslimitchecker/tests/metrics/test_base.py new file mode 100644 index 00000000..15273ed6 --- /dev/null +++ b/awslimitchecker/tests/metrics/test_base.py @@ -0,0 +1,86 @@ +""" +awslimitchecker/tests/metrics/test_base.py + +The latest version of this package is available at: + + +################################################################################ +Copyright 2015-2019 Jason Antman + + This file is part of awslimitchecker, also known as awslimitchecker. + + awslimitchecker is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + awslimitchecker is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with awslimitchecker. If not, see . + +The Copyright and Authors attributions contained herein may not be removed or +otherwise altered, except to add the Author attribution of a contributor to +this work. (Additional Terms pursuant to Section 7b of the AGPL v3) +################################################################################ +While not legally required, I sincerely request that anyone who finds +bugs please submit them at or +to me via email, and that you send any contributions or improvements +either as a pull request on GitHub, or to me via email. +################################################################################ + +AUTHORS: +Jason Antman +################################################################################ +""" + +from awslimitchecker.metrics.base import MetricsProvider +from awslimitchecker.metrics import Dummy + +import pytest + + +class MPTester(MetricsProvider): + + def flush(self): + pass + + +class TestMetricsProvider(object): + + def test_init(self): + cls = MPTester('foo') + assert cls._region_name == 'foo' + assert cls._duration == 0.0 + assert cls._limits == [] + + def test_set_run_duration(self): + cls = MPTester('foo') + assert cls._duration == 0.0 + cls.set_run_duration(123.45) + assert cls._duration == 123.45 + + def test_add_limit(self): + cls = MPTester('foo') + assert cls._limits == [] + cls.add_limit(1) + cls.add_limit(2) + assert cls._limits == [1, 2] + + def test_providers_by_name(self): + assert MetricsProvider.providers_by_name() == { + 'Dummy': Dummy, + 'MPTester': MPTester + } + + def test_get_provider_by_name(self): + assert MetricsProvider.get_provider_by_name('Dummy') == Dummy + + def test_get_provider_by_name_exception(self): + with pytest.raises(RuntimeError) as exc: + MetricsProvider.get_provider_by_name('3993fhej') + assert str(exc.value) == 'ERROR: "3993fhej" is not a valid ' \ + 'MetricsProvider class name' diff --git a/awslimitchecker/tests/metrics/test_dummy.py b/awslimitchecker/tests/metrics/test_dummy.py new file mode 100644 index 00000000..dda6c16c --- /dev/null +++ b/awslimitchecker/tests/metrics/test_dummy.py @@ -0,0 +1,85 @@ +""" +awslimitchecker/tests/metrics/test_dummy.py + +The latest version of this package is available at: + + +################################################################################ +Copyright 2015-2019 Jason Antman + + This file is part of awslimitchecker, also known as awslimitchecker. + + awslimitchecker is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + awslimitchecker is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with awslimitchecker. If not, see . + +The Copyright and Authors attributions contained herein may not be removed or +otherwise altered, except to add the Author attribution of a contributor to +this work. (Additional Terms pursuant to Section 7b of the AGPL v3) +################################################################################ +While not legally required, I sincerely request that anyone who finds +bugs please submit them at or +to me via email, and that you send any contributions or improvements +either as a pull request on GitHub, or to me via email. +################################################################################ + +AUTHORS: +Jason Antman +################################################################################ +""" + +import sys +from awslimitchecker.metrics import Dummy + +if ( + sys.version_info[0] < 3 or + sys.version_info[0] == 3 and sys.version_info[1] < 4 +): + from mock import Mock +else: + from unittest.mock import Mock + + +class TestDummyInit(object): + + def test_happy_path(self): + cls = Dummy('foo') + assert cls._region_name == 'foo' + assert cls._duration == 0.0 + assert cls._limits == [] + + def test_flush(self, capsys): + cls = Dummy('foo') + cls.set_run_duration(123.45) + limA = Mock( + name='limitA', service=Mock(service_name='SVC1') + ) + type(limA).name = 'limitA' + limA.get_current_usage.return_value = [] + limA.get_limit.return_value = None + cls.add_limit(limA) + limB = Mock( + name='limitB', service=Mock(service_name='SVC1') + ) + type(limB).name = 'limitB' + mocku = Mock() + mocku.get_value.return_value = 6 + limB.get_current_usage.return_value = [mocku] + limB.get_limit.return_value = 10 + cls.add_limit(limB) + cls.flush() + out, err = capsys.readouterr() + assert err == '' + assert out == 'DummyMetrics Provider flush for region=foo\n' \ + 'Duration: 123.45\n' \ + 'SVC1 / limitA: limit=unknown max_usage=0\n' \ + 'SVC1 / limitB: limit=10 max_usage=6\n' diff --git a/awslimitchecker/tests/test_checker.py b/awslimitchecker/tests/test_checker.py index 1d24e956..af12b632 100644 --- a/awslimitchecker/tests/test_checker.py +++ b/awslimitchecker/tests/test_checker.py @@ -53,9 +53,9 @@ sys.version_info[0] < 3 or sys.version_info[0] == 3 and sys.version_info[1] < 4 ): - from mock import patch, call, Mock, DEFAULT + from mock import patch, call, Mock, DEFAULT, PropertyMock else: - from unittest.mock import patch, call, Mock, DEFAULT + from unittest.mock import patch, call, Mock, DEFAULT, PropertyMock pbm = 'awslimitchecker.checker' # patch base path - module pb = '%s.AwsLimitChecker' % pbm # patch base path @@ -868,3 +868,16 @@ def test_check_thresholds_no_ta(self): call._update_limits_from_api(), call.check_thresholds() ] + + def test_region_name(self): + mock_client = Mock( + _client_config=Mock(region_name='rname') + ) + with patch( + '%s._boto_conn_kwargs' % pb, new_callable=PropertyMock + ) as mock_bck: + mock_bck.return_value = {'foo': 'bar'} + with patch('%s.boto3.client' % pbm) as m_client: + m_client.return_value = mock_client + res = self.cls.region_name + assert res == 'rname' diff --git a/awslimitchecker/tests/test_runner.py b/awslimitchecker/tests/test_runner.py index 1a060d45..0d898432 100644 --- a/awslimitchecker/tests/test_runner.py +++ b/awslimitchecker/tests/test_runner.py @@ -43,6 +43,7 @@ import logging import json import termcolor +from freezegun import freeze_time from awslimitchecker.runner import Runner, console_entry_point from awslimitchecker.checker import AwsLimitChecker @@ -56,9 +57,9 @@ sys.version_info[0] < 3 or sys.version_info[0] == 3 and sys.version_info[1] < 4 ): - from mock import patch, call, Mock, mock_open + from mock import patch, call, Mock, mock_open, PropertyMock else: - from unittest.mock import patch, call, Mock, mock_open + from unittest.mock import patch, call, Mock, mock_open, PropertyMock def red(s): @@ -118,6 +119,9 @@ def test_simple(self): assert res.limit == {} assert res.limit_override_json is None assert res.threshold_override_json is None + assert res.list_metrics_providers is False + assert res.metrics_provider is None + assert res.metrics_config == {} def test_parser(self): argv = ['-V'] @@ -269,6 +273,22 @@ def test_parser(self): action='store_true', default=False, help='print version number and exit.'), + call().add_argument('--list-metrics-providers', + dest='list_metrics_providers', + action='store_true', default=False, + help='List available metrics providers and exit' + ), + call().add_argument('--metrics-provider', dest='metrics_provider', + type=str, + action='store', default=None, + help='Metrics provider class name, to enable ' + 'sending metrics' + ), + call().add_argument('--metrics-config', action=StoreKeyValuePair, + dest='metrics_config', + help='Specify key/value parameters for the ' + 'metrics provider constructor. See ' + 'documentation for further information.'), call().parse_args(argv) ] @@ -332,6 +352,19 @@ def test_skip_check_multiple(self): 'EC2/Running On-Demand c5.9xlarge instances', ] + def test_list_metrics_providers(self): + res = self.cls.parse_args(['--list-metrics-providers']) + assert res.list_metrics_providers is True + + def test_metrics_provider(self): + res = self.cls.parse_args([ + '--metrics-provider=ClassName', + '--metrics-config=foo=bar', + '--metrics-config=baz=blam' + ]) + assert res.metrics_provider == 'ClassName' + assert res.metrics_config == {'foo': 'bar', 'baz': 'blam'} + class TestListServices(RunnerTester): @@ -719,6 +752,7 @@ def test_ok(self, capsys): """no problems, return 0 and print nothing""" mock_checker = Mock(spec_set=AwsLimitChecker) mock_checker.check_thresholds.return_value = {} + mock_checker.get_limits.return_value = {} self.cls.checker = mock_checker with patch('awslimitchecker.runner.dict2cols') as mock_d2c: mock_d2c.return_value = '' @@ -730,6 +764,40 @@ def test_ok(self, capsys): ] assert res == 0 + def test_metrics(self, capsys): + """no problems, return 0 and print nothing; send metrics""" + mock_checker = Mock(spec_set=AwsLimitChecker) + mock_checker.check_thresholds.return_value = {} + mock_lim1 = Mock() + mock_lim2 = Mock() + mock_lim3 = Mock() + mock_checker.get_limits.return_value = { + 'S1': { + 'lim1': mock_lim1, + 'lim2': mock_lim2 + }, + 'S2': { + 'lim3': mock_lim3 + } + } + mock_metrics = Mock() + self.cls.checker = mock_checker + self.cls.service_name = ['S1'] + with patch('awslimitchecker.runner.dict2cols') as mock_d2c: + mock_d2c.return_value = '' + res = self.cls.check_thresholds(metrics=mock_metrics) + out, err = capsys.readouterr() + assert out == '\n' + assert mock_checker.mock_calls == [ + call.check_thresholds(use_ta=True, service=['S1']), + call.get_limits() + ] + assert res == 0 + assert mock_metrics.mock_calls == [ + call.add_limit(mock_lim1), + call.add_limit(mock_lim2) + ] + def test_many_problems(self): """lots of problems""" mock_limit1 = Mock(spec_set=AwsLimit) @@ -768,6 +836,7 @@ def test_many_problems(self): 'limit2': mock_limit2, }, } + mock_checker.get_limits.return_value = {} def se_print(cls, s, l, c, w): return ('{s}/{l}'.format(s=s, l=l.name), '') @@ -820,6 +889,7 @@ def test_when_skip_check(self): 'limit2': mock_limit2, }, } + mock_checker.get_limits.return_value = {} def se_print(cls, s, l, c, w): return ('{s}/{l}'.format(s=s, l=l.name), '') @@ -868,6 +938,7 @@ def test_warn(self): 'limit1': mock_limit1, }, } + mock_checker.get_limits.return_value = {} self.cls.checker = mock_checker with patch('awslimitchecker.runner.Runner.print_issue', @@ -904,6 +975,7 @@ def test_warn_one_service(self): 'limit2': mock_limit2, }, } + mock_checker.get_limits.return_value = {} self.cls.checker = mock_checker self.cls.service_name = ['svc2'] @@ -935,6 +1007,7 @@ def test_crit(self): 'limit1': mock_limit1, }, } + mock_checker.get_limits.return_value = {} self.cls.checker = mock_checker self.cls.skip_ta = True @@ -1636,15 +1709,70 @@ def test_critical_ta_refresh(self): def test_check_thresholds(self): argv = ['awslimitchecker'] with patch.object(sys, 'argv', argv): - with patch('%s.Runner.check_thresholds' % pb, - autospec=True) as mock_ct: + with patch( + '%s.Runner.check_thresholds' % pb, autospec=True + ) as mock_ct: with pytest.raises(SystemExit) as excinfo: mock_ct.return_value = 10 self.cls.console_entry_point() assert excinfo.value.code == 10 assert mock_ct.mock_calls == [ - call(self.cls) + call(self.cls, None) + ] + + @freeze_time("2016-12-16 10:40:42", tz_offset=0, auto_tick_seconds=6) + def test_check_thresholds_with_metrics(self): + argv = [ + 'awslimitchecker', + '--metrics-provider=FooProvider', + '--metrics-config=foo=bar', + '--metrics-config=baz=blam' + ] + mock_prov = Mock() + mock_rn = PropertyMock(return_value='rname') + with patch.object(sys, 'argv', argv): + with patch( + '%s.Runner.check_thresholds' % pb, autospec=True + ) as mock_ct: + with patch( + '%s.MetricsProvider.get_provider_by_name' % pb + ) as m_gpbn: + m_gpbn.return_value = mock_prov + with patch( + '%s.AwsLimitChecker' % pb, spec_set=AwsLimitChecker + ) as mock_alc: + type(mock_alc.return_value).region_name = mock_rn + with pytest.raises(SystemExit) as excinfo: + mock_ct.return_value = 10 + self.cls.console_entry_point() + assert excinfo.value.code == 10 + assert mock_ct.mock_calls == [ + call(self.cls, mock_prov.return_value) ] + assert mock_prov.mock_calls == [ + call('rname', foo='bar', baz='blam'), + call().set_run_duration(6), + call().flush() + ] + + def test_list_metrics_providers(self, capsys): + argv = ['awslimitchecker', '--list-metrics-providers'] + with patch.object(sys, 'argv', argv): + with patch( + '%s.MetricsProvider.providers_by_name' % pb, + ) as mock_list: + mock_list.return_value = { + 'Prov2': None, + 'Prov1': None + } + with pytest.raises(SystemExit) as excinfo: + self.cls.console_entry_point() + assert excinfo.value.code == 0 + assert mock_list.mock_calls == [ + call() + ] + out, err = capsys.readouterr() + assert out == 'Available metrics providers:\nProv1\nProv2\n' def test_no_color(self): argv = ['awslimitchecker', '--no-color'] diff --git a/docs/source/awslimitchecker.metrics.base.rst b/docs/source/awslimitchecker.metrics.base.rst new file mode 100644 index 00000000..c80bc5f5 --- /dev/null +++ b/docs/source/awslimitchecker.metrics.base.rst @@ -0,0 +1,7 @@ +awslimitchecker.metrics.base module +=================================== + +.. automodule:: awslimitchecker.metrics.base + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/awslimitchecker.metrics.dummy.rst b/docs/source/awslimitchecker.metrics.dummy.rst new file mode 100644 index 00000000..c25b1c7c --- /dev/null +++ b/docs/source/awslimitchecker.metrics.dummy.rst @@ -0,0 +1,7 @@ +awslimitchecker.metrics.dummy module +==================================== + +.. automodule:: awslimitchecker.metrics.dummy + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/awslimitchecker.metrics.rst b/docs/source/awslimitchecker.metrics.rst new file mode 100644 index 00000000..749e32dc --- /dev/null +++ b/docs/source/awslimitchecker.metrics.rst @@ -0,0 +1,16 @@ +awslimitchecker.metrics package +=============================== + +.. automodule:: awslimitchecker.metrics + :members: + :undoc-members: + :show-inheritance: + +Submodules +---------- + +.. toctree:: + + awslimitchecker.metrics.base + awslimitchecker.metrics.dummy + diff --git a/docs/source/awslimitchecker.rst b/docs/source/awslimitchecker.rst index 1c5e8e87..5489608f 100644 --- a/docs/source/awslimitchecker.rst +++ b/docs/source/awslimitchecker.rst @@ -11,6 +11,7 @@ Subpackages .. toctree:: + awslimitchecker.metrics awslimitchecker.services Submodules diff --git a/docs/source/development.rst b/docs/source/development.rst index 4865022b..630926bd 100644 --- a/docs/source/development.rst +++ b/docs/source/development.rst @@ -204,6 +204,28 @@ are needed to support new limit checks from TA. For further information, see :ref:`Internals / Trusted Advisor `. +.. _development.metrics_providers: + +Adding Metrics Providers +------------------------ + +Metrics providers are subclasses of :py:class:`~.MetricsProvider` that take key/value +configuration items via constructor keyword arguments and implement a +:py:meth:`~.MetricsProvider.flush` method to send all metrics to the configured provider. +It is probably easiest to look at the other existing providers for an example of how to +implement a new one, but there are a few important things to keep in mind: + +* All configuration must be able to be bassed as keyword arguments to the class + constructor (which come from ``--metrics-config=key=value`` CLI arguments). + It is recommended that any secrets/API keys also be able to be set via + environment variables, but the CLI arguments should have precedence. +* All dependency imports must be made inside the constructor, not at the module + level. +* If the provider requires additional dependencies, they should be added as + extras but installed in the Docker image. +* The constructor should do as much validation (i.e. authentication test) as + possible. + .. _development.tests: Unit Testing diff --git a/pytest.ini b/pytest.ini index 1611c60a..438d9b07 100644 --- a/pytest.ini +++ b/pytest.ini @@ -5,5 +5,6 @@ pep8ignore = pep8maxlinelength = 80 flakes-ignore = awslimitchecker/services/__init__.py UnusedImport + awslimitchecker/metrics/__init__.py UnusedImport markers = integration: actual integration tests that connect to AWS APIs From 4ebbff60ab6ef1208a4a2122310af31e720552dd Mon Sep 17 00:00:00 2001 From: Jason Antman Date: Thu, 29 Aug 2019 14:37:54 -0400 Subject: [PATCH 2/5] Issue #418 - WIP implementation of Datadog metrics provider --- awslimitchecker/metrics/__init__.py | 1 + awslimitchecker/metrics/datadog.py | 169 ++++++++++++++++++++++++++++ 2 files changed, 170 insertions(+) create mode 100644 awslimitchecker/metrics/datadog.py diff --git a/awslimitchecker/metrics/__init__.py b/awslimitchecker/metrics/__init__.py index 6fda0bd2..bd29239d 100644 --- a/awslimitchecker/metrics/__init__.py +++ b/awslimitchecker/metrics/__init__.py @@ -39,3 +39,4 @@ from .base import MetricsProvider from .dummy import Dummy +from .datadog import Datadog diff --git a/awslimitchecker/metrics/datadog.py b/awslimitchecker/metrics/datadog.py new file mode 100644 index 00000000..64c2c78f --- /dev/null +++ b/awslimitchecker/metrics/datadog.py @@ -0,0 +1,169 @@ +""" +awslimitchecker/metrics/datadog.py + +The latest version of this package is available at: + + +################################################################################ +Copyright 2015-2018 Jason Antman + + This file is part of awslimitchecker, also known as awslimitchecker. + + awslimitchecker is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + awslimitchecker is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with awslimitchecker. If not, see . + +The Copyright and Authors attributions contained herein may not be removed or +otherwise altered, except to add the Author attribution of a contributor to +this work. (Additional Terms pursuant to Section 7b of the AGPL v3) +################################################################################ +While not legally required, I sincerely request that anyone who finds +bugs please submit them at or +to me via email, and that you send any contributions or improvements +either as a pull request on GitHub, or to me via email. +################################################################################ + +AUTHORS: +Jason Antman +################################################################################ +""" + +import os +import logging +import urllib3 +import time +import re +import json +from awslimitchecker.metrics.base import MetricsProvider + +logger = logging.getLogger(__name__) + + +class Datadog(MetricsProvider): + """Send metrics to Datadog.""" + + def __init__( + self, region_name, prefix='awslimitchecker.', api_key=None, + extra_tags=None + ): + """ + Initialize the Datadog metrics provider. This class does not have any + additional requirements. You must specify at least the ``api_key`` + configuration option. + + :param region_name: the name of the region we're connected to. This + parameter is automatically passed in by the Runner class. + :type region_name: str + :param prefix: Datadog metric prefix + :type prefix: str + :param api_key: Datadog API key. May alternatively be specified by the + ``DATADOG_API_KEY`` environment variable. + :type api_key: str + :param extra_tags: CSV list of additional tags to send with metrics. + All metrics will automatically be tagged with ``region:`` + :type extra_tags: str + """ + super(Datadog, self).__init__(region_name) + self._prefix = prefix + self._tags = ['region:%s' % region_name] + if extra_tags is not None: + self._tags.extend(','.split(extra_tags)) + self._api_key = os.environ.get('DATADOG_API_KEY') + if api_key is not None: + self._api_key = api_key + if self._api_key is None: + raise RuntimeError( + 'ERROR: Datadog metrics provider requires datadog API key.' + ) + self._http = urllib3.PoolManager() + self._validate_auth(self._api_key) + + def _validate_auth(self, api_key): + url = 'https://api.datadoghq.com/api/v1/validate?api_key=%s' + logger.debug('Validating Datadog API key: GET %s', url) + url = url % api_key + r = self._http.request('GET', url) + if r.status != 200: + raise RuntimeError( + 'ERROR: Datadog API key validation failed with HTTP %s: %s' % ( + r.status, r.data + ) + ) + + def _name_for_metric(self, service, limit): + """ + Return a metric name that's safe for datadog + + :param service: service name + :type service: str + :param limit: limit name + :type limit: str + :return: datadog metric name + :rtype: str + """ + return ('%s%s.%s' % ( + self._prefix, + re.sub(r'[^0-9a-zA-Z]+', '_', service), + re.sub(r'[^0-9a-zA-Z]+', '_', limit) + )).lower() + + def flush(self): + ts = int(time.time()) + logger.debug('Flushing metrics to Datadog.') + series = [{ + 'metric': '%sruntime' % self._prefix, + 'points': [[ts, self._duration]], + 'type': 'gauge', + 'tags': self._tags + }] + for lim in self._limits: + u = lim.get_current_usage() + if len(u) == 0: + max_usage = 0 + else: + max_usage = max(u).get_value() + mname = self._name_for_metric(lim.service.service_name, lim.name) + series.append({ + 'metric': '%s.max_usage' % mname, + 'points': [[ts, max_usage]], + 'type': 'gauge', + 'tags': self._tags + }) + limit = lim.get_limit() + if limit is not None: + series.append({ + 'metric': '%s.limit' % mname, + 'points': [[ts, limit]], + 'type': 'gauge', + 'tags': self._tags + }) + logger.info('POSTing %d metrics to datadog', len(series)) + data = {'series': series} + encoded = json.dumps(data).encode('utf-8') + url = 'https://api.datadoghq.com/api/v1/series' \ + '?api_key=%s' % self._api_key + resp = self._http.request( + 'POST', url, + headers={'Content-type': 'application/json'}, + body=encoded + ) + if resp.status < 300: + logger.debug( + 'Successfully POSTed to Datadog; HTTP %d: %s', + resp.status, resp.data + ) + return + raise RuntimeError( + 'ERROR sending metrics to Datadog; API responded HTTP %d: %s' % ( + resp.status, resp.data + ) + ) From 3f17333f3bf1e87af5b4389a70b0d0a8b6a85e83 Mon Sep 17 00:00:00 2001 From: Jason Antman Date: Fri, 30 Aug 2019 08:35:00 -0400 Subject: [PATCH 3/5] Issue #418 - Metrics provider documentation --- README.rst | 1 + docs/build_generated_docs.py | 71 +++++++++++++++++- .../awslimitchecker.metrics.datadog.rst | 7 ++ docs/source/awslimitchecker.metrics.rst | 1 + docs/source/cli_usage.rst | 74 ++++++++++++++++++- docs/source/cli_usage.rst.template | 71 +++++++++--------- docs/source/getting_started.rst | 4 + docs/source/iam_policy.rst | 6 +- 8 files changed, 190 insertions(+), 45 deletions(-) create mode 100644 docs/source/awslimitchecker.metrics.datadog.rst diff --git a/README.rst b/README.rst index 86ec599d..a7d72d76 100644 --- a/README.rst +++ b/README.rst @@ -101,6 +101,7 @@ What It Does an optional maximum time limit). See `Getting Started - Trusted Advisor `_ for more information. +- Optionally send current usage and limit metrics to a metrics store, such as Datadog. Requirements ------------ diff --git a/docs/build_generated_docs.py b/docs/build_generated_docs.py index 2614051b..8888ea49 100644 --- a/docs/build_generated_docs.py +++ b/docs/build_generated_docs.py @@ -51,10 +51,12 @@ sys.path.insert(0, os.path.join(my_dir, '..')) from awslimitchecker.checker import AwsLimitChecker +from awslimitchecker.metrics import MetricsProvider logger = logging.getLogger() logging.basicConfig(level=logging.INFO) + def build_iam_policy(checker): logger.info("Beginning build of iam_policy.rst") # get the policy dict @@ -77,6 +79,13 @@ def build_iam_policy(checker): Required IAM Permissions ======================== + .. important:: + The required IAM policy output by awslimitchecker includes only the permissions + required to check limits and usage. If you are loading + :ref:`limit overrides ` and/or + :ref:`threshold overrides ` from S3, you will + need to run awslimitchecker with additional permissions to access those objects. + Below is the sample IAM policy from this version of awslimitchecker, listing the IAM permissions required for it to function correctly. Please note that in some cases awslimitchecker may cause AWS services to make additional API calls on your behalf @@ -97,6 +106,7 @@ def build_iam_policy(checker): with open(fname, 'w') as fh: fh.write(doc) + def build_limits(checker): logger.info("Beginning build of limits.rst") logger.info("Getting Limits") @@ -202,6 +212,7 @@ def build_limits(checker): with open(fname, 'w') as fh: fh.write(doc) + def build_runner_examples(): logger.info("Beginning build of runner examples") # read in the template file @@ -225,7 +236,8 @@ def build_runner_examples(): 'check_thresholds': ['awslimitchecker', '--no-color'], 'check_thresholds_custom': ['awslimitchecker', '-W', '97', '--critical=98', '--no-color'], - 'iam_policy': ['awslimitchecker', '--iam-policy'] + 'iam_policy': ['awslimitchecker', '--iam-policy'], + 'list_metrics': ['awslimitchecker', '--list-metrics-providers'], } results = {} # run the commands @@ -238,6 +250,58 @@ def build_runner_examples(): output = e.output results[name] = format_cmd_output(cmd_str, output, name) results['%s-output-only' % name] = format_cmd_output(None, output, name) + results['metrics-providers'] = '' + for m in MetricsProvider.providers_by_name().keys(): + results['metrics-providers'] += '* :py:class:`~.%s`\n' % m + results['limit-override-json'] = dedent(""" + { + "AutoScaling": { + "Auto Scaling groups": 321, + "Launch configurations": 456 + } + } + """) + results['threshold-override-json'] = dedent(""" + { + "S3": { + "Buckets": { + "warning": { + "percent": 97 + }, + "critical": { + "percent": 99 + } + } + }, + "EC2": { + "Security groups per VPC": { + "warning": { + "percent": 80, + "count": 800 + }, + "critical": { + "percent": 90, + "count": 900 + } + }, + "VPC security groups per elastic network interface": { + "warning": { + "percent": 101 + }, + "critical": { + "percent": 101 + } + } + } + } + """) + for x in ['limit-override-json', 'threshold-override-json']: + tmp = '' + for line in results[x].split('\n'): + if line.strip() == '': + continue + tmp += ' %s\n' % line + results[x] = tmp tmpl = tmpl.format(**results) # write out the final .rst @@ -245,6 +309,7 @@ def build_runner_examples(): fh.write(tmpl) logger.critical("WARNING - some output may need to be fixed to provide good examples") + def format_cmd_output(cmd, output, name): """format command output for docs""" if cmd is None: @@ -259,7 +324,7 @@ def format_cmd_output(cmd, output, name): lines[idx] = line[:100] + ' (...)' if len(lines) > 12: tmp_lines = lines[:5] + ['(...)'] + lines[-5:] - if ' -l' in cmd or ' --list-defaults' in cmd: + if cmd is not None and (' -l' in cmd or ' --list-defaults' in cmd): # find a line that uses a limit from the API, # and a line with None (unlimited) api_line = None @@ -291,6 +356,7 @@ def format_cmd_output(cmd, output, name): formatted += '\n' return formatted + def build_docs(): """ Trigger rebuild of all documentation that is dynamically generated @@ -309,5 +375,6 @@ def build_docs(): build_limits(c) build_runner_examples() + if __name__ == "__main__": build_docs() diff --git a/docs/source/awslimitchecker.metrics.datadog.rst b/docs/source/awslimitchecker.metrics.datadog.rst new file mode 100644 index 00000000..7977c7c2 --- /dev/null +++ b/docs/source/awslimitchecker.metrics.datadog.rst @@ -0,0 +1,7 @@ +awslimitchecker.metrics.datadog module +====================================== + +.. automodule:: awslimitchecker.metrics.datadog + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/awslimitchecker.metrics.rst b/docs/source/awslimitchecker.metrics.rst index 749e32dc..92f58ed7 100644 --- a/docs/source/awslimitchecker.metrics.rst +++ b/docs/source/awslimitchecker.metrics.rst @@ -12,5 +12,6 @@ Submodules .. toctree:: awslimitchecker.metrics.base + awslimitchecker.metrics.datadog awslimitchecker.metrics.dummy diff --git a/docs/source/cli_usage.rst b/docs/source/cli_usage.rst index 5d65c3bc..9e70f6a0 100644 --- a/docs/source/cli_usage.rst +++ b/docs/source/cli_usage.rst @@ -26,8 +26,10 @@ use as a Nagios-compatible plugin). (venv)$ awslimitchecker --help usage: awslimitchecker [-h] [-S [SERVICE [SERVICE ...]]] [--skip-service SKIP_SERVICE] [--skip-check SKIP_CHECK] - [-s] [-l] [--list-defaults] [-L LIMIT] [-u] - [--iam-policy] [-W WARNING_THRESHOLD] + [-s] [-l] [--list-defaults] [-L LIMIT] + [--limit-override-json LIMIT_OVERRIDE_JSON] + [--threshold-override-json THRESHOLD_OVERRIDE_JSON] + [-u] [--iam-policy] [-W WARNING_THRESHOLD] [-C CRITICAL_THRESHOLD] [-P PROFILE_NAME] [-A STS_ACCOUNT_ID] [-R STS_ACCOUNT_ROLE] [-E EXTERNAL_ID] [-M MFA_SERIAL_NUMBER] [-T MFA_TOKEN] @@ -35,6 +37,9 @@ use as a Nagios-compatible plugin). [--ta-refresh-wait | --ta-refresh-trigger | --ta-refresh-older TA_REFRESH_OLDER] [--ta-refresh-timeout TA_REFRESH_TIMEOUT] [--no-color] [--no-check-version] [-v] [-V] + [--list-metrics-providers] + [--metrics-provider METRICS_PROVIDER] + [--metrics-config METRICS_CONFIG] Report on AWS service limits and usage via boto3, optionally warn about any services with usage nearing or exceeding their limits. For further help, see @@ -58,6 +63,14 @@ use as a Nagios-compatible plugin). override a single AWS limit, specified in "service_name/limit_name=value" format; can be specified multiple times. + --limit-override-json LIMIT_OVERRIDE_JSON + Absolute or relative path, or s3:// URL, to a JSON + file specifying limit overrides. See docs for expected + format. + --threshold-override-json THRESHOLD_OVERRIDE_JSON + Absolute or relative path, or s3:// URL, to a JSON + file specifying threshold overrides. See docs for + expected format. -u, --show-usage find and print the current usage of all AWS services with known limits --iam-policy output a JSON serialized IAM Policy listing the @@ -108,6 +121,14 @@ use as a Nagios-compatible plugin). --no-check-version do not check latest version at startup -v, --verbose verbose output. specify twice for debug-level output. -V, --version print version number and exit. + --list-metrics-providers + List available metrics providers and exit + --metrics-provider METRICS_PROVIDER + Metrics provider class name, to enable sending metrics + --metrics-config METRICS_CONFIG + Specify key/value parameters for the metrics provider + constructor. See documentation for further + information. awslimitchecker is AGPLv3-licensed Free Software. Anyone using this program, even remotely over a network, is entitled to a copy of the source code. Use `--version` for information on the source code location. @@ -445,6 +466,7 @@ You can also set custom thresholds on a per-limit basis using the } } + Using a command like: .. code-block:: console @@ -460,6 +482,48 @@ Using a command like: VPC/NAT Gateways per AZ (limit 5) CRITICAL: us-east-1d=7, us-east-1c= (...) VPC/Virtual private gateways (limit 5) CRITICAL: 5 + + +.. _cli_usage.metrics: + +Enable Metrics Provider ++++++++++++++++++++++++ + +awslimitchecker is capable of sending metrics for the overall runtime of checking +thresholds, as well as the current limit values and current usage, to various metrics +stores. The list of metrics providers supported by your version of awslimitchecker +can be seen with the ``--list-metrics-providers`` option: + +.. code-block:: console + + (venv)$ awslimitchecker --list-metrics-providers + Available metrics providers: + Datadog + Dummy + + + +The configuration options required by each metrics provider are specified in the +providers' documentation: + +* :py:class:`~.Dummy` +* :py:class:`~.Datadog` + + +For example, to use the :py:class:`~awslimitchecker.metrics.datadog.Datadog` +metrics provider which requires an ``api_key`` paramater (also accepted as an +environment variable) and an optional ``extra_tags`` parameter: + +.. code-block:: console + + (venv)$ awslimitchecker \ + --metrics-provider=Datadog \ + --metrics-config=api_key=123456 \ + --metrics-config=extra_tags=foo,bar,baz:blam + +Metrics will be pushed to the provider only when awslimitchecker is done checking +all limits. + Required IAM Policy +++++++++++++++++++ @@ -474,13 +538,15 @@ permissions for it to perform all limit checks. This can be viewed with the "Statement": [ { "Action": [ - "apigateway:GET", + "apigateway:GET", (...) } - ], + ], "Version": "2012-10-17" } + + For the current IAM Policy required by this version of awslimitchecker, see :ref:`IAM Policy `. diff --git a/docs/source/cli_usage.rst.template b/docs/source/cli_usage.rst.template index 40335d1b..1060daf2 100644 --- a/docs/source/cli_usage.rst.template +++ b/docs/source/cli_usage.rst.template @@ -144,12 +144,7 @@ You could also set the same limit overrides using a JSON file stored at ``limit_ .. code-block:: json - { - "AutoScaling": { - "Auto Scaling groups": 321, - "Launch configurations": 456 - } - } +{limit-override-json} Using a command like: @@ -199,38 +194,7 @@ You can also set custom thresholds on a per-limit basis using the .. code-block:: json - { - "S3": { - "Buckets": { - "warning": { - "percent": 97 - }, - "critical": { - "percent": 99 - } - } - }, - "EC2": { - "Security groups per VPC": { - "warning": { - "percent": 80, - "count": 800 - }, - "critical": { - "percent": 90, - "count": 900 - } - }, - "VPC security groups per elastic network interface": { - "warning": { - "percent": 101 - }, - "critical": { - "percent": 101 - } - } - } - } +{threshold-override-json} Using a command like: @@ -239,6 +203,37 @@ Using a command like: (venv)$ awslimitchecker -W 97 --critical=98 --no-color --threshold-overrides-json=s3://bucketname/path/overrides.json {check_thresholds_custom-output-only} +.. _cli_usage.metrics: + +Enable Metrics Provider ++++++++++++++++++++++++ + +awslimitchecker is capable of sending metrics for the overall runtime of checking +thresholds, as well as the current limit values and current usage, to various metrics +stores. The list of metrics providers supported by your version of awslimitchecker +can be seen with the ``--list-metrics-providers`` option: + +{list_metrics} + +The configuration options required by each metrics provider are specified in the +providers' documentation: + +{metrics-providers} + +For example, to use the :py:class:`~awslimitchecker.metrics.datadog.Datadog` +metrics provider which requires an ``api_key`` paramater (also accepted as an +environment variable) and an optional ``extra_tags`` parameter: + +.. code-block:: console + + (venv)$ awslimitchecker \ + --metrics-provider=Datadog \ + --metrics-config=api_key=123456 \ + --metrics-config=extra_tags=foo,bar,baz:blam + +Metrics will be pushed to the provider only when awslimitchecker is done checking +all limits. + Required IAM Policy +++++++++++++++++++ diff --git a/docs/source/getting_started.rst b/docs/source/getting_started.rst index d85746ca..5ef597b4 100644 --- a/docs/source/getting_started.rst +++ b/docs/source/getting_started.rst @@ -23,6 +23,7 @@ What It Does an optional maximum time limit). See :ref:`Getting Started - Trusted Advisor ` for more information. +- Optionally send current usage and limit metrics to a :ref:`metrics store ` such as Datadog. .. _getting_started.nomenclature: @@ -208,6 +209,9 @@ information on the implementation of Trusted Advisor polling. Required Permissions -------------------- +.. important:: + The required IAM policy output by awslimitchecker includes only the permissions required to check limits and usage. If you are loading :ref:`limit overrides ` and/or :ref:`threshold overrides ` from S3, you will need to run awslimitchecker with additional permissions to access those objects. + You can view a sample IAM policy listing the permissions required for awslimitchecker to function properly either via the CLI client: diff --git a/docs/source/iam_policy.rst b/docs/source/iam_policy.rst index f10ef94b..e08749e7 100644 --- a/docs/source/iam_policy.rst +++ b/docs/source/iam_policy.rst @@ -10,7 +10,11 @@ Required IAM Permissions ======================== .. important:: - The required IAM policy output by awslimitchecker includes only the permissions required to check limits and usage. If you are loading :ref:`limit overrides ` and/or :ref:`threshold overrides ` from S3, you will need to run awslimitchecker with additional permissions to access those objects. + The required IAM policy output by awslimitchecker includes only the permissions + required to check limits and usage. If you are loading + :ref:`limit overrides ` and/or + :ref:`threshold overrides ` from S3, you will + need to run awslimitchecker with additional permissions to access those objects. Below is the sample IAM policy from this version of awslimitchecker, listing the IAM permissions required for it to function correctly. Please note that in some cases From 9f9f9052890fd5da2fb6b04fdb3b56137541219b Mon Sep 17 00:00:00 2001 From: Jason Antman Date: Fri, 30 Aug 2019 08:44:27 -0400 Subject: [PATCH 4/5] Fix failing test --- awslimitchecker/tests/metrics/test_base.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/awslimitchecker/tests/metrics/test_base.py b/awslimitchecker/tests/metrics/test_base.py index 15273ed6..81ee12b9 100644 --- a/awslimitchecker/tests/metrics/test_base.py +++ b/awslimitchecker/tests/metrics/test_base.py @@ -38,7 +38,7 @@ """ from awslimitchecker.metrics.base import MetricsProvider -from awslimitchecker.metrics import Dummy +from awslimitchecker.metrics import Dummy, Datadog import pytest @@ -73,7 +73,8 @@ def test_add_limit(self): def test_providers_by_name(self): assert MetricsProvider.providers_by_name() == { 'Dummy': Dummy, - 'MPTester': MPTester + 'MPTester': MPTester, + 'Datadog': Datadog } def test_get_provider_by_name(self): From 3e022f9e35b52833e7ddfbfc49e62e7fe5f3ce74 Mon Sep 17 00:00:00 2001 From: Jason Antman Date: Fri, 30 Aug 2019 09:46:48 -0400 Subject: [PATCH 5/5] Issue #418 - finish unit tests for Datadog metrics provider --- awslimitchecker/metrics/datadog.py | 2 +- awslimitchecker/tests/metrics/test_datadog.py | 305 ++++++++++++++++++ 2 files changed, 306 insertions(+), 1 deletion(-) create mode 100644 awslimitchecker/tests/metrics/test_datadog.py diff --git a/awslimitchecker/metrics/datadog.py b/awslimitchecker/metrics/datadog.py index 64c2c78f..82d1aad0 100644 --- a/awslimitchecker/metrics/datadog.py +++ b/awslimitchecker/metrics/datadog.py @@ -76,7 +76,7 @@ def __init__( self._prefix = prefix self._tags = ['region:%s' % region_name] if extra_tags is not None: - self._tags.extend(','.split(extra_tags)) + self._tags.extend(extra_tags.split(',')) self._api_key = os.environ.get('DATADOG_API_KEY') if api_key is not None: self._api_key = api_key diff --git a/awslimitchecker/tests/metrics/test_datadog.py b/awslimitchecker/tests/metrics/test_datadog.py new file mode 100644 index 00000000..4bf06a97 --- /dev/null +++ b/awslimitchecker/tests/metrics/test_datadog.py @@ -0,0 +1,305 @@ +""" +awslimitchecker/tests/metrics/test_datadog.py + +The latest version of this package is available at: + + +################################################################################ +Copyright 2015-2019 Jason Antman + + This file is part of awslimitchecker, also known as awslimitchecker. + + awslimitchecker is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + awslimitchecker is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with awslimitchecker. If not, see . + +The Copyright and Authors attributions contained herein may not be removed or +otherwise altered, except to add the Author attribution of a contributor to +this work. (Additional Terms pursuant to Section 7b of the AGPL v3) +################################################################################ +While not legally required, I sincerely request that anyone who finds +bugs please submit them at or +to me via email, and that you send any contributions or improvements +either as a pull request on GitHub, or to me via email. +################################################################################ + +AUTHORS: +Jason Antman +################################################################################ +""" + +import sys +import json +from awslimitchecker.metrics import Datadog +import pytest +from freezegun import freeze_time + +if ( + sys.version_info[0] < 3 or + sys.version_info[0] == 3 and sys.version_info[1] < 4 +): + from mock import Mock, patch, call +else: + from unittest.mock import Mock, patch, call + + +pbm = 'awslimitchecker.metrics.datadog' +pb = '%s.Datadog' % pbm + + +class TestInit(object): + + @patch.dict('os.environ', {}, clear=True) + def test_happy_path(self): + mock_http = Mock() + with patch('%s.urllib3.PoolManager' % pbm, autospec=True) as m_pm: + m_pm.return_value = mock_http + with patch('%s._validate_auth' % pb, autospec=True) as m_va: + cls = Datadog( + 'foo', api_key='1234', extra_tags='foo,bar,baz:blam' + ) + assert cls._region_name == 'foo' + assert cls._duration == 0.0 + assert cls._limits == [] + assert cls._api_key == '1234' + assert cls._prefix == 'awslimitchecker.' + assert cls._tags == [ + 'region:foo', 'foo', 'bar', 'baz:blam' + ] + assert cls._http == mock_http + assert m_pm.mock_calls == [call()] + assert m_va.mock_calls == [call(cls, '1234')] + + @patch.dict('os.environ', {'DATADOG_API_KEY': '5678'}, clear=True) + def test_api_key_env_var(self): + mock_http = Mock() + with patch('%s.urllib3.PoolManager' % pbm, autospec=True) as m_pm: + m_pm.return_value = mock_http + with patch('%s._validate_auth' % pb, autospec=True) as m_va: + cls = Datadog( + 'foo', prefix='myprefix.' + ) + assert cls._region_name == 'foo' + assert cls._duration == 0.0 + assert cls._limits == [] + assert cls._api_key == '5678' + assert cls._prefix == 'myprefix.' + assert cls._tags == [ + 'region:foo' + ] + assert cls._http == mock_http + assert m_pm.mock_calls == [call()] + assert m_va.mock_calls == [call(cls, '5678')] + + @patch.dict('os.environ', {}, clear=True) + def test_no_api_key(self): + mock_http = Mock() + with patch('%s.urllib3.PoolManager' % pbm, autospec=True) as m_pm: + m_pm.return_value = mock_http + with patch('%s._validate_auth' % pb, autospec=True) as m_va: + with pytest.raises(RuntimeError) as exc: + Datadog( + 'foo', extra_tags='foo,bar,baz:blam', prefix='myprefix.' + ) + assert str(exc.value) == 'ERROR: Datadog metrics provider ' \ + 'requires datadog API key.' + assert m_pm.mock_calls == [] + assert m_va.mock_calls == [] + + +class DatadogTester(object): + + def setup(self): + with patch('%s.__init__' % pb) as m_init: + m_init.return_value = None + self.cls = Datadog() + + +class TestValidateAuth(DatadogTester): + + def test_happy_path(self): + mock_http = Mock() + mock_resp = Mock(status=200, data=b'{"success": "ok"}') + mock_http.request.return_value = mock_resp + self.cls._http = mock_http + self.cls._validate_auth('1234') + assert mock_http.mock_calls == [ + call.request( + 'GET', + 'https://api.datadoghq.com/api/v1/validate?api_key=1234' + ) + ] + + def test_failure(self): + mock_http = Mock() + mock_resp = Mock(status=401, data='{"success": "NO"}') + mock_http.request.return_value = mock_resp + self.cls._http = mock_http + with pytest.raises(RuntimeError) as exc: + self.cls._validate_auth('1234') + assert str(exc.value) == 'ERROR: Datadog API key validation failed ' \ + 'with HTTP 401: {"success": "NO"}' + assert mock_http.mock_calls == [ + call.request( + 'GET', + 'https://api.datadoghq.com/api/v1/validate?api_key=1234' + ) + ] + + +class TestNameForMetric(DatadogTester): + + def test_simple(self): + self.cls._prefix = 'foobar.' + assert self.cls._name_for_metric( + 'Service Name*', 'limit NAME .' + ) == 'foobar.service_name_.limit_name_' + + +class TestFlush(DatadogTester): + + @freeze_time("2016-12-16 10:40:42", tz_offset=0, auto_tick_seconds=6) + def test_happy_path(self): + self.cls._prefix = 'prefix.' + self.cls._tags = ['tag1', 'tag:2'] + self.cls._limits = [] + self.cls._api_key = 'myKey' + self.cls.set_run_duration(123.45) + limA = Mock( + name='limitA', service=Mock(service_name='SVC1') + ) + type(limA).name = 'limitA' + limA.get_current_usage.return_value = [] + limA.get_limit.return_value = None + self.cls.add_limit(limA) + limB = Mock( + name='limitB', service=Mock(service_name='SVC1') + ) + type(limB).name = 'limitB' + mocku = Mock() + mocku.get_value.return_value = 6 + limB.get_current_usage.return_value = [mocku] + limB.get_limit.return_value = 10 + self.cls.add_limit(limB) + mock_http = Mock() + mock_resp = Mock(status=200, data='{"status": "ok"}') + mock_http.request.return_value = mock_resp + self.cls._http = mock_http + self.cls.flush() + ts = 1481884842 + expected = { + 'series': [ + { + 'metric': 'prefix.runtime', + 'points': [[ts, 123.45]], + 'type': 'gauge', + 'tags': ['tag1', 'tag:2'] + }, + { + 'metric': 'prefix.svc1.limita.max_usage', + 'points': [[ts, 0]], + 'type': 'gauge', + 'tags': ['tag1', 'tag:2'] + }, + { + 'metric': 'prefix.svc1.limitb.max_usage', + 'points': [[ts, 6]], + 'type': 'gauge', + 'tags': ['tag1', 'tag:2'] + }, + { + 'metric': 'prefix.svc1.limitb.limit', + 'points': [[ts, 10]], + 'type': 'gauge', + 'tags': ['tag1', 'tag:2'] + } + ] + } + assert len(mock_http.mock_calls) == 1 + c = mock_http.mock_calls[0] + assert c[0] == 'request' + assert c[1] == ( + 'POST', 'https://api.datadoghq.com/api/v1/series?api_key=myKey' + ) + assert len(c[2]) == 2 + assert c[2]['headers'] == {'Content-type': 'application/json'} + assert json.loads(c[2]['body'].decode()) == expected + + @freeze_time("2016-12-16 10:40:42", tz_offset=0, auto_tick_seconds=6) + def test_api_error(self): + self.cls._prefix = 'prefix.' + self.cls._tags = ['tag1', 'tag:2'] + self.cls._limits = [] + self.cls._api_key = 'myKey' + self.cls.set_run_duration(123.45) + limA = Mock( + name='limitA', service=Mock(service_name='SVC1') + ) + type(limA).name = 'limitA' + limA.get_current_usage.return_value = [] + limA.get_limit.return_value = None + self.cls.add_limit(limA) + limB = Mock( + name='limitB', service=Mock(service_name='SVC1') + ) + type(limB).name = 'limitB' + mocku = Mock() + mocku.get_value.return_value = 6 + limB.get_current_usage.return_value = [mocku] + limB.get_limit.return_value = 10 + self.cls.add_limit(limB) + mock_http = Mock() + mock_resp = Mock(status=503, data='{"status": "NG"}') + mock_http.request.return_value = mock_resp + self.cls._http = mock_http + with pytest.raises(RuntimeError) as exc: + self.cls.flush() + assert str(exc.value) == 'ERROR sending metrics to Datadog; API ' \ + 'responded HTTP 503: {"status": "NG"}' + ts = 1481884842 + expected = { + 'series': [ + { + 'metric': 'prefix.runtime', + 'points': [[ts, 123.45]], + 'type': 'gauge', + 'tags': ['tag1', 'tag:2'] + }, + { + 'metric': 'prefix.svc1.limita.max_usage', + 'points': [[ts, 0]], + 'type': 'gauge', + 'tags': ['tag1', 'tag:2'] + }, + { + 'metric': 'prefix.svc1.limitb.max_usage', + 'points': [[ts, 6]], + 'type': 'gauge', + 'tags': ['tag1', 'tag:2'] + }, + { + 'metric': 'prefix.svc1.limitb.limit', + 'points': [[ts, 10]], + 'type': 'gauge', + 'tags': ['tag1', 'tag:2'] + } + ] + } + assert len(mock_http.mock_calls) == 1 + c = mock_http.mock_calls[0] + assert c[0] == 'request' + assert c[1] == ( + 'POST', 'https://api.datadoghq.com/api/v1/series?api_key=myKey' + ) + assert len(c[2]) == 2 + assert c[2]['headers'] == {'Content-type': 'application/json'} + assert json.loads(c[2]['body'].decode()) == expected