Merge pull request #17 from rackerlabs/skip_ids

Add support for ignoring certain volumes

DESIGN.md

@@ -27,6 +27,9 @@ Note: All instances will be filtered by whether they are running or stopped, usi
     - Frequency of snapshots (F hours, days, weeks, minimum is 1 hour) *or* a
     crontab expression [as described here](https://github.com/josiahcarlson/parse-crontab#description)
 
+  - Ignore section
+    - An array of instance or volume ids to ignore when doing snapshots or cleanups
+
 [1] http://boto3.readthedocs.io/en/latest/reference/services/ec2.html#EC2.Client.describe_instances
 
 Example of a JSON document from the DynamoDB table's `configuration` field (see [cloudformation template](cloudformation.json)):

README.md

@@ -85,7 +85,8 @@ lambda-uploader --no-upload -r requirements.txt -x ebs_snapper/lambdas.py .
     "retention": "4 days",
     "minimum": 5,
     "frequency": "12 hours"
-  }
+  },
+  "ignore": []
 }
 ```
 1. The CLI has a nice method for interacting with these configuration stanzas, but you must still provide them as JSON.

ebs_snapper/clean.py

@@ -68,6 +68,9 @@ def clean_snapshot(context, region, default_min_snaps=5, installed_region='us-ea
     configurations = dynamo.list_configurations(context, installed_region)
     LOG.debug('Fetched all possible configuration rules from DynamoDB')
 
+    # build a list of any IDs (anywhere) that we should ignore
+    ignore_ids = utils.build_ignore_list(configurations)
+
     # setup some lookup tables
     cache_data = utils.build_cache_maps(context, configurations, region, installed_region)
     instance_configs = cache_data['instance_id_to_config']
@@ -114,6 +117,9 @@ def clean_snapshot(context, region, default_min_snaps=5, installed_region='us-ea
             snapshot_volume = snap['VolumeId']
             minimum_snaps = default_min_snaps
 
+            if snapshot_volume in ignore_ids:
+                continue
+
             # attempt to identify the instance this applies to, so we can check minimums
             try:
                 # given volume id, get the instance for it

ebs_snapper/snapshot.py

@@ -73,6 +73,9 @@ def perform_snapshot(context, region, installed_region='us-east-1'):
     configurations = dynamo.list_configurations(context, installed_region)
     LOG.debug('Fetched all possible configuration rules from DynamoDB')
 
+    # build a list of any IDs (anywhere) that we should ignore
+    ignore_ids = utils.build_ignore_list(configurations)
+
     # setup some lookup tables
     cache_data = utils.build_cache_maps(context, configurations, region, installed_region)
     all_instances = cache_data['instance_id_to_data']
@@ -84,6 +87,9 @@ def perform_snapshot(context, region, installed_region='us-east-1'):
         if timeout_check(context, 'perform_snapshot'):
             break
 
+        if instance_id in ignore_ids:
+            continue
+
         snapshot_settings = instance_configs[instance_id]
 
         # parse out snapshot settings
@@ -100,9 +106,14 @@ def perform_snapshot(context, region, installed_region='us-east-1'):
             if timeout_check(context, 'perform_snapshot'):
                 break
 
+            # we probably should have been using volume keys from one of the
+            # caches here, but since we're not, we're going to have to check here too
             LOG.debug('Considering device %s', dev)
             volume_id = dev['Ebs']['VolumeId']
 
+            if volume_id in ignore_ids:
+                continue
+
             # find snapshots
             recent = volume_snap_recent.get(volume_id)
             now = datetime.datetime.now(dateutil.tz.tzutc())

ebs_snapper/utils.py

@@ -32,8 +32,8 @@
 from time import sleep
 import dateutil
 import boto3
-from pytimeparse.timeparse import timeparse
 from crontab import CronTab
+from pytimeparse.timeparse import timeparse
 import ebs_snapper
 
 LOG = logging.getLogger()
@@ -113,6 +113,20 @@ def get_owner_id(context, region=None):
     return list(set(owners))
 
 
+def build_ignore_list(configurations):
+    """Given a bunch of configs, build a list of ids to ignore"""
+    ignore_ids = []
+    for config in configurations:
+        # if it's missing the match section, ignore it
+        if not validate_snapshot_settings(config):
+            continue
+
+        ignored = config.get('ignore', [])
+        ignore_ids.extend(ignored)
+
+    return ignore_ids
+
+
 def get_regions(must_contain_instances=False):
     """Get regions, optionally filtering by regions containing instances."""
     LOG.debug('get_regions(must_contain_instances=%s)', must_contain_instances)
@@ -510,6 +524,10 @@ def build_cache_maps(context, configurations, region, installed_region):
     # populate them
     LOG.info("Retrieved %s DynamoDB configurations for caching",
              str(len(configurations)))
+
+    # build a list of any IDs (anywhere) that we should ignore
+    ignore_ids = build_ignore_list(configurations)
+
     for config in configurations:
         # stop if we're running out of time
         if ebs_snapper.timeout_check(context, 'build_cache_maps'):
@@ -542,10 +560,19 @@ def build_cache_maps(context, configurations, region, installed_region):
             for instance_data in inst_list:
                 instance_id = instance_data['InstanceId']
 
+                # skip if we're ignoring this
+                if instance_id in ignore_ids:
+                    continue
+
                 cache_data['instance_id_to_config'][instance_id] = config
                 cache_data['instance_id_to_data'][instance_id] = instance_data
                 for dev in instance_data.get('BlockDeviceMappings', []):
                     vid = dev['Ebs']['VolumeId']
+
+                    # skip if we're ignoring this
+                    if vid in ignore_ids:
+                        continue
+
                     cache_data['volume_id_to_instance_id'][vid] = instance_id
 
     LOG.info("Retrieved %s instances for caching",

pylintrc

@@ -59,7 +59,7 @@ confidence=
 # --enable=similarities". If you want to run only the classes checker, but have
 # no Warning level messages displayed, use"--disable=all --enable=classes
 # --disable=W"
-disable=next-method-called,coerce-method,old-division,round-builtin,long-builtin,using-cmp-argument,dict-iter-method,old-raise-syntax,intern-builtin,cmp-builtin,file-builtin,buffer-builtin,dict-view-method,long-suffix,suppressed-message,oct-method,raising-string,hex-method,xrange-builtin,raw_input-builtin,metaclass-assignment,import-star-module-level,useless-suppression,reload-builtin,old-ne-operator,filter-builtin-not-iterating,setslice-method,standarderror-builtin,unpacking-in-except,indexing-exception,parameter-unpacking,execfile-builtin,cmp-method,nonzero-method,backtick,zip-builtin-not-iterating,reduce-builtin,print-statement,range-builtin-not-iterating,map-builtin-not-iterating,delslice-method,unicode-builtin,apply-builtin,basestring-builtin,input-builtin,old-octal-literal,no-absolute-import,unichr-builtin,coerce-builtin,getslice-method,invalid-name,too-many-locals,fixme,unused-argument,locally-disabled,unidiomatic-typecheck,bare-except,too-many-branches,consider-iterating-dictionary,too-many-arguments
+disable=next-method-called,coerce-method,old-division,round-builtin,long-builtin,using-cmp-argument,dict-iter-method,old-raise-syntax,intern-builtin,cmp-builtin,file-builtin,buffer-builtin,dict-view-method,long-suffix,suppressed-message,oct-method,raising-string,hex-method,xrange-builtin,raw_input-builtin,metaclass-assignment,import-star-module-level,useless-suppression,reload-builtin,old-ne-operator,filter-builtin-not-iterating,setslice-method,standarderror-builtin,unpacking-in-except,indexing-exception,parameter-unpacking,execfile-builtin,cmp-method,nonzero-method,backtick,zip-builtin-not-iterating,reduce-builtin,print-statement,range-builtin-not-iterating,map-builtin-not-iterating,delslice-method,unicode-builtin,apply-builtin,basestring-builtin,input-builtin,old-octal-literal,no-absolute-import,unichr-builtin,coerce-builtin,getslice-method,invalid-name,too-many-locals,fixme,unused-argument,locally-disabled,unidiomatic-typecheck,bare-except,too-many-branches,consider-iterating-dictionary,too-many-arguments,too-many-statements
 
 
 [REPORTS]

tests/test_clean.py

@@ -161,3 +161,95 @@ def test_clean_snapshots_tagged_timeout(mocker):
 
     # ensure we DO NOT take a snapshot if our runtime was 5 minutes
     assert not utils.delete_snapshot.called
+
+
+@mock_ec2
+@mock_dynamodb2
+@mock_iam
+@mock_sts
+def test_clean_tagged_snapshots_ignore_instance(mocker):
+    """Test for method of the same name."""
+    # default settings
+    region = 'us-east-1'
+    mocks.create_dynamodb(region)
+
+    # create an instance and record the id
+    instance_id = mocks.create_instances(region, count=1)[0]
+    ctx = utils.MockContext()
+
+    # setup the min # snaps for the instance
+    config_data = {
+        "match": {"instance-id": instance_id},
+        "snapshot": {
+            "retention": "6 days", "minimum": 0, "frequency": "13 hours"
+        },
+        "ignore": [instance_id]
+    }
+
+    # put it in the table, be sure it succeeded
+    dynamo.store_configuration(region, 'foo', '111122223333', config_data)
+
+    # figure out the EBS volume that came with our instance
+    volume_id = utils.get_volumes([instance_id], region)[0]['VolumeId']
+
+    # make a snapshot that should be deleted today too
+    now = datetime.datetime.now(dateutil.tz.tzutc())
+    delete_on = now.strftime('%Y-%m-%d')
+    utils.snapshot_and_tag(instance_id, 'ami-123abc', volume_id, delete_on, region)
+    utils.most_recent_snapshot(volume_id, region)['SnapshotId']
+
+    mocker.patch('ebs_snapper.utils.delete_snapshot')
+    clean.clean_snapshot(ctx, region)
+
+    # ensure we ignored the instance from this volume
+    utils.delete_snapshot.assert_not_called()  # pylint: disable=E1103
+
+
+@mock_ec2
+@mock_dynamodb2
+@mock_iam
+@mock_sts
+def test_clean_tagged_snapshots_ignore_volume(mocker):
+    """Test for method of the same name."""
+    # default settings
+    region = 'us-east-1'
+    mocks.create_dynamodb(region)
+
+    # create an instance and record the id
+    instance_id = mocks.create_instances(region, count=1)[0]
+    ctx = utils.MockContext()
+
+    # setup the min # snaps for the instance
+    config_data = {
+        "match": {"instance-id": instance_id},
+        "snapshot": {
+            "retention": "6 days", "minimum": 0, "frequency": "13 hours"
+        },
+        "ignore": []
+    }
+
+    # put it in the table, be sure it succeeded
+    dynamo.store_configuration(region, 'foo', '111122223333', config_data)
+
+    # figure out the EBS volume that came with our instance
+    volume_id = utils.get_volumes([instance_id], region)[0]['VolumeId']
+    config_data["ignore"].append(volume_id)
+
+    # make a snapshot that should be deleted today too
+    now = datetime.datetime.now(dateutil.tz.tzutc())
+    delete_on = now.strftime('%Y-%m-%d')
+    utils.snapshot_and_tag(instance_id, 'ami-123abc', volume_id, delete_on, region)
+    snapshot_id = utils.most_recent_snapshot(volume_id, region)['SnapshotId']
+
+    mocker.patch('ebs_snapper.utils.delete_snapshot')
+    clean.clean_snapshot(ctx, region)
+
+    # ensure we deleted this snapshot if it was ready to die today
+    utils.delete_snapshot.assert_any_call(snapshot_id, region)  # pylint: disable=E1103
+
+    # now raise the minimum, and check to be sure we didn't delete
+    utils.delete_snapshot.reset_mock()  # pylint: disable=E1103
+    config_data['snapshot']['minimum'] = 5
+    dynamo.store_configuration(region, 'foo', '111122223333', config_data)
+    clean.clean_snapshot(ctx, region)
+    utils.delete_snapshot.assert_not_called()  # pylint: disable=E1103

tests/test_snapshot.py

@@ -235,3 +235,78 @@ def test_should_perform_snapshot():
         'volume-foo',
         recent=datetime.datetime(2016, 7, 24, 02, 35)  # 2016-07-24 at 2:35 UTC
         ) is False
+
+
+@mock_ec2
+@mock_dynamodb2
+@mock_sns
+@mock_iam
+@mock_sts
+def test_perform_snapshot_ignore_instance(mocker):
+    """Test for method of the same name."""
+    # some default settings for this test
+    region = 'us-west-2'
+
+    snapshot_settings = {
+        'snapshot': {'minimum': 5, 'frequency': '2 hours', 'retention': '5 days'},
+        'match': {'tag:backup': 'yes'},
+        'ignore': []
+    }
+
+    # create an instance and record the id
+    instance_id = mocks.create_instances(region, count=1)[0]
+    snapshot_settings['ignore'].append(instance_id)
+
+    # need to filter instances, so need dynamodb present
+    mocks.create_dynamodb('us-east-1')
+    dynamo.store_configuration('us-east-1', 'some_unique_id', '111122223333', snapshot_settings)
+
+    # patch the final method that takes a snapshot
+    mocker.patch('ebs_snapper.utils.snapshot_and_tag')
+
+    # since there are no snapshots, we should expect this to trigger one
+    ctx = utils.MockContext()
+    snapshot.perform_snapshot(ctx, region)
+
+    # test results
+    utils.snapshot_and_tag.assert_not_called()  # pylint: disable=E1103
+
+
+@mock_ec2
+@mock_dynamodb2
+@mock_sns
+@mock_iam
+@mock_sts
+def test_perform_snapshot_ignore_volume(mocker):
+    """Test for method of the same name."""
+    # some default settings for this test
+    region = 'us-west-2'
+
+    snapshot_settings = {
+        'snapshot': {'minimum': 5, 'frequency': '2 hours', 'retention': '5 days'},
+        'match': {'tag:backup': 'yes'},
+        'ignore': []
+    }
+
+    # create an instance and record the id
+    instance_id = mocks.create_instances(region, count=1)[0]
+
+    # need to filter instances, so need dynamodb present
+    mocks.create_dynamodb('us-east-1')
+    dynamo.store_configuration('us-east-1', 'some_unique_id', '111122223333', snapshot_settings)
+
+    # figure out the EBS volume that came with our instance
+    instance_details = utils.get_instance(instance_id, region)
+    block_devices = instance_details.get('BlockDeviceMappings', [])
+    volume_id = block_devices[0]['Ebs']['VolumeId']
+    snapshot_settings['ignore'].append(volume_id)
+
+    # patch the final method that takes a snapshot
+    mocker.patch('ebs_snapper.utils.snapshot_and_tag')
+
+    # since there are no snapshots, we should expect this to trigger one
+    ctx = utils.MockContext()
+    snapshot.perform_snapshot(ctx, region)
+
+    # test results
+    utils.snapshot_and_tag.assert_not_called()  # pylint: disable=E1103