From 65c7f5c29e4491cc75a95b04821cdc274b7e5134 Mon Sep 17 00:00:00 2001 From: Andrew Grumet Date: Wed, 22 Jun 2016 22:09:42 -0700 Subject: [PATCH 1/4] Added script that manages a set of rolling backups. --- compute/backup/manage-backups.py | 152 +++++++++++++++++++++++++++++++ 1 file changed, 152 insertions(+) create mode 100755 compute/backup/manage-backups.py diff --git a/compute/backup/manage-backups.py b/compute/backup/manage-backups.py new file mode 100755 index 000000000000..76b95b06343a --- /dev/null +++ b/compute/backup/manage-backups.py @@ -0,0 +1,152 @@ +#!/usr/bin/python + +# pip install --upgrade google-api-python-client +# pip install --upgrade iso8601 +# pip install --upgrade rfc3339 +# pip install --upgrade pytz + +import simplejson as json +import iso8601 +import datetime +import pytz +import sys +from oauth2client.client import GoogleCredentials +from googleapiclient import discovery +import re +import argparse + +DISK_ZONE_MAP = {} + +def list_snapshots(compute, project, filter=None, pageToken=None): + result = compute.snapshots().list(project=project, pageToken=pageToken, filter=filter).execute() + return result + +# Given a list of snapshot items, return True if a snapshot should be +# taken, False if not. +def should_snapshot(items, minimum_delta): + _items = items[:] + _items.sort(key=lambda x: x['creationTimestamp']) + _items.reverse() + if datetime.datetime.now(pytz.utc) > iso8601.parse_date(_items[0]['creationTimestamp']) \ + + minimum_delta: + return True + return False + + +# Given a list of snapshot items, return the snapshots than can be +# deleted. +def deletable_items(items): + _items = items[:] + _items.sort(key=lambda x: x['creationTimestamp']) + _items.reverse() + + result = [] + now = datetime.datetime.now(pytz.utc) + one_week = datetime.timedelta(days=7) + three_months = datetime.timedelta(weeks=13) + one_year = datetime.timedelta(weeks=52) + minimum_number = 1 + + # Strategy: look for a reason not to delete. If none found, + # add to list. + + # Global reasons + + if len(items) < minimum_number: + print "Fewer than %d snapshots, not deleting any" % minimum_number + return result + + # Item-specific reasons + + for item in _items[1:]: #always skip newest snapshot + + item_timestamp = iso8601.parse_date(item['creationTimestamp']) + + if now - item_timestamp < one_week: + print "Snapshot '%s' too new, not deleting." % item['name'] + continue + + if item_timestamp.weekday() == 1 and now - item_timestamp < three_months: + print "Snapshot '%s' is weekly timestamp and too new, not deleting." % item['name'] + continue + + if item_timestamp.day == 1 and now - item_timestamp < one_year: + print "Snapshot '%s' is monthly timestamp and too new, not deleting." % item['name'] + continue + + print "Adding snapshot '%s' to the delete list" % item['name'] + result.append(item) + + return result + +def create_snapshot(compute,project,disk,dry_run): + + now = datetime.datetime.now(pytz.utc) + name = "%s-%s" % (disk,now.strftime('%Y-%m-%d')) + zone = zone_from_disk(disk) + print "Creating snapshot '%s' in zone '%s'" % (disk,zone) + if not dry_run: + result = compute.disks().createSnapshot(project=project, disk=disk, body={"name":name}, zone=zone).execute() + +def delete_snapshots(compute,project,snapshots,dry_run): + for snapshot in snapshots: + print "Deleting snapshot '%s'" % snapshot['name'] + if not dry_run: + result = compute.snapshots().delete(project=project, snapshot=snapshot['name']).execute() + +def zone_from_disk(disk): + return DISK_ZONE_MAP[disk] + +def update_snapshots(compute,project,disk,dry_run): + + filter = "name eq %s-[0-9]{4}-[0-9]{2}-[0-9]{2}" % disk + result = list_snapshots(compute,project,filter=filter) + + if not result.has_key('items'): + print "Disk '%s' has no snapshots. Possibly it's new or you have a typo." % disk + snapshot_p = True + items_to_delete = [] + else: + snapshot_p = should_snapshot(result['items'],datetime.timedelta(days=1)) + items_to_delete = deletable_items(result['items']) + + if snapshot_p: + create_snapshot(compute,project,disk,dry_run) + + if len(items_to_delete): + delete_snapshots(compute,project,items_to_deelete,dry_run) + +def main(args): + + disks = [] + for diskzone in args.disk: + disk, zone = diskzone.split(',') + DISK_ZONE_MAP[disk] = zone + disks.append(disk) + + credentials = GoogleCredentials.get_application_default() + compute = discovery.build('compute', 'v1', credentials=credentials) + project = args.project + + for disk in disks: + update_snapshots(compute,project,disk,args.dry_run) + +if __name__ == "__main__": + + parser = argparse.ArgumentParser(description='Make and manage disk snapshots.') + parser.add_argument('--dry-run', help="Show what actions would be run, but don't actually run them.", + action="store_true") + parser.add_argument('--project', help="GCE project.", required=True) + parser.add_argument('--disk', help="Disk and zone, comma-separated.", + action="append", required=True) + + args = parser.parse_args() + + for diskzone in args.disk: + try: + diskzone.index(',') + except ValueError: + print "Disk '%s' has no comma. Should be ." % disk + sys.exit(1) + + main(args) From ee74fb9b0a7bb53c97f3f2feb0da5218a9c53400 Mon Sep 17 00:00:00 2001 From: Andrew Grumet Date: Sat, 25 Jun 2016 11:47:41 -0700 Subject: [PATCH 2/4] Applied first round of pull request review changes. --- compute/backup/manage-backups.py | 112 +++++++++++++++++++------------ compute/requirements.txt | 4 ++ 2 files changed, 73 insertions(+), 43 deletions(-) create mode 100644 compute/requirements.txt diff --git a/compute/backup/manage-backups.py b/compute/backup/manage-backups.py index 76b95b06343a..e546aa36ba62 100755 --- a/compute/backup/manage-backups.py +++ b/compute/backup/manage-backups.py @@ -1,41 +1,66 @@ -#!/usr/bin/python +#!/usr/bin/env python + +# Copyright (C) 2016 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Example of using the Compute Engine API to manage a set of rolling +disk snapshots. + +./manage-backups.py --disk , \ + --disk , \ + --project + +""" -# pip install --upgrade google-api-python-client -# pip install --upgrade iso8601 -# pip install --upgrade rfc3339 -# pip install --upgrade pytz - -import simplejson as json -import iso8601 +import argparse import datetime -import pytz +import json +import re import sys -from oauth2client.client import GoogleCredentials + from googleapiclient import discovery -import re -import argparse +import iso8601 +from oauth2client.client import GoogleCredentials +import pytz + DISK_ZONE_MAP = {} def list_snapshots(compute, project, filter=None, pageToken=None): - result = compute.snapshots().list(project=project, pageToken=pageToken, filter=filter).execute() - return result + return compute.snapshots().list(project=project, pageToken=pageToken, filter=filter).execute() -# Given a list of snapshot items, return True if a snapshot should be -# taken, False if not. def should_snapshot(items, minimum_delta): - _items = items[:] - _items.sort(key=lambda x: x['creationTimestamp']) - _items.reverse() - if datetime.datetime.now(pytz.utc) > iso8601.parse_date(_items[0]['creationTimestamp']) \ - + minimum_delta: + """Given a list of snapshot items, return True if a snapshot should be + taken, False if not. + """ + sorted_items = items[:] + sorted_items.sort(key=lambda x: x['creationTimestamp']) + sorted_items.reverse() + + now = datetime.datetime.now(pytz.utc) + created = iso8601.parse_date(sorted_items[0]['creationTimestamp']) + + if now > created + minimum_delta: return True + return False -# Given a list of snapshot items, return the snapshots than can be -# deleted. def deletable_items(items): + """Given a list of snapshot items, return the snapshots than can be + deleted. + """ _items = items[:] _items.sort(key=lambda x: x['creationTimestamp']) _items.reverse() @@ -53,7 +78,7 @@ def deletable_items(items): # Global reasons if len(items) < minimum_number: - print "Fewer than %d snapshots, not deleting any" % minimum_number + print("Fewer than {0} snapshots, not deleting any".format(minimum_number)) return result # Item-specific reasons @@ -63,58 +88,58 @@ def deletable_items(items): item_timestamp = iso8601.parse_date(item['creationTimestamp']) if now - item_timestamp < one_week: - print "Snapshot '%s' too new, not deleting." % item['name'] + print("Snapshot '{0}' too new, not deleting.".format(item['name'])) continue if item_timestamp.weekday() == 1 and now - item_timestamp < three_months: - print "Snapshot '%s' is weekly timestamp and too new, not deleting." % item['name'] + print("Snapshot '{0}' is weekly timestamp and too new, not deleting.".format(item['name'])) continue if item_timestamp.day == 1 and now - item_timestamp < one_year: - print "Snapshot '%s' is monthly timestamp and too new, not deleting." % item['name'] + print("Snapshot '{0}' is monthly timestamp and too new, not deleting.".format(item['name'])) continue - print "Adding snapshot '%s' to the delete list" % item['name'] + print("Adding snapshot '{0}' to the delete list".format(item['name'])) result.append(item) return result -def create_snapshot(compute,project,disk,dry_run): +def create_snapshot(compute, project, disk, dry_run): now = datetime.datetime.now(pytz.utc) - name = "%s-%s" % (disk,now.strftime('%Y-%m-%d')) + name = "{0}-{1}" % (disk, now.strftime('%Y-%m-%d')) zone = zone_from_disk(disk) - print "Creating snapshot '%s' in zone '%s'" % (disk,zone) + print("Creating snapshot '{0}' in zone '{1}'".format(disk, zone)) if not dry_run: result = compute.disks().createSnapshot(project=project, disk=disk, body={"name":name}, zone=zone).execute() -def delete_snapshots(compute,project,snapshots,dry_run): +def delete_snapshots(compute, project, snapshots, dry_run): for snapshot in snapshots: - print "Deleting snapshot '%s'" % snapshot['name'] + print("Deleting snapshot '{0}'".format(snapshot['name'])) if not dry_run: result = compute.snapshots().delete(project=project, snapshot=snapshot['name']).execute() def zone_from_disk(disk): return DISK_ZONE_MAP[disk] -def update_snapshots(compute,project,disk,dry_run): +def update_snapshots(compute, project, disk, dry_run): - filter = "name eq %s-[0-9]{4}-[0-9]{2}-[0-9]{2}" % disk - result = list_snapshots(compute,project,filter=filter) + filter = "name eq {0}-[0-9]{{4}}-[0-9]{{2}}-[0-9]{{2}}".format(disk) + result = list_snapshots(compute, project, filter=filter) if not result.has_key('items'): - print "Disk '%s' has no snapshots. Possibly it's new or you have a typo." % disk + print("Disk '{0}' has no snapshots. Possibly it's new or you have a typo.".format(disk)) snapshot_p = True items_to_delete = [] else: - snapshot_p = should_snapshot(result['items'],datetime.timedelta(days=1)) + snapshot_p = should_snapshot(result['items'], datetime.timedelta(days=1)) items_to_delete = deletable_items(result['items']) if snapshot_p: - create_snapshot(compute,project,disk,dry_run) + create_snapshot(compute, project, disk, dry_run) if len(items_to_delete): - delete_snapshots(compute,project,items_to_deelete,dry_run) + delete_snapshots(compute, project, items_to_delete, dry_run) def main(args): @@ -129,11 +154,12 @@ def main(args): project = args.project for disk in disks: - update_snapshots(compute,project,disk,args.dry_run) + update_snapshots(compute, project, disk, args.dry_run) if __name__ == "__main__": - parser = argparse.ArgumentParser(description='Make and manage disk snapshots.') + parser = argparse.ArgumentParser( + description=__doc__) parser.add_argument('--dry-run', help="Show what actions would be run, but don't actually run them.", action="store_true") parser.add_argument('--project', help="GCE project.", required=True) @@ -146,7 +172,7 @@ def main(args): try: diskzone.index(',') except ValueError: - print "Disk '%s' has no comma. Should be ." % disk + print("Disk '{0}' has no comma. Should be .".format(disk)) sys.exit(1) main(args) diff --git a/compute/requirements.txt b/compute/requirements.txt new file mode 100644 index 000000000000..89a6fd84a537 --- /dev/null +++ b/compute/requirements.txt @@ -0,0 +1,4 @@ +google-api-python-client==1.5.1 +iso8601==0.1.11 +rfc3339==5 +pytz==2014.7 From a575b0d7554281fec936a793792b3500e74024e9 Mon Sep 17 00:00:00 2001 From: Andrew Grumet Date: Thu, 30 Jun 2016 20:58:26 -0700 Subject: [PATCH 3/4] Added next round of changes. --- compute/backup/manage-backups.py | 73 +++++++++++++++++++++----------- 1 file changed, 48 insertions(+), 25 deletions(-) diff --git a/compute/backup/manage-backups.py b/compute/backup/manage-backups.py index e546aa36ba62..5c947cca510e 100755 --- a/compute/backup/manage-backups.py +++ b/compute/backup/manage-backups.py @@ -37,8 +37,11 @@ DISK_ZONE_MAP = {} + def list_snapshots(compute, project, filter=None, pageToken=None): - return compute.snapshots().list(project=project, pageToken=pageToken, filter=filter).execute() + return compute.snapshots().list( + project=project, pageToken=pageToken, filter=filter).execute() + def should_snapshot(items, minimum_delta): """Given a list of snapshot items, return True if a snapshot should be @@ -78,7 +81,8 @@ def deletable_items(items): # Global reasons if len(items) < minimum_number: - print("Fewer than {0} snapshots, not deleting any".format(minimum_number)) + print('Fewer than {0} snapshots, not deleting any'.format( + minimum_number)) return result # Item-specific reasons @@ -86,53 +90,68 @@ def deletable_items(items): for item in _items[1:]: #always skip newest snapshot item_timestamp = iso8601.parse_date(item['creationTimestamp']) + snapshot_age = now - item_timestamp - if now - item_timestamp < one_week: - print("Snapshot '{0}' too new, not deleting.".format(item['name'])) + if snapshot_age < one_week: + print('Snapshot "{0}" too new, not deleting.'.format(item['name'])) continue - if item_timestamp.weekday() == 1 and now - item_timestamp < three_months: - print("Snapshot '{0}' is weekly timestamp and too new, not deleting.".format(item['name'])) + if item_timestamp.weekday() == 1 and snapshot_age < three_months: + message = 'Snapshot "{}" is weekly timestamp and too new,' + message += ' not deleting.' + print(message.format(item['name'])) continue - if item_timestamp.day == 1 and now - item_timestamp < one_year: - print("Snapshot '{0}' is monthly timestamp and too new, not deleting.".format(item['name'])) + if item_timestamp.day == 1 and snapshot_age < one_year: + message = 'Snapshot "{}" is monthly timestamp and too new,' + message += ' not deleting.' + print(message.format(item['name'])) continue - print("Adding snapshot '{0}' to the delete list".format(item['name'])) + print('Adding snapshot "{}" to the delete list'.format(item['name'])) result.append(item) return result -def create_snapshot(compute, project, disk, dry_run): +def create_snapshot(compute, project, disk, dry_run): now = datetime.datetime.now(pytz.utc) - name = "{0}-{1}" % (disk, now.strftime('%Y-%m-%d')) + name = '{}-{}'.format(disk, now.strftime('%Y-%m-%d')) zone = zone_from_disk(disk) - print("Creating snapshot '{0}' in zone '{1}'".format(disk, zone)) + print('Creating snapshot "{}" in zone "{}"'.format(disk, zone)) + if not dry_run: - result = compute.disks().createSnapshot(project=project, disk=disk, body={"name":name}, zone=zone).execute() + result = compute.disks().createSnapshot(project=project, disk=disk, + body={'name':name}, zone=zone).execute() + def delete_snapshots(compute, project, snapshots, dry_run): for snapshot in snapshots: - print("Deleting snapshot '{0}'".format(snapshot['name'])) + print('Deleting snapshot "{0}"'.format(snapshot['name'])) + if not dry_run: - result = compute.snapshots().delete(project=project, snapshot=snapshot['name']).execute() + result = compute.snapshots().delete(project=project, + snapshot=snapshot['name']).execute() + def zone_from_disk(disk): return DISK_ZONE_MAP[disk] + def update_snapshots(compute, project, disk, dry_run): - filter = "name eq {0}-[0-9]{{4}}-[0-9]{{2}}-[0-9]{{2}}".format(disk) + filter = 'name eq {}-[0-9]{{4}}-[0-9]{{2}}-[0-9]{{2}}'.format(disk) result = list_snapshots(compute, project, filter=filter) if not result.has_key('items'): - print("Disk '{0}' has no snapshots. Possibly it's new or you have a typo.".format(disk)) + message = 'Disk "{}" has no snapshots.' + message += ' Possibly it\'s new or you have a typo.' + print(message.format(disk)) snapshot_p = True items_to_delete = [] else: - snapshot_p = should_snapshot(result['items'], datetime.timedelta(days=1)) + snapshot_p = should_snapshot(result['items'], + datetime.timedelta(days=1)) items_to_delete = deletable_items(result['items']) if snapshot_p: @@ -141,6 +160,7 @@ def update_snapshots(compute, project, disk, dry_run): if len(items_to_delete): delete_snapshots(compute, project, items_to_delete, dry_run) + def main(args): disks = [] @@ -156,15 +176,17 @@ def main(args): for disk in disks: update_snapshots(compute, project, disk, args.dry_run) -if __name__ == "__main__": + +if __name__ == '__main__': parser = argparse.ArgumentParser( description=__doc__) - parser.add_argument('--dry-run', help="Show what actions would be run, but don't actually run them.", - action="store_true") - parser.add_argument('--project', help="GCE project.", required=True) - parser.add_argument('--disk', help="Disk and zone, comma-separated.", - action="append", required=True) + parser.add_argument('--dry-run', + help='Show what actions would be run, but don\'t actually run them.', + action='store_true') + parser.add_argument('--project', help='GCE project.', required=True) + parser.add_argument('--disk', help='Disk and zone, comma-separated.', + action='append', required=True) args = parser.parse_args() @@ -172,7 +194,8 @@ def main(args): try: diskzone.index(',') except ValueError: - print("Disk '{0}' has no comma. Should be .".format(disk)) + message = 'Disk "{}" has no comma. Should be .' + print(message.format(disk)) sys.exit(1) main(args) From 1e0ed5f75dddb6aacbdc1870e8e6cb7993d6660c Mon Sep 17 00:00:00 2001 From: Andrew Grumet Date: Thu, 30 Jun 2016 20:59:58 -0700 Subject: [PATCH 4/4] Fixed a few more {0}s --- compute/backup/manage-backups.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/compute/backup/manage-backups.py b/compute/backup/manage-backups.py index 5c947cca510e..2f0b45db18cd 100755 --- a/compute/backup/manage-backups.py +++ b/compute/backup/manage-backups.py @@ -81,7 +81,7 @@ def deletable_items(items): # Global reasons if len(items) < minimum_number: - print('Fewer than {0} snapshots, not deleting any'.format( + print('Fewer than {} snapshots, not deleting any'.format( minimum_number)) return result @@ -93,7 +93,7 @@ def deletable_items(items): snapshot_age = now - item_timestamp if snapshot_age < one_week: - print('Snapshot "{0}" too new, not deleting.'.format(item['name'])) + print('Snapshot "{}" too new, not deleting.'.format(item['name'])) continue if item_timestamp.weekday() == 1 and snapshot_age < three_months: @@ -127,7 +127,7 @@ def create_snapshot(compute, project, disk, dry_run): def delete_snapshots(compute, project, snapshots, dry_run): for snapshot in snapshots: - print('Deleting snapshot "{0}"'.format(snapshot['name'])) + print('Deleting snapshot "{}"'.format(snapshot['name'])) if not dry_run: result = compute.snapshots().delete(project=project,