From 245a090fced5889c36266383ddbe30ad523d3a75 Mon Sep 17 00:00:00 2001 From: Tres Seaver Date: Fri, 1 May 2015 16:29:16 -0400 Subject: [PATCH 1/4] Document 'extra_params'. --- gcloud/storage/bucket.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/gcloud/storage/bucket.py b/gcloud/storage/bucket.py index 6d1a502d5c95..4589d4ee3f62 100644 --- a/gcloud/storage/bucket.py +++ b/gcloud/storage/bucket.py @@ -60,6 +60,9 @@ class _BlobIterator(Iterator): :type bucket: :class:`gcloud.storage.bucket.Bucket` :param bucket: The bucket from which to list blobs. + + :type extra_params: dict or None + :param extra_params: Extra query string parameters for the API call. """ def __init__(self, bucket, extra_params=None): self.bucket = bucket From f97c62c19d2c7b693da1280308d871db0f1a0520 Mon Sep 17 00:00:00 2001 From: Tres Seaver Date: Fri, 1 May 2015 16:30:24 -0400 Subject: [PATCH 2/4] #825: Allow passing explicit connection to '_BlobIterator.__init__'. --- gcloud/storage/bucket.py | 10 ++++++++-- gcloud/storage/test_bucket.py | 11 +++++++++++ 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/gcloud/storage/bucket.py b/gcloud/storage/bucket.py index 4589d4ee3f62..fe533ba34e55 100644 --- a/gcloud/storage/bucket.py +++ b/gcloud/storage/bucket.py @@ -63,12 +63,18 @@ class _BlobIterator(Iterator): :type extra_params: dict or None :param extra_params: Extra query string parameters for the API call. + + :type connection: :class:`gcloud.storage.connection.Connection` + :param connection: The connection to use when sending requests. Defaults + to the bucket's connection """ - def __init__(self, bucket, extra_params=None): + def __init__(self, bucket, extra_params=None, connection=None): + if connection is None: + connection = bucket.connection self.bucket = bucket self.prefixes = () super(_BlobIterator, self).__init__( - connection=bucket.connection, path=bucket.path + '/o', + connection=connection, path=bucket.path + '/o', extra_params=extra_params) def get_items_from_response(self, response): diff --git a/gcloud/storage/test_bucket.py b/gcloud/storage/test_bucket.py index 8274b2e6e16e..6a74fbccd8f2 100644 --- a/gcloud/storage/test_bucket.py +++ b/gcloud/storage/test_bucket.py @@ -37,6 +37,17 @@ def test_ctor(self): self.assertEqual(iterator.next_page_token, None) self.assertEqual(iterator.prefixes, ()) + def test_ctor_w_explicit_connection(self): + connection = _Connection() + bucket = _Bucket(None) + iterator = self._makeOne(bucket, connection=connection) + self.assertTrue(iterator.bucket is bucket) + self.assertTrue(iterator.connection is connection) + self.assertEqual(iterator.path, '%s/o' % bucket.path) + self.assertEqual(iterator.page_number, 0) + self.assertEqual(iterator.next_page_token, None) + self.assertEqual(iterator.prefixes, ()) + def test_get_items_from_response_empty(self): connection = _Connection() bucket = _Bucket(connection) From 5ed6c7e3cec6f97821facf6f2c329f624430e95b Mon Sep 17 00:00:00 2001 From: Tres Seaver Date: Fri, 1 May 2015 16:52:26 -0400 Subject: [PATCH 3/4] #825: Allow passing explicit connection to 'Bucket.{list_blobs,make_public}'. --- gcloud/storage/bucket.py | 30 ++++++++++++++++++++++-------- gcloud/storage/test_bucket.py | 28 +++++++++++++++++++++------- 2 files changed, 43 insertions(+), 15 deletions(-) diff --git a/gcloud/storage/bucket.py b/gcloud/storage/bucket.py index fe533ba34e55..c8afd69f4a8e 100644 --- a/gcloud/storage/bucket.py +++ b/gcloud/storage/bucket.py @@ -266,7 +266,7 @@ def get_blob(self, blob_name, connection=None): def list_blobs(self, max_results=None, page_token=None, prefix=None, delimiter=None, versions=None, - projection='noAcl', fields=None): + projection='noAcl', fields=None, connection=None): """Return an iterator used to find blobs in the bucket. :type max_results: integer or ``NoneType`` @@ -298,6 +298,11 @@ def list_blobs(self, max_results=None, page_token=None, prefix=None, and the language of each blob returned: 'items/contentLanguage,nextPageToken' + :type connection: :class:`gcloud.storage.connection.Connection` or + ``NoneType`` + :param connection: Optional. The connection to use when sending + requests. If not provided, falls back to default. + :rtype: :class:`_BlobIterator`. :returns: An iterator of blobs. """ @@ -320,7 +325,8 @@ def list_blobs(self, max_results=None, page_token=None, prefix=None, if fields is not None: extra_params['fields'] = fields - result = self._iterator_class(self, extra_params=extra_params) + result = self._iterator_class( + self, extra_params=extra_params, connection=connection) # Page token must be handled specially since the base `Iterator` # class has it as a reserved property. if page_token is not None: @@ -853,7 +859,7 @@ def disable_website(self): """ return self.configure_website(None, None) - def make_public(self, recursive=False, future=False): + def make_public(self, recursive=False, future=False, connection=None): """Make a bucket public. :type recursive: boolean @@ -863,18 +869,26 @@ def make_public(self, recursive=False, future=False): :type future: boolean :param future: If True, this will make all objects created in the future public as well. + + :type connection: :class:`gcloud.storage.connection.Connection` or + ``NoneType`` + :param connection: Optional. The connection to use when sending + requests. If not provided, falls back to default. """ + connection = _require_connection(connection) + self.acl.all().grant_read() - self.acl.save() + self.acl.save(connection=connection) if future: doa = self.default_object_acl if not doa.loaded: - doa.reload() + doa.reload(connection=connection) doa.all().grant_read() - doa.save() + doa.save(connection=connection) if recursive: - for blob in self: + for blob in self.list_blobs(projection='full', + connection=connection): blob.acl.all().grant_read() - blob.save_acl() + blob.acl.save(connection=connection) diff --git a/gcloud/storage/test_bucket.py b/gcloud/storage/test_bucket.py index 6a74fbccd8f2..ea46d9d157a0 100644 --- a/gcloud/storage/test_bucket.py +++ b/gcloud/storage/test_bucket.py @@ -337,6 +337,18 @@ def test_list_blobs_explicit(self): self.assertEqual(kw['path'], '/b/%s/o' % NAME) self.assertEqual(kw['query_params'], EXPECTED) + def test_list_blobs_w_explicit_connection(self): + NAME = 'name' + connection = _Connection({'items': []}) + bucket = self._makeOne(NAME, None) + iterator = bucket.list_blobs(connection=connection) + blobs = list(iterator) + self.assertEqual(blobs, []) + kw, = connection._requested + self.assertEqual(kw['method'], 'GET') + self.assertEqual(kw['path'], '/b/%s/o' % NAME) + self.assertEqual(kw['query_params'], {'projection': 'noAcl'}) + def test_delete_default_miss(self): from gcloud.exceptions import NotFound NAME = 'name' @@ -906,7 +918,7 @@ def test_make_public_defaults(self): permissive = [{'entity': 'allUsers', 'role': _ACLEntity.READER_ROLE}] after = {'acl': permissive, 'defaultObjectAcl': []} connection = _Connection(after) - bucket = self._makeOne(NAME, connection) + bucket = self._makeOne(NAME, None) bucket.acl.loaded = True bucket.default_object_acl.loaded = True with _monkey_defaults(connection=connection): @@ -935,7 +947,7 @@ def _make_public_w_future_helper(self, default_object_acl_loaded=True): # We return the same value for default_object_acl.reload() # to consume. connection = _Connection(after1, after1, after2) - bucket = self._makeOne(NAME, connection) + bucket = self._makeOne(NAME, None) bucket.acl.loaded = True bucket.default_object_acl.loaded = default_object_acl_loaded with _monkey_defaults(connection=connection): @@ -980,14 +992,16 @@ def __init__(self, bucket, name): def acl(self): return self + # Faux ACL methods def all(self): return self def grant_read(self): self._granted = True - def save_acl(self): - _saved.append((self._bucket, self._name, self._granted)) + def save(self, connection=None): + _saved.append( + (self._bucket, self._name, self._granted, connection)) class _Iterator(_BlobIterator): def get_items_from_response(self, response): @@ -999,7 +1013,7 @@ def get_items_from_response(self, response): permissive = [{'entity': 'allUsers', 'role': _ACLEntity.READER_ROLE}] after = {'acl': permissive, 'defaultObjectAcl': []} connection = _Connection(after, {'items': [{'name': BLOB_NAME}]}) - bucket = self._makeOne(NAME, connection) + bucket = self._makeOne(NAME, None) bucket.acl.loaded = True bucket.default_object_acl.loaded = True bucket._iterator_class = _Iterator @@ -1007,7 +1021,7 @@ def get_items_from_response(self, response): bucket.make_public(recursive=True) self.assertEqual(list(bucket.acl), permissive) self.assertEqual(list(bucket.default_object_acl), []) - self.assertEqual(_saved, [(bucket, BLOB_NAME, True)]) + self.assertEqual(_saved, [(bucket, BLOB_NAME, True, connection)]) kw = connection._requested self.assertEqual(len(kw), 2) self.assertEqual(kw[0]['method'], 'PATCH') @@ -1016,7 +1030,7 @@ def get_items_from_response(self, response): self.assertEqual(kw[0]['query_params'], {'projection': 'full'}) self.assertEqual(kw[1]['method'], 'GET') self.assertEqual(kw[1]['path'], '/b/%s/o' % NAME) - self.assertEqual(kw[1]['query_params'], {'projection': 'noAcl'}) + self.assertEqual(kw[1]['query_params'], {'projection': 'full'}) class _Connection(object): From 8d6ef054cb911174e42836f9fca8dbf19a00424b Mon Sep 17 00:00:00 2001 From: Tres Seaver Date: Thu, 7 May 2015 12:26:59 -0400 Subject: [PATCH 4/4] Iterator uses implicit connection, rather than 'bucket.connection'. Addresses: https://github.com/GoogleCloudPlatform/gcloud-python/pull/856#discussion_r29804670 --- gcloud/storage/bucket.py | 6 ++-- gcloud/storage/test_bucket.py | 65 +++++++++++++++++++++-------------- 2 files changed, 43 insertions(+), 28 deletions(-) diff --git a/gcloud/storage/bucket.py b/gcloud/storage/bucket.py index c8afd69f4a8e..f75d866df456 100644 --- a/gcloud/storage/bucket.py +++ b/gcloud/storage/bucket.py @@ -69,8 +69,7 @@ class _BlobIterator(Iterator): to the bucket's connection """ def __init__(self, bucket, extra_params=None, connection=None): - if connection is None: - connection = bucket.connection + connection = _require_connection(connection) self.bucket = bucket self.prefixes = () super(_BlobIterator, self).__init__( @@ -363,7 +362,8 @@ def delete(self, force=False, connection=None): connection = _require_connection(connection) if force: blobs = list(self.list_blobs( - max_results=self._MAX_OBJECTS_FOR_BUCKET_DELETE + 1)) + max_results=self._MAX_OBJECTS_FOR_BUCKET_DELETE + 1, + connection=connection)) if len(blobs) > self._MAX_OBJECTS_FOR_BUCKET_DELETE: message = ( 'Refusing to delete bucket with more than ' diff --git a/gcloud/storage/test_bucket.py b/gcloud/storage/test_bucket.py index ea46d9d157a0..cbcded03f973 100644 --- a/gcloud/storage/test_bucket.py +++ b/gcloud/storage/test_bucket.py @@ -26,10 +26,12 @@ def _getTargetClass(self): def _makeOne(self, *args, **kw): return self._getTargetClass()(*args, **kw) - def test_ctor(self): + def test_ctor_w_implicit_connection(self): + from gcloud.storage._testing import _monkey_defaults connection = _Connection() - bucket = _Bucket(connection) - iterator = self._makeOne(bucket) + bucket = _Bucket(None) + with _monkey_defaults(connection=connection): + iterator = self._makeOne(bucket) self.assertTrue(iterator.bucket is bucket) self.assertTrue(iterator.connection is connection) self.assertEqual(iterator.path, '%s/o' % bucket.path) @@ -49,20 +51,25 @@ def test_ctor_w_explicit_connection(self): self.assertEqual(iterator.prefixes, ()) def test_get_items_from_response_empty(self): + from gcloud.storage._testing import _monkey_defaults connection = _Connection() bucket = _Bucket(connection) - iterator = self._makeOne(bucket) - self.assertEqual(list(iterator.get_items_from_response({})), []) + with _monkey_defaults(connection=connection): + iterator = self._makeOne(bucket) + blobs = list(iterator.get_items_from_response({})) + self.assertEqual(blobs, []) self.assertEqual(iterator.prefixes, ()) def test_get_items_from_response_non_empty(self): from gcloud.storage.blob import Blob + from gcloud.storage._testing import _monkey_defaults BLOB_NAME = 'blob-name' response = {'items': [{'name': BLOB_NAME}], 'prefixes': ['foo']} connection = _Connection() bucket = _Bucket(connection) - iterator = self._makeOne(bucket) - blobs = list(iterator.get_items_from_response(response)) + with _monkey_defaults(connection=connection): + iterator = self._makeOne(bucket) + blobs = list(iterator.get_items_from_response(response)) self.assertEqual(len(blobs), 1) blob = blobs[0] self.assertTrue(isinstance(blob, Blob)) @@ -104,10 +111,12 @@ def test_ctor_explicit(self): self.assertTrue(bucket._default_object_acl.bucket is bucket) def test___iter___empty(self): + from gcloud.storage._testing import _monkey_defaults NAME = 'name' connection = _Connection({'items': []}) - bucket = self._makeOne(NAME, connection) - blobs = list(bucket) + bucket = self._makeOne(NAME) + with _monkey_defaults(connection=connection): + blobs = list(bucket) self.assertEqual(blobs, []) kw, = connection._requested self.assertEqual(kw['method'], 'GET') @@ -115,11 +124,13 @@ def test___iter___empty(self): self.assertEqual(kw['query_params'], {'projection': 'noAcl'}) def test___iter___non_empty(self): + from gcloud.storage._testing import _monkey_defaults NAME = 'name' BLOB_NAME = 'blob-name' connection = _Connection({'items': [{'name': BLOB_NAME}]}) - bucket = self._makeOne(NAME, connection) - blobs = list(bucket) + bucket = self._makeOne(NAME) + with _monkey_defaults(connection=connection): + blobs = list(bucket) blob, = blobs self.assertTrue(blob.bucket is bucket) self.assertEqual(blob.name, BLOB_NAME) @@ -290,11 +301,13 @@ def test_get_blob_hit(self): self.assertEqual(kw['path'], '/b/%s/o/%s' % (NAME, BLOB_NAME)) def test_list_blobs_defaults(self): + from gcloud.storage._testing import _monkey_defaults NAME = 'name' connection = _Connection({'items': []}) - bucket = self._makeOne(NAME, connection) - iterator = bucket.list_blobs() - blobs = list(iterator) + bucket = self._makeOne(NAME) + with _monkey_defaults(connection=connection): + iterator = bucket.list_blobs() + blobs = list(iterator) self.assertEqual(blobs, []) kw, = connection._requested self.assertEqual(kw['method'], 'GET') @@ -302,6 +315,7 @@ def test_list_blobs_defaults(self): self.assertEqual(kw['query_params'], {'projection': 'noAcl'}) def test_list_blobs_explicit(self): + from gcloud.storage._testing import _monkey_defaults NAME = 'name' MAX_RESULTS = 10 PAGE_TOKEN = 'ABCD' @@ -320,17 +334,18 @@ def test_list_blobs_explicit(self): 'fields': FIELDS, } connection = _Connection({'items': []}) - bucket = self._makeOne(NAME, connection) - iterator = bucket.list_blobs( - max_results=MAX_RESULTS, - page_token=PAGE_TOKEN, - prefix=PREFIX, - delimiter=DELIMITER, - versions=VERSIONS, - projection=PROJECTION, - fields=FIELDS, - ) - blobs = list(iterator) + bucket = self._makeOne(NAME) + with _monkey_defaults(connection=connection): + iterator = bucket.list_blobs( + max_results=MAX_RESULTS, + page_token=PAGE_TOKEN, + prefix=PREFIX, + delimiter=DELIMITER, + versions=VERSIONS, + projection=PROJECTION, + fields=FIELDS, + ) + blobs = list(iterator) self.assertEqual(blobs, []) kw, = connection._requested self.assertEqual(kw['method'], 'GET')