Skip to content

Commit

Permalink
Merge pull request #856 from tseaver/825-storage_bucket_list_blobs_ma…
Browse files Browse the repository at this point in the history 
…ke_public-explicit_connection

#825: Allow passing explicit connection to 'Bucket.{list_blobs,make public}'
  • Loading branch information
@tseaver
tseaver committed May 11, 2015
2 parents a59dc9d + 8d6ef05 commit 290b09e
Show file tree
Hide file tree
Showing 2 changed files with 106 additions and 43 deletions.
45 changes: 34 additions & 11 deletions gcloud/storage/bucket.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,12 +60,20 @@ class _BlobIterator(Iterator):
:type bucket: :class:`gcloud.storage.bucket.Bucket`
:param bucket: The bucket from which to list blobs.
:type extra_params: dict or None
:param extra_params: Extra query string parameters for the API call.
:type connection: :class:`gcloud.storage.connection.Connection`
:param connection: The connection to use when sending requests. Defaults
to the bucket's connection
"""
def __init__(self, bucket, extra_params=None):
def __init__(self, bucket, extra_params=None, connection=None):
connection = _require_connection(connection)
self.bucket = bucket
self.prefixes = ()
super(_BlobIterator, self).__init__(
connection=bucket.connection, path=bucket.path + '/o',
connection=connection, path=bucket.path + '/o',
extra_params=extra_params)

def get_items_from_response(self, response):
Expand Down Expand Up @@ -257,7 +265,7 @@ def get_blob(self, blob_name, connection=None):

def list_blobs(self, max_results=None, page_token=None, prefix=None,
delimiter=None, versions=None,
projection='noAcl', fields=None):
projection='noAcl', fields=None, connection=None):
"""Return an iterator used to find blobs in the bucket.
:type max_results: integer or ``NoneType``
Expand Down Expand Up @@ -289,6 +297,11 @@ def list_blobs(self, max_results=None, page_token=None, prefix=None,
and the language of each blob returned:
'items/contentLanguage,nextPageToken'
:type connection: :class:`gcloud.storage.connection.Connection` or
``NoneType``
:param connection: Optional. The connection to use when sending
requests. If not provided, falls back to default.
:rtype: :class:`_BlobIterator`.
:returns: An iterator of blobs.
"""
Expand All @@ -311,7 +324,8 @@ def list_blobs(self, max_results=None, page_token=None, prefix=None,
if fields is not None:
extra_params['fields'] = fields

result = self._iterator_class(self, extra_params=extra_params)
result = self._iterator_class(
self, extra_params=extra_params, connection=connection)
# Page token must be handled specially since the base `Iterator`
# class has it as a reserved property.
if page_token is not None:
Expand Down Expand Up @@ -348,7 +362,8 @@ def delete(self, force=False, connection=None):
connection = _require_connection(connection)
if force:
blobs = list(self.list_blobs(
max_results=self._MAX_OBJECTS_FOR_BUCKET_DELETE + 1))
max_results=self._MAX_OBJECTS_FOR_BUCKET_DELETE + 1,
connection=connection))
if len(blobs) > self._MAX_OBJECTS_FOR_BUCKET_DELETE:
message = (
'Refusing to delete bucket with more than '
Expand Down Expand Up @@ -844,7 +859,7 @@ def disable_website(self):
"""
return self.configure_website(None, None)

def make_public(self, recursive=False, future=False):
def make_public(self, recursive=False, future=False, connection=None):
"""Make a bucket public.
:type recursive: boolean
Expand All @@ -854,18 +869,26 @@ def make_public(self, recursive=False, future=False):
:type future: boolean
:param future: If True, this will make all objects created in the
future public as well.
:type connection: :class:`gcloud.storage.connection.Connection` or
``NoneType``
:param connection: Optional. The connection to use when sending
requests. If not provided, falls back to default.
"""
connection = _require_connection(connection)

self.acl.all().grant_read()
self.acl.save()
self.acl.save(connection=connection)

if future:
doa = self.default_object_acl
if not doa.loaded:
doa.reload()
doa.reload(connection=connection)
doa.all().grant_read()
doa.save()
doa.save(connection=connection)

if recursive:
for blob in self:
for blob in self.list_blobs(projection='full',
connection=connection):
blob.acl.all().grant_read()
blob.save_acl()
blob.acl.save(connection=connection)
104 changes: 72 additions & 32 deletions gcloud/storage/test_bucket.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,10 +26,23 @@ def _getTargetClass(self):
def _makeOne(self, *args, **kw):
return self._getTargetClass()(*args, **kw)

def test_ctor(self):
def test_ctor_w_implicit_connection(self):
from gcloud.storage._testing import _monkey_defaults
connection = _Connection()
bucket = _Bucket(connection)
iterator = self._makeOne(bucket)
bucket = _Bucket(None)
with _monkey_defaults(connection=connection):
iterator = self._makeOne(bucket)
self.assertTrue(iterator.bucket is bucket)
self.assertTrue(iterator.connection is connection)
self.assertEqual(iterator.path, '%s/o' % bucket.path)
self.assertEqual(iterator.page_number, 0)
self.assertEqual(iterator.next_page_token, None)
self.assertEqual(iterator.prefixes, ())

def test_ctor_w_explicit_connection(self):
connection = _Connection()
bucket = _Bucket(None)
iterator = self._makeOne(bucket, connection=connection)
self.assertTrue(iterator.bucket is bucket)
self.assertTrue(iterator.connection is connection)
self.assertEqual(iterator.path, '%s/o' % bucket.path)
Expand All @@ -38,20 +51,25 @@ def test_ctor(self):
self.assertEqual(iterator.prefixes, ())

def test_get_items_from_response_empty(self):
from gcloud.storage._testing import _monkey_defaults
connection = _Connection()
bucket = _Bucket(connection)
iterator = self._makeOne(bucket)
self.assertEqual(list(iterator.get_items_from_response({})), [])
with _monkey_defaults(connection=connection):
iterator = self._makeOne(bucket)
blobs = list(iterator.get_items_from_response({}))
self.assertEqual(blobs, [])
self.assertEqual(iterator.prefixes, ())

def test_get_items_from_response_non_empty(self):
from gcloud.storage.blob import Blob
from gcloud.storage._testing import _monkey_defaults
BLOB_NAME = 'blob-name'
response = {'items': [{'name': BLOB_NAME}], 'prefixes': ['foo']}
connection = _Connection()
bucket = _Bucket(connection)
iterator = self._makeOne(bucket)
blobs = list(iterator.get_items_from_response(response))
with _monkey_defaults(connection=connection):
iterator = self._makeOne(bucket)
blobs = list(iterator.get_items_from_response(response))
self.assertEqual(len(blobs), 1)
blob = blobs[0]
self.assertTrue(isinstance(blob, Blob))
Expand Down Expand Up @@ -93,22 +111,26 @@ def test_ctor_explicit(self):
self.assertTrue(bucket._default_object_acl.bucket is bucket)

def test___iter___empty(self):
from gcloud.storage._testing import _monkey_defaults
NAME = 'name'
connection = _Connection({'items': []})
bucket = self._makeOne(NAME, connection)
blobs = list(bucket)
bucket = self._makeOne(NAME)
with _monkey_defaults(connection=connection):
blobs = list(bucket)
self.assertEqual(blobs, [])
kw, = connection._requested
self.assertEqual(kw['method'], 'GET')
self.assertEqual(kw['path'], '/b/%s/o' % NAME)
self.assertEqual(kw['query_params'], {'projection': 'noAcl'})

def test___iter___non_empty(self):
from gcloud.storage._testing import _monkey_defaults
NAME = 'name'
BLOB_NAME = 'blob-name'
connection = _Connection({'items': [{'name': BLOB_NAME}]})
bucket = self._makeOne(NAME, connection)
blobs = list(bucket)
bucket = self._makeOne(NAME)
with _monkey_defaults(connection=connection):
blobs = list(bucket)
blob, = blobs
self.assertTrue(blob.bucket is bucket)
self.assertEqual(blob.name, BLOB_NAME)
Expand Down Expand Up @@ -279,18 +301,21 @@ def test_get_blob_hit(self):
self.assertEqual(kw['path'], '/b/%s/o/%s' % (NAME, BLOB_NAME))

def test_list_blobs_defaults(self):
from gcloud.storage._testing import _monkey_defaults
NAME = 'name'
connection = _Connection({'items': []})
bucket = self._makeOne(NAME, connection)
iterator = bucket.list_blobs()
blobs = list(iterator)
bucket = self._makeOne(NAME)
with _monkey_defaults(connection=connection):
iterator = bucket.list_blobs()
blobs = list(iterator)
self.assertEqual(blobs, [])
kw, = connection._requested
self.assertEqual(kw['method'], 'GET')
self.assertEqual(kw['path'], '/b/%s/o' % NAME)
self.assertEqual(kw['query_params'], {'projection': 'noAcl'})

def test_list_blobs_explicit(self):
from gcloud.storage._testing import _monkey_defaults
NAME = 'name'
MAX_RESULTS = 10
PAGE_TOKEN = 'ABCD'
Expand All @@ -309,23 +334,36 @@ def test_list_blobs_explicit(self):
'fields': FIELDS,
}
connection = _Connection({'items': []})
bucket = self._makeOne(NAME, connection)
iterator = bucket.list_blobs(
max_results=MAX_RESULTS,
page_token=PAGE_TOKEN,
prefix=PREFIX,
delimiter=DELIMITER,
versions=VERSIONS,
projection=PROJECTION,
fields=FIELDS,
)
blobs = list(iterator)
bucket = self._makeOne(NAME)
with _monkey_defaults(connection=connection):
iterator = bucket.list_blobs(
max_results=MAX_RESULTS,
page_token=PAGE_TOKEN,
prefix=PREFIX,
delimiter=DELIMITER,
versions=VERSIONS,
projection=PROJECTION,
fields=FIELDS,
)
blobs = list(iterator)
self.assertEqual(blobs, [])
kw, = connection._requested
self.assertEqual(kw['method'], 'GET')
self.assertEqual(kw['path'], '/b/%s/o' % NAME)
self.assertEqual(kw['query_params'], EXPECTED)

def test_list_blobs_w_explicit_connection(self):
NAME = 'name'
connection = _Connection({'items': []})
bucket = self._makeOne(NAME, None)
iterator = bucket.list_blobs(connection=connection)
blobs = list(iterator)
self.assertEqual(blobs, [])
kw, = connection._requested
self.assertEqual(kw['method'], 'GET')
self.assertEqual(kw['path'], '/b/%s/o' % NAME)
self.assertEqual(kw['query_params'], {'projection': 'noAcl'})

def test_delete_default_miss(self):
from gcloud.exceptions import NotFound
NAME = 'name'
Expand Down Expand Up @@ -895,7 +933,7 @@ def test_make_public_defaults(self):
permissive = [{'entity': 'allUsers', 'role': _ACLEntity.READER_ROLE}]
after = {'acl': permissive, 'defaultObjectAcl': []}
connection = _Connection(after)
bucket = self._makeOne(NAME, connection)
bucket = self._makeOne(NAME, None)
bucket.acl.loaded = True
bucket.default_object_acl.loaded = True
with _monkey_defaults(connection=connection):
Expand Down Expand Up @@ -924,7 +962,7 @@ def _make_public_w_future_helper(self, default_object_acl_loaded=True):
# We return the same value for default_object_acl.reload()
# to consume.
connection = _Connection(after1, after1, after2)
bucket = self._makeOne(NAME, connection)
bucket = self._makeOne(NAME, None)
bucket.acl.loaded = True
bucket.default_object_acl.loaded = default_object_acl_loaded
with _monkey_defaults(connection=connection):
Expand Down Expand Up @@ -969,14 +1007,16 @@ def __init__(self, bucket, name):
def acl(self):
return self

# Faux ACL methods
def all(self):
return self

def grant_read(self):
self._granted = True

def save_acl(self):
_saved.append((self._bucket, self._name, self._granted))
def save(self, connection=None):
_saved.append(
(self._bucket, self._name, self._granted, connection))

class _Iterator(_BlobIterator):
def get_items_from_response(self, response):
Expand All @@ -988,15 +1028,15 @@ def get_items_from_response(self, response):
permissive = [{'entity': 'allUsers', 'role': _ACLEntity.READER_ROLE}]
after = {'acl': permissive, 'defaultObjectAcl': []}
connection = _Connection(after, {'items': [{'name': BLOB_NAME}]})
bucket = self._makeOne(NAME, connection)
bucket = self._makeOne(NAME, None)
bucket.acl.loaded = True
bucket.default_object_acl.loaded = True
bucket._iterator_class = _Iterator
with _monkey_defaults(connection=connection):
bucket.make_public(recursive=True)
self.assertEqual(list(bucket.acl), permissive)
self.assertEqual(list(bucket.default_object_acl), [])
self.assertEqual(_saved, [(bucket, BLOB_NAME, True)])
self.assertEqual(_saved, [(bucket, BLOB_NAME, True, connection)])
kw = connection._requested
self.assertEqual(len(kw), 2)
self.assertEqual(kw[0]['method'], 'PATCH')
Expand All @@ -1005,7 +1045,7 @@ def get_items_from_response(self, response):
self.assertEqual(kw[0]['query_params'], {'projection': 'full'})
self.assertEqual(kw[1]['method'], 'GET')
self.assertEqual(kw[1]['path'], '/b/%s/o' % NAME)
self.assertEqual(kw[1]['query_params'], {'projection': 'noAcl'})
self.assertEqual(kw[1]['query_params'], {'projection': 'full'})


class _Connection(object):
Expand Down

0 comments on commit 290b09e

Please sign in to comment.