From 1ff8e8bd0789d797f8bce4833a57d970b4c61fb0 Mon Sep 17 00:00:00 2001 From: Danny Hermes Date: Wed, 25 Mar 2015 12:41:54 -0700 Subject: [PATCH 1/4] Documenting buckets in full. --- docs/index.rst | 1 + docs/storage-surface.rst | 318 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 319 insertions(+) create mode 100644 docs/storage-surface.rst diff --git a/docs/index.rst b/docs/index.rst index e97e005076d6..52691f0a1fe7 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -23,6 +23,7 @@ :hidden: :caption: Storage + storage-surface Client storage-blobs storage-buckets diff --git a/docs/storage-surface.rst b/docs/storage-surface.rst new file mode 100644 index 000000000000..7f82a15afb4f --- /dev/null +++ b/docs/storage-surface.rst @@ -0,0 +1,318 @@ +``gcloud.storage`` API +====================== + +Connection / Authorization +-------------------------- + +- Inferred defaults used to create connection if none configured explicitly: + + - credentials (derived from GAE / GCE environ if present). + + - ``project`` (derived from GAE / GCE environ if present). + + - ``bucket`` (derived from GAE / GCE environ if present). + +- By calling methods which require authentication + + .. code-block:: python + + >>> from gcloud import storage + >>> bucket = storage.get_bucket('my-bucket') + + the default connection "just works" out of the box and will be lazily + loaded and stored after the first use. + +- Set defaults in a declarative fashion + + .. code-block:: python + + >>> storage.set_defaults() + + or instead of using implicit behavior, pass in defaults + + .. code-block:: python + + >>> storage.set_defaults(connection=connection, project='some-project') + ... bucket=some_bucket) + + though not all are needed + + .. code-block:: python + + >>> storage.set_defaults(project='some-project', bucket=some_bucket) + +- Set defaults one-by-one + + .. code-block:: python + + >>> storage.set_default_bucket(some_bucket) + >>> storage.set_default_connection(connection) + >>> storage.set_default_project(project) + + +Manage buckets +-------------- + +Create a new bucket: + +.. code-block:: python + + >>> from gcloud import storage + >>> new_bucket = storage.create_bucket(bucket_name) + +if you desire to be declarative, you may pass in a connection to +override the default: + +.. code-block:: python + + >>> new_bucket = storage.create_bucket(bucket_name, connection=connection) + +Retrieve an existing bucket: + +.. code-block:: python + + >>> existing_bucket = storage.get_bucket(bucket_name) + +but if the bucket does not exist an exception will occur + +.. code-block:: python + + >>> existing_bucket = storage.get_bucket(bucket_name) + Traceback (most recent call last): + File "", line 1, in + gcloud.exceptions.NotFound: 404 Some Message + +To get a null response instead of an exception, use +:func:`lookup_bucket `: + +.. code-block:: python + + >>> non_existent = storage.lookup_bucket(bucket_name) + >>> print non_existent + None + +To list all buckets: + +.. code-block:: python + + >>> for bucket in storage.get_buckets(): + ... print bucket + + + + +or to use a project other than the default + +.. code-block:: python + + >>> for bucket in storage.get_buckets('some-project'): + ... print bucket + +To limit the list of buckets returned, +:func:`get_buckets() ` accepts optional +arguments + +.. code-block:: python + + >>> bucket_iterator = storage.get_buckets(max_results=2, + ... page_token='next-bucket-name', + ... prefix='foo', + ... projection='noAcl', + ... fields=None) + >>> for bucket in bucket_iterator: + ... print bucket + +See the `buckets list`_ documenation for details. + +.. _buckets list: https://cloud.google.com/storage/docs/json_api/v1/buckets/list + +To delete a bucket + +.. code-block:: python + + >>> storage.delete_bucket(bucket_name) + +.. note:: + Deleting a bucket should happen very infrequently. Be careful that you + actually mean to delete the bucket. + +In the case that the bucket has existing objects (``Blob`` here), the backend +will return a `409 conflict`_ and raise + +.. code-block:: python + + >>> storage.delete_bucket(bucket_name) + Traceback (most recent call last): + File "", line 1, in + gcloud.exceptions.Conflict: 409 Some Message + +this can be addressed by using the ``force`` keyword: + + >>> storage.delete_bucket(bucket_name, force=True) + +This too will fail if the bucket contains more than 256 blobs. +In this case, the blobs should be deleted manually first. + +.. _409 conflict: http://en.wikipedia.org/wiki/List_of_HTTP_status_codes#4xx_Client_Error + +Working with Buckets +-------------------- + +To create a bucket directly + +.. code-block:: python + + >>> bucket = storage.Bucket('bucket-name') + >>> bucket.exists() + False + >>> bucket.create() + >>> bucket.exists() + True + +You can also use an explicit connection + +.. code-block:: python + + >>> bucket = storage.Bucket('bucket-name', connection=connection) + +.. note:: + An explicitly passed connection will be bound to the ``bucket`` and + all objects associated with the bucket. This means that within a batch of + updates, the ``connection`` will be used to make the request instead of + the batch. + +To load all bucket properties + +.. code-block:: python + + >>> bucket = storage.Bucket('bucket-name') + >>> print bucket.last_sync + None + >>> bucket.properties + {} + >>> bucket.reload() + >>> bucket.last_sync + datetime.datetime(2015, 1, 1, 12, 0) + >>> bucket.properties + {u'etag': u'CAE=', + u'id': u'bucket-name', + ...} + >>> bucket.acl.loaded + False + >>> bucket.acl.reload() + >>> bucket.acl.loaded + True + >>> bucket.acl.entities + {'project-editors-111111': , + 'project-owners-111111': , + 'project-viewers-111111': , + 'user-01234': } + +Instead of calling +:meth:`Bucket.reload() ` and +:meth:`BucketACL.reload() `, you +can load the properties when the object is instantiated by using the +``eager`` keyword: + +.. code-block:: python + + >>> bucket = storage.Bucket('bucket-name', eager=True) + >>> bucket.last_sync + datetime.datetime(2015, 1, 1, 12, 0) + >>> bucket.acl.loaded + True + +To delete a bucket + +.. code-block:: python + + >>> bucket.delete() + +or + +.. code-block:: python + + >>> bucket.delete(force=True) + +as above. + +To make updates to the bucket use +:meth:`Bucket.patch() ` + +.. code-block:: python + + >>> bucket.versioning_enabled = True + >>> bucket.patch() + +If there are no updates to send, an exception will occur + +.. code-block:: python + + >>> bucket.patch() + Traceback (most recent call last): + File "", line 1, in + ValueError: No updates to send. + +In total, the properties that can be updated are + +.. code-block:: python + + >>> bucket.cors = [ + ... { + ... 'origin': ['http://example.appspot.com'], + ... 'responseHeader': ['Content-Type'], + ... 'method': ['GET', 'HEAD', 'DELETE'], + ... 'maxAgeSeconds': 3600, + ... } + ... ] + >>> bucket.lifecycle = [ + ... { + ... 'action': {'type': 'Delete'}, + ... 'condition': {'age': 365}, + ... }, + ... ] + >>> bucket.location = 'ASIA' + >>> bucket.logging = { + ... 'logBucket': 'bucket-name', + ... 'logObjectPrefix': 'foo/', + ... } + >>> bucket.versioning_enabled = True + >>> bucket.website = { + ... 'mainPageSuffix': 'index.html', + ... 'notFoundPage': '404.html', + ... } + >>> bucket.storage_class = 'DURABLE_REDUCED_AVAILABILITY' + +See `buckets`_ specification for more details. In general, many of these +properties are optional and will not need to be used (or changed from the +defaults). + +Other data -- namely `access control`_ -- is associated with buckets, but +this data is handled through ``Bucket.acl``. + +.. _buckets: https://cloud.google.com/storage/docs/json_api/v1/buckets +.. _access control: https://cloud.google.com/storage/docs/access-control + +Manage Blobs +------------ + +Interacting with blobs requires an associated bucket. + +To retrieve a blob, a bucket can be used directly + +.. code-block:: python + + >>> bucket.get_blob('blob-name') + + +or the default bucket can be used implicitly + +.. code-block:: python + + >>> storage.get_blob('blob-name') + + +Dealing with ACLs +----------------- + +To do From 8024de1853cec63f731cce0057a1a90bbfd4d03b Mon Sep 17 00:00:00 2001 From: Danny Hermes Date: Thu, 26 Mar 2015 00:40:25 -0700 Subject: [PATCH 2/4] Documenting Blob API surface. --- docs/storage-surface.rst | 421 ++++++++++++++++++++++++++++++++++++--- 1 file changed, 388 insertions(+), 33 deletions(-) diff --git a/docs/storage-surface.rst b/docs/storage-surface.rst index 7f82a15afb4f..d8b2013e39a5 100644 --- a/docs/storage-surface.rst +++ b/docs/storage-surface.rst @@ -4,7 +4,7 @@ Connection / Authorization -------------------------- -- Inferred defaults used to create connection if none configured explicitly: +- Inferred defaults used to create connection if none configured explicitly - credentials (derived from GAE / GCE environ if present). @@ -41,7 +41,7 @@ Connection / Authorization >>> storage.set_defaults(project='some-project', bucket=some_bucket) -- Set defaults one-by-one +- Declaratively set defaults one-by-one .. code-block:: python @@ -53,7 +53,7 @@ Connection / Authorization Manage buckets -------------- -Create a new bucket: +Create a new bucket .. code-block:: python @@ -61,13 +61,17 @@ Create a new bucket: >>> new_bucket = storage.create_bucket(bucket_name) if you desire to be declarative, you may pass in a connection to -override the default: +override the default .. code-block:: python >>> new_bucket = storage.create_bucket(bucket_name, connection=connection) -Retrieve an existing bucket: +.. note:: + All methods in the ``storage`` package accept ``connection`` as an optional + parameter. + +Retrieve an existing bucket .. code-block:: python @@ -83,7 +87,7 @@ but if the bucket does not exist an exception will occur gcloud.exceptions.NotFound: 404 Some Message To get a null response instead of an exception, use -:func:`lookup_bucket `: +:func:`lookup_bucket ` .. code-block:: python @@ -91,11 +95,28 @@ To get a null response instead of an exception, use >>> print non_existent None -To list all buckets: +To retrieve multiple buckets in a single request + +.. code-block:: python + + >>> bucket1, bucket2, bucket3 = storage.get_buckets('bucket-name1', + ... 'bucket-name2', + ... 'bucket-name3') + +This is equivalent to + +.. code-block:: python + + >>> with storage.Batch(): + ... bucket1 = storage.get_bucket('bucket-name1') + ... bucket2 = storage.get_bucket('bucket-name2') + ... bucket3 = storage.get_bucket('bucket-name3') + +To list all buckets associated to the default project .. code-block:: python - >>> for bucket in storage.get_buckets(): + >>> for bucket in storage.list_buckets(): ... print bucket @@ -105,24 +126,24 @@ or to use a project other than the default .. code-block:: python - >>> for bucket in storage.get_buckets('some-project'): + >>> for bucket in storage.list_buckets('some-project'): ... print bucket To limit the list of buckets returned, -:func:`get_buckets() ` accepts optional +:func:`list_buckets() ` accepts optional arguments .. code-block:: python - >>> bucket_iterator = storage.get_buckets(max_results=2, - ... page_token='next-bucket-name', - ... prefix='foo', - ... projection='noAcl', - ... fields=None) + >>> bucket_iterator = storage.list_buckets(max_results=2, + ... page_token='next-bucket-name', + ... prefix='foo', + ... projection='noAcl', + ... fields=None) >>> for bucket in bucket_iterator: ... print bucket -See the `buckets list`_ documenation for details. +See the `buckets list`_ documentation for details. .. _buckets list: https://cloud.google.com/storage/docs/json_api/v1/buckets/list @@ -132,11 +153,16 @@ To delete a bucket >>> storage.delete_bucket(bucket_name) -.. note:: +.. warning:: Deleting a bucket should happen very infrequently. Be careful that you actually mean to delete the bucket. -In the case that the bucket has existing objects (``Blob`` here), the backend +.. note:: + We use the term blob interchangeably with "object" when referring to the + API. The Google Cloud Storage documentation use object, but we use ``blob`` + instead to avoid confusion with the Python builtin ``object``. + +In the case that the bucket has existing blobs, the backend will return a `409 conflict`_ and raise .. code-block:: python @@ -146,19 +172,19 @@ will return a `409 conflict`_ and raise File "", line 1, in gcloud.exceptions.Conflict: 409 Some Message -this can be addressed by using the ``force`` keyword: +this can be addressed by using the ``force`` keyword >>> storage.delete_bucket(bucket_name, force=True) -This too will fail if the bucket contains more than 256 blobs. -In this case, the blobs should be deleted manually first. +Even using ``force=True`` will fail if the bucket contains more than 256 +blobs. In this case, the blobs should be deleted manually first. .. _409 conflict: http://en.wikipedia.org/wiki/List_of_HTTP_status_codes#4xx_Client_Error Working with Buckets -------------------- -To create a bucket directly +To create a bucket object directly .. code-block:: python @@ -181,7 +207,9 @@ You can also use an explicit connection updates, the ``connection`` will be used to make the request instead of the batch. -To load all bucket properties +By default, just constructing a :class:`Bucket ` +does not load any of the associated bucket metadata. To load all bucket +properties .. code-block:: python @@ -212,7 +240,7 @@ Instead of calling :meth:`Bucket.reload() ` and :meth:`BucketACL.reload() `, you can load the properties when the object is instantiated by using the -``eager`` keyword: +``eager`` keyword .. code-block:: python @@ -276,42 +304,369 @@ In total, the properties that can be updated are ... 'logBucket': 'bucket-name', ... 'logObjectPrefix': 'foo/', ... } + >>> bucket.storage_class = 'DURABLE_REDUCED_AVAILABILITY' >>> bucket.versioning_enabled = True >>> bucket.website = { ... 'mainPageSuffix': 'index.html', ... 'notFoundPage': '404.html', ... } - >>> bucket.storage_class = 'DURABLE_REDUCED_AVAILABILITY' -See `buckets`_ specification for more details. In general, many of these -properties are optional and will not need to be used (or changed from the -defaults). +In general, many of these properties are optional and will not need to be +used (or changed from the defaults). -Other data -- namely `access control`_ -- is associated with buckets, but -this data is handled through ``Bucket.acl``. +In addition, a bucket has several read-only properties + +.. code-block:: python + + >>> bucket.etag + u'CAI=' + >>> bucket.id + u'bucket-name' + >>> bucket.metageneration + 2L + >>> bucket.name + u'bucket-name' + >>> bucket.owner + {u'entity': u'project-owners-111111'} + >>> bucket.project_number + u'111111' + >>> bucket.self_link + u'https://www.googleapis.com/storage/v1/b/bucket-name' + >>> bucket.time_created + u'2015-01-01T12:00:00.000Z' + +See `buckets`_ specification for more details. + +Other data -- namely `access control`_ data -- is associated with buckets, +but is handled through ``Bucket.acl``. .. _buckets: https://cloud.google.com/storage/docs/json_api/v1/buckets .. _access control: https://cloud.google.com/storage/docs/access-control +.. note:: + **BREAKING THE FOURTH WALL**: Note that ``storage.buckets.update`` is + absent. This doesn't seem necessary to implement given the presence of + ``patch``. + Manage Blobs ------------ -Interacting with blobs requires an associated bucket. +Interacting with blobs requires an associated bucket. Either methods on an +explicit :class:`Bucket ` instance can be used +or the default bucket can be used via methods in the ``storage`` package. -To retrieve a blob, a bucket can be used directly +To retrieve a blob with a bucket .. code-block:: python >>> bucket.get_blob('blob-name') -or the default bucket can be used implicitly +or from the package via the default bucket .. code-block:: python >>> storage.get_blob('blob-name') +Simply calling ``get_blob`` will not actually retrieve the contents stored +for the given blob. Instead, it retrieves the metadata associated with +the blob. + +For the remainder of this section we will just illustrate the methods +on :class:`Bucket `. For each such method, +an equivalent function is defined in the ``storage`` package and each +such function relies on the default bucket. + +To retrieve multiple blobs in a single request + +.. code-block:: python + + >>> blob1, blob2, blob3 = bucket.get_blobs('blob-name1', + ... 'blob-name2', + ... 'blob-name3') + +This is equivalent to + +.. code-block:: python + + >>> with storage.Batch(): + ... blob1 = bucket.get_blob('blob-name1') + ... blob2 = bucket.get_blob('blob-name2') + ... blob3 = bucket.get_blob('blob-name3') + +however, recall that if ``bucket`` has a ``connection`` explicitly bound +to it, the ``bucket.get_blob`` requests will be executed individually +rather than as part of the batch. + +To list all blobs in a bucket + +.. code-block:: python + + >>> for blob in bucket.list_blobs(): + ... print blob + + + + +.. warning:: + In a production application, a typical bucket may very likely have thousands + or even millions of blobs. Iterating through all of them in such an + application is a very bad idea. + +To limit the list of blobs returned, +:meth:`list_blobs() ` accepts optional +arguments + +.. code-block:: python + + >>> blob_iterator = bucket.list_blobs(max_results=2, + ... page_token='next-blob-name', + ... prefix='foo', + ... delimiter='/', + ... versions=True, + ... projection='noAcl', + ... fields=None) + >>> for blob in blob_iterator: + ... print blob + +See the `objects list`_ documentation for details. + +.. _objects list: https://cloud.google.com/storage/docs/json_api/v1/objects/list + +To delete a blob + +.. code-block:: python + + >>> bucket.delete_blob(blob_name) + +As with retrieving, you may also delete multiple blobs in a single request + +.. code-block:: python + + >>> bucket.delete_blobs('blob-name1', 'blob-name2', 'blob-name3') + +This is equivalent to + +.. code-block:: python + + >>> with storage.Batch(): + ... bucket.delete_blob('blob-name1') + ... bucket.delete_blob('blob-name2') + ... bucket.delete_blob('blob-name3') + +To copy an existing blob to a new location, potentially even in +a new bucket + +.. code-block:: python + + >>> new_blob = bucket.copy_blob(blob, new_bucket, new_name='new-blob-name') + +To compose multiple blobs together + +.. code-block:: python + + >>> blob1, blob2 = bucket.get_blobs('blob-name1', 'blob-name2') + >>> new_blob = bucket.compose('composed-blob', parts=[blob1, blob2]) + +To upload and download blob data, :class:`Blob ` +objects should be used directly. + +Working with Blobs +------------------ + +To create a blob object directly + +.. code-block:: python + + >>> blob = storage.Blob('blob-name', bucket=bucket) + >>> blob.exists() + False + >>> blob.create() + >>> blob.exists() + True + >>> blob + + +If no ``bucket`` is provided, the default bucket will be used + +.. code-block:: python + + >>> storage.Blob('blob-name') + + +By default, just constructing a :class:`Blob ` +does not load any of the associated blob metadata. To load all blob +properties + +.. code-block:: python + + >>> blob = storage.Blob('blob-name') + >>> print blob.last_sync + None + >>> blob.properties + {} + >>> blob.reload() + >>> blob.last_sync + datetime.datetime(2015, 1, 1, 12, 0) + >>> blob.properties + {u'bucket': u'default-bucket-name', + u'contentType': u'text/plain', + ...} + >>> blob.acl.loaded + False + >>> blob.acl.reload() + >>> blob.acl.loaded + True + >>> blob.acl.entities + {'project-editors-111111': , + 'project-owners-111111': , + 'project-viewers-111111': , + 'user-01234': } + +Instead of calling +:meth:`Blob.reload() ` and +:meth:`ObjectACL.reload() `, you +can load the properties when the object is instantiated by using the +``eager`` keyword + +.. code-block:: python + + >>> blob = storage.Blob('blob-name') + >>> blob.last_sync + datetime.datetime(2015, 1, 1, 12, 0) + >>> blob.acl.loaded + True + +To delete a blob + +.. code-block:: python + + >>> blob.delete() + +To generate a signed URL for temporary privileged access to the +contents of a blob + +.. code-block:: python + + >>> expiration_seconds = 600 + >>> signed_url = blob.generate_signed_url(expiration_seconds) + +To make updates to the blob use +:meth:`Blob.patch() ` + +.. code-block:: python + + >>> blob.versioning_enabled = True + >>> blob.patch() + +If there are no updates to send, an exception will occur + +.. code-block:: python + + >>> blob.patch() + Traceback (most recent call last): + File "", line 1, in + ValueError: No updates to send. + +In total, the properties that can be updated are + +.. code-block:: python + + >>> blob.cache_control = 'private, max-age=0, no-cache' + >>> blob.content_disposition = 'Attachment; filename=example.html' + >>> blob.content_encoding = 'gzip' + >>> blob.content_language = 'en-US' + >>> blob.content_type = 'text/plain' + >>> blob.crc32c = u'z8SuHQ==' # crc32-c of "foo" + >>> blob.md5_hash = u'rL0Y20zC+Fzt72VPzMSk2A==' # md5 of "foo" + >>> blob.metadata = {'foo': 'bar', 'baz': 'qux'} + +.. note:: + **BREAKING THE FOURTH WALL**: Why are ``crc32c`` and ``md5_hash`` writable? + +In general, many of these properties are optional and will not need to be +used (or changed from the defaults). + +In addition, a blob has several read-only properties + +.. code-block:: python + + >>> blob.bucket + + >>> blob.component_count + 1 + >>> blob.etag + u'CNiOr665xcQCEAE=' + >>> blob.generation + 12345L + >>> blob.id + u'bucket-name/blob-name/12345' + >>> blob.media_link + u'https://www.googleapis.com/download/storage/v1/b/bucket-name/o/blob-name?generation=12345&alt=media' + >>> blob.metageneration + 1L + >>> blob.name + 'blob-name' + >>> blob.owner + {u'entity': u'user-01234', + u'entityId': u'01234'} + >>> blob.self_link + u'https://www.googleapis.com/storage/v1/b/bucket-name/o/blob-name' + >>> blob.size + 3L + >>> blob.storage_class + u'STANDARD' + >>> print blob.time_deleted + None + >>> blob.updated + u'2015-01-01T12:00:00.000Z' + +See `objects`_ specification for more details. + +Other data -- namely `access control`_ data -- is associated with blobs, +but is handled through ``Blob.acl``. + +.. _objects: https://cloud.google.com/storage/docs/json_api/v1/objects + +Working with Blob Data +---------------------- + +The most important use of a blob is not accessing and updating the metadata, +it is storing data in the cloud (hence Cloud Storage). + +To upload string data into a blob + + .. code-block:: python + + >>> blob.upload_from_string('foo') + +If the data has a known content-type, set it on the blob before +uploading: + + .. code-block:: python + + >>> blob.content_type = 'application/zip' + >>> blob.upload_from_string('foo') + +To upload instead from a file + + .. code-block:: python + + >>> blob.upload_from_filename('/path/on/local/machine.file') + +To download blob data into a string + + .. code-block:: python + + >>> blob_contents = blob.download_as_string() + +To download instead to a file + + .. code-block:: python + + >>> blob.download_to_filename('/path/on/local/machine.file') + Dealing with ACLs ----------------- From 508d0bb82632243e935a1f10594f667aa959e3ad Mon Sep 17 00:00:00 2001 From: Danny Hermes Date: Sat, 28 Mar 2015 14:05:37 -0700 Subject: [PATCH 3/4] Updating after review comments from @tseaver. --- docs/storage-surface.rst | 41 +++++++++++++++++++++++++++++++++------- 1 file changed, 34 insertions(+), 7 deletions(-) diff --git a/docs/storage-surface.rst b/docs/storage-surface.rst index d8b2013e39a5..5206fb9eb676 100644 --- a/docs/storage-surface.rst +++ b/docs/storage-surface.rst @@ -60,16 +60,22 @@ Create a new bucket >>> from gcloud import storage >>> new_bucket = storage.create_bucket(bucket_name) -if you desire to be declarative, you may pass in a connection to -override the default +.. warning:: + If the bucket already exists, this method will throw an exception + corresponding to the `409 conflict`_ status code in the response. + +If you desire to be declarative, you may pass in a connection and a project +to override the defaults .. code-block:: python - >>> new_bucket = storage.create_bucket(bucket_name, connection=connection) + >>> new_bucket = storage.create_bucket(bucket_name, connection=connection, + ... project=project) .. note:: All methods in the ``storage`` package accept ``connection`` as an optional - parameter. + parameter. (Only ``create_bucket`` and ``list_buckets`` accept ``project`` + to override the default.) Retrieve an existing bucket @@ -177,7 +183,7 @@ this can be addressed by using the ``force`` keyword >>> storage.delete_bucket(bucket_name, force=True) Even using ``force=True`` will fail if the bucket contains more than 256 -blobs. In this case, the blobs should be deleted manually first. +blobs. In this case, delete the blobs manually before deleting the bucket. .. _409 conflict: http://en.wikipedia.org/wiki/List_of_HTTP_status_codes#4xx_Client_Error @@ -649,19 +655,40 @@ uploading: >>> blob.content_type = 'application/zip' >>> blob.upload_from_string('foo') -To upload instead from a file +To upload instead from a file-like object + + .. code-block:: python + + >>> blob.upload_from_stream(file_object) + +To upload directly from a file .. code-block:: python >>> blob.upload_from_filename('/path/on/local/machine.file') +This is roughly equivalent to + + .. code-block:: python + + >>> with open('/path/on/local/machine.file', 'w') as file_object: + ... blob.upload_from_stream(file_object) + +with some extra behavior to set local file properties. + To download blob data into a string .. code-block:: python >>> blob_contents = blob.download_as_string() -To download instead to a file +To download instead to a file-like object + + .. code-block:: python + + >>> blob.download_to_stream(file_object) + +To download directly to a file .. code-block:: python From ca5122dff1bfe7abb644c0a43cddf30e19deaa7c Mon Sep 17 00:00:00 2001 From: Danny Hermes Date: Mon, 30 Mar 2015 00:04:20 -0700 Subject: [PATCH 4/4] Finishing storage surface API docs. --- docs/storage-surface.rst | 386 ++++++++++++++++++++++++++++++--------- 1 file changed, 301 insertions(+), 85 deletions(-) diff --git a/docs/storage-surface.rst b/docs/storage-surface.rst index 5206fb9eb676..bdfd2b65a76a 100644 --- a/docs/storage-surface.rst +++ b/docs/storage-surface.rst @@ -114,9 +114,13 @@ This is equivalent to .. code-block:: python >>> with storage.Batch(): - ... bucket1 = storage.get_bucket('bucket-name1') - ... bucket2 = storage.get_bucket('bucket-name2') - ... bucket3 = storage.get_bucket('bucket-name3') + ... bucket_future1 = storage.get_bucket('bucket-name1') + ... bucket_future2 = storage.get_bucket('bucket-name2') + ... bucket_future3 = storage.get_bucket('bucket-name3') + ... + >>> bucket1 = bucket_future1.get() + >>> bucket2 = bucket_future2.get() + >>> bucket3 = bucket_future3.get() To list all buckets associated to the default project @@ -201,17 +205,16 @@ To create a bucket object directly >>> bucket.exists() True -You can also use an explicit connection +You can also use an explicit connection when calling +:class:`Bucket ` methods which make HTTP +requests .. code-block:: python - >>> bucket = storage.Bucket('bucket-name', connection=connection) - -.. note:: - An explicitly passed connection will be bound to the ``bucket`` and - all objects associated with the bucket. This means that within a batch of - updates, the ``connection`` will be used to make the request instead of - the batch. + >>> bucket = storage.Bucket('bucket-name') + >>> bucket.exists(connection=connection) + False + >>> bucket.create(connection=connection) By default, just constructing a :class:`Bucket ` does not load any of the associated bucket metadata. To load all bucket @@ -222,29 +225,17 @@ properties >>> bucket = storage.Bucket('bucket-name') >>> print bucket.last_sync None - >>> bucket.properties - {} + >>> print bucket.self_link + None >>> bucket.reload() + >>> # May be necessary to include projection and fields for last sync >>> bucket.last_sync datetime.datetime(2015, 1, 1, 12, 0) - >>> bucket.properties - {u'etag': u'CAE=', - u'id': u'bucket-name', - ...} - >>> bucket.acl.loaded - False - >>> bucket.acl.reload() - >>> bucket.acl.loaded - True - >>> bucket.acl.entities - {'project-editors-111111': , - 'project-owners-111111': , - 'project-viewers-111111': , - 'user-01234': } + >>> bucket.self_link + u'https://www.googleapis.com/storage/v1/b/bucket-name' Instead of calling -:meth:`Bucket.reload() ` and -:meth:`BucketACL.reload() `, you +:meth:`Bucket.reload() `, you can load the properties when the object is instantiated by using the ``eager`` keyword @@ -253,8 +244,9 @@ can load the properties when the object is instantiated by using the >>> bucket = storage.Bucket('bucket-name', eager=True) >>> bucket.last_sync datetime.datetime(2015, 1, 1, 12, 0) - >>> bucket.acl.loaded - True + +As with using :class:`Bucket ` methods, +you can also pass an explicit ``connection`` in when using ``eager=True``. To delete a bucket @@ -291,6 +283,12 @@ In total, the properties that can be updated are .. code-block:: python + >>> bucket.acl = [ + ... ACLEntity('project-editors-111111', 'OWNER'), + ... ACLEntity('project-owners-111111', 'OWNER'), + ... ACLEntity('project-viewers-111111, 'READER'), + ... ACLEntity('user-01234, 'OWNER'), + ... ] >>> bucket.cors = [ ... { ... 'origin': ['http://example.appspot.com'], @@ -299,6 +297,10 @@ In total, the properties that can be updated are ... 'maxAgeSeconds': 3600, ... } ... ] + >>> bucket.default_object_acl = [ + ... ACLEntity('project-owners-111111', 'OWNER'), + ... ACLEntity('user-01234, 'OWNER'), + ... ] >>> bucket.lifecycle = [ ... { ... 'action': {'type': 'Delete'}, @@ -333,21 +335,21 @@ In addition, a bucket has several read-only properties >>> bucket.name u'bucket-name' >>> bucket.owner - {u'entity': u'project-owners-111111'} + >>> bucket.project_number - u'111111' + 111111L >>> bucket.self_link u'https://www.googleapis.com/storage/v1/b/bucket-name' >>> bucket.time_created - u'2015-01-01T12:00:00.000Z' - -See `buckets`_ specification for more details. + datetime.datetime(2015, 1, 1, 12, 0) -Other data -- namely `access control`_ data -- is associated with buckets, -but is handled through ``Bucket.acl``. +See `buckets`_ specification for more details. `Access control`_ data is +complex enough to be a topic of its own. We provide the +:class:`ACLEntity ` class to represent these +objects and will discuss more further on. .. _buckets: https://cloud.google.com/storage/docs/json_api/v1/buckets -.. _access control: https://cloud.google.com/storage/docs/access-control +.. _Access control: https://cloud.google.com/storage/docs/access-control .. note:: **BREAKING THE FOURTH WALL**: Note that ``storage.buckets.update`` is @@ -397,13 +399,13 @@ This is equivalent to .. code-block:: python >>> with storage.Batch(): - ... blob1 = bucket.get_blob('blob-name1') - ... blob2 = bucket.get_blob('blob-name2') - ... blob3 = bucket.get_blob('blob-name3') - -however, recall that if ``bucket`` has a ``connection`` explicitly bound -to it, the ``bucket.get_blob`` requests will be executed individually -rather than as part of the batch. + ... blob_future1 = bucket.get_blob('blob-name1') + ... blob_future2 = bucket.get_blob('blob-name2') + ... blob_future3 = bucket.get_blob('blob-name3') + ... + >>> blob1 = blob_future1.get() + >>> blob2 = blob_future2.get() + >>> blob3 = blob_future3.get() To list all blobs in a bucket @@ -466,6 +468,7 @@ a new bucket .. code-block:: python + >>> new_bucket = storage.Bucket('new-bucket') >>> new_blob = bucket.copy_blob(blob, new_bucket, new_name='new-blob-name') To compose multiple blobs together @@ -494,6 +497,17 @@ To create a blob object directly >>> blob +You can pass the arguments ``if_generation_match`` or +``if_generation_not_match`` (mutually exclusive) and ``if_metageneration_match`` +or ``if_metageneration_not_match`` (also mutually exclusive). See documentation +for `objects.insert`_ for more details. + +.. _objects.insert: https://cloud.google.com/storage/docs/json_api/v1/objects/insert + +As with :class:`Bucket `, a +:class:`Connection ` can be passed to +methods which make HTTP requests. + If no ``bucket`` is provided, the default bucket will be used .. code-block:: python @@ -510,39 +524,24 @@ properties >>> blob = storage.Blob('blob-name') >>> print blob.last_sync None - >>> blob.properties - {} + >>> print blob.content_type + None >>> blob.reload() >>> blob.last_sync datetime.datetime(2015, 1, 1, 12, 0) - >>> blob.properties - {u'bucket': u'default-bucket-name', - u'contentType': u'text/plain', - ...} - >>> blob.acl.loaded - False - >>> blob.acl.reload() - >>> blob.acl.loaded - True - >>> blob.acl.entities - {'project-editors-111111': , - 'project-owners-111111': , - 'project-viewers-111111': , - 'user-01234': } + >>> print blob.content_type + u'text/plain' Instead of calling -:meth:`Blob.reload() ` and -:meth:`ObjectACL.reload() `, you +:meth:`Blob.reload() `, you can load the properties when the object is instantiated by using the ``eager`` keyword .. code-block:: python - >>> blob = storage.Blob('blob-name') + >>> blob = storage.Blob('blob-name', eager=True) >>> blob.last_sync datetime.datetime(2015, 1, 1, 12, 0) - >>> blob.acl.loaded - True To delete a blob @@ -550,14 +549,6 @@ To delete a blob >>> blob.delete() -To generate a signed URL for temporary privileged access to the -contents of a blob - -.. code-block:: python - - >>> expiration_seconds = 600 - >>> signed_url = blob.generate_signed_url(expiration_seconds) - To make updates to the blob use :meth:`Blob.patch() ` @@ -579,6 +570,10 @@ In total, the properties that can be updated are .. code-block:: python + >>> blob.acl = [ + ... ACLEntity('project-owners-111111', 'OWNER'), + ... ACLEntity('user-01234, 'OWNER'), + ... ] >>> blob.cache_control = 'private, max-age=0, no-cache' >>> blob.content_disposition = 'Attachment; filename=example.html' >>> blob.content_encoding = 'gzip' @@ -615,8 +610,7 @@ In addition, a blob has several read-only properties >>> blob.name 'blob-name' >>> blob.owner - {u'entity': u'user-01234', - u'entityId': u'01234'} + >>> blob.self_link u'https://www.googleapis.com/storage/v1/b/bucket-name/o/blob-name' >>> blob.size @@ -626,12 +620,12 @@ In addition, a blob has several read-only properties >>> print blob.time_deleted None >>> blob.updated - u'2015-01-01T12:00:00.000Z' - -See `objects`_ specification for more details. + datetime.datetime(2015, 1, 1, 12, 0) -Other data -- namely `access control`_ data -- is associated with blobs, -but is handled through ``Blob.acl``. +See `objects`_ specification for more details. `Access control`_ data is +complex enough to be a topic of its own. We provide the +:class:`ACLEntity ` class to represent these +objects and will discuss more further on. .. _objects: https://cloud.google.com/storage/docs/json_api/v1/objects @@ -676,6 +670,10 @@ This is roughly equivalent to with some extra behavior to set local file properties. +.. note:: + If you ``upload`` a blob which didn't already exist, it will also be + created with all the properties you have set locally. + To download blob data into a string .. code-block:: python @@ -694,7 +692,225 @@ To download directly to a file >>> blob.download_to_filename('/path/on/local/machine.file') -Dealing with ACLs ------------------ +Dealing with Sharing and ACLs +----------------------------- + +To generate a signed URL for temporary privileged access to the +contents of a blob + +.. code-block:: python + + >>> expiration_seconds = 600 + >>> signed_url = blob.generate_signed_url(expiration_seconds) + +A :class:`Bucket ` has both its own ACLs +and a set of default ACLs to be used for newly created blobs. + +.. code-block:: python + + >>> bucket.acl + [, + , + , + ] + >>> bucket.default_object_acl + [, + ] + +This will be updated when calling ``bucket.reload()``, since by +default ``projection=full`` is used to get the bucket properties. + +To update these directly + +.. code-block:: python + + >>> bucket.update_acl() + >>> bucket.acl + [, + , + , + , + ] + >>> bucket.update_default_object_acl() + >>> bucket.default_object_acl + [, + , + ] + +These methods call `bucketAccessControls.list`_ and +`defaultObjectAccessControls.list`_ instead of updating +every single property associated with the bucket. + +.. _bucketAccessControls.list: https://cloud.google.com/storage/docs/json_api/v1/bucketAccessControls/list +.. _defaultObjectAccessControls.list: https://cloud.google.com/storage/docs/json_api/v1/defaultObjectAccessControls/list + +You can limit the results of +:meth:`Bucket.update_default_object_acl() ` +by using + +.. code-block:: python + + >>> bucket.update_default_object_acl(if_metageneration_match=1) + +or + +.. code-block:: python + + >>> bucket.update_default_object_acl(if_metageneration_not_match=1) + +Similarly, a :class:`Blob ` has its own ACLs + +.. code-block:: python + + >>> blob.acl + [, + ] + +This will be updated when calling ``blob.reload()``, since by +default ``projection=full`` is used to get the blob properties. + +To update these directly + +.. code-block:: python + + >>> blob.update_acl() + >>> blob.acl + [, + , + ] + +When sending the `objectAccessControls.list`_ request, the blob's current +generation is sent. + +.. _objectAccessControls.list: https://cloud.google.com/storage/docs/json_api/v1/objectAccessControls/list + +Individual :class:`ACLEntity ` objects can be +edited and updated directly + +.. code-block:: python + + >>> entity = bucket.acl[1] + >>> entity + + >>> entity.role = storage.ROLES.WRITER + + >>> entity.patch() + +A :class:`ACLEntity ` objects has two +properties that can be updated + +.. code-block:: python + + >>> entity.entity = 'user-01234' + >>> entity.role = 'WRITER' + +and several read-only properties + +.. code-block:: python + + >>> entity.bucket + u'bucket-name' + >>> entity.domain + u'foo.com' + >>> entity.email + u'foo@gmail.com' + >>> entity.entityId + u'00b4903a9708670FAKEDATA3109ed94bFAKEDATA3e3090f8c566691bFAKEDATA' + >>> entity.etag + u'CAI=' + >>> entity.generation + 1L + >>> entity.id + u'bucket-name/project-owners-111111' + >>> entity.project_team + {u'projectNumber': u'111111', u'team': u'owners'} + >>> entity.self_link + u'https://www.googleapis.com/storage/v1/b/bucket-name/acl/project-owners-111111' + +To update the values in an ACL, you can either update the entire parent + +.. code-block:: python + + >>> blob.acl + [, + ] + >>> blob.reload() + >>> blob.acl + [, + ] + +or just reload the individual ACL + +.. code-block:: python + + >>> blob.acl + [, + ] + >>> blob.acl[1].reload() + >>> blob.acl + [, + ] + +To add an ACL to an existing object + +.. code-block:: python + + >>> bucket.add_acl_entity('group-foo@googlegroups.com', 'WRITER') + >>> bucket.add_default_object_acl_entity('domain-foo.com', 'OWNER') + >>> blob.add_acl_entity('user-01234', 'READER') + +To remove an ACL, you can either reduce the list and update + +.. code-block:: python + + >>> blob.acl + [, + , + ] + >>> blob.acl.remove(blob.acl[1]) + >>> blob.patch() + >>> blob.acl + [, + ] + +or delete the ACL directly + +.. code-block:: python + + >>> blob.acl + [, + , + ] + >>> blob.acl[1].delete() + >>> blob.acl + [, + ] + +.. note:: + **BREAKING THE FOURTH WALL**: Note that ``storage.*AccessControls.insert`` + and ``storage.*AccessControls.update`` are absent. This is done + intentionally, with the philosophy that an + :class:`ACLEntity ` must be attached to either + a :class:`Bucket ` or + :class:`Blob ` + +Predefined ACLs +--------------- + +When creating a new bucket, you can set predefined ACLs + +.. code-block:: python + + >>> bucket.create(predefined_acl=storage.ACLS.PROJECT_PRIVATE, + ... predefined_default_object_acl=storage.ACLS.PRIVATE) + +The enum variable ``storage.ACLS`` contains all acceptable values. See +documentation for `buckets.insert`_ for more details. + +.. _buckets.insert: https://cloud.google.com/storage/docs/json_api/v1/buckets/insert + +When creating a new blob, you can set a predefined ACL + +.. code-block:: python -To do + >>> blob.create(predefined_acl=storage.ACLS.AUTHENTICATED_READ)