From deecc14fe1015d7f3e62323da57471f9d7dd8ec1 Mon Sep 17 00:00:00 2001 From: Marco Ziccardi Date: Mon, 2 Nov 2015 13:40:39 +0100 Subject: [PATCH 1/7] Add support for selected fields to blob get and list - Add AclField and BlobField classes to Storage - Add BlobGetOption class to Storage that supports the .fields option - Use BlobGetOption in Storage.get instead of BlobSourceOption - Add .fields option to BlobListOption - Updated Blob.exists to use selected fields - Use BlobGetOption in BatchRequest.toGet instead of BlobSourceOption - Add unit and integration tests for selected fields --- .../google/gcloud/spi/DefaultStorageRpc.java | 5 +- .../com/google/gcloud/spi/StorageRpc.java | 3 +- .../google/gcloud/storage/BatchRequest.java | 11 +- .../java/com/google/gcloud/storage/Blob.java | 33 ++- .../com/google/gcloud/storage/Bucket.java | 4 +- .../com/google/gcloud/storage/Storage.java | 261 +++++++++++++++++- .../google/gcloud/storage/StorageImpl.java | 8 +- .../gcloud/storage/BatchRequestTest.java | 11 +- .../com/google/gcloud/storage/BlobTest.java | 8 +- .../com/google/gcloud/storage/BucketTest.java | 2 +- .../google/gcloud/storage/ITStorageTest.java | 83 +++++- .../gcloud/storage/StorageImplTest.java | 121 +++++++- 12 files changed, 519 insertions(+), 31 deletions(-) diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java b/gcloud-java-storage/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java index 70cad8c7773e..29552d5121e5 100644 --- a/gcloud-java-storage/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java +++ b/gcloud-java-storage/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java @@ -15,6 +15,7 @@ package com.google.gcloud.spi; import static com.google.gcloud.spi.StorageRpc.Option.DELIMITER; +import static com.google.gcloud.spi.StorageRpc.Option.FIELDS; import static com.google.gcloud.spi.StorageRpc.Option.IF_GENERATION_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.IF_GENERATION_NOT_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.IF_METAGENERATION_MATCH; @@ -168,6 +169,7 @@ public Tuple> list(String bucket, Map .setPrefix(PREFIX.getString(options)) .setMaxResults(MAX_RESULTS.getLong(options)) .setPageToken(PAGE_TOKEN.getString(options)) + .setFields(FIELDS.getString(options)) .execute(); return Tuple.>of( objects.getNextPageToken(), objects.getItems()); @@ -207,7 +209,8 @@ private Storage.Objects.Get getRequest(StorageObject object, Map opti .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options)) - .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options)); + .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options)) + .setFields(FIELDS.getString(options)); } @Override diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/spi/StorageRpc.java b/gcloud-java-storage/src/main/java/com/google/gcloud/spi/StorageRpc.java index 40382a857fca..e4b1be785951 100644 --- a/gcloud-java-storage/src/main/java/com/google/gcloud/spi/StorageRpc.java +++ b/gcloud-java-storage/src/main/java/com/google/gcloud/spi/StorageRpc.java @@ -47,7 +47,8 @@ enum Option { MAX_RESULTS("maxResults"), PAGE_TOKEN("pageToken"), DELIMITER("delimiter"), - VERSIONS("versions"); + VERSIONS("versions"), + FIELDS("fields"); private final String value; diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BatchRequest.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BatchRequest.java index 6e815648497a..bf77c731754e 100644 --- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BatchRequest.java +++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BatchRequest.java @@ -18,6 +18,7 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; +import com.google.gcloud.storage.Storage.BlobGetOption; import com.google.gcloud.storage.Storage.BlobSourceOption; import com.google.gcloud.storage.Storage.BlobTargetOption; @@ -35,13 +36,13 @@ public final class BatchRequest implements Serializable { private final Map> toDelete; private final Map> toUpdate; - private final Map> toGet; + private final Map> toGet; public static class Builder { private Map> toDelete = new LinkedHashMap<>(); private Map> toUpdate = new LinkedHashMap<>(); - private Map> toGet = new LinkedHashMap<>(); + private Map> toGet = new LinkedHashMap<>(); private Builder() {} @@ -72,7 +73,7 @@ public Builder update(BlobInfo blobInfo, BlobTargetOption... options) { /** * Retrieve metadata for the given blob. */ - public Builder get(String bucket, String blob, BlobSourceOption... options) { + public Builder get(String bucket, String blob, BlobGetOption... options) { toGet.put(BlobId.of(bucket, blob), Lists.newArrayList(options)); return this; } @@ -80,7 +81,7 @@ public Builder get(String bucket, String blob, BlobSourceOption... options) { /** * Retrieve metadata for the given blob. */ - public Builder get(BlobId blob, BlobSourceOption... options) { + public Builder get(BlobId blob, BlobGetOption... options) { toGet.put(blob, Lists.newArrayList(options)); return this; } @@ -120,7 +121,7 @@ public Map> toUpdate() { return toUpdate; } - public Map> toGet() { + public Map> toGet() { return toGet; } diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Blob.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Blob.java index 8f988922aad9..88f25c7e5cbd 100644 --- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Blob.java +++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Blob.java @@ -19,6 +19,7 @@ import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkNotNull; import static com.google.gcloud.storage.Blob.BlobSourceOption.convert; +import static com.google.gcloud.storage.Blob.BlobSourceOption.toGetOptions; import com.google.common.base.Function; import com.google.common.collect.Lists; @@ -29,6 +30,7 @@ import com.google.gcloud.storage.Storage.SignUrlOption; import java.net.URL; +import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Objects; @@ -71,6 +73,21 @@ private Storage.BlobSourceOption convert(BlobInfo blobInfo) { } } + private Storage.BlobGetOption toGetOption(BlobInfo blobInfo) { + switch (rpcOption()) { + case IF_GENERATION_MATCH: + return Storage.BlobGetOption.generationMatch(blobInfo.generation()); + case IF_GENERATION_NOT_MATCH: + return Storage.BlobGetOption.generationNotMatch(blobInfo.generation()); + case IF_METAGENERATION_MATCH: + return Storage.BlobGetOption.metagenerationMatch(blobInfo.metageneration()); + case IF_METAGENERATION_NOT_MATCH: + return Storage.BlobGetOption.metagenerationNotMatch(blobInfo.metageneration()); + default: + throw new AssertionError("Unexpected enum value"); + } + } + public static BlobSourceOption generationMatch() { return new BlobSourceOption(StorageRpc.Option.IF_GENERATION_MATCH); } @@ -95,6 +112,15 @@ static Storage.BlobSourceOption[] convert(BlobInfo blobInfo, BlobSourceOption... } return convertedOptions; } + + static Storage.BlobGetOption[] toGetOptions(BlobInfo blobInfo, BlobSourceOption... options) { + Storage.BlobGetOption[] convertedOptions = new Storage.BlobGetOption[options.length]; + int index = 0; + for (BlobSourceOption option : options) { + convertedOptions[index++] = option.toGetOption(blobInfo); + } + return convertedOptions; + } } /** @@ -159,7 +185,10 @@ public BlobId id() { * @throws StorageException upon failure */ public boolean exists(BlobSourceOption... options) { - return storage.get(info.blobId(), convert(info, options)) != null; + int length = options.length; + Storage.BlobGetOption[] getOptions = Arrays.copyOf(toGetOptions(info, options), length + 1); + getOptions[length] = Storage.BlobGetOption.fields(); + return storage.get(info.blobId(), getOptions) != null; } /** @@ -180,7 +209,7 @@ public byte[] content(Storage.BlobSourceOption... options) { * @throws StorageException upon failure */ public Blob reload(BlobSourceOption... options) { - return new Blob(storage, storage.get(info.blobId(), convert(info, options))); + return new Blob(storage, storage.get(info.blobId(), toGetOptions(info, options))); } /** diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Bucket.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Bucket.java index 1f696d07ac33..3b659f0cd355 100644 --- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Bucket.java +++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Bucket.java @@ -24,7 +24,7 @@ import com.google.common.collect.Iterators; import com.google.gcloud.PageImpl; import com.google.gcloud.Page; -import com.google.gcloud.storage.Storage.BlobSourceOption; +import com.google.gcloud.storage.Storage.BlobGetOption; import com.google.gcloud.storage.Storage.BlobTargetOption; import com.google.gcloud.storage.Storage.BlobWriteOption; import com.google.gcloud.storage.Storage.BucketSourceOption; @@ -221,7 +221,7 @@ public Page list(Storage.BlobListOption... options) { * @param options blob search options * @throws StorageException upon failure */ - public Blob get(String blob, BlobSourceOption... options) { + public Blob get(String blob, BlobGetOption... options) { return new Blob(storage, storage.get(BlobId.of(info.name(), blob), options)); } diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Storage.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Storage.java index 2fd4df0cc6ba..4438342b853f 100644 --- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Storage.java +++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Storage.java @@ -19,9 +19,11 @@ import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkNotNull; +import com.google.common.base.Joiner; import com.google.common.collect.ImmutableList; import com.google.common.collect.Iterables; import com.google.common.collect.Lists; +import com.google.common.collect.Sets; import com.google.gcloud.AuthCredentials.ServiceAccountAuthCredentials; import com.google.gcloud.Service; import com.google.gcloud.Page; @@ -33,6 +35,7 @@ import java.net.URL; import java.util.Arrays; import java.util.Collections; +import java.util.HashSet; import java.util.LinkedHashSet; import java.util.LinkedList; import java.util.List; @@ -70,6 +73,209 @@ String entry() { } } + public static abstract class EntityField { + + private final String selector; + + public EntityField(String selector) { + this.selector = selector; + } + + public String selector() { + return selector; + } + } + + public static class AclField extends EntityField { + + public AclField(String selector) { + super(selector); + } + + public static AclField bucket() { + return new AclField("bucket"); + } + + public static AclField domain() { + return new AclField("domain"); + } + + public static AclField email() { + return new AclField("email"); + } + + public static AclField entity() { + return new AclField("entity"); + } + + public static AclField entityId() { + return new AclField("entityId"); + } + + public static AclField etag() { + return new AclField("etag"); + } + + public static AclField generation() { + return new AclField("generation"); + } + + public static AclField id() { + return new AclField("id"); + } + + public static AclField kind() { + return new AclField("kind"); + } + + public static AclField object() { + return new AclField("object"); + } + + public static AclField projectTeam() { + return new AclField("projectTeam"); + } + + public static AclField role() { + return new AclField("role"); + } + + public static AclField selfLink() { + return new AclField("selfLink"); + } + + static String selector(AclField... aclFields) { + HashSet fieldStrings = Sets.newHashSetWithExpectedSize(aclFields.length + 2); + fieldStrings.add(AclField.role().selector()); + fieldStrings.add(AclField.entity().selector()); + for (AclField field : aclFields) { + fieldStrings.add(field.selector()); + } + return new StringBuffer() + .append("acl(") + .append(Joiner.on(",").join(fieldStrings)) + .append(")") + .toString(); + } + } + + public static class BlobField extends EntityField { + + public BlobField(String selector) { + super(selector); + } + + public static BlobField acl(AclField... aclFields) { + return new BlobField(AclField.selector(aclFields)); + } + + public static BlobField bucket() { + return new BlobField("bucket"); + } + + public static BlobField cacheControl() { + return new BlobField("cacheControl"); + } + + public static BlobField componentCount() { + return new BlobField("componentCount"); + } + + public static BlobField contentDisposition() { + return new BlobField("contentDisposition"); + } + + public static BlobField contentEncoding() { + return new BlobField("contentEncoding"); + } + + public static BlobField contentLanguage() { + return new BlobField("contentLanguage"); + } + + public static BlobField contentType() { + return new BlobField("contentType"); + } + + public static BlobField crc32c() { + return new BlobField("crc32c"); + } + + public static BlobField etag() { + return new BlobField("etag"); + } + + public static BlobField generation() { + return new BlobField("generation"); + } + + public static BlobField id() { + return new BlobField("id"); + } + + public static BlobField kind() { + return new BlobField("kind"); + } + + public static BlobField md5Hash() { + return new BlobField("md5Hash"); + } + + public static BlobField mediaLink() { + return new BlobField("mediaLink"); + } + + public static BlobField metadata() { + return new BlobField("metadata"); + } + + public static BlobField metageneration() { + return new BlobField("metageneration"); + } + + public static BlobField name() { + return new BlobField("name"); + } + + public static BlobField owner() { + return new BlobField("owner"); + } + + public static BlobField selfLink() { + return new BlobField("selfLink"); + } + + public static BlobField size() { + return new BlobField("size"); + } + + public static BlobField storageClass() { + return new BlobField("storageClass"); + } + + public static BlobField timeCreated() { + return new BlobField("timeCreated"); + } + + public static BlobField timeDeleted() { + return new BlobField("timeDeleted"); + } + + public static BlobField updated() { + return new BlobField("updated"); + } + + static String selector(BlobField... fields) { + HashSet fieldStrings = Sets.newHashSetWithExpectedSize(fields.length + 2); + fieldStrings.add(bucket().selector()); + fieldStrings.add(name().selector()); + for (BlobField field : fields) { + fieldStrings.add(field.selector()); + } + return Joiner.on(',').join(fieldStrings); + } + } + class BucketTargetOption extends Option { private static final long serialVersionUID = -5880204616982900975L; @@ -277,6 +483,45 @@ public static BlobSourceOption metagenerationNotMatch(long metageneration) { } } + class BlobGetOption extends Option { + + private static final long serialVersionUID = 803817709703661480L; + + private BlobGetOption(StorageRpc.Option rpcOption, long value) { + super(rpcOption, value); + } + + private BlobGetOption(StorageRpc.Option rpcOption, String value) { + super(rpcOption, value); + } + + public static BlobGetOption generationMatch(long generation) { + return new BlobGetOption(StorageRpc.Option.IF_GENERATION_MATCH, generation); + } + + public static BlobGetOption generationNotMatch(long generation) { + return new BlobGetOption(StorageRpc.Option.IF_GENERATION_NOT_MATCH, generation); + } + + public static BlobGetOption metagenerationMatch(long metageneration) { + return new BlobGetOption(StorageRpc.Option.IF_METAGENERATION_MATCH, metageneration); + } + + public static BlobGetOption metagenerationNotMatch(long metageneration) { + return new BlobGetOption(StorageRpc.Option.IF_METAGENERATION_NOT_MATCH, metageneration); + } + + /** + * Returns an option to specify the blob's fields to be returned by the RPC call. If this option + * is not provided all blob's fields are returned. {@code BlobGetOption.fields}) can be used to + * specify only the fields of interest. Blob name and bucket are always returned, even if not + * specified. + */ + public static BlobGetOption fields(BlobField... fields) { + return new BlobGetOption(StorageRpc.Option.FIELDS, BlobField.selector(fields)); + } + } + class BucketListOption extends Option { private static final long serialVersionUID = 8754017079673290353L; @@ -321,6 +566,18 @@ public static BlobListOption prefix(String prefix) { public static BlobListOption recursive(boolean recursive) { return new BlobListOption(StorageRpc.Option.DELIMITER, recursive); } + + /** + * Returns an option to specify the blob's fields to be returned by the RPC call. If this option + * is not provided all blob's fields are returned. {@code BlobListOption.fields}) can be used to + * specify only the fields of interest. Blob name and bucket are always returned, even if not + * specified. + */ + public static BlobListOption fields(BlobField... fields) { + StringBuilder builder = new StringBuilder(); + builder.append("items(").append(BlobField.selector(fields)).append(")"); + return new BlobListOption(StorageRpc.Option.FIELDS, builder.toString()); + } } class SignUrlOption implements Serializable { @@ -807,14 +1064,14 @@ private static void checkContentType(BlobInfo blobInfo) throws IllegalArgumentEx * * @throws StorageException upon failure */ - BlobInfo get(String bucket, String blob, BlobSourceOption... options); + BlobInfo get(String bucket, String blob, BlobGetOption... options); /** * Return the requested blob or {@code null} if not found. * * @throws StorageException upon failure */ - BlobInfo get(BlobId blob, BlobSourceOption... options); + BlobInfo get(BlobId blob, BlobGetOption... options); /** * Return the requested blob or {@code null} if not found. diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageImpl.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageImpl.java index d1e85368fecb..4d1a2ba09957 100644 --- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageImpl.java +++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageImpl.java @@ -192,12 +192,12 @@ public com.google.api.services.storage.model.Bucket call() { } @Override - public BlobInfo get(String bucket, String blob, BlobSourceOption... options) { + public BlobInfo get(String bucket, String blob, BlobGetOption... options) { return get(BlobId.of(bucket, blob), options); } @Override - public BlobInfo get(BlobId blob, BlobSourceOption... options) { + public BlobInfo get(BlobId blob, BlobGetOption... options) { final StorageObject storedObject = blob.toPb(); final Map optionsMap = optionMap(options); try { @@ -222,7 +222,7 @@ public StorageObject call() { @Override public BlobInfo get(BlobId blob) { - return get(blob, new BlobSourceOption[0]); + return get(blob, new BlobGetOption[0]); } private abstract static class BasePageFetcher @@ -510,7 +510,7 @@ public BatchResponse apply(BatchRequest batchRequest) { } List>> toGet = Lists.newArrayListWithCapacity(batchRequest.toGet().size()); - for (Map.Entry> entry : batchRequest.toGet().entrySet()) { + for (Map.Entry> entry : batchRequest.toGet().entrySet()) { BlobId blob = entry.getKey(); Map optionsMap = optionMap(null, null, entry.getValue()); toGet.add(Tuple.>of(blob.toPb(), optionsMap)); diff --git a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BatchRequestTest.java b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BatchRequestTest.java index 06b1105d7b9b..600c8af0d554 100644 --- a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BatchRequestTest.java +++ b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BatchRequestTest.java @@ -23,6 +23,7 @@ import static org.junit.Assert.assertTrue; import com.google.common.collect.Iterables; +import com.google.gcloud.storage.Storage.BlobGetOption; import com.google.gcloud.storage.Storage.BlobSourceOption; import com.google.gcloud.storage.Storage.BlobTargetOption; @@ -42,7 +43,7 @@ public void testBatchRequest() { .update(BlobInfo.builder("b2", "o1").build(), BlobTargetOption.predefinedAcl(PUBLIC_READ)) .update(BlobInfo.builder("b2", "o2").build()) .get("b3", "o1") - .get("b3", "o2", BlobSourceOption.generationMatch(1)) + .get("b3", "o2", BlobGetOption.generationMatch(1)) .get("b3", "o3") .build(); @@ -68,16 +69,14 @@ public void testBatchRequest() { assertTrue(Iterables.isEmpty(update.getValue())); assertFalse(updates.hasNext()); - Iterator>> gets = request - .toGet().entrySet().iterator(); - Entry> get = gets.next(); + Iterator>> gets = request.toGet().entrySet().iterator(); + Entry> get = gets.next(); assertEquals(BlobId.of("b3", "o1"), get.getKey()); assertTrue(Iterables.isEmpty(get.getValue())); get = gets.next(); assertEquals(BlobId.of("b3", "o2"), get.getKey()); assertEquals(1, Iterables.size(get.getValue())); - assertEquals(BlobSourceOption.generationMatch(1), - Iterables.getFirst(get.getValue(), null)); + assertEquals(BlobGetOption.generationMatch(1), Iterables.getFirst(get.getValue(), null)); get = gets.next(); assertEquals(BlobId.of("b3", "o3"), get.getKey()); assertTrue(Iterables.isEmpty(get.getValue())); diff --git a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BlobTest.java b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BlobTest.java index defb1d35e3f4..02e325716c8b 100644 --- a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BlobTest.java +++ b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BlobTest.java @@ -72,14 +72,16 @@ public void testInfo() throws Exception { @Test public void testExists_True() throws Exception { - expect(storage.get(BLOB_INFO.blobId(), new Storage.BlobSourceOption[0])).andReturn(BLOB_INFO); + Storage.BlobGetOption[] expectedOptions = {Storage.BlobGetOption.fields()}; + expect(storage.get(BLOB_INFO.blobId(), expectedOptions)).andReturn(BLOB_INFO); replay(storage); assertTrue(blob.exists()); } @Test public void testExists_False() throws Exception { - expect(storage.get(BLOB_INFO.blobId(), new Storage.BlobSourceOption[0])).andReturn(null); + Storage.BlobGetOption[] expectedOptions = {Storage.BlobGetOption.fields()}; + expect(storage.get(BLOB_INFO.blobId(), expectedOptions)).andReturn(null); replay(storage); assertFalse(blob.exists()); } @@ -95,7 +97,7 @@ public void testContent() throws Exception { @Test public void testReload() throws Exception { BlobInfo updatedInfo = BLOB_INFO.toBuilder().cacheControl("c").build(); - expect(storage.get(BLOB_INFO.blobId(), new Storage.BlobSourceOption[0])).andReturn(updatedInfo); + expect(storage.get(BLOB_INFO.blobId(), new Storage.BlobGetOption[0])).andReturn(updatedInfo); replay(storage); Blob updatedBlob = blob.reload(); assertSame(storage, blob.storage()); diff --git a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BucketTest.java b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BucketTest.java index decac7b1e0d2..99c3e6a8737b 100644 --- a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BucketTest.java +++ b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BucketTest.java @@ -137,7 +137,7 @@ public void testList() throws Exception { @Test public void testGet() throws Exception { BlobInfo info = BlobInfo.builder("b", "n").build(); - expect(storage.get(BlobId.of(bucket.info().name(), "n"), new Storage.BlobSourceOption[0])) + expect(storage.get(BlobId.of(bucket.info().name(), "n"), new Storage.BlobGetOption[0])) .andReturn(info); replay(storage); Blob blob = bucket.get("n"); diff --git a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/ITStorageTest.java b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/ITStorageTest.java index d15e07dfeff7..ad003cb5f887 100644 --- a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/ITStorageTest.java +++ b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/ITStorageTest.java @@ -160,6 +160,87 @@ public void testCreateBlobMd5Fail() throws UnsupportedEncodingException { } } + @Test + public void testGetBlobEmptySelectedFields() { + String blobName = "test-get-empty-selected-fields-blob"; + BlobInfo blob = BlobInfo.builder(BUCKET, blobName).contentType(CONTENT_TYPE).build(); + assertNotNull(storage.create(blob)); + BlobInfo remoteBlob = storage.get(blob.blobId(), Storage.BlobGetOption.fields()); + assertEquals(blob.blobId(), remoteBlob.blobId()); + assertNull(remoteBlob.contentType()); + assertTrue(storage.delete(BUCKET, blobName)); + } + + @Test + public void testGetBlobSelectedFields() { + String blobName = "test-get-selected-fields-blob"; + BlobInfo blob = BlobInfo.builder(BUCKET, blobName) + .contentType(CONTENT_TYPE) + .metadata(ImmutableMap.of("k", "v")) + .build(); + assertNotNull(storage.create(blob)); + BlobInfo remoteBlob = storage.get(blob.blobId(), Storage.BlobGetOption.fields( + Storage.BlobField.metadata())); + assertEquals(blob.blobId(), remoteBlob.blobId()); + assertEquals(ImmutableMap.of("k", "v"), remoteBlob.metadata()); + assertNull(remoteBlob.contentType()); + assertTrue(storage.delete(BUCKET, blobName)); + } + + @Test + public void testListBlobsSelectedFields() { + String[] blobNames = {"test-list-blobs-selected-fields-blob1", + "test-list-blobs-selected-fields-blob2"}; + ImmutableMap metadata = ImmutableMap.of("k", "v"); + BlobInfo blob1 = BlobInfo.builder(BUCKET, blobNames[0]) + .contentType(CONTENT_TYPE) + .metadata(metadata) + .build(); + BlobInfo blob2 = BlobInfo.builder(BUCKET, blobNames[1]) + .contentType(CONTENT_TYPE) + .metadata(metadata) + .build(); + assertNotNull(storage.create(blob1)); + assertNotNull(storage.create(blob2)); + ListResult result = storage.list(BUCKET, + Storage.BlobListOption.prefix("test-list-blobs-selected-fields-blob"), + Storage.BlobListOption.fields(Storage.BlobField.metadata())); + int index = 0; + for (BlobInfo remoteBlob : result) { + assertEquals(BUCKET, remoteBlob.bucket()); + assertEquals(blobNames[index++], remoteBlob.name()); + assertEquals(metadata, remoteBlob.metadata()); + assertNull(remoteBlob.contentType()); + } + assertTrue(storage.delete(BUCKET, blobNames[0])); + assertTrue(storage.delete(BUCKET, blobNames[1])); + } + + @Test + public void testListBlobsEmptySelectedFields() { + String[] blobNames = {"test-list-blobs-empty-selected-fields-blob1", + "test-list-blobs-empty-selected-fields-blob2"}; + BlobInfo blob1 = BlobInfo.builder(BUCKET, blobNames[0]) + .contentType(CONTENT_TYPE) + .build(); + BlobInfo blob2 = BlobInfo.builder(BUCKET, blobNames[1]) + .contentType(CONTENT_TYPE) + .build(); + assertNotNull(storage.create(blob1)); + assertNotNull(storage.create(blob2)); + ListResult result = storage.list(BUCKET, + Storage.BlobListOption.prefix("test-list-blobs-empty-selected-fields-blob"), + Storage.BlobListOption.fields()); + int index = 0; + for (BlobInfo remoteBlob : result) { + assertEquals(BUCKET, remoteBlob.bucket()); + assertEquals(blobNames[index++], remoteBlob.name()); + assertNull(remoteBlob.contentType()); + } + assertTrue(storage.delete(BUCKET, blobNames[0])); + assertTrue(storage.delete(BUCKET, blobNames[1])); + } + @Test public void testUpdateBlob() { String blobName = "test-update-blob"; @@ -442,7 +523,7 @@ public void testBatchRequestFail() { BatchRequest batchRequest = BatchRequest.builder() .update(updatedBlob, Storage.BlobTargetOption.generationMatch()) .delete(BUCKET, blobName, Storage.BlobSourceOption.generationMatch(-1L)) - .get(BUCKET, blobName, Storage.BlobSourceOption.generationMatch(-1L)) + .get(BUCKET, blobName, Storage.BlobGetOption.generationMatch(-1L)) .build(); BatchResponse updateResponse = storage.apply(batchRequest); assertEquals(1, updateResponse.updates().size()); diff --git a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/StorageImplTest.java b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/StorageImplTest.java index 6c77b4fcfc99..9abb96c63948 100644 --- a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/StorageImplTest.java +++ b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/StorageImplTest.java @@ -144,7 +144,18 @@ public class StorageImplTest { private static final Map BUCKET_SOURCE_OPTIONS = ImmutableMap.of( StorageRpc.Option.IF_METAGENERATION_MATCH, BUCKET_SOURCE_METAGENERATION.value()); - // Blob source options + // Blob read/source options + private static final Storage.BlobGetOption BLOB_GET_METAGENERATION = + Storage.BlobGetOption.metagenerationMatch(BLOB_INFO1.metageneration()); + private static final Storage.BlobGetOption BLOB_GET_GENERATION = + Storage.BlobGetOption.generationMatch(BLOB_INFO1.generation()); + private static final Storage.BlobGetOption BLOB_GET_FIELDS = + Storage.BlobGetOption.fields(Storage.BlobField.contentType(), Storage.BlobField.crc32c()); + private static final Storage.BlobGetOption BLOB_GET_EMPTY_FIELDS = + Storage.BlobGetOption.fields(); + private static final Map BLOB_GET_OPTIONS = ImmutableMap.of( + StorageRpc.Option.IF_METAGENERATION_MATCH, BLOB_GET_METAGENERATION.value(), + StorageRpc.Option.IF_GENERATION_MATCH, BLOB_GET_GENERATION.value()); private static final Storage.BlobSourceOption BLOB_SOURCE_METAGENERATION = Storage.BlobSourceOption.metagenerationMatch(BLOB_INFO1.metageneration()); private static final Storage.BlobSourceOption BLOB_SOURCE_GENERATION = @@ -170,6 +181,10 @@ public class StorageImplTest { Storage.BlobListOption.maxResults(42L); private static final Storage.BlobListOption BLOB_LIST_PREFIX = Storage.BlobListOption.prefix("prefix"); + private static final Storage.BlobListOption BLOB_LIST_FIELDS = + Storage.BlobListOption.fields(Storage.BlobField.contentType(), Storage.BlobField.md5Hash()); + private static final Storage.BlobListOption BLOB_LIST_EMPTY_FIELDS = + Storage.BlobListOption.fields(); private static final Map BLOB_LIST_OPTIONS = ImmutableMap.of( StorageRpc.Option.MAX_RESULTS, BLOB_LIST_MAX_RESULT.value(), StorageRpc.Option.PREFIX, BLOB_LIST_PREFIX.value()); @@ -383,12 +398,56 @@ public void testGetBlob() { @Test public void testGetBlobWithOptions() { EasyMock.expect( - storageRpcMock.get(BlobId.of(BUCKET_NAME1, BLOB_NAME1).toPb(), BLOB_SOURCE_OPTIONS)) + storageRpcMock.get(BlobId.of(BUCKET_NAME1, BLOB_NAME1).toPb(), BLOB_GET_OPTIONS)) .andReturn(BLOB_INFO1.toPb()); EasyMock.replay(storageRpcMock); storage = options.service(); BlobInfo blob = - storage.get(BUCKET_NAME1, BLOB_NAME1, BLOB_SOURCE_METAGENERATION, BLOB_SOURCE_GENERATION); + storage.get(BUCKET_NAME1, BLOB_NAME1, BLOB_GET_METAGENERATION, BLOB_GET_GENERATION); + assertEquals(BLOB_INFO1, blob); + } + + @Test + public void testGetWithSelectedFields() { + Capture> capturedOptions = + Capture.>newInstance(); + EasyMock.expect(storageRpcMock.get(EasyMock.eq(BlobId.of(BUCKET_NAME1, BLOB_NAME1).toPb()), + EasyMock.capture(capturedOptions))).andReturn(BLOB_INFO1.toPb()); + EasyMock.replay(storageRpcMock); + storage = options.service(); + BlobInfo blob = storage.get(BUCKET_NAME1, BLOB_NAME1, BLOB_GET_METAGENERATION, + BLOB_GET_GENERATION, BLOB_GET_FIELDS); + assertEquals(BLOB_GET_METAGENERATION.value(), + capturedOptions.getValue().get(BLOB_GET_METAGENERATION.rpcOption())); + assertEquals(BLOB_GET_GENERATION.value(), + capturedOptions.getValue().get(BLOB_GET_GENERATION.rpcOption())); + String selector = (String) capturedOptions.getValue().get(BLOB_GET_FIELDS.rpcOption()); + assertTrue(selector.contains("bucket")); + assertTrue(selector.contains("name")); + assertTrue(selector.contains("contentType")); + assertTrue(selector.contains("crc32c")); + assertEquals(30, selector.length()); + assertEquals(BLOB_INFO1, blob); + } + + @Test + public void testGetWithEmptyFields() { + Capture> capturedOptions = + Capture.>newInstance(); + EasyMock.expect(storageRpcMock.get(EasyMock.eq(BlobId.of(BUCKET_NAME1, BLOB_NAME1).toPb()), + EasyMock.capture(capturedOptions))).andReturn(BLOB_INFO1.toPb()); + EasyMock.replay(storageRpcMock); + storage = options.service(); + BlobInfo blob = storage.get(BUCKET_NAME1, BLOB_NAME1, BLOB_GET_METAGENERATION, + BLOB_GET_GENERATION, BLOB_GET_EMPTY_FIELDS); + assertEquals(BLOB_GET_METAGENERATION.value(), + capturedOptions.getValue().get(BLOB_GET_METAGENERATION.rpcOption())); + assertEquals(BLOB_GET_GENERATION.value(), + capturedOptions.getValue().get(BLOB_GET_GENERATION.rpcOption())); + String selector = (String) capturedOptions.getValue().get(BLOB_GET_FIELDS.rpcOption()); + assertTrue(selector.contains("bucket")); + assertTrue(selector.contains("name")); + assertEquals(11, selector.length()); assertEquals(BLOB_INFO1, blob); } @@ -473,6 +532,62 @@ public void testListBlobsWithOptions() { assertArrayEquals(blobList.toArray(), Iterables.toArray(page.values(), BlobInfo.class)); } + @Test + public void testListBlobsWithSelectedFields() { + String cursor = "cursor"; + Capture> capturedOptions = + Capture.>newInstance(); + ImmutableList blobList = ImmutableList.of(BLOB_INFO1, BLOB_INFO2); + Tuple> result = + Tuple.of(cursor, Iterables.transform(blobList, BlobInfo.TO_PB_FUNCTION)); + EasyMock.expect( + storageRpcMock.list(EasyMock.eq(BUCKET_NAME1), EasyMock.capture(capturedOptions))) + .andReturn(result); + EasyMock.replay(storageRpcMock); + storage = options.service(); + ListResult listResult = + storage.list(BUCKET_NAME1, BLOB_LIST_MAX_RESULT, BLOB_LIST_PREFIX, BLOB_LIST_FIELDS); + assertEquals(BLOB_LIST_MAX_RESULT.value(), + capturedOptions.getValue().get(BLOB_LIST_MAX_RESULT.rpcOption())); + assertEquals(BLOB_LIST_PREFIX.value(), + capturedOptions.getValue().get(BLOB_LIST_PREFIX.rpcOption())); + String selector = (String) capturedOptions.getValue().get(BLOB_LIST_FIELDS.rpcOption()); + assertTrue(selector.contains("bucket")); + assertTrue(selector.contains("name")); + assertTrue(selector.contains("contentType")); + assertTrue(selector.contains("md5Hash")); + assertEquals(38, selector.length()); + assertEquals(cursor, listResult.nextPageCursor()); + assertArrayEquals(blobList.toArray(), Iterables.toArray(listResult, BlobInfo.class)); + } + + @Test + public void testListBlobsWithEmptyFields() { + String cursor = "cursor"; + Capture> capturedOptions = + Capture.>newInstance(); + ImmutableList blobList = ImmutableList.of(BLOB_INFO1, BLOB_INFO2); + Tuple> result = + Tuple.of(cursor, Iterables.transform(blobList, BlobInfo.TO_PB_FUNCTION)); + EasyMock.expect( + storageRpcMock.list(EasyMock.eq(BUCKET_NAME1), EasyMock.capture(capturedOptions))) + .andReturn(result); + EasyMock.replay(storageRpcMock); + storage = options.service(); + ListResult listResult = + storage.list(BUCKET_NAME1, BLOB_LIST_MAX_RESULT, BLOB_LIST_PREFIX, BLOB_LIST_EMPTY_FIELDS); + assertEquals(BLOB_LIST_MAX_RESULT.value(), + capturedOptions.getValue().get(BLOB_LIST_MAX_RESULT.rpcOption())); + assertEquals(BLOB_LIST_PREFIX.value(), + capturedOptions.getValue().get(BLOB_LIST_PREFIX.rpcOption())); + String selector = (String) capturedOptions.getValue().get(BLOB_LIST_EMPTY_FIELDS.rpcOption()); + assertTrue(selector.contains("bucket")); + assertTrue(selector.contains("name")); + assertEquals(18, selector.length()); + assertEquals(cursor, listResult.nextPageCursor()); + assertArrayEquals(blobList.toArray(), Iterables.toArray(listResult, BlobInfo.class)); + } + @Test public void testUpdateBucket() { BucketInfo updatedBucketInfo = BUCKET_INFO1.toBuilder().indexPage("some-page").build(); From bea01573f3aec62ff7c96867467089eebf872975 Mon Sep 17 00:00:00 2001 From: Marco Ziccardi Date: Tue, 3 Nov 2015 16:22:23 +0100 Subject: [PATCH 2/7] Add support for selected fields to bucket get and list - Remove AclField and make BlobField an enum type - Add BucketField enum to Storage - Add BucketGetOption class and use if for storage.get(bucket) - Add BucketSourceOption class to Bucket - Updated and add tests --- .../java/com/google/gcloud/storage/Blob.java | 21 +- .../com/google/gcloud/storage/Bucket.java | 67 ++++- .../com/google/gcloud/storage/Storage.java | 268 +++++++----------- .../google/gcloud/storage/StorageImpl.java | 2 +- .../google/gcloud/storage/ITStorageTest.java | 4 +- .../gcloud/storage/StorageImplTest.java | 111 +++++++- 6 files changed, 282 insertions(+), 191 deletions(-) diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Blob.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Blob.java index 88f25c7e5cbd..d35fcef026c8 100644 --- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Blob.java +++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Blob.java @@ -18,7 +18,7 @@ import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkNotNull; -import static com.google.gcloud.storage.Blob.BlobSourceOption.convert; +import static com.google.gcloud.storage.Blob.BlobSourceOption.toSourceOptions; import static com.google.gcloud.storage.Blob.BlobSourceOption.toGetOptions; import com.google.common.base.Function; @@ -58,7 +58,7 @@ private BlobSourceOption(StorageRpc.Option rpcOption) { super(rpcOption, null); } - private Storage.BlobSourceOption convert(BlobInfo blobInfo) { + private Storage.BlobSourceOption toSourceOptions(BlobInfo blobInfo) { switch (rpcOption()) { case IF_GENERATION_MATCH: return Storage.BlobSourceOption.generationMatch(blobInfo.generation()); @@ -104,11 +104,12 @@ public static BlobSourceOption metagenerationNotMatch() { return new BlobSourceOption(StorageRpc.Option.IF_METAGENERATION_NOT_MATCH); } - static Storage.BlobSourceOption[] convert(BlobInfo blobInfo, BlobSourceOption... options) { + static Storage.BlobSourceOption[] toSourceOptions(BlobInfo blobInfo, + BlobSourceOption... options) { Storage.BlobSourceOption[] convertedOptions = new Storage.BlobSourceOption[options.length]; int index = 0; for (BlobSourceOption option : options) { - convertedOptions[index++] = option.convert(blobInfo); + convertedOptions[index++] = option.toSourceOptions(blobInfo); } return convertedOptions; } @@ -126,7 +127,7 @@ static Storage.BlobGetOption[] toGetOptions(BlobInfo blobInfo, BlobSourceOption. /** * Constructs a {@code Blob} object for the provided {@code BlobInfo}. The storage service is used * to issue requests. - * + * * @param storage the storage service used for issuing requests * @param info blob's info */ @@ -138,7 +139,7 @@ public Blob(Storage storage, BlobInfo info) { /** * Creates a {@code Blob} object for the provided bucket and blob names. Performs an RPC call to * get the latest blob information. - * + * * @param storage the storage service used for issuing requests * @param bucket bucket's name * @param blob blob's name @@ -152,7 +153,7 @@ public static Blob load(Storage storage, String bucket, String blob) { /** * Creates a {@code Blob} object for the provided {@code blobId}. Performs an RPC call to get the * latest blob information. - * + * * @param storage the storage service used for issuing requests * @param blobId blob's identifier * @return the {@code Blob} object or {@code null} if not found. @@ -253,7 +254,7 @@ public Blob update(BlobInfo blobInfo, BlobTargetOption... options) { */ public CopyWriter copyTo(BlobId targetBlob, BlobSourceOption... options) { CopyRequest copyRequest = CopyRequest.builder().source(info.bucket(), info.name()) - .sourceOptions(convert(info, options)).target(targetBlob).build(); + .sourceOptions(toSourceOptions(info, options)).target(targetBlob).build(); return storage.copy(copyRequest); } @@ -265,7 +266,7 @@ public CopyWriter copyTo(BlobId targetBlob, BlobSourceOption... options) { * @throws StorageException upon failure */ public boolean delete(BlobSourceOption... options) { - return storage.delete(info.blobId(), convert(info, options)); + return storage.delete(info.blobId(), toSourceOptions(info, options)); } /** @@ -304,7 +305,7 @@ public CopyWriter copyTo(String targetBucket, String targetBlob, BlobSourceOptio * @throws StorageException upon failure */ public BlobReadChannel reader(BlobSourceOption... options) { - return storage.reader(info.blobId(), convert(info, options)); + return storage.reader(info.blobId(), toSourceOptions(info, options)); } /** diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Bucket.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Bucket.java index 3b659f0cd355..4f2b5731e47a 100644 --- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Bucket.java +++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Bucket.java @@ -24,10 +24,10 @@ import com.google.common.collect.Iterators; import com.google.gcloud.PageImpl; import com.google.gcloud.Page; +import com.google.gcloud.spi.StorageRpc; import com.google.gcloud.storage.Storage.BlobGetOption; import com.google.gcloud.storage.Storage.BlobTargetOption; import com.google.gcloud.storage.Storage.BlobWriteOption; -import com.google.gcloud.storage.Storage.BucketSourceOption; import com.google.gcloud.storage.Storage.BucketTargetOption; import java.io.IOException; @@ -119,6 +119,66 @@ public boolean equals(Object obj) { } } + public static class BucketSourceOption extends Option { + + private static final long serialVersionUID = 6928872234155522371L; + + private BucketSourceOption(StorageRpc.Option rpcOption) { + super(rpcOption, null); + } + + private Storage.BucketSourceOption toSourceOptions(BucketInfo bucketInfo) { + switch (rpcOption()) { + case IF_METAGENERATION_MATCH: + return Storage.BucketSourceOption.metagenerationMatch(bucketInfo.metageneration()); + case IF_METAGENERATION_NOT_MATCH: + return Storage.BucketSourceOption.metagenerationNotMatch(bucketInfo.metageneration()); + default: + throw new AssertionError("Unexpected enum value"); + } + } + + private Storage.BucketGetOption toGetOption(BucketInfo bucketInfo) { + switch (rpcOption()) { + case IF_METAGENERATION_MATCH: + return Storage.BucketGetOption.metagenerationMatch(bucketInfo.metageneration()); + case IF_METAGENERATION_NOT_MATCH: + return Storage.BucketGetOption.metagenerationNotMatch(bucketInfo.metageneration()); + default: + throw new AssertionError("Unexpected enum value"); + } + } + + public static BucketSourceOption metagenerationMatch() { + return new BucketSourceOption(StorageRpc.Option.IF_METAGENERATION_MATCH); + } + + public static BucketSourceOption metagenerationNotMatch() { + return new BucketSourceOption(StorageRpc.Option.IF_METAGENERATION_NOT_MATCH); + } + + static Storage.BucketSourceOption[] toSourceOptions(BucketInfo bucketInfo, + BucketSourceOption... options) { + Storage.BucketSourceOption[] convertedOptions = + new Storage.BucketSourceOption[options.length]; + int index = 0; + for (BucketSourceOption option : options) { + convertedOptions[index++] = option.toSourceOptions(bucketInfo); + } + return convertedOptions; + } + + static Storage.BucketGetOption[] toGetOptions(BucketInfo bucketInfo, + BucketSourceOption... options) { + Storage.BucketGetOption[] convertedOptions = new Storage.BucketGetOption[options.length]; + int index = 0; + for (BucketSourceOption option : options) { + convertedOptions[index++] = option.toGetOption(bucketInfo); + } + return convertedOptions; + } + } + /** * Constructs a {@code Bucket} object for the provided {@code BucketInfo}. The storage service is * used to issue requests. @@ -170,7 +230,8 @@ public boolean exists() { * @throws StorageException upon failure */ public Bucket reload(BucketSourceOption... options) { - return new Bucket(storage, storage.get(info.name(), options)); + return new Bucket(storage, storage.get(info.name(), + BucketSourceOption.toGetOptions(info, options))); } /** @@ -198,7 +259,7 @@ public Bucket update(BucketInfo bucketInfo, BucketTargetOption... options) { * @throws StorageException upon failure */ public boolean delete(BucketSourceOption... options) { - return storage.delete(info.name(), options); + return storage.delete(info.name(), BucketSourceOption.toSourceOptions(info, options)); } /** diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Storage.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Storage.java index 4438342b853f..04e3a6bc687f 100644 --- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Storage.java +++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Storage.java @@ -86,189 +86,84 @@ public String selector() { } } - public static class AclField extends EntityField { + enum BucketField { + ID("id"), + SELF_LINK("selfLink"), + NAME("name"), + TIME_CREATED("timeCreated"), + UPDATED("updated"), + METAGENERATION("metageneration"), + ACL("acl"), + DEFAULT_OBJECT_ACL("defaultObjectAcl"), + OWNER("owner"), + LOCATION("location"), + WEBSITE("website"), + VERSIONING("versioning"), + CORS("cors"), + STORAGE_CLASS("storageClass"), + ETAG("etag"); - public AclField(String selector) { - super(selector); - } - - public static AclField bucket() { - return new AclField("bucket"); - } - - public static AclField domain() { - return new AclField("domain"); - } - - public static AclField email() { - return new AclField("email"); - } - - public static AclField entity() { - return new AclField("entity"); - } - - public static AclField entityId() { - return new AclField("entityId"); - } - - public static AclField etag() { - return new AclField("etag"); - } - - public static AclField generation() { - return new AclField("generation"); - } - - public static AclField id() { - return new AclField("id"); - } - - public static AclField kind() { - return new AclField("kind"); - } - - public static AclField object() { - return new AclField("object"); - } - - public static AclField projectTeam() { - return new AclField("projectTeam"); - } + private final String selector; - public static AclField role() { - return new AclField("role"); + BucketField(String selector) { + this.selector = selector; } - public static AclField selfLink() { - return new AclField("selfLink"); + public String selector() { + return selector; } - static String selector(AclField... aclFields) { - HashSet fieldStrings = Sets.newHashSetWithExpectedSize(aclFields.length + 2); - fieldStrings.add(AclField.role().selector()); - fieldStrings.add(AclField.entity().selector()); - for (AclField field : aclFields) { + static String selector(BucketField... fields) { + HashSet fieldStrings = Sets.newHashSetWithExpectedSize(fields.length + 2); + fieldStrings.add(NAME.selector()); + for (BucketField field : fields) { fieldStrings.add(field.selector()); } - return new StringBuffer() - .append("acl(") - .append(Joiner.on(",").join(fieldStrings)) - .append(")") - .toString(); + return Joiner.on(',').join(fieldStrings); } } - public static class BlobField extends EntityField { - - public BlobField(String selector) { - super(selector); - } - - public static BlobField acl(AclField... aclFields) { - return new BlobField(AclField.selector(aclFields)); - } - - public static BlobField bucket() { - return new BlobField("bucket"); - } - - public static BlobField cacheControl() { - return new BlobField("cacheControl"); - } - - public static BlobField componentCount() { - return new BlobField("componentCount"); - } - - public static BlobField contentDisposition() { - return new BlobField("contentDisposition"); - } - - public static BlobField contentEncoding() { - return new BlobField("contentEncoding"); - } - - public static BlobField contentLanguage() { - return new BlobField("contentLanguage"); - } - - public static BlobField contentType() { - return new BlobField("contentType"); - } - - public static BlobField crc32c() { - return new BlobField("crc32c"); - } - - public static BlobField etag() { - return new BlobField("etag"); - } - - public static BlobField generation() { - return new BlobField("generation"); - } - - public static BlobField id() { - return new BlobField("id"); - } - - public static BlobField kind() { - return new BlobField("kind"); - } - - public static BlobField md5Hash() { - return new BlobField("md5Hash"); - } - - public static BlobField mediaLink() { - return new BlobField("mediaLink"); - } - - public static BlobField metadata() { - return new BlobField("metadata"); - } - - public static BlobField metageneration() { - return new BlobField("metageneration"); - } + enum BlobField { + ACL("acl"), + BUCKET("bucket"), + CACHE_CONTROL("cacheControl"), + COMPONENT_COUNT("componentCount"), + CONTENT_DISPOSITION("contentDisposition"), + CONTENT_ENCODING("contentEncoding"), + CONTENT_LANGUAGE("contentLanguage"), + CONTENT_TYPE("contentType"), + CRC32C("crc32c"), + ETAG("etag"), + GENERATION("generation"), + ID("id"), + KIND("kind"), + MD5HASH("md5Hash"), + MEDIA_LINK("mediaLink"), + METADATA("metadata"), + METAGENERATION("metageneration"), + NAME("name"), + OWNER("owner"), + SELF_LINK("selfLink"), + SIZE("size"), + STORAGE_CLASS("storageClass"), + TIME_CREATED("timeCreated"), + TIME_DELETED("timeDeleted"), + UPDATED("updated"); - public static BlobField name() { - return new BlobField("name"); - } - - public static BlobField owner() { - return new BlobField("owner"); - } - - public static BlobField selfLink() { - return new BlobField("selfLink"); - } - - public static BlobField size() { - return new BlobField("size"); - } - - public static BlobField storageClass() { - return new BlobField("storageClass"); - } - - public static BlobField timeCreated() { - return new BlobField("timeCreated"); - } + private final String selector; - public static BlobField timeDeleted() { - return new BlobField("timeDeleted"); + BlobField(String selector) { + this.selector = selector; } - public static BlobField updated() { - return new BlobField("updated"); + public String selector() { + return selector; } static String selector(BlobField... fields) { HashSet fieldStrings = Sets.newHashSetWithExpectedSize(fields.length + 2); - fieldStrings.add(bucket().selector()); - fieldStrings.add(name().selector()); + fieldStrings.add(BUCKET.selector()); + fieldStrings.add(NAME.selector()); for (BlobField field : fields) { fieldStrings.add(field.selector()); } @@ -322,6 +217,37 @@ public static BucketSourceOption metagenerationNotMatch(long metageneration) { } } + class BucketGetOption extends Option { + + private static final long serialVersionUID = 1901844869484087395L; + + private BucketGetOption(StorageRpc.Option rpcOption, long metageneration) { + super(rpcOption, metageneration); + } + + private BucketGetOption(StorageRpc.Option rpcOption, String value) { + super(rpcOption, value); + } + + public static BucketGetOption metagenerationMatch(long metageneration) { + return new BucketGetOption(StorageRpc.Option.IF_METAGENERATION_MATCH, metageneration); + } + + public static BucketGetOption metagenerationNotMatch(long metageneration) { + return new BucketGetOption(StorageRpc.Option.IF_METAGENERATION_NOT_MATCH, metageneration); + } + + /** + * Returns an option to specify the bucket's fields to be returned by the RPC call. If this + * option is not provided all bucket's fields are returned. {@code BucketGetOption.fields}) can + * be used to specify only the fields of interest. Bucket name is always returned, even if not + * specified. + */ + public static BucketGetOption fields(BucketField... fields) { + return new BucketGetOption(StorageRpc.Option.FIELDS, BucketField.selector(fields)); + } + } + class BlobTargetOption extends Option { private static final long serialVersionUID = 214616862061934846L; @@ -541,6 +467,18 @@ public static BucketListOption startPageToken(String pageToken) { public static BucketListOption prefix(String prefix) { return new BucketListOption(StorageRpc.Option.PREFIX, prefix); } + + /** + * Returns an option to specify the bucket's fields to be returned by the RPC call. If this + * option is not provided all bucket's fields are returned. {@code BucketListOption.fields}) can + * be used to specify only the fields of interest. Bucket name is always returned, even if not + * specified. + */ + public static BucketListOption fields(BucketField... fields) { + StringBuilder builder = new StringBuilder(); + builder.append("items(").append(BucketField.selector(fields)).append(")"); + return new BucketListOption(StorageRpc.Option.FIELDS, builder.toString()); + } } class BlobListOption extends Option { @@ -1057,7 +995,7 @@ private static void checkContentType(BlobInfo blobInfo) throws IllegalArgumentEx * * @throws StorageException upon failure */ - BucketInfo get(String bucket, BucketSourceOption... options); + BucketInfo get(String bucket, BucketGetOption... options); /** * Return the requested blob or {@code null} if not found. diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageImpl.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageImpl.java index 4d1a2ba09957..4c85113e940e 100644 --- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageImpl.java +++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageImpl.java @@ -167,7 +167,7 @@ public StorageObject call() { } @Override - public BucketInfo get(String bucket, BucketSourceOption... options) { + public BucketInfo get(String bucket, BucketGetOption... options) { final com.google.api.services.storage.model.Bucket bucketPb = BucketInfo.of(bucket).toPb(); final Map optionsMap = optionMap(options); try { diff --git a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/ITStorageTest.java b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/ITStorageTest.java index ad003cb5f887..8a9a747f948d 100644 --- a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/ITStorageTest.java +++ b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/ITStorageTest.java @@ -180,7 +180,7 @@ public void testGetBlobSelectedFields() { .build(); assertNotNull(storage.create(blob)); BlobInfo remoteBlob = storage.get(blob.blobId(), Storage.BlobGetOption.fields( - Storage.BlobField.metadata())); + Storage.BlobField.METADATA)); assertEquals(blob.blobId(), remoteBlob.blobId()); assertEquals(ImmutableMap.of("k", "v"), remoteBlob.metadata()); assertNull(remoteBlob.contentType()); @@ -204,7 +204,7 @@ public void testListBlobsSelectedFields() { assertNotNull(storage.create(blob2)); ListResult result = storage.list(BUCKET, Storage.BlobListOption.prefix("test-list-blobs-selected-fields-blob"), - Storage.BlobListOption.fields(Storage.BlobField.metadata())); + Storage.BlobListOption.fields(Storage.BlobField.METADATA)); int index = 0; for (BlobInfo remoteBlob : result) { assertEquals(BUCKET, remoteBlob.bucket()); diff --git a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/StorageImplTest.java b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/StorageImplTest.java index 9abb96c63948..f5c8cff1a335 100644 --- a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/StorageImplTest.java +++ b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/StorageImplTest.java @@ -138,19 +138,27 @@ public class StorageImplTest { private static final Storage.BlobWriteOption BLOB_WRITE_CRC2C = Storage.BlobWriteOption.crc32cMatch(); - // Bucket source options + // Bucket get/source options private static final Storage.BucketSourceOption BUCKET_SOURCE_METAGENERATION = Storage.BucketSourceOption.metagenerationMatch(BUCKET_INFO1.metageneration()); private static final Map BUCKET_SOURCE_OPTIONS = ImmutableMap.of( StorageRpc.Option.IF_METAGENERATION_MATCH, BUCKET_SOURCE_METAGENERATION.value()); + private static final Storage.BucketGetOption BUCKET_GET_METAGENERATION = + Storage.BucketGetOption.metagenerationMatch(BUCKET_INFO1.metageneration()); + private static final Storage.BucketGetOption BUCKET_GET_FIELDS = + Storage.BucketGetOption.fields(Storage.BucketField.LOCATION, Storage.BucketField.ACL); + private static final Storage.BucketGetOption BUCKET_GET_EMPTY_FIELDS = + Storage.BucketGetOption.fields(); + private static final Map BUCKET_GET_OPTIONS = ImmutableMap.of( + StorageRpc.Option.IF_METAGENERATION_MATCH, BUCKET_SOURCE_METAGENERATION.value()); - // Blob read/source options + // Blob get/source options private static final Storage.BlobGetOption BLOB_GET_METAGENERATION = Storage.BlobGetOption.metagenerationMatch(BLOB_INFO1.metageneration()); private static final Storage.BlobGetOption BLOB_GET_GENERATION = Storage.BlobGetOption.generationMatch(BLOB_INFO1.generation()); private static final Storage.BlobGetOption BLOB_GET_FIELDS = - Storage.BlobGetOption.fields(Storage.BlobField.contentType(), Storage.BlobField.crc32c()); + Storage.BlobGetOption.fields(Storage.BlobField.CONTENT_TYPE, Storage.BlobField.CRC32C); private static final Storage.BlobGetOption BLOB_GET_EMPTY_FIELDS = Storage.BlobGetOption.fields(); private static final Map BLOB_GET_OPTIONS = ImmutableMap.of( @@ -172,6 +180,10 @@ public class StorageImplTest { Storage.BucketListOption.maxResults(42L); private static final Storage.BucketListOption BUCKET_LIST_PREFIX = Storage.BucketListOption.prefix("prefix"); + private static final Storage.BucketListOption BUCKET_LIST_FIELDS = + Storage.BucketListOption.fields(Storage.BucketField.LOCATION, Storage.BucketField.ACL); + private static final Storage.BucketListOption BUCKET_LIST_EMPTY_FIELDS = + Storage.BucketListOption.fields(); private static final Map BUCKET_LIST_OPTIONS = ImmutableMap.of( StorageRpc.Option.MAX_RESULTS, BUCKET_LIST_MAX_RESULT.value(), StorageRpc.Option.PREFIX, BUCKET_LIST_PREFIX.value()); @@ -182,7 +194,7 @@ public class StorageImplTest { private static final Storage.BlobListOption BLOB_LIST_PREFIX = Storage.BlobListOption.prefix("prefix"); private static final Storage.BlobListOption BLOB_LIST_FIELDS = - Storage.BlobListOption.fields(Storage.BlobField.contentType(), Storage.BlobField.md5Hash()); + Storage.BlobListOption.fields(Storage.BlobField.CONTENT_TYPE, Storage.BlobField.MD5HASH); private static final Storage.BlobListOption BLOB_LIST_EMPTY_FIELDS = Storage.BlobListOption.fields(); private static final Map BLOB_LIST_OPTIONS = ImmutableMap.of( @@ -374,16 +386,51 @@ public void testGetBucket() { @Test public void testGetBucketWithOptions() { - EasyMock.expect(storageRpcMock.get(BucketInfo.of(BUCKET_NAME1).toPb(), BUCKET_SOURCE_OPTIONS)) + EasyMock.expect(storageRpcMock.get(BucketInfo.of(BUCKET_NAME1).toPb(), BUCKET_GET_OPTIONS)) .andReturn(BUCKET_INFO1.toPb()); EasyMock.replay(storageRpcMock); storage = options.service(); - BucketInfo bucket = - storage.get(BUCKET_NAME1, - Storage.BucketSourceOption.metagenerationMatch(BUCKET_INFO1.metageneration())); + BucketInfo bucket = storage.get(BUCKET_NAME1, BUCKET_GET_METAGENERATION); assertEquals(BUCKET_INFO1, bucket); } + @Test + public void testGetBucketWithSelectedFields() { + Capture> capturedOptions = + Capture.>newInstance(); + EasyMock.expect(storageRpcMock.get(EasyMock.eq(BucketInfo.of(BUCKET_NAME1).toPb()), + EasyMock.capture(capturedOptions))).andReturn(BUCKET_INFO1.toPb()); + EasyMock.replay(storageRpcMock); + storage = options.service(); + BucketInfo bucket = storage.get(BUCKET_NAME1, BUCKET_GET_METAGENERATION, BUCKET_GET_FIELDS); + assertEquals(BUCKET_GET_METAGENERATION.value(), + capturedOptions.getValue().get(BUCKET_GET_METAGENERATION.rpcOption())); + String selector = (String) capturedOptions.getValue().get(BLOB_GET_FIELDS.rpcOption()); + assertTrue(selector.contains("name")); + assertTrue(selector.contains("location")); + assertTrue(selector.contains("acl")); + assertEquals(17, selector.length()); + assertEquals(BUCKET_INFO1.name(), bucket.name()); + } + + @Test + public void testGetBucketWithEmptyFields() { + Capture> capturedOptions = + Capture.>newInstance(); + EasyMock.expect(storageRpcMock.get(EasyMock.eq(BucketInfo.of(BUCKET_NAME1).toPb()), + EasyMock.capture(capturedOptions))).andReturn(BUCKET_INFO1.toPb()); + EasyMock.replay(storageRpcMock); + storage = options.service(); + BucketInfo bucket = storage.get(BUCKET_NAME1, BUCKET_GET_METAGENERATION, + BUCKET_GET_EMPTY_FIELDS); + assertEquals(BUCKET_GET_METAGENERATION.value(), + capturedOptions.getValue().get(BUCKET_GET_METAGENERATION.rpcOption())); + String selector = (String) capturedOptions.getValue().get(BLOB_GET_FIELDS.rpcOption()); + assertTrue(selector.contains("name")); + assertEquals(4, selector.length()); + assertEquals(BUCKET_INFO1.name(), bucket.name()); + } + @Test public void testGetBlob() { EasyMock.expect( @@ -408,7 +455,7 @@ public void testGetBlobWithOptions() { } @Test - public void testGetWithSelectedFields() { + public void testGetBlobWithSelectedFields() { Capture> capturedOptions = Capture.>newInstance(); EasyMock.expect(storageRpcMock.get(EasyMock.eq(BlobId.of(BUCKET_NAME1, BLOB_NAME1).toPb()), @@ -431,7 +478,7 @@ public void testGetWithSelectedFields() { } @Test - public void testGetWithEmptyFields() { + public void testGetBlobWithEmptyFields() { Capture> capturedOptions = Capture.>newInstance(); EasyMock.expect(storageRpcMock.get(EasyMock.eq(BlobId.of(BUCKET_NAME1, BLOB_NAME1).toPb()), @@ -491,6 +538,48 @@ public void testListBucketsWithOptions() { assertArrayEquals(bucketList.toArray(), Iterables.toArray(page.values(), BucketInfo.class)); } + @Test + public void testListBucketsWithSelectedFields() { + String cursor = "cursor"; + Capture> capturedOptions = + Capture.>newInstance(); + ImmutableList bucketList = ImmutableList.of(BUCKET_INFO1, BUCKET_INFO2); + Tuple> result = + Tuple.of(cursor, Iterables.transform(bucketList, BucketInfo.TO_PB_FUNCTION)); + EasyMock.expect(storageRpcMock.list(EasyMock.capture(capturedOptions))).andReturn(result); + EasyMock.replay(storageRpcMock); + storage = options.service(); + ListResult listResult = storage.list(BUCKET_LIST_FIELDS); + String selector = (String) capturedOptions.getValue().get(BLOB_LIST_FIELDS.rpcOption()); + assertTrue(selector.contains("items")); + assertTrue(selector.contains("name")); + assertTrue(selector.contains("acl")); + assertTrue(selector.contains("location")); + assertEquals(24, selector.length()); + assertEquals(cursor, listResult.nextPageCursor()); + assertArrayEquals(bucketList.toArray(), Iterables.toArray(listResult, BucketInfo.class)); + } + + @Test + public void testListBucketsWithEmptyFields() { + String cursor = "cursor"; + Capture> capturedOptions = + Capture.>newInstance(); + ImmutableList bucketList = ImmutableList.of(BUCKET_INFO1, BUCKET_INFO2); + Tuple> result = + Tuple.of(cursor, Iterables.transform(bucketList, BucketInfo.TO_PB_FUNCTION)); + EasyMock.expect(storageRpcMock.list(EasyMock.capture(capturedOptions))).andReturn(result); + EasyMock.replay(storageRpcMock); + storage = options.service(); + ListResult listResult = storage.list(BUCKET_LIST_EMPTY_FIELDS); + String selector = (String) capturedOptions.getValue().get(BLOB_LIST_FIELDS.rpcOption()); + assertTrue(selector.contains("items")); + assertTrue(selector.contains("name")); + assertEquals(11, selector.length()); + assertEquals(cursor, listResult.nextPageCursor()); + assertArrayEquals(bucketList.toArray(), Iterables.toArray(listResult, BucketInfo.class)); + } + @Test public void testListBlobs() { String cursor = "cursor"; @@ -552,6 +641,7 @@ public void testListBlobsWithSelectedFields() { assertEquals(BLOB_LIST_PREFIX.value(), capturedOptions.getValue().get(BLOB_LIST_PREFIX.rpcOption())); String selector = (String) capturedOptions.getValue().get(BLOB_LIST_FIELDS.rpcOption()); + assertTrue(selector.contains("items")); assertTrue(selector.contains("bucket")); assertTrue(selector.contains("name")); assertTrue(selector.contains("contentType")); @@ -581,6 +671,7 @@ public void testListBlobsWithEmptyFields() { assertEquals(BLOB_LIST_PREFIX.value(), capturedOptions.getValue().get(BLOB_LIST_PREFIX.rpcOption())); String selector = (String) capturedOptions.getValue().get(BLOB_LIST_EMPTY_FIELDS.rpcOption()); + assertTrue(selector.contains("items")); assertTrue(selector.contains("bucket")); assertTrue(selector.contains("name")); assertEquals(18, selector.length()); From f11c05a4846d598984fee5c132493d962fdf92e0 Mon Sep 17 00:00:00 2001 From: Marco Ziccardi Date: Wed, 4 Nov 2015 08:47:27 +0100 Subject: [PATCH 3/7] Add field selection to StorageRpc bucket get and list --- .../src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java b/gcloud-java-storage/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java index 29552d5121e5..b1e188f1d1fb 100644 --- a/gcloud-java-storage/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java +++ b/gcloud-java-storage/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java @@ -151,6 +151,7 @@ public Tuple> list(Map options) { .setPrefix(PREFIX.getString(options)) .setMaxResults(MAX_RESULTS.getLong(options)) .setPageToken(PAGE_TOKEN.getString(options)) + .setFields(FIELDS.getString(options)) .execute(); return Tuple.>of(buckets.getNextPageToken(), buckets.getItems()); } catch (IOException ex) { @@ -186,6 +187,7 @@ public Bucket get(Bucket bucket, Map options) { .setProjection(DEFAULT_PROJECTION) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) + .setFields(FIELDS.getString(options)) .execute(); } catch (IOException ex) { throw translate(ex); From ba4bfa6017b39800abfda831ee071664802e6186 Mon Sep 17 00:00:00 2001 From: Marco Ziccardi Date: Wed, 4 Nov 2015 08:49:43 +0100 Subject: [PATCH 4/7] Add field selection to Bucket.exist --- .../java/com/google/gcloud/storage/Bucket.java | 15 ++++++++++----- .../com/google/gcloud/storage/BucketTest.java | 6 ++++-- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Bucket.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Bucket.java index 4f2b5731e47a..7c386373995f 100644 --- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Bucket.java +++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Bucket.java @@ -18,6 +18,8 @@ import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkNotNull; +import static com.google.gcloud.storage.Bucket.BucketSourceOption.toGetOptions; +import static com.google.gcloud.storage.Bucket.BucketSourceOption.toSourceOptions; import com.google.common.base.Function; import com.google.common.base.MoreObjects; @@ -35,6 +37,7 @@ import java.io.ObjectInputStream; import java.io.Serializable; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.Iterator; import java.util.List; @@ -218,8 +221,11 @@ public BucketInfo info() { * @return true if this bucket exists, false otherwise * @throws StorageException upon failure */ - public boolean exists() { - return storage.get(info.name()) != null; + public boolean exists(BucketSourceOption... options) { + int length = options.length; + Storage.BucketGetOption[] getOptions = Arrays.copyOf(toGetOptions(info, options), length + 1); + getOptions[length] = Storage.BucketGetOption.fields(); + return storage.get(info.name(), getOptions) != null; } /** @@ -230,8 +236,7 @@ public boolean exists() { * @throws StorageException upon failure */ public Bucket reload(BucketSourceOption... options) { - return new Bucket(storage, storage.get(info.name(), - BucketSourceOption.toGetOptions(info, options))); + return new Bucket(storage, storage.get(info.name(), toGetOptions(info, options))); } /** @@ -259,7 +264,7 @@ public Bucket update(BucketInfo bucketInfo, BucketTargetOption... options) { * @throws StorageException upon failure */ public boolean delete(BucketSourceOption... options) { - return storage.delete(info.name(), BucketSourceOption.toSourceOptions(info, options)); + return storage.delete(info.name(), toSourceOptions(info, options)); } /** diff --git a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BucketTest.java b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BucketTest.java index 99c3e6a8737b..5d8fc5a9dffc 100644 --- a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BucketTest.java +++ b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BucketTest.java @@ -75,14 +75,16 @@ public void testInfo() throws Exception { @Test public void testExists_True() throws Exception { - expect(storage.get(BUCKET_INFO.name())).andReturn(BUCKET_INFO); + Storage.BucketGetOption[] expectedOptions = {Storage.BucketGetOption.fields()}; + expect(storage.get(BUCKET_INFO.name(), expectedOptions)).andReturn(BUCKET_INFO); replay(storage); assertTrue(bucket.exists()); } @Test public void testExists_False() throws Exception { - expect(storage.get(BUCKET_INFO.name())).andReturn(null); + Storage.BucketGetOption[] expectedOptions = {Storage.BucketGetOption.fields()}; + expect(storage.get(BUCKET_INFO.name(), expectedOptions)).andReturn(null); replay(storage); assertFalse(bucket.exists()); } From 46b60df4a132b669bb8752bc7bf02bd5f24c57d1 Mon Sep 17 00:00:00 2001 From: Marco Ziccardi Date: Wed, 4 Nov 2015 08:51:40 +0100 Subject: [PATCH 5/7] Delete EntityField class and remove non-accessible fields from Bucket/BlobField --- .../java/com/google/gcloud/storage/Storage.java | 17 +---------------- 1 file changed, 1 insertion(+), 16 deletions(-) diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Storage.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Storage.java index 04e3a6bc687f..52387310de71 100644 --- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Storage.java +++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Storage.java @@ -73,25 +73,11 @@ String entry() { } } - public static abstract class EntityField { - - private final String selector; - - public EntityField(String selector) { - this.selector = selector; - } - - public String selector() { - return selector; - } - } - enum BucketField { ID("id"), SELF_LINK("selfLink"), NAME("name"), TIME_CREATED("timeCreated"), - UPDATED("updated"), METAGENERATION("metageneration"), ACL("acl"), DEFAULT_OBJECT_ACL("defaultObjectAcl"), @@ -114,7 +100,7 @@ public String selector() { } static String selector(BucketField... fields) { - HashSet fieldStrings = Sets.newHashSetWithExpectedSize(fields.length + 2); + HashSet fieldStrings = Sets.newHashSetWithExpectedSize(fields.length + 1); fieldStrings.add(NAME.selector()); for (BucketField field : fields) { fieldStrings.add(field.selector()); @@ -146,7 +132,6 @@ enum BlobField { SELF_LINK("selfLink"), SIZE("size"), STORAGE_CLASS("storageClass"), - TIME_CREATED("timeCreated"), TIME_DELETED("timeDeleted"), UPDATED("updated"); From 603d50c0bddbc5c816ced52211cdfb6ae0e55a64 Mon Sep 17 00:00:00 2001 From: Marco Ziccardi Date: Wed, 4 Nov 2015 09:02:47 +0100 Subject: [PATCH 6/7] Add IT tests for bucket's field selection, add test for selecting all fields --- .../google/gcloud/storage/ITStorageTest.java | 77 ++++++++++++++++--- .../gcloud/storage/StorageImplTest.java | 24 +++--- 2 files changed, 79 insertions(+), 22 deletions(-) diff --git a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/ITStorageTest.java b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/ITStorageTest.java index 8a9a747f948d..659c3d94ad4d 100644 --- a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/ITStorageTest.java +++ b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/ITStorageTest.java @@ -26,7 +26,10 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import com.google.gcloud.Page; import com.google.gcloud.RestorableState; +import com.google.gcloud.storage.Storage.BlobField; +import com.google.gcloud.storage.Storage.BucketField; import com.google.gcloud.storage.testing.RemoteGcsHelper; import org.junit.AfterClass; @@ -80,17 +83,49 @@ public static void afterClass() @Test(timeout = 5000) public void testListBuckets() throws InterruptedException { - Iterator bucketIterator = - storage.list(Storage.BucketListOption.prefix(BUCKET)).values().iterator(); + Iterator bucketIterator = storage.list(Storage.BucketListOption.prefix(BUCKET), + Storage.BucketListOption.fields()).values().iterator(); while (!bucketIterator.hasNext()) { Thread.sleep(500); - bucketIterator = storage.list(Storage.BucketListOption.prefix(BUCKET)).values().iterator(); + bucketIterator = storage.list(Storage.BucketListOption.prefix(BUCKET), + Storage.BucketListOption.fields()).values().iterator(); } while (bucketIterator.hasNext()) { - assertTrue(bucketIterator.next().name().startsWith(BUCKET)); + BucketInfo remoteBucket = bucketIterator.next(); + assertTrue(remoteBucket.name().startsWith(BUCKET)); + assertNull(remoteBucket.createTime()); + assertNull(remoteBucket.selfLink()); } } + @Test + public void testGetBucketSelectedFields() { + BucketInfo remoteBucket = storage.get(BUCKET, Storage.BucketGetOption.fields(BucketField.ID)); + assertEquals(BUCKET, remoteBucket.name()); + assertNull(remoteBucket.createTime()); + assertNotNull(remoteBucket.id()); + } + + @Test + public void testGetBucketAllSelectedFields() { + BucketInfo remoteBucket = storage.get(BUCKET, Storage.BucketGetOption.fields(BucketField.ACL, + BucketField.CORS, BucketField.DEFAULT_OBJECT_ACL, BucketField.ETAG, BucketField.ID, + BucketField.LOCATION, BucketField.METAGENERATION, BucketField.NAME, BucketField.OWNER, + BucketField.SELF_LINK, BucketField.STORAGE_CLASS, BucketField.TIME_CREATED, + BucketField.VERSIONING, BucketField.WEBSITE)); + assertEquals(BUCKET, remoteBucket.name()); + assertNotNull(remoteBucket.createTime()); + assertNotNull(remoteBucket.selfLink()); + } + + @Test + public void testGetBucketEmptyFields() { + BucketInfo remoteBucket = storage.get(BUCKET, Storage.BucketGetOption.fields()); + assertEquals(BUCKET, remoteBucket.name()); + assertNull(remoteBucket.createTime()); + assertNull(remoteBucket.selfLink()); + } + @Test public void testCreateBlob() { String blobName = "test-create-blob"; @@ -180,13 +215,35 @@ public void testGetBlobSelectedFields() { .build(); assertNotNull(storage.create(blob)); BlobInfo remoteBlob = storage.get(blob.blobId(), Storage.BlobGetOption.fields( - Storage.BlobField.METADATA)); + BlobField.METADATA)); assertEquals(blob.blobId(), remoteBlob.blobId()); assertEquals(ImmutableMap.of("k", "v"), remoteBlob.metadata()); assertNull(remoteBlob.contentType()); assertTrue(storage.delete(BUCKET, blobName)); } + @Test + public void testGetBlobAllSelectedFields() { + String blobName = "test-get-all-selected-fields-blob"; + BlobInfo blob = BlobInfo.builder(BUCKET, blobName) + .contentType(CONTENT_TYPE) + .metadata(ImmutableMap.of("k", "v")) + .build(); + assertNotNull(storage.create(blob)); + BlobInfo remoteBlob = storage.get(blob.blobId(), Storage.BlobGetOption.fields( + BlobField.ACL, BlobField.BUCKET, BlobField.CACHE_CONTROL, BlobField.COMPONENT_COUNT, + BlobField.CONTENT_DISPOSITION, BlobField.CONTENT_ENCODING, BlobField.CONTENT_LANGUAGE, + BlobField.CONTENT_TYPE, BlobField.CRC32C, BlobField.ETAG, BlobField.GENERATION, + BlobField.ID, BlobField.KIND, BlobField.MD5HASH, BlobField.MEDIA_LINK, BlobField.METADATA, + BlobField.METAGENERATION, BlobField.NAME, BlobField.OWNER, BlobField.SELF_LINK, + BlobField.SIZE, BlobField.STORAGE_CLASS, BlobField.TIME_DELETED, BlobField.UPDATED)); + assertEquals(blob.blobId(), remoteBlob.blobId()); + assertEquals(ImmutableMap.of("k", "v"), remoteBlob.metadata()); + assertNotNull(remoteBlob.id()); + assertNotNull(remoteBlob.selfLink()); + assertTrue(storage.delete(BUCKET, blobName)); + } + @Test public void testListBlobsSelectedFields() { String[] blobNames = {"test-list-blobs-selected-fields-blob1", @@ -202,11 +259,11 @@ public void testListBlobsSelectedFields() { .build(); assertNotNull(storage.create(blob1)); assertNotNull(storage.create(blob2)); - ListResult result = storage.list(BUCKET, + Page page = storage.list(BUCKET, Storage.BlobListOption.prefix("test-list-blobs-selected-fields-blob"), - Storage.BlobListOption.fields(Storage.BlobField.METADATA)); + Storage.BlobListOption.fields(BlobField.METADATA)); int index = 0; - for (BlobInfo remoteBlob : result) { + for (BlobInfo remoteBlob : page.values()) { assertEquals(BUCKET, remoteBlob.bucket()); assertEquals(blobNames[index++], remoteBlob.name()); assertEquals(metadata, remoteBlob.metadata()); @@ -228,11 +285,11 @@ public void testListBlobsEmptySelectedFields() { .build(); assertNotNull(storage.create(blob1)); assertNotNull(storage.create(blob2)); - ListResult result = storage.list(BUCKET, + Page page = storage.list(BUCKET, Storage.BlobListOption.prefix("test-list-blobs-empty-selected-fields-blob"), Storage.BlobListOption.fields()); int index = 0; - for (BlobInfo remoteBlob : result) { + for (BlobInfo remoteBlob : page.values()) { assertEquals(BUCKET, remoteBlob.bucket()); assertEquals(blobNames[index++], remoteBlob.name()); assertNull(remoteBlob.contentType()); diff --git a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/StorageImplTest.java b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/StorageImplTest.java index f5c8cff1a335..d5e2f8de7397 100644 --- a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/StorageImplTest.java +++ b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/StorageImplTest.java @@ -549,15 +549,15 @@ public void testListBucketsWithSelectedFields() { EasyMock.expect(storageRpcMock.list(EasyMock.capture(capturedOptions))).andReturn(result); EasyMock.replay(storageRpcMock); storage = options.service(); - ListResult listResult = storage.list(BUCKET_LIST_FIELDS); + Page page = storage.list(BUCKET_LIST_FIELDS); String selector = (String) capturedOptions.getValue().get(BLOB_LIST_FIELDS.rpcOption()); assertTrue(selector.contains("items")); assertTrue(selector.contains("name")); assertTrue(selector.contains("acl")); assertTrue(selector.contains("location")); assertEquals(24, selector.length()); - assertEquals(cursor, listResult.nextPageCursor()); - assertArrayEquals(bucketList.toArray(), Iterables.toArray(listResult, BucketInfo.class)); + assertEquals(cursor, page.nextPageCursor()); + assertArrayEquals(bucketList.toArray(), Iterables.toArray(page.values(), BucketInfo.class)); } @Test @@ -571,13 +571,13 @@ public void testListBucketsWithEmptyFields() { EasyMock.expect(storageRpcMock.list(EasyMock.capture(capturedOptions))).andReturn(result); EasyMock.replay(storageRpcMock); storage = options.service(); - ListResult listResult = storage.list(BUCKET_LIST_EMPTY_FIELDS); + Page page = storage.list(BUCKET_LIST_EMPTY_FIELDS); String selector = (String) capturedOptions.getValue().get(BLOB_LIST_FIELDS.rpcOption()); assertTrue(selector.contains("items")); assertTrue(selector.contains("name")); assertEquals(11, selector.length()); - assertEquals(cursor, listResult.nextPageCursor()); - assertArrayEquals(bucketList.toArray(), Iterables.toArray(listResult, BucketInfo.class)); + assertEquals(cursor, page.nextPageCursor()); + assertArrayEquals(bucketList.toArray(), Iterables.toArray(page.values(), BucketInfo.class)); } @Test @@ -634,7 +634,7 @@ public void testListBlobsWithSelectedFields() { .andReturn(result); EasyMock.replay(storageRpcMock); storage = options.service(); - ListResult listResult = + Page page = storage.list(BUCKET_NAME1, BLOB_LIST_MAX_RESULT, BLOB_LIST_PREFIX, BLOB_LIST_FIELDS); assertEquals(BLOB_LIST_MAX_RESULT.value(), capturedOptions.getValue().get(BLOB_LIST_MAX_RESULT.rpcOption())); @@ -647,8 +647,8 @@ public void testListBlobsWithSelectedFields() { assertTrue(selector.contains("contentType")); assertTrue(selector.contains("md5Hash")); assertEquals(38, selector.length()); - assertEquals(cursor, listResult.nextPageCursor()); - assertArrayEquals(blobList.toArray(), Iterables.toArray(listResult, BlobInfo.class)); + assertEquals(cursor, page.nextPageCursor()); + assertArrayEquals(blobList.toArray(), Iterables.toArray(page.values(), BlobInfo.class)); } @Test @@ -664,7 +664,7 @@ public void testListBlobsWithEmptyFields() { .andReturn(result); EasyMock.replay(storageRpcMock); storage = options.service(); - ListResult listResult = + Page page = storage.list(BUCKET_NAME1, BLOB_LIST_MAX_RESULT, BLOB_LIST_PREFIX, BLOB_LIST_EMPTY_FIELDS); assertEquals(BLOB_LIST_MAX_RESULT.value(), capturedOptions.getValue().get(BLOB_LIST_MAX_RESULT.rpcOption())); @@ -675,8 +675,8 @@ public void testListBlobsWithEmptyFields() { assertTrue(selector.contains("bucket")); assertTrue(selector.contains("name")); assertEquals(18, selector.length()); - assertEquals(cursor, listResult.nextPageCursor()); - assertArrayEquals(blobList.toArray(), Iterables.toArray(listResult, BlobInfo.class)); + assertEquals(cursor, page.nextPageCursor()); + assertArrayEquals(blobList.toArray(), Iterables.toArray(page.values(), BlobInfo.class)); } @Test From af7a08b4e03506fd830fa8d0fcbed8b7e279e8e1 Mon Sep 17 00:00:00 2001 From: Marco Ziccardi Date: Wed, 4 Nov 2015 23:05:53 +0100 Subject: [PATCH 7/7] Use BlobField and BucketField .values() in IT tests --- .../com/google/gcloud/storage/ITStorageTest.java | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/ITStorageTest.java b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/ITStorageTest.java index 659c3d94ad4d..423e972a8de6 100644 --- a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/ITStorageTest.java +++ b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/ITStorageTest.java @@ -108,11 +108,8 @@ public void testGetBucketSelectedFields() { @Test public void testGetBucketAllSelectedFields() { - BucketInfo remoteBucket = storage.get(BUCKET, Storage.BucketGetOption.fields(BucketField.ACL, - BucketField.CORS, BucketField.DEFAULT_OBJECT_ACL, BucketField.ETAG, BucketField.ID, - BucketField.LOCATION, BucketField.METAGENERATION, BucketField.NAME, BucketField.OWNER, - BucketField.SELF_LINK, BucketField.STORAGE_CLASS, BucketField.TIME_CREATED, - BucketField.VERSIONING, BucketField.WEBSITE)); + BucketInfo remoteBucket = storage.get(BUCKET, + Storage.BucketGetOption.fields(BucketField.values())); assertEquals(BUCKET, remoteBucket.name()); assertNotNull(remoteBucket.createTime()); assertNotNull(remoteBucket.selfLink()); @@ -230,13 +227,8 @@ public void testGetBlobAllSelectedFields() { .metadata(ImmutableMap.of("k", "v")) .build(); assertNotNull(storage.create(blob)); - BlobInfo remoteBlob = storage.get(blob.blobId(), Storage.BlobGetOption.fields( - BlobField.ACL, BlobField.BUCKET, BlobField.CACHE_CONTROL, BlobField.COMPONENT_COUNT, - BlobField.CONTENT_DISPOSITION, BlobField.CONTENT_ENCODING, BlobField.CONTENT_LANGUAGE, - BlobField.CONTENT_TYPE, BlobField.CRC32C, BlobField.ETAG, BlobField.GENERATION, - BlobField.ID, BlobField.KIND, BlobField.MD5HASH, BlobField.MEDIA_LINK, BlobField.METADATA, - BlobField.METAGENERATION, BlobField.NAME, BlobField.OWNER, BlobField.SELF_LINK, - BlobField.SIZE, BlobField.STORAGE_CLASS, BlobField.TIME_DELETED, BlobField.UPDATED)); + BlobInfo remoteBlob = storage.get(blob.blobId(), + Storage.BlobGetOption.fields(BlobField.values())); assertEquals(blob.blobId(), remoteBlob.blobId()); assertEquals(ImmutableMap.of("k", "v"), remoteBlob.metadata()); assertNotNull(remoteBlob.id());