From 0f84dbb67ea104b19245fb535847f9e6fb4effff Mon Sep 17 00:00:00 2001 From: Joe Giardino Date: Tue, 1 May 2012 10:54:18 -0700 Subject: [PATCH] Issue # 55 CloudTableClient should use Iterator instead of Iterable. Signed-off-by: Joe Giardino --- .../services/blob/client/CloudBlobClient.java | 4 +- .../blob/client/CloudBlobContainer.java | 4 +- .../implementation/LazySegmentedIterable.java | 76 +++++++++++++++++++ .../implementation/LazySegmentedIterator.java | 11 +-- .../queue/client/CloudQueueClient.java | 4 +- .../table/client/CloudTableClient.java | 6 +- .../table/client/TableClientTests.java | 22 +++++- .../table/client/TableQueryTests.java | 49 ++++++++++++ 8 files changed, 155 insertions(+), 21 deletions(-) create mode 100644 microsoft-azure-api/src/main/java/com/microsoft/windowsazure/services/core/storage/utils/implementation/LazySegmentedIterable.java diff --git a/microsoft-azure-api/src/main/java/com/microsoft/windowsazure/services/blob/client/CloudBlobClient.java b/microsoft-azure-api/src/main/java/com/microsoft/windowsazure/services/blob/client/CloudBlobClient.java index 7b7ccd4e461f..ed74ed27450c 100644 --- a/microsoft-azure-api/src/main/java/com/microsoft/windowsazure/services/blob/client/CloudBlobClient.java +++ b/microsoft-azure-api/src/main/java/com/microsoft/windowsazure/services/blob/client/CloudBlobClient.java @@ -36,7 +36,7 @@ import com.microsoft.windowsazure.services.core.storage.utils.PathUtility; import com.microsoft.windowsazure.services.core.storage.utils.Utility; import com.microsoft.windowsazure.services.core.storage.utils.implementation.ExecutionEngine; -import com.microsoft.windowsazure.services.core.storage.utils.implementation.LazySegmentedIterator; +import com.microsoft.windowsazure.services.core.storage.utils.implementation.LazySegmentedIterable; import com.microsoft.windowsazure.services.core.storage.utils.implementation.ListingContext; import com.microsoft.windowsazure.services.core.storage.utils.implementation.SegmentedStorageOperation; import com.microsoft.windowsazure.services.core.storage.utils.implementation.StorageOperation; @@ -534,7 +534,7 @@ public ResultSegment execute(final CloudBlobClient client, f } }; - return new LazySegmentedIterator(impl, this, null, + return new LazySegmentedIterable(impl, this, null, options.getRetryPolicyFactory(), opContext); } diff --git a/microsoft-azure-api/src/main/java/com/microsoft/windowsazure/services/blob/client/CloudBlobContainer.java b/microsoft-azure-api/src/main/java/com/microsoft/windowsazure/services/blob/client/CloudBlobContainer.java index 3892a425c148..dffa565ca014 100644 --- a/microsoft-azure-api/src/main/java/com/microsoft/windowsazure/services/blob/client/CloudBlobContainer.java +++ b/microsoft-azure-api/src/main/java/com/microsoft/windowsazure/services/blob/client/CloudBlobContainer.java @@ -42,7 +42,7 @@ import com.microsoft.windowsazure.services.core.storage.utils.UriQueryBuilder; import com.microsoft.windowsazure.services.core.storage.utils.Utility; import com.microsoft.windowsazure.services.core.storage.utils.implementation.ExecutionEngine; -import com.microsoft.windowsazure.services.core.storage.utils.implementation.LazySegmentedIterator; +import com.microsoft.windowsazure.services.core.storage.utils.implementation.LazySegmentedIterable; import com.microsoft.windowsazure.services.core.storage.utils.implementation.SegmentedStorageOperation; import com.microsoft.windowsazure.services.core.storage.utils.implementation.StorageOperation; @@ -1273,7 +1273,7 @@ public ResultSegment execute(final CloudBlobClient client, } }; - return new LazySegmentedIterator(impl, + return new LazySegmentedIterable(impl, this.blobServiceClient, this, options.getRetryPolicyFactory(), opContext); } diff --git a/microsoft-azure-api/src/main/java/com/microsoft/windowsazure/services/core/storage/utils/implementation/LazySegmentedIterable.java b/microsoft-azure-api/src/main/java/com/microsoft/windowsazure/services/core/storage/utils/implementation/LazySegmentedIterable.java new file mode 100644 index 000000000000..d57bb296fe28 --- /dev/null +++ b/microsoft-azure-api/src/main/java/com/microsoft/windowsazure/services/core/storage/utils/implementation/LazySegmentedIterable.java @@ -0,0 +1,76 @@ +/** + * Copyright 2011 Microsoft Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.microsoft.windowsazure.services.core.storage.utils.implementation; + +import java.util.Iterator; + +import com.microsoft.windowsazure.services.core.storage.OperationContext; +import com.microsoft.windowsazure.services.core.storage.ResultSegment; +import com.microsoft.windowsazure.services.core.storage.RetryPolicyFactory; + +/** + * RESERVED FOR INTERNAL USE. Provides a lazy iterator which will retrieve the next segment of a result as the iterator + * is consumed + * + * @param + * The service client type + * @param + * The type of the parent object, i.e. CloudBlobClient for ListContainers etc. + * @param + * The type of the objects the resulting iterable objects + */ +public final class LazySegmentedIterable implements Iterable { + /** + * Holds the service client associated with the operations. + */ + private final CLIENT_TYPE client; + + /** + * Holds a reference to the parent object, i.e. CloudBlobContainer for list blobs. + */ + private final PARENT_TYPE parentObject; + + /** + * Holds the reference to the RetryPolicyFactory object. + */ + private final RetryPolicyFactory policyFactory; + + /** + * Holds the SegmentedStorageOperation which is used to retrieve the next segment of results. + */ + private final SegmentedStorageOperation> segmentGenerator; + + /** + * Holds an object used to track the execution of the operation + */ + private final OperationContext opContext; + + public LazySegmentedIterable( + final SegmentedStorageOperation> segmentGenerator, + final CLIENT_TYPE client, final PARENT_TYPE parent, final RetryPolicyFactory policyFactory, + final OperationContext opContext) { + this.segmentGenerator = segmentGenerator; + this.parentObject = parent; + this.opContext = opContext; + this.policyFactory = policyFactory; + this.client = client; + } + + @Override + public Iterator iterator() { + return new LazySegmentedIterator(this.segmentGenerator, this.client, + this.parentObject, this.policyFactory, this.opContext); + } +} diff --git a/microsoft-azure-api/src/main/java/com/microsoft/windowsazure/services/core/storage/utils/implementation/LazySegmentedIterator.java b/microsoft-azure-api/src/main/java/com/microsoft/windowsazure/services/core/storage/utils/implementation/LazySegmentedIterator.java index b0525448bd38..e1f9f6ab116f 100644 --- a/microsoft-azure-api/src/main/java/com/microsoft/windowsazure/services/core/storage/utils/implementation/LazySegmentedIterator.java +++ b/microsoft-azure-api/src/main/java/com/microsoft/windowsazure/services/core/storage/utils/implementation/LazySegmentedIterator.java @@ -34,8 +34,7 @@ * @param * The type of the objects the resulting iterable objects */ -public final class LazySegmentedIterator implements Iterator, - Iterable { +public final class LazySegmentedIterator implements Iterator { /** * Holds the current segment of results. @@ -126,14 +125,6 @@ public boolean hasNext() { return this.currentSegmentIterator.hasNext(); } - /** - * Gets a reference to the iterator. - */ - @Override - public Iterator iterator() { - return this; - } - /** * Returns the next element. */ diff --git a/microsoft-azure-api/src/main/java/com/microsoft/windowsazure/services/queue/client/CloudQueueClient.java b/microsoft-azure-api/src/main/java/com/microsoft/windowsazure/services/queue/client/CloudQueueClient.java index 8b8fe0e571f2..8971ba912032 100644 --- a/microsoft-azure-api/src/main/java/com/microsoft/windowsazure/services/queue/client/CloudQueueClient.java +++ b/microsoft-azure-api/src/main/java/com/microsoft/windowsazure/services/queue/client/CloudQueueClient.java @@ -36,7 +36,7 @@ import com.microsoft.windowsazure.services.core.storage.StorageException; import com.microsoft.windowsazure.services.core.storage.utils.Utility; import com.microsoft.windowsazure.services.core.storage.utils.implementation.ExecutionEngine; -import com.microsoft.windowsazure.services.core.storage.utils.implementation.LazySegmentedIterator; +import com.microsoft.windowsazure.services.core.storage.utils.implementation.LazySegmentedIterable; import com.microsoft.windowsazure.services.core.storage.utils.implementation.ListingContext; import com.microsoft.windowsazure.services.core.storage.utils.implementation.SegmentedStorageOperation; import com.microsoft.windowsazure.services.core.storage.utils.implementation.StorageOperation; @@ -164,7 +164,7 @@ public ResultSegment execute(final CloudQueueClient client, final Vo } }; - return new LazySegmentedIterator(impl, this, null, + return new LazySegmentedIterable(impl, this, null, options.getRetryPolicyFactory(), opContext); } diff --git a/microsoft-azure-api/src/main/java/com/microsoft/windowsazure/services/table/client/CloudTableClient.java b/microsoft-azure-api/src/main/java/com/microsoft/windowsazure/services/table/client/CloudTableClient.java index b471594261c6..8932790d3c2a 100644 --- a/microsoft-azure-api/src/main/java/com/microsoft/windowsazure/services/table/client/CloudTableClient.java +++ b/microsoft-azure-api/src/main/java/com/microsoft/windowsazure/services/table/client/CloudTableClient.java @@ -39,7 +39,7 @@ import com.microsoft.windowsazure.services.core.storage.StorageException; import com.microsoft.windowsazure.services.core.storage.utils.Utility; import com.microsoft.windowsazure.services.core.storage.utils.implementation.ExecutionEngine; -import com.microsoft.windowsazure.services.core.storage.utils.implementation.LazySegmentedIterator; +import com.microsoft.windowsazure.services.core.storage.utils.implementation.LazySegmentedIterable; import com.microsoft.windowsazure.services.core.storage.utils.implementation.SegmentedStorageOperation; import com.microsoft.windowsazure.services.core.storage.utils.implementation.StorageOperation; @@ -1328,7 +1328,7 @@ public ResultSegment execute(final CloudTableClient client, final TableQuery< } }; - return new LazySegmentedIterator, T>(impl, this, queryRef, + return new LazySegmentedIterable, T>(impl, this, queryRef, options.getRetryPolicyFactory(), opContext); } else { @@ -1352,7 +1352,7 @@ public ResultSegment execute(final CloudTableClient client, final TableQuery< return result; } }; - return new LazySegmentedIterator, R>(impl, this, queryRef, + return new LazySegmentedIterable, R>(impl, this, queryRef, options.getRetryPolicyFactory(), opContext); } } diff --git a/microsoft-azure-api/src/test/java/com/microsoft/windowsazure/services/table/client/TableClientTests.java b/microsoft-azure-api/src/test/java/com/microsoft/windowsazure/services/table/client/TableClientTests.java index 7dc7bd3c4dbd..8060002ca8ae 100644 --- a/microsoft-azure-api/src/test/java/com/microsoft/windowsazure/services/table/client/TableClientTests.java +++ b/microsoft-azure-api/src/test/java/com/microsoft/windowsazure/services/table/client/TableClientTests.java @@ -142,17 +142,35 @@ public void listTablesWithIterator() throws IOException, URISyntaxException, Sto try { // With prefix int currTable = 0; - for (String s : tClient.listTables(tableBaseName, null, null)) { + Iterable listTables = tClient.listTables(tableBaseName, null, null); + for (String s : listTables) { Assert.assertEquals(s, String.format("%s%s", tableBaseName, new DecimalFormat("#0000").format(currTable))); currTable++; } + Assert.assertEquals(20, currTable); + // Second Iteration + currTable = 0; + for (String s : listTables) { + Assert.assertEquals(s, + String.format("%s%s", tableBaseName, new DecimalFormat("#0000").format(currTable))); + currTable++; + } Assert.assertEquals(20, currTable); // Without prefix currTable = 0; - for (String s : tClient.listTables()) { + Iterable listTablesNoPrefix = tClient.listTables(); + for (String s : listTablesNoPrefix) { + if (s.startsWith(tableBaseName)) { + currTable++; + } + } + + Assert.assertEquals(20, currTable); + currTable = 0; + for (String s : listTablesNoPrefix) { if (s.startsWith(tableBaseName)) { currTable++; } diff --git a/microsoft-azure-api/src/test/java/com/microsoft/windowsazure/services/table/client/TableQueryTests.java b/microsoft-azure-api/src/test/java/com/microsoft/windowsazure/services/table/client/TableQueryTests.java index dd46b99bae99..75740c9fff23 100644 --- a/microsoft-azure-api/src/test/java/com/microsoft/windowsazure/services/table/client/TableQueryTests.java +++ b/microsoft-azure-api/src/test/java/com/microsoft/windowsazure/services/table/client/TableQueryTests.java @@ -20,6 +20,7 @@ import java.net.HttpURLConnection; import java.net.URISyntaxException; import java.security.InvalidKeyException; +import java.util.ArrayList; import java.util.Arrays; import java.util.Date; import java.util.HashMap; @@ -59,6 +60,54 @@ public static void setup() throws URISyntaxException, StorageException, InvalidK } } + @Test + public void tableQueryIterateTwice() { + // Create entity to check against + class1 randEnt = TableTestBase.generateRandomEnitity(null); + + final Iterable result = tClient.execute(TableQuery.from(testSuiteTableName, + DynamicTableEntity.class).take(50)); + + ArrayList firstIteration = new ArrayList(); + ArrayList secondIteration = new ArrayList(); + + // Validate results + for (DynamicTableEntity ent : result) { + Assert.assertEquals(ent.getProperties().size(), 4); + Assert.assertEquals(ent.getProperties().get("A").getValueAsString(), randEnt.getA()); + Assert.assertEquals(ent.getProperties().get("B").getValueAsString(), randEnt.getB()); + Assert.assertEquals(ent.getProperties().get("C").getValueAsString(), randEnt.getC()); + Assert.assertTrue(Arrays.equals(ent.getProperties().get("D").getValueAsByteArray(), randEnt.getD())); + firstIteration.add(ent); + } + + // Validate results + for (DynamicTableEntity ent : result) { + Assert.assertEquals(ent.getProperties().size(), 4); + Assert.assertEquals(ent.getProperties().get("A").getValueAsString(), randEnt.getA()); + Assert.assertEquals(ent.getProperties().get("B").getValueAsString(), randEnt.getB()); + Assert.assertEquals(ent.getProperties().get("C").getValueAsString(), randEnt.getC()); + Assert.assertTrue(Arrays.equals(ent.getProperties().get("D").getValueAsByteArray(), randEnt.getD())); + secondIteration.add(ent); + } + + Assert.assertEquals(firstIteration.size(), secondIteration.size()); + for (int m = 0; m < firstIteration.size(); m++) { + Assert.assertEquals(firstIteration.get(m).getPartitionKey(), secondIteration.get(m).getPartitionKey()); + Assert.assertEquals(firstIteration.get(m).getRowKey(), secondIteration.get(m).getRowKey()); + Assert.assertEquals(firstIteration.get(m).getProperties().size(), secondIteration.get(m).getProperties() + .size()); + Assert.assertEquals(firstIteration.get(m).getProperties().get("A").getValueAsString(), + secondIteration.get(m).getProperties().get("A").getValueAsString()); + Assert.assertEquals(firstIteration.get(m).getProperties().get("B").getValueAsString(), + secondIteration.get(m).getProperties().get("B").getValueAsString()); + Assert.assertEquals(firstIteration.get(m).getProperties().get("C").getValueAsString(), + secondIteration.get(m).getProperties().get("C").getValueAsString()); + Assert.assertTrue(Arrays.equals(firstIteration.get(m).getProperties().get("D").getValueAsByteArray(), + secondIteration.get(m).getProperties().get("D").getValueAsByteArray())); + } + } + @Test public void tableQueryWithDynamicEntity() { // Create entity to check against