Skip to content

Commit

Permalink
Add support download latest index metadata from remote
Browse files Browse the repository at this point in the history
Signed-off-by: bansvaru <bansvaru@amazon.com>
  • Loading branch information
linuxpi committed Aug 22, 2023
1 parent f3957ea commit 8791d9e
Show file tree
Hide file tree
Showing 5 changed files with 570 additions and 17 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -343,30 +343,25 @@ private static DeleteObjectsRequest bulkDelete(String bucket, List<String> blobs
}

@Override
public void listBlobsByPrefixInSortedOrder(
public List<BlobMetadata> listBlobsByPrefixInSortedOrder(
String blobNamePrefix,
int limit,
BlobNameSortOrder blobNameSortOrder,
ActionListener<List<BlobMetadata>> listener
) {
// As AWS S3 returns list of keys in Lexicographic order, we don't have to fetch all the keys in order to sort them
// We fetch only keys as per the given limit to optimize the fetch. If provided sort order is not Lexicographic,
// we fall-back to default implementation of fetching all the keys and sorting them.
BlobNameSortOrder blobNameSortOrder
) throws IOException {
if (blobNameSortOrder != BlobNameSortOrder.LEXICOGRAPHIC) {
super.listBlobsByPrefixInSortedOrder(blobNamePrefix, limit, blobNameSortOrder, listener);
return super.listBlobsByPrefixInSortedOrder(blobNamePrefix, limit, blobNameSortOrder);
} else {
if (limit < 0) {
throw new IllegalArgumentException("limit should not be a negative value");
}
String prefix = blobNamePrefix == null ? keyPath : buildKey(blobNamePrefix);
try (AmazonS3Reference clientReference = blobStore.clientReference()) {
List<BlobMetadata> blobs = executeListing(clientReference, listObjectsRequest(prefix, limit), limit).stream()
return executeListing(clientReference, listObjectsRequest(prefix, limit), limit).stream()
.flatMap(listing -> listing.contents().stream())
.map(s3Object -> new PlainBlobMetadata(s3Object.key().substring(keyPath.length()), s3Object.size()))
.collect(Collectors.toList());
listener.onResponse(blobs.subList(0, Math.min(limit, blobs.size())));
} catch (final Exception e) {
listener.onFailure(new IOException("Exception when listing blobs by prefix [" + prefix + "]", e));
throw new IOException("Exception when listing blobs by prefix [" + prefix + "]", e);
}
}
}
Expand Down
Loading

0 comments on commit 8791d9e

Please sign in to comment.