Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Storage] Consolidate offset and range parameters #7598

Merged
merged 27 commits into from
Oct 5, 2019
Merged

[Storage] Consolidate offset and range parameters #7598

Show file tree
Hide file tree
Changes from 25 commits
Commits
Show all changes
27 commits
Select commit Hold shift + click to select a range
039f124
Updated blob sync tests
annatisch Oct 3, 2019
abae738
Offset refactor APIS (#19)
Oct 3, 2019
c0c6ace
Temp block async tests
annatisch Oct 3, 2019
22af611
Fix upload page behaviour
annatisch Oct 4, 2019
c58c400
Fix clear page behaviour
annatisch Oct 4, 2019
5861f03
Update from_url offset behaviour
annatisch Oct 4, 2019
9d8e86e
Update page ranges
annatisch Oct 4, 2019
09525c4
Fix download blob behaviour
annatisch Oct 4, 2019
6501b20
Some cleanup
annatisch Oct 4, 2019
795acbf
Fixed page size
annatisch Oct 4, 2019
2c3e686
More test fixes
annatisch Oct 4, 2019
c484282
Some more fixes
annatisch Oct 4, 2019
2c5e984
Fixed page tests
annatisch Oct 4, 2019
70fb84a
Fixed encryption tests
annatisch Oct 4, 2019
3b20ffc
Fix common test
annatisch Oct 4, 2019
6c1074c
append anf page blob async
rakshith91 Oct 4, 2019
3129af7
some more test fixes
rakshith91 Oct 4, 2019
1715f21
Fix live tests
annatisch Oct 4, 2019
ddb6137
Merge branch 'storage-offset-range' of https://github.com/annatisch/a…
annatisch Oct 4, 2019
b8d7c68
Merge remote-tracking branch 'upstream/feature/storage-preview4' into…
annatisch Oct 4, 2019
2228766
more changes async
rakshith91 Oct 4, 2019
c9d80ee
Merge branch 'storage-offset-range' of https://github.com/annatisch/a…
annatisch Oct 4, 2019
58a84af
Fix sparse blob test
annatisch Oct 4, 2019
d139d0e
Last tests
annatisch Oct 4, 2019
790e7ec
update docstrings
rakshith91 Oct 4, 2019
33be60f
Merge branch 'feature/storage-preview4' into storage-offset-range
rakshith91 Oct 5, 2019
3110969
pylint
rakshith91 Oct 5, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
123 changes: 58 additions & 65 deletions sdk/storage/azure-storage-blob/azure/storage/blob/aio/blob_client_async.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1316,29 +1316,27 @@ async def set_premium_page_blob_tier(self, premium_page_blob_tier, **kwargs):

@distributed_trace_async
async def get_page_ranges( # type: ignore
self, start_range=None, # type: Optional[int]
end_range=None, # type: Optional[int]
self, offset=None, # type: Optional[int]
length=None, # type: Optional[int]
previous_snapshot_diff=None, # type: Optional[Union[str, Dict[str, Any]]]
**kwargs
):
# type: (...) -> List[dict[str, int]]
"""Returns the list of valid page ranges for a Page Blob or snapshot
of a page blob.

:param int start_range:
Start of byte range to use for getting valid page ranges.
If no end_range is given, all bytes after the start_range will be searched.
:param int offset:
Start of byte range to use for downloading a section of the blob.
Must be set if length is provided.
Pages must be aligned with 512-byte boundaries, the start offset
must be a modulus of 512 and the end offset must be a modulus of
512-1. Examples of valid byte ranges are 0-511, 512-, etc.
:param int end_range:
End of byte range to use for getting valid page ranges.
If end_range is given, start_range must be provided.
This range will return valid page ranges for from the offset start up to
offset end.
must be a modulus of 512 and the length must be a modulus of
512.
:param int length:
Number of bytes to use for getting valid page ranges.
If length is given, offset must be provided.
Pages must be aligned with 512-byte boundaries, the start offset
must be a modulus of 512 and the end offset must be a modulus of
512-1. Examples of valid byte ranges are 0-511, 512-, etc.
must be a modulus of 512 and the length must be a modulus of
512.
:param lease:
Required if the blob has an active lease. Value can be a LeaseClient object
or the lease ID as a string.
Expand Down Expand Up @@ -1376,8 +1374,8 @@ async def get_page_ranges( # type: ignore
:rtype: tuple(list(dict(str, str), list(dict(str, str))
"""
options = self._get_page_ranges_options(
start_range=start_range,
end_range=end_range,
offset=offset,
length=length,
previous_snapshot_diff=previous_snapshot_diff,
**kwargs)
try:
Expand Down Expand Up @@ -1495,9 +1493,8 @@ async def resize_blob(self, size, **kwargs):
@distributed_trace_async
async def upload_page( # type: ignore
self, page, # type: bytes
start_range, # type: int
end_range, # type: int
length=None, # type: Optional[int]
offset, # type: int
length, # type: int
validate_content=False, # type: Optional[bool]
**kwargs
):
Expand All @@ -1506,18 +1503,17 @@ async def upload_page( # type: ignore

:param bytes page:
Content of the page.
:param int start_range:
Start of byte range to use for writing to a section of the blob.
Pages must be aligned with 512-byte boundaries, the start offset
must be a modulus of 512 and the end offset must be a modulus of
512-1. Examples of valid byte ranges are 0-511, 512-1023, etc.
:param int end_range:
End of byte range to use for writing to a section of the blob.
:param int offset:
Start of byte range to use for downloading a section of the blob.
Must be set if length is provided.
Pages must be aligned with 512-byte boundaries, the start offset
must be a modulus of 512 and the end offset must be a modulus of
512-1. Examples of valid byte ranges are 0-511, 512-1023, etc.
must be a modulus of 512 and the length must be a modulus of
512.
:param int length:
Length of the page
Number of bytes to use for writing to a section of the blob.
Pages must be aligned with 512-byte boundaries, the start offset
must be a modulus of 512 and the length must be a modulus of
512.
:param lease:
Required if the blob has an active lease. Value can be a LeaseClient object
or the lease ID as a string.
Expand Down Expand Up @@ -1573,8 +1569,7 @@ async def upload_page( # type: ignore
"""
options = self._upload_page_options(
page=page,
start_range=start_range,
end_range=end_range,
offset=offset,
length=length,
validate_content=validate_content,
**kwargs)
Expand All @@ -1585,9 +1580,9 @@ async def upload_page( # type: ignore

@distributed_trace_async
async def upload_pages_from_url(self, source_url, # type: str
range_start, # type: int
range_end, # type: int
source_range_start, # type: int
offset, # type: int
length, # type: int
source_offset, # type: int
source_content_md5=None, # type: Optional[bytes]
**kwargs
):
Expand All @@ -1598,19 +1593,17 @@ async def upload_pages_from_url(self, source_url, # type: str
:param str source_url:
The URL of the source data. It can point to any Azure Blob or File, that is either public or has a
shared access signature attached.
:param int range_start:
Start of byte range to use for writing to a section of the blob.
Pages must be aligned with 512-byte boundaries, the start offset
must be a modulus of 512 and the end offset must be a modulus of
512-1. Examples of valid byte ranges are 0-511, 512-1023, etc.
:param int range_end:
End of byte range to use for writing to a section of the blob.
:param int offset:
Start of byte range to use for downloading a section of the blob.
Must be set if length is provided.
:param int length:
Number of bytes to use for writing to a section of the blob.
Pages must be aligned with 512-byte boundaries, the start offset
must be a modulus of 512 and the end offset must be a modulus of
512-1. Examples of valid byte ranges are 0-511, 512-1023, etc.
:param int source_range_start:
must be a modulus of 512 and the length must be a modulus of
512.
:param int source_offset:
This indicates the start of the range of bytes(inclusive) that has to be taken from the copy source.
The service will read the same number of bytes as the destination range (end_range-start_range).
The service will read the same number of bytes as the destination range (length-offset).
:param bytes source_content_md5:
If given, the service will calculate the MD5 hash of the block content and compare against this value.
:param datetime source_if_modified_since:
Expand Down Expand Up @@ -1677,9 +1670,9 @@ async def upload_pages_from_url(self, source_url, # type: str

options = self._upload_pages_from_url_options(
source_url=source_url,
range_start=range_start,
range_end=range_end,
source_range_start=source_range_start,
offset=offset,
length=length,
source_offset=source_offset,
source_content_md5=source_content_md5,
**kwargs
)
Expand All @@ -1689,20 +1682,20 @@ async def upload_pages_from_url(self, source_url, # type: str
process_storage_error(error)

@distributed_trace_async
async def clear_page(self, start_range, end_range, **kwargs):
async def clear_page(self, offset, length, **kwargs):
# type: (int, int) -> Dict[str, Union[str, datetime]]
"""Clears a range of pages.

:param int start_range:
Start of byte range to use for writing to a section of the blob.
:param int offset:
Start of byte range to use for downloading a section of the blob.
Must be set if length is provided.
Pages must be aligned with 512-byte boundaries, the start offset
must be a modulus of 512 and the end offset must be a modulus of
512-1. Examples of valid byte ranges are 0-511, 512-1023, etc.
:param int end_range:
End of byte range to use for writing to a section of the blob.
must be a modulus of 512.
:param int length:
Number of bytes to use for writing to a section of the blob.
Pages must be aligned with 512-byte boundaries, the start offset
must be a modulus of 512 and the end offset must be a modulus of
512-1. Examples of valid byte ranges are 0-511, 512-1023, etc.
must be a modulus of 512 and the length must be a modulus of
512.
:param lease:
Required if the blob has an active lease. Value can be a LeaseClient object
or the lease ID as a string.
Expand Down Expand Up @@ -1746,7 +1739,7 @@ async def clear_page(self, start_range, end_range, **kwargs):
:returns: Blob-updated property dict (Etag and last modified).
:rtype: dict(str, Any)
"""
options = self._clear_page_options(start_range, end_range, **kwargs)
options = self._clear_page_options(offset, length, **kwargs)
try:
return await self._client.page_blob.clear_pages(**options) # type: ignore
except StorageErrorException as error:
Expand Down Expand Up @@ -1839,8 +1832,8 @@ async def append_block( # type: ignore

@distributed_trace_async()
async def append_block_from_url(self, copy_source_url, # type: str
source_range_start=None, # type Optional[int]
source_range_end=None, # type Optional[int]
source_offset=None, # type Optional[int]
source_length=None, # type Optional[int]
source_content_md5=None, # type: Optional[bytearray]
maxsize_condition=None, # type: Optional[int]
appendpos_condition=None, # type: Optional[int]
Expand All @@ -1852,10 +1845,10 @@ async def append_block_from_url(self, copy_source_url, # type: str
:param str copy_source_url:
The URL of the source data. It can point to any Azure Blob or File, that is either public or has a
shared access signature attached.
:param int source_range_start:
:param int source_offset:
This indicates the start of the range of bytes(inclusive) that has to be taken from the copy source.
:param int source_range_end:
This indicates the end of the range of bytes(inclusive) that has to be taken from the copy source.
:param int source_length:
This indicates the end of the range of bytes that has to be taken from the copy source.
:param bytearray source_content_md5:
If given, the service will calculate the MD5 hash of the block content and compare against this value.
:param int maxsize_condition:
Expand Down Expand Up @@ -1926,8 +1919,8 @@ async def append_block_from_url(self, copy_source_url, # type: str
"""
options = self._append_block_from_url_options(
copy_source_url,
source_range_start=source_range_start,
source_range_end=source_range_end,
source_offset=source_offset,
source_length=source_length,
source_content_md5=source_content_md5,
maxsize_condition=maxsize_condition,
appendpos_condition=appendpos_condition,
Expand Down
Loading