Azure Storage Blob Track2 (#5896)

* Storage blob track2 (#5894) * Blob storage track 2 * Started pylint fixes * Clean pylint * Started blob walk * Started docstrings * Support walk blob * Pylint * Refactored upload * Pylint * Packaging update of azure-storage-blob * Fixes to storage dependencies in CI (#5900) * Revert "Packaging update of azure-storage-blob" This reverts commit aeeb3ce. * Fix Storage CI * Fixed imports * updating ccount_key_arg -> account_key_arg * updating ordering * Packaging update of azure-storage-blob * Revert "Packaging update of azure-storage-blob" This reverts commit 66d6877. * reverting 66d6877. adding sdk_packaging.toml file * docstrings for blobs (#5954) * [storage-blob] Refactor shared code (#5960) * Fix Storage CI * Fixed imports * Refactor shared storage code * Fixed XML error * Reverted change to models * Merged models docstrings * Fix setup.py version * Missing recordings * Fix storage SDK conflict * Packaging update of azure-eventhubs * Revert "Packaging update of azure-eventhubs" This reverts commit df844f7. * Fix EH packaging * Updated shared code * Review feedback * [azure-storage-blob] Policy updates + simple upload/download (#6162) * Last blob updates * Azure core dependency version * Update to user-agent policy * Review feedback * Removed old python tags * Code samples for blobs (#6093) * adds code samples for blobs * fixes blob code samples and adds literalincludes for docs * Storage Blobs Readme [wip] (#6011) * blobs readme wip * adding some samples and fixes for readme * fix settings import * moving code samples to a different PR * small fixes to readme * fill links for pypi and identity * populating TODO links * Fix CI * [Storage Blob] pylint + mypy passs (#6175) * pylint + mypy passs * comments * [storage-blob] Docs updates (#6184) * Docs updates * Some docstring fixes * [storage-blob] Test rename (#6197) * Test rename * Removed test init files * More renames * Try flattening tests * Blob test imports * Flattened queue tests * Renamed queue test case
Azure · Jul 2, 2019 · c93e46e · c93e46e
1 parent 22eeb3d
commit c93e46e
Show file tree

Hide file tree

Showing 555 changed files with 490,198 additions and 450 deletions.
diff --git a/scripts/dev_setup.py b/scripts/dev_setup.py
@@ -88,6 +88,11 @@ def pip_command(command, additional_dir=".", error_ok=False):
     content_packages.remove("azure-common")
 content_packages.insert(2, "azure-common")
 
+if 'azure-core' in content_packages:
+    content_packages.remove('azure-core')
+content_packages.insert(3, 'azure-core')
+
+
 print("Running dev setup...")
 print("Root directory '{}'\n".format(root_dir))
 

diff --git a/sdk/core/azure-servicemanagement-legacy/dev_requirements.txt b/sdk/core/azure-servicemanagement-legacy/dev_requirements.txt
@@ -1 +1,2 @@
 -e ../../../tools/azure-sdk-tools
+-e ../../storage/azure-storage-blob
diff --git a/sdk/core/azure-servicemanagement-legacy/tests/legacy_mgmt_testcase.py b/sdk/core/azure-servicemanagement-legacy/tests/legacy_mgmt_testcase.py
@@ -82,13 +82,18 @@ def create_service_management(self, service_class):
         return service
 
     def _create_storage_service(self, service_class, settings, account_name=None, account_key=None):
-        account_name = account_name or settings.STORAGE_ACCOUNT_NAME
-        account_key = account_key or settings.STORAGE_ACCOUNT_KEY
+        try:
+            account_name_arg = account_name or settings.STORAGE_ACCOUNT_NAME
+        except NameError:
+            account_name_arg = settings.STORAGE_ACCOUNT_NAME
+        try:
+            account_key_arg = account_key or settings.STORAGE_ACCOUNT_KEY
+        except NameError:
+            account_key_arg = settings.STORAGE_ACCOUNT_KEY
         session = Session()
         service = service_class(
-            account_name,
-            account_key,
-            request_session=session,
+            "https://{}.blob.core.windows.net".format(account_name_arg),
+            credentials=account_key_arg
         )
         self._set_service_options(service, settings)
         return service

diff --git a/sdk/core/azure-servicemanagement-legacy/tests/test_legacy_mgmt_misc.py b/sdk/core/azure-servicemanagement-legacy/tests/test_legacy_mgmt_misc.py
@@ -40,8 +40,12 @@
     parse_response_for_async_op,
     get_certificate_from_publish_settings,
 )
-from azure.storage.blob import PageBlobService, BlockBlobService
-from azure.storage.blob.models import PublicAccess
+from azure.core.exceptions import HttpResponseError, ResourceExistsError
+from azure.storage.blob import (
+    BlobServiceClient,
+    BlobType,
+    PublicAccess
+)
 from testutils.common_recordingtestcase import (
     TestMode,
     record,
@@ -85,9 +89,7 @@ def setUp(self):
         super(LegacyMgmtMiscTest, self).setUp()
 
         self.sms = self.create_service_management(ServiceManagementService)
-
-        self.bc = self._create_storage_service(PageBlobService, self.settings)
-        self.bbc = self._create_storage_service(BlockBlobService, self.settings)
+        self.bsc = self._create_storage_service(BlobServiceClient, self.settings)
 
         self.hosted_service_name = self.get_resource_name('utsvc')
         self.container_name = self.get_resource_name('utctnr')
@@ -166,7 +168,7 @@ def tearDown(self):
                     pass
 
             try:
-                self.bc.delete_container(self.container_name)
+                self.bsc.delete_container(self.container_name)
             except:
                 pass
 
@@ -284,14 +286,19 @@ def _make_blob_url(self, storage_account_name, container_name, blob_name):
 
     def _create_container_and_block_blob(self, container_name, blob_name,
                                          blob_data):
-        self.bc.create_container(container_name, None, 'container', False)
-        self.bbc.create_blob_from_bytes(
-            container_name, blob_name, blob_data)
+        try:
+            container = self.bsc.create_container(container_name, public_access='container')
+        except ResourceExistsError:
+            container = self.bsc.get_container_client(container_name)
+        container.upload_blob(blob_name, blob_data)
 
     def _create_container_and_page_blob(self, container_name, blob_name,
                                         content_length):
-        self.bc.create_container(container_name, None, 'container', False)
-        self.bc.create_blob_from_bytes(container_name, blob_name, b'')
+        try:
+            container = self.bsc.create_container(container_name, public_access='container')
+        except ResourceExistsError:
+            container = self.bsc.get_container_client(container_name)
+        container.upload_blob(blob_name, b'', blob_type=BlobType.PageBlob)
 
     def _upload_file_to_block_blob(self, file_path, blob_name):
         data = open(file_path, 'rb').read()
@@ -303,14 +310,13 @@ def _upload_file_to_block_blob(self, file_path, blob_name):
 
     def _upload_chunks(self, file_path, blob_name, chunk_size):
         index = 0
+        blob = self.bsc.get_blob_client(self.container_name, blob_name)
         with open(file_path, 'rb') as f:
             while True:
                 data = f.read(chunk_size)
                 if data:
                     length = len(data)
-                    self.bc.update_page(
-                        self.container_name, blob_name, data,
-                        index, index + length - 1)
+                    blob.upload_page(data, index, index + length - 1)
                     index += length
                 else:
                     break
@@ -379,7 +385,8 @@ def _os_image_exists(self, image_name):
 
     def _blob_exists(self, container_name, blob_name):
         try:
-            props = self.bc.get_blob_properties(container_name, blob_name)
+            blob = self.bsc.get_blob_client(container_name, blob_name)
+            props = blob.get_blob_properties()
             return props is not None
         except:
             return False
@@ -564,11 +571,13 @@ def _create_vm_image(self, image_name):
 
     def _copy_linux_os_vhd_to_container(self):
         blob_name = 'imagecopy.vhd'
-        self.bc.create_container(self.container_name,
-                                 public_access=PublicAccess.Blob)
-        resp = self.bc.copy_blob(self.container_name, blob_name,
-                                 self.settings.LINUX_OS_VHD)
-        return self.bc.make_blob_url(self.container_name, blob_name)
+        try:
+            container = self.bsc.create_container(self.container_name, public_access=PublicAccess.Blob)
+        except ResourceExistsError:
+            container = self.bsc.get_container_client(self.container_name)
+        blob = container.get_blob_client(blob_name)
+        blob.copy_blob_from_url(self.settings.LINUX_OS_VHD)
+        return blob.url
 
     #--Test cases for http passthroughs --------------------------------------
     @record

diff --git a/sdk/eventhub/azure-eventhubs/conftest.py b/sdk/eventhub/azure-eventhubs/conftest.py
@@ -19,14 +19,23 @@
     collect_ignore.append("features")
     collect_ignore.append("examples/async_examples")
 else:
-    from azure.eventprocessorhost import EventProcessorHost
-    from azure.eventprocessorhost import EventHubPartitionPump
-    from azure.eventprocessorhost import AzureStorageCheckpointLeaseManager
-    from azure.eventprocessorhost import AzureBlobLease
-    from azure.eventprocessorhost import EventHubConfig
-    from azure.eventprocessorhost.lease import Lease
-    from azure.eventprocessorhost.partition_pump import PartitionPump
-    from azure.eventprocessorhost.partition_manager import PartitionManager
+    try:
+        from azure.eventprocessorhost import EventProcessorHost
+        from azure.eventprocessorhost import EventHubPartitionPump
+        from azure.eventprocessorhost import AzureStorageCheckpointLeaseManager
+        from azure.eventprocessorhost import AzureBlobLease
+        from azure.eventprocessorhost import EventHubConfig
+        from azure.eventprocessorhost.lease import Lease
+        from azure.eventprocessorhost.partition_pump import PartitionPump
+        from azure.eventprocessorhost.partition_manager import PartitionManager
+    except ImportError:
+        # Due to storage SDK conflict, temporarily skipping EPH tests
+        collect_ignore.append("tests/asynctests/test_checkpoint_manager.py")
+        collect_ignore.append("tests/asynctests/test_eh_partition_pump.py")
+        collect_ignore.append("tests/asynctests/test_longrunning_eph.py")
+        collect_ignore.append("tests/asynctests/test_longrunning_eph_with_context.py")
+        collect_ignore.append("tests/asynctests/test_partition_manager.py")
+        collect_ignore.append("tests/asynctests/test_partition_pump.py")
 
 from azure.eventhub import EventHubClient, EventHubConsumer, EventPosition
 

diff --git a/sdk/storage/azure-storage-blob/HISTORY.md b/sdk/storage/azure-storage-blob/HISTORY.md
@@ -0,0 +1,65 @@
+# Change Log azure-storage-blob
+
+## Version 12.0.0b1:
+
+For release notes and more information please visit
+https://aka.ms/azure-sdk-preview1-python
+
+## Version 2.0.1:
+
+- Updated dependency on azure-storage-common.
+
+## Version 2.0.0:
+
+- Support for 2018-11-09 REST version. Please see our REST API documentation and blog for information about the related added features.
+- Added support for append block from URL(synchronously) for append blobs.
+- Added support for update page from URL(synchronously) for page blobs.
+- Added support for generating and using blob snapshot SAS tokens.
+- Added support for generating user delegation SAS tokens.
+
+## Version 1.5.0:
+
+- Added new method list_blob_names to efficiently list only blob names in an efficient way.
+
+## Version 1.4.0:
+
+- azure-storage-nspkg is not installed anymore on Python 3 (PEP420-based namespace package)
+- copy_blob method added to BlockBlobService to enable support for deep sync copy.
+
+## Version 1.3.1:
+
+- Fixed design flaw where get_blob_to_* methods buffer entire blob when max_connections is set to 1.
+- Added support for access conditions on append_blob_from_* methods.
+
+## Version 1.3.0:
+
+- Support for 2018-03-28 REST version. Please see our REST API documentation and blog for information about the related added features.
+- Added support for setting static website service properties.
+- Added support for getting account information, such as SKU name and account kind.
+- Added support for put block from URL(synchronously).
+
+## Version 1.2.0rc1:
+
+- Support for 2017-11-09 REST version. Please see our REST API documentation and blog for information about the related added features.
+- Support for write-once read-many containers.
+- Added support for OAuth authentication for HTTPS requests(Please note that this feature is available in preview).
+
+## Version 1.1.0:
+
+- Support for 2017-07-29 REST version. Please see our REST API documentation and blogs for information about the related added features.
+- Added support for soft delete feature. If a delete retention policy is enabled through the set service properties API, then blobs or snapshots could be deleted softly and retained for a specified number of days, before being permanently removed by garbage collection.
+- Error message now contains the ErrorCode from the x-ms-error-code header value.
+
+## Version 1.0.0:
+
+- The package has switched from Apache 2.0 to the MIT license.
+- Fixed bug where get_blob_to_* cannot get a single byte when start_range and end_range are both equal to 0.
+- Optimized page blob upload for create_blob_from_* methods, by skipping the empty chunks.
+- Added convenient method to generate container url (make_container_url).
+- Metadata keys are now case-preserving when fetched from the service. Previously they were made lower-case by the library.
+
+## Version 0.37.1:
+
+- Enabling MD5 validation no longer uses the memory-efficient algorithm for large block blobs, since computing the MD5 hash requires reading the entire block into memory.
+- Fixed a bug in the _SubStream class which was at risk of causing data corruption when using the memory-efficient algorithm for large block blobs.
+- Support for AccessTierChangeTime to get the last time a tier was modified on an individual blob.
diff --git a/sdk/storage/azure-storage-blob/LICENSE.txt b/sdk/storage/azure-storage-blob/LICENSE.txt
@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2017 Microsoft
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
diff --git a/sdk/storage/azure-storage-blob/MANIFEST.in b/sdk/storage/azure-storage-blob/MANIFEST.in
@@ -0,0 +1,4 @@
+include *.md
+include azure/__init__.py
+include azure/storage/__init__.py
+include LICENSE.txt
Original file line number	Diff line number	Diff line change
		@@ -1 +1,2 @@
		-e ../../../tools/azure-sdk-tools
		-e ../../storage/azure-storage-blob