Oauth

.github/CODEOWNERS

@@ -27,3 +27,4 @@
 /src/botservice/ @swagatmishra2007
 
 /src/storage-preview/ @williexu
+

scripts/ci/test_static.sh

@@ -9,12 +9,12 @@ PYLINT_EXCLUDES=$(echo "$AZURE_SDK_AUTOGEN_FILES" | sed -e s=\./src/=src/=g -e '
 
 # Run pylint/flake8 on extensions
 echo "Running pylint on extensions..."
-pylint ./src/rdbms/azext_rdbms/ --ignore=$PYLINT_EXCLUDES --ignore-patterns=test_* --rcfile=./pylintrc -j $proc_number
+pylint ./src/rdbms/azext_rdbms/ --ignore=$PYLINT_EXCLUDES,vendored_sdks --ignore-patterns=test_* --rcfile=./pylintrc -j $proc_number
 
 
 echo "Pylint OK."
 echo "Running flake8 on extensions..."
-flake8 --statistics --exclude=$FLAKE8_EXCLUDES --append-config=./.flake8 ./src/*/azext_*/
+flake8 --statistics --exclude=$FLAKE8_EXCLUDES,vendored_sdks --append-config=./.flake8 ./src/*/azext_*/
 echo "Flake8 OK."
 
 # Run pylint/flake8 on CI files
@@ -23,4 +23,4 @@ flake8 --append-config=./.flake8 ./scripts/ci/*.py
 
 # Other static checks
 python ./scripts/ci/verify_codeowners.py
-python ./scripts/ci/verify_license.py
+python ./scripts/ci/verify_license.py 'src/storage-preview/azext_storage_preview/vendored_sdks'

scripts/ci/verify_license.py

@@ -7,6 +7,7 @@
 
 import os
 import sys
+import argparse
 
 from util import get_repo_root
 
@@ -31,12 +32,14 @@
 """
 
 
-def main():
-    env_path = os.path.join(REPO_ROOT, 'env')
+def main(args):
+    excluded_paths = args.excluded_paths
+    excluded_paths.append('env')
+    excluded_paths = tuple([os.path.join(REPO_ROOT, relative_path) for relative_path in excluded_paths])
 
     files_without_header = []
     for current_dir, _, files in os.walk(get_repo_root()):
-        if current_dir.startswith(env_path):
+        if current_dir.startswith(excluded_paths):
             continue
         file_itr = (os.path.join(current_dir, p) for p in files if p.endswith('.py'))
         for python_file in file_itr:
@@ -52,4 +55,6 @@ def main():
 
 
 if __name__ == '__main__':
-    main()
+    parser = argparse.ArgumentParser()
+    parser.add_argument('excluded_paths', nargs='*')
+    main(parser.parse_args())

src/index.json

@@ -823,6 +823,53 @@
                     "version": "0.0.1"
                 }
             }
+        ],
+        "storage-preview": [
+            {
+                "filename": "storage_preview-0.1.0-py2.py3-none-any.whl",
+                "sha256Digest": "36768962d09c65b9668581f5bc01f1ad252acf832f22afdb04ed00c3333379cc",
+                "downloadUrl": "https://azurecliprod.blob.core.windows.net/cli-extensions/storage_preview-0.1.0-py2.py3-none-any.whl",
+                "metadata": {
+                    "azext.isPreview": true,
+                    "azext.minCliCoreVersion": "2.0.32.dev0",
+                    "classifiers": [
+                        "Development Status :: 4 - Beta",
+                        "Intended Audience :: Developers",
+                        "Intended Audience :: System Administrators",
+                        "Programming Language :: Python",
+                        "Programming Language :: Python :: 2",
+                        "Programming Language :: Python :: 2.7",
+                        "Programming Language :: Python :: 3",
+                        "Programming Language :: Python :: 3.4",
+                        "Programming Language :: Python :: 3.5",
+                        "Programming Language :: Python :: 3.6",
+                        "License :: OSI Approved :: MIT License"
+                    ],
+                    "extensions": {
+                        "python.details": {
+                            "contacts": [
+                                {
+                                    "email": "azpycli@microsoft.com",
+                                    "name": "Microsoft Corporation",
+                                    "role": "author"
+                                }
+                            ],
+                            "document_names": {
+                                "description": "DESCRIPTION.rst"
+                            },
+                            "project_urls": {
+                                "Home": "https://github.com/Azure/azure-cli-extensions"
+                            }
+                        }
+                    },
+                    "generator": "bdist_wheel (0.29.0)",
+                    "license": "MIT",
+                    "metadata_version": "2.0",
+                    "name": "storage-preview",
+                    "summary": "Provides a preview for upcoming storage features.",
+                    "version": "0.1.0"
+                }
+            }
         ]
     }
 }

src/storage-preview/azext_storage_preview/__init__.py

@@ -4,19 +4,22 @@
 # --------------------------------------------------------------------------------------------
 
 from azure.cli.core import AzCommandsLoader
-from azure.cli.core.profiles import ResourceType
+from azure.cli.core.profiles import ResourceType, register_resource_type
 from azure.cli.core.commands import AzCommandGroup, AzArgumentContext
 
 import azext_storage_preview._help  # pylint: disable=unused-import
+from .profiles import CUSTOM_DATA_STORAGE
 
 
 class StorageCommandsLoader(AzCommandsLoader):
     def __init__(self, cli_ctx=None):
         from azure.cli.core.commands import CliCommandType
 
+        register_resource_type('latest', CUSTOM_DATA_STORAGE, '2017-11-09')
         storage_custom = CliCommandType(operations_tmpl='azext_storage_preview.custom#{}')
+
         super(StorageCommandsLoader, self).__init__(cli_ctx=cli_ctx,
-                                                    resource_type=ResourceType.DATA_STORAGE,
+                                                    resource_type=CUSTOM_DATA_STORAGE,
                                                     custom_command_type=storage_custom,
                                                     command_group_cls=StorageCommandGroup,
                                                     argument_context_cls=StorageArgumentContext)
@@ -127,30 +130,66 @@ def register_common_storage_account_options(self):
 
 
 class StorageCommandGroup(AzCommandGroup):
-    def storage_command(self, name, method_name=None, command_type=None, **kwargs):
+    def storage_command(self, name, method_name=None, command_type=None, oauth=False, **kwargs):
         """ Registers an Azure CLI Storage Data Plane command. These commands always include the four parameters which
         can be used to obtain a storage client: account-name, account-key, connection-string, and sas-token. """
         if command_type:
             command_name = self.command(name, method_name, command_type=command_type, **kwargs)
         else:
             command_name = self.command(name, method_name, **kwargs)
         self._register_data_plane_account_arguments(command_name)
+        if oauth:
+            self._register_data_plane_oauth_arguments(command_name)
+
+    def storage_command_oauth(self, *args, **kwargs):
+        _merge_new_exception_handler(kwargs, self.get_handler_suppress_403())
+        self.storage_command(*args, oauth=True, **kwargs)
 
-    def storage_custom_command(self, name, method_name, **kwargs):
+    def storage_custom_command(self, name, method_name, oauth=False, **kwargs):
         command_name = self.custom_command(name, method_name, **kwargs)
         self._register_data_plane_account_arguments(command_name)
+        if oauth:
+            self._register_data_plane_oauth_arguments(command_name)
 
-    def get_handler_suppress_404(self):
+    def storage_custom_command_oauth(self, *args, **kwargs):
+        _merge_new_exception_handler(kwargs, self.get_handler_suppress_403())
+        self.storage_custom_command(*args, oauth=True, **kwargs)
 
-        # pylint: disable=inconsistent-return-statements
+    def get_handler_suppress_404(self):
         def handler(ex):
             from azure.cli.core.profiles import get_sdk
 
             t_error = get_sdk(self.command_loader.cli_ctx,
-                              ResourceType.DATA_STORAGE,
+                              CUSTOM_DATA_STORAGE,
                               'common._error#AzureMissingResourceHttpError')
             if isinstance(ex, t_error):
-                return None
+                return
+            raise ex
+
+        return handler
+
+    def get_handler_suppress_403(self):
+        def handler(ex):
+            from azure.cli.core.profiles import get_sdk
+            from knack.log import get_logger
+
+            logger = get_logger(__name__)
+            t_error = get_sdk(self.command_loader.cli_ctx,
+                              CUSTOM_DATA_STORAGE,
+                              'common._error#AzureHttpError')
+            if isinstance(ex, t_error) and ex.status_code == 403:
+                message = """
+You do not have the required permissions needed to perform this operation.
+Depending on your operation, you may need to be assigned one of the following roles:
+    "Storage Blob Data Contributor (Preview)"
+    "Storage Blob Data Reader (Preview)"
+    "Storage Queue Data Contributor (Preview)"
+    "Storage Queue Data Reader (Preview)"
+
+If you want to use the old authentication method and allow querying for the right account key, please use the "--auth-mode" parameter and "key" value.
+                """
+                logger.error(message)
+                return
             raise ex
 
         return handler
@@ -183,5 +222,30 @@ def _register_data_plane_account_arguments(self, command_name):
                              help='A Shared Access Signature (SAS). Must be used in conjunction with storage account '
                                   'name. Environment variable: AZURE_STORAGE_SAS_TOKEN')
 
+    def _register_data_plane_oauth_arguments(self, command_name):
+        from azure.cli.core.commands.parameters import get_enum_type
+
+        # workaround to allow use of AzArgumentContext.extra()
+        self.command_loader.command_name = command_name
+        with self.command_loader.argument_context(command_name) as c:
+            c.extra('auth_mode', arg_type=get_enum_type(['login', 'key']),
+                    help='The mode in which to run the command. "login" mode will directly use your login credentials '
+                         'for the authentication. The legacy "key" mode will attempt to query for '
+                         'an account key if no authentication parameters for the account are provided. '
+                         'Environment variable: AZURE_STORAGE_AUTH_MODE')
+
+
+def _merge_new_exception_handler(kwargs, handler):
+    if kwargs.get('exception_handler'):
+        def new_handler(ex):
+            first = kwargs['exception_handler']
+            try:
+                first(ex)
+            except Exception as raised_ex:  # pylint: disable=broad-except
+                handler(raised_ex)
+        kwargs['exception_handler'] = new_handler
+    else:
+        kwargs['exception_handler'] = handler
+
 
 COMMAND_LOADER_CLS = StorageCommandsLoader

src/storage-preview/azext_storage_preview/_client_factory.py

@@ -3,10 +3,13 @@
 # Licensed under the MIT License. See License.txt in the project root for license information.
 # --------------------------------------------------------------------------------------------
 
-from azure.cli.core.commands.client_factory import get_mgmt_service_client, get_data_service_client
+from azure.cli.core.commands.client_factory import get_mgmt_service_client, _get_add_headers_callback
 from azure.cli.core.profiles import ResourceType, get_sdk
+from knack.util import CLIError
+from knack.log import get_logger
 
 from .sdkutil import get_table_data_type
+from .profiles import CUSTOM_DATA_STORAGE
 
 NO_CREDENTIALS_ERROR_MESSAGE = """
 No credentials specified to access storage service. Please provide any of the following:
@@ -18,26 +21,56 @@
         option or AZURE_STORAGE_ACCOUNT environment variable)
 """
 
+logger = get_logger(__name__)
+
+
+# edited from azure.cli.core.commands.client_factory
+def get_data_service_client(cli_ctx, service_type, account_name, account_key, connection_string=None,
+                            sas_token=None, socket_timeout=None, token_credential=None, endpoint_suffix=None):
+    logger.debug('Getting data service client service_type=%s', service_type.__name__)
+    try:
+        client_kwargs = {'account_name': account_name,
+                         'account_key': account_key,
+                         'connection_string': connection_string,
+                         'sas_token': sas_token}
+        if socket_timeout:
+            client_kwargs['socket_timeout'] = socket_timeout
+        if token_credential:
+            client_kwargs['token_credential'] = token_credential
+        if endpoint_suffix:
+            client_kwargs['endpoint_suffix'] = endpoint_suffix
+        client = service_type(**client_kwargs)
+    except ValueError as exc:
+        _ERROR_STORAGE_MISSING_INFO = get_sdk(cli_ctx, CUSTOM_DATA_STORAGE,
+                                              'common._error#_ERROR_STORAGE_MISSING_INFO')
+        if _ERROR_STORAGE_MISSING_INFO in str(exc):
+            raise ValueError(exc)
+        else:
+            raise CLIError('Unable to obtain data client. Check your connection parameters.')
+    # TODO: enable Fiddler
+    client.request_callback = _get_add_headers_callback(cli_ctx)
+    return client
+
 
 def get_storage_data_service_client(cli_ctx, service, name=None, key=None, connection_string=None, sas_token=None,
-                                    socket_timeout=None):
+                                    socket_timeout=None, token_credential=None):
     return get_data_service_client(cli_ctx, service, name, key, connection_string, sas_token,
                                    socket_timeout=socket_timeout,
+                                   token_credential=token_credential,
                                    endpoint_suffix=cli_ctx.cloud.suffixes.storage_endpoint)
 
 
 def generic_data_service_factory(cli_ctx, service, name=None, key=None, connection_string=None, sas_token=None,
-                                 socket_timeout=None):
+                                 socket_timeout=None, token_credential=None):
     try:
         return get_storage_data_service_client(cli_ctx, service, name, key, connection_string, sas_token,
-                                               socket_timeout)
+                                               socket_timeout, token_credential)
     except ValueError as val_exception:
-        _ERROR_STORAGE_MISSING_INFO = get_sdk(cli_ctx, ResourceType.DATA_STORAGE,
+        _ERROR_STORAGE_MISSING_INFO = get_sdk(cli_ctx, CUSTOM_DATA_STORAGE,
                                               'common._error#_ERROR_STORAGE_MISSING_INFO')
         message = str(val_exception)
         if message == _ERROR_STORAGE_MISSING_INFO:
             message = NO_CREDENTIALS_ERROR_MESSAGE
-        from knack.util import CLIError
         raise CLIError(message)
 
 
@@ -46,15 +79,15 @@ def storage_client_factory(cli_ctx, **_):
 
 
 def file_data_service_factory(cli_ctx, kwargs):
-    t_file_svc = get_sdk(cli_ctx, ResourceType.DATA_STORAGE, 'file#FileService')
+    t_file_svc = get_sdk(cli_ctx, CUSTOM_DATA_STORAGE, 'file#FileService')
     return generic_data_service_factory(cli_ctx, t_file_svc, kwargs.pop('account_name', None),
                                         kwargs.pop('account_key', None),
                                         connection_string=kwargs.pop('connection_string', None),
                                         sas_token=kwargs.pop('sas_token', None))
 
 
 def page_blob_service_factory(cli_ctx, kwargs):
-    t_page_blob_service = get_sdk(cli_ctx, ResourceType.DATA_STORAGE, 'blob.pageblobservice#PageBlobService')
+    t_page_blob_service = get_sdk(cli_ctx, CUSTOM_DATA_STORAGE, 'blob.pageblobservice#PageBlobService')
     return generic_data_service_factory(cli_ctx, t_page_blob_service, kwargs.pop('account_name', None),
                                         kwargs.pop('account_key', None),
                                         connection_string=kwargs.pop('connection_string', None),
@@ -70,7 +103,8 @@ def blob_data_service_factory(cli_ctx, kwargs):
                                         kwargs.pop('account_key', None),
                                         connection_string=kwargs.pop('connection_string', None),
                                         sas_token=kwargs.pop('sas_token', None),
-                                        socket_timeout=kwargs.pop('socket_timeout', None))
+                                        socket_timeout=kwargs.pop('socket_timeout', None),
+                                        token_credential=kwargs.pop('token_credential', None))
 
 
 def table_data_service_factory(cli_ctx, kwargs):
@@ -83,17 +117,18 @@ def table_data_service_factory(cli_ctx, kwargs):
 
 
 def queue_data_service_factory(cli_ctx, kwargs):
-    t_queue_service = get_sdk(cli_ctx, ResourceType.DATA_STORAGE, 'queue#QueueService')
+    t_queue_service = get_sdk(cli_ctx, CUSTOM_DATA_STORAGE, 'queue#QueueService')
     return generic_data_service_factory(
         cli_ctx, t_queue_service,
         kwargs.pop('account_name', None),
         kwargs.pop('account_key', None),
         connection_string=kwargs.pop('connection_string', None),
-        sas_token=kwargs.pop('sas_token', None))
+        sas_token=kwargs.pop('sas_token', None),
+        token_credential=kwargs.pop('token_credential', None))
 
 
 def cloud_storage_account_service_factory(cli_ctx, kwargs):
-    t_cloud_storage_account = get_sdk(cli_ctx, ResourceType.DATA_STORAGE, 'common#CloudStorageAccount')
+    t_cloud_storage_account = get_sdk(cli_ctx, CUSTOM_DATA_STORAGE, 'common#CloudStorageAccount')
     account_name = kwargs.pop('account_name', None)
     account_key = kwargs.pop('account_key', None)
     sas_token = kwargs.pop('sas_token', None)
@@ -105,7 +140,7 @@ def multi_service_properties_factory(cli_ctx, kwargs):
     """Create multiple data services properties instance based on the services option"""
     from .services_wrapper import ServiceProperties
 
-    t_base_blob_service, t_file_service, t_queue_service, = get_sdk(cli_ctx, ResourceType.DATA_STORAGE,
+    t_base_blob_service, t_file_service, t_queue_service, = get_sdk(cli_ctx, CUSTOM_DATA_STORAGE,
                                                                     'blob.baseblobservice#BaseBlobService',
                                                                     'file#FileService', 'queue#QueueService')
 

src/storage-preview/azext_storage_preview/_format.py

@@ -3,7 +3,8 @@
 # Licensed under the MIT License. See License.txt in the project root for license information.
 # --------------------------------------------------------------------------------------------
 
-from azure.cli.core.profiles import get_sdk, ResourceType
+from azure.cli.core.profiles import get_sdk
+from .profiles import CUSTOM_DATA_STORAGE
 
 
 def build_table_output(result, projection):
@@ -134,7 +135,7 @@ def transform_file_directory_result(cli_ctx):
     list.
     """
     def transformer(result):
-        t_file, t_dir = get_sdk(cli_ctx, ResourceType.DATA_STORAGE, 'File', 'Directory', mod='file.models')
+        t_file, t_dir = get_sdk(cli_ctx, CUSTOM_DATA_STORAGE, 'File', 'Directory', mod='file.models')
         return_list = []
         for each in result:
             if isinstance(each, t_file):