Get latest master

.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,29 @@
+---
+name: Bug report
+about: Create a report to help us improve
+title: ''
+labels: ''
+assignees: ''
+
+---
+
+- **Package Name**: 
+- **Package Version**: 
+- **Operating System**:
+- **Python Version**: 
+
+**Describe the bug**
+A clear and concise description of what the bug is.
+
+**To Reproduce**
+Steps to reproduce the behavior:
+1. 
+
+**Expected behavior**
+A clear and concise description of what you expected to happen.
+
+**Screenshots**
+If applicable, add screenshots to help explain your problem.
+
+**Additional context**
+Add any other context about the problem here.

.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,20 @@
+---
+name: Feature request
+about: Suggest an idea for this project
+title: ''
+labels: ''
+assignees: ''
+
+---
+
+**Is your feature request related to a problem? Please describe.**
+A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
+
+**Describe the solution you'd like**
+A clear and concise description of what you want to happen.
+
+**Describe alternatives you've considered**
+A clear and concise description of any alternative solutions or features you've considered.
+
+**Additional context**
+Add any other context or screenshots about the feature request here.

README.md

@@ -2,7 +2,7 @@
 
 [![Packages](https://img.shields.io/badge/packages-latest-blue.svg)](https://azure.github.io/azure-sdk/releases/latest/python.html) [![Dependencies](https://img.shields.io/badge/dependencies-analyzed-blue.svg)](https://azuresdkartifacts.blob.core.windows.net/azure-sdk-for-python/dependencies/dependencies.html) [![Python](https://img.shields.io/pypi/pyversions/azure-core.svg?maxAge=2592000)](https://pypi.python.org/pypi/azure/) [![Build Status](https://dev.azure.com/azure-sdk/public/_apis/build/status/python/python%20-%20core%20-%20ci?branchName=master)](https://dev.azure.com/azure-sdk/public/_build/latest?definitionId=458&branchName=master)
 
-This repository is intended for active development of the Azure SDK for Python. For consumers of the SDK we recommend visiting our [public developer docs](https://docs.microsoft.com/en-us/python/azure/) or our versioned [developer docs](https://azure.github.io/azure-sdk-for-python).
+This repository is for active development of the Azure SDK for Python. For consumers of the SDK we recommend visiting our [public developer docs](https://docs.microsoft.com/en-us/python/azure/) or our versioned [developer docs](https://azure.github.io/azure-sdk-for-python).
 
 ## Getting started
 

doc/sphinx/_static/css/default_overrides.css

@@ -16,4 +16,11 @@
 
 .wy-nav-content-wrap {
     margin-left: 380px;
+}
+
+@font-face{
+    font-family:"Roboto Slab";
+    font-style:normal;
+    font-weight:400;
+    src:url("../fonts/RobotoSlab/roboto-slab-v7-regular.eot?#iefix") format("embedded-opentype"),url("../fonts/RobotoSlab/roboto-slab-v7-regular.woff2") format("woff2"),url("../fonts/RobotoSlab/roboto-slab-v7-regular.woff") format("woff"),url("../fonts/RobotoSlab/roboto-slab-v7-regular.ttf") format("truetype")
 }

doc/sphinx/generate_doc.py

@@ -70,6 +70,7 @@ def make_title(title):
 # Update the code to compute this list automatically
 MULTIAPI_VERSION_NAMESPACE = [
     "azure.mgmt.storage",
+    "azure.mgmt.web",
     "azure.mgmt.network",
     "azure.mgmt.compute",
     "azure.mgmt.containerregistry",

doc/sphinx/generate_versioned_index.py

@@ -155,7 +155,7 @@ def write_toc_tree(categorized_menu_items):
 
 
 def get_categorized_menu_items(package_names):
-    categorized_menu_items = {}
+    categorized_menu_items = {"Other": {"Client":[], "Management":[], "Other": []}}
 
     for pkg in package_names:
         # add to the categorized menu items

doc/sphinx/index.rst

@@ -2,7 +2,7 @@
 Azure SDK for Python
 ====================
 
-.. important:: The most part of this documentation has moved to https://docs.microsoft.com/python/azure
+.. important:: You can find all of the Python on Azure documentation at https://docs.microsoft.com/python/azure
 
 The Azure SDK for Python is a set of libraries which allow you to work on Azure for your management, runtime or data needs.
 
@@ -59,14 +59,6 @@ If you encounter any bugs with the library please file an issue in the
 `Issues <https://github.com/Azure/azure-sdk-for-python/issues>`__
 section of the project.
 
-
-Indices and tables
-------------------
-
-* :ref:`genindex`
-* :ref:`modindex`
-* :ref:`search`
-
 .. toctree::
   :glob:
   :caption: User Documentation

doc/sphinx/package_service_mapping.json

@@ -811,11 +811,16 @@
   "azure-mgmt-web": {
     "category": "Management",
     "namespaces": [
-      "azure.mgmt.web"
+      "azure.mgmt.web.v2015_04_01",
+      "azure.mgmt.web.v2015_08_01",
+      "azure.mgmt.web.v2016_03_01",
+      "azure.mgmt.web.v2016_08_01",
+      "azure.mgmt.web.v2016_09_01",
+      "azure.mgmt.web.v2018_02_01",
+      "azure.mgmt.web.v2018_11_01"
     ],
     "service_name": "Web Apps"
   },
-
   "azure-servicebus": {
     "category": "Client",
     "service_name": "Service Bus",
@@ -866,6 +871,11 @@
     "service_name": "Keyvault",
     "manually_generated": true
   },
+  "azure-ai-textanalytics": {
+    "category": "Client",
+    "service_name": "Text Analytics",
+    "manually_generated": true
+  },
   "azure-storage-blob": {
     "category": "Client",
     "service_name": "Storage",
@@ -991,7 +1001,7 @@
   },
   "azure-mgmt-storagesync": {
     "category": "Management",
-    "service_name": "Storage",
+    "service_name": "Storage Sync",
     "namespaces": [
       "azure.mgmt.storagesync"
     ]

doc/sphinx/servicemanagement.rst

@@ -116,9 +116,9 @@ To make a new deployment to Azure you must store the package file in a
 Microsoft Azure Blob Storage account under the same subscription as the
 hosted service to which the package is being uploaded. You can create a
 deployment package with the `Microsoft Azure PowerShell
-cmdlets <https://www.windowsazure.com/en-us/develop/php/how-to-guides/powershell-cmdlets/>`__,
+cmdlets <https://docs.microsoft.com/en-us/powershell/azure/?view=azps-3.2.0>`__,
 or with the `cspack commandline
-tool <http://msdn.microsoft.com/en-us/library/gg432988.aspx>`__.
+tool <https://docs.microsoft.com/en-us/azure/cloud-services/cloud-services-model-and-package#servicepackagecspkg>`__.
 
 .. code:: python
 

eng/pipelines/create-doc-index.yml

@@ -27,10 +27,17 @@ jobs:
         inputs:
           versionSpec: $(PythonVersion)
 
+      - template: eng/pipelines/templates/scripts/replace-relative-links.yml@azure-sdk-tools
+        parameters:
+          TargetFolder: '.'
+          RootFolder: '.'
+          BuildSHA: $(Build.SourceVersion)
+          RepoId: 'Azure/azure-sdk-for-python'
+
       - pwsh: |
           pip install -r $(Build.SourcesDirectory)/doc/sphinx/requirements.txt
           python $(Build.SourcesDirectory)/doc/sphinx/generate_versioned_index.py
-          mkdir $(Build.ArtifactStagingDirectory)
+          sphinx-build -b html -c $(Build.SourcesDirectory)/doc/sphinx $(Build.SourcesDirectory)/_docs $(Build.SourcesDirectory)/_docs/html
           Copy-Item -Path $(Build.SourcesDirectory)/_docs/html/* -Destination $(Build.ArtifactStagingDirectory) -Recurse -Force
         displayName: 'Generate Documentation Index'
 

eng/pipelines/templates/jobs/archetype-sdk-client.yml

@@ -4,6 +4,7 @@ parameters:
   TestMarkArgument: ''
   BuildTargetingString: 'azure-*'
   TestTimeoutInMinutes: 0
+  ToxEnvParallel: '--tenvparallel'
   TestMatrix:
     Linux_Python27:
       OSName: 'Linux'
@@ -117,6 +118,7 @@ jobs:
         PythonVersion: $(PythonVersion)
         BuildTargetingString: ${{ parameters.BuildTargetingString }}
         ToxTestEnv: 'whl,sdist'
+        ToxEnvParallel: ${{ parameters.ToxEnvParallel }}
         BeforeTestSteps: 
           - task: DownloadPipelineArtifact@0
             inputs:

eng/pipelines/templates/stages/archetype-sdk-client.yml

@@ -1,13 +1,15 @@
 parameters:
   Artifacts: []
   ServiceDirectory: not-specified
+  ToxEnvParallel: '--tenvparallel'
 
 stages:
   - stage: Build
     jobs:
     - template: ../jobs/archetype-sdk-client.yml
       parameters:
         ServiceDirectory: ${{parameters.ServiceDirectory}}
+        ToxEnvParallel: ${{parameters.ToxEnvParallel}}
 
   # The Prerelease and Release stages are conditioned on whether we are building a pull request and the branch.
   - ${{if and(ne(variables['Build.Reason'], 'PullRequest'), eq(variables['System.TeamProject'], 'internal'))}}:

eng/pipelines/templates/steps/build-test.yml

@@ -10,6 +10,7 @@ parameters:
   BuildTargetingString: 'azure-*'
   ToxTestEnv: ""
   RunCoverage: ne(variables['CoverageArg'], '--disablecov')
+  ToxEnvParallel: ''
 
 steps:
   - pwsh: |
@@ -41,6 +42,7 @@ steps:
         --mark_arg="${{ parameters.TestMarkArgument }}" 
         --service="${{ parameters.ServiceDirectory }}" 
         --toxenv="${{ parameters.ToxTestEnv }}"
+        ${{ parameters.ToxEnvParallel }}
     env: ${{ parameters.EnvVars }}
 
   - task: PublishTestResults@2

eng/tox/tox.ini

@@ -2,6 +2,7 @@
 # note that this envlist is the default set of environments that will run if a target environment is not selected.
 envlist = whl,sdist
 
+
 [tools]
 deps =
   -r ../../../eng/test_tools.txt
@@ -21,6 +22,7 @@ deps =
 
 [testenv]
 default_pytest_params = --junitxml={toxinidir}/test-junit-{envname}.xml --verbose --durations=10 --ignore=azure
+parallel_show_output =True
 pre-deps =
   wheel
 skip_install = true
@@ -35,10 +37,11 @@ setenv =
 deps = {[base]deps}
 changedir = {toxinidir}
 commands = 
-    {envbindir}/python {toxinidir}/../../../eng/tox/create_wheel_and_install.py -d {distdir} -p {toxinidir}
+    {envbindir}/python {toxinidir}/../../../eng/tox/create_wheel_and_install.py -d {envtmpdir} -p {toxinidir}
     pip freeze
     pytest \
       {[testenv]default_pytest_params} \
+      --ignore=.tox \
       {posargs} \
       {toxinidir}
 
@@ -79,6 +82,8 @@ commands =
     pip freeze
     pytest \
       {posargs} \
+      --no-cov \
+      --ignore=.tox \
       {toxinidir}
 
 
@@ -92,8 +97,10 @@ deps =
 commands =
     pytest \
       {posargs} \
+      --ignore=.tox \
       {toxinidir}
 
+
 [testenv:sphinx]
 skipsdist = false
 skip_install = false
@@ -127,7 +134,7 @@ changedir = {toxinidir}
 deps =
   {[tools]deps}
 commands = 
-    {envbindir}/python {toxinidir}/../../../eng/tox/create_wheel_and_install.py -d {distdir} -p {toxinidir}
+    {envbindir}/python {toxinidir}/../../../eng/tox/create_wheel_and_install.py -d {envtmpdir} -p {toxinidir}
     {envbindir}/python {toxinidir}/../../../eng/tox/import_all.py -t {toxinidir}
 
 
@@ -137,8 +144,8 @@ skip_install = true
 changedir = {toxinidir}
 deps = 
 commands =
-    {envbindir}/python {toxinidir}/../../../eng/tox/create_wheel_and_install.py -d {distdir} -p {toxinidir} --skip-install True
-    {envbindir}/python {toxinidir}/../../../eng/tox/verify_whl.py -d {distdir} -t {toxinidir}
+    {envbindir}/python {toxinidir}/../../../eng/tox/create_wheel_and_install.py -d {envtmpdir} -p {toxinidir} --skip-install True
+    {envbindir}/python {toxinidir}/../../../eng/tox/verify_whl.py -d {envtmpdir} -t {toxinidir}
 
 
 [testenv:verifysdist]
@@ -147,8 +154,8 @@ skip_install = true
 changedir = {toxinidir}
 deps = 
 commands =
-    {envbindir}/python {toxinidir}/setup.py --q sdist --format zip -d {distdir}
-    {envbindir}/python {toxinidir}/../../../eng/tox/verify_sdist.py -d {distdir} -t {toxinidir}
+    {envbindir}/python {toxinidir}/setup.py --q sdist --format zip -d {envtmpdir}
+    {envbindir}/python {toxinidir}/../../../eng/tox/verify_sdist.py -d {envtmpdir} -t {toxinidir}
 
 
 [testenv:devtest]
@@ -157,9 +164,10 @@ pre-deps =
 deps = {[base]deps}
 changedir = {toxinidir}
 commands = 
-    {envbindir}/python {toxinidir}/../../../eng/tox/create_wheel_and_install.py -d {distdir} -p {toxinidir}
+    {envbindir}/python {toxinidir}/../../../eng/tox/create_wheel_and_install.py -d {envtmpdir} -p {toxinidir}
     {envbindir}/python {toxinidir}/../../../eng/tox/install_dev_build_dependency.py -t {toxinidir}
     pytest \
       {[testenv]default_pytest_params} \
+      --ignore=.tox \
       {posargs} \
       {toxinidir}

scripts/devops_tasks/common_tasks.py

@@ -40,7 +40,8 @@
     "mgmt",
     "azure-cognitiveservices",
     "azure-servicefabric",
-    "nspkg"
+    "nspkg",
+    "azure-keyvault"
 ]
 NON_MANAGEMENT_CODE_5_ALLOWED = [
     "azure-keyvault"

scripts/devops_tasks/setup_execute_tests.py

@@ -226,6 +226,13 @@ def execute_global_install_and_test(
         action="store_true",
     )
 
+    parser.add_argument(
+        "--tenvparallel",
+        default=False,
+        help=("Run individual tox env for each package in parallel."),
+        action="store_true",
+    )
+
     parser.add_argument(
         "--service",
         help=(

scripts/devops_tasks/tox_harness.py

@@ -199,6 +199,26 @@ def execute_tox_parallel(tox_command_tuples):
         exit(1)
 
 
+def replace_dev_reqs(file):
+    adjusted_req_lines = []
+
+    with open(file, "r") as f:
+        for line in f:
+            args = [
+                part.strip()
+                for part in line.split()
+                if part and not part.strip() == "-e"
+            ]
+            amended_line = " ".join(args)
+            adjusted_req_lines.append(amended_line)
+
+    with open(file, "w") as f:
+        # note that we directly use '\n' here instead of os.linesep due to how f.write() actually handles this stuff internally
+        # If a file is opened in text mode (the default), during write python will accidentally double replace due to "\r" being
+        # replaced with "\r\n" on Windows. Result: "\r\n\n". Extra line breaks!
+        f.write("\n".join(adjusted_req_lines))
+
+
 def execute_tox_serial(tox_command_tuples):
     for index, cmd_tuple in enumerate(tox_command_tuples):
         tox_dir = os.path.join(cmd_tuple[1], "./.tox/")
@@ -260,9 +280,16 @@ def prep_and_run_tox(targeted_packages, parsed_args, options_array=[]):
             with open(destination_dev_req, "w+") as file:
                 file.write("\n")
 
+        if in_ci():
+            replace_dev_reqs(destination_dev_req)
+            os.environ["TOX_PARALLEL_NO_SPINNER"] = "1"
+
         if parsed_args.tox_env:
             tox_execution_array.extend(["-e", parsed_args.tox_env])
 
+        if parsed_args.tenvparallel:
+            tox_execution_array.extend(["-p", "all"])
+
         if local_options_array:
             tox_execution_array.extend(["--"] + local_options_array)