Merge remote-tracking branch 'origin/master' into feature/storage-pre…

…view4

.gitignore

@@ -20,6 +20,7 @@ bin/
 obj/
 dist/
 MANIFEST
+_docs
 
 # Result of running python setup.py install/pip install -e
 RECORD.txt

doc/sphinx/conf.py

@@ -262,5 +262,3 @@
 
 # If false, no module index is generated.
 #latex_domain_indices = True
-
-

doc/sphinx/generate_doc.py

@@ -2,8 +2,9 @@
 import logging
 import json
 from pathlib import Path
+import os
 
-CONFIG_FILE = './package_service_mapping.json'
+CONFIG_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)), './package_service_mapping.json')
 GENERATED_PACKAGES_LIST_FILE = 'autorest_generated_packages.rst'
 
 _LOGGER = logging.getLogger(__name__)
@@ -83,14 +84,15 @@ def make_title(title):
     "azure.mgmt.resource.subscriptions",
 ]
 
-def generate_doc(config_path, project_pattern=None):
-
+def generate_doc(config_path, output_directory = "./ref/", project_pattern=None):
     multiapi_found_apiversion = {}
 
+    rst_path_template = os.path.join(output_directory, '{}.rst')
+    rst_namespace_template = os.path.join(output_directory, '{}.{}.rst')
+
     with Path(config_path).open() as config_fd:
         config = json.load(config_fd)
     package_list_path = []
-    rst_path_template = './ref/{}.rst'
 
     namespaces = [n for pack in config.values() for n in pack.get("namespaces", {})]
 
@@ -109,7 +111,7 @@ def generate_doc(config_path, project_pattern=None):
             ))
 
         for module in ["operations", "models"]:
-            with Path('./ref/{}.{}.rst'.format(namespace, module)).open('w') as rst_file:
+            with Path(rst_namespace_template.format(namespace, module)).open('w') as rst_file:
                 rst_file.write(SUBMODULE_TEMPLATE.format(
                     title=make_title(namespace+"."+module+" module"),
                     namespace=namespace,
@@ -167,6 +169,9 @@ def main():
     parser.add_argument("--debug",
                         dest="debug", action="store_true",
                         help="Verbosity in DEBUG mode")
+    parser.add_argument("--out", "-o",
+                        dest="output_directory", 
+                        help="The final resting place for the generated sphinx source files.")
 
     args = parser.parse_args()
 
@@ -175,7 +180,7 @@ def main():
         logging.basicConfig()
         main_logger.setLevel(logging.DEBUG if args.debug else logging.INFO)
 
-    generate_doc(args.config_path, args.project)
+    generate_doc(args.config_path, args.output_directory, args.project)
 
 if __name__ == "__main__":
     main()

doc/sphinx/individual_build_conf.py

@@ -0,0 +1,264 @@
+# -*- coding: utf-8 -*-
+#
+# azure-sdk-for-python documentation build configuration file, created by
+# sphinx-quickstart on Fri Jun 27 15:42:45 2014.
+#
+# This file is execfile()d with the current directory set to its
+# containing dir.
+#
+# Note that not all possible configuration values are present in this
+# autogenerated file.
+#
+# All configuration values have a default; values that are commented out
+# serve to show the default.
+
+import sys
+import os
+import sphinx_rtd_theme
+
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
+# sys.path.insert(0, os.path.abspath('.'))
+
+# FIX FOR EXAMPLE REFERENCES
+# REPO_ROOT = os.path.abspath(os.path.join(os.path.abspath(__file__), '..', '..', '..'))
+# examples_tests_glob_expansion = os.path.join(REPO_ROOT, 'sdk/*/*/examples/**/test*example*.py')
+# samples_glob_expansion = os.path.join(REPO_ROOT, 'sdk/*/*/tests/**/test*sample*.py')
+# test_examples_glob_expansion = os.path.join(REPO_ROOT, 'sdk/*/*/tests/**/test*example*.py')
+
+# example_tests_files = glob.glob(examples_tests_glob_expansion, recursive=True)
+# test_examples_files = glob.glob(test_examples_glob_expansion, recursive=True)
+# samples_files = glob.glob(samples_glob_expansion, recursive=True)
+
+# all_files = [os.path.relpath(file, REPO_ROOT) for file in list(set(example_tests_files + test_examples_files + samples_files))]
+
+# # now for each package, we need to copy it and write it to the relative path FROM THE CURRENT CWD
+# for example_file in all_files:
+#     relative_path_in_pkg = os.path.join(*(example_file.split(os.path.sep)[3:]))
+#     final_destination = os.path.abspath(os.path.join(os.path.dirname(__file__), relative_path_in_pkg))
+
+#     # create the directories if they don't already exist
+#     os.makedirs(os.path.dirname(final_destination), exist_ok=True)
+
+#     # finally copy the example file into the sphinx folder
+#     copyfile(os.path.join(REPO_ROOT, example_file), final_destination)
+
+# -- General configuration ------------------------------------------------
+
+# If your documentation needs a minimal Sphinx version, state it here.
+#needs_sphinx = '1.0'
+
+# Add any Sphinx extension module names here, as strings. They can be
+# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
+# ones.
+extensions = ['sphinx.ext.autodoc', 'sphinx.ext.autosummary', 'sphinx.ext.doctest',
+              'sphinx.ext.viewcode', 'sphinx.ext.intersphinx', 'sphinx.ext.napoleon']
+
+intersphinx_mapping = {
+    'python': ('https://docs.python.org/3.5', None),
+    'msrestazure': ('http://msrestazure.readthedocs.io/en/latest/', None),
+    'msrest': ('http://msrest.readthedocs.io/en/latest/', None)
+}
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ['_templates']
+
+source_parsers = {
+   '.md': 'recommonmark.parser.CommonMarkParser',
+}
+
+# The suffix of source filenames.
+source_suffix = ['.rst', '.md']
+
+# The encoding of source files.
+#source_encoding = 'utf-8-sig'
+
+# The master toctree document.
+master_doc = 'index'
+
+# General information about the project.
+project = u'Azure SDK for Python'
+copyright = u'2019, Microsoft'
+
+# The version info for the project you're documenting, acts as replacement for
+# |version| and |release|, also used in various other places throughout the
+# built documents.
+#
+# The short X.Y version.
+version = '2.0.0'
+# The full version, including alpha/beta/rc tags.
+release = '2.0.0'
+
+# The language for content autogenerated by Sphinx. Refer to documentation
+# for a list of supported languages.
+#language = None
+
+# There are two options for replacing |today|: either, you set today to some
+# non-false value, then it is used:
+#today = ''
+# Else, today_fmt is used as the format for a strftime call.
+#today_fmt = '%B %d, %Y'
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+exclude_patterns = ['_build']
+
+# The reST default role (used for this markup: `text`) to use for all
+# documents.
+#default_role = None
+
+# If true, '()' will be appended to :func: etc. cross-reference text.
+#add_function_parentheses = True
+
+# If true, the current module name will be prepended to all description
+# unit titles (such as .. function::).
+#add_module_names = True
+
+# If true, sectionauthor and moduleauthor directives will be shown in the
+# output. They are ignored by default.
+#show_authors = False
+
+# The name of the Pygments (syntax highlighting) style to use.
+pygments_style = 'sphinx'
+
+# A list of ignored prefixes for module index sorting.
+#modindex_common_prefix = []
+
+# If true, keep warnings as "system message" paragraphs in the built documents.
+#keep_warnings = False
+
+# -- Options for extensions ----------------------------------------------------
+autoclass_content = 'both'
+
+# -- Options for HTML output ----------------------------------------------
+
+# The theme to use for HTML and HTML Help pages.  See the documentation for
+# a list of builtin themes.
+html_theme = 'sphinx_rtd_theme'
+#html_theme_options = {'collapsiblesidebar': True}
+
+# Activate the theme.
+html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]
+
+# Theme options are theme-specific and customize the look and feel of a theme
+# further.  For a list of options available for each theme, see the
+# documentation.
+#html_theme_options = {}
+
+# Add any paths that contain custom themes here, relative to this directory.
+#html_theme_path = []
+
+# The name for this set of Sphinx documents.  If None, it defaults to
+# "<project> v<release> documentation".
+#html_title = None
+
+# A shorter title for the navigation bar.  Default is the same as html_title.
+#html_short_title = None
+
+# The name of an image file (relative to this directory) to place at the top
+# of the sidebar.
+#html_logo = None
+
+# The name of an image file (within the static path) to use as favicon of the
+# docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
+# pixels large.
+#html_favicon = None
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+# html_static_path = ['_static']
+
+# Add any extra paths that contain custom files (such as robots.txt or
+# .htaccess) here, relative to this directory. These files are copied
+# directly to the root of the documentation.
+#html_extra_path = []
+
+# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
+# using the given strftime format.
+#html_last_updated_fmt = '%b %d, %Y'
+
+# If true, SmartyPants will be used to convert quotes and dashes to
+# typographically correct entities.
+#html_use_smartypants = True
+
+# Custom sidebar templates, maps document names to template names.
+#html_sidebars = {}
+
+# Additional templates that should be rendered to pages, maps page names to
+# template names.
+#html_additional_pages = {}
+
+# If false, no module index is generated.
+#html_domain_indices = True
+
+# If false, no index is generated.
+#html_use_index = True
+
+# If true, the index is split into individual pages for each letter.
+#html_split_index = False
+
+# If true, links to the reST sources are added to the pages.
+#html_show_sourcelink = True
+
+# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
+#html_show_sphinx = True
+
+# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
+#html_show_copyright = True
+
+# If true, an OpenSearch description file will be output, and all pages will
+# contain a <link> tag referring to it.  The value of this option must be the
+# base URL from which the finished HTML is served.
+#html_use_opensearch = ''
+
+# This is the file name suffix for HTML files (e.g. ".xhtml").
+#html_file_suffix = None
+
+# Output file base name for HTML help builder.
+#htmlhelp_basename = 'azure-sdk-for-python-doc'
+
+
+# -- Options for LaTeX output ---------------------------------------------
+
+#latex_elements = {
+# The paper size ('letterpaper' or 'a4paper').
+#'papersize': 'letterpaper',
+
+# The font size ('10pt', '11pt' or '12pt').
+#'pointsize': '10pt',
+
+# Additional stuff for the LaTeX preamble.
+#'preamble': '',
+#}
+
+# Grouping the document tree into LaTeX files. List of tuples
+# (source start file, target name, title,
+#  author, documentclass [howto, manual, or own class]).
+# latex_documents = [
+#   ('index', 'azure-sdk-for-python.tex', u'Azure SDK for Python Documentation',
+#    u'Microsoft', 'manual'),
+# ]
+
+# The name of an image file (relative to this directory) to place at the top of
+# the title page.
+#latex_logo = None
+
+# For "manual" documents, if this is true, then toplevel headings are parts,
+# not chapters.
+#latex_use_parts = False
+
+# If true, show page references after internal links.
+#latex_show_pagerefs = False
+
+# If true, show URL addresses after external links.
+#latex_show_urls = False
+
+# Documents to append as an appendix to all manuals.
+#latex_appendices = []
+
+# If false, no module index is generated.
+#latex_domain_indices = True
+
+

eng/pipelines/templates/jobs/archetype-sdk-nightly.yml

@@ -13,7 +13,7 @@ jobs:
       vmImage: 'ubuntu-16.04'
 
     steps:
-    - template: ../steps/build-artifacts.yml
+    - template: ../steps/build-package-artifacts.yml
       parameters: 
         ServiceDirectory: ${{ parameters.ServiceDirectory }}
         BuildTargetingString: ${{ parameters.BuildTargetingString }}

eng/pipelines/templates/stages/archetype-sdk-client.yml

@@ -0,0 +1,18 @@
+parameters:
+  Artifacts: []
+  ServiceDirectory: not-specified
+
+stages:
+  - stage: Build
+    jobs:
+    - template: ../jobs/archetype-sdk-client.yml
+      parameters:
+        ServiceDirectory: ${{parameters.ServiceDirectory}}
+
+  # The Prerelease and Release stages are conditioned on whether we are building a pull request and the branch.
+  - ${{if and(ne(variables['Build.Reason'], 'PullRequest'), eq(variables['System.TeamProject'], 'internal'))}}:
+    - template: pipelines/stages/archetype-python-release.yml@azure-sdk-build-tools
+      parameters:
+        DependsOn: Build
+        Artifacts: ${{parameters.Artifacts}}
+        ArtifactName: packages

eng/pipelines/templates/steps/build-artifacts.yml

@@ -30,7 +30,7 @@ steps:
       versionSpec: $(PythonVersion)
 
   - script: |
-      pip install wheel setuptools pathlib twine readme-renderer[md] packaging
+      pip install wheel setuptools pathlib twine readme-renderer[md] packaging tox tox-monorepo
     displayName: 'Prep Environment'
 
   - task: PythonScript@0
@@ -43,6 +43,15 @@ steps:
       twine check $(Build.ArtifactStagingDirectory)/*
     displayName: 'Verify Readme'
 
+  - task: PythonScript@0
+    displayName: 'Generate Docs'
+    inputs:
+      scriptPath: 'scripts/devops_tasks/setup_execute_tests.py'
+      arguments: >-
+        "${{ parameters.BuildTargetingString }}" 
+        --service="${{ parameters.ServiceDirectory }}" 
+        --toxenv=sphinx
+
   - ${{ parameters.BeforePublishSteps }}
 
   - task: PublishPipelineArtifact@0
@@ -54,4 +63,5 @@ steps:
     condition: succeededOrFailed()
     displayName: 'Publish Artifacts'
     inputs:
-      artifactName: packages
+      artifactName: 'docs'
+      pathtoPublish: '$(Build.SourcesDirectory)/_docs'