[T1] new pipeline for Python (#16131)

* new pipeline

* update sh

ci_template.yml

@@ -0,0 +1,33 @@
+# DO NOT EDIT THIS FILE
+# This file is generated automatically and any changes will be lost.
+
+trigger:
+  branches:
+    include:
+    - master
+    - hotfix/*
+    - release/*
+    - restapi*
+  paths:
+    include:
+    - sdk/MyService/
+
+pr:
+  branches:
+    include:
+    - master
+    - feature/*
+    - hotfix/*
+    - release/*
+    - restapi*
+  paths:
+    include:
+    - sdk/MyService/
+
+extends:
+  template: ../../eng/pipelines/templates/stages/archetype-sdk-client.yml
+  parameters:
+    ServiceDirectory: MyService
+    Artifacts:
+    - name: azure_mgmt_MyService
+      safeName: azuremgmtMyService

scripts/automation_generate.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+VIRTUAL_ENV=$TMPDIR/venv-sdk
+export VIRTUAL_ENV
+PATH="$VIRTUAL_ENV/bin:$PATH"
+export PATH
+
+# node version degrade
+sudo npm install -g n
+sudo n 10.15.0
+export PATH=/usr/local/n/versions/node/10.15.0/bin:$PATH
+
+# generate code and package
+python -m packaging_tools.auto_codegen "$1" "$TMPDIR/venv-sdk/auto_temp.json" 2>&1
+echo "[Generate] codegen done!!!"
+python -m packaging_tools.auto_package "$TMPDIR/venv-sdk/auto_temp.json" "$2" 2>&1
+echo "[Generate] generate done!!!"

scripts/automation_init.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+rm -rf $TMPDIR/venv-sdk
+python3 -m venv $TMPDIR/venv-sdk
+VIRTUAL_ENV=$TMPDIR/venv-sdk
+export VIRTUAL_ENV
+PATH="$VIRTUAL_ENV/bin:$PATH"
+export PATH
+python scripts/dev_setup.py -p azure-core
+echo "{}" >> $2
+echo "[Generate] init success!!!"

swagger_to_sdk_config.json

@@ -1,21 +1,26 @@
 {
-  "$schema": "https://openapistorageprod.blob.core.windows.net/sdkautomation/prod/schemas/swagger_to_sdk_config.schema.json",
-  "meta": {
-    "autorest_options": {
-      "version": "V2",
-      "use": "@microsoft.azure/autorest.python@~4.0.71",
-      "python": "",
-      "python-mode": "update",
-      "sdkrel:python-sdks-folder": "./sdk/.",
-      "multiapi": "",
-      "keep-version-file" :"",
-      "no-async": ""
+  "advancedOptions": {
+    "createSdkPullRequests": true,
+    "generationCallMode": "one-for-all-configs"
+  },
+
+  "initOptions": {
+    "initScript": {
+      "path": "sh scripts/automation_init.sh"
+    }
+  },
+
+  "generateOptions": {
+    "generateScript": {
+      "path": "sh scripts/automation_generate.sh",
+      "stderr": {
+        "showInComment": true
+      },
+      "stdout": {
+        "showInComment": "^\\[Autorest\\]"
+      }
     },
-    "advanced_options": {
-      "create_sdk_pull_requests": true,
-      "sdk_generation_pull_request_base": "integration_branch"
-    },
-    "repotag": "azure-sdk-for-python",
-    "version": "0.2.0"
+
+    "parseGenerateOutput": true
   }
-}
+}

swagger_to_sdk_config_autorest.json

@@ -0,0 +1,20 @@
+{
+  "meta": {
+    "autorest_options": {
+      "version": "V2",
+      "use": "@microsoft.azure/autorest.python@~4.0.71",
+      "python": "",
+      "python-mode": "update",
+      "sdkrel:python-sdks-folder": "./sdk/.",
+      "multiapi": "",
+      "keep-version-file" :"",
+      "no-async": ""
+    },
+    "advanced_options": {
+      "create_sdk_pull_requests": true,
+      "sdk_generation_pull_request_base": "integration_branch"
+    },
+    "repotag": "azure-sdk-for-python",
+    "version": "0.2.0"
+  }
+}

tools/azure-devtools/src/azure_devtools/ci_tools/git_tools.py

@@ -88,3 +88,11 @@ def get_files_in_commit(git_folder, commit_id="HEAD"):
     repo = Repo(str(git_folder))
     output = repo.git.diff("--name-only", commit_id+"^", commit_id)
     return output.splitlines()
+
+def get_diff_file_list(git_folder):
+    """List of new files.
+    """
+    repo = Repo(str(git_folder))
+    repo.git.add("sdk")
+    output = repo.git.diff("HEAD", "--name-only")
+    return output.splitlines()

tools/azure-sdk-tools/packaging_tools/auto_codegen.py

@@ -0,0 +1,121 @@
+import argparse
+import json
+import logging
+from pathlib import Path
+import re
+from subprocess import check_call
+
+from .swaggertosdk.SwaggerToSdkCore import (
+    CONFIG_FILE,
+)
+from azure_devtools.ci_tools.git_tools import get_diff_file_list
+from .generate_sdk import generate
+
+_LOGGER = logging.getLogger(__name__)
+_SDK_FOLDER_RE = re.compile(r"^(sdk/[\w-]+)/(azure[\w-]+)/", re.ASCII)
+
+DEFAULT_DEST_FOLDER = "./dist"
+
+
+def get_package_names(sdk_folder):
+    files = get_diff_file_list(sdk_folder)
+    matches = {_SDK_FOLDER_RE.search(f) for f in files}
+    package_names = {match.groups() for match in matches if match is not None}
+    return package_names
+
+
+def init_new_service(package_name, folder_name):
+    ci = Path(folder_name, 'ci.yml')
+    if not ci.exists():
+        check_call(f'python -m packaging_tools --build-conf {package_name} -o {folder_name}', shell=True)
+        with open('ci_template.yml', 'r') as file_in:
+            content = file_in.readlines()
+        name = package_name.replace('azure-', '').replace('mgmt-', '')
+        content = [line.replace('MyService', name) for line in content]
+        with open(str(ci), 'w') as file_out:
+            file_out.writelines(content)
+
+
+def main(generate_input, generate_output):
+    with open(generate_input, "r") as reader:
+        data = json.load(reader)
+
+    spec_folder = data['specFolder']
+    sdk_folder = "."
+    result = {}
+    package_total = set()
+    for input_readme in data["relatedReadmeMdFiles"]:
+        relative_path_readme = str(Path(spec_folder, input_readme))
+        _LOGGER.info(f'[CODEGEN]({input_readme})codegen begin')
+        generate(CONFIG_FILE,
+                 sdk_folder,
+                 [],
+                 relative_path_readme,
+                 spec_folder,
+                 force_generation=True
+                 )
+        package_names = get_package_names(sdk_folder)
+        _LOGGER.info(f'[CODEGEN]({input_readme})codegen end. [(packages:{str(package_names)})]')
+
+        for folder_name, package_name in package_names:
+            if package_name in package_total:
+                continue
+
+            package_total.add(package_name)
+            if package_name not in result:
+                package_entry = {}
+                package_entry['packageName'] = package_name
+                package_entry["path"] = [folder_name]
+                package_entry['readmeMd'] = [input_readme]
+                result[package_name] = package_entry
+            else:
+                result[package_name]["path"].append(folder_name)
+                result[package_name]["readmeMd"].append(input_readme)
+
+            # Generate some necessary file for new service
+            init_new_service(package_name, folder_name)
+
+            # Setup package locally
+            check_call(f'pip install --ignore-requires-python -e {str(Path(sdk_folder, folder_name, package_name))}',
+                       shell=True)
+
+
+    # remove duplicates
+    for value in result.values():
+        value['path'] = list(set(value['path']))
+        value['readmeMd'] = list(set(value['readmeMd']))
+
+    with open(generate_output, "w") as writer:
+        json.dump(result, writer)
+
+
+def generate_main():
+    """Main method"""
+
+    parser = argparse.ArgumentParser(
+        description='Build SDK using Autorest, offline version.',
+        formatter_class=argparse.RawTextHelpFormatter)
+    parser.add_argument('generate_input',
+                        help='Generate input file path')
+    parser.add_argument('generate_output',
+                        help='Generate output file path')
+    parser.add_argument("-v", "--verbose",
+                        dest="verbose", action="store_true",
+                        help="Verbosity in INFO mode")
+    parser.add_argument("--debug",
+                        dest="debug", action="store_true",
+                        help="Verbosity in DEBUG mode")
+    parser.add_argument("-c", "--codegen",
+                        dest="debug", action="store_true",
+                        help="Verbosity in DEBUG mode")
+
+    args = parser.parse_args()
+    main_logger = logging.getLogger()
+    logging.basicConfig()
+    main_logger.setLevel(logging.DEBUG if args.verbose or args.debug else logging.INFO)
+
+    main(args.generate_input, args.generate_output)
+
+
+if __name__ == "__main__":
+    generate_main()

tools/azure-sdk-tools/packaging_tools/auto_package.py

@@ -0,0 +1,114 @@
+import argparse
+import json
+import glob
+import logging
+import os
+from pathlib import Path
+import re
+from subprocess import check_call
+
+from azure_devtools.ci_tools.git_tools import get_diff_file_list
+from .change_log import main as change_log_main
+
+_LOGGER = logging.getLogger(__name__)
+_SDK_FOLDER_RE = re.compile(r"^(sdk/[\w-]+)/(azure[\w-]+)/", re.ASCII)
+
+DEFAULT_DEST_FOLDER = "./dist"
+
+
+def create_package(name, dest_folder=DEFAULT_DEST_FOLDER):
+    # a package will exist in either one, or the other folder. this is why we can resolve both at the same time.
+    absdirs = [os.path.dirname(package) for package in
+               (glob.glob('{}/setup.py'.format(name)) + glob.glob('sdk/*/{}/setup.py'.format(name)))]
+
+    absdirpath = os.path.abspath(absdirs[0])
+    check_call(['python', 'setup.py', 'bdist_wheel', '-d', dest_folder], cwd=absdirpath)
+    check_call(['python', 'setup.py', "sdist", "--format", "zip", '-d', dest_folder], cwd=absdirpath)
+
+
+def get_package_names(sdk_folder):
+    files = get_diff_file_list(sdk_folder)
+    matches = {_SDK_FOLDER_RE.search(f) for f in files}
+    package_names = {match.groups() for match in matches if match is not None}
+    return package_names
+
+
+def change_log_generate(package_name):
+    from pypi_tools.pypi import PyPIClient
+    client = PyPIClient()
+    try:
+        client.get_ordered_versions(package_name)
+    except:
+        return "  - Initial Release"
+    else:
+        return change_log_main(f"{package_name}:pypi", f"{package_name}:latest")
+
+
+def main(generate_input, generate_output):
+    with open(generate_input, "r") as reader:
+        data = json.load(reader)
+    if not data:
+        return
+
+    sdk_folder = '.'
+    result = {
+        'packages': []
+    }
+    for package in data.values():
+        package_name = package['packageName']
+        # Changelog
+        md_output = change_log_generate(package_name)
+        package["changelog"] = {
+            "content": md_output,
+            "hasBreakingChange": "Breaking changes" in md_output or "Initial Release" in md_output
+        }
+        _LOGGER.info(f'[PACKAGE]({package_name})[CHANGELOG]:{md_output}')
+        # Built package
+        create_package(package_name)
+        folder_name = package['path'][0]
+        dist_path = Path(sdk_folder, folder_name, package_name, "dist")
+        package["artifacts"] = [
+            str(dist_path / package_file) for package_file in os.listdir(dist_path)
+        ]
+        # Installation package
+        package["installInstructions"] = {
+            "full": "You can install the use using pip install of the artificats.",
+            "lite": f"pip install {package_name}"
+        }
+        package["result"]: "success"
+        result['packages'].append(package)
+
+    with open(generate_output, "w") as writer:
+        json.dump(result, writer)
+
+
+def generate_main():
+    """Main method"""
+
+    parser = argparse.ArgumentParser(
+        description='Build SDK using Autorest, offline version.',
+        formatter_class=argparse.RawTextHelpFormatter)
+    parser.add_argument('generate_input',
+                        help='Generate input file path')
+    parser.add_argument('generate_output',
+                        help='Generate output file path')
+    parser.add_argument("-v", "--verbose",
+                        dest="verbose", action="store_true",
+                        help="Verbosity in INFO mode")
+    parser.add_argument("--debug",
+                        dest="debug", action="store_true",
+                        help="Verbosity in DEBUG mode")
+    parser.add_argument("-c", "--codegen",
+                        dest="debug", action="store_true",
+                        help="Verbosity in DEBUG mode")
+
+    args = parser.parse_args()
+    main_logger = logging.getLogger()
+    logging.basicConfig()
+    main_logger.setLevel(logging.DEBUG if args.verbose or args.debug else logging.INFO)
+
+    main(args.generate_input, args.generate_output)
+
+
+if __name__ == "__main__":
+    generate_main()