diff --git a/ci_template.yml b/ci_template.yml new file mode 100644 index 00000000000..18a73a5fc69 --- /dev/null +++ b/ci_template.yml @@ -0,0 +1,33 @@ +# DO NOT EDIT THIS FILE +# This file is generated automatically and any changes will be lost. + +trigger: + branches: + include: + - master + - hotfix/* + - release/* + - restapi* + paths: + include: + - sdk/MyService/ + +pr: + branches: + include: + - master + - feature/* + - hotfix/* + - release/* + - restapi* + paths: + include: + - sdk/MyService/ + +extends: + template: ../../eng/pipelines/templates/stages/archetype-sdk-client.yml + parameters: + ServiceDirectory: MyService + Artifacts: + - name: azure_mgmt_MyService + safeName: azuremgmtMyService \ No newline at end of file diff --git a/scripts/automation_generate.sh b/scripts/automation_generate.sh new file mode 100644 index 00000000000..40a1e1ff876 --- /dev/null +++ b/scripts/automation_generate.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +VIRTUAL_ENV=$TMPDIR/venv-sdk +export VIRTUAL_ENV +PATH="$VIRTUAL_ENV/bin:$PATH" +export PATH + +# node version degrade +sudo npm install -g n +sudo n 10.15.0 +export PATH=/usr/local/n/versions/node/10.15.0/bin:$PATH + +# generate code and package +python -m packaging_tools.auto_codegen "$1" "$TMPDIR/venv-sdk/auto_temp.json" 2>&1 +echo "[Generate] codegen done!!!" +python -m packaging_tools.auto_package "$TMPDIR/venv-sdk/auto_temp.json" "$2" 2>&1 +echo "[Generate] generate done!!!" diff --git a/scripts/automation_init.sh b/scripts/automation_init.sh new file mode 100644 index 00000000000..fdbe319cb48 --- /dev/null +++ b/scripts/automation_init.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +rm -rf $TMPDIR/venv-sdk +python3 -m venv $TMPDIR/venv-sdk +VIRTUAL_ENV=$TMPDIR/venv-sdk +export VIRTUAL_ENV +PATH="$VIRTUAL_ENV/bin:$PATH" +export PATH +python scripts/dev_setup.py -p azure-core +echo "{}" >> $2 +echo "[Generate] init success!!!" diff --git a/swagger_to_sdk_config.json b/swagger_to_sdk_config.json index eeb46999ae5..2ac5d9b9d14 100644 --- a/swagger_to_sdk_config.json +++ b/swagger_to_sdk_config.json @@ -1,21 +1,26 @@ { - "$schema": "https://openapistorageprod.blob.core.windows.net/sdkautomation/prod/schemas/swagger_to_sdk_config.schema.json", - "meta": { - "autorest_options": { - "version": "V2", - "use": "@microsoft.azure/autorest.python@~4.0.71", - "python": "", - "python-mode": "update", - "sdkrel:python-sdks-folder": "./sdk/.", - "multiapi": "", - "keep-version-file" :"", - "no-async": "" + "advancedOptions": { + "createSdkPullRequests": true, + "generationCallMode": "one-for-all-configs" + }, + + "initOptions": { + "initScript": { + "path": "sh scripts/automation_init.sh" + } + }, + + "generateOptions": { + "generateScript": { + "path": "sh scripts/automation_generate.sh", + "stderr": { + "showInComment": true + }, + "stdout": { + "showInComment": "^\\[Autorest\\]" + } }, - "advanced_options": { - "create_sdk_pull_requests": true, - "sdk_generation_pull_request_base": "integration_branch" - }, - "repotag": "azure-sdk-for-python", - "version": "0.2.0" + + "parseGenerateOutput": true } -} +} \ No newline at end of file diff --git a/swagger_to_sdk_config_autorest.json b/swagger_to_sdk_config_autorest.json new file mode 100644 index 00000000000..0bc0992e3aa --- /dev/null +++ b/swagger_to_sdk_config_autorest.json @@ -0,0 +1,20 @@ +{ + "meta": { + "autorest_options": { + "version": "V2", + "use": "@microsoft.azure/autorest.python@~4.0.71", + "python": "", + "python-mode": "update", + "sdkrel:python-sdks-folder": "./sdk/.", + "multiapi": "", + "keep-version-file" :"", + "no-async": "" + }, + "advanced_options": { + "create_sdk_pull_requests": true, + "sdk_generation_pull_request_base": "integration_branch" + }, + "repotag": "azure-sdk-for-python", + "version": "0.2.0" + } +} diff --git a/tools/azure-devtools/src/azure_devtools/ci_tools/git_tools.py b/tools/azure-devtools/src/azure_devtools/ci_tools/git_tools.py index 8c29dd842dc..37c4854e3b6 100644 --- a/tools/azure-devtools/src/azure_devtools/ci_tools/git_tools.py +++ b/tools/azure-devtools/src/azure_devtools/ci_tools/git_tools.py @@ -88,3 +88,11 @@ def get_files_in_commit(git_folder, commit_id="HEAD"): repo = Repo(str(git_folder)) output = repo.git.diff("--name-only", commit_id+"^", commit_id) return output.splitlines() + +def get_diff_file_list(git_folder): + """List of new files. + """ + repo = Repo(str(git_folder)) + repo.git.add("sdk") + output = repo.git.diff("HEAD", "--name-only") + return output.splitlines() diff --git a/tools/azure-sdk-tools/packaging_tools/auto_codegen.py b/tools/azure-sdk-tools/packaging_tools/auto_codegen.py new file mode 100644 index 00000000000..f58da0579f5 --- /dev/null +++ b/tools/azure-sdk-tools/packaging_tools/auto_codegen.py @@ -0,0 +1,121 @@ +import argparse +import json +import logging +from pathlib import Path +import re +from subprocess import check_call + +from .swaggertosdk.SwaggerToSdkCore import ( + CONFIG_FILE, +) +from azure_devtools.ci_tools.git_tools import get_diff_file_list +from .generate_sdk import generate + +_LOGGER = logging.getLogger(__name__) +_SDK_FOLDER_RE = re.compile(r"^(sdk/[\w-]+)/(azure[\w-]+)/", re.ASCII) + +DEFAULT_DEST_FOLDER = "./dist" + + +def get_package_names(sdk_folder): + files = get_diff_file_list(sdk_folder) + matches = {_SDK_FOLDER_RE.search(f) for f in files} + package_names = {match.groups() for match in matches if match is not None} + return package_names + + +def init_new_service(package_name, folder_name): + ci = Path(folder_name, 'ci.yml') + if not ci.exists(): + check_call(f'python -m packaging_tools --build-conf {package_name} -o {folder_name}', shell=True) + with open('ci_template.yml', 'r') as file_in: + content = file_in.readlines() + name = package_name.replace('azure-', '').replace('mgmt-', '') + content = [line.replace('MyService', name) for line in content] + with open(str(ci), 'w') as file_out: + file_out.writelines(content) + + +def main(generate_input, generate_output): + with open(generate_input, "r") as reader: + data = json.load(reader) + + spec_folder = data['specFolder'] + sdk_folder = "." + result = {} + package_total = set() + for input_readme in data["relatedReadmeMdFiles"]: + relative_path_readme = str(Path(spec_folder, input_readme)) + _LOGGER.info(f'[CODEGEN]({input_readme})codegen begin') + generate(CONFIG_FILE, + sdk_folder, + [], + relative_path_readme, + spec_folder, + force_generation=True + ) + package_names = get_package_names(sdk_folder) + _LOGGER.info(f'[CODEGEN]({input_readme})codegen end. [(packages:{str(package_names)})]') + + for folder_name, package_name in package_names: + if package_name in package_total: + continue + + package_total.add(package_name) + if package_name not in result: + package_entry = {} + package_entry['packageName'] = package_name + package_entry["path"] = [folder_name] + package_entry['readmeMd'] = [input_readme] + result[package_name] = package_entry + else: + result[package_name]["path"].append(folder_name) + result[package_name]["readmeMd"].append(input_readme) + + # Generate some necessary file for new service + init_new_service(package_name, folder_name) + + # Setup package locally + check_call(f'pip install --ignore-requires-python -e {str(Path(sdk_folder, folder_name, package_name))}', + shell=True) + + + # remove duplicates + for value in result.values(): + value['path'] = list(set(value['path'])) + value['readmeMd'] = list(set(value['readmeMd'])) + + with open(generate_output, "w") as writer: + json.dump(result, writer) + + +def generate_main(): + """Main method""" + + parser = argparse.ArgumentParser( + description='Build SDK using Autorest, offline version.', + formatter_class=argparse.RawTextHelpFormatter) + parser.add_argument('generate_input', + help='Generate input file path') + parser.add_argument('generate_output', + help='Generate output file path') + parser.add_argument("-v", "--verbose", + dest="verbose", action="store_true", + help="Verbosity in INFO mode") + parser.add_argument("--debug", + dest="debug", action="store_true", + help="Verbosity in DEBUG mode") + parser.add_argument("-c", "--codegen", + dest="debug", action="store_true", + help="Verbosity in DEBUG mode") + + args = parser.parse_args() + main_logger = logging.getLogger() + logging.basicConfig() + main_logger.setLevel(logging.DEBUG if args.verbose or args.debug else logging.INFO) + + main(args.generate_input, args.generate_output) + + +if __name__ == "__main__": + generate_main() diff --git a/tools/azure-sdk-tools/packaging_tools/auto_package.py b/tools/azure-sdk-tools/packaging_tools/auto_package.py new file mode 100644 index 00000000000..323fc2d305f --- /dev/null +++ b/tools/azure-sdk-tools/packaging_tools/auto_package.py @@ -0,0 +1,114 @@ +import argparse +import json +import glob +import logging +import os +from pathlib import Path +import re +from subprocess import check_call + +from azure_devtools.ci_tools.git_tools import get_diff_file_list +from .change_log import main as change_log_main + +_LOGGER = logging.getLogger(__name__) +_SDK_FOLDER_RE = re.compile(r"^(sdk/[\w-]+)/(azure[\w-]+)/", re.ASCII) + +DEFAULT_DEST_FOLDER = "./dist" + + +def create_package(name, dest_folder=DEFAULT_DEST_FOLDER): + # a package will exist in either one, or the other folder. this is why we can resolve both at the same time. + absdirs = [os.path.dirname(package) for package in + (glob.glob('{}/setup.py'.format(name)) + glob.glob('sdk/*/{}/setup.py'.format(name)))] + + absdirpath = os.path.abspath(absdirs[0]) + check_call(['python', 'setup.py', 'bdist_wheel', '-d', dest_folder], cwd=absdirpath) + check_call(['python', 'setup.py', "sdist", "--format", "zip", '-d', dest_folder], cwd=absdirpath) + + +def get_package_names(sdk_folder): + files = get_diff_file_list(sdk_folder) + matches = {_SDK_FOLDER_RE.search(f) for f in files} + package_names = {match.groups() for match in matches if match is not None} + return package_names + + +def change_log_generate(package_name): + from pypi_tools.pypi import PyPIClient + client = PyPIClient() + try: + client.get_ordered_versions(package_name) + except: + return " - Initial Release" + else: + return change_log_main(f"{package_name}:pypi", f"{package_name}:latest") + + +def main(generate_input, generate_output): + with open(generate_input, "r") as reader: + data = json.load(reader) + if not data: + return + + sdk_folder = '.' + result = { + 'packages': [] + } + for package in data.values(): + package_name = package['packageName'] + # Changelog + md_output = change_log_generate(package_name) + package["changelog"] = { + "content": md_output, + "hasBreakingChange": "Breaking changes" in md_output or "Initial Release" in md_output + } + _LOGGER.info(f'[PACKAGE]({package_name})[CHANGELOG]:{md_output}') + # Built package + create_package(package_name) + folder_name = package['path'][0] + dist_path = Path(sdk_folder, folder_name, package_name, "dist") + package["artifacts"] = [ + str(dist_path / package_file) for package_file in os.listdir(dist_path) + ] + # Installation package + package["installInstructions"] = { + "full": "You can install the use using pip install of the artificats.", + "lite": f"pip install {package_name}" + } + package["result"]: "success" + result['packages'].append(package) + + with open(generate_output, "w") as writer: + json.dump(result, writer) + + +def generate_main(): + """Main method""" + + parser = argparse.ArgumentParser( + description='Build SDK using Autorest, offline version.', + formatter_class=argparse.RawTextHelpFormatter) + parser.add_argument('generate_input', + help='Generate input file path') + parser.add_argument('generate_output', + help='Generate output file path') + parser.add_argument("-v", "--verbose", + dest="verbose", action="store_true", + help="Verbosity in INFO mode") + parser.add_argument("--debug", + dest="debug", action="store_true", + help="Verbosity in DEBUG mode") + parser.add_argument("-c", "--codegen", + dest="debug", action="store_true", + help="Verbosity in DEBUG mode") + + args = parser.parse_args() + main_logger = logging.getLogger() + logging.basicConfig() + main_logger.setLevel(logging.DEBUG if args.verbose or args.debug else logging.INFO) + + main(args.generate_input, args.generate_output) + + +if __name__ == "__main__": + generate_main() diff --git a/tools/azure-sdk-tools/packaging_tools/change_log.py b/tools/azure-sdk-tools/packaging_tools/change_log.py index d7c76d1e860..d13534aeff1 100644 --- a/tools/azure-sdk-tools/packaging_tools/change_log.py +++ b/tools/azure-sdk-tools/packaging_tools/change_log.py @@ -171,6 +171,18 @@ def get_report_from_parameter(input_parameter): return json.load(fd) +def main(base, latest): + old_report = get_report_from_parameter(base) + new_report = get_report_from_parameter(latest) + + # result = diff(old_report, new_report) + # with open("result.json", "w") as fd: + # json.dump(result, fd) + + change_log = build_change_log(old_report, new_report) + return change_log.build_md() + + if __name__ == "__main__": import argparse @@ -190,12 +202,4 @@ def get_report_from_parameter(input_parameter): logging.basicConfig(level=logging.DEBUG if args.debug else logging.INFO) - old_report = get_report_from_parameter(args.base) - new_report = get_report_from_parameter(args.latest) - - # result = diff(old_report, new_report) - # with open("result.json", "w") as fd: - # json.dump(result, fd) - - change_log = build_change_log(old_report, new_report) - print(change_log.build_md()) + print(main(args.base, args.latest)) diff --git a/tools/azure-sdk-tools/packaging_tools/code_report.py b/tools/azure-sdk-tools/packaging_tools/code_report.py index 0970753ec7d..047859ddb1e 100644 --- a/tools/azure-sdk-tools/packaging_tools/code_report.py +++ b/tools/azure-sdk-tools/packaging_tools/code_report.py @@ -9,6 +9,7 @@ import subprocess import sys import types +import tempfile from typing import Dict, Any, Optional # Because I'm subprocessing myself, I need to do weird thing as import. @@ -140,7 +141,7 @@ def merge_report(report_paths): merged_report = create_empty_report() for report in sorted(report_paths): - with report.open() as report_fd: + with open(report, 'r') as report_fd: report_json = json.load(report_fd) merged_report["models"]["enums"].update(report_json["models"]["enums"]) @@ -249,10 +250,14 @@ def filter_track2_versions(package_name, versions): return versions return list(filter(lambda x: version.parse(x) < version.parse(upbound), versions)) -def main(input_parameter: str, version: Optional[str] = None, no_venv: bool = False, pypi: bool = False, last_pypi: bool = False, output: str = None): + +def main(input_parameter: str, version: Optional[str] = None, no_venv: bool = False, pypi: bool = False, + last_pypi: bool = False, output: str = None, metadata_path: str = None): package_name, module_name = parse_input(input_parameter) path_to_package = resolve_package_directory(package_name) + output_filename = '' + result = [] if (version or pypi or last_pypi) and not no_venv: if version: versions = [version] @@ -269,13 +274,16 @@ def main(input_parameter: str, version: Optional[str] = None, no_venv: bool = Fa for version in versions: _LOGGER.info(f"Installing version {version} of {package_name} in a venv") - with create_venv_with_package([f"{package_name}=={version}"]) as venv: + with create_venv_with_package([f"{package_name}=={version}"]) as venv, tempfile.TemporaryDirectory() as temp_dir: + metadata_path = str(Path(temp_dir, f"metadata_{version}.json")) args = [ venv.env_exe, __file__, "--no-venv", "--version", version, + "--metadata", + metadata_path, input_parameter ] if output is not None: @@ -285,11 +293,12 @@ def main(input_parameter: str, version: Optional[str] = None, no_venv: bool = Fa except subprocess.CalledProcessError: # If it fail, just assume this version is too old to get an Autorest report _LOGGER.warning(f"Version {version} seems to be too old to build a report (probably not Autorest based)") + with open(metadata_path, "r") as metadata_fd: + result.extend(json.load(metadata_fd)["reports_path"]) # Files have been written by the subprocess - return + return result modules = find_autorest_generated_folder(module_name) - result = [] version = version or "latest" output_folder = Path(path_to_package) / Path("code_reports") / Path(version) output_folder.mkdir(parents=True, exist_ok=True) @@ -311,7 +320,7 @@ def main(input_parameter: str, version: Optional[str] = None, no_venv: bool = Fa with open(output_filename, "w") as fd: json.dump(report, fd, indent=2) _LOGGER.info(f"Report written to {output_filename}") - result.append(output_filename) + result.append(str(output_filename)) if len(result) > 1: merged_report = merge_report(result) @@ -322,6 +331,13 @@ def main(input_parameter: str, version: Optional[str] = None, no_venv: bool = Fa with open(output_filename, "w") as fd: json.dump(merged_report, fd, indent=2) _LOGGER.info(f"Merged report written to {output_filename}") + result = [str(output_filename)] + + if metadata_path: + metadata = {"reports_path": result} # Prepare metadata + with open(metadata_path, "w") as metadata_fd: + _LOGGER.info(f"Writing metadata: {metadata}") + json.dump(metadata, metadata_fd) return result @@ -392,8 +408,11 @@ def get_sub_module_part(package_name, module_name): parser.add_argument("--output", dest="output", help="Override output path.") + parser.add_argument("--metadata-path", + dest="metadata", + help="Write a metadata file about what happen. Mostly used for automation.") args = parser.parse_args() logging.basicConfig(level=logging.DEBUG if args.debug else logging.INFO) - main(args.package_name, args.version, args.no_venv, args.pypi, args.last_pypi, args.output) + main(args.package_name, args.version, args.no_venv, args.pypi, args.last_pypi, args.output, args.metadata) diff --git a/tools/azure-sdk-tools/packaging_tools/swaggertosdk/SwaggerToSdkCore.py b/tools/azure-sdk-tools/packaging_tools/swaggertosdk/SwaggerToSdkCore.py index 47914bdd453..c94d64ce6e8 100644 --- a/tools/azure-sdk-tools/packaging_tools/swaggertosdk/SwaggerToSdkCore.py +++ b/tools/azure-sdk-tools/packaging_tools/swaggertosdk/SwaggerToSdkCore.py @@ -24,7 +24,7 @@ _LOGGER = logging.getLogger(__name__) -CONFIG_FILE = 'swagger_to_sdk_config.json' +CONFIG_FILE = 'swagger_to_sdk_config_autorest.json' DEFAULT_COMMIT_MESSAGE = 'Generated from {hexsha}' diff --git a/tools/azure-sdk-tools/packaging_tools/swaggertosdk/autorest_tools.py b/tools/azure-sdk-tools/packaging_tools/swaggertosdk/autorest_tools.py index c51325dde2e..863824ed79d 100644 --- a/tools/azure-sdk-tools/packaging_tools/swaggertosdk/autorest_tools.py +++ b/tools/azure-sdk-tools/packaging_tools/swaggertosdk/autorest_tools.py @@ -133,7 +133,7 @@ def execute_simple_command(cmd_line, cwd=None, shell=False, env=None): output_buffer = [] for line in process.stdout: output_buffer.append(line.rstrip()) - _LOGGER.info(output_buffer[-1]) + _LOGGER.info(f"==[autorest]"+output_buffer[-1]) process.wait() output = "\n".join(output_buffer) if process.returncode: diff --git a/tools/azure-sdk-tools/setup.py b/tools/azure-sdk-tools/setup.py index 13e65b579bf..ce0af12e128 100644 --- a/tools/azure-sdk-tools/setup.py +++ b/tools/azure-sdk-tools/setup.py @@ -37,6 +37,8 @@ 'console_scripts': [ 'generate_package=packaging_tools.generate_package:generate_main', 'generate_sdk=packaging_tools.generate_sdk:generate_main', + 'auto_codegen=packaging_tools.auto_codegen:generate_main', + 'auto_package=packaging_tools.auto_package:generate_main', ], }, extras_require={