diff --git a/build_package.py b/build_package.py index af628a155799..cb0064c33d7a 100644 --- a/build_package.py +++ b/build_package.py @@ -19,7 +19,7 @@ def create_package(name, dest_folder=DEFAULT_DEST_FOLDER): absdirs = [os.path.dirname(package) for package in (glob.glob('{}/setup.py'.format(name)) + glob.glob('sdk/*/{}/setup.py'.format(name)))] absdirpath = os.path.abspath(absdirs[0]) - check_call(['python', 'setup.py', 'bdist_wheel', '--universal', '-d', dest_folder], cwd=absdirpath) + check_call(['python', 'setup.py', 'bdist_wheel', '-d', dest_folder], cwd=absdirpath) check_call(['python', 'setup.py', "sdist", "--format", "zip", '-d', dest_folder], cwd=absdirpath) diff --git a/doc/dev/engineering_assumptions.md b/doc/dev/engineering_assumptions.md index 7cf9840f78f9..7b910c9111c5 100644 --- a/doc/dev/engineering_assumptions.md +++ b/doc/dev/engineering_assumptions.md @@ -1,6 +1,12 @@ ## Engineering System Assumptions, Gotchas, and Minutiae -1. All wheels are generated with `--universal` flag passed to `bdist_wheel`. We do not currently allow non-universal packages to be shipped out of this repository. +1. All wheels are generated with influence from `setup.cfg` at the built package root. This means that for most of our packages, the `universal` flag is set within the `setup.cfg`. + +``` +[bdist_wheel] +universal=1 +``` +2. If a package does _not_ have `universal` set, then the default build will produce a wheel for `py3`. This is due to the fact that wheels are currently generated by a CI step running `Python 3.6`. ## Build @@ -15,3 +21,7 @@ Build CI for `azure-sdk-for-python` essentially builds and tests packages in one 1. Install on packages (and their dev_requirements!) in one go. 2. Run `pytest , pytest ` where folders correspond to package folders 1. While all packages are installed alongside each other, each test run is individual to the package. This has the benefit of not allowing `packageA`'s `conftest.py` to mess with `packageB`'s environment.' + +### So when do they run? + +A standard pull request will option target the `Individual Packages` option. On a nightly cadence, the `python - client` build installs the entire world and tests in _both_ methodologies outlined above. diff --git a/eng/pipelines/templates/steps/build-artifacts.yml b/eng/pipelines/templates/steps/build-artifacts.yml index 18e5ebdca8b2..1d21d34003d9 100644 --- a/eng/pipelines/templates/steps/build-artifacts.yml +++ b/eng/pipelines/templates/steps/build-artifacts.yml @@ -15,7 +15,7 @@ steps: versionSpec: $(PythonVersion) - script: | - pip install wheel setuptools pathlib twine readme-renderer[md] + pip install wheel setuptools pathlib twine readme-renderer[md] packaging displayName: 'Prep Environment' - task: PythonScript@0 diff --git a/eng/pipelines/templates/steps/build-test.yml b/eng/pipelines/templates/steps/build-test.yml index e614254fe18e..b31e30f405ad 100644 --- a/eng/pipelines/templates/steps/build-test.yml +++ b/eng/pipelines/templates/steps/build-test.yml @@ -35,7 +35,7 @@ steps: arguments: ${{ parameters.OSName }} - script: | - pip install pathlib twine codecov beautifulsoup4 tox tox-monorepo + pip install pathlib twine codecov beautifulsoup4 tox tox-monorepo packaging displayName: 'Prep Environment' - ${{ parameters.BeforeTestSteps }} diff --git a/eng/pipelines/templates/steps/test-nightly.yml b/eng/pipelines/templates/steps/test-nightly.yml index 367a7ad894e1..c5c3f327853a 100644 --- a/eng/pipelines/templates/steps/test-nightly.yml +++ b/eng/pipelines/templates/steps/test-nightly.yml @@ -34,7 +34,7 @@ steps: arguments: ${{ parameters.OSName }} - script: | - pip install pathlib twine codecov beautifulsoup4 tox tox-monorepo + pip install pathlib twine codecov beautifulsoup4 tox tox-monorepo packaging displayName: 'Prep Environment' - ${{ parameters.BeforeTestSteps }} diff --git a/eng/tox/create_wheel_and_install.py b/eng/tox/create_wheel_and_install.py index 8151b6886aef..50d4e87cc6de 100644 --- a/eng/tox/create_wheel_and_install.py +++ b/eng/tox/create_wheel_and_install.py @@ -59,7 +59,6 @@ def cleanup_build_artifacts(build_folder): "python", os.path.join(args.target_setup, "setup.py"), "bdist_wheel", - "--universal", "-d", args.distribution_directory, ] diff --git a/scripts/devops_tasks/common_tasks.py b/scripts/devops_tasks/common_tasks.py index 45e7b1a1e4cf..bc6c5eefc178 100644 --- a/scripts/devops_tasks/common_tasks.py +++ b/scripts/devops_tasks/common_tasks.py @@ -9,18 +9,29 @@ # package targeting during release. import glob -from pathlib import Path from subprocess import check_call, CalledProcessError import os import errno import shutil import sys import logging +import ast +import textwrap +import io +import re +import pdb + +# ssumes the presence of setuptools +from pkg_resources import parse_version + +# this assumes the presence of "packaging" +from packaging.specifiers import SpecifierSet +from packaging.version import Version + logging.getLogger().setLevel(logging.INFO) OMITTED_CI_PACKAGES = ["azure-mgmt-documentdb", "azure-servicemanagement-legacy"] - MANAGEMENT_PACKAGE_IDENTIFIERS = [ "mgmt", "azure-cognitiveservices", @@ -28,17 +39,14 @@ "nspkg" ] - def log_file(file_location, is_error=False): with open(file_location, "r") as file: for line in file: sys.stdout.write(line) sys.stdout.write("\n") - # CI consistently sees outputs in the wrong location. Trying this to see if it helps sys.stdout.flush() - def read_file(file_location): str_buffer = "" with open(file_location, "r") as file: @@ -68,6 +76,57 @@ def clean_coverage(coverage_dir): else: raise +def parse_setup_requires(setup_path): + setup_filename = os.path.join(setup_path, 'setup.py') + mock_setup = textwrap.dedent('''\ + def setup(*args, **kwargs): + __setup_calls__.append((args, kwargs)) + ''') + parsed_mock_setup = ast.parse(mock_setup, filename=setup_filename) + with io.open(setup_filename, 'r', encoding='utf-8-sig') as setup_file: + parsed = ast.parse(setup_file.read()) + for index, node in enumerate(parsed.body[:]): + if ( + not isinstance(node, ast.Expr) or + not isinstance(node.value, ast.Call) or + not hasattr(node.value.func, 'id') or + node.value.func.id != 'setup' + ): + continue + parsed.body[index:index] = parsed_mock_setup.body + break + + fixed = ast.fix_missing_locations(parsed) + codeobj = compile(fixed, setup_filename, 'exec') + local_vars = {} + global_vars = {'__setup_calls__': []} + current_dir = os.getcwd() + working_dir = os.path.dirname(setup_filename) + os.chdir(working_dir) + exec(codeobj, global_vars, local_vars) + os.chdir(current_dir) + _, kwargs = global_vars['__setup_calls__'][0] + + try: + python_requires = kwargs['python_requires'] + # most do not define this, fall back to what we define as universal + except KeyError as e: + python_requires = ">=2.7" + + return python_requires + +def filter_for_compatibility(package_set): + collected_packages = [] + v = sys.version_info + running_major_version = Version(".".join([str(v[0]), str(v[1]), str(v[2])])) + + for pkg in package_set: + spec_set = SpecifierSet(parse_setup_requires(pkg)) + + if running_major_version in spec_set: + collected_packages.append(pkg) + + return collected_packages # this function is where a glob string gets translated to a list of packages # It is called by both BUILD (package) and TEST. In the future, this function will be the central location @@ -91,12 +150,17 @@ def process_glob_string(glob_string, target_root_dir): # if we have individually queued this specific package, it's obvious that we want to build it specifically # in this case, do not honor the omission list if len(collected_directories) == 1: - return collected_directories + return filter_for_compatibility(collected_directories) # however, if there are multiple packages being built, we should honor the omission list and NOT build the omitted # packages else: - return sorted(remove_omitted_packages(collected_directories)) + allowed_package_set = remove_omitted_packages(collected_directories) + logging.info("Target packages after filtering by omission list: {}".format(allowed_package_set)) + pkg_set_ci_filtered = filter_for_compatibility(allowed_package_set) + logging.info("Package(s) omitted by CI filter: {}".format(list(set(allowed_package_set) - set(pkg_set_ci_filtered)))) + + return sorted(pkg_set_ci_filtered) def remove_omitted_packages(collected_directories): return [ @@ -105,7 +169,6 @@ def remove_omitted_packages(collected_directories): if os.path.basename(package_dir) not in OMITTED_CI_PACKAGES ] - def run_check_call( command_array, working_directory, diff --git a/sdk/redis/azure-mgmt-redis/dev_requirements.txt b/sdk/redis/azure-mgmt-redis/dev_requirements.txt index 6ccb7f031ddd..f6457a93d5e8 100644 --- a/sdk/redis/azure-mgmt-redis/dev_requirements.txt +++ b/sdk/redis/azure-mgmt-redis/dev_requirements.txt @@ -1 +1 @@ --e ../../../tools/azure-sdk-tools +-e ../../../tools/azure-sdk-tools \ No newline at end of file diff --git a/sdk/storage/azure-storage-file/dev_requirements.txt b/sdk/storage/azure-storage-file/dev_requirements.txt index abd2604ca38c..49237d8ca1f3 100644 --- a/sdk/storage/azure-storage-file/dev_requirements.txt +++ b/sdk/storage/azure-storage-file/dev_requirements.txt @@ -2,4 +2,4 @@ -e ../../../tools/azure-sdk-tools -e ../../core/azure-core aiohttp>=3.0; python_version >= '3.5' -pytest +pytest \ No newline at end of file