diff --git a/.github/workflows/daily.yml b/.github/workflows/daily.yml index 5522c8cb66ca..04b97e54b0d8 100644 --- a/.github/workflows/daily.yml +++ b/.github/workflows/daily.yml @@ -44,9 +44,10 @@ jobs: stubtest-third-party: name: Check third party stubs with stubtest if: ${{ github.repository == 'python/typeshed' || github.event_name == 'workflow_dispatch' }} - runs-on: ubuntu-20.04 + runs-on: ${{ matrix.os }} strategy: matrix: + os: ["ubuntu-latest", "windows-latest", "macos-latest"] shard-index: [0, 1, 2, 3] fail-fast: false steps: @@ -56,12 +57,17 @@ jobs: python-version: "3.9" - name: Install dependencies run: pip install -r requirements-tests.txt - - name: Install apt packages - run: | - sudo apt update - sudo apt install -y $(python tests/get_apt_packages.py) - name: Run stubtest - run: xvfb-run python tests/stubtest_third_party.py --num-shards 4 --shard-index ${{ matrix.shard-index }} + shell: bash + run: | + if [ "${{ matrix.os }}" = "ubuntu-latest" ]; then + sudo apt update + sudo apt install -y $(python tests/get_packages.py) + + xvfb-run python tests/stubtest_third_party.py --num-shards 4 --shard-index ${{ matrix.shard-index }} + else + python tests/stubtest_third_party.py --num-shards 4 --shard-index ${{ matrix.shard-index }} + fi stub-uploader: name: Run the stub_uploader tests diff --git a/.github/workflows/stubtest_stdlib.yml b/.github/workflows/stubtest_stdlib.yml index f0f020af0c52..e470be761637 100644 --- a/.github/workflows/stubtest_stdlib.yml +++ b/.github/workflows/stubtest_stdlib.yml @@ -1,4 +1,4 @@ -name: Run stubtest +name: Stdlib stubtest on: workflow_dispatch: diff --git a/.github/workflows/stubtest_third_party.yml b/.github/workflows/stubtest_third_party.yml new file mode 100644 index 000000000000..29cac9b3f95f --- /dev/null +++ b/.github/workflows/stubtest_third_party.yml @@ -0,0 +1,70 @@ +name: Third-party stubtest + +on: + pull_request: + paths-ignore: + - '**/*.md' + - 'scripts/**' + +permissions: + contents: read + +env: + PIP_DISABLE_PIP_VERSION_CHECK: 1 + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +jobs: + stubtest-third-party: + name: Check third party stubs with stubtest + + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: ["ubuntu-latest", "windows-latest", "macos-latest"] + fail-fast: false + + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + - uses: actions/setup-python@v4 + with: + python-version: "3.9" + - name: Install dependencies + run: pip install -r requirements-tests.txt + - name: Run stubtest + shell: bash + run: | + STUBS=$( + git diff --name-only origin/${{ github.base_ref }} HEAD | + # Use the daily.yml workflow to run stubtest on all third party stubs + egrep ^stubs/ | cut -d "/" -f 2 | sort -u | (while read stub; do [ -d stubs/$stub ] && echo $stub || true; done) + ) + + if [ -n "$STUBS" ]; then + echo "Testing $STUBS..." + PACKAGES=$(python tests/get_packages.py $STUBS) + + if [ "${{ matrix.os }}" = "ubuntu-latest" ]; then + if [ -n "$PACKAGES" ]; then + echo "Installing apt packages: $PACKAGES" + sudo apt update && sudo apt install -y $PACKAGES + fi + xvfb-run python tests/stubtest_third_party.py $STUBS + fi + + if [ "${{ matrix.os }}" = "macos-latest" ]; then + # Could install brew packages here if we run into stubs that need it + python tests/stubtest_third_party.py $STUBS + fi + + if [ "${{ matrix.os }}" = "windows-latest" ]; then + # Could install choco packages here if we run into stubs that need it + python tests/stubtest_third_party.py $STUBS + fi + else + echo "Nothing to test" + fi diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 42ac66a3cc92..59790c4635cc 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -146,36 +146,3 @@ jobs: cd stub_uploader pip install -r requirements.txt python -m pytest tests - - stubtest-third-party: - name: Check third party stubs with stubtest - runs-on: ubuntu-20.04 - if: github.event_name == 'pull_request' - steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - uses: actions/setup-python@v4 - with: - python-version: "3.9" - - name: Install dependencies - run: pip install -r requirements-tests.txt - - name: Run stubtest - run: | - STUBS=$( - git diff --name-only origin/${{ github.base_ref }} HEAD | - # Uncomment the following to (very slowly) run on all third party stubs: - # git ls-files | - egrep ^stubs/ | cut -d "/" -f 2 | sort -u | (while read stub; do [ -d stubs/$stub ] && echo $stub || true; done) - ) - if test -n "$STUBS"; then - echo "Testing $STUBS..." - APT_PACKAGES=$(python tests/get_apt_packages.py $STUBS) - if test -n "$APT_PACKAGES"; then - echo "Installing apt packages: $APT_PACKAGES" - sudo apt update && sudo apt install -y $APT_PACKAGES - fi - xvfb-run python tests/stubtest_third_party.py $STUBS - else - echo "Nothing to test" - fi diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index dee7be8e89fb..ceba9b87ab07 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -192,6 +192,11 @@ This has the following keys: * `apt_dependencies` (default: `[]`): A list of Ubuntu APT packages that need to be installed for stubtest to run successfully. These are usually packages needed to pip install the implementation distribution. +* `platforms` (default: `["linux"]`): A list of OSes on which to run stubtest. + Can contain `win32`, `linux`, and `darwin` values. + If not specified, stubtest is run only on `linux`. + Only add extra OSes to the test + if there are platform-specific branches in a stubs package. The format of all `METADATA.toml` files can be checked by running `python3 ./tests/check_consistent.py`. diff --git a/stubs/D3DShot/METADATA.toml b/stubs/D3DShot/METADATA.toml index d6bdd1ba0ba9..b66c346f9a8b 100644 --- a/stubs/D3DShot/METADATA.toml +++ b/stubs/D3DShot/METADATA.toml @@ -2,6 +2,7 @@ version = "0.1.*" requires = ["types-Pillow"] [tool.stubtest] -# The library only works on Windows; we currently only run stubtest on Ubuntu for third-party stubs in CI. -# See #8660 +# TODO: figure out how to run stubtest for this package +# (the package pins Pillow in a problematic way) skip = true +platforms = ["win32"] diff --git a/stubs/JACK-Client/METADATA.toml b/stubs/JACK-Client/METADATA.toml index 5e65c85ad252..6052c937d493 100644 --- a/stubs/JACK-Client/METADATA.toml +++ b/stubs/JACK-Client/METADATA.toml @@ -1,4 +1,5 @@ version = "0.5.*" [tool.stubtest] +platforms = ["linux"] apt_dependencies = ["libjack-dev"] diff --git a/stubs/pyaudio/METADATA.toml b/stubs/pyaudio/METADATA.toml index a259ccae9200..f1492c2a5753 100644 --- a/stubs/pyaudio/METADATA.toml +++ b/stubs/pyaudio/METADATA.toml @@ -1,4 +1,5 @@ version = "0.2.*" [tool.stubtest] +platforms = ["linux"] apt_dependencies = ["portaudio19-dev"] diff --git a/stubs/pycurl/METADATA.toml b/stubs/pycurl/METADATA.toml index 575cf2229e09..06b23b2b8a2f 100644 --- a/stubs/pycurl/METADATA.toml +++ b/stubs/pycurl/METADATA.toml @@ -1,4 +1,5 @@ version = "7.45.*" [tool.stubtest] +platforms = ["linux"] apt_dependencies = ["libcurl4-openssl-dev"] diff --git a/stubs/pywin32/@tests/stubtest_allowlist.txt b/stubs/pywin32/@tests/stubtest_allowlist_win32.txt similarity index 100% rename from stubs/pywin32/@tests/stubtest_allowlist.txt rename to stubs/pywin32/@tests/stubtest_allowlist_win32.txt diff --git a/stubs/pywin32/METADATA.toml b/stubs/pywin32/METADATA.toml index b2d4e9598153..69102f70711c 100644 --- a/stubs/pywin32/METADATA.toml +++ b/stubs/pywin32/METADATA.toml @@ -1,6 +1,5 @@ version = "305.*" + [tool.stubtest] +platforms = ["win32"] ignore_missing_stub = false -# The library only works on Windows; we currently only run stubtest on Ubuntu for third-party stubs in CI. -# See #8660 -skip = true diff --git a/tests/check_consistent.py b/tests/check_consistent.py index a22d908d61f1..b0a55d265a2f 100755 --- a/tests/check_consistent.py +++ b/tests/check_consistent.py @@ -18,7 +18,7 @@ from utils import VERSIONS_RE, get_all_testcase_directories, get_gitignore_spec, spec_matches_path, strip_comments metadata_keys = {"version", "requires", "extra_description", "obsolete_since", "no_longer_updated", "tool"} -tool_keys = {"stubtest": {"skip", "apt_dependencies", "extras", "ignore_missing_stub"}} +tool_keys = {"stubtest": {"skip", "apt_dependencies", "extras", "ignore_missing_stub", "platforms"}} extension_descriptions = {".pyi": "stub", ".py": ".py"} diff --git a/tests/get_apt_packages.py b/tests/get_apt_packages.py deleted file mode 100755 index 9a902ee99d1d..000000000000 --- a/tests/get_apt_packages.py +++ /dev/null @@ -1,14 +0,0 @@ -#!/usr/bin/env python3 -import os -import sys - -import tomli - -distributions = sys.argv[1:] -if not distributions: - distributions = os.listdir("stubs") - -for distribution in distributions: - with open(f"stubs/{distribution}/METADATA.toml", "rb") as file: - for apt_package in tomli.load(file).get("tool", {}).get("stubtest", {}).get("apt_dependencies", []): - print(apt_package) diff --git a/tests/get_packages.py b/tests/get_packages.py new file mode 100755 index 000000000000..dcd33c606334 --- /dev/null +++ b/tests/get_packages.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python3 +import os +import sys + +import tomli + +platform = sys.platform +distributions = sys.argv[1:] +if not distributions: + distributions = os.listdir("stubs") + +metadata_mapping = { + "linux": "apt_dependencies", + # We could add others here if we run into stubs that need it: + # "darwin": "brew_dependencies", + # "win32": "choco_dependencies", +} + +if platform in metadata_mapping: + for distribution in distributions: + with open(f"stubs/{distribution}/METADATA.toml", "rb") as file: + for package in tomli.load(file).get("tool", {}).get("stubtest", {}).get(metadata_mapping[platform], []): + print(package) diff --git a/tests/stubtest_third_party.py b/tests/stubtest_third_party.py index 2f3e55845ab6..49892435b795 100755 --- a/tests/stubtest_third_party.py +++ b/tests/stubtest_third_party.py @@ -34,6 +34,11 @@ def run_stubtest(dist: Path, *, verbose: bool = False) -> bool: print(colored("skipping", "yellow")) return True + platforms_to_test = stubtest_meta.get("platforms", ["linux"]) + if sys.platform not in platforms_to_test: + print(colored(f"skipping, unsupported platform: {sys.platform}, supported: {platforms_to_test}", "yellow")) + return True + with tempfile.TemporaryDirectory() as tmp: venv_dir = Path(tmp) venv.create(venv_dir, with_pip=True, clear=True) @@ -57,8 +62,8 @@ def run_stubtest(dist: Path, *, verbose: bool = False) -> bool: # If @tests/requirements-stubtest.txt exists, run "pip install" on it. req_path = dist / "@tests" / "requirements-stubtest.txt" if req_path.exists(): + pip_cmd = [pip_exe, "install", "-r", str(req_path)] try: - pip_cmd = [pip_exe, "install", "-r", str(req_path)] subprocess.run(pip_cmd, check=True, capture_output=True) except subprocess.CalledProcessError as e: print_command_failure("Failed to install requirements", e) @@ -102,6 +107,9 @@ def run_stubtest(dist: Path, *, verbose: bool = False) -> bool: allowlist_path = dist / "@tests/stubtest_allowlist.txt" if allowlist_path.exists(): stubtest_cmd.extend(["--allowlist", str(allowlist_path)]) + platform_allowlist = dist / f"@tests/stubtest_allowlist_{sys.platform}.txt" + if platform_allowlist.exists(): + stubtest_cmd.extend(["--allowlist", str(platform_allowlist)]) try: subprocess.run(stubtest_cmd, env=stubtest_env, check=True, capture_output=True)