Support third-party stub external dependencies in pytype

.github/workflows/daily.yml

@@ -71,7 +71,7 @@ jobs:
       - name: Run stubtest
         shell: bash
         run: |
-          PACKAGES=$(python tests/get_packages.py)
+          PACKAGES=$(python tests/get_stubtest_system_requirements.py)
 
           if [ "${{ runner.os }}" = "Linux" ]; then
             if [ -n "$PACKAGES" ]; then

.github/workflows/stubtest_third_party.yml

@@ -58,7 +58,7 @@ jobs:
 
           if [ -n "$STUBS" ]; then
             echo "Testing $STUBS..."
-            PACKAGES=$(python tests/get_packages.py $STUBS)
+            PACKAGES=$(python tests/get_stubtest_system_requirements.py $STUBS)
 
             if [ "${{ runner.os }}" = "Linux" ]; then
               if [ -n "$PACKAGES" ]; then

.github/workflows/tests.yml

@@ -7,8 +7,8 @@ on:
       - main
   pull_request:
     paths-ignore:
-      - '**/*.md'
-      - 'scripts/**'
+      - "**/*.md"
+      - "scripts/**"
 
 permissions:
   contents: read
@@ -69,6 +69,13 @@ jobs:
           cache: pip
           cache-dependency-path: requirements-tests.txt
       - run: pip install -r requirements-tests.txt
+      - name: Install external dependencies for 3rd-party stubs
+        run: |
+          DEPENDENCIES=$(python tests/get_external_stub_requirements.py)
+          if [ -n "$DEPENDENCIES" ]; then
+            echo "Installing packages: $DEPENDENCIES"
+            pip install $DEPENDENCIES
+          fi
       - run: ./tests/pytype_test.py --print-stderr
 
   mypy:
@@ -112,12 +119,26 @@ jobs:
       fail-fast: false
     steps:
       - uses: actions/checkout@v3
+      - uses: actions/setup-python@v4
+        with:
+          python-version: "3.10"
+          cache: pip
+          cache-dependency-path: requirements-tests.txt
+      - name: Install external dependencies for 3rd-party stubs
+        run: |
+          pip install -r requirements-tests.txt
+          DEPENDENCIES=$(python tests/get_external_stub_requirements.py)
+          if [ -n "$DEPENDENCIES" ]; then
+            echo "Installing packages: $DEPENDENCIES"
+            pip install $DEPENDENCIES
+          fi
+      - run: pip freeze --all
       - name: Get pyright version
         uses: SebRollen/toml-action@v1.0.2
         id: pyright_version
         with:
-          file: 'pyproject.toml'
-          field: 'tool.typeshed.pyright_version'
+          file: "pyproject.toml"
+          field: "tool.typeshed.pyright_version"
       - uses: jakebailey/pyright-action@v1
         with:
           version: ${{ steps.pyright_version.outputs.value }}
@@ -137,7 +158,7 @@ jobs:
           version: ${{ steps.pyright_version.outputs.value }}
           python-platform: ${{ matrix.python-platform }}
           python-version: ${{ matrix.python-version }}
-          no-comments: ${{ matrix.python-version != '3.10' || matrix.python-platform != 'Linux' }}  # Having each job create the same comment is too noisy.
+          no-comments: ${{ matrix.python-version != '3.10' || matrix.python-platform != 'Linux' }} # Having each job create the same comment is too noisy.
 
   stub-uploader:
     name: Run the stub_uploader tests

CONTRIBUTING.md

@@ -46,9 +46,11 @@ virtual environment. If you're not familiar with what it is and how it works,
 please refer to this
 [documentation](https://packaging.python.org/guides/installing-using-pip-and-virtual-environments/).
 
+Some tests require extra setup steps to install the required dependencies.
+
 ### Linux/Mac OS
 
-On Linux and Mac OS, you will be able to run the full test suite on Python 3.8,
+On Linux and Mac OS, you will be able to run the full test suite on Python
 3.9 or 3.10.
 To install the necessary requirements, run the following commands from a
 terminal window:

stubs/D3DShot/METADATA.toml

@@ -1,5 +1,5 @@
 version = "0.1.*"
-requires = ["types-Pillow"]
+requires = ["types-Pillow", "numpy"]
 
 [tool.stubtest]
 # TODO: figure out how to run stubtest for this package

stubs/JACK-Client/METADATA.toml

@@ -1,4 +1,5 @@
 version = "0.5.*"
+requires = ["numpy"]
 
 [tool.stubtest]
 ignore_missing_stub = false

stubs/pycocotools/METADATA.toml

@@ -1,4 +1,5 @@
 version = "2.0.*"
+requires = ["numpy"]
 
 [tool.stubtest]
 ignore_missing_stub = false

tests/README.md

@@ -17,7 +17,18 @@ objects at runtime.
 in the `tests` and `scripts` directories.
 
 To run the tests, follow the [setup instructions](../CONTRIBUTING.md#preparing-the-environment)
-in the `CONTRIBUTING.md` document. In particular, we recommend running with Python 3.9+.
+in the `CONTRIBUTING.md` document. In particular, you have to run with Python 3.9+.
+
+In order for `pytype_test` and `pyright_test` to work correctly, some third-party stubs may require dependencies external to typeshed to be installed in your virtual environment prior to running the test.
+You can list or install all of a stubs package's external dependencies using the following script:
+```bash
+(.venv3)$ python tests/get_external_stub_requirements.py <third_party_stub>  # List external dependencies for <third_party_stub>
+(.venv3)$ python tests/get_external_stub_requirements.py <third_party_stub1> <third_party_stub2>  # List external dependencies for <third_party_stub1> and <third_party_stub2>
+(.venv3)$ python tests/get_external_stub_requirements.py  # List external dependencies for all third-party stubs in typeshed
+# Install external dependencies for all third-party stubs in typeshed
+(.venv3)$ DEPENDENCIES=$(python tests/get_external_stub_requirements.py)
+(.venv3)$ if [ -n "$DEPENDENCIES" ]; then pip install $DEPENDENCIES; fi
+```
 
 ## Run all tests for a specific stub
 

tests/get_external_stub_requirements.py

@@ -0,0 +1,18 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+
+import os
+import sys
+
+from utils import read_dependencies
+
+distributions = sys.argv[1:]
+if not distributions:
+    distributions = os.listdir("stubs")
+
+requirements = set[str]()
+for distribution in distributions:
+    requirements.update(read_dependencies(distribution).external_pkgs)
+
+for requirement in sorted(requirements):
+    print(requirement)

tests/utils.py

@@ -7,10 +7,10 @@
 import subprocess
 import sys
 import venv
-from collections.abc import Mapping
+from collections.abc import Iterable, Mapping
 from functools import cache
 from pathlib import Path
-from typing import Iterable, NamedTuple
+from typing import NamedTuple
 from typing_extensions import Annotated
 
 import pathspec  # type: ignore[import]