Allow the use of arbitrary Pyodide versions

cibuildwheel/pyodide.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import json
 import os
 import shutil
 import sys
@@ -41,6 +42,7 @@ class PythonConfiguration:
     version: str
     identifier: str
     pyodide_version: str
+    pyodide_build_version: str
     emscripten_version: str
     node_version: str
 
@@ -65,11 +67,58 @@ def install_emscripten(tmp: Path, version: str) -> Path:
     return emcc_path
 
 
-def install_xbuildenv(env: dict[str, str], pyodide_version: str) -> str:
+def search_xbuildenv(env: dict[str, str]) -> list[str]:
+    """Searches for the compatible xbuildenvs for the current pyodide-build version"""
+    with FileLock(CIBW_CACHE_PATH / "xbuildenv.lock"):
+        xbuildenvs = call(
+            "pyodide",
+            "xbuildenv",
+            "search",
+            "--json",
+            "--all",
+            env=env,
+            cwd=CIBW_CACHE_PATH,
+            capture_stdout=True,
+        ).strip()
+        xbuildenvs_dict = json.loads(xbuildenvs)
+        compatible_xbuildenv_versions = [
+            env["version"] for env in xbuildenvs_dict["environments"] if env["compatible"]
+        ]
+
+        return compatible_xbuildenv_versions
+
+
+# The xbuildenv version is brought in sync with the pyodide-build version in build-platforms.toml,
+# which will always be compatible. Hence, this condition really checks only for the case where the
+# version is supplied manually through a CIBW_PYODIDE_VERSION environment variable and raises an
+# error as appropriate.
+def validate_xbuildenv(
+    cibw_pyodide_version: str, pyodide_build_version: str, compatible_versions: list[str]
+) -> None:
+    """Validate the Pyodide version if set manually for the current pyodide-build version
+    against a list of compatible versions."""
+
+    if cibw_pyodide_version not in compatible_versions:
+        msg = (
+            f"The xbuildenv version {cibw_pyodide_version} is not compatible with the pyodide-build"
+            f" version {pyodide_build_version}. The compatible versions available to download are:"
+            f" {compatible_versions}. Please use the 'pyodide xbuildenv search' command to"
+            f" find the compatible versions for {pyodide_build_version}"
+        )
+        raise errors.FatalError(msg)
+
+
+def install_xbuildenv(env: dict[str, str], pyodide_build_version: str, pyodide_version: str) -> str:
+    """Install a particular Pyodide xbuildenv version and set a path to the Pyodide root."""
+    # Since pyodide-build was unvendored from Pyodide v0.27.0, the versions of pyodide-build are
+    # not guaranteed to match the versions of Pyodide or be in sync with them. Hence, we shall
+    # specify the pyodide-build version in the root path, which will set up the xbuildenv for
+    # the requested Pyodide version.
     pyodide_root = (
         CIBW_CACHE_PATH
-        / f".pyodide-xbuildenv-{pyodide_version}/{pyodide_version}/xbuildenv/pyodide-root"
+        / f".pyodide-xbuildenv-{pyodide_build_version}/{pyodide_version}/xbuildenv/pyodide-root"
     )
+
     with FileLock(CIBW_CACHE_PATH / "xbuildenv.lock"):
         if pyodide_root.exists():
             return str(pyodide_root)
@@ -78,6 +127,8 @@ def install_xbuildenv(env: dict[str, str], pyodide_version: str) -> str:
         # PYODIDE_ROOT so copy it first.
         env = dict(env)
         env.pop("PYODIDE_ROOT", None)
+
+        # 3. Install the xbuildenv
         call(
             "pyodide",
             "xbuildenv",
@@ -165,13 +216,28 @@ def setup_python(
         env=env,
     )
 
-    log.step("Installing emscripten...")
+    log.step(f"Installing Emscripten version: {python_configuration.emscripten_version} ...")
     emcc_path = install_emscripten(tmp, python_configuration.emscripten_version)
 
     env["PATH"] = os.pathsep.join([str(emcc_path.parent), env["PATH"]])
 
-    log.step("Installing Pyodide xbuildenv...")
-    env["PYODIDE_ROOT"] = install_xbuildenv(env, python_configuration.pyodide_version)
+    # Allow overriding the xbuildenv version with an environment variable. This allows
+    # testing new Pyodide xbuildenv versions before they are officially released, or for
+    # using a different Pyodide version other than the one that is listed to be compatible
+    # with the pyodide-build version in the build-platforms.toml file.
+    cibw_pyodide_version = os.environ.get(
+        "CIBW_PYODIDE_VERSION", python_configuration.pyodide_version
+    )
+    log.step(f"Installing Pyodide xbuildenv version: {cibw_pyodide_version} ...")
+    # Search for compatible xbuildenv versions
+    compatible_versions = search_xbuildenv(env)
+    # and then validate the xbuildenv version
+    validate_xbuildenv(
+        cibw_pyodide_version, python_configuration.pyodide_build_version, compatible_versions
+    )
+    env["PYODIDE_ROOT"] = install_xbuildenv(
+        env, python_configuration.pyodide_build_version, cibw_pyodide_version
+    )
 
     return env
 
@@ -219,7 +285,10 @@ def build(options: Options, tmp_path: Path) -> None:
 
             log.build_start(config.identifier)
 
-            identifier_tmp_dir = tmp_path / config.identifier
+            # Include both the identifier and the Pyodide version in the temp directory name
+            cibw_pyodide_version = os.environ.get("CIBW_PYODIDE_VERSION", config.pyodide_version)
+            identifier_tmp_dir = tmp_path / f"{config.identifier}_{cibw_pyodide_version}"
+
             built_wheel_dir = identifier_tmp_dir / "built_wheel"
             repaired_wheel_dir = identifier_tmp_dir / "repaired_wheel"
             identifier_tmp_dir.mkdir()

cibuildwheel/resources/build-platforms.toml

@@ -190,5 +190,5 @@ python_configurations = [
 
 [pyodide]
 python_configurations = [
-  { identifier = "cp312-pyodide_wasm32", version = "3.12.1", pyodide_version = "0.26.1", emscripten_version = "3.1.58", node_version = "v20" },
+  { identifier = "cp312-pyodide_wasm32", version = "3.12.1", pyodide_version = "0.26.4", pyodide_build_version = "0.29.0", emscripten_version = "3.1.58", node_version = "v20" },
 ]

cibuildwheel/resources/cibuildwheel.schema.json

@@ -890,6 +890,11 @@
         },
         "test-requires": {
           "$ref": "#/properties/test-requires"
+        },
+        "pyodide-version": {
+          "description": "Specify the Pyodide xbuildenv version to use for building",
+          "type": "string",
+          "title": "CIBW_PYODIDE_VERSION"
         }
       }
     }

cibuildwheel/resources/constraints-pyodide312.txt

@@ -19,8 +19,6 @@ charset-normalizer==3.4.0
     # via requests
 click==8.1.7
     # via typer
-cloudpickle==3.1.0
-    # via loky
 cmake==3.31.0.1
     # via pyodide-build
 distlib==0.3.9
@@ -40,8 +38,6 @@ idna==3.10
     #   requests
 leb128==1.0.8
     # via auditwheel-emscripten
-loky==3.4.1
-    # via pyodide-build
 markdown-it-py==3.0.0
     # via rich
 mdurl==0.1.2
@@ -56,26 +52,24 @@ pip==24.3.1
     # via -r .nox/update_constraints/tmp/constraints-pyodide.in
 platformdirs==4.3.6
     # via virtualenv
-pydantic==2.9.2
+pydantic==2.10.0
     # via
     #   pyodide-build
     #   pyodide-lock
-pydantic-core==2.23.4
+pydantic-core==2.27.0
     # via pydantic
 pygments==2.18.0
     # via rich
-pyodide-build==0.26.1
+pyodide-build==0.29.0
     # via -r .nox/update_constraints/tmp/constraints-pyodide.in
 pyodide-cli==0.2.4
     # via
     #   auditwheel-emscripten
     #   pyodide-build
-pyodide-lock==0.1.0a6
+pyodide-lock==0.1.0a7
     # via pyodide-build
 pyproject-hooks==1.2.0
     # via build
-pyyaml==6.0.2
-    # via pyodide-build
 requests==2.32.3
     # via pyodide-build
 resolvelib==1.1.0
@@ -95,13 +89,11 @@ sniffio==1.3.1
     # via
     #   anyio
     #   httpx
-typer==0.13.0
+typer==0.13.1
     # via
     #   auditwheel-emscripten
     #   pyodide-build
     #   pyodide-cli
-types-requests==2.32.0.20241016
-    # via pyodide-build
 typing-extensions==4.12.2
     # via
     #   pydantic
@@ -110,9 +102,7 @@ typing-extensions==4.12.2
 unearth==0.17.2
     # via pyodide-build
 urllib3==2.2.3
-    # via
-    #   requests
-    #   types-requests
+    # via requests
 virtualenv==20.27.1
     # via
     #   build

docs/options.md

@@ -255,7 +255,7 @@ Default: `auto`
 
 - For `linux`, you need [Docker or Podman](#container-engine) running, on Linux, macOS, or Windows.
 - For `macos` and `windows`, you need to be running on the respective system, with a working compiler toolchain installed - Xcode Command Line tools for macOS, and MSVC for Windows.
-- For `pyodide` you need to be on an x86-64 linux runner and `python3.12` must be available in `PATH`.
+- For `pyodide` `python3.12` must be available in `PATH` and you need to be on one of the following runners: x86-64 Linux, arm64 Linux. Intel and Silicon macOS hosts may work, too. See [the section on Pyodide](setup.md#pyodide-(WebAssembly)-builds-(experimental)) for more information.
 
 This option can also be set using the [command-line option](#command-line) `--platform`. This option is not available in the `pyproject.toml` config.
 
@@ -321,7 +321,7 @@ If you set the value lower, cibuildwheel will cap it to the lowest supported val
 Windows arm64 platform support is experimental.
 
 For an experimental WebAssembly build with `--platform pyodide`,
-`cp312-pyodide_wasm32` is the only platform identifier.
+`cp312-pyodide_wasm32` is the only platform identifier, corresponding to [Pyodide version `0.26.4`](https://github.com/pyodide/pyodide/releases/tag/0.26.4).
 
 See the [cibuildwheel 1 documentation](https://cibuildwheel.pypa.io/en/1.x/) for past end-of-life versions of Python, and PyPy2.7.
 

docs/setup.md

@@ -113,7 +113,14 @@ Pre-requisite: you need to have a matching host version of Python (unlike all
 other cibuildwheel platforms). Linux host highly recommended; macOS hosts may
 work (e.g. invoking `pytest` directly in [`CIBW_TEST_COMMAND`](options.md#test-command) is [currently failing](https://github.com/pyodide/pyodide/issues/4802)) and Windows hosts will not work.
 
-You must target pyodide with `--platform pyodide` (or use `--only` on the identifier).
+You must target Pyodide with `--platform pyodide` (or use `--only` on the identifier).
+
+!!!tip
+
+    It is also possible to target a specific Pyodide version by setting the `CIBW_PYODIDE_VERSION` environment variable to the desired version. Users are responsible for setting an appropriate Pyodide version according to the `pyodide-build` version. A list is available in Pyodide's [cross-build environments metadata file](https://github.com/pyodide/pyodide/blob/main/pyodide-cross-build-environments.json) or can be
+    referenced using the `pyodide xbuildenv search` command.
+
+    This option is **not available** in the `pyproject.toml` config.
 
 ## Configure a CI service
 

noxfile.py

@@ -106,10 +106,10 @@ def update_constraints(session: nox.Session) -> None:
     pyodides = build_platforms["pyodide"]["python_configurations"]
     for pyodide in pyodides:
         python_version = ".".join(pyodide["version"].split(".")[:2])
-        pyodide_version = pyodide["pyodide_version"]
+        pyodide_build_version = pyodide["pyodide_build_version"]
         output_file = resources / f"constraints-pyodide{python_version.replace('.', '')}.txt"
         tmp_file = Path(session.create_tmp()) / "constraints-pyodide.in"
-        tmp_file.write_text(f"pip\nbuild[virtualenv]\npyodide-build=={pyodide_version}")
+        tmp_file.write_text(f"pip\nbuild[virtualenv]\npyodide-build=={pyodide_build_version}")
         session.run(
             "uv",
             "pip",

test/test_emscripten.py

@@ -20,7 +20,7 @@
 
 def check_node():
     # cibuildwheel adds a pinned node version to the PATH
-    # check it's in the PATH then, check it's the one that runs pyoodide
+    # check it's in the PATH then, check it's the one that runs pyodide
     cibw_cache_path = Path(sys.argv[1]).resolve(strict=True)
     # find the node executable in PATH
     node = shutil.which("node")