From 44926067eb5b64d8ae8617f751208d4c1ea1360b Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Fri, 23 Feb 2024 17:58:52 +0530 Subject: [PATCH] CI: Re-enable Emscripten/Pyodide CI job for NumPy This commit performs the following actions: 1. Adds WASM builds to the CPU family for Meson configurations, but without SSE or SIMD instructions. 2. Removes CPU feature detection message for an unsupported architecture. 3. Enables `IEEE_QUAD_LE` longdouble format for the wasm32 target (cross-builds). 4. Enables run for Emscripten/Pyodide wheels by setting the `if:` condition to `true`. 5. Uses recursive submodules to ensure that vendored-meson is received. 6. Moves the Meson cross file to `tools/ci/emscripten/` (i.e., creates a separate Emscripten folder to store relevant files) 7. Adds a patch for vendored-meson detection for Pyodide and applies this Pyodide-meson patch in the Emscripten CI jobs 8. Builds wasm32 wheels without BLAS and LAPACK support (see https://github.com/numpy/numpy/issues/24750#issuecomment-1727328397) 9. Forces coloured and prettified outputs for test runs Some of these changes have been copied with updates and suggestions received from #24603 on 23/02/2024 and authorship is preserved with this commit. Co-Authored-By: Ralf Gommers --- .github/workflows/emscripten.yml | 57 ++++++++++++------- .gitignore | 4 ++ meson_cpu/meson.build | 11 ++-- ...environment-variable-pyodide-gh-4502.patch | 55 ++++++++++++++++++ tools/ci/emscripten/emscripten.meson.cross | 15 +++++ 5 files changed, 115 insertions(+), 27 deletions(-) create mode 100644 tools/ci/emscripten/0001-do-not-set-meson-environment-variable-pyodide-gh-4502.patch create mode 100644 tools/ci/emscripten/emscripten.meson.cross diff --git a/.github/workflows/emscripten.yml b/.github/workflows/emscripten.yml index 5a2dc72def88..297cae5a8f57 100644 --- a/.github/workflows/emscripten.yml +++ b/.github/workflows/emscripten.yml @@ -8,6 +8,12 @@ on: branches: - main - maintenance/** +# TODO: remove both of these after everything is ready + push: + workflow_dispatch: + +env: + FORCE_COLOR: 3 concurrency: group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} @@ -19,50 +25,61 @@ permissions: jobs: build-wasm-emscripten: runs-on: ubuntu-22.04 - if: false # NOTE: job disabled, it needs to be moved to Meson (see gh-24603) # if: "github.repository == 'numpy/numpy'" env: - PYODIDE_VERSION: 0.23.1 + PYODIDE_VERSION: 0.25.0 # PYTHON_VERSION and EMSCRIPTEN_VERSION are determined by PYODIDE_VERSION. # The appropriate versions can be found in the Pyodide repodata.json # "info" field, or in Makefile.envs: # https://github.com/pyodide/pyodide/blob/main/Makefile.envs#L2 - PYTHON_VERSION: 3.11.2 - EMSCRIPTEN_VERSION: 3.1.32 + PYTHON_VERSION: 3.11.3 + EMSCRIPTEN_VERSION: 3.1.46 NODE_VERSION: 18 steps: - name: Checkout numpy uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 with: - submodules: true - # versioneer.py requires the latest tag to be reachable. Here we - # fetch the complete history to get access to the tags. - # A shallow clone can work when the following issue is resolved: - # https://github.com/actions/checkout/issues/338 - fetch-depth: 0 + submodules: recursive + # The fetch-tags input shall fetch tags for versioneer.py + # without the need to fetch the entire history, see + # https://github.com/actions/checkout#usage + fetch-tags: true - - name: set up python + - name: Set up Python ${{ env.PYTHON_VERSION }} id: setup-python uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5.0.0 with: python-version: ${{ env.PYTHON_VERSION }} - - uses: mymindstorm/setup-emsdk@6ab9eb1bda2574c4ddb79809fc9247783eaf9021 # v14 + - name: Set up Emscripten toolchain + uses: mymindstorm/setup-emsdk@6ab9eb1bda2574c4ddb79809fc9247783eaf9021 # v14 with: version: ${{ env.EMSCRIPTEN_VERSION }} actions-cache-folder: emsdk-cache - name: Install pyodide-build - run: pip install "pydantic<2" pyodide-build==$PYODIDE_VERSION + run: pip install "pydantic<2" pyodide-build==${{ env.PYODIDE_VERSION }} + + - name: Find installation for pyodide-build + shell: python + run: | + import os + import pyodide_build + print(pyodide_build.__file__[:-12]) + env_file = os.getenv('GITHUB_ENV') + with open(env_file, "a") as myfile: + myfile.write(f"PYODIDE_BUILD_PATH={pyodide_build.__file__[:-12]}\n") + + - name: Apply patch(es) for pyodide-build installation + run: | + ls -a ${{ env.PYODIDE_BUILD_PATH }} + patch -d "${{ env.PYODIDE_BUILD_PATH }}" -p1 < tools/ci/emscripten/0001-do-not-set-meson-environment-variable-pyodide-gh-4502.patch - - name: Build + - name: Build NumPy for Pyodide run: | - # Pyodide is still in the process of adding better/easier support for - # non-setup.py based builds. - cp pyproject.toml.setuppy pyproject.toml - CFLAGS=-g2 LDFLAGS=-g2 pyodide build + pyodide build -Cbuild-dir=build -Csetup-args="--cross-file=$PWD/tools/ci/emscripten/emscripten.meson.cross" -Csetup-args="-Dblas=none" -Csetup-args="-Dlapack=none" - - name: set up node + - name: Set up Node.js uses: actions/setup-node@60edb5dd545a775178f52524783378180af0d1f8 # v4.0.2 with: node-version: ${{ env.NODE_VERSION }} @@ -80,4 +97,4 @@ jobs: run: | source .venv-pyodide/bin/activate cd .. - python numpy/runtests.py -n -vv + pytest --pyargs numpy -m "not slow" \ No newline at end of file diff --git a/.gitignore b/.gitignore index 109a3c49e543..c9598f7e598c 100644 --- a/.gitignore +++ b/.gitignore @@ -97,6 +97,10 @@ pip-wheel-metadata *.patch *.diff +# Do not ignore the following patches: # +######################################## +!tools/ci/emscripten/0001-do-not-set-meson-environment-variable-pyodide-gh-4502.patch + # OS generated files # ###################### .DS_Store* diff --git a/meson_cpu/meson.build b/meson_cpu/meson.build index 186f3e079e0c..2152b0b6aa58 100644 --- a/meson_cpu/meson.build +++ b/meson_cpu/meson.build @@ -92,13 +92,14 @@ min_features = { 'ppc64': [], 's390x': [], 'arm': [], - 'aarch64': [ASIMD] + 'aarch64': [ASIMD], + 'wasm32': [], }.get(cpu_family, []) if host_machine.endian() == 'little' and cpu_family == 'ppc64' min_features = [VSX2] endif -# Used by build option 'max/native/detect' +# Used by build option 'max' max_features_dict = { 'x86': X86_FEATURES, 'x86_64': X86_FEATURES, @@ -106,16 +107,12 @@ max_features_dict = { 's390x': S390X_FEATURES, 'arm': ARM_FEATURES, 'aarch64': ARM_FEATURES, + 'wasm32': {}, }.get(cpu_family, {}) max_features = [] foreach fet_name, fet_obj : max_features_dict max_features += [fet_obj] endforeach -if max_features.length() == 0 - message('Disabling CPU feature detection due to unsupported architecture: "' + cpu_family + '"') - CPU_CONF_BASELINE = 'none' - CPU_CONF_DISPATCH = 'none' -endif parse_options = { 'cpu-baseline': CPU_CONF_BASELINE, diff --git a/tools/ci/emscripten/0001-do-not-set-meson-environment-variable-pyodide-gh-4502.patch b/tools/ci/emscripten/0001-do-not-set-meson-environment-variable-pyodide-gh-4502.patch new file mode 100644 index 000000000000..f06ea4eead19 --- /dev/null +++ b/tools/ci/emscripten/0001-do-not-set-meson-environment-variable-pyodide-gh-4502.patch @@ -0,0 +1,55 @@ +From e08ebf0e90f632547c8ff5b396ec0c4ddd65aad4 Mon Sep 17 00:00:00 2001 +From: Gyeongjae Choi +Date: Sat, 10 Feb 2024 03:28:01 +0900 +Subject: [PATCH] Update numpy to 1.26.4 and don't set MESON env variable + (#4502) + +From meson-python 0.15, $MESON env variable is used to overwrite the meson binary +path. We don't want that behavior. +--- + pypabuild.py | 22 +++++++++++++++------- + 1 file changed, 15 insertions(+), 7 deletions(-) + +diff --git a/pypabuild.py b/pypabuild.py +index 9d0107a8..6961b14e 100644 +--- a/pypabuild.py ++++ b/pypabuild.py +@@ -40,6 +40,19 @@ AVOIDED_REQUIREMENTS = [ + "patchelf", + ] + ++# corresponding env variables for symlinks ++SYMLINK_ENV_VARS = { ++ "cc": "CC", ++ "c++": "CXX", ++ "ld": "LD", ++ "lld": "LLD", ++ "ar": "AR", ++ "gcc": "GCC", ++ "ranlib": "RANLIB", ++ "strip": "STRIP", ++ "gfortran": "FC", # https://mesonbuild.com/Reference-tables.html#compiler-and-linker-selection-variables ++} ++ + + def _gen_runner( + cross_build_env: Mapping[str, str], +@@ -207,13 +220,8 @@ def make_command_wrapper_symlinks(symlink_dir: Path) -> dict[str, str]: + symlink_path.unlink() + + symlink_path.symlink_to(pywasmcross_exe) +- if symlink == "c++": +- var = "CXX" +- elif symlink == "gfortran": +- var = "FC" # https://mesonbuild.com/Reference-tables.html#compiler-and-linker-selection-variables +- else: +- var = symlink.upper() +- env[var] = str(symlink_path) ++ if symlink in SYMLINK_ENV_VARS: ++ env[SYMLINK_ENV_VARS[symlink]] = str(symlink_path) + + return env + +-- +2.39.3 (Apple Git-145) + diff --git a/tools/ci/emscripten/emscripten.meson.cross b/tools/ci/emscripten/emscripten.meson.cross new file mode 100644 index 000000000000..15a483ea12fe --- /dev/null +++ b/tools/ci/emscripten/emscripten.meson.cross @@ -0,0 +1,15 @@ +# compiler paths are omitted intentionally so we can override the compiler using environment variables +[binaries] +exe_wrapper = 'node' +pkgconfig = 'pkg-config' + +[properties] +needs_exe_wrapper = true +skip_sanity_check = true +longdouble_format = 'IEEE_QUAD_LE' # for numpy + +[host_machine] +system = 'emscripten' +cpu_family = 'wasm32' +cpu = 'wasm' +endian = 'little'