Merge pull request #183 from asmeurer/performance-playground

Performance playground

.circleci/config.yml

@@ -26,6 +26,10 @@ jobs:
           command: |
             cd docs
             pip install -r requirements.txt
+      - run:
+          name: In-place build
+          command: |
+            python setup.py build_ext --inplace
       - run:
           name: Build docs
           no_output_timeout: 25m

.coveragerc

@@ -3,6 +3,7 @@ omit =
     ndindex/_version.py
     ndindex/tests/doctest.py
     ndindex/tests/test_no_dependencies.py
+plugins = Cython.Coverage
 
 [report]
 exclude_lines =

.github/workflows/docs.yml

@@ -17,6 +17,9 @@ jobs:
           set -x
           set -e
           python -m pip install -r docs/requirements.txt
+      - name: In-place build
+        run: |
+          python setup.py build_ext --inplace
 
       - name: Build Docs
         run: |

.github/workflows/publish-package.yml

@@ -25,7 +25,7 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  build:
+  build_sdist:
     name: Build Python distribution
     runs-on: ubuntu-latest
 
@@ -48,29 +48,49 @@ jobs:
     - name: Install dependencies
       run: python -m pip install -r requirements-dev.txt
 
-    - name: Build a wheel and a sdist
+    - name: Build an sdist
       run: |
-        CYTHONIZE_NDINDEX=0 PYTHONWARNINGS=error,default::DeprecationWarning python -m build .
+        PYTHONWARNINGS=error,default::DeprecationWarning python -m build --sdist .
 
     - name: Verify the distribution
       run: twine check --strict dist/*
 
     - name: List contents of sdist
       run: python -m tarfile --list dist/ndindex-*.tar.gz
 
-    - name: List contents of wheel
-      run: python -m zipfile --list dist/ndindex-*.whl
-
     - name: Upload distribution artifact
       uses: actions/upload-artifact@v4
       with:
         name: dist-artifact
         path: dist
 
+  build_wheels:
+    name: Build wheels on ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        # macos-13 is an intel runner, macos-14 is apple silicon
+        os: [ubuntu-latest, windows-latest, macos-13, macos-14]
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Build wheels
+        uses: pypa/cibuildwheel@v2.21.0
+        with:
+          output-dir: dist/
+        env:
+          CIBW_TEST_COMMAND: 'python -c "import ndindex"'
+
+      - uses: actions/upload-artifact@v4
+        with:
+          name: cibw-wheels-${{ matrix.os }}-${{ strategy.job-index }}
+          path: ./wheelhouse/*.whl
+
   publish:
     name: Publish Python distribution to (Test)PyPI
     if: github.event_name != 'pull_request' && github.repository == 'Quansight-Labs/ndindex'
-    needs: build
+    needs: [build_sdist, build_wheels]
     runs-on: ubuntu-latest
     # Mandatory for publishing with a trusted publisher
     # c.f. https://docs.pypi.org/trusted-publishers/using-a-publisher/

.github/workflows/tests.yml

@@ -25,20 +25,35 @@ jobs:
           if [[ ${{ matrix.numpy-version }} == 'latest' ]]; then
               python -m pip install --upgrade numpy
           elif [[ ${{ matrix.numpy-version }} == 'dev' ]]; then
-              python -m pip install --pre --upgrade --extra-index https://pypi.anaconda.org/scientific-python-nightly-wheels/simple numpy -r requirements-dev.txt
+              python -m pip install --pre --upgrade --extra-index https://pypi.anaconda.org/scientific-python-nightly-wheels/simple numpy
           else
               python -m pip install --upgrade numpy==${{ matrix.numpy-version }}.*
           fi
+      - name: Test Installation
+        run: |
+          python -m pip install --verbose .
+        # COVERAGE_CORE=sysmon does not work with Cython. Since coverage is
+        # really slow without it (see
+        # https://github.com/nedbat/coveragepy/issues/1665), just omit Cython
+        # coverage in 3.12. Hopefully by the time it is our minimal version it
+        # will work with Cython, or trace coverage will be faster again.
+      - name: Set CYTHON_COVERAGE=1
+        run: echo "CYTHON_COVERAGE=1" >> $GITHUB_ENV
+        if: matrix.python-version != 3.12
+      - name: Disable Coverage Plugin
+        run: sed -i '/plugins = Cython.Coverage/d' .coveragerc
+        if: matrix.python-version == 3.12
+      - name: In-place build
+        run: |
+          python setup.py clean
+          python setup.py build_ext --inplace
       - name: Run Doctests
         run: |
           ./run_doctests
         # A NumPy 2.0 compatible skimage doesn't support 3.9, and we need a
         # version of matplotlib that requires NumPy 1.23. Easiest to just
         # skip this for now.
         if: matrix.numpy-version != 'dev' && matrix.python-version != '3.9' && matrix.numpy-version != '1.22'
-      - name: Test Installation
-        run: |
-          python -m pip install .
       - name: Run Slotscheck
         run: |
           python -m slotscheck ndindex

.gitignore

@@ -146,3 +146,7 @@ dmypy.json
 /scratch/
 
 .DS_Store
+
+# Cython
+ndindex/*.c
+ndindex/*.html

MANIFEST.in

@@ -1,5 +1,6 @@
 include versioneer.py
 include ndindex/_version.py
+include ndindex/*.pyx
 include LICENSE
 include pytest.ini
 include conftest.py

benchmarks/slice.py

@@ -17,6 +17,18 @@ def time_constructor_invalid(self):
         except TypeError:
             pass
 
+    def time_args(self):
+        self.s1.args
+        self.s2.args
+
+    def time_start(self):
+        self.s1.start
+        self.s2.start
+
+    def time_raw(self):
+        self.s1.raw
+        self.s2.raw
+
     def time_reduce(self):
         self.s1.reduce()
         self.s2.reduce()

benchmarks/tuple.py

@@ -30,6 +30,16 @@ def time_constructor_invalid_array_broadcast(self):
         except IndexError:
             pass
 
+    def time_args(self):
+        self.t.args
+        self.t_arrays.args
+        self.t_boolean_scalars.args
+
+    def time_raw(self):
+        self.t.raw
+        self.t_arrays.raw
+        self.t_boolean_scalars.raw
+
     def time_reduce(self):
         self.t.reduce()
 

docs/api/internal.rst

@@ -21,6 +21,7 @@ Base Classes
 
 .. autoclass:: ndindex.ndindex.NDIndex
    :members:
+   :inherited-members:
 
 .. autoclass:: ndindex.array.ArrayIndex
    :members:

docs/index.md

@@ -77,27 +77,6 @@ their limitations:
 ndindex can be installed using pip (`pip install ndindex`) or conda (`conda
 install -f conda-forge ndindex`).
 
-### Experimental Cythonization
-
-When installing ndindex from source (using `python setup.py install`) all
-Python modules (except tests) will be Cythonized when Cython and a working
-compiler are installed. This can improve the general performance of ndindex.
-The environment variable `CYTHONIZE_NDINDEX` is used to explicitly control
-this default behavior:
-
-- `CYTHONIZE_NDINDEX=0`: disables Cythonization (even if a
-  working Cython environment is available).
-
-- `CYTHONIZE_NDINDEX=1`: force Cythonization (will fail when Cython or a
-  compiler isn't present).
-
-- `CYTHONIZE_NDINDEX` not set: the default behavior (Cythonize if Cython is
-  installed and working).
-
-Cythonization is still experimental, and is only enabled for direct source
-installations. The pip and conda packages are not Cythonized. Future versions
-of ndindex may enable Cythonization for all installs.
-
 ## Features
 
 ndindex is still a work in progress. The following things are currently

docs/requirements.txt

@@ -1,10 +1,12 @@
+Cython
 furo
 linkify-it-py
 matplotlib @ git+https://github.com/asmeurer/matplotlib.git@output-base-name
 myst-parser
 scikit-image
+setuptools
 sphinx
+sphinx-autobuild
 sphinx-copybutton
-sphinx-reredirects
 sphinx-design
-sphinx-autobuild
+sphinx-reredirects

ndindex/_crt.py

@@ -44,9 +44,7 @@
 DAMAGE.
 """
 
-from functools import reduce
-from operator import mul
-from math import gcd
+from math import gcd, prod
 
 def gcdex(a, b):
     """Returns x, y, g such that g = x*a + y*b = gcd(a, b).
@@ -93,11 +91,6 @@ def gcdex(a, b):
 
     return (x*x_sign, y*y_sign, a)
 
-
-# np.prod has overflow and math.prod is Python 3.8+ only
-def prod(seq):
-    return reduce(mul, seq, 1)
-
 def _crt(U, M):
     """
     Chinese Remainder Theorem.
@@ -183,7 +176,7 @@ def combine(c1, c2):
         a1, m1 = c1
         a2, m2 = c2
         a, b, c = m1, a2 - a1, m2
-        g = reduce(gcd, [a, b, c])
+        g = gcd(a, b, c)
         a, b, c = [i//g for i in [a, b, c]]
         if a != 1:
             inv_a, _, g = gcdex(a, c)