From 1a7a78a920a107db997863ad134d9001d36b6506 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20H=C3=B8xbro=20Hansen?= Date: Tue, 21 May 2024 11:20:35 +0200 Subject: [PATCH] Clean up infrastructure --- .github/workflows/test.yaml | 141 ++++++++++++++++++++++-------------- setup.py | 66 ++++++++++++++--- tox.ini | 2 +- 3 files changed, 142 insertions(+), 67 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index a6cd25e23..897ac9fd2 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -65,13 +65,22 @@ jobs: if: env.MATRIX_OPTION == 'default' run: | MATRIX=$(jq -nsc '{ - # "os": ["ubuntu-latest", "macos-latest", "windows-latest"], - # "python-version": ["3.9", "3.10", "3.11", "3.12"], - "os": ["ubuntu-latest", "macos-latest"], - "python-version": ["3.12"], + "os": ["ubuntu-latest", "macos-latest", "windows-latest"], + "python-version": ["3.9", "3.10", "3.11", "3.12"], + "numpy-version": ["1"], "exclude": [{ "os": "windows-latest", "python-version": "3.9" + }], + "include": [{ + "os": "ubuntu-latest", + "python-version": "3.12", + "numpy-version": "2" + }, + { + "os": "macos-latest", + "python-version": "3.12", + "numpy-version": "2" }] }') # Won't solve on Windows + Python 3.9 @@ -82,9 +91,20 @@ jobs: MATRIX=$(jq -nsc '{ "os": ["ubuntu-latest", "macos-latest", "windows-latest"], "python-version": ["3.9", "3.10", "3.11", "3.12"], + "numpy-version": ["1"], "exclude": [{ "os": "windows-latest", "python-version": "3.9" + }], + "include": [{ + "os": "ubuntu-latest", + "python-version": "3.12", + "numpy-version": "2" + }, + { + "os": "macos-latest", + "python-version": "3.12", + "numpy-version": "2" }] }') # Won't solve on Windows + Python 3.9 @@ -99,7 +119,7 @@ jobs: echo "MATRIX=$MATRIX" >> $GITHUB_ENV test_suite: - name: Tests on ${{ matrix.os }} with Python ${{ matrix.python-version }} + name: Tests on ${{ matrix.os }} with Python ${{ matrix.python-version }}, numpy {{ matrix.numpy-version }} needs: [pre_commit, setup] runs-on: ${{ matrix.os }} strategy: @@ -112,13 +132,24 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} steps: + - name: Set channels and envs + run: | + if [[ ${{ matrix.numpy-version }} == "2" ]]; then + channels="pyviz/label/dev,conda-forge/label/numpy_rc,numba/label/dev,conda-forge,nodefaults" + envs="-o numpy2" + else + channels="pyviz/label/dev,numba,conda-forge,nodefaults" + envs="-o tests -o examples" + fi + echo "CHANNELS=$channels" >> $GITHUB_ENV + echo "ENVS=$envs" >> $GITHUB_ENV - uses: holoviz-dev/holoviz_tasks/install@v0 with: name: unit_test_suite python-version: ${{ matrix.python-version }} channel-priority: flexible - channels: pyviz/label/dev,conda-forge/label/numpy_rc,numba/label/dev,conda-forge,nodefaults - envs: "-o tests -o examples" + channels: ${{ env.CHANNELS }} + envs: ${{ env.ENVS }} cache: ${{ github.event.inputs.cache || github.event.inputs.cache == '' }} conda-update: true id: install @@ -151,51 +182,51 @@ jobs: conda activate test-environment codecov - # test_pip: - # name: Pip tests on ${{ matrix.os }} with Python ${{ matrix.python-version }} - # needs: [pre_commit] - # runs-on: ${{ matrix.os }} - # strategy: - # fail-fast: false - # matrix: - # os: ['ubuntu-latest', 'macos-latest'] - # python-version: ["3.12"] - # steps: - # - name: Checkout source - # uses: actions/checkout@v3 - # with: - # fetch-depth: 0 - # - name: Install Python ${{ matrix.python-version }} - # uses: actions/setup-python@v4 - # with: - # python-version: ${{ matrix.python-version }} - # - name: Update setuptools - # run: | - # pip install --upgrade setuptools - # - name: Install pyctdev - # run: | - # pip install pyctdev - # - name: doit develop_install - # run: | - # doit ecosystem=pip develop_install -o tests -o examples - # - name: doit env_capture - # run: | - # doit ecosystem=pip env_capture - # - name: doit test_lint - # if: runner.os == 'Linux' - # run: | - # doit ecosystem=pip test_lint - # - name: doit test_unit - # run: | - # doit ecosystem=pip test_unit - # - name: doit test_unit_nojit - # run: | - # doit ecosystem=pip test_unit_nojit - # env: - # NUMBA_DISABLE_JIT: 1 - # - name: doit test_examples - # run: | - # doit ecosystem=pip test_examples - # - name: codecov - # run: | - # codecov + test_pip: + name: Pip tests on ${{ matrix.os }} with Python ${{ matrix.python-version }} + needs: [pre_commit] + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: ['ubuntu-latest', 'macos-latest'] + python-version: ["3.12"] + steps: + - name: Checkout source + uses: actions/checkout@v3 + with: + fetch-depth: 0 + - name: Install Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + - name: Update setuptools + run: | + pip install --upgrade setuptools + - name: Install pyctdev + run: | + pip install pyctdev + - name: doit develop_install + run: | + doit ecosystem=pip develop_install -o tests -o examples + - name: doit env_capture + run: | + doit ecosystem=pip env_capture + - name: doit test_lint + if: runner.os == 'Linux' + run: | + doit ecosystem=pip test_lint + - name: doit test_unit + run: | + doit ecosystem=pip test_unit + - name: doit test_unit_nojit + run: | + doit ecosystem=pip test_unit_nojit + env: + NUMBA_DISABLE_JIT: 1 + - name: doit test_examples + run: | + doit ecosystem=pip test_examples + - name: codecov + run: | + codecov diff --git a/setup.py b/setup.py index dec17bab7..d8e8585b2 100644 --- a/setup.py +++ b/setup.py @@ -30,20 +30,63 @@ 'shapely >=2.0.0', ] -# examples = geopandas + [ -examples = [ +examples = geopandas + [ 'bokeh >3.1', 'geodatasets', 'holoviews', 'matplotlib >=3.3', 'panel >1.1', + 'scikit-image', + 'spatialpandas', +] + +# Numpy 2 packages, should be removed when all commented out packages works with Numpy 2 +numpy2 = [ + # [geopandas] + # 'dask-geopandas', + # 'geopandas', + # 'shapely >=2.0.0', + + # [examples] + 'bokeh >3.1', + 'geodatasets', + 'holoviews', + 'matplotlib >=3.3', + 'panel >1.1', + # 'scikit-image', + # 'spatialpandas', + + # [tests] + 'codecov', + 'geodatasets', + 'flake8', + 'nbconvert', + 'nbformat', + 'nbsmoke[verify] >0.5', + 'netcdf4', + # 'pyarrow', + 'pytest <8', # Fails lint with IPynbFile is deprecated + 'pytest-benchmark', + 'pytest-cov', + # 'rasterio', + # 'rioxarray', # rasterio # 'scikit-image', # 'spatialpandas', + # 'dask-expr', # pyarrow + + # [examples_extras] + 'networkx', + 'streamz', + ### conda only below here + 'fastparquet', + 'graphviz', + 'python-graphviz', + 'python-snappy', + # 'rasterio', ] extras_require = { - # 'tests': geopandas + [ - 'tests': [ + 'tests': geopandas + [ 'codecov', 'geodatasets', 'flake8', @@ -51,15 +94,15 @@ 'nbformat', 'nbsmoke[verify] >0.5', 'netcdf4', - # 'pyarrow', + 'pyarrow', 'pytest <8', # Fails lint with IPynbFile is deprecated 'pytest-benchmark', 'pytest-cov', - # 'rasterio', - # 'rioxarray', # rasterio - # 'scikit-image', - # 'spatialpandas', - # 'dask-expr', # pyarrow + 'rasterio', + 'rioxarray', + 'scikit-image', + 'spatialpandas', + 'dask-expr', ], 'examples': examples, 'examples_extra': examples + [ @@ -70,9 +113,10 @@ 'graphviz', 'python-graphviz', 'python-snappy', - # 'rasterio', + 'rasterio', ], 'geopandas': geopandas, + 'numpy2': numpy2, } diff --git a/tox.ini b/tox.ini index 0ec3bbd96..a6bdc77b0 100644 --- a/tox.ini +++ b/tox.ini @@ -17,7 +17,7 @@ commands = flake8 [_unit] description = Run unit tests deps = .[tests] -commands = pytest datashader --cov=./datashader --cov-append -k "not benchmarks" +commands = pytest datashader --cov=./datashader --cov-append [_unit_deploy] description = Run unit tests without coverage