Add gpu marker and test both Classic/dask-expr Dask DataFrames

.github/workflows/test.yaml

@@ -146,7 +146,7 @@ jobs:
           echo "ENVS=$envs" >> $GITHUB_ENV
       - uses: holoviz-dev/holoviz_tasks/install@v0
         with:
-          name: unit_test_suite
+          name: unit_test_suite_np${{ matrix.numpy-version }}
           python-version: ${{ matrix.python-version }}
           channel-priority: flexible
           channels: ${{ env.CHANNELS }}
@@ -159,11 +159,6 @@ jobs:
           conda activate test-environment
           python -c "import numba; print('Numba', numba.__version__)"
           python -c "import numpy; print('Numpy', numpy.__version__)"
-      - name: doit test_lint
-        if: runner.os == 'Linux'
-        run: |
-          conda activate test-environment
-          doit test_lint
       - name: doit test_unit
         run: |
           conda activate test-environment
@@ -178,56 +173,6 @@ jobs:
         run: |
           conda activate test-environment
           doit test_examples
-      - name: codecov
-        run: |
-          conda activate test-environment
-          codecov
-
-  test_pip:
-    name: Pip tests on ${{ matrix.os }} with Python ${{ matrix.python-version }}
-    needs: [pre_commit]
-    runs-on: ${{ matrix.os }}
-    strategy:
-      fail-fast: false
-      matrix:
-        os: ['ubuntu-latest', 'macos-latest']
-        python-version: ["3.12"]
-    steps:
-      - name: Checkout source
-        uses: actions/checkout@v3
-        with:
-          fetch-depth: 0
-      - name: Install Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v4
+      - uses: codecov/codecov-action@v4
         with:
-          python-version: ${{ matrix.python-version }}
-      - name: Update setuptools
-        run: |
-          pip install --upgrade setuptools
-      - name: Install pyctdev
-        run: |
-          pip install pyctdev
-      - name: doit develop_install
-        run: |
-          doit ecosystem=pip develop_install -o tests -o examples
-      - name: doit env_capture
-        run: |
-          doit ecosystem=pip env_capture
-      - name: doit test_lint
-        if: runner.os == 'Linux'
-        run: |
-          doit ecosystem=pip test_lint
-      - name: doit test_unit
-        run: |
-          doit ecosystem=pip test_unit
-      - name: doit test_unit_nojit
-        run: |
-          doit ecosystem=pip test_unit_nojit
-        env:
-          NUMBA_DISABLE_JIT: 1
-      - name: doit test_examples
-        run: |
-          doit ecosystem=pip test_examples
-      - name: codecov
-        run: |
-          codecov
+          token: ${{ secrets.CODECOV_TOKEN }}

datashader/data_libraries/cudf.py

@@ -1,9 +1,14 @@
 from __future__ import annotations
+from contextlib import suppress
 from datashader.data_libraries.pandas import default
 from datashader.core import bypixel
-import cudf
 
 
-@bypixel.pipeline.register(cudf.DataFrame)
 def cudf_pipeline(df, schema, canvas, glyph, summary, *, antialias=False):
     return default(glyph, df, schema, canvas, summary, antialias=antialias, cuda=True)
+
+
+with suppress(ImportError):
+    import cudf
+
+    cudf_pipeline = bypixel.pipeline.register(cudf.DataFrame)(cudf_pipeline)

datashader/data_libraries/dask.py

@@ -1,5 +1,7 @@
 from __future__ import annotations
 
+from contextlib import suppress
+
 import numpy as np
 import pandas as pd
 import dask
@@ -16,7 +18,6 @@
 __all__ = ()
 
 
-@bypixel.pipeline.register(dd.DataFrame)
 def dask_pipeline(df, schema, canvas, glyph, summary, *, antialias=False, cuda=False):
     dsk, name = glyph_dispatch(glyph, df, schema, canvas, summary, antialias=antialias, cuda=cuda)
 
@@ -35,6 +36,15 @@ def dask_pipeline(df, schema, canvas, glyph, summary, *, antialias=False, cuda=F
     return scheduler(dsk, name)
 
 
+# Classic Dask.DataFrame
+bypixel.pipeline.register(dd.core.DataFrame)(dask_pipeline)
+
+with suppress(ImportError):
+    import dask_expr
+
+    bypixel.pipeline.register(dask_expr.DataFrame)(dask_pipeline)
+
+
 def shape_bounds_st_and_axis(df, canvas, glyph):
     if not canvas.x_range or not canvas.y_range:
         x_extents, y_extents = glyph.compute_bounds_dask(df)

datashader/data_libraries/dask_cudf.py

@@ -1,9 +1,14 @@
 from __future__ import annotations
+from contextlib import suppress
 from datashader.data_libraries.dask import dask_pipeline
 from datashader.core import bypixel
-import dask_cudf
 
 
-@bypixel.pipeline.register(dask_cudf.DataFrame)
 def dask_cudf_pipeline(df, schema, canvas, glyph, summary, *, antialias=False):
     return dask_pipeline(df, schema, canvas, glyph, summary, antialias=antialias, cuda=True)
+
+
+with suppress(ImportError):
+    import dask_cudf
+
+    dask_cudf_pipeline = bypixel.pipeline.register(dask_cudf.DataFrame)(dask_cudf_pipeline)

datashader/data_libraries/dask_xarray.py

@@ -173,8 +173,8 @@ def dask_raster(glyph, xr_ds, schema, canvas, summary, *, antialias=False, cuda=
     src_y0, src_y1 = glyph._compute_bounds_from_1d_centers(
         xr_ds, y_name, maybe_expand=False, orient=False
     )
-    xbinsize = float(xr_ds[x_name][1] - xr_ds[x_name][0])
-    ybinsize = float(xr_ds[y_name][1] - xr_ds[y_name][0])
+    xbinsize = abs(float(xr_ds[x_name][1] - xr_ds[x_name][0]))
+    ybinsize = abs(float(xr_ds[y_name][1] - xr_ds[y_name][0]))
 
     # Compute scale/translate
     out_h, out_w = shape

datashader/datashape/coretypes.py

@@ -254,7 +254,7 @@ def to_numpy_dtype(self):
         return np.dtype('datetime64[us]')
 
 
-_units = set(['ns', 'us', 'ms', 's', 'm', 'h', 'D', 'W', 'M', 'Y'])
+_units = ('ns', 'us', 'ms', 's', 'm', 'h', 'D', 'W', 'M', 'Y')
 
 
 _unit_aliases = {

datashader/tests/benchmarks/__init__.py

@@ -0,0 +1,3 @@
+import pytest
+
+pytestmark = pytest.mark.benchmark

datashader/tests/benchmarks/test_canvas.py

@@ -1,12 +1,9 @@
 import pytest
-import os
 import numpy as np
 import pandas as pd
 
 import datashader as ds
 
-test_gpu = bool(int(os.getenv("DATASHADER_TEST_GPU", 0)))
-
 
 @pytest.fixture
 def time_series():
@@ -33,7 +30,7 @@ def test_points(benchmark, time_series):
     benchmark(cvs.points, time_series, 'x', 'y')
 
 
-@pytest.mark.skipif(not test_gpu, reason="DATASHADER_TEST_GPU not set")
+@pytest.mark.gpu
 @pytest.mark.benchmark(group="canvas")
 def test_line_gpu(benchmark, time_series):
     from cudf import from_pandas
@@ -42,7 +39,7 @@ def test_line_gpu(benchmark, time_series):
     benchmark(cvs.line, time_series, 'x', 'y')
 
 
-@pytest.mark.skipif(not test_gpu, reason="DATASHADER_TEST_GPU not set")
+@pytest.mark.gpu
 @pytest.mark.benchmark(group="canvas")
 def test_points_gpu(benchmark, time_series):
     from cudf import from_pandas

datashader/tests/conftest.py

@@ -0,0 +1,34 @@
+CUSTOM_MARKS = ("benchmark", "gpu")
+
+
+def pytest_addoption(parser):
+    for marker in CUSTOM_MARKS:
+        parser.addoption(
+            f"--{marker}",
+            action="store_true",
+            default=False,
+            help=f"Run {marker} related tests",
+        )
+
+
+def pytest_configure(config):
+    for marker in CUSTOM_MARKS:
+        config.addinivalue_line("markers", f"{marker}: {marker} test marker")
+
+
+def pytest_collection_modifyitems(config, items):
+    skipped, selected = [], []
+    markers = [m for m in CUSTOM_MARKS if config.getoption(f"--{m}")]
+    empty = not markers
+    for item in items:
+        if empty and any(m in item.keywords for m in CUSTOM_MARKS):
+            skipped.append(item)
+        elif empty:
+            selected.append(item)
+        elif not empty and all(m in item.keywords for m in markers):
+            selected.append(item)
+        else:
+            skipped.append(item)
+
+    config.hook.pytest_deselected(items=skipped)
+    items[:] = selected