From 35fc54090b3ac3b081a7e6afe2840223aa4cbe9d Mon Sep 17 00:00:00 2001 From: Ajay Thorve Date: Tue, 13 Aug 2024 17:12:55 -0700 Subject: [PATCH 01/13] avoid duplicate columns when node_aggregate_col is the same as node_id --- python/cuxfilter/charts/core/non_aggregate/core_graph.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/cuxfilter/charts/core/non_aggregate/core_graph.py b/python/cuxfilter/charts/core/non_aggregate/core_graph.py index 13d30a14..af07e87b 100644 --- a/python/cuxfilter/charts/core/non_aggregate/core_graph.py +++ b/python/cuxfilter/charts/core/non_aggregate/core_graph.py @@ -50,7 +50,7 @@ def edge_columns(self): @property def node_columns(self): - if self.node_aggregate_col: + if self.node_aggregate_col != self.node_id: return [ self.node_id, self.node_x, From a199a2fc4d3a8652f3daa7ca967a6d26e2221b7c Mon Sep 17 00:00:00 2001 From: Ajay Thorve Date: Tue, 13 Aug 2024 17:27:09 -0700 Subject: [PATCH 02/13] use the updated --gpu flag for external tests --- ci/test_external.sh | 43 ++++++++++++------------------------------- 1 file changed, 12 insertions(+), 31 deletions(-) diff --git a/ci/test_external.sh b/ci/test_external.sh index 613b7d1f..085b1b8e 100755 --- a/ci/test_external.sh +++ b/ci/test_external.sh @@ -19,6 +19,11 @@ LIBRARIES=("datashader" "holoviews") # Change directory to /tmp pushd /tmp + +EXITCODE=0 +trap "EXITCODE=1" ERR +set +e + # Clone the specified Python libraries if [ "$PROJECT" = "all" ]; then # Loop through each library and install dependencies @@ -35,6 +40,10 @@ if [ "$PROJECT" = "all" ]; then # Run setup.py with test dependencies python -m pip install -e .[tests] + rapids-logger "Run GPU tests for $LIBRARY" + + pytest --numprocesses=8 --dist=worksteal $LIBRARY/$LIBRARY/tests --benchmark-skip --gpu + popd done else @@ -54,41 +63,13 @@ else pushd $PROJECT # Run setup.py with test dependencies python -m pip install -e .[tests] - popd -fi -FILES="" -# Install and run tests -if [ "$PROJECT" = "all" ]; then - # Loop through each library and install dependencies - for LIBRARY in "${LIBRARIES[@]}" - do - rapids-logger "gathering GPU tests for $LIBRARY" - TEST_DIR="$LIBRARY/$LIBRARY/tests" - # Find all Python scripts containing the keywords cudf or dask_cudf except test_quadmesh.py - FILES+=" $(grep -l -R -e 'cudf' --include='*.py' "$TEST_DIR" | grep -v test_quadmesh.py)" - done -else - rapids-logger "gathering GPU tests for $PROJECT" - TEST_DIR="$PROJECT/$PROJECT/tests" - # Find all Python scripts containing the keywords cudf or dask_cudf - FILES+=$(grep -l -R -e 'cudf' --include='*.py' "$TEST_DIR") -fi -EXITCODE=0 -trap "EXITCODE=1" ERR -set +e + rapids-logger "Run GPU tests for $LIBRARY" -rapids-logger "running all gathered tests" -DATASHADER_TEST_GPU=1 pytest \ - --numprocesses=8 \ - --dist=worksteal \ - $FILES + pytest --numprocesses=8 --dist=worksteal $LIBRARY/$LIBRARY/tests --benchmark-skip --gpu -if [[ "$PROJECT" = "all" ]] || [[ "$PROJECT" = "datashader" ]]; then - # run test_quadmesh.py separately as dask.array tests fail with numprocesses - rapids-logger "running test_quadmesh.py" - DATASHADER_TEST_GPU=1 pytest datashader/datashader/tests/test_quadmesh.py + popd fi rapids-logger "Test script exiting with value: $EXITCODE" From d72dda1999a210179f1c8cb6d557e5a04f31fb09 Mon Sep 17 00:00:00 2001 From: Ajay Thorve Date: Tue, 13 Aug 2024 17:29:16 -0700 Subject: [PATCH 03/13] temporary trigger to test external_testing workflow --- .github/workflows/test-external.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/test-external.yaml b/.github/workflows/test-external.yaml index 614d65b5..09d302fa 100644 --- a/.github/workflows/test-external.yaml +++ b/.github/workflows/test-external.yaml @@ -1,6 +1,9 @@ name: GPU testing for external party dependencies on: + push: + branches: + - "pull-request/[0-9]+" workflow_dispatch: inputs: external-project: From dae1ddd362a769d65d89bc48c457e53a4f02c1d1 Mon Sep 17 00:00:00 2001 From: Ajay Thorve Date: Tue, 13 Aug 2024 17:54:43 -0700 Subject: [PATCH 04/13] add xdist --- ci/test_external.sh | 4 ++-- ci/utils/external_dependencies.yaml | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/ci/test_external.sh b/ci/test_external.sh index 085b1b8e..244b9832 100755 --- a/ci/test_external.sh +++ b/ci/test_external.sh @@ -42,7 +42,7 @@ if [ "$PROJECT" = "all" ]; then rapids-logger "Run GPU tests for $LIBRARY" - pytest --numprocesses=8 --dist=worksteal $LIBRARY/$LIBRARY/tests --benchmark-skip --gpu + python -m pytest $LIBRARY/tests/ --numprocesses=8 --dist=worksteal --gpu popd done @@ -67,7 +67,7 @@ else rapids-logger "Run GPU tests for $LIBRARY" - pytest --numprocesses=8 --dist=worksteal $LIBRARY/$LIBRARY/tests --benchmark-skip --gpu + python -m pytest $LIBRARY/tests/ --numprocesses=8 --dist=worksteal --gpu popd fi diff --git a/ci/utils/external_dependencies.yaml b/ci/utils/external_dependencies.yaml index 518c0a96..9bc5e35e 100644 --- a/ci/utils/external_dependencies.yaml +++ b/ci/utils/external_dependencies.yaml @@ -15,3 +15,4 @@ dependencies: - dash - dask-sql - pytest-benchmark + - pytest-xdist From afecc722072da1e1461b484e2548fef69dc7d4d5 Mon Sep 17 00:00:00 2001 From: Ajay Thorve Date: Tue, 13 Aug 2024 18:14:41 -0700 Subject: [PATCH 05/13] add coverage as a dependency --- ci/utils/external_dependencies.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/ci/utils/external_dependencies.yaml b/ci/utils/external_dependencies.yaml index 9bc5e35e..1ea11b8e 100644 --- a/ci/utils/external_dependencies.yaml +++ b/ci/utils/external_dependencies.yaml @@ -16,3 +16,4 @@ dependencies: - dask-sql - pytest-benchmark - pytest-xdist + - coverage From 9e0a030443a353861ced9af2041d160181ddc723 Mon Sep 17 00:00:00 2001 From: Ajay Thorve Date: Tue, 13 Aug 2024 18:30:55 -0700 Subject: [PATCH 06/13] rm temporary trigger --- .github/workflows/test-external.yaml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/test-external.yaml b/.github/workflows/test-external.yaml index 09d302fa..614d65b5 100644 --- a/.github/workflows/test-external.yaml +++ b/.github/workflows/test-external.yaml @@ -1,9 +1,6 @@ name: GPU testing for external party dependencies on: - push: - branches: - - "pull-request/[0-9]+" workflow_dispatch: inputs: external-project: From d882975919f6b6c57b0181f6853f400551039a23 Mon Sep 17 00:00:00 2001 From: Ajay Thorve Date: Tue, 13 Aug 2024 18:31:06 -0700 Subject: [PATCH 07/13] simplify install --- ci/test_external.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/test_external.sh b/ci/test_external.sh index 244b9832..b9e88e47 100755 --- a/ci/test_external.sh +++ b/ci/test_external.sh @@ -38,7 +38,7 @@ if [ "$PROJECT" = "all" ]; then # Change directory to the library pushd $LIBRARY # Run setup.py with test dependencies - python -m pip install -e .[tests] + python -m pip install .[tests] rapids-logger "Run GPU tests for $LIBRARY" @@ -62,7 +62,7 @@ else # Change directory to the specified project pushd $PROJECT # Run setup.py with test dependencies - python -m pip install -e .[tests] + python -m pip install .[tests] rapids-logger "Run GPU tests for $LIBRARY" From dd241de0e1f83c271bda7464b6e42885beebfc19 Mon Sep 17 00:00:00 2001 From: Ajay Thorve Date: Wed, 14 Aug 2024 11:35:56 -0700 Subject: [PATCH 08/13] rm external_dependencies.yaml in favor of adding test_external to dependencies.yaml --- ci/test_external.sh | 12 +++++++++++- ci/utils/external_dependencies.yaml | 19 ------------------- dependencies.yaml | 22 ++++++++++++++++++++++ 3 files changed, 33 insertions(+), 20 deletions(-) delete mode 100644 ci/utils/external_dependencies.yaml diff --git a/ci/test_external.sh b/ci/test_external.sh index b9e88e47..981b3b01 100755 --- a/ci/test_external.sh +++ b/ci/test_external.sh @@ -7,9 +7,19 @@ rapids-logger "Create test_external conda environment" . /opt/conda/etc/profile.d/conda.sh # Install external dependencies into test_external conda environment -rapids-conda-retry env update -f ./ci/utils/external_dependencies.yaml +rapids-dependency-file-generator \ + --output conda \ + --file-key test_notebooks \ + --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee env.yaml +rapids-mamba-retry env create --yes -f env.yaml -n test_external + +# Temporarily allow unbound variables for conda activation. +set +u conda activate test_external +set -u + +rapids-print-env # Define input parameter PROJECT=$1 diff --git a/ci/utils/external_dependencies.yaml b/ci/utils/external_dependencies.yaml deleted file mode 100644 index 1ea11b8e..00000000 --- a/ci/utils/external_dependencies.yaml +++ /dev/null @@ -1,19 +0,0 @@ -name: test_external -channels: - - rapidsai-nightly - - conda-forge - - nvidia -dependencies: - - cudf==24.10.*,>=0.0.0a0 - - dask-cudf==24.10.*,>=0.0.0a0 - - cuxfilter==24.10.*,>=0.0.0a0 - - cuda-version=12.0 - - python>=3.9,<3.12 - - xarray-spatial - - pycaret - - graphistry - - dash - - dask-sql - - pytest-benchmark - - pytest-xdist - - coverage diff --git a/dependencies.yaml b/dependencies.yaml index faaacb67..7f9f36fd 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -70,6 +70,10 @@ files: key: test includes: - test_python + test_external: + output: none + includes: + - test_external channels: - rapidsai - rapidsai-nightly @@ -257,3 +261,21 @@ dependencies: - output_types: conda packages: - panel>=1.0,<=1.3.4 + test_external: + common: + - output_types: [conda] + packages: + - coverage + - cudf-cu12==24.10.*,>=0.0.0a0 + - cupy-cuda12x>=12.0.0 + - cuspatial-cu12==24.10.*,>=0.0.0a0 + - dask-cudf-cu12==24.10.*,>=0.0.0a0 + - cuxfilter==24.10.*,>=0.0.0a0 + - python>=3.9,<3.12 + - xarray-spatial + - pycaret + - graphistry + - dash + - dask-sql + - pytest-benchmark + - pytest-xdist From 2e9f79b2206158b9203c559b88660f12a06aec93 Mon Sep 17 00:00:00 2001 From: Ajay Thorve Date: Wed, 14 Aug 2024 11:36:36 -0700 Subject: [PATCH 09/13] temporary PR trigger --- .github/workflows/test-external.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/test-external.yaml b/.github/workflows/test-external.yaml index 614d65b5..09d302fa 100644 --- a/.github/workflows/test-external.yaml +++ b/.github/workflows/test-external.yaml @@ -1,6 +1,9 @@ name: GPU testing for external party dependencies on: + push: + branches: + - "pull-request/[0-9]+" workflow_dispatch: inputs: external-project: From 7c8403cad73be1add2bce801c10012b63c135fc2 Mon Sep 17 00:00:00 2001 From: Ajay Thorve Date: Wed, 14 Aug 2024 12:29:51 -0700 Subject: [PATCH 10/13] fix dependency name --- ci/test_external.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/test_external.sh b/ci/test_external.sh index 981b3b01..ce8b42b6 100755 --- a/ci/test_external.sh +++ b/ci/test_external.sh @@ -9,7 +9,7 @@ rapids-logger "Create test_external conda environment" # Install external dependencies into test_external conda environment rapids-dependency-file-generator \ --output conda \ - --file-key test_notebooks \ + --file-key test_external \ --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee env.yaml rapids-mamba-retry env create --yes -f env.yaml -n test_external From bbc65714f2d5812261fc72d19668e1a37b2a9f1f Mon Sep 17 00:00:00 2001 From: Ajay Thorve Date: Wed, 14 Aug 2024 12:32:37 -0700 Subject: [PATCH 11/13] fix conda dependencies --- dependencies.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/dependencies.yaml b/dependencies.yaml index 7f9f36fd..ae83599d 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -266,10 +266,10 @@ dependencies: - output_types: [conda] packages: - coverage - - cudf-cu12==24.10.*,>=0.0.0a0 - - cupy-cuda12x>=12.0.0 - - cuspatial-cu12==24.10.*,>=0.0.0a0 - - dask-cudf-cu12==24.10.*,>=0.0.0a0 + - *cudf_unsuffixed + - cupy>=12.0.0 + - *cuspatial_unsuffixed + - *dask_cudf_unsuffixed - cuxfilter==24.10.*,>=0.0.0a0 - python>=3.9,<3.12 - xarray-spatial From e7d1bead8dd9a9cd120a1b5b9596e2b3e7eea033 Mon Sep 17 00:00:00 2001 From: Ajay Thorve Date: Wed, 14 Aug 2024 12:39:32 -0700 Subject: [PATCH 12/13] remove trigger on PR --- .github/workflows/test-external.yaml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/test-external.yaml b/.github/workflows/test-external.yaml index 09d302fa..614d65b5 100644 --- a/.github/workflows/test-external.yaml +++ b/.github/workflows/test-external.yaml @@ -1,9 +1,6 @@ name: GPU testing for external party dependencies on: - push: - branches: - - "pull-request/[0-9]+" workflow_dispatch: inputs: external-project: From 703b41a03b66f838df8bd63f66315b863030f20f Mon Sep 17 00:00:00 2001 From: Ajay Thorve Date: Wed, 14 Aug 2024 14:12:22 -0700 Subject: [PATCH 13/13] apply suggestion --- .../charts/core/non_aggregate/core_graph.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/python/cuxfilter/charts/core/non_aggregate/core_graph.py b/python/cuxfilter/charts/core/non_aggregate/core_graph.py index af07e87b..ab9b4101 100644 --- a/python/cuxfilter/charts/core/non_aggregate/core_graph.py +++ b/python/cuxfilter/charts/core/non_aggregate/core_graph.py @@ -50,14 +50,14 @@ def edge_columns(self): @property def node_columns(self): + cols = [ + self.node_id, + self.node_x, + self.node_y, + ] if self.node_aggregate_col != self.node_id: - return [ - self.node_id, - self.node_x, - self.node_y, - self.node_aggregate_col, - ] - return [self.node_id, self.node_x, self.node_y] + cols.append(self.node_aggregate_col) + return cols def __init__( self,