Test narwhals in CI (#17884)

Contributes to #17662.

@MarcoGorelli provided very helpful instructions for running the narwhals test suite. We should examine and correct the failing tests. We are down to 39 failures, shown in [this Kaggle notebook](https://www.kaggle.com/code/marcogorelli/testing-cudf-in-narwhals?scriptVersionId=220053914).

Authors:
  - Bradley Dice (https://github.com/bdice)

Approvers:
  - Matthew Murray (https://github.com/Matt711)
  - Gil Forsyth (https://github.com/gforsyth)

URL: #17884

.github/workflows/pr.yaml

@@ -40,6 +40,7 @@ jobs:
       - unit-tests-cudf-pandas
       - pandas-tests
       - pandas-tests-diff
+      - narwhals-tests
       - telemetry-setup
       - third-party-integration-tests-cudf-pandas
     secrets: inherit
@@ -358,6 +359,20 @@ jobs:
         node_type: "cpu4"
         build_type: pull-request
         run_script: "ci/cudf_pandas_scripts/pandas-tests/diff.sh"
+  narwhals-tests:
+    needs: [conda-python-build, changed-files]
+    secrets: inherit
+    uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-25.04
+    if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python
+    with:
+      build_type: pull-request
+      branch: ${{ inputs.branch }}
+      date: ${{ inputs.date }}
+      sha: ${{ inputs.sha }}
+      node_type: "gpu-l4-latest-1"
+      continue-on-error: true
+      container_image: "rapidsai/ci-conda:latest"
+      run_script: ci/test_narwhals.sh
   spark-rapids-jni:
     needs: changed-files
     uses: ./.github/workflows/spark-rapids-jni.yaml

ci/test_narwhals.sh

@@ -0,0 +1,44 @@
+#!/bin/bash
+# Copyright (c) 2025, NVIDIA CORPORATION.
+
+# Support invoking test_python_cudf.sh outside the script directory
+cd "$(dirname "$(realpath "${BASH_SOURCE[0]}")")"/../ || exit 1
+
+# Common setup steps shared by Python test jobs
+source ./ci/test_python_common.sh test_python_narwhals
+
+rapids-logger "Check GPU usage"
+nvidia-smi
+rapids-print-env
+EXITCODE=0
+trap "EXITCODE=1" ERR
+set +e
+
+rapids-logger "pytest narwhals"
+git clone https://github.com/narwhals-dev/narwhals --depth=1
+pushd narwhals || exit 1
+rapids-pip-retry install -U -e ".[dev]"
+
+rapids-logger "Check narwhals versions"
+python -c "import narwhals; print(narwhals.show_versions())"
+
+rapids-logger "Run narwhals tests for cuDF"
+python -m pytest \
+    --cache-clear \
+    --junitxml="${RAPIDS_TESTS_DIR}/junit-cudf-narwhals.xml" \
+    --numprocesses=8 \
+    --dist=worksteal \
+    --constructors=cudf
+
+rapids-logger "Run narwhals tests for cuDF Polars"
+NARWHALS_POLARS_GPU=1 python -m pytest \
+    --cache-clear \
+    --junitxml="${RAPIDS_TESTS_DIR}/junit-cudf-polars-narwhals.xml" \
+    --numprocesses=8 \
+    --dist=worksteal \
+    --constructors=polars[lazy]
+
+popd || exit 1
+
+rapids-logger "Test script exiting with value: $EXITCODE"
+exit ${EXITCODE}

dependencies.yaml

@@ -379,6 +379,16 @@ files:
     includes:
       - test_python_common
       - test_python_cudf_common
+  test_python_narwhals:
+    output: none
+    includes:
+      - cuda_version
+      - py_version
+      - test_python_common
+      - test_python_cudf_common
+      - test_python_cudf
+      - depends_on_cudf
+      - depends_on_cudf_polars
 channels:
   - rapidsai
   - rapidsai-nightly