From e2dcd4e486c13d5f5e62e82f7331d506b84c41a2 Mon Sep 17 00:00:00 2001 From: Oliver Borchert Date: Mon, 11 Mar 2024 12:24:42 +0100 Subject: [PATCH 1/8] [ci] Fix tests for R 3.6 --- .ci/test_r_package_windows.ps1 | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.ci/test_r_package_windows.ps1 b/.ci/test_r_package_windows.ps1 index c4ae84f49dae..6af4dbb04173 100644 --- a/.ci/test_r_package_windows.ps1 +++ b/.ci/test_r_package_windows.ps1 @@ -91,7 +91,11 @@ if ($env:R_MAJOR_VERSION -eq "3") { $env:R_LIB_PATH = "$env:BUILD_SOURCESDIRECTORY/RLibrary" -replace '[\\]', '/' $env:R_LIBS = "$env:R_LIB_PATH" $env:PATH = "$env:RTOOLS_BIN;" + "$env:RTOOLS_MINGW_BIN;" + "$env:R_LIB_PATH/R/bin/x64;"+ $env:PATH -$env:CRAN_MIRROR = "https://cran.rstudio.com" +if ([version]$env:R_VERSION -lt [version]"4.0") { + $env:CRAN_MIRROR = "https://cran-archive.r-project.org" +} else { + $env:CRAN_MIRROR = "https://cran.rstudio.com" +} $env:MIKTEX_EXCEPTION_PATH = "$env:TEMP\miktex" # don't fail builds for long-running examples unless they're very long. From 52815dd46a4d3bc2506fdd2c5901d191fe3aab05 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Sat, 16 Mar 2024 13:06:01 -0500 Subject: [PATCH 2/8] remove clang-18 job --- .github/workflows/r_package.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/r_package.yml b/.github/workflows/r_package.yml index db3e3c8a713b..7d79cdab1d21 100644 --- a/.github/workflows/r_package.yml +++ b/.github/workflows/r_package.yml @@ -275,7 +275,6 @@ jobs: clang-version: - 16 - 17 - - 18 runs-on: ubuntu-latest container: rhub/debian-clang-devel env: From 119f44c2263cf889c2952c9d3e42316c9727f2a9 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Sat, 16 Mar 2024 20:06:47 -0500 Subject: [PATCH 3/8] always run 'brew update' onn macOS jobs --- .ci/setup.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/.ci/setup.sh b/.ci/setup.sh index ab47ad70745a..91669e39da70 100755 --- a/.ci/setup.sh +++ b/.ci/setup.sh @@ -1,6 +1,7 @@ #!/bin/bash if [[ $OS_NAME == "macos" ]]; then + brew update-reset && brew update if [[ $COMPILER == "clang" ]]; then brew install libomp if [[ $AZURE == "true" ]]; then From 09b3eb70196402a99219df1682866a78467d4471 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Sat, 16 Mar 2024 20:09:54 -0500 Subject: [PATCH 4/8] switch from dask-core to dask --- .ci/setup.sh | 3 ++- .ci/test.sh | 6 +++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.ci/setup.sh b/.ci/setup.sh index 91669e39da70..5e6e16ceb278 100755 --- a/.ci/setup.sh +++ b/.ci/setup.sh @@ -1,7 +1,8 @@ #!/bin/bash if [[ $OS_NAME == "macos" ]]; then - brew update-reset && brew update + brew update-reset + brew update if [[ $COMPILER == "clang" ]]; then brew install libomp if [[ $AZURE == "true" ]]; then diff --git a/.ci/test.sh b/.ci/test.sh index 79b2748e41fb..9d9c6e653b55 100755 --- a/.ci/test.sh +++ b/.ci/test.sh @@ -126,9 +126,9 @@ fi # older versions of Dask are incompatible with pandas>=2.0, but not all conda packages' metadata accurately reflects that # # ref: https://github.com/microsoft/LightGBM/issues/6030 -CONSTRAINED_DEPENDENCIES="'dask-core>=2023.5.0' 'distributed>=2023.5.0' 'pandas>=2.0'" +CONSTRAINED_DEPENDENCIES="'dask>=2023.5.0' 'distributed>=2023.5.0' 'pandas>=2.0'" if [[ $PYTHON_VERSION == "3.7" ]]; then - CONSTRAINED_DEPENDENCIES="'dask-core' 'distributed' 'pandas<2.0'" + CONSTRAINED_DEPENDENCIES="'dask' 'distributed' 'pandas<2.0'" fi # including python=version[build=*cpython] to ensure that conda doesn't fall back to pypy @@ -322,7 +322,7 @@ matplotlib.use\(\"Agg\"\)\ # importing the library should succeed even if all optional dependencies are not present conda uninstall -n $CONDA_ENV --force --yes \ cffi \ - dask-core \ + dask \ distributed \ joblib \ matplotlib \ From f1aa403f50fdcb77a2359b36ebdefef3438c6f79 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Sat, 16 Mar 2024 23:34:34 -0500 Subject: [PATCH 5/8] pre-compute inputs in Dask test functions --- tests/python_package_test/test_dask.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/tests/python_package_test/test_dask.py b/tests/python_package_test/test_dask.py index 9fe4da18faaf..37d6db2541f5 100644 --- a/tests/python_package_test/test_dask.py +++ b/tests/python_package_test/test_dask.py @@ -213,13 +213,17 @@ def _create_data(objective, n_samples=1_000, output="array", chunk_size=500, **k def _r2_score(dy_true, dy_pred): - numerator = ((dy_true - dy_pred) ** 2).sum(axis=0, dtype=np.float64) - denominator = ((dy_true - dy_true.mean(axis=0)) ** 2).sum(axis=0, dtype=np.float64) - return (1 - numerator / denominator).compute() + y_true = dy_true.compute() + y_pred = dy_pred.compute() + numerator = ((y_true - y_pred) ** 2).sum(axis=0) + denominator = ((y_true - y_true.mean(axis=0)) ** 2).sum(axis=0) + return 1 - numerator / denominator def _accuracy_score(dy_true, dy_pred): - return da.average(dy_true == dy_pred).compute() + y_true = dy_true.compute() + y_pred = dy_pred.compute() + return (y_true == y_pred).mean() def _constant_metric(y_true, y_pred): From e789b15690d91f66ed0b48ddb18a104f5a635ccb Mon Sep 17 00:00:00 2001 From: James Lamb Date: Sun, 17 Mar 2024 01:41:12 -0500 Subject: [PATCH 6/8] fix dask imports, try skipping r-sanitizers jobs --- .github/workflows/r_package.yml | 2 +- python-package/lightgbm/compat.py | 12 +++++++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/.github/workflows/r_package.yml b/.github/workflows/r_package.yml index 7d79cdab1d21..387b8c52eca2 100644 --- a/.github/workflows/r_package.yml +++ b/.github/workflows/r_package.yml @@ -315,7 +315,7 @@ jobs: all-r-package-jobs-successful: if: always() runs-on: ubuntu-latest - needs: [test, test-r-sanitizers, test-r-debian-clang] + needs: [test, test-r-debian-clang] steps: - name: Note that all tests succeeded uses: re-actors/alls-green@v1.2.2 diff --git a/python-package/lightgbm/compat.py b/python-package/lightgbm/compat.py index 965dd332525f..7656a0458315 100644 --- a/python-package/lightgbm/compat.py +++ b/python-package/lightgbm/compat.py @@ -164,7 +164,17 @@ class _LGBMRegressorBase: # type: ignore from dask.distributed import Client, Future, default_client, wait DASK_INSTALLED = True -except ImportError: +# catching 'ValueError' here because of this: +# https://github.com/microsoft/LightGBM/issues/6365#issuecomment-2002330003 +# +# That's potentially risky as dask does some significant import-time processing, +# like loading configuration from environment variables and files, and catching +# ValueError here might hide issues with that config-loading. +# +# But in exchange, it's less likely that 'import lightgbm' will fail for +# dask-related reasons, which is beneficial for any workloads that are using +# lightgbm but not its Dask functionality. +except (ImportError, ValueError): DASK_INSTALLED = False dask_array_from_delayed = None # type: ignore[assignment] From c875bd839e4a8c5137995568084e505908403e70 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Sun, 17 Mar 2024 11:34:39 -0500 Subject: [PATCH 7/8] fix brew updates --- .ci/setup.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.ci/setup.sh b/.ci/setup.sh index 5e6e16ceb278..c0e7dbf9dfcc 100755 --- a/.ci/setup.sh +++ b/.ci/setup.sh @@ -1,8 +1,8 @@ #!/bin/bash if [[ $OS_NAME == "macos" ]]; then - brew update-reset - brew update + brew update-reset --auto-update + brew update --auto-update if [[ $COMPILER == "clang" ]]; then brew install libomp if [[ $AZURE == "true" ]]; then From 45914c4687c33ec0d0d7db65e9bde83d1d317783 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Sun, 17 Mar 2024 11:36:40 -0500 Subject: [PATCH 8/8] just run brew update in R-package jobs --- .ci/setup.sh | 2 -- .ci/test_r_package.sh | 2 ++ 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.ci/setup.sh b/.ci/setup.sh index c0e7dbf9dfcc..ab47ad70745a 100755 --- a/.ci/setup.sh +++ b/.ci/setup.sh @@ -1,8 +1,6 @@ #!/bin/bash if [[ $OS_NAME == "macos" ]]; then - brew update-reset --auto-update - brew update --auto-update if [[ $COMPILER == "clang" ]]; then brew install libomp if [[ $AZURE == "true" ]]; then diff --git a/.ci/test_r_package.sh b/.ci/test_r_package.sh index 4fda67a1fc7b..d303d0ad20a7 100755 --- a/.ci/test_r_package.sh +++ b/.ci/test_r_package.sh @@ -81,6 +81,8 @@ fi # Installing R precompiled for Mac OS 10.11 or higher if [[ $OS_NAME == "macos" ]]; then + brew update-reset --auto-update + brew update --auto-update if [[ $R_BUILD_TYPE == "cran" ]]; then brew install automake || exit 1 fi