From b2be6cd2aebd419d93d32ce7a32b441b8b34f01e Mon Sep 17 00:00:00 2001 From: Kai Fricke Date: Thu, 15 Jun 2023 14:34:27 +0100 Subject: [PATCH] [data] Upgrade modin (and pandas) Signed-off-by: Kai Fricke --- .buildkite/pipeline.ml.yml | 3 +-- python/ray/data/tests/test_ecosystem.py | 7 ------- python/ray/data/tests/test_execution_optimizer.py | 6 ------ python/requirements/data_processing/requirements.txt | 3 +-- python/requirements/ml/requirements_upstream.txt | 3 +-- 5 files changed, 3 insertions(+), 19 deletions(-) diff --git a/.buildkite/pipeline.ml.yml b/.buildkite/pipeline.ml.yml index d96dc5c385e1..4ec81482a564 100644 --- a/.buildkite/pipeline.ml.yml +++ b/.buildkite/pipeline.ml.yml @@ -394,8 +394,7 @@ - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT - DATA_PROCESSING_TESTING=1 ./ci/env/install-dependencies.sh - ./ci/env/env_info.sh - # Todo: enable again when we upgraded modin and pandas - # - bazel test --config=ci $(./ci/run/bazel_export_options) --build_tests_only python/ray/tests/modin/... + - bazel test --config=ci $(./ci/run/bazel_export_options) --build_tests_only python/ray/tests/modin/... # Dask tests and examples. - bazel test --config=ci $(./ci/run/bazel_export_options) --build_tests_only --test_tag_filters=-client python/ray/util/dask/... diff --git a/python/ray/data/tests/test_ecosystem.py b/python/ray/data/tests/test_ecosystem.py index bb908c21402c..45d33a9fbac5 100644 --- a/python/ray/data/tests/test_ecosystem.py +++ b/python/ray/data/tests/test_ecosystem.py @@ -113,12 +113,6 @@ def test_to_dask_tensor_column_cast_arrow(ray_start_regular_shared): ctx.enable_tensor_extension_casting = original -# We are currently testing with modin 0.12.1, which uses stale Ray core -# APIs. Upgrading to a later version will also update pandas, and that upgrade -# will break other tests. We skip modin until we can resolve the issues of the -# pandas upgrade. -# Todo: upgrade modin + pandas -@pytest.mark.skip("Needs modin + pandas upgrade") @pytest.mark.skipif(sys.version_info < (3, 8), reason="requires python3.8 or higher") def test_from_modin(ray_start_regular_shared): import modin.pandas as mopd @@ -132,7 +126,6 @@ def test_from_modin(ray_start_regular_shared): assert df.equals(dfds) -@pytest.mark.skip("Needs modin + pandas upgrade") @pytest.mark.skipif(sys.version_info < (3, 8), reason="requires python3.8 or higher") def test_to_modin(ray_start_regular_shared): # create two modin dataframes diff --git a/python/ray/data/tests/test_execution_optimizer.py b/python/ray/data/tests/test_execution_optimizer.py index 40542c1404a1..f00676bc4105 100644 --- a/python/ray/data/tests/test_execution_optimizer.py +++ b/python/ray/data/tests/test_execution_optimizer.py @@ -1002,12 +1002,6 @@ def test_from_dask_e2e(ray_start_regular_shared, enable_optimizer): _check_usage_record(["FromPandas"]) -# We are currently testing with modin 0.12.1, which uses stale Ray core -# APIs. Upgrading to a later version will also update pandas, and that upgrade -# will break other tests. We skip modin until we can resolve the issues of the -# pandas upgrade. -# Todo: upgrade modin + pandas -@pytest.mark.skip("Needs modin + pandas upgrade") @pytest.mark.skipif(sys.version_info < (3, 8), reason="requires python3.8 or higher") def test_from_modin_e2e(ray_start_regular_shared, enable_optimizer): import modin.pandas as mopd diff --git a/python/requirements/data_processing/requirements.txt b/python/requirements/data_processing/requirements.txt index 3a5b24aea13f..655bdce2046c 100644 --- a/python/requirements/data_processing/requirements.txt +++ b/python/requirements/data_processing/requirements.txt @@ -7,9 +7,8 @@ aioboto3==11.0.1 crc32c==2.3 flask_cors s3fs==2023.1.0 -# Todo: Upgrade and fix pandas errors modin==0.12.1; python_version < '3.8' -modin==0.12.1; python_version >= '3.8' +modin==0.22.2; python_version >= '3.8' pytest-repeat raydp>=0.0.dev0 responses==0.13.4 diff --git a/python/requirements/ml/requirements_upstream.txt b/python/requirements/ml/requirements_upstream.txt index b9d063f5863a..32331123a5f3 100644 --- a/python/requirements/ml/requirements_upstream.txt +++ b/python/requirements/ml/requirements_upstream.txt @@ -6,6 +6,5 @@ ray_lightning==0.3.0 tune-sklearn==0.4.4 xgboost_ray==0.1.15 lightgbm_ray==0.1.8 -# Todo: Upgrade and fix pandas errors modin==0.12.1; python_version < '3.8' -modin==0.12.1; python_version >= '3.8' +modin==0.22.2; python_version >= '3.8'