[data] Upgrade modin (and pandas)

.buildkite/pipeline.ml.yml

@@ -394,8 +394,7 @@
     - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
     - DATA_PROCESSING_TESTING=1 ./ci/env/install-dependencies.sh
     - ./ci/env/env_info.sh
-    # Todo: enable again when we upgraded modin and pandas
-    # - bazel test --config=ci $(./ci/run/bazel_export_options) --build_tests_only python/ray/tests/modin/...
+     - bazel test --config=ci $(./ci/run/bazel_export_options) --build_tests_only python/ray/tests/modin/...
     # Dask tests and examples.
     - bazel test --config=ci $(./ci/run/bazel_export_options) --build_tests_only --test_tag_filters=-client python/ray/util/dask/...
 

python/ray/data/tests/test_ecosystem.py

@@ -113,12 +113,6 @@ def test_to_dask_tensor_column_cast_arrow(ray_start_regular_shared):
         ctx.enable_tensor_extension_casting = original
 
 
-# We are currently testing with modin 0.12.1, which uses stale Ray core
-# APIs. Upgrading to a later version will also update pandas, and that upgrade
-# will break other tests. We skip modin until we can resolve the issues of the
-# pandas upgrade.
-# Todo: upgrade modin + pandas
-@pytest.mark.skip("Needs modin + pandas upgrade")
 @pytest.mark.skipif(sys.version_info < (3, 8), reason="requires python3.8 or higher")
 def test_from_modin(ray_start_regular_shared):
     import modin.pandas as mopd
@@ -132,7 +126,6 @@ def test_from_modin(ray_start_regular_shared):
     assert df.equals(dfds)
 
 
-@pytest.mark.skip("Needs modin + pandas upgrade")
 @pytest.mark.skipif(sys.version_info < (3, 8), reason="requires python3.8 or higher")
 def test_to_modin(ray_start_regular_shared):
     # create two modin dataframes

python/ray/data/tests/test_execution_optimizer.py

@@ -1002,12 +1002,6 @@ def test_from_dask_e2e(ray_start_regular_shared, enable_optimizer):
     _check_usage_record(["FromPandas"])
 
 
-# We are currently testing with modin 0.12.1, which uses stale Ray core
-# APIs. Upgrading to a later version will also update pandas, and that upgrade
-# will break other tests. We skip modin until we can resolve the issues of the
-# pandas upgrade.
-# Todo: upgrade modin + pandas
-@pytest.mark.skip("Needs modin + pandas upgrade")
 @pytest.mark.skipif(sys.version_info < (3, 8), reason="requires python3.8 or higher")
 def test_from_modin_e2e(ray_start_regular_shared, enable_optimizer):
     import modin.pandas as mopd

python/requirements/data_processing/requirements.txt

@@ -7,9 +7,8 @@ aioboto3==11.0.1
 crc32c==2.3
 flask_cors
 s3fs==2023.1.0
-# Todo: Upgrade and fix pandas errors
 modin==0.12.1; python_version < '3.8'
-modin==0.12.1; python_version >= '3.8'
+modin==0.22.2; python_version >= '3.8'
 pytest-repeat
 raydp>=0.0.dev0
 responses==0.13.4

python/requirements/ml/requirements_upstream.txt

@@ -6,6 +6,5 @@ ray_lightning==0.3.0
 tune-sklearn==0.4.4
 xgboost_ray==0.1.15
 lightgbm_ray==0.1.8
-# Todo: Upgrade and fix pandas errors
 modin==0.12.1; python_version < '3.8'
-modin==0.12.1; python_version >= '3.8'
+modin==0.22.2; python_version >= '3.8'