upstream changes (#6)

<!-- Thank you for your contribution! Please review
https://github.com/ray-project/ray/blob/master/CONTRIBUTING.rst before
opening a pull request. -->

<!-- Please add a reviewer to the assignee section when you create a PR.
If you don't have the access to it, we will shortly find a reviewer and
assign them to your PR. -->

## Why are these changes needed?

<!-- Please give a short summary of the change and the problem this
solves. -->

## Related issue number

<!-- For example: "Closes ray-project#1234" -->

## Checks

- [ ] I've signed off every commit(by using the -s flag, i.e., `git
commit -s`) in this PR.
- [ ] I've run `scripts/format.sh` to lint the changes in this PR.
- [ ] I've included any doc changes needed for
https://docs.ray.io/en/master/.
- [ ] I've made sure the tests are passing. Note that there might be a
few flaky tests, see the recent failures at https://flakey-tests.ray.io/
- Testing Strategy
   - [ ] Unit tests
   - [ ] Release tests
   - [ ] This PR is not tested :(

.bazeliskrc

@@ -0,0 +1 @@
+USE_BAZEL_VERSION=5.x

.bazelrc

@@ -29,6 +29,9 @@ test --test_env=PYENV_VERSION
 test --test_env=PYENV_SHELL
 # Do not send usage stats to the server for tests
 test --test_env=RAY_USAGE_STATS_REPORT_URL="http://127.0.0.1:8000"
+# Enable cluster mode for OSX and Windows. By default, Ray
+# will not allow multinode OSX and Windows clusters.
+test --test_env=RAY_ENABLE_WINDOWS_OR_OSX_CLUSTER="1"
 # This is needed for some core tests to run correctly
 build:windows --enable_runfiles
 # TODO(mehrdadn): Revert the "-\\.(asm|S)$" exclusion when this Bazel bug
@@ -97,6 +100,7 @@ build:tsan --copt -g
 build:tsan --copt -fno-omit-frame-pointer
 build:tsan --copt -Wno-uninitialized
 build:tsan --linkopt -fsanitize=thread
+build:tsan --cxxopt="-D_RAY_TSAN_BUILD"
 # This config is only for running TSAN with LLVM toolchain on Linux.
 build:tsan-clang --config=tsan
 build:tsan-clang --config=llvm

.buildkite/pipeline.arm64.yml

@@ -54,3 +54,83 @@
 #    - python .buildkite/copy_files.py --destination branch_wheels --path ./.whl
 #    # Upload to latest directory.
 #    - if [ "$BUILDKITE_BRANCH" == "master" ]; then python .buildkite/copy_files.py --destination wheels --path ./.whl; fi
+
+
+- label: ":mechanical_arm: :docker: Build Images: py37 [aarch64] (1/2)"
+  conditions: ["RAY_CI_LINUX_WHEELS_AFFECTED"]
+  instance_size: arm64-medium
+  commands:
+    - LINUX_WHEELS=1 ./ci/ci.sh build
+    - pip install -q docker aws_requests_auth boto3
+    - ./ci/env/env_info.sh
+    - if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then python .buildkite/copy_files.py --destination docker_login; fi
+    - python ./ci/build/build-docker-images.py --py-versions py37 --device-types cpu cu112 --build-type BUILDKITE --build-base --suffix aarch64
+
+- label: ":mechanical_arm: :docker: Build Images: py37 [aarch64] (2/2)"
+  conditions: ["RAY_CI_LINUX_WHEELS_AFFECTED"]
+  instance_size: arm64-medium
+  commands:
+    - LINUX_WHEELS=1 ./ci/ci.sh build
+    - pip install -q docker aws_requests_auth boto3
+    - ./ci/env/env_info.sh
+    - if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then python .buildkite/copy_files.py --destination docker_login; fi
+    - python ./ci/build/build-docker-images.py --py-versions py37 --device-types cu113 cu116 --build-type BUILDKITE --build-base --suffix aarch64
+
+- label: ":mechanical_arm: :docker: Build Images: py38 [aarch64] (1/2)"
+  conditions: ["RAY_CI_LINUX_WHEELS_AFFECTED"]
+  instance_size: arm64-medium
+  commands:
+    - LINUX_WHEELS=1 ./ci/ci.sh build
+    - pip install -q docker aws_requests_auth boto3
+    - ./ci/env/env_info.sh
+    - if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then python .buildkite/copy_files.py --destination docker_login; fi
+    - python ./ci/build/build-docker-images.py --py-versions py38 --device-types cpu cu112 --build-type BUILDKITE --build-base --suffix aarch64
+
+- label: ":mechanical_arm: :docker: Build Images: py38 [aarch64] (2/2)"
+  conditions: ["RAY_CI_LINUX_WHEELS_AFFECTED"]
+  instance_size: arm64-medium
+  commands:
+    - LINUX_WHEELS=1 ./ci/ci.sh build
+    - pip install -q docker aws_requests_auth boto3
+    - ./ci/env/env_info.sh
+    - if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then python .buildkite/copy_files.py --destination docker_login; fi
+    - python ./ci/build/build-docker-images.py --py-versions py38 --device-types cu113 cu116 cu118 --build-type BUILDKITE --build-base --suffix aarch64
+
+- label: ":mechanical_arm: :docker: Build Images: py39 [aarch64] (1/2)"
+  conditions: ["RAY_CI_LINUX_WHEELS_AFFECTED"]
+  instance_size: arm64-medium
+  commands:
+    - LINUX_WHEELS=1 ./ci/ci.sh build
+    - pip install -q docker aws_requests_auth boto3
+    - ./ci/env/env_info.sh
+    - if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then python .buildkite/copy_files.py --destination docker_login; fi
+    - python ./ci/build/build-docker-images.py --py-versions py39 --device-types cpu cu112 --build-type BUILDKITE --build-base --suffix aarch64
+
+- label: ":mechanical_arm: :docker: Build Images: py39 [aarch64] (2/2)"
+  conditions: ["RAY_CI_LINUX_WHEELS_AFFECTED"]
+  instance_size: arm64-medium
+  commands:
+    - LINUX_WHEELS=1 ./ci/ci.sh build
+    - pip install -q docker aws_requests_auth boto3
+    - ./ci/env/env_info.sh
+    - if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then python .buildkite/copy_files.py --destination docker_login; fi
+    - python ./ci/build/build-docker-images.py --py-versions py39 --device-types cu113 cu116 cu118 --build-type BUILDKITE --build-base --suffix aarch64
+
+- label: ":mechanical_arm: :docker: Build Images: py310 [aarch64] (1/2)"
+  conditions: ["RAY_CI_LINUX_WHEELS_AFFECTED"]
+  instance_size: arm64-medium
+  commands:
+    - LINUX_WHEELS=1 ./ci/ci.sh build
+    - pip install -q docker aws_requests_auth boto3
+    - ./ci/env/env_info.sh
+    - if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then python .buildkite/copy_files.py --destination docker_login; fi
+    - python ./ci/build/build-docker-images.py --py-versions py310 --device-types cpu cu112 --build-type BUILDKITE --build-base --suffix aarch64
+
+- label: ":mechanical_arm: :docker: Build Images: py310 [aarch64] (2/2)"
+  conditions: ["RAY_CI_LINUX_WHEELS_AFFECTED"]
+  instance_size: arm64-medium
+  commands:
+    - LINUX_WHEELS=1 ./ci/ci.sh build
+    - pip install -q docker aws_requests_auth boto3
+    - if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then python .buildkite/copy_files.py --destination docker_login; fi
+    - python ./ci/build/build-docker-images.py --py-versions py310 --device-types cu113 cu116 cu118 --build-type BUILDKITE --build-base --suffix aarch64

.buildkite/pipeline.build.yml

@@ -122,7 +122,7 @@
     - pip install -q docker aws_requests_auth boto3
     - ./ci/env/env_info.sh
     - if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then python .buildkite/copy_files.py --destination docker_login; fi
-    - python ./ci/build/build-docker-images.py --py-versions py38 --device-types cu111 cu112 cu113 cu116 --build-type BUILDKITE --build-base
+    - python ./ci/build/build-docker-images.py --py-versions py38 --device-types cu111 cu112 cu113 cu116 cu118 --build-type BUILDKITE --build-base
 
 - label: ":docker: Build Images: py39 (1/2)"
   conditions: ["RAY_CI_LINUX_WHEELS_AFFECTED"]
@@ -142,7 +142,7 @@
     - pip install -q docker aws_requests_auth boto3
     - ./ci/env/env_info.sh
     - if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then python .buildkite/copy_files.py --destination docker_login; fi
-    - python ./ci/build/build-docker-images.py --py-versions py39 --device-types cu111 cu112 cu113 cu116 --build-type BUILDKITE --build-base
+    - python ./ci/build/build-docker-images.py --py-versions py39 --device-types cu111 cu112 cu113 cu116 cu118 --build-type BUILDKITE --build-base
 
 - label: ":docker: Build Images: py310 (1/2)"
   conditions: ["RAY_CI_LINUX_WHEELS_AFFECTED"]
@@ -161,7 +161,7 @@
     - LINUX_WHEELS=1 ./ci/ci.sh build
     - pip install -q docker aws_requests_auth boto3
     - if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then python .buildkite/copy_files.py --destination docker_login; fi
-    - python ./ci/build/build-docker-images.py --py-versions py310 --device-types cu111 cu112 cu113 cu116 --build-type BUILDKITE --build-base
+    - python ./ci/build/build-docker-images.py --py-versions py310 --device-types cu111 cu112 cu113 cu116 cu118 --build-type BUILDKITE --build-base
 
 - label: ":java: Java"
   conditions: ["RAY_CI_JAVA_AFFECTED"]
@@ -574,6 +574,10 @@
     - pushd /ray && git clean -f -f -x -d -e .whl -e python/ray/dashboard/client && popd
     - bazel clean --expunge
     - export WANDB_MODE=offline
+    # Horovod needs to be installed separately (needed for API ref imports)
+    - ./ci/env/install-horovod.sh
+    # See https://stackoverflow.com/questions/63383400/error-cannot-uninstall-ruamel-yaml-while-creating-docker-image-for-azure-ml-a
+    - pip install mosaicml==0.10.1 --ignore-installed
     - ./ci/ci.sh build
 
 - label: ":octopus: Tune multinode tests"
@@ -599,6 +603,15 @@
       --test_env=DOCKER_CERT_PATH=/certs/client
       --test_env=DOCKER_TLS_CERTDIR=/certs
 
+- label: ":hadoop: Ray AIR HDFS tests"
+  conditions: ["RAY_CI_ML_AFFECTED"]
+  instance_size: medium
+  commands:
+    - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
+    - INSTALL_HDFS=1 ./ci/env/install-dependencies.sh
+    - ./ci/env/env_info.sh
+    - cat /tmp/hdfs_env
+    - bazel test --config=ci $(./ci/run/bazel_export_options) --test_tag_filters=hdfs python/ray/air/...
 
 
 # Test to see if Train can be used without torch, tf, etc. installed
@@ -639,7 +652,7 @@
   commands:
     - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
     - ./ci/env/install-minimal.sh 3.7
-    - DATA_PROCESSING_TESTING=1 TUNE_TESTING=1 TRAIN_TESTING=1 ./ci/env/install-dependencies.sh
+    - DATA_PROCESSING_TESTING=1 TUNE_TESTING=1 TRAIN_TESTING=1 INSTALL_HDFS=1 ./ci/env/install-dependencies.sh
     - pip install -r python/requirements/compat/requirements_legacy_compat.txt
     - pip install -U typing-extensions
     - HOROVOD_WITH_GLOO=1 HOROVOD_WITHOUT_MPI=1 HOROVOD_WITHOUT_MXNET=1 HOROVOD_WITH_TENSORFLOW=1 HOROVOD_WITH_PYTORCH=1 pip install horovod
@@ -649,8 +662,8 @@
       set -x;
       {
         python ./ci/run/bazel_sharding/bazel_sharding.py --exclude_manual --index "\${BUILDKITE_PARALLEL_JOB}" --count "\${BUILDKITE_PARALLEL_JOB_COUNT}" --tag_filters=compat python/ray/tests/horovod/... python/ray/tests/lightgbm/... python/ray/tests/ml_py36_compat/... python/ray/tests/xgboost/... python/ray/tests/ray_lightning/... &&
-        python ./ci/run/bazel_sharding/bazel_sharding.py --exclude_manual --index "\${BUILDKITE_PARALLEL_JOB}" --count "\${BUILDKITE_PARALLEL_JOB_COUNT}" --tag_filters=-gpu,-needs_credentials python/ray/air/... &&
-        python ./ci/run/bazel_sharding/bazel_sharding.py --exclude_manual --index "\${BUILDKITE_PARALLEL_JOB}" --count "\${BUILDKITE_PARALLEL_JOB_COUNT}" --tag_filters=ray_air,-torch_1_11,-gpu_only,-gpu,-needs_credentials python/ray/train/... &&
+        python ./ci/run/bazel_sharding/bazel_sharding.py --exclude_manual --index "\${BUILDKITE_PARALLEL_JOB}" --count "\${BUILDKITE_PARALLEL_JOB_COUNT}" --tag_filters=-gpu,-needs_credentials,-hdfs python/ray/air/... &&
+        python ./ci/run/bazel_sharding/bazel_sharding.py --exclude_manual --index "\${BUILDKITE_PARALLEL_JOB}" --count "\${BUILDKITE_PARALLEL_JOB_COUNT}" --tag_filters=ray_air,-torch_1_11,-gpu_only,-gpu,-needs_credentials,-hdfs python/ray/train/... &&
         python ./ci/run/bazel_sharding/bazel_sharding.py --exclude_manual --index "\${BUILDKITE_PARALLEL_JOB}" --count "\${BUILDKITE_PARALLEL_JOB_COUNT}" --tag_filters=ray_air python/ray/data/...;
       } > test_shard.txt
     - cat test_shard.txt

.buildkite/pipeline.macos.yml

@@ -46,6 +46,7 @@ steps:
     - export MAC_WHEELS=1
     - export MAC_JARS=1
     - export RAY_INSTALL_JAVA=1
+    - export RAY_ENABLE_WINDOWS_OR_OSX_CLUSTER=1
     - . ./ci/ci.sh init && source ~/.zshenv
     - ./ci/ci.sh build
     # Test wheels

.buildkite/pipeline.ml.yml

@@ -5,7 +5,7 @@
     - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
     - DATA_PROCESSING_TESTING=1 INSTALL_HOROVOD=1 ./ci/env/install-dependencies.sh
     - ./ci/env/env_info.sh
-    - bazel test --config=ci $(./ci/run/bazel_export_options) --build_tests_only --test_tag_filters=-gpu,-needs_credentials
+    - bazel test --config=ci $(./ci/run/bazel_export_options) --build_tests_only --test_tag_filters=-gpu,-needs_credentials,-hdfs
       python/ray/air/...
     - bazel test --config=ci $(./ci/run/bazel_export_options) --build_tests_only --test_tag_filters=ray_air python/ray/data/...
 
@@ -177,7 +177,7 @@
     - RLLIB_TESTING=1 ./ci/env/install-dependencies.sh
     - ./ci/env/env_info.sh
     - ./ci/run/run_bazel_test_with_sharding.sh --config=ci $(./ci/run/bazel_export_options) --build_tests_only
-      --test_tag_filters=examples,-multi_gpu --test_env=RAY_USE_MULTIPROCESSING_CPU_COUNT=1 rllib/...
+      --test_tag_filters=examples,-multi_gpu,-gpu --test_env=RAY_USE_MULTIPROCESSING_CPU_COUNT=1 rllib/...
 
 - label: ":brain: RLlib: tests/ dir"
   conditions: ["NO_WHEELS_REQUIRED", "RAY_CI_RLLIB_DIRECTLY_AFFECTED"]

.buildkite/pipeline.windows.yml

@@ -35,6 +35,7 @@ steps:
     - conda init
     - . ./ci/ci.sh init
     - ./ci/ci.sh build
+    - export RAY_ENABLE_WINDOWS_OR_OSX_CLUSTER="1"
     - if [ "${BUILDKITE_PARALLEL_JOB}" = "0" ]; then ./ci/ci.sh test_core; fi
     # The next command will be sharded into $parallelism shards.
     - ./ci/ci.sh test_python

.github/CODEOWNERS

@@ -10,6 +10,7 @@
 # Authors responsible for copy-editing of the documentation.
 # NOTE: Add @ray-project/ray-docs to all following docs subdirs.
 /doc/ @ray-project/ray-docs
+/doc/source/use-cases.rst @ericl @pcmoritz
 
 # ==== Ray core ====
 
@@ -24,28 +25,31 @@
 /python/ray/autoscaler/ @wuisawesome @DmitriGekhtman @ericl
 
 # Metrics
-/src/ray/stats/metric_defs.h @ericl @scv119 @rkooo567
-/src/ray/stats/metric_defs.cc @ericl @scv119 @rkooo567
+/src/ray/stats/metric_defs.h @ray-project/ray-core
+/src/ray/stats/metric_defs.cc @ray-project/ray-core
 
 # Telemetry
 /src/ray/protobuf/usage.proto @pcmoritz @thomasdesr
 
 # All C++ code.
 # /src/ray @ray-project/ray-core-cpp
 
+# GCS
+/src/ray/gcs/ @ray-project/ray-core
+
 # Dependencies
 /python/setup.py @richardliaw @ericl @edoakes
 
 # Formatting tool
 /ci/lint/format.sh @richardliaw @ericl @edoakes
 
 # Docker image build script.
-/ci/build/build-docker-images.py @amogkam @krfricke
+/ci/build/build-docker-images.py @amogkam @krfricke @ray-project/ray-core
 
 # Python worker.
-#/python/ray/ @ray-project/ray-core-python
-#!/python/ray/tune/ @ray-project/ray-core-python
-#!/python/ray/rllib/ @ray-project/ray-core-python
+#/python/ray/ @ray-project/ray-core
+#!/python/ray/tune/ @ray-project/ray-core
+#!/python/ray/rllib/ @ray-project/ray-core
 
 # Java worker.
 /java/dependencies.bzl @jovany-wang @kfstorm @raulchen @ericl @iycheng @WangTaoTheTonic
@@ -55,15 +59,15 @@
 /java/api/ @jovany-wang @kfstorm @raulchen @ericl @iycheng @WangTaoTheTonic
 
 # C++ worker
-/cpp/include/ray @SongGuyang @raulchen @kfstorm
+/cpp/include/ray @SongGuyang @raulchen @kfstorm @ray-project/ray-core
 
 # Ray Client
 /src/ray/protobuf/ray_client.proto @ijrsvt @ameerhajali @ckw017
 
 # Runtime Env
 # TODO(SongGuyang): Add new items to guarantee runtime env API compatibility in multiple languages.
-/src/ray/protobuf/runtime_env_common.proto @SongGuyang @raulchen @edoakes @architkulkarni
-/src/ray/protobuf/runtime_env_agent.proto @SongGuyang @raulchen @edoakes @architkulkarni
+/src/ray/protobuf/runtime_env_common.proto @SongGuyang @raulchen @edoakes @architkulkarni @ray-project/ray-core
+/src/ray/protobuf/runtime_env_agent.proto @SongGuyang @raulchen @edoakes @architkulkarni @ray-project/ray-core
 
 # ==== Libraries and frameworks ====
 

.github/ISSUE_TEMPLATE/documentation-issue.yml

@@ -1,7 +1,7 @@
 name: Documentation
 title: "[<Ray component: Core|RLlib|etc...>] "
 description: Report an issue with the Ray documentation
-labels: [docs]
+labels: [docs, triage]
 body:
   - type: markdown
     attributes:

.github/ISSUE_TEMPLATE/feature-request.yml

@@ -1,7 +1,7 @@
 name: Ray feature request
 description: Suggest an idea for Ray project
 title: "[<Ray component: Core|RLlib|etc...>] "
-labels: [enhancement]
+labels: [enhancement, triage]
 body:
   - type: markdown
     attributes:

.github/stale.yml

@@ -12,6 +12,7 @@ onlyLabels: []
 
 # Issues or Pull Requests with these labels will never be considered stale. Set to `[]` to disable
 exemptLabels:
+  - triage
   - P0
   - P1
   - P2
@@ -37,29 +38,29 @@ staleLabel: stale
 # Comment to post when marking as stale. Set to `false` to disable
 markComment: |
   Hi, I'm a bot from the Ray team :)
-  
+
   To help human contributors to focus on more relevant issues, I will automatically add the stale label to issues that have had no activity for more than 4 months.
 
   If there is no further activity in the 14 days, the issue will be closed!
-  
+
   - If you'd like to keep the issue open, just leave any comment, and the stale label will be removed!
   - If you'd like to get more attention to the issue, please tag one of Ray's contributors.
 
-  You can always ask for help on our [discussion forum](https://discuss.ray.io/) or [Ray's public slack channel](https://github.com/ray-project/ray#getting-involved). 
-  
+  You can always ask for help on our [discussion forum](https://discuss.ray.io/) or [Ray's public slack channel](https://github.com/ray-project/ray#getting-involved).
+
 # Comment to post when removing the stale label.
 # unmarkComment: >
 #   Your comment here.
 
 # Comment to post when closing a stale Issue or Pull Request.
 closeComment: |
   Hi again! The issue will be closed because there has been no more activity in the 14 days since the last message.
-  
+
   Please feel free to reopen or open a new issue if you'd still like it to be addressed.
 
-  Again, you can always ask for help on our [discussion forum](https://discuss.ray.io) or [Ray's public slack channel](https://github.com/ray-project/ray#getting-involved). 
+  Again, you can always ask for help on our [discussion forum](https://discuss.ray.io) or [Ray's public slack channel](https://github.com/ray-project/ray#getting-involved).
 
-  Thanks again for opening the issue! 
+  Thanks again for opening the issue!
 
 # Limit the number of actions per hour, from 1-30. Default is 30
 limitPerRun: 30
@@ -75,7 +76,7 @@ pulls:
      This pull request has been automatically marked as stale because it has not had
      recent activity. It will be closed in 14 days if no further activity occurs. Thank you
      for your contributions.
-     
+
      - If you'd like to keep this open, just leave any comment, and the stale label will be removed.
 
 # issues:

.gitignore

@@ -119,6 +119,7 @@ scripts/nodes.txt
 /doc/_build
 /doc/source/_static/thumbs
 /doc/source/tune/generated_guides/
+/doc/source/**/doc/
 
 # User-specific stuff:
 .idea/**/workspace.xml
@@ -226,3 +227,5 @@ workflow_data/
 
 # Auto-generated tag mapping
 tag-mapping.json
+
+.bazeliskrc