diff --git a/.buildkite/pipeline.arm64.yml b/.buildkite/pipeline.arm64.yml
index eea53233fb27e..379727179b2c2 100644
--- a/.buildkite/pipeline.arm64.yml
+++ b/.buildkite/pipeline.arm64.yml
@@ -56,81 +56,32 @@
 #    - if [ "$BUILDKITE_BRANCH" == "master" ]; then python .buildkite/copy_files.py --destination wheels --path ./.whl; fi
 
 
-- label: ":mechanical_arm: :docker: Build Images: py37 [aarch64] (1/2)"
+- label: ":mechanical_arm: :docker: Build Images: {{matrix}} [aarch64] - cpu/cu115/cu116"
   conditions: ["RAY_CI_PYTHON_DEPENDENCIES_AFFECTED", "RAY_CI_DOCKER_AFFECTED", "RAY_CI_CORE_CPP_AFFECTED"]
   instance_size: arm64-medium
   commands:
-    - LINUX_WHEELS=1 BUILD_ONE_PYTHON_ONLY=3.7 ./ci/ci.sh build
+    - LINUX_WHEELS=1 BUILD_ONE_PYTHON_ONLY={{matrix}} ./ci/ci.sh build
     - pip install -q docker aws_requests_auth boto3
     - ./ci/env/env_info.sh
     - if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then python .buildkite/copy_files.py --destination docker_login; fi
-    - python ./ci/build/build-docker-images.py --py-versions py37 -T cpu -T cu115 -T cu116 --build-type BUILDKITE --build-base --suffix aarch64
+    - python ./ci/build/build-docker-images.py --py-versions {{matrix}} -T cpu -T cu115 -T cu116 --build-type BUILDKITE --build-base --suffix aarch64
+  matrix:
+    - py37
+    - py38
+    - py39
+    - py310
 
-- label: ":mechanical_arm: :docker: Build Images: py37 [aarch64] (2/2)"
+- label: ":mechanical_arm: :docker: Build Images: {{matrix}} [aarch64] cu117/cu118"
   conditions: ["RAY_CI_PYTHON_DEPENDENCIES_AFFECTED", "RAY_CI_DOCKER_AFFECTED", "RAY_CI_CORE_CPP_AFFECTED"]
   instance_size: arm64-medium
   commands:
-    - LINUX_WHEELS=1 BUILD_ONE_PYTHON_ONLY=3.7 ./ci/ci.sh build
+    - LINUX_WHEELS=1 BUILD_ONE_PYTHON_ONLY={{matrix}} ./ci/ci.sh build
     - pip install -q docker aws_requests_auth boto3
     - ./ci/env/env_info.sh
     - if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then python .buildkite/copy_files.py --destination docker_login; fi
-    - python ./ci/build/build-docker-images.py --py-versions py37 -T cu117 -T cu118 --build-type BUILDKITE --build-base --suffix aarch64
-
-- label: ":mechanical_arm: :docker: Build Images: py38 [aarch64] (1/2)"
-  conditions: ["RAY_CI_PYTHON_DEPENDENCIES_AFFECTED", "RAY_CI_DOCKER_AFFECTED", "RAY_CI_CORE_CPP_AFFECTED"]
-  instance_size: arm64-medium
-  commands:
-    - LINUX_WHEELS=1 BUILD_ONE_PYTHON_ONLY=3.8 ./ci/ci.sh build
-    - pip install -q docker aws_requests_auth boto3
-    - ./ci/env/env_info.sh
-    - if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then python .buildkite/copy_files.py --destination docker_login; fi
-    - python ./ci/build/build-docker-images.py --py-versions py38 -T cpu -T cu115 -T cu116 --build-type BUILDKITE --build-base --suffix aarch64
-
-- label: ":mechanical_arm: :docker: Build Images: py38 [aarch64] (2/2)"
-  conditions: ["RAY_CI_PYTHON_DEPENDENCIES_AFFECTED", "RAY_CI_DOCKER_AFFECTED", "RAY_CI_CORE_CPP_AFFECTED"]
-  instance_size: arm64-medium
-  commands:
-    - LINUX_WHEELS=1 BUILD_ONE_PYTHON_ONLY=3.8 ./ci/ci.sh build
-    - pip install -q docker aws_requests_auth boto3
-    - ./ci/env/env_info.sh
-    - if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then python .buildkite/copy_files.py --destination docker_login; fi
-    - python ./ci/build/build-docker-images.py --py-versions py38 -T cu117 -T cu118 --build-type BUILDKITE --build-base --suffix aarch64
-
-- label: ":mechanical_arm: :docker: Build Images: py39 [aarch64] (1/2)"
-  conditions: ["RAY_CI_PYTHON_DEPENDENCIES_AFFECTED", "RAY_CI_DOCKER_AFFECTED", "RAY_CI_CORE_CPP_AFFECTED"]
-  instance_size: arm64-medium
-  commands:
-    - LINUX_WHEELS=1 BUILD_ONE_PYTHON_ONLY=3.9 ./ci/ci.sh build
-    - pip install -q docker aws_requests_auth boto3
-    - ./ci/env/env_info.sh
-    - if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then python .buildkite/copy_files.py --destination docker_login; fi
-    - python ./ci/build/build-docker-images.py --py-versions py39 -T cpu -T cu115 -T cu116 --build-type BUILDKITE --build-base --suffix aarch64
-
-- label: ":mechanical_arm: :docker: Build Images: py39 [aarch64] (2/2)"
-  conditions: ["RAY_CI_PYTHON_DEPENDENCIES_AFFECTED", "RAY_CI_DOCKER_AFFECTED", "RAY_CI_CORE_CPP_AFFECTED"]
-  instance_size: arm64-medium
-  commands:
-    - LINUX_WHEELS=1 BUILD_ONE_PYTHON_ONLY=3.9 ./ci/ci.sh build
-    - pip install -q docker aws_requests_auth boto3
-    - ./ci/env/env_info.sh
-    - if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then python .buildkite/copy_files.py --destination docker_login; fi
-    - python ./ci/build/build-docker-images.py --py-versions py39 -T cu117 -T cu118 --build-type BUILDKITE --build-base --suffix aarch64
-
-- label: ":mechanical_arm: :docker: Build Images: py310 [aarch64] (1/2)"
-  conditions: ["RAY_CI_PYTHON_DEPENDENCIES_AFFECTED", "RAY_CI_DOCKER_AFFECTED", "RAY_CI_CORE_CPP_AFFECTED"]
-  instance_size: arm64-medium
-  commands:
-    - LINUX_WHEELS=1 BUILD_ONE_PYTHON_ONLY=3.10 ./ci/ci.sh build
-    - pip install -q docker aws_requests_auth boto3
-    - ./ci/env/env_info.sh
-    - if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then python .buildkite/copy_files.py --destination docker_login; fi
-    - python ./ci/build/build-docker-images.py --py-versions py310 -T cpu -T cu115 -T cu116 --build-type BUILDKITE --build-base --suffix aarch64
-
-- label: ":mechanical_arm: :docker: Build Images: py310 [aarch64] (2/2)"
-  conditions: ["RAY_CI_PYTHON_DEPENDENCIES_AFFECTED", "RAY_CI_DOCKER_AFFECTED", "RAY_CI_CORE_CPP_AFFECTED"]
-  instance_size: arm64-medium
-  commands:
-    - LINUX_WHEELS=1 BUILD_ONE_PYTHON_ONLY=3.10 ./ci/ci.sh build
-    - pip install -q docker aws_requests_auth boto3
-    - if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then python .buildkite/copy_files.py --destination docker_login; fi
-    - python ./ci/build/build-docker-images.py --py-versions py310 -T cu117 -T cu118 --build-type BUILDKITE --build-base --suffix aarch64
+    - python ./ci/build/build-docker-images.py --py-versions {{matrix}} -T cu117 -T cu118 --build-type BUILDKITE --build-base --suffix aarch64
+  matrix:
+    - py37
+    - py38
+    - py39
+    - py310
diff --git a/.buildkite/pipeline.build.yml b/.buildkite/pipeline.build.yml
index 9915ce89a8b0e..1f30039f5a4c1 100644
--- a/.buildkite/pipeline.build.yml
+++ b/.buildkite/pipeline.build.yml
@@ -68,80 +68,31 @@
   conditions: ["RAY_CI_PYTHON_DEPENDENCIES_AFFECTED", "RAY_CI_DOCKER_AFFECTED", "RAY_CI_CORE_CPP_AFFECTED"]
   instance_size: medium
   commands:
-    - LINUX_WHEELS=1 BUILD_ONE_PYTHON_ONLY=3.7 ./ci/ci.sh build
+    - LINUX_WHEELS=1 BUILD_ONE_PYTHON_ONLY={{matrix}} ./ci/ci.sh build
     - pip install -q docker aws_requests_auth boto3
     - ./ci/env/env_info.sh
-    - if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then python .buildkite/copy_files.py --destination docker_login; fi
-    - python ./ci/build/build-docker-images.py --py-versions py37 -T cpu -T cu115 -T cu116 --build-type BUILDKITE --build-base
-
-- label: ":docker: Build Images: py37 (2/2)"
-  conditions: ["RAY_CI_PYTHON_DEPENDENCIES_AFFECTED", "RAY_CI_DOCKER_AFFECTED", "RAY_CI_CORE_CPP_AFFECTED"]
-  instance_size: medium
-  commands:
-    - LINUX_WHEELS=1 BUILD_ONE_PYTHON_ONLY=3.7 ./ci/ci.sh build
-    - pip install -q docker aws_requests_auth boto3
-    - ./ci/env/env_info.sh
-    - if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then python .buildkite/copy_files.py --destination docker_login; fi
-    - python ./ci/build/build-docker-images.py --py-versions py37 -T cu117 -T cu118 --build-type BUILDKITE --build-base
-
-- label: ":docker: Build Images: py38 (1/2)"
-  conditions: ["RAY_CI_PYTHON_DEPENDENCIES_AFFECTED", "RAY_CI_DOCKER_AFFECTED", "RAY_CI_CORE_CPP_AFFECTED"]
-  instance_size: medium
-  commands:
-    - LINUX_WHEELS=1 BUILD_ONE_PYTHON_ONLY=3.8 ./ci/ci.sh build
-    - pip install -q docker aws_requests_auth boto3
-    - ./ci/env/env_info.sh
-    - if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then python .buildkite/copy_files.py --destination docker_login; fi
-    - python ./ci/build/build-docker-images.py --py-versions py38 -T cpu -T cu115 -T cu116 --build-type BUILDKITE --build-base
-
-- label: ":docker: Build Images: py38 (2/2)"
-  conditions: ["RAY_CI_PYTHON_DEPENDENCIES_AFFECTED", "RAY_CI_DOCKER_AFFECTED", "RAY_CI_CORE_CPP_AFFECTED"]
-  instance_size: medium
-  commands:
-    - LINUX_WHEELS=1 BUILD_ONE_PYTHON_ONLY=3.8 ./ci/ci.sh build
-    - pip install -q docker aws_requests_auth boto3
-    - ./ci/env/env_info.sh
-    - if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then python .buildkite/copy_files.py --destination docker_login; fi
-    - python ./ci/build/build-docker-images.py --py-versions py38 -T cu117 -T cu118 --build-type BUILDKITE --build-base
-
-- label: ":docker: Build Images: py39 (1/2)"
-  conditions: ["RAY_CI_PYTHON_DEPENDENCIES_AFFECTED", "RAY_CI_DOCKER_AFFECTED", "RAY_CI_CORE_CPP_AFFECTED"]
-  instance_size: medium
-  commands:
-    - LINUX_WHEELS=1 BUILD_ONE_PYTHON_ONLY=3.9 ./ci/ci.sh build
-    - pip install -q docker aws_requests_auth boto3
-    - ./ci/env/env_info.sh
-    - if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then python .buildkite/copy_files.py --destination docker_login; fi
-    - python ./ci/build/build-docker-images.py --py-versions py39 -T cpu -T cu115 -T cu116 --build-type BUILDKITE --build-base
-
-- label: ":docker: Build Images: py39 (2/2)"
-  conditions: ["RAY_CI_PYTHON_DEPENDENCIES_AFFECTED", "RAY_CI_DOCKER_AFFECTED", "RAY_CI_CORE_CPP_AFFECTED"]
-  instance_size: medium
-  commands:
-    - LINUX_WHEELS=1 BUILD_ONE_PYTHON_ONLY=3.9 ./ci/ci.sh build
-    - pip install -q docker aws_requests_auth boto3
-    - ./ci/env/env_info.sh
-    - if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then python .buildkite/copy_files.py --destination docker_login; fi
-    - python ./ci/build/build-docker-images.py --py-versions py39 -T cu117 -T cu118 --build-type BUILDKITE --build-base
+    - if [[ "${BUILDKITE_PULL_REQUEST}" == "false" ]]; then python .buildkite/copy_files.py --destination docker_login; fi
+    - python ./ci/build/build-docker-images.py --py-versions {{matrix}} -T cpu -T cu115 -T cu116 --build-type BUILDKITE --build-base
+  matrix:
+    - py37
+    - py38
+    - py39
+    - py310
 
-- label: ":docker: Build Images: py310 (1/2)"
+- label: ":docker: Build Images: {{matrix}} - cu117/cu118"
   conditions: ["RAY_CI_PYTHON_DEPENDENCIES_AFFECTED", "RAY_CI_DOCKER_AFFECTED", "RAY_CI_CORE_CPP_AFFECTED"]
   instance_size: medium
   commands:
-    - LINUX_WHEELS=1 BUILD_ONE_PYTHON_ONLY=3.10 ./ci/ci.sh build
+    - LINUX_WHEELS=1 BUILD_ONE_PYTHON_ONLY={{matrix}} ./ci/ci.sh build
     - pip install -q docker aws_requests_auth boto3
     - ./ci/env/env_info.sh
-    - if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then python .buildkite/copy_files.py --destination docker_login; fi
-    - python ./ci/build/build-docker-images.py --py-versions py310 -T cpu -T cu115 -T cu116 --build-type BUILDKITE --build-base
-
-- label: ":docker: Build Images: py310 (2/2)"
-  conditions: ["RAY_CI_PYTHON_DEPENDENCIES_AFFECTED", "RAY_CI_DOCKER_AFFECTED", "RAY_CI_CORE_CPP_AFFECTED"]
-  instance_size: medium
-  commands:
-    - LINUX_WHEELS=1 BUILD_ONE_PYTHON_ONLY=3.10 ./ci/ci.sh build
-    - pip install -q docker aws_requests_auth boto3
-    - if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then python .buildkite/copy_files.py --destination docker_login; fi
-    - python ./ci/build/build-docker-images.py --py-versions py310 -T cu117 -T cu118 --build-type BUILDKITE --build-base
+    - if [[ "${BUILDKITE_PULL_REQUEST}" == "false" ]]; then python .buildkite/copy_files.py --destination docker_login; fi
+    - python ./ci/build/build-docker-images.py --py-versions {{matrix}} -T cu117 -T cu118 --build-type BUILDKITE --build-base
+  matrix:
+    - py37
+    - py38
+    - py39
+    - py310
 
 - label: ":java: Java"
   conditions: ["RAY_CI_JAVA_AFFECTED"]
diff --git a/python/build-wheel-manylinux2014.sh b/python/build-wheel-manylinux2014.sh
index ca5733a480999..5cfb4eb5e5eee 100755
--- a/python/build-wheel-manylinux2014.sh
+++ b/python/build-wheel-manylinux2014.sh
@@ -15,11 +15,11 @@ NODE_VERSION="14"
 
 # Python version key, interpreter version code, numpy tuples.
 PYTHON_NUMPYS=(
-  "3.7 cp37-cp37m 1.14.5"
-  "3.8 cp38-cp38 1.14.5"
-  "3.9 cp39-cp39 1.19.3"
-  "3.10 cp310-cp310 1.22.0"
-  "3.11 cp311-cp311 1.22.0"
+  "py37 cp37-cp37m 1.14.5"
+  "py38 cp38-cp38 1.14.5"
+  "py39 cp39-cp39 1.19.3"
+  "py310 cp310-cp310 1.22.0"
+  "py311 cp311-cp311 1.22.0"
 )
 
 yum -y install unzip zip sudo