From 90b5a8ee52f5389501628b2628eaad286ee25284 Mon Sep 17 00:00:00 2001
From: can <can@anyscale.com>
Date: Thu, 8 Jun 2023 20:40:25 +0000
Subject: [PATCH] Support docker build for 3.11

Signed-off-by: can <can@anyscale.com>
---
 .buildkite/pipeline.build.yml           | 19 +++++++++++++
 ci/build/build-docker-images.py         |  1 +
 docker/base-deps/Dockerfile             | 37 +++++++++++++++++++------
 docker/retag-lambda/python_versions.txt |  1 +
 4 files changed, 49 insertions(+), 9 deletions(-)

diff --git a/.buildkite/pipeline.build.yml b/.buildkite/pipeline.build.yml
index 5d92d37bfa297..554ff5ffae55b 100644
--- a/.buildkite/pipeline.build.yml
+++ b/.buildkite/pipeline.build.yml
@@ -162,6 +162,25 @@
     - if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then python .buildkite/copy_files.py --destination docker_login; fi
     - python ./ci/build/build-docker-images.py --py-versions py310 -T cu111 -T cu112 -T cu113 -T cu116 -T cu118 --build-type BUILDKITE --build-base
 
+- label: ":docker: Build Images: py311 (1/2)"
+  conditions: ["RAY_CI_PYTHON_DEPENDENCIES_AFFECTED", "RAY_CI_DOCKER_AFFECTED", "RAY_CI_CORE_CPP_AFFECTED"]
+  instance_size: medium
+  commands:
+    - LINUX_WHEELS=1 BUILD_ONE_PYTHON_ONLY=3.11 ./ci/ci.sh build
+    - pip install -q docker aws_requests_auth boto3
+    - ./ci/env/env_info.sh
+    - if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then python .buildkite/copy_files.py --destination docker_login; fi
+    - python ./ci/build/build-docker-images.py --py-versions py311 -T cpu -T cu101 -T cu102 -T cu110 --build-type BUILDKITE --build-base
+
+- label: ":docker: Build Images: py311 (2/2)"
+  conditions: ["RAY_CI_PYTHON_DEPENDENCIES_AFFECTED", "RAY_CI_DOCKER_AFFECTED", "RAY_CI_CORE_CPP_AFFECTED"]
+  instance_size: medium
+  commands:
+    - LINUX_WHEELS=1 BUILD_ONE_PYTHON_ONLY=3.11 ./ci/ci.sh build
+    - pip install -q docker aws_requests_auth boto3
+    - if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then python .buildkite/copy_files.py --destination docker_login; fi
+    - python ./ci/build/build-docker-images.py --py-versions py311 -T cu111 -T cu112 -T cu113 -T cu116 -T cu118 --build-type BUILDKITE --build-base
+
 - label: ":java: Java"
   conditions: ["RAY_CI_JAVA_AFFECTED"]
   instance_size: medium
diff --git a/ci/build/build-docker-images.py b/ci/build/build-docker-images.py
index d4f0baf39b142..a0d1c9f1c9318 100644
--- a/ci/build/build-docker-images.py
+++ b/ci/build/build-docker-images.py
@@ -42,6 +42,7 @@
     "py38": "3.8",
     "py39": "3.9",
     "py310": "3.10",
+    "py311": "3.11",
 }
 
 BASE_IMAGES = {
diff --git a/docker/base-deps/Dockerfile b/docker/base-deps/Dockerfile
index 63c9d42c47e76..57831362c561d 100644
--- a/docker/base-deps/Dockerfile
+++ b/docker/base-deps/Dockerfile
@@ -1,3 +1,6 @@
+# syntax=docker/dockerfile:1.3-labs
+# shellcheck disable=SC2148
+
 # The base-deps Docker image installs main libraries needed to run Ray
 
 # The GPU options are NVIDIA CUDA developer images.
@@ -33,7 +36,10 @@ ENV HOME=/home/ray
 # images for Python 3.7.
 # https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-${HOSTTYPE}.sh
 
-RUN sudo apt-get update -y && sudo apt-get upgrade -y \
+RUN <<EOF
+#!/bin/bash
+
+sudo apt-get update -y && sudo apt-get upgrade -y \
     && sudo apt-get install -y \
         git \
         libjemalloc-dev \
@@ -47,18 +53,29 @@ RUN sudo apt-get update -y && sudo apt-get upgrade -y \
         rsync \
         netbase \
         openssh-client \
-        gnupg; fi) \
-    && wget \
-        --quiet "https://repo.anaconda.com/miniconda/Miniconda3-py37_23.1.0-1-Linux-${HOSTTYPE}.sh" \
-        -O /tmp/miniconda.sh \
+        gnupg; fi)
+
+if [ "$PYTHON_VERSION" == "3.11" ]; then
+    MINICONDA="https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-${HOSTTYPE}.sh"
+else
+    MINICONDA="https://repo.anaconda.com/miniconda/Miniconda3-py37_23.1.0-1-Linux-${HOSTTYPE}.sh"
+fi
+
+wget --quiet "$MINICONDA" -O /tmp/miniconda.sh \
     && /bin/bash /tmp/miniconda.sh -b -u -p $HOME/anaconda3 \
     && $HOME/anaconda3/bin/conda init \ 
     && echo 'export PATH=$HOME/anaconda3/bin:$PATH' >> /home/ray/.bashrc \
-    && rm /tmp/miniconda.sh \
-    && $HOME/anaconda3/bin/conda install -y \
+    && rm /tmp/miniconda.sh 
+
+if [ "$PYTHON_VERSION" == "3.11" ]; then
+    $HOME/anaconda3/bin/conda install -y libgcc-ng python=$PYTHON_VERSION
+else
+    $HOME/anaconda3/bin/conda install -y \
         # remove libffi pinning when fixed upstream #33299
-        libgcc-ng libffi=3.3 python=$PYTHON_VERSION \
-    && $HOME/anaconda3/bin/conda clean -y --all \
+        libgcc-ng libffi=3.3 python=$PYTHON_VERSION
+fi
+
+$HOME/anaconda3/bin/conda clean -y --all \
     && $HOME/anaconda3/bin/pip install --no-cache-dir \
         flatbuffers \
         cython==0.29.32 \
@@ -85,4 +102,6 @@ RUN sudo apt-get update -y && sudo apt-get upgrade -y \
     && sudo rm -rf /var/lib/apt/lists/* \
     && sudo apt-get clean
 
+EOF
+
 WORKDIR $HOME
diff --git a/docker/retag-lambda/python_versions.txt b/docker/retag-lambda/python_versions.txt
index 8d4b09f3bd1d0..f7ed4ea62206a 100644
--- a/docker/retag-lambda/python_versions.txt
+++ b/docker/retag-lambda/python_versions.txt
@@ -2,3 +2,4 @@ py37
 py38
 py39
 py310
+py311