From b1853f37917afecdb6a7f76ba77c0ae489a8096d Mon Sep 17 00:00:00 2001 From: Tim Felgentreff Date: Mon, 17 Jul 2023 16:40:11 +0200 Subject: [PATCH 1/2] Add graalpy --- docker/Dockerfile | 3 ++ docker/build_scripts/finalize.sh | 4 +- docker/build_scripts/graalpy.sha256 | 2 + docker/build_scripts/install-graalpy.sh | 66 +++++++++++++++++++++++++ tests/run_tests.sh | 2 + 5 files changed, 76 insertions(+), 1 deletion(-) create mode 100644 docker/build_scripts/graalpy.sha256 create mode 100755 docker/build_scripts/install-graalpy.sh diff --git a/docker/Dockerfile b/docker/Dockerfile index da8a64aa..725562d2 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -141,12 +141,15 @@ RUN manylinux-entrypoint /build_scripts/build-cpython.sh 3.12.0b4 FROM build_cpython AS all_python COPY build_scripts/install-pypy.sh \ build_scripts/pypy.sha256 \ + build_scripts/install-graalpy.sh \ + build_scripts/graalpy.sha256 \ build_scripts/finalize-python.sh \ /build_scripts/ RUN manylinux-entrypoint /build_scripts/install-pypy.sh 3.7 7.3.9 RUN manylinux-entrypoint /build_scripts/install-pypy.sh 3.8 7.3.11 RUN manylinux-entrypoint /build_scripts/install-pypy.sh 3.9 7.3.12 RUN manylinux-entrypoint /build_scripts/install-pypy.sh 3.10 7.3.12 +RUN manylinux-entrypoint /build_scripts/install-graalpy.sh 3.10 graal 23.0.0 graalpython COPY --from=build_cpython36 /opt/_internal /opt/_internal/ COPY --from=build_cpython37 /opt/_internal /opt/_internal/ COPY --from=build_cpython38 /opt/_internal /opt/_internal/ diff --git a/docker/build_scripts/finalize.sh b/docker/build_scripts/finalize.sh index 621eab92..9c9ae1e2 100755 --- a/docker/build_scripts/finalize.sh +++ b/docker/build_scripts/finalize.sh @@ -10,7 +10,7 @@ MY_DIR=$(dirname "${BASH_SOURCE[0]}") source $MY_DIR/build_utils.sh mkdir /opt/python -for PREFIX in $(find /opt/_internal/ -mindepth 1 -maxdepth 1 \( -name 'cpython*' -o -name 'pypy*' \)); do +for PREFIX in $(find /opt/_internal/ -mindepth 1 -maxdepth 1 \( -name 'cpython*' -o -name 'pypy*' -o -name 'graalpy*' \)); do # Some python's install as bin/python3. Make them available as # bin/python. if [ -e ${PREFIX}/bin/python3 ] && [ ! -e ${PREFIX}/bin/python ]; then @@ -30,6 +30,8 @@ for PREFIX in $(find /opt/_internal/ -mindepth 1 -maxdepth 1 \( -name 'cpython*' # Make versioned python commands available directly in environment. if [[ "${PREFIX}" == *"/pypy"* ]]; then ln -s ${PREFIX}/bin/python /usr/local/bin/pypy${PY_VER} + elif [[ "${PREFIX}" == *"/graalpy"* ]]; then + ln -s ${PREFIX}/bin/python /usr/local/bin/graalpy${PY_VER} else ln -s ${PREFIX}/bin/python /usr/local/bin/python${PY_VER} fi diff --git a/docker/build_scripts/graalpy.sha256 b/docker/build_scripts/graalpy.sha256 new file mode 100644 index 00000000..361ffc9d --- /dev/null +++ b/docker/build_scripts/graalpy.sha256 @@ -0,0 +1,2 @@ +e2a00b2b6485282b4a04aa382e30d696e00d20eb2fe1736debbe2d9df2a8737a graalpython-23.0.0-linux-aarch64.tar.gz +25e4fa7c1d45db6dcac5bfa4d1a0aa9ef5581623dc5903ce98d246c5d394639c graalpython-23.0.0-linux-amd64.tar.gz diff --git a/docker/build_scripts/install-graalpy.sh b/docker/build_scripts/install-graalpy.sh new file mode 100755 index 00000000..bb79a113 --- /dev/null +++ b/docker/build_scripts/install-graalpy.sh @@ -0,0 +1,66 @@ +#!/bin/bash + +# Stop at any error, show all commands +set -exuo pipefail + +# Get script directory +MY_DIR=$(dirname "${BASH_SOURCE[0]}") + +# Get build utilities +source $MY_DIR/build_utils.sh + +if [ "${BASE_POLICY}" == "musllinux" ]; then + echo "Skip GraalPy build on musllinux" + exit 0 +fi + +PYTHON_VERSION=$1 +VERSION_PREFIX=$2 +GRAALPY_VERSION=$3 +ARCHIVE_PREFIX=$4 +GRAALPY_DOWNLOAD_URL=https://github.com/oracle/graalpython/releases/download/${VERSION_PREFIX}-${GRAALPY_VERSION}/ +# graal-23.0.0/graalpython-23.0.0-linux-amd64.tar.gz + + +function get_shortdir { + local exe=$1 + $exe -c 'import sys; print(sys.implementation.cache_tag)' +} + + +mkdir -p /tmp +cd /tmp + +case ${AUDITWHEEL_ARCH} in + x86_64) GRAALPY_ARCH=amd64;; + aarch64) GRAALPY_ARCH=aarch64;; + *) echo "No PyPy for ${AUDITWHEEL_ARCH}"; exit 0;; +esac + +EXPAND_NAME=graalpy-${GRAALPY_VERSION}-linux-${GRAALPY_ARCH} +TMPDIR=/tmp/${EXPAND_NAME} +TARBALL=graalpython-${GRAALPY_VERSION}-linux-${GRAALPY_ARCH}.tar.gz +PREFIX="/opt/_internal" + +mkdir -p ${PREFIX} + +fetch_source ${TARBALL} ${GRAALPY_DOWNLOAD_URL} + +# We only want to check the current tarball sha256sum +grep " ${TARBALL}\$" ${MY_DIR}/graalpy.sha256 > ${TARBALL}.sha256 +# then check sha256 sum +sha256sum -c ${TARBALL}.sha256 + +tar -xf ${TARBALL} + +# rename the directory to something shorter like graalpy230-310 +PREFIX=${PREFIX}/$(get_shortdir ${TMPDIR}/bin/graalpy) +mv ${TMPDIR} ${PREFIX} + +# add a generic "python" symlink +if [ ! -f "${PREFIX}/bin/python" ]; then + ln -s graalpy ${PREFIX}/bin/python +fi + +# We do not need precompiled .pyc and .pyo files. +clean_pyc ${PREFIX} diff --git a/tests/run_tests.sh b/tests/run_tests.sh index 3cbaf62b..24634b82 100755 --- a/tests/run_tests.sh +++ b/tests/run_tests.sh @@ -28,6 +28,8 @@ for PYTHON in /opt/python/*/bin/python; do PYVERS=$(${PYTHON} -c "import sys; print('.'.join(map(str, sys.version_info[:2])))") if [ "${IMPLEMENTATION}" == "pypy" ]; then LINK_PREFIX=pypy + elif [ "${IMPLEMENTATION}" == "graalpy" ]; then + LINK_PREFIX=graalpy else LINK_PREFIX=python # Make sure sqlite3 module can be loaded properly and is the manylinux version one From 505e645b8f33e7eeb1635f2be4d20561bf9cd4b8 Mon Sep 17 00:00:00 2001 From: Tim Felgentreff Date: Tue, 25 Jul 2023 01:57:05 +0200 Subject: [PATCH 2/2] mostly remove graalpy from the final image to reduce size --- docker/build_scripts/finalize.sh | 7 +++ docker/build_scripts/install-graalpy.sh | 69 ++++++++++--------------- tests/run_tests.sh | 4 ++ 3 files changed, 38 insertions(+), 42 deletions(-) diff --git a/docker/build_scripts/finalize.sh b/docker/build_scripts/finalize.sh index 9c9ae1e2..c6b93698 100755 --- a/docker/build_scripts/finalize.sh +++ b/docker/build_scripts/finalize.sh @@ -32,6 +32,13 @@ for PREFIX in $(find /opt/_internal/ -mindepth 1 -maxdepth 1 \( -name 'cpython*' ln -s ${PREFIX}/bin/python /usr/local/bin/pypy${PY_VER} elif [[ "${PREFIX}" == *"/graalpy"* ]]; then ln -s ${PREFIX}/bin/python /usr/local/bin/graalpy${PY_VER} + # we remove most of graalpy itself, but symlink the install script to + # get the same version (same sha) back in the same place + ln -s ${PREFIX}/install-graalpy.sh /usr/local/bin/install-graalpy${PY_VER} + rm -rf ${PREFIX}/lib/graalpy* + rm -rf ${PREFIX}/lib/sulong* + rm -rf ${PREFIX}/lib/llvm* + rm -rf ${PREFIX}/lib/*.so else ln -s ${PREFIX}/bin/python /usr/local/bin/python${PY_VER} fi diff --git a/docker/build_scripts/install-graalpy.sh b/docker/build_scripts/install-graalpy.sh index bb79a113..530a524b 100755 --- a/docker/build_scripts/install-graalpy.sh +++ b/docker/build_scripts/install-graalpy.sh @@ -14,53 +14,38 @@ if [ "${BASE_POLICY}" == "musllinux" ]; then exit 0 fi -PYTHON_VERSION=$1 -VERSION_PREFIX=$2 -GRAALPY_VERSION=$3 -ARCHIVE_PREFIX=$4 -GRAALPY_DOWNLOAD_URL=https://github.com/oracle/graalpython/releases/download/${VERSION_PREFIX}-${GRAALPY_VERSION}/ -# graal-23.0.0/graalpython-23.0.0-linux-amd64.tar.gz - - -function get_shortdir { - local exe=$1 - $exe -c 'import sys; print(sys.implementation.cache_tag)' -} - - -mkdir -p /tmp -cd /tmp - case ${AUDITWHEEL_ARCH} in x86_64) GRAALPY_ARCH=amd64;; aarch64) GRAALPY_ARCH=aarch64;; - *) echo "No PyPy for ${AUDITWHEEL_ARCH}"; exit 0;; + *) echo "No GraalPy for ${AUDITWHEEL_ARCH}"; exit 0;; esac -EXPAND_NAME=graalpy-${GRAALPY_VERSION}-linux-${GRAALPY_ARCH} -TMPDIR=/tmp/${EXPAND_NAME} +PYTHON_VERSION=$1 +VERSION_PREFIX=$2 +GRAALPY_VERSION=$3 +ARCHIVE_PREFIX=$4 +GRAALPY_DOWNLOAD_URL=https://github.com/oracle/graalpython/releases/download/${VERSION_PREFIX}-${GRAALPY_VERSION}/ # e.g. graal-23.0.0/graalpython-23.0.0-linux-amd64.tar.gz +TMPDIR=/tmp/ TARBALL=graalpython-${GRAALPY_VERSION}-linux-${GRAALPY_ARCH}.tar.gz -PREFIX="/opt/_internal" +TARBALL_SHA=`grep " ${TARBALL}\$" ${MY_DIR}/graalpy.sha256` +PREFIX="/opt/_internal/graalpy-${GRAALPY_VERSION}" +# create a download script that will download and extract graalpy. we leave +# this script in the image to avoid the large distribution to use up space in +# the default image. mkdir -p ${PREFIX} - -fetch_source ${TARBALL} ${GRAALPY_DOWNLOAD_URL} - -# We only want to check the current tarball sha256sum -grep " ${TARBALL}\$" ${MY_DIR}/graalpy.sha256 > ${TARBALL}.sha256 -# then check sha256 sum -sha256sum -c ${TARBALL}.sha256 - -tar -xf ${TARBALL} - -# rename the directory to something shorter like graalpy230-310 -PREFIX=${PREFIX}/$(get_shortdir ${TMPDIR}/bin/graalpy) -mv ${TMPDIR} ${PREFIX} - -# add a generic "python" symlink -if [ ! -f "${PREFIX}/bin/python" ]; then - ln -s graalpy ${PREFIX}/bin/python -fi - -# We do not need precompiled .pyc and .pyo files. -clean_pyc ${PREFIX} +cat < ${PREFIX}/install-graalpy.sh +#!/bin/bash +set -exuo pipefail +mkdir -p ${PREFIX} +mkdir -p ${TMPDIR} +curl -fsSL -o "${TMPDIR}/${TARBALL}" "${GRAALPY_DOWNLOAD_URL}/${TARBALL}" +cd ${TMPDIR} +echo "${TARBALL_SHA}" | sha256sum -c +tar -xf "${TMPDIR}/${TARBALL}" --overwrite --strip-components=1 -C "${PREFIX}" +rm -f "${TMPDIR}/${TARBALL}" +EOF + +# call the download script right now. +chmod +x ${PREFIX}/install-graalpy.sh +${PREFIX}/install-graalpy.sh diff --git a/tests/run_tests.sh b/tests/run_tests.sh index 24634b82..3e62dcb8 100755 --- a/tests/run_tests.sh +++ b/tests/run_tests.sh @@ -17,6 +17,10 @@ else exit 1 fi +# make sure all graalpy versions are available for testing +for INSTALLER in $(find /opt/_internal/ -mindepth 2 -maxdepth 2 \( -name 'install-graalpy.sh' \)); do + $INSTALLER +done for PYTHON in /opt/python/*/bin/python; do # Smoke test to make sure that our Pythons work, and do indeed detect as