From 2ca4f695e7806145226ff06f76ec81b320f5f415 Mon Sep 17 00:00:00 2001 From: Marc Helbling Date: Wed, 28 Aug 2024 09:40:41 +0000 Subject: [PATCH] feat: build python shared objects **Summary:** See https://github.com/pypa/manylinux/pull/1185 We propose to: * update the branch and "bake" the shared libraries build * remove old Python (<3.8) versions that we do not care * remove the GIL build * remove compression for static libs --- docker/Dockerfile | 19 +------------------ docker/build_scripts/build-cpython.sh | 4 ++-- docker/build_scripts/finalize.sh | 12 ++++++------ 3 files changed, 9 insertions(+), 26 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 40cb125c..b27baabf 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -107,16 +107,6 @@ RUN export OPENSSL_ROOT=openssl-3.0.14 && \ export OPENSSL_DOWNLOAD_URL=https://www.openssl.org/source && \ manylinux-entrypoint /build_scripts/build-openssl.sh - -FROM build_cpython_system_ssl AS build_cpython36 -COPY build_scripts/cpython-pubkeys.txt /build_scripts/cpython-pubkeys.txt -RUN manylinux-entrypoint /build_scripts/build-cpython.sh 3.6.15 - -FROM build_cpython_system_ssl AS build_cpython37 -COPY build_scripts/cpython-pubkeys.txt /build_scripts/cpython-pubkeys.txt -RUN manylinux-entrypoint /build_scripts/build-cpython.sh 3.7.17 - - FROM build_cpython AS build_cpython38 COPY build_scripts/ambv-pubkey.txt /build_scripts/cpython-pubkeys.txt RUN manylinux-entrypoint /build_scripts/build-cpython.sh 3.8.19 @@ -141,24 +131,17 @@ FROM build_cpython AS build_cpython313 COPY build_scripts/cpython-pubkey-312-313.txt /build_scripts/cpython-pubkeys.txt RUN manylinux-entrypoint /build_scripts/build-cpython.sh 3.13.0rc1 -FROM build_cpython AS build_cpython313_nogil -COPY build_scripts/cpython-pubkey-312-313.txt /build_scripts/cpython-pubkeys.txt -RUN manylinux-entrypoint /build_scripts/build-cpython.sh 3.13.0rc1 nogil - FROM runtime_base COPY --from=build_git /manylinux-rootfs / COPY --from=build_cpython_system_ssl /manylinux-rootfs / COPY build_scripts /opt/_internal/build_scripts/ -RUN --mount=type=bind,target=/build_cpython36,from=build_cpython36 \ - --mount=type=bind,target=/build_cpython37,from=build_cpython37 \ - --mount=type=bind,target=/build_cpython38,from=build_cpython38 \ +RUN --mount=type=bind,target=/build_cpython38,from=build_cpython38 \ --mount=type=bind,target=/build_cpython39,from=build_cpython39 \ --mount=type=bind,target=/build_cpython310,from=build_cpython310 \ --mount=type=bind,target=/build_cpython311,from=build_cpython311 \ --mount=type=bind,target=/build_cpython312,from=build_cpython312 \ --mount=type=bind,target=/build_cpython313,from=build_cpython313 \ - --mount=type=bind,target=/build_cpython313_nogil,from=build_cpython313_nogil \ mkdir -p /opt/_internal && \ cp -rf /build_cpython*/opt/_internal/* /opt/_internal/ && \ manylinux-entrypoint /opt/_internal/build_scripts/finalize.sh \ diff --git a/docker/build_scripts/build-cpython.sh b/docker/build_scripts/build-cpython.sh index ea7485a9..b7e9cffc 100755 --- a/docker/build_scripts/build-cpython.sh +++ b/docker/build_scripts/build-cpython.sh @@ -33,7 +33,7 @@ pushd Python-${CPYTHON_VERSION} PREFIX="/opt/_internal/cpython-${CPYTHON_VERSION}" mkdir -p ${PREFIX}/lib CFLAGS_EXTRA="" -CONFIGURE_ARGS="--disable-shared --with-ensurepip=no" +CONFIGURE_ARGS="--enable-shared --with-ensurepip=no" if [ "${2:-}" == "nogil" ]; then PREFIX="${PREFIX}-nogil" @@ -63,7 +63,7 @@ fi # do not change the default for user built extension (yet?) ./configure \ CFLAGS_NODIST="${MANYLINUX_CFLAGS} ${MANYLINUX_CPPFLAGS} ${CFLAGS_EXTRA}" \ - LDFLAGS_NODIST="${MANYLINUX_LDFLAGS}" \ + LDFLAGS_NODIST="${MANYLINUX_LDFLAGS} -Wl,-rpath=${PREFIX}/lib" \ --prefix=${PREFIX} ${CONFIGURE_ARGS} > /dev/null make > /dev/null make install > /dev/null diff --git a/docker/build_scripts/finalize.sh b/docker/build_scripts/finalize.sh index fc5a9955..abafce83 100755 --- a/docker/build_scripts/finalize.sh +++ b/docker/build_scripts/finalize.sh @@ -9,12 +9,12 @@ MY_DIR=$(dirname "${BASH_SOURCE[0]}") # Get build utilities source $MY_DIR/build_utils.sh -# most people don't need libpython*.a, and they're many megabytes. -# compress them all together for best efficiency -pushd /opt/_internal -XZ_OPT=-9e tar -cJf static-libs-for-embedding-only.tar.xz cpython-*/lib/libpython*.a -popd -find /opt/_internal -name '*.a' -print0 | xargs -0 rm -f +# # most people don't need libpython*.a, and they're many megabytes. +# # compress them all together for best efficiency +# pushd /opt/_internal +# XZ_OPT=-9e tar -cJf static-libs-for-embedding-only.tar.xz cpython-*/lib/libpython*.a +# popd +# find /opt/_internal -name '*.a' -print0 | xargs -0 rm -f # disable some pip warnings export PIP_ROOT_USER_ACTION=ignore