Skip to content

Commit

Permalink
Build hardening (#1009)
Browse files Browse the repository at this point in the history
* Use hardening for building all tools & libraries

This does not affect the wheels that are produced by end users as proposed in #59 but mitigates potential security issues in the tools used by manylinux images as mentioned in #1005

* Always update system packages in the final step

Since docker cache is used, system packages are not updated when cache is present. Always update them in the final step.
  • Loading branch information
mayeut authored Feb 27, 2021
1 parent 2c345f8 commit 114e2c5
Show file tree
Hide file tree
Showing 11 changed files with 140 additions and 64 deletions.
13 changes: 6 additions & 7 deletions docker/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -21,18 +21,17 @@ ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH_ARG}
ENV PATH=${PREPEND_PATH}${PATH}
ENV PKG_CONFIG_PATH=/usr/local/lib/pkgconfig

# first fixup mirrors, keep the script around
# first copy the fixup mirrors script, keep the script around
COPY build_scripts/fixup-mirrors.sh /usr/local/sbin/fixup-mirrors
RUN fixup-mirrors

# setup entrypoint, this will wrap commands with `linux32` with i686 images
COPY build_scripts/install-entrypoint.sh /build_scripts/
RUN bash /build_scripts/install-entrypoint.sh && rm -rf build_scripts
COPY build_scripts/install-entrypoint.sh build_scripts/update-system-packages.sh /build_scripts/
RUN bash /build_scripts/install-entrypoint.sh && rm -rf /build_scripts
COPY manylinux-entrypoint /usr/local/bin/manylinux-entrypoint
ENTRYPOINT ["manylinux-entrypoint"]

COPY build_scripts/install-runtime-packages.sh /build_scripts/
RUN manylinux-entrypoint /build_scripts/install-runtime-packages.sh && rm /build_scripts/install-runtime-packages.sh
COPY build_scripts/install-runtime-packages.sh build_scripts/update-system-packages.sh /build_scripts/
RUN manylinux-entrypoint /build_scripts/install-runtime-packages.sh && rm -rf /build_scripts/

COPY build_scripts/build_utils.sh /build_scripts/

Expand Down Expand Up @@ -160,7 +159,7 @@ COPY --from=build_cmake /manylinux-rootfs /
COPY --from=build_swig /manylinux-rootfs /
COPY --from=build_cpython /manylinux-rootfs /
COPY --from=all_cpython /opt/_internal /opt/_internal/
COPY build_scripts/finalize.sh build_scripts/python-tag-abi-tag.py build_scripts/requirements.txt build_scripts/requirements-tools.txt /build_scripts/
COPY build_scripts/finalize.sh build_scripts/update-system-packages.sh build_scripts/python-tag-abi-tag.py build_scripts/requirements.txt build_scripts/requirements-tools.txt /build_scripts/
RUN manylinux-entrypoint /build_scripts/finalize.sh && rm -rf /build_scripts

ENV SSL_CERT_FILE=/opt/_internal/certs.pem
Expand Down
4 changes: 4 additions & 0 deletions docker/build_scripts/build-cmake.sh
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,10 @@ fetch_source cmake-${CMAKE_VERSION}.tar.gz ${CMAKE_DOWNLOAD_URL}/v${CMAKE_VERSIO
check_sha256sum cmake-${CMAKE_VERSION}.tar.gz ${CMAKE_HASH}
tar -xzf cmake-${CMAKE_VERSION}.tar.gz
pushd cmake-${CMAKE_VERSION}
export CPPFLAGS="${MANYLINUX_CPPFLAGS}"
export CFLAGS="${MANYLINUX_CFLAGS} ${CPPFLAGS}"
export CXXFLAGS="${MANYLINUX_CXXFLAGS} ${CPPFLAGS}"
export LDFLAGS="${MANYLINUX_LDFLAGS}"
./bootstrap --system-curl --parallel=$(nproc)
make -j$(nproc)
make install DESTDIR=/manylinux-rootfs
Expand Down
15 changes: 14 additions & 1 deletion docker/build_scripts/build-cpython.sh
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,20 @@ tar -xzf Python-${CPYTHON_VERSION}.tgz
pushd Python-${CPYTHON_VERSION}
PREFIX="/opt/_internal/cpython-${CPYTHON_VERSION}"
mkdir -p ${PREFIX}/lib
./configure --prefix=${PREFIX} --disable-shared --with-ensurepip=no > /dev/null
# configure with hardening options only for the interpreter & stdlib C extensions
# do not change the default for user built extension (yet?)
if [ "${CPYTHON_VERSION:0:4}" == "3.5." ]; then
./configure \
CFLAGS_NODIST="${MANYLINUX_CFLAGS} ${MANYLINUX_CPPFLAGS}" \
--prefix=${PREFIX} --disable-shared --with-ensurepip=no > /dev/null
# those are not picked-up by distutils in CPython 3.5 which has no LDFLAGS_NODIST option in configure
export LDFLAGS="${MANYLINUX_LDFLAGS}"
else
./configure \
CFLAGS_NODIST="${MANYLINUX_CFLAGS} ${MANYLINUX_CPPFLAGS}" \
LDFLAGS_NODIST="${MANYLINUX_LDFLAGS}" \
--prefix=${PREFIX} --disable-shared --with-ensurepip=no > /dev/null
fi
make -j$(nproc) > /dev/null
make -j$(nproc) install > /dev/null
popd
Expand Down
2 changes: 1 addition & 1 deletion docker/build_scripts/build-git.sh
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ fetch_source ${GIT_ROOT}.tar.gz ${GIT_DOWNLOAD_URL}
check_sha256sum ${GIT_ROOT}.tar.gz ${GIT_HASH}
tar -xzf ${GIT_ROOT}.tar.gz
pushd ${GIT_ROOT}
make -j$(nproc) install prefix=/usr/local NO_GETTEXT=1 NO_TCLTK=1 DESTDIR=/manylinux-rootfs
make -j$(nproc) install prefix=/usr/local NO_GETTEXT=1 NO_TCLTK=1 DESTDIR=/manylinux-rootfs CPPFLAGS="${MANYLINUX_CPPFLAGS}" CFLAGS="${MANYLINUX_CFLAGS}" CXXFLAGS="${MANYLINUX_CXXFLAGS}" LDFLAGS="${MANYLINUX_LDFLAGS}"
popd
rm -rf ${GIT_ROOT} ${GIT_ROOT}.tar.gz

Expand Down
2 changes: 1 addition & 1 deletion docker/build_scripts/build-openssl.sh
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ fetch_source ${OPENSSL_ROOT}.tar.gz ${OPENSSL_DOWNLOAD_URL}
check_sha256sum ${OPENSSL_ROOT}.tar.gz ${OPENSSL_HASH}
tar -xzf ${OPENSSL_ROOT}.tar.gz
pushd ${OPENSSL_ROOT}
./config no-shared -fPIC --prefix=/usr/local/ssl --openssldir=/usr/local/ssl > /dev/null
./config no-shared --prefix=/usr/local/ssl --openssldir=/usr/local/ssl CPPFLAGS="${MANYLINUX_CPPFLAGS}" CFLAGS="${MANYLINUX_CFLAGS} -fPIC" CXXFLAGS="${MANYLINUX_CXXFLAGS} -fPIC" LDFLAGS="${MANYLINUX_LDFLAGS} -fPIC" > /dev/null
make > /dev/null
make install_sw > /dev/null
popd
Expand Down
4 changes: 4 additions & 0 deletions docker/build_scripts/build-swig.sh
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,10 @@ tar -xzf ${SWIG_ROOT}.tar.gz
pushd ${SWIG_ROOT}
fetch_source ${PCRE_ROOT}.tar.gz ${PCRE_DOWNLOAD_URL}
check_sha256sum ${PCRE_ROOT}.tar.gz ${PCRE_HASH}
export CPPFLAGS="${MANYLINUX_CPPFLAGS}"
export CFLAGS="${MANYLINUX_CFLAGS}"
export CXXFLAGS="${MANYLINUX_CXXFLAGS}"
export LDFLAGS="${MANYLINUX_LDFLAGS}"
./Tools/pcre-build.sh
./configure
make -j$(nproc)
Expand Down
11 changes: 10 additions & 1 deletion docker/build_scripts/build_utils.sh
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,15 @@
# Helper utilities for build


# use all flags used by ubuntu 20.04 for hardening builds, dpkg-buildflags --export
# other flags mentioned in https://wiki.ubuntu.com/ToolChain/CompilerFlags can't be
# used because the distros used here are too old
MANYLINUX_CPPFLAGS="-Wdate-time -D_FORTIFY_SOURCE=2"
MANYLINUX_CFLAGS="-g -O2 -Wall -fdebug-prefix-map=/=. -fstack-protector-strong -Wformat -Werror=format-security"
MANYLINUX_CXXFLAGS="-g -O2 -Wall -fdebug-prefix-map=/=. -fstack-protector-strong -Wformat -Werror=format-security"
MANYLINUX_LDFLAGS="-Wl,-Bsymbolic-functions -Wl,-z,relro -Wl,-z,now"


function check_var {
if [ -z "$1" ]; then
echo "required variable not defined"
Expand Down Expand Up @@ -38,7 +47,7 @@ function check_sha256sum {


function do_standard_install {
./configure "$@" > /dev/null
./configure "$@" CPPFLAGS="${MANYLINUX_CPPFLAGS}" CFLAGS="${MANYLINUX_CFLAGS}" "CXXFLAGS=${MANYLINUX_CXXFLAGS}" LDFLAGS="${MANYLINUX_LDFLAGS}" > /dev/null
make -j$(nproc) > /dev/null
make -j$(nproc) install > /dev/null
}
Expand Down
4 changes: 4 additions & 0 deletions docker/build_scripts/finalize.sh
Original file line number Diff line number Diff line change
Expand Up @@ -59,3 +59,7 @@ clean_pyc /opt/_internal
rm -rf /root/.cache

hardlink -cv /opt/_internal

# update system packages
LC_ALL=C ${MY_DIR}/update-system-packages.sh

13 changes: 11 additions & 2 deletions docker/build_scripts/install-entrypoint.sh
Original file line number Diff line number Diff line change
Expand Up @@ -5,10 +5,19 @@
# Stop at any error, show all commands
set -exuo pipefail

# Set build environment variables
MY_DIR=$(dirname "${BASH_SOURCE[0]}")


if [ "${AUDITWHEEL_PLAT}" == "manylinux2010_i686" ] || [ "${AUDITWHEEL_PLAT}" == "manylinux2014_i686" ]; then
echo "i386" > /etc/yum/vars/basearch
fixup-mirrors
yum -y update
fixup-mirrors
yum install -y util-linux-ng
yum clean all
rm -rf /var/cache/yum
# update system packages, we already updated them but
# the following script takes care of cleaning-up some things
# and since it's also needed in the finalize step, everything's
# centralized in this script to avoid code duplication
LC_ALL=C ${MY_DIR}/update-system-packages.sh
fi
58 changes: 7 additions & 51 deletions docker/build_scripts/install-runtime-packages.sh
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,9 @@ if [ "${AUDITWHEEL_POLICY}" == "manylinux2010" ]; then
BASETOOLS="${BASETOOLS} which"
# See https://unix.stackexchange.com/questions/41784/can-yum-express-a-preference-for-x86-64-over-i386-packages
echo "multilib_policy=best" >> /etc/yum.conf
fixup-mirrors
yum -y update
fixup-mirrors
yum -y install https://archives.fedoraproject.org/pub/archive/epel/6/x86_64/epel-release-6-8.noarch.rpm curl
fixup-mirrors
TOOLCHAIN_DEPS="devtoolset-8-binutils devtoolset-8-gcc devtoolset-8-gcc-c++ devtoolset-8-gcc-gfortran yasm"
Expand Down Expand Up @@ -108,63 +110,17 @@ else
exit 1
fi

if ! which localedef &> /dev/null; then
# somebody messed up glibc-common package to squeeze image size, reinstall the package
if [ "${PACKAGE_MANAGER}" == "yum" ]; then
yum -y reinstall glibc-common
else
echo "Not implemented"
exit 1
fi
fi

# upgrading glibc-common can end with removal on en_US.UTF-8 locale
localedef -i en_US -f UTF-8 en_US.UTF-8

if [ "${PACKAGE_MANAGER}" == "yum" ]; then
yum -y install ${BASETOOLS} ${TOOLCHAIN_DEPS} ${MANYLINUX_DEPS} ${RUNTIME_DEPS}
yum clean all
rm -rf /var/cache/yum
elif [ "${PACKAGE_MANAGER}" == "apt" ]; then
apt-get install -qq -y --no-install-recommends ${BASETOOLS} ${TOOLCHAIN_DEPS} ${MANYLINUX_DEPS} ${RUNTIME_DEPS}
apt-get clean -qq
rm -rf /var/lib/apt/lists/*
else
echo "Not implemented"
exit 1
fi


# Fix libc headers to remain compatible with C99 compilers.
find /usr/include/ -type f -exec sed -i 's/\bextern _*inline_*\b/extern __inline __attribute__ ((__gnu_inline__))/g' {} +

if [ "${DEVTOOLSET_ROOTPATH:-}" != "" ]; then
# remove useless things that have been installed by devtoolset
rm -rf $DEVTOOLSET_ROOTPATH/usr/share/man
find $DEVTOOLSET_ROOTPATH/usr/share/locale -mindepth 1 -maxdepth 1 -not \( -name 'en*' -or -name 'locale.alias' \) | xargs rm -rf
fi

if [ -d /usr/share/backgrounds ]; then
rm -rf /usr/share/backgrounds
fi

# if we updated glibc, we need to strip locales again...
if localedef --list-archive | grep -sq -v -i ^en_US.utf8; then
localedef --list-archive | grep -v -i ^en_US.utf8 | xargs localedef --delete-from-archive
fi
if [ "${AUDITWHEEL_POLICY}" == "manylinux2014" ]; then
mv -f /usr/lib/locale/locale-archive /usr/lib/locale/locale-archive.tmpl
build-locale-archive
elif [ "${AUDITWHEEL_POLICY}" == "manylinux_2_24" ]; then
rm /usr/lib/locale/locale-archive
localedef -i en_US -f UTF-8 en_US.UTF-8
update-locale LANG=en_US.UTF-8
fi

find /usr/share/locale -mindepth 1 -maxdepth 1 -not \( -name 'en*' -or -name 'locale.alias' \) | xargs rm -rf
if [ -d /usr/local/share/locale ]; then
find /usr/local/share/locale -mindepth 1 -maxdepth 1 -not \( -name 'en*' -or -name 'locale.alias' \) | xargs rm -rf
fi
if [ -d /usr/local/share/man ]; then
rm -rf /usr/local/share/man
fi
# update system packages, we already updated them but
# the following script takes care of cleaning-up some things
# and since it's also needed in the finalize step, everything's
# centralized in this script to avoid code duplication
LC_ALL=C ${MY_DIR}/update-system-packages.sh
78 changes: 78 additions & 0 deletions docker/build_scripts/update-system-packages.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,78 @@
#!/bin/bash
# Update system packages

# Stop at any error, show all commands
set -exuo pipefail


fixup-mirrors
if [ "${AUDITWHEEL_POLICY}" == "manylinux2010" ] || [ "${AUDITWHEEL_POLICY}" == "manylinux2014" ]; then
yum -y update
if ! localedef -V &> /dev/null; then
# somebody messed up glibc-common package to squeeze image size, reinstall the package
fixup-mirrors
yum -y reinstall glibc-common
fi
yum clean all
rm -rf /var/cache/yum
elif [ "${AUDITWHEEL_POLICY}" == "manylinux_2_24" ]; then
export DEBIAN_FRONTEND=noninteractive
apt-get update -qq
apt-get upgrade -qq -y
apt-get clean -qq
rm -rf /var/lib/apt/lists/*
else
echo "Unsupported policy: '${AUDITWHEEL_POLICY}'"
exit 1
fi
fixup-mirrors

# do we want to update locales ?
LOCALE_ARCHIVE=/usr/lib/locale/locale-archive
TIMESTAMP_FILE=${LOCALE_ARCHIVE}.ml.timestamp
if [ ! -f ${TIMESTAMP_FILE} ] || [ ${LOCALE_ARCHIVE} -nt ${TIMESTAMP_FILE} ]; then
# upgrading glibc-common can end with removal on en_US.UTF-8 locale
localedef -i en_US -f UTF-8 en_US.UTF-8

# if we updated glibc, we need to strip locales again...
if localedef --list-archive | grep -sq -v -i ^en_US.utf8; then
localedef --list-archive | grep -v -i ^en_US.utf8 | xargs localedef --delete-from-archive
fi
if [ "${AUDITWHEEL_POLICY}" == "manylinux2014" ] || [ "${AUDITWHEEL_POLICY}" == "manylinux2010" ]; then
mv -f ${LOCALE_ARCHIVE} ${LOCALE_ARCHIVE}.tmpl
build-locale-archive --install-langs="en_US.utf8"
elif [ "${AUDITWHEEL_POLICY}" == "manylinux_2_24" ]; then
rm ${LOCALE_ARCHIVE}
localedef -i en_US -f UTF-8 en_US.UTF-8
update-locale LANG=en_US.UTF-8
fi
touch ${TIMESTAMP_FILE}
fi

if [ -d /usr/share/locale ]; then
find /usr/share/locale -mindepth 1 -maxdepth 1 -not \( -name 'en*' -or -name 'locale.alias' \) | xargs rm -rf
fi
if [ -d /usr/local/share/locale ]; then
find /usr/local/share/locale -mindepth 1 -maxdepth 1 -not \( -name 'en*' -or -name 'locale.alias' \) | xargs rm -rf
fi

# Fix libc headers to remain compatible with C99 compilers.
find /usr/include/ -type f -exec sed -i 's/\bextern _*inline_*\b/extern __inline __attribute__ ((__gnu_inline__))/g' {} +

if [ "${DEVTOOLSET_ROOTPATH:-}" != "" ]; then
# remove useless things that have been installed/updated by devtoolset
if [ -d $DEVTOOLSET_ROOTPATH/usr/share/man ]; then
rm -rf $DEVTOOLSET_ROOTPATH/usr/share/man
fi
if [ -d $DEVTOOLSET_ROOTPATH/usr/share/locale ]; then
find $DEVTOOLSET_ROOTPATH/usr/share/locale -mindepth 1 -maxdepth 1 -not \( -name 'en*' -or -name 'locale.alias' \) | xargs rm -rf
fi
fi

if [ -d /usr/share/backgrounds ]; then
rm -rf /usr/share/backgrounds
fi

if [ -d /usr/local/share/man ]; then
rm -rf /usr/local/share/man
fi

0 comments on commit 114e2c5

Please sign in to comment.