From 084adc5fa48ef637148a1442b618851738fab907 Mon Sep 17 00:00:00 2001 From: Scott Todd Date: Mon, 30 Sep 2024 15:14:20 -0700 Subject: [PATCH] [libshortfin] Add package build script for Linux using manylinux. (#230) Progress on https://github.com/nod-ai/SHARK-Platform/issues/130 This is forked from * https://github.com/llvm/torch-mlir/blob/main/build_tools/python_deploy/build_linux_packages.sh * https://github.com/iree-org/iree/blob/main/build_tools/python_deploy/build_linux_packages.sh TODO (future PRs?): - [x] Clean up script further - [x] Mention in docs - [x] Integrate into a github workflow - [ ] Update manylinux docker image (2_28, not 2014) - [ ] Test the wheels this builds (including tracing support) --- shortfin/README.md | 22 ++-- shortfin/build_tools/build_linux_package.sh | 122 ++++++++++++++++++++ 2 files changed, 136 insertions(+), 8 deletions(-) create mode 100755 shortfin/build_tools/build_linux_package.sh diff --git a/shortfin/README.md b/shortfin/README.md index 5cabe0304..957658400 100644 --- a/shortfin/README.md +++ b/shortfin/README.md @@ -33,16 +33,22 @@ pip install -v -e . ## Package Python Release Builds -```bash -# Build shortfin.*.whl into the dist/ directory -# e.g. `shortfin-0.9-cp312-cp312-linux_x86_64.whl` -python3 -m pip wheel -v -w dist . +* To build wheels for Linux using a manylinux Docker container: -# Install the built wheel. -python3 -m pip install dist/*.whl -``` + ```bash + sudo ./build_tools/build_linux_package.sh + ``` + +* To build a wheel for your host OS/arch manually: + + ```bash + # Build shortfin.*.whl into the dist/ directory + # e.g. `shortfin-0.9-cp312-cp312-linux_x86_64.whl` + python3 -m pip wheel -v -w dist . -TODO: helper script to build under manylinux using Docker + # Install the built wheel. + python3 -m pip install dist/*.whl + ``` ## Python Dev Builds diff --git a/shortfin/build_tools/build_linux_package.sh b/shortfin/build_tools/build_linux_package.sh new file mode 100755 index 000000000..6f5c67d46 --- /dev/null +++ b/shortfin/build_tools/build_linux_package.sh @@ -0,0 +1,122 @@ +#!/bin/bash +# Copyright 2024 Advanced Micro Devices, Inc. +# Copyright 2022 The IREE Authors +# +# Licensed under the Apache License v2.0 with LLVM Exceptions. +# See https://llvm.org/LICENSE.txt for license information. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +# build_linux_package.sh +# One stop build of shortfin Python package for Linux. The Linux build is +# complicated because it has to be done via a manylinux docker container. +# +# Usage: +# Build everything (all python versions): +# sudo ./build_tools/build_linux_package.sh +# +# Build specific Python versions to custom directory: +# OVERRIDE_PYTHON_VERSIONS="cp312-cp312 cp313-cp313" \ +# OUTPUT_DIR="/tmp/wheelhouse" \ +# sudo -E ./build_tools/build_linux_package.sh +# +# Valid Python versions match a subdirectory under /opt/python in the docker +# image. Typically: +# cp312-cp312 cp313-cp313 +# +# Note that this script is meant to be run on CI and it will pollute both the +# output directory and in-tree build/ directories with docker created, root +# owned builds. Sorry - there is no good way around it. +# +# It can be run on a workstation but recommend using a git worktree dedicated +# to packaging to avoid stomping on development artifacts. +set -xeu -o errtrace + +THIS_DIR="$(cd $(dirname $0) && pwd)" +REPO_ROOT="$(cd "$THIS_DIR"/../../ && pwd)" +SCRIPT_NAME="$(basename $0)" +ARCH="$(uname -m)" + +# TODO(#130): Update to manylinux_2_28, upstream or a fork +# * upstream uses a version of gcc that has build warnings/errors +# * https://github.com/nod-ai/base-docker-images is a bit out of date but can include a recent clang +# MANYLINUX_DOCKER_IMAGE="${MANYLINUX_DOCKER_IMAGE:-quay.io/pypa/manylinux_2_28_${ARCH}:latest}" +MANYLINUX_DOCKER_IMAGE="${MANYLINUX_DOCKER_IMAGE:-quay.io/pypa/manylinux2014_${ARCH}:latest}" +PYTHON_VERSIONS="${OVERRIDE_PYTHON_VERSIONS:-cp312-cp312 cp313-cp313}" +OUTPUT_DIR="${OUTPUT_DIR:-${THIS_DIR}/wheelhouse}" + +function run_on_host() { + echo "Running on host" + echo "Launching docker image ${MANYLINUX_DOCKER_IMAGE}" + + # Canonicalize paths. + mkdir -p "${OUTPUT_DIR}" + OUTPUT_DIR="$(cd "${OUTPUT_DIR}" && pwd)" + echo "Outputting to ${OUTPUT_DIR}" + mkdir -p "${OUTPUT_DIR}" + docker run --rm \ + -v "${REPO_ROOT}:${REPO_ROOT}" \ + -v "${OUTPUT_DIR}:${OUTPUT_DIR}" \ + -e __MANYLINUX_BUILD_WHEELS_IN_DOCKER=1 \ + -e "OVERRIDE_PYTHON_VERSIONS=${PYTHON_VERSIONS}" \ + -e "OUTPUT_DIR=${OUTPUT_DIR}" \ + "${MANYLINUX_DOCKER_IMAGE}" \ + -- ${THIS_DIR}/${SCRIPT_NAME} + + echo "******************** BUILD COMPLETE ********************" + echo "Generated binaries:" + ls -l "${OUTPUT_DIR}" +} + +function run_in_docker() { + echo "Running in docker" + echo "Marking git safe.directory" + git config --global --add safe.directory '*' + + echo "Using python versions: ${PYTHON_VERSIONS}" + local orig_path="${PATH}" + + # Build phase. + echo "******************** BUILDING PACKAGE ********************" + for python_version in ${PYTHON_VERSIONS}; do + python_dir="/opt/python/${python_version}" + if ! [ -x "${python_dir}/bin/python" ]; then + echo "ERROR: Could not find python: ${python_dir} (skipping)" + continue + fi + export PATH="${python_dir}/bin:${orig_path}" + echo ":::: Python version $(python --version)" + clean_wheels "shortfin" "${python_version}" + build_shortfin + run_audit_wheel "shortfin" "${python_version}" + done +} + +function build_shortfin() { + export SHORTFIN_ENABLE_TRACING=ON + python -m pip wheel --disable-pip-version-check -v -w "${OUTPUT_DIR}" "${REPO_ROOT}/shortfin" +} + +function run_audit_wheel() { + local wheel_basename="$1" + local python_version="$2" + # Force wildcard expansion here + generic_wheel="$(echo "${OUTPUT_DIR}/${wheel_basename}-"*"-${python_version}-linux_${ARCH}.whl")" + ls "${generic_wheel}" + echo ":::: Auditwheel ${generic_wheel}" + auditwheel repair -w "${OUTPUT_DIR}" "${generic_wheel}" + rm -v "${generic_wheel}" +} + +function clean_wheels() { + local wheel_basename="$1" + local python_version="$2" + echo ":::: Clean wheels ${wheel_basename} ${python_version}" + rm -f -v "${OUTPUT_DIR}/${wheel_basename}-"*"-${python_version}-"*".whl" +} + +# Trampoline to the docker container if running on the host. +if [ -z "${__MANYLINUX_BUILD_WHEELS_IN_DOCKER-}" ]; then + run_on_host "$@" +else + run_in_docker "$@" +fi