From 72665fc0e3e8d1ed3b4e7eacdfa9c493e2ba0852 Mon Sep 17 00:00:00 2001 From: Scott Todd Date: Mon, 30 Sep 2024 15:59:38 -0700 Subject: [PATCH] [libshortfin] Add workflow to build nightly packages. (#238) Progress on https://github.com/nod-ai/SHARK-Platform/issues/130, follow-up to https://github.com/nod-ai/SHARK-Platform/pull/230. ## Overview This gets us the general structure of a release pipeline that: 1. Computes some version information metadata based on build id and date 2. Calls `shortfin/build_tools/build_linux_package.sh`, which runs `python -m pip wheel` under a manylinux Docker container 3. Uploads wheels to both GitHub artifacts (for the workflow run itself) and GitHub releases (for archival and developer usage) This follows how stablehlo (https://github.com/openxla/stablehlo/releases/tag/dev-wheels) and torch-mlir (https://github.com/llvm/torch-mlir-release/releases/tag/dev-wheels) both publish to a constantly growing "dev-wheels" release rather than separate releases like IREE does. That makes it harder to directly "promote" a candidate release, but it avoids polluting the release history with nightly builds. ## Testing * Sample run: https://github.com/ScottTodd/SHARK-Platform/actions/runs/11078636913 * Dev wheels release: https://github.com/ScottTodd/SHARK-Platform/releases/tag/dev-wheels * Install with `python -m pip install shortfin -f https://github.com/ScottTodd/SHARK-Platform/releases/expanded_assets/dev-wheels` ## What's next In no particular order, - [ ] Verify that the Tracy build enabled by `SHORTFIN_ENABLE_TRACING` is functional - [ ] Set up dependencies and optional components such that a user can just `pip install shortfin` and have it bring along a compatible version of packages like `iree-runtime` and `iree-compiler` - [ ] Update the dockerfile version from manylinux2014 (that is independent of this workflow) - [x] Build for Python 3.13 free threaded and test that - [ ] Build for other platforms/architectures (macOS, Windows, arm64, etc.) - [ ] Set up a workflow like IREE's pkgci that installs the packages and runs some tests on them - [ ] Document how to install the developer packages --------- Co-authored-by: Marius Brehler --- .github/workflows/build_packages.yml | 111 +++++++++++++++++++++++++++ shortfin/.gitignore | 2 + shortfin/setup.py | 47 ++++++++++-- 3 files changed, 154 insertions(+), 6 deletions(-) create mode 100644 .github/workflows/build_packages.yml create mode 100644 shortfin/.gitignore diff --git a/.github/workflows/build_packages.yml b/.github/workflows/build_packages.yml new file mode 100644 index 000000000..44a637f8c --- /dev/null +++ b/.github/workflows/build_packages.yml @@ -0,0 +1,111 @@ +# Copyright 2024 Advanced Micro Devices, Inc. +# +# Licensed under the Apache License v2.0 with LLVM Exceptions. +# See https://llvm.org/LICENSE.txt for license information. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +name: Build packages + +on: + workflow_dispatch: + schedule: + # Runs at 11:00 AM UTC, which is 3:00 AM PST (UTC-8) + - cron: '0 11 * * *' + +jobs: + # Note: metadata generation could happen in a separate trigger/schedule + # workflow. For cross platform builds, it's useful to just generate the + # metadata on Linux and pass that to later jobs using artifacts. + setup_metadata: + runs-on: ubuntu-24.04 + outputs: + shark_package_version: ${{ steps.version.outputs.shark_package_version }} + steps: + # For now the version is just a calendar date + an automatically + # incrementing value. We may want different versions for nightly/dev + # builds and stable releases published to official places like pypi. + - name: Compute version + id: version + run: | + shark_package_version="$(printf '%(%Y%m%d)T.${{ github.run_number }}')" + echo "shark_package_version=${shark_package_version}" >> $GITHUB_OUTPUT + cat << EOF > ./version_info.json + { + "package-version": "${shark_package_version}" + } + EOF + cat ./version_info.json + - name: Upload version_info.json + uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0 + with: + name: version_info + path: version_info.json + + build_packages: + name: "${{ matrix.package }} :: ${{ matrix.platform }} :: ${{ matrix.python-version }}" + runs-on: ${{ matrix.runs-on }} + needs: [setup_metadata] + strategy: + fail-fast: false + matrix: + include: + # Ubuntu packages. + - runs-on: ubuntu-24.04 + platform: linux-x86_64 + package: shortfin + python-version: cp312-cp312 + - runs-on: ubuntu-24.04 + platform: linux-x86_64 + package: shortfin + python-version: cp313-cp313 + - runs-on: ubuntu-24.04 + platform: linux-x86_64 + package: shortfin + python-version: cp313-cp313t + + # TODO(#130): macOS platform + # TODO(#130): Windows platform + # TODO(#130): sharktank packages + + steps: + - name: Checkout repository + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + with: + path: "c" # Windows can hit path length limits, so use a short path. + submodules: false + + - name: Download version_info.json + uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 + with: + name: version_info + path: ./c/shortfin/ + merge-multiple: true + + - name: Build shortfin (Linux x86_64, ${{ matrix.python-version }}) + if: "matrix.package == 'shortfin' && matrix.platform == 'linux-x86_64'" + env: + OUTPUT_DIR: "${{ github.workspace }}/bindist" + OVERRIDE_PYTHON_VERSIONS: "${{ matrix.python-version }}" + run: | + [ -e ./bindist/* ] && rm ./bindist/* + ./c/shortfin/build_tools/build_linux_package.sh + + - name: Upload python wheels + uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0 + with: + if-no-files-found: error + name: snapshot-${{ matrix.package }}-${{ matrix.platform }}-${{ matrix.python-version }} + path: bindist + + - name: Release python wheels + uses: ncipollo/release-action@2c591bcc8ecdcd2db72b97d6147f871fcd833ba5 # v1.14.0 + with: + artifacts: bindist/*.whl + token: "${{ secrets.GITHUB_TOKEN }}" + tag: "dev-wheels" + name: "dev-wheels" + body: "Automatic snapshot release of SHARK-Platform python wheels." + removeArtifacts: false + allowUpdates: true + replacesArtifacts: true + makeLatest: false diff --git a/shortfin/.gitignore b/shortfin/.gitignore new file mode 100644 index 000000000..5469650d5 --- /dev/null +++ b/shortfin/.gitignore @@ -0,0 +1,2 @@ +# Local-only config options +version_info.json diff --git a/shortfin/setup.py b/shortfin/setup.py index deb9b745b..a4f3e31ec 100644 --- a/shortfin/setup.py +++ b/shortfin/setup.py @@ -4,16 +4,17 @@ # See https://llvm.org/LICENSE.txt for license information. # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -from distutils.core import setup, Extension -import sys +import json +import os import shutil import subprocess -import os -from pathlib import Path +import sys from distutils.command.build import build as _build +from distutils.core import setup, Extension +from pathlib import Path from setuptools import find_namespace_packages -from setuptools.command.build_ext import build_ext as _build_ext from setuptools.command.build_py import build_py as _build_py +from setuptools.command.build_ext import build_ext as _build_ext def get_env_boolean(name: str, default_value: bool = False) -> bool: @@ -133,6 +134,40 @@ def copy_extensions_to_source(self, *args, **kwargs): ... +# Setup and get version information. +VERSION_INFO_FILE = os.path.join(SOURCE_DIR, "version_info.json") + + +def load_version_info(): + with open(VERSION_INFO_FILE, "rt") as f: + return json.load(f) + + +def find_git_version(): + try: + return ( + subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=SOURCE_DIR) + .decode("utf-8") + .strip() + ) + except subprocess.SubprocessError as e: + print(f"ERROR: Could not get git revision: {e}", file=sys.stderr) + return None + + +try: + version_info = load_version_info() +except FileNotFoundError: + print("version_info.json not found. Using defaults", file=sys.stderr) + version_info = {} +git_version = find_git_version() + +PACKAGE_VERSION = version_info.get("package-version") +if not PACKAGE_VERSION: + PACKAGE_VERSION = f"0.dev0+{git_version or '0'}" +print(f"Using PACKAGE_VERSION: '{PACKAGE_VERSION}'") + + def maybe_nuke_cmake_cache(cmake_build_dir): # From run to run under pip, we can end up with different paths to ninja, # which isn't great and will confuse cmake. Detect if the location of @@ -324,7 +359,7 @@ def populate_built_package(abs_dir): setup( name="shortfin", - version="0.9", + version=f"{PACKAGE_VERSION}", description="Shortfin native library implementation", author="SHARK Authors", packages=packages,