Build and publish nightly Python packages. (#314)

Fixes #305.

This forks and adapts the similar scripting from
https://github.com/iree-org/iree and https://github.com/nod-ai/shark-ai.

* `build_tools/compute_local_version.py` reads from `version.json` and
writes a `[-stable,-rc,-dev]` version to `version_local.json`
* `build_tools/build_release.py` builds a `.whl` into `wheelhouse/` (I
made the script more robust and removed the version handling, delegating
that to the dedicated version script and local version file)
* `.github/workflows/build_packages.yml` calls `build_release.py` on a
schedule, pushing to a github release
* Sample:
https://github.com/ScottTodd/iree-turbine/releases/tag/dev-wheels
  * These can be installed for now with
    ```
    pip install iree-turbine --pre \
--find-links
https://github.com/iree-org/iree-turbine/releases/expanded_assets/dev-wheels
    ```
Once a common index page is added
(iree-org/iree#19193), that link will be
replaced with https://iree.dev/pip-release-links.html
* `build_tools/pypi_deploy.sh` and
`build_tools/promote_whl_from_rc_to_final.py` can be used to promote
from a nightly release to pypi

.github/workflows/build_packages.yml

@@ -0,0 +1,55 @@
+# Copyright 2024 The IREE Authors
+#
+# Licensed under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+name: Build packages
+
+on:
+  workflow_dispatch:
+  schedule:
+    # Runs at 11:00 AM UTC, which is 3:00 AM PST (UTC-8)
+    - cron: '0 11 * * *'
+
+jobs:
+  build_packages:
+    if: ${{ github.repository_owner == 'iree-org' || github.event_name != 'schedule' }}
+    runs-on: ubuntu-22.04
+    permissions:
+      contents: write
+
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+
+      - name: "Setting up Python"
+        uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0
+        with:
+          python-version: 3.11
+
+      - name: Install dependencies
+        run: pip install -r ./build_tools/requirements-packaging.txt
+
+      - name: Build iree-turbine release candidate
+        run: |
+          ./build_tools/compute_local_version.py -rc --write-json
+          ./build_tools/build_release.py --no-download
+
+      - name: Upload python wheels
+        uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
+        with:
+          if-no-files-found: error
+          name: snapshot
+          path: wheelhouse
+
+      - name: Release python wheels
+        uses: ncipollo/release-action@2c591bcc8ecdcd2db72b97d6147f871fcd833ba5 # v1.14.0
+        with:
+          artifacts: wheelhouse/*.whl
+          tag: "dev-wheels"
+          name: "dev-wheels"
+          body: "Automatic snapshot release of iree-turbine python wheels."
+          removeArtifacts: false
+          allowUpdates: true
+          replacesArtifacts: true
+          makeLatest: false

.github/workflows/test_build_release.yml

@@ -4,7 +4,7 @@
 # See https://llvm.org/LICENSE.txt for license information.
 # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
-name: Build Release
+name: Test Build Release
 
 on:
   workflow_dispatch:
@@ -37,10 +37,15 @@ jobs:
         id: setup_python
         uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0
         with:
-          python-version: ${{matrix.version}}
+          python-version: ${{ matrix.version }}
+
+      - name: Install dependencies
+        run: pip install -r ./build_tools/requirements-packaging.txt
 
       - name: Build Release Wheels
-        run: ./build_tools/build_release.py --package-version 2.5.0.dev
+        run: |
+          ./build_tools/compute_local_version.py -dev --write-json
+          ./build_tools/build_release.py
 
       - name: Validate Release Build
         if: ${{ !cancelled() }}

.gitignore

@@ -25,7 +25,10 @@ wheelhouse
 *.whl
 *.venv
 
-#Model artifacts
+# Local-only config options
+version_local.json
+
+# Model artifacts
 *.pt
 *.safetensors
 *.gguf

build_tools/build_release.py

@@ -1,5 +1,6 @@
 #!/usr/bin/env python
 # Copyright 2024 Advanced Micro Devices, Inc.
+# Copyright 2024 The IREE Authors
 #
 # Licensed under the Apache License v2.0 with LLVM Exceptions.
 # See https://llvm.org/LICENSE.txt for license information.
@@ -10,17 +11,15 @@
 See docs/releasing.md for usage.
 """
 
+from pathlib import Path
 import argparse
-from datetime import date
-import json
 import os
-from pathlib import Path
 import shlex
 import subprocess
+import sys
 
 
 REPO_ROOT = Path(__file__).resolve().parent.parent
-VERSION_INFO_FILE = REPO_ROOT / "version_info.json"
 WHEEL_DIR = REPO_ROOT / "wheelhouse"
 
 # The platform flags that we will download IREE wheels for. This must match
@@ -54,30 +53,6 @@
 ]
 
 
-def eval_version(version_spec: str):
-    date_stamp = date.today().strftime("%Y%m%d")
-    return version_spec.replace("YYYYMMDD", date_stamp)
-
-
-def write_version_info(args):
-    with open(VERSION_INFO_FILE, "rt") as f:
-        info_dict = json.load(f)
-
-    # Compute package-version.
-    package_version = eval_version(args.package_version)
-    if args.package_pre_version:
-        package_version += eval_version(args.package_pre_version)
-    if args.package_post_version:
-        package_version += f".{eval_version(args.package_post_version)}"
-    info_dict["package-version"] = package_version
-
-    with open(VERSION_INFO_FILE, "wt") as f:
-        json.dump(info_dict, f, indent=2)
-        f.write("\n")
-
-    print(f"Updated version_info.json:\n{json.dumps(info_dict, indent=2)}")
-
-
 def exec(args, env=None):
     args = [str(s) for s in args]
     print(f": Exec: {shlex.join(args)}")
@@ -91,6 +66,8 @@ def exec(args, env=None):
 
 def download_requirements(requirements_file, platforms=()):
     args = [
+        sys.executable,
+        "-m",
         "pip",
         "download",
         "-d",
@@ -113,6 +90,8 @@ def download_iree_binaries():
     for platform_args in IREE_PLATFORM_ARGS:
         print("Downloading for platform:", platform_args)
         args = [
+            sys.executable,
+            "-m",
             "pip",
             "download",
             "-d",
@@ -135,31 +114,33 @@ def download_iree_binaries():
         exec(args)
 
 
-def build_wheel(path, env=None):
-    exec(
-        ["pip", "wheel", "--no-index", "-f", WHEEL_DIR, "-w", WHEEL_DIR, path], env=env
-    )
+def build_wheel(args, path):
+    build_args = [
+        sys.executable,
+        "-m",
+        "pip",
+        "wheel",
+        "-f",
+        WHEEL_DIR,
+        "-w",
+        WHEEL_DIR,
+        path,
+    ]
+    if args.no_download:
+        build_args.extend(["--disable-pip-version-check", "--no-deps"])
+    else:
+        build_args.extend(["--no-index"])
+
+    exec(build_args, env=None)
 
 
 def main():
     parser = argparse.ArgumentParser()
-    parser.add_argument("--package-version", help="Version to use", required=True)
-    parser.add_argument(
-        "--package-pre-version",
-        help="Pre-release version segment or (YYYYMMDD)",
-        default="",
-    )
-    parser.add_argument(
-        "--package-post-version",
-        help="Post-release version segment or (YYYYMMDD)",
-        default="",
-    )
     parser.add_argument(
         "--no-download", help="Disable dep download", action="store_true"
     )
     args = parser.parse_args()
 
-    write_version_info(args)
     WHEEL_DIR.mkdir(parents=True, exist_ok=True)
 
     if not args.no_download:
@@ -171,7 +152,7 @@ def main():
         download_requirements(REPO_ROOT / "requirements.txt")
 
     print("Building iree-turbine")
-    build_wheel(REPO_ROOT)
+    build_wheel(args, REPO_ROOT)
 
 
 if __name__ == "__main__":

build_tools/compute_local_version.py

@@ -0,0 +1,64 @@
+#!/usr/bin/env python3
+# Copyright 2024 The IREE Authors
+#
+# Licensed under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+# This scripts grabs the X.Y.Z[.dev]` version identifier from `version.json`
+# and writes the corresponding `X.Y.ZrcYYYYMMDD` version identifier to
+# `version_local.json`.
+
+from datetime import datetime
+from packaging.version import Version
+from pathlib import Path
+import argparse
+import json
+import subprocess
+
+
+parser = argparse.ArgumentParser()
+parser.add_argument("--write-json", action="store_true")
+
+release_type = parser.add_mutually_exclusive_group(required=True)
+release_type.add_argument("-stable", "--stable-release", action="store_true")
+release_type.add_argument("-rc", "--nightly-release", action="store_true")
+release_type.add_argument("-dev", "--development-release", action="store_true")
+release_type.add_argument("--custom-string", action="store", type=str)
+
+args = parser.parse_args()
+
+REPO_ROOT = Path(__file__).parent.parent
+VERSION_FILE_PATH = REPO_ROOT / "version.json"
+VERSION_LOCAL_FILE_PATH = REPO_ROOT / "version_local.json"
+
+
+def load_version_from_file(version_file):
+    with open(version_file, "rt") as f:
+        return json.load(f)
+
+
+def write_version_to_file(version_file, version):
+    with open(version_file, "w") as f:
+        json.dump({"package-version": version}, f, indent=2)
+        f.write("\n")
+
+
+version_info = load_version_from_file(VERSION_FILE_PATH)
+package_version = version_info.get("package-version")
+current_version = Version(package_version).base_version
+
+if args.nightly_release:
+    current_version += "rc" + datetime.today().strftime("%Y%m%d")
+elif args.development_release:
+    current_version += (
+        ".dev+"
+        + subprocess.check_output(["git", "rev-parse", "HEAD"]).decode("ascii").strip()
+    )
+elif args.custom_string:
+    current_version += args.custom_string
+
+if args.write_json:
+    write_version_to_file(VERSION_LOCAL_FILE_PATH, current_version)
+
+print(current_version)

build_tools/promote_whl_from_rc_to_final.py

@@ -0,0 +1,69 @@
+#!/usr/bin/env python3
+# Copyright 2024 The IREE Authors
+#
+# Licensed under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+# This scripts takes a file like
+# 'iree_turbine-3.1.0rc20241204-py3-none-any.whl'
+# with embedded version '3.1.0rc20241204' as input and then drops the
+# 'rcYYYYMMDD' suffix from both the embedded version and file name.
+#
+# Typical usage:
+#   pip install -r requirements-packaging.txt
+#   ./promote_whl_from_rc_to_final.py /path/to/file.whl --delete-old-wheel
+
+import argparse
+from change_wheel_version import change_wheel_version
+from packaging.version import Version
+from pathlib import Path
+from pkginfo import Wheel
+
+
+def parse_arguments():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "input_file",
+        help="Path to the input .whl file to promote",
+        type=Path,
+    )
+    parser.add_argument(
+        "--delete-old-wheel",
+        help="Deletes the original wheel after successfully promoting it",
+        action="store_true",
+        default=False,
+    )
+    return parser.parse_args()
+
+
+def main(args):
+    original_wheel_path = args.input_file
+    print(f"Promoting whl from rc to final: '{original_wheel_path}'")
+
+    original_wheel = Wheel(original_wheel_path)
+    original_version = Version(original_wheel.version)
+    base_version = original_version.base_version
+    print(
+        f"  Original wheel version is '{original_version}' with base '{base_version}'"
+    )
+
+    if str(base_version) == str(original_version):
+        print("  Version is already a release version, skipping")
+        return
+
+    print(f"  Changing to base version: '{base_version}'")
+    new_wheel_path = change_wheel_version(original_wheel_path, str(base_version), None)
+    print(f"  New wheel path is '{new_wheel_path}'")
+
+    new_wheel = Wheel(new_wheel_path)
+    new_version = Version(new_wheel.version)
+    print(f"  New wheel version is '{new_version}'")
+
+    if args.delete_old_wheel:
+        print("  Deleting original wheel")
+        original_wheel_path.unlink()
+
+
+if __name__ == "__main__":
+    main(parse_arguments())