Github action to update PyTorch version

.github/workflows/RollPyTorch.yml

@@ -0,0 +1,65 @@
+name: Roll PyTorch
+
+on:
+  workflow_dispatch:
+
+jobs:
+  build_linux:
+    name: Manylinux Build
+    runs-on: ubuntu-latest
+    steps:
+    - name: Get torch-mlir
+      uses: actions/checkout@v2
+      with:
+        submodules: 'true'
+    - name: Setup ccache
+      uses: ./.github/actions/setup-build
+      with:
+        cache-suffix: x86_64-out-of-tree-OFF
+    - name: Determine nightly PyTorch version
+      run: |
+        cd ${GITHUB_WORKSPACE}
+        python -m pip install wheel
+        # Fetch the most recent nightly PyTorch release
+        PT_RELEASE=$(python -m pip index versions -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html --pre torch | grep "Available versions" | tr ' ' '\n' | grep "^[0-9]" | sort --version-sort --reverse | head -n1 | tr -d ',' | sed 's/\([^+]*\).*/\1/')
+        printf -- "-f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html\n--pre\ntorch==%s\n" "${PT_RELEASE}" > pytorch-requirements.txt
+        # Fetch the whl file associated with the nightly release
+        rm -f torch-"${PT_RELEASE}"*.whl
+        python -m pip download -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html --pre "torch==${PT_RELEASE}"
+        # Read the commit hash from the downloaded whl file without extracting it
+        PT_HASH=$(unzip -p torch-"${PT_RELEASE}"*.whl torch/version.py | grep git_version | awk '{ print $3 }' | tr -d "'")
+        echo "${PT_HASH}" > pytorch-version.txt
+        rm torch-"${PT_RELEASE}"*.whl
+        # Write the release and hash to the environment file so that we can
+        # retrieve them when creating a PR
+        echo "PT_HASH=${PT_HASH}" >> ${GITHUB_ENV}
+        echo "PT_RELEASE=${PT_RELEASE}" >> ${GITHUB_ENV}
+    - name: Build and test
+      run: |
+        cd ${GITHUB_WORKSPACE}
+        TM_PACKAGES="out-of-tree" TM_USE_PYTORCH_BINARY="OFF" \
+        TORCH_MLIR_SRC_PYTORCH_BRANCH="${{ env.PT_HASH }}" \
+        TORCH_MLIR_SRC_PYTORCH_RELEASE="${{ env.PT_RELEASE }}" \
+        ./build_tools/python_deploy/build_linux_packages.sh
+    - name: Push changes to new branch
+      run: |
+        BRANCH="merge/pytorch-update-${{ env.PT_RELEASE }}"
+        TITLE="update PyTorch version to ${{ env.PT_RELEASE }}"
+        echo "BRANCH=${BRANCH}" >> ${GITHUB_ENV}
+        echo "TITLE=${TITLE}" >> ${GITHUB_ENV}
+        cd ${GITHUB_WORKSPACE}
+        git config user.email "torch-mlir@users.noreply.github.com"
+        git config user.name "Roll PyTorch Action"
+        git checkout -b "${BRANCH}"
+        git add pytorch-version.txt pytorch-requirements.txt
+        git commit -m "${TITLE}"
+        git push --set-upstream origin "${BRANCH}"
+    - name: Create PR to push new PyTorch version
+      run: |
+        URL="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
+        BODY="PyTorch commit hash: \`${{ env.PT_HASH }}\` -- CI link: ${URL}"
+        cd ${GITHUB_WORKSPACE}
+        gh pr create -H "${{ env.BRANCH }}" -B main --title "${{ env.TITLE }}" \
+          --body "${BODY}" --reviewer powderluv
+      env:
+        GITHUB_TOKEN: ${{ secrets.WORKFLOW_INVOCATION_TOKEN }}

build_tools/build_libtorch.sh

@@ -45,10 +45,21 @@ install_requirements() {
 
 checkout_pytorch() {
   if [[ ! -d "$PYTORCH_ROOT" ]]; then
-    git clone --depth 1 --single-branch --branch "${TORCH_MLIR_SRC_PYTORCH_BRANCH}" https://github.com/"$TORCH_MLIR_SRC_PYTORCH_REPO" "$PYTORCH_ROOT"
+    # ${TORCH_MLIR_SRC_PYTORCH_BRANCH} could be a branch name or a commit hash.
+    # Althought `git clone` can accept a branch name, the same command does not
+    # accept a commit hash, so we instead use `git fetch`.  The alternative is
+    # to clone the entire repository and then `git checkout` the requested
+    # branch or commit hash, but that's too expensive.
+    mkdir "${PYTORCH_ROOT}"
+    cd "${PYTORCH_ROOT}"
+    git init
+    git remote add origin "https://github.com/${TORCH_MLIR_SRC_PYTORCH_REPO}"
+    git fetch --depth=1 origin "${TORCH_MLIR_SRC_PYTORCH_BRANCH}"
+    git reset --hard FETCH_HEAD
+  else
+    cd "${PYTORCH_ROOT}"
+    git reset --hard HEAD
   fi
-  cd "$PYTORCH_ROOT"
-  git reset --hard HEAD
   git clean -df
   git submodule update --init --depth 1 --recursive
 }

build_tools/python_deploy/build_linux_packages.sh

@@ -55,9 +55,14 @@ TM_USE_PYTORCH_BINARY="${TM_USE_PYTORCH_BINARY:-ON}"
 TM_SKIP_TESTS="${TM_SKIP_TESTS:-OFF}"
 
 PKG_VER_FILE="${repo_root}"/torch_mlir_package_version ; [ -f "$PKG_VER_FILE" ] && . "$PKG_VER_FILE"
-export TORCH_MLIR_PYTHON_PACKAGE_VERSION="${TORCH_MLIR_PYTHON_PACKAGE_VERSION:-0.0.1}"
+TORCH_MLIR_PYTHON_PACKAGE_VERSION="${TORCH_MLIR_PYTHON_PACKAGE_VERSION:-0.0.1}"
 echo "Setting torch-mlir Python Package version to: ${TORCH_MLIR_PYTHON_PACKAGE_VERSION}"
 
+TORCH_MLIR_SRC_PYTORCH_REPO="${TORCH_MLIR_SRC_PYTORCH_REPO:-pytorch/pytorch}"
+echo "Setting torch-mlir PyTorch Repo for source builds to: ${TORCH_MLIR_SRC_PYTORCH_REPO}"
+TORCH_MLIR_SRC_PYTORCH_BRANCH="${TORCH_MLIR_SRC_PYTORCH_BRANCH:-master}"
+echo "Setting torch-mlir PyTorch version for source builds to: ${TORCH_MLIR_SRC_PYTORCH_BRANCH}"
+
 function run_on_host() {
   echo "Running on host for $1:$@"
   echo "Outputting to ${TM_OUTPUT_DIR}"
@@ -105,6 +110,8 @@ function run_on_host() {
     -e "TM_PACKAGES=${package}" \
     -e "TM_SKIP_TESTS=${TM_SKIP_TESTS}" \
     -e "TM_USE_PYTORCH_BINARY=${TM_USE_PYTORCH_BINARY}" \
+    -e "TORCH_MLIR_SRC_PYTORCH_REPO=${TORCH_MLIR_SRC_PYTORCH_REPO}" \
+    -e "TORCH_MLIR_SRC_PYTORCH_BRANCH=${TORCH_MLIR_SRC_PYTORCH_BRANCH}" \
     -e "CCACHE_DIR=/main_checkout/torch-mlir/.ccache" \
     "${TM_CURRENT_DOCKER_IMAGE}" \
     /bin/bash /main_checkout/torch-mlir/build_tools/python_deploy/build_linux_packages.sh
@@ -160,9 +167,9 @@ function run_in_docker() {
 
 
 function build_in_tree() {
-  local torch_from_src="$1"
+  local torch_from_bin="$1"
   local python_version="$2"
-  echo ":::: Build in-tree Torch from source: $torch_from_src with Python: $python_version"
+  echo ":::: Build in-tree Torch from binary: $torch_from_bin with Python: $python_version"
   cmake -GNinja -B/main_checkout/torch-mlir/build \
       -DCMAKE_BUILD_TYPE=Release \
       -DCMAKE_C_COMPILER=clang \
@@ -178,7 +185,9 @@ function build_in_tree() {
       -DLLVM_TARGETS_TO_BUILD=host \
       -DMLIR_ENABLE_BINDINGS_PYTHON=ON \
       -DTORCH_MLIR_ENABLE_LTC=ON \
-      -DTORCH_MLIR_USE_INSTALLED_PYTORCH="$torch_from_src" \
+      -DTORCH_MLIR_USE_INSTALLED_PYTORCH="$torch_from_bin" \
+      -DTORCH_MLIR_SRC_PYTORCH_REPO=${TORCH_MLIR_SRC_PYTORCH_REPO} \
+      -DTORCH_MLIR_SRC_PYTORCH_BRANCH=${TORCH_MLIR_SRC_PYTORCH_BRANCH} \
       -DPython3_EXECUTABLE="$(which python3)" \
       /main_checkout/torch-mlir/externals/llvm-project/llvm
   cmake --build /main_checkout/torch-mlir/build
@@ -249,15 +258,15 @@ function setup_venv() {
   source /main_checkout/torch-mlir/docker_venv/bin/activate
 
   echo ":::: pip installing dependencies"
-  python3 -m pip install -r /main_checkout/torch-mlir/externals/llvm-project/mlir/python/requirements.txt
-  python3 -m pip install -r /main_checkout/torch-mlir/requirements.txt
+  python3 -m pip install --upgrade -r /main_checkout/torch-mlir/externals/llvm-project/mlir/python/requirements.txt
+  python3 -m pip install --upgrade -r /main_checkout/torch-mlir/requirements.txt
 
 }
 
 function build_out_of_tree() {
-  local torch_from_src="$1"
+  local torch_from_bin="$1"
   local python_version="$2"
-  echo ":::: Build out-of-tree Torch from source: $torch_from_src with Python: $python_version"
+  echo ":::: Build out-of-tree Torch from binary: $torch_from_bin with Python: $python_version"
 
   if [ ! -d "/main_checkout/torch-mlir/llvm-build/lib/cmake/mlir/" ]
   then
@@ -289,7 +298,9 @@ function build_out_of_tree() {
       -DMLIR_DIR="/main_checkout/torch-mlir/llvm-build/lib/cmake/mlir/" \
       -DMLIR_ENABLE_BINDINGS_PYTHON=OFF \
       -DTORCH_MLIR_ENABLE_LTC=ON \
-      -DTORCH_MLIR_USE_INSTALLED_PYTORCH="$torch_from_src" \
+      -DTORCH_MLIR_USE_INSTALLED_PYTORCH="$torch_from_bin" \
+      -DTORCH_MLIR_SRC_PYTORCH_REPO=${TORCH_MLIR_SRC_PYTORCH_REPO} \
+      -DTORCH_MLIR_SRC_PYTORCH_BRANCH=${TORCH_MLIR_SRC_PYTORCH_BRANCH} \
       -DPython3_EXECUTABLE="$(which python3)" \
       /main_checkout/torch-mlir
   cmake --build /main_checkout/torch-mlir/build_oot
@@ -310,7 +321,7 @@ function clean_build() {
 }
 
 function build_torch_mlir() {
-  python -m pip install -r /main_checkout/torch-mlir/requirements.txt --extra-index-url https://download.pytorch.org/whl/nightly/cpu
+  python -m pip install --upgrade -r /main_checkout/torch-mlir/requirements.txt --extra-index-url https://download.pytorch.org/whl/nightly/cpu
   CMAKE_GENERATOR=Ninja \
   TORCH_MLIR_PYTHON_PACKAGE_VERSION=${TORCH_MLIR_PYTHON_PACKAGE_VERSION} \
   python -m pip wheel -v -w /wheelhouse /main_checkout/torch-mlir/ \

pytorch-requirements.txt

@@ -0,0 +1,3 @@
+-f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html
+--pre
+torch==1.13.0.dev20220927

pytorch-version.txt

@@ -0,0 +1 @@
+04bb9533d516919190b80a0682e93dd44da9ef6d

requirements.txt

@@ -1,6 +1,4 @@
--f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html
---pre
-torch
+-r pytorch-requirements.txt
 
 numpy
 torchvision