From 48b25c2436739c0a60c3b6edfab6f7c352dbeec0 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Wed, 24 May 2023 12:57:50 +0200 Subject: [PATCH] [REVERTME] disable unrelated CI --- .circleci/.gitignore | 1 - .circleci/build_docs/commit_docs.sh | 35 -- .circleci/config.yml | 271 -------------- .circleci/config.yml.in | 251 ------------- .circleci/regenerate.py | 58 --- .circleci/smoke_test/docker/Dockerfile | 34 -- .../android/scripts/binary_android_build.sh | 27 -- .../android/scripts/binary_android_upload.sh | 34 -- .../android/scripts/install_gradle.sh | 19 - .../unittest/ios/scripts/binary_ios_build.sh | 47 --- .../unittest/ios/scripts/binary_ios_upload.sh | 42 --- .../unittest/linux/scripts/environment.yml | 16 - .circleci/unittest/linux/scripts/install.sh | 45 --- .../unittest/linux/scripts/post_process.sh | 6 - .../linux/scripts/run-clang-format.py | 331 ------------------ .circleci/unittest/linux/scripts/run_test.sh | 22 -- .circleci/unittest/linux/scripts/setup_env.sh | 47 --- .../unittest/windows/scripts/environment.yml | 19 - .circleci/unittest/windows/scripts/install.sh | 53 --- .../windows/scripts/install_conda.bat | 1 - .../unittest/windows/scripts/post_process.sh | 6 - .../unittest/windows/scripts/run_test.sh | 12 - .../unittest/windows/scripts/set_cuda_envs.sh | 48 --- .../unittest/windows/scripts/setup_env.sh | 45 --- .../windows/scripts/vc_env_helper.bat | 39 --- .github/scripts/unittest.sh | 3 +- .github/workflows/build-cmake.yml | 92 ----- .github/workflows/build-conda-linux.yml | 51 --- .github/workflows/build-conda-m1.yml | 52 --- .github/workflows/build-conda-macos.yml | 52 --- .github/workflows/build-conda-windows.yml | 51 --- .github/workflows/build-wheels-linux.yml | 50 --- .github/workflows/build-wheels-m1.yml | 51 --- .github/workflows/build-wheels-macos.yml | 51 --- .github/workflows/build-wheels-windows.yml | 52 --- .github/workflows/docs.yml | 115 ------ .github/workflows/lint.yml | 107 ------ .github/workflows/pr-labels.yml | 35 -- .../workflows/prototype-tests-linux-gpu.yml | 54 --- .github/workflows/tests-schedule.yml | 57 --- .github/workflows/update-viablestrict.yml | 23 -- 41 files changed, 2 insertions(+), 2403 deletions(-) delete mode 100644 .circleci/.gitignore delete mode 100755 .circleci/build_docs/commit_docs.sh delete mode 100644 .circleci/config.yml delete mode 100644 .circleci/config.yml.in delete mode 100755 .circleci/regenerate.py delete mode 100644 .circleci/smoke_test/docker/Dockerfile delete mode 100644 .circleci/unittest/android/scripts/binary_android_build.sh delete mode 100644 .circleci/unittest/android/scripts/binary_android_upload.sh delete mode 100755 .circleci/unittest/android/scripts/install_gradle.sh delete mode 100755 .circleci/unittest/ios/scripts/binary_ios_build.sh delete mode 100644 .circleci/unittest/ios/scripts/binary_ios_upload.sh delete mode 100644 .circleci/unittest/linux/scripts/environment.yml delete mode 100755 .circleci/unittest/linux/scripts/install.sh delete mode 100755 .circleci/unittest/linux/scripts/post_process.sh delete mode 100755 .circleci/unittest/linux/scripts/run-clang-format.py delete mode 100755 .circleci/unittest/linux/scripts/run_test.sh delete mode 100755 .circleci/unittest/linux/scripts/setup_env.sh delete mode 100644 .circleci/unittest/windows/scripts/environment.yml delete mode 100644 .circleci/unittest/windows/scripts/install.sh delete mode 100644 .circleci/unittest/windows/scripts/install_conda.bat delete mode 100644 .circleci/unittest/windows/scripts/post_process.sh delete mode 100644 .circleci/unittest/windows/scripts/run_test.sh delete mode 100644 .circleci/unittest/windows/scripts/set_cuda_envs.sh delete mode 100644 .circleci/unittest/windows/scripts/setup_env.sh delete mode 100644 .circleci/unittest/windows/scripts/vc_env_helper.bat delete mode 100644 .github/workflows/build-cmake.yml delete mode 100644 .github/workflows/build-conda-linux.yml delete mode 100644 .github/workflows/build-conda-m1.yml delete mode 100644 .github/workflows/build-conda-macos.yml delete mode 100644 .github/workflows/build-conda-windows.yml delete mode 100644 .github/workflows/build-wheels-linux.yml delete mode 100644 .github/workflows/build-wheels-m1.yml delete mode 100644 .github/workflows/build-wheels-macos.yml delete mode 100644 .github/workflows/build-wheels-windows.yml delete mode 100644 .github/workflows/docs.yml delete mode 100644 .github/workflows/lint.yml delete mode 100644 .github/workflows/pr-labels.yml delete mode 100644 .github/workflows/prototype-tests-linux-gpu.yml delete mode 100644 .github/workflows/tests-schedule.yml delete mode 100644 .github/workflows/update-viablestrict.yml diff --git a/.circleci/.gitignore b/.circleci/.gitignore deleted file mode 100644 index 485dee64bcf..00000000000 --- a/.circleci/.gitignore +++ /dev/null @@ -1 +0,0 @@ -.idea diff --git a/.circleci/build_docs/commit_docs.sh b/.circleci/build_docs/commit_docs.sh deleted file mode 100755 index 04e3538fefc..00000000000 --- a/.circleci/build_docs/commit_docs.sh +++ /dev/null @@ -1,35 +0,0 @@ -#!/usr/bin/env bash - -set -ex - - -if [ "$2" == "" ]; then - echo call as "$0" "" "" - echo where src is the root of the built documentation git checkout and - echo branch should be "main" or "1.7" or so - exit 1 -fi - -src=$1 -target=$2 - -echo "committing docs from ${src} to ${target}" - -pushd "${src}" -git checkout gh-pages -mkdir -p ./"${target}" -rm -rf ./"${target}"/* -cp -r "${src}/docs/build/html/"* ./"$target" -if [ "${target}" == "main" ]; then - mkdir -p ./_static - rm -rf ./_static/* - cp -r "${src}/docs/build/html/_static/"* ./_static - git add --all ./_static || true -fi -git add --all ./"${target}" || true -git config user.email "soumith+bot@pytorch.org" -git config user.name "pytorchbot" -# If there aren't changes, don't make a commit; push is no-op -git commit -m "auto-generating sphinx docs" || true -git remote add https https://github.com/pytorch/vision.git -git push -u https gh-pages diff --git a/.circleci/config.yml b/.circleci/config.yml deleted file mode 100644 index 0463007af46..00000000000 --- a/.circleci/config.yml +++ /dev/null @@ -1,271 +0,0 @@ -version: 2.1 - -# How to test the Linux jobs: -# - Install CircleCI local CLI: https://circleci.com/docs/2.0/local-cli/ -# - circleci config process .circleci/config.yml > gen.yml && circleci local execute -c gen.yml --job binary_win_wheel_py3.8 -# - Replace binary_win_wheel_py3.8 with the name of the job you want to test. -# Job names are 'name:' key. - -executors: - windows-cpu: - machine: - resource_class: windows.xlarge - image: windows-server-2019-vs2019:stable - shell: bash.exe - - windows-gpu: - machine: - resource_class: windows.gpu.nvidia.medium - image: windows-server-2019-nvidia:stable - shell: bash.exe - -commands: - checkout_merge: - description: "checkout merge branch" - steps: - - checkout -# - run: -# name: Checkout merge branch -# command: | -# set -ex -# BRANCH=$(git rev-parse --abbrev-ref HEAD) -# if [[ "$BRANCH" != "main" ]]; then -# git fetch --force origin ${CIRCLE_BRANCH}/merge:merged/${CIRCLE_BRANCH} -# git checkout "merged/$CIRCLE_BRANCH" -# fi - designate_upload_channel: - description: "inserts the correct upload channel into ${BASH_ENV}" - steps: - - run: - name: adding UPLOAD_CHANNEL to BASH_ENV - command: | - our_upload_channel=nightly - # On tags upload to test instead - if [[ -n "${CIRCLE_TAG}" ]]; then - our_upload_channel=test - fi - echo "export UPLOAD_CHANNEL=${our_upload_channel}" >> ${BASH_ENV} - - pip_install: - parameters: - args: - type: string - descr: - type: string - default: "" - user: - type: boolean - default: true - steps: - - run: - name: > - <<^ parameters.descr >> pip install << parameters.args >> <> - <<# parameters.descr >> << parameters.descr >> <> - command: > - pip install - <<# parameters.user >> --user <> - --progress-bar=off - << parameters.args >> - -binary_common: &binary_common - parameters: - # Edit these defaults to do a release - build_version: - description: "version number of release binary; by default, build a nightly" - type: string - default: "" - pytorch_version: - description: "PyTorch version to build against; by default, use a nightly" - type: string - default: "" - # Don't edit these - python_version: - description: "Python version to build against (e.g., 3.8)" - type: string - cu_version: - description: "CUDA version to build against, in CU format (e.g., cpu or cu100)" - type: string - default: "cpu" - unicode_abi: - description: "Python 2.7 wheel only: whether or not we are cp27mu (default: no)" - type: string - default: "" - wheel_docker_image: - description: "Wheel only: what docker image to use" - type: string - default: "" - conda_docker_image: - description: "Conda only: what docker image to use" - type: string - default: "pytorch/conda-builder:cpu" - environment: - PYTHON_VERSION: << parameters.python_version >> - PYTORCH_VERSION: << parameters.pytorch_version >> - UNICODE_ABI: << parameters.unicode_abi >> - CU_VERSION: << parameters.cu_version >> - MACOSX_DEPLOYMENT_TARGET: 10.9 - -smoke_test_common: &smoke_test_common - <<: *binary_common - docker: - - image: torchvision/smoke_test:latest - -jobs: - circleci_consistency: - docker: - - image: cimg/python:3.8 - steps: - - checkout - - pip_install: - args: jinja2 pyyaml - - run: - name: Check CircleCI config consistency - command: | - python .circleci/regenerate.py - git diff --exit-code || (echo ".circleci/config.yml not in sync with config.yml.in! Run .circleci/regenerate.py to update config"; exit 1) - - smoke_test_docker_image_build: - machine: - image: ubuntu-2004:202104-01 - resource_class: large - environment: - image_name: torchvision/smoke_test - steps: - - checkout - - designate_upload_channel - - run: - name: Build and push Docker image - no_output_timeout: "1h" - command: | - set +x - echo "${DOCKER_HUB_TOKEN}" | docker login --username "${DOCKER_HUB_USERNAME}" --password-stdin - set -x - cd .circleci/smoke_test/docker && docker build . -t ${image_name}:${CIRCLE_WORKFLOW_ID} - docker tag ${image_name}:${CIRCLE_WORKFLOW_ID} ${image_name}:latest - docker push ${image_name}:${CIRCLE_WORKFLOW_ID} - docker push ${image_name}:latest - - cmake_linux_cpu: - <<: *binary_common - docker: - - image: "pytorch/manylinux-cpu" - resource_class: 2xlarge+ - steps: - - checkout_merge - - designate_upload_channel - - run: - name: Setup conda - command: .circleci/unittest/linux/scripts/setup_env.sh - - run: packaging/build_cmake.sh - - cmake_linux_gpu: - <<: *binary_common - machine: - image: ubuntu-2004-cuda-11.4:202110-01 - resource_class: gpu.nvidia.small - steps: - - checkout_merge - - designate_upload_channel - - run: - name: Setup conda - command: docker run -e CU_VERSION -e PYTHON_VERSION -e UNICODE_ABI -e PYTORCH_VERSION -t --gpus all -v $PWD:$PWD -w $PWD << parameters.wheel_docker_image >> .circleci/unittest/linux/scripts/setup_env.sh - - run: - name: Build torchvision C++ distribution and test - no_output_timeout: 30m - command: docker run -e CU_VERSION -e PYTHON_VERSION -e UNICODE_ABI -e PYTORCH_VERSION -e UPLOAD_CHANNEL -t --gpus all -v $PWD:$PWD -w $PWD << parameters.wheel_docker_image >> packaging/build_cmake.sh - - cmake_macos_cpu: - <<: *binary_common - macos: - xcode: "14.0" - steps: - - checkout_merge - - designate_upload_channel - - run: - command: | - curl -o conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh - sh conda.sh -b - source $HOME/miniconda3/bin/activate - conda install -yq conda-build cmake python=<< parameters.python_version >> - packaging/build_cmake.sh - - cmake_windows_cpu: - <<: *binary_common - executor: - name: windows-cpu - steps: - - checkout_merge - - designate_upload_channel - - run: - command: | - set -ex - source packaging/windows/internal/vc_install_helper.sh - eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')" - conda activate base - conda create -yn python39 python=3.9 - conda activate python39 - packaging/build_cmake.sh - - cmake_windows_gpu: - <<: *binary_common - executor: - name: windows-gpu - steps: - - checkout_merge - - designate_upload_channel - - run: - name: Update CUDA driver - command: packaging/windows/internal/driver_update.bat - - run: - command: | - set -ex - source packaging/windows/internal/vc_install_helper.sh - packaging/windows/internal/cuda_install.bat - eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')" - conda activate - conda update -y conda - conda create -yn python39 python=3.9 - conda activate python39 - packaging/build_cmake.sh - - -workflows: - lint: - jobs: - - circleci_consistency - - cmake: - jobs: - - cmake_linux_cpu: - cu_version: cpu - name: cmake_linux_cpu - python_version: '3.8' - - cmake_linux_gpu: - cu_version: cu117 - name: cmake_linux_gpu - python_version: '3.8' - wheel_docker_image: pytorch/manylinux-cuda117 - - cmake_windows_cpu: - cu_version: cpu - name: cmake_windows_cpu - python_version: '3.8' - - cmake_windows_gpu: - cu_version: cu117 - name: cmake_windows_gpu - python_version: '3.8' - - cmake_macos_cpu: - cu_version: cpu - name: cmake_macos_cpu - python_version: '3.8' - - docker_build: - triggers: - - schedule: - cron: "0 10 * * 0" - filters: - branches: - only: - - main - jobs: - - smoke_test_docker_image_build: - context: org-member diff --git a/.circleci/config.yml.in b/.circleci/config.yml.in deleted file mode 100644 index f3a88f2d987..00000000000 --- a/.circleci/config.yml.in +++ /dev/null @@ -1,251 +0,0 @@ -version: 2.1 - -# How to test the Linux jobs: -# - Install CircleCI local CLI: https://circleci.com/docs/2.0/local-cli/ -# - circleci config process .circleci/config.yml > gen.yml && circleci local execute -c gen.yml --job binary_win_wheel_py3.8 -# - Replace binary_win_wheel_py3.8 with the name of the job you want to test. -# Job names are 'name:' key. - -executors: - windows-cpu: - machine: - resource_class: windows.xlarge - image: windows-server-2019-vs2019:stable - shell: bash.exe - - windows-gpu: - machine: - resource_class: windows.gpu.nvidia.medium - image: windows-server-2019-nvidia:stable - shell: bash.exe - -commands: - checkout_merge: - description: "checkout merge branch" - steps: - - checkout -# - run: -# name: Checkout merge branch -# command: | -# set -ex -# BRANCH=$(git rev-parse --abbrev-ref HEAD) -# if [[ "$BRANCH" != "main" ]]; then -# git fetch --force origin ${CIRCLE_BRANCH}/merge:merged/${CIRCLE_BRANCH} -# git checkout "merged/$CIRCLE_BRANCH" -# fi - designate_upload_channel: - description: "inserts the correct upload channel into ${BASH_ENV}" - steps: - - run: - name: adding UPLOAD_CHANNEL to BASH_ENV - command: | - our_upload_channel=nightly - # On tags upload to test instead - if [[ -n "${CIRCLE_TAG}" ]]; then - our_upload_channel=test - fi - echo "export UPLOAD_CHANNEL=${our_upload_channel}" >> ${BASH_ENV} - - pip_install: - parameters: - args: - type: string - descr: - type: string - default: "" - user: - type: boolean - default: true - steps: - - run: - name: > - <<^ parameters.descr >> pip install << parameters.args >> <> - <<# parameters.descr >> << parameters.descr >> <> - command: > - pip install - <<# parameters.user >> --user <> - --progress-bar=off - << parameters.args >> - -binary_common: &binary_common - parameters: - # Edit these defaults to do a release - build_version: - description: "version number of release binary; by default, build a nightly" - type: string - default: "" - pytorch_version: - description: "PyTorch version to build against; by default, use a nightly" - type: string - default: "" - # Don't edit these - python_version: - description: "Python version to build against (e.g., 3.8)" - type: string - cu_version: - description: "CUDA version to build against, in CU format (e.g., cpu or cu100)" - type: string - default: "cpu" - unicode_abi: - description: "Python 2.7 wheel only: whether or not we are cp27mu (default: no)" - type: string - default: "" - wheel_docker_image: - description: "Wheel only: what docker image to use" - type: string - default: "" - conda_docker_image: - description: "Conda only: what docker image to use" - type: string - default: "pytorch/conda-builder:cpu" - environment: - PYTHON_VERSION: << parameters.python_version >> - PYTORCH_VERSION: << parameters.pytorch_version >> - UNICODE_ABI: << parameters.unicode_abi >> - CU_VERSION: << parameters.cu_version >> - MACOSX_DEPLOYMENT_TARGET: 10.9 - -smoke_test_common: &smoke_test_common - <<: *binary_common - docker: - - image: torchvision/smoke_test:latest - -jobs: - circleci_consistency: - docker: - - image: cimg/python:3.8 - steps: - - checkout - - pip_install: - args: jinja2 pyyaml - - run: - name: Check CircleCI config consistency - command: | - python .circleci/regenerate.py - git diff --exit-code || (echo ".circleci/config.yml not in sync with config.yml.in! Run .circleci/regenerate.py to update config"; exit 1) - - smoke_test_docker_image_build: - machine: - image: ubuntu-2004:202104-01 - resource_class: large - environment: - image_name: torchvision/smoke_test - steps: - - checkout - - designate_upload_channel - - run: - name: Build and push Docker image - no_output_timeout: "1h" - command: | - set +x - echo "${DOCKER_HUB_TOKEN}" | docker login --username "${DOCKER_HUB_USERNAME}" --password-stdin - set -x - cd .circleci/smoke_test/docker && docker build . -t ${image_name}:${CIRCLE_WORKFLOW_ID} - docker tag ${image_name}:${CIRCLE_WORKFLOW_ID} ${image_name}:latest - docker push ${image_name}:${CIRCLE_WORKFLOW_ID} - docker push ${image_name}:latest - - cmake_linux_cpu: - <<: *binary_common - docker: - - image: "pytorch/manylinux-cpu" - resource_class: 2xlarge+ - steps: - - checkout_merge - - designate_upload_channel - - run: - name: Setup conda - command: .circleci/unittest/linux/scripts/setup_env.sh - - run: packaging/build_cmake.sh - - cmake_linux_gpu: - <<: *binary_common - machine: - image: ubuntu-2004-cuda-11.4:202110-01 - resource_class: gpu.nvidia.small - steps: - - checkout_merge - - designate_upload_channel - - run: - name: Setup conda - command: docker run -e CU_VERSION -e PYTHON_VERSION -e UNICODE_ABI -e PYTORCH_VERSION -t --gpus all -v $PWD:$PWD -w $PWD << parameters.wheel_docker_image >> .circleci/unittest/linux/scripts/setup_env.sh - - run: - name: Build torchvision C++ distribution and test - no_output_timeout: 30m - command: docker run -e CU_VERSION -e PYTHON_VERSION -e UNICODE_ABI -e PYTORCH_VERSION -e UPLOAD_CHANNEL -t --gpus all -v $PWD:$PWD -w $PWD << parameters.wheel_docker_image >> packaging/build_cmake.sh - - cmake_macos_cpu: - <<: *binary_common - macos: - xcode: "14.0" - steps: - - checkout_merge - - designate_upload_channel - - run: - command: | - curl -o conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh - sh conda.sh -b - source $HOME/miniconda3/bin/activate - conda install -yq conda-build cmake python=<< parameters.python_version >> - packaging/build_cmake.sh - - cmake_windows_cpu: - <<: *binary_common - executor: - name: windows-cpu - steps: - - checkout_merge - - designate_upload_channel - - run: - command: | - set -ex - source packaging/windows/internal/vc_install_helper.sh - eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')" - conda activate base - conda create -yn python39 python=3.9 - conda activate python39 - packaging/build_cmake.sh - - cmake_windows_gpu: - <<: *binary_common - executor: - name: windows-gpu - steps: - - checkout_merge - - designate_upload_channel - - run: - name: Update CUDA driver - command: packaging/windows/internal/driver_update.bat - - run: - command: | - set -ex - source packaging/windows/internal/vc_install_helper.sh - packaging/windows/internal/cuda_install.bat - eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')" - conda activate - conda update -y conda - conda create -yn python39 python=3.9 - conda activate python39 - packaging/build_cmake.sh - - -workflows: - lint: - jobs: - - circleci_consistency - - cmake: - jobs: - {{ cmake_workflows() }} - - docker_build: - triggers: - - schedule: - cron: "0 10 * * 0" - filters: - branches: - only: - - main - jobs: - - smoke_test_docker_image_build: - context: org-member diff --git a/.circleci/regenerate.py b/.circleci/regenerate.py deleted file mode 100755 index e7b8db3d56e..00000000000 --- a/.circleci/regenerate.py +++ /dev/null @@ -1,58 +0,0 @@ -#!/usr/bin/env python3 - -""" -This script should use a very simple, functional programming style. -Avoid Jinja macros in favor of native Python functions. - -Don't go overboard on code generation; use Python only to generate -content that can't be easily declared statically using CircleCI's YAML API. - -Data declarations (e.g. the nested loops for defining the configuration matrix) -should be at the top of the file for easy updating. - -See this comment for design rationale: -https://github.com/pytorch/vision/pull/1321#issuecomment-531033978 -""" - -import os.path - -import jinja2 -import yaml -from jinja2 import select_autoescape - - -def indent(indentation, data_list): - return ("\n" + " " * indentation).join(yaml.dump(data_list, default_flow_style=False).splitlines()) - - -def cmake_workflows(indentation=6): - jobs = [] - python_version = "3.8" - for os_type in ["linux", "windows", "macos"]: - # Skip OSX CUDA - device_types = ["cpu", "gpu"] if os_type != "macos" else ["cpu"] - for device in device_types: - job = {"name": f"cmake_{os_type}_{device}", "python_version": python_version} - - job["cu_version"] = "cu117" if device == "gpu" else "cpu" - if device == "gpu" and os_type == "linux": - job["wheel_docker_image"] = "pytorch/manylinux-cuda117" - jobs.append({f"cmake_{os_type}_{device}": job}) - return indent(indentation, jobs) - - -if __name__ == "__main__": - d = os.path.dirname(__file__) - env = jinja2.Environment( - loader=jinja2.FileSystemLoader(d), - lstrip_blocks=True, - autoescape=select_autoescape(enabled_extensions=("html", "xml")), - keep_trailing_newline=True, - ) - - with open(os.path.join(d, "config.yml"), "w") as f: - f.write( - env.get_template("config.yml.in").render( - cmake_workflows=cmake_workflows, - ) - ) diff --git a/.circleci/smoke_test/docker/Dockerfile b/.circleci/smoke_test/docker/Dockerfile deleted file mode 100644 index 34bdcda1053..00000000000 --- a/.circleci/smoke_test/docker/Dockerfile +++ /dev/null @@ -1,34 +0,0 @@ -# this Dockerfile is for torchvision smoke test, it will be created periodically via CI system -# if you need to do it locally, follow below steps once you have Docker installed -# assuming you're within the directory where this Dockerfile located -# $ docker build . -t torchvision/smoketest - -# if you want to push to aws ecr, make sure you have the rights to write to ECR, then run -# $ eval $(aws ecr get-login --region us-east-1 --no-include-email) -# $ export MYTAG=localbuild ## you can choose whatever tag you like -# $ docker tag torchvision/smoketest 308535385114.dkr.ecr.us-east-1.amazonaws.com/torchvision/smoke_test:${MYTAG} -# $ docker push 308535385114.dkr.ecr.us-east-1.amazonaws.com/torchvision/smoke_test:${MYTAG} - -FROM ubuntu:latest - -RUN apt-get -qq update && apt-get -qq -y install curl bzip2 libsox-fmt-all \ - && curl -sSL https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -o /tmp/miniconda.sh \ - && bash /tmp/miniconda.sh -bfp /usr/local \ - && rm -rf /tmp/miniconda.sh \ - && conda install -y python=3 \ - && conda update conda \ - && apt-get -qq -y remove curl bzip2 \ - && apt-get -qq -y autoremove \ - && apt-get autoclean \ - && rm -rf /var/lib/apt/lists/* /var/log/dpkg.log \ - && conda clean --all --yes - -ENV PATH /opt/conda/bin:$PATH - -RUN conda create -y --name python3.7 python=3.7 -RUN conda create -y --name python3.8 python=3.8 -RUN conda create -y --name python3.9 python=3.9 -RUN conda create -y --name python3.10 python=3.10 -SHELL [ "/bin/bash", "-c" ] -RUN echo "source /usr/local/etc/profile.d/conda.sh" >> ~/.bashrc -CMD [ "/bin/bash"] diff --git a/.circleci/unittest/android/scripts/binary_android_build.sh b/.circleci/unittest/android/scripts/binary_android_build.sh deleted file mode 100644 index 0d8c0d47d8a..00000000000 --- a/.circleci/unittest/android/scripts/binary_android_build.sh +++ /dev/null @@ -1,27 +0,0 @@ -#!/bin/bash -set -ex -o pipefail - -echo "DIR: $(pwd)" -echo "ANDROID_HOME=${ANDROID_HOME}" -echo "ANDROID_NDK_HOME=${ANDROID_NDK_HOME}" -echo "JAVA_HOME=${JAVA_HOME}" - -WORKSPACE=/home/circleci/workspace -VISION_ANDROID=/home/circleci/project/android - -. /home/circleci/project/.circleci/unittest/android/scripts/install_gradle.sh - -GRADLE_LOCAL_PROPERTIES=${VISION_ANDROID}/local.properties -rm -f $GRADLE_LOCAL_PROPERTIES - -echo "sdk.dir=${ANDROID_HOME}" >> $GRADLE_LOCAL_PROPERTIES -echo "ndk.dir=${ANDROID_NDK_HOME}" >> $GRADLE_LOCAL_PROPERTIES - -echo "GRADLE_PATH $GRADLE_PATH" -echo "GRADLE_HOME $GRADLE_HOME" - -${GRADLE_PATH} --scan --stacktrace --debug --no-daemon -p ${VISION_ANDROID} assemble || true - -mkdir -p ~/workspace/artifacts -find . -type f -name *aar -print | xargs tar cfvz ~/workspace/artifacts/artifacts-aars.tgz -find . -type f -name *apk -print | xargs tar cfvz ~/workspace/artifacts/artifacts-apks.tgz diff --git a/.circleci/unittest/android/scripts/binary_android_upload.sh b/.circleci/unittest/android/scripts/binary_android_upload.sh deleted file mode 100644 index 1472a877d90..00000000000 --- a/.circleci/unittest/android/scripts/binary_android_upload.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/bin/bash -set -ex -o pipefail - -echo "DIR: $(pwd)" -echo "ANDROID_HOME=${ANDROID_HOME}" -echo "ANDROID_NDK_HOME=${ANDROID_NDK_HOME}" -echo "JAVA_HOME=${JAVA_HOME}" - -WORKSPACE=/home/circleci/workspace -VISION_ANDROID=/home/circleci/project/android - -. /home/circleci/project/.circleci/unittest/android/scripts/install_gradle.sh - -GRADLE_LOCAL_PROPERTIES=${VISION_ANDROID}/local.properties -rm -f $GRADLE_LOCAL_PROPERTIES -GRADLE_PROPERTIES=/home/circleci/project/android/gradle.properties - -echo "sdk.dir=${ANDROID_HOME}" >> $GRADLE_LOCAL_PROPERTIES -echo "ndk.dir=${ANDROID_NDK_HOME}" >> $GRADLE_LOCAL_PROPERTIES - -echo "SONATYPE_NEXUS_USERNAME=${SONATYPE_NEXUS_USERNAME}" >> $GRADLE_PROPERTIES -echo "mavenCentralRepositoryUsername=${SONATYPE_NEXUS_USERNAME}" >> $GRADLE_PROPERTIES -echo "SONATYPE_NEXUS_PASSWORD=${SONATYPE_NEXUS_PASSWORD}" >> $GRADLE_PROPERTIES -echo "mavenCentralRepositoryPassword=${SONATYPE_NEXUS_PASSWORD}" >> $GRADLE_PROPERTIES - -echo "signing.keyId=${ANDROID_SIGN_KEY}" >> $GRADLE_PROPERTIES -echo "signing.password=${ANDROID_SIGN_PASS}" >> $GRADLE_PROPERTIES - -cat /home/circleci/project/android/gradle.properties | grep VERSION - -${GRADLE_PATH} --scan --stacktrace --debug --no-daemon -p ${VISION_ANDROID} ops:uploadArchives - -mkdir -p ~/workspace/artifacts -find . -type f -name *aar -print | xargs tar cfvz ~/workspace/artifacts/artifacts-aars.tgz diff --git a/.circleci/unittest/android/scripts/install_gradle.sh b/.circleci/unittest/android/scripts/install_gradle.sh deleted file mode 100755 index ff90c657e65..00000000000 --- a/.circleci/unittest/android/scripts/install_gradle.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/bash -set -ex - -_https_amazon_aws=https://downloads.gradle-dn.com/distributions -GRADLE_VERSION=6.8.3 - -_gradle_home=/opt/gradle -sudo rm -rf $gradle_home -sudo mkdir -p $_gradle_home - -curl --silent --output /tmp/gradle.zip --retry 3 $_https_amazon_aws/gradle-${GRADLE_VERSION}-bin.zip - -sudo unzip -q /tmp/gradle.zip -d $_gradle_home -rm /tmp/gradle.zip - -sudo chmod -R 777 $_gradle_home - -export GRADLE_HOME=$_gradle_home/gradle-$GRADLE_VERSION -export GRADLE_PATH=${GRADLE_HOME}/bin/gradle diff --git a/.circleci/unittest/ios/scripts/binary_ios_build.sh b/.circleci/unittest/ios/scripts/binary_ios_build.sh deleted file mode 100755 index 1f117481f2d..00000000000 --- a/.circleci/unittest/ios/scripts/binary_ios_build.sh +++ /dev/null @@ -1,47 +0,0 @@ -#!/bin/bash -set -ex -o pipefail - -echo "" -echo "DIR: $(pwd)" -WORKSPACE=/Users/distiller/workspace -PROJ_ROOT_IOS=/Users/distiller/project/ios -PYTORCH_IOS_NIGHTLY_NAME=libtorch_ios_nightly_build.zip -export TCLLIBPATH="/usr/local/lib" - -# install conda -curl --retry 3 -o ~/conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh -chmod +x ~/conda.sh -/bin/bash ~/conda.sh -b -p ~/anaconda -export PATH="~/anaconda/bin:${PATH}" -source ~/anaconda/bin/activate - -# install dependencies -conda install numpy ninja pyyaml mkl mkl-include setuptools cmake cffi requests wget --yes -conda install -c conda-forge valgrind --yes -export CMAKE_PREFIX_PATH=${CONDA_PREFIX:-"$(dirname $(which conda))/../"} - -# sync submodules -cd ${PROJ_ROOT_IOS} -git submodule sync -git submodule update --init --recursive - -# download pytorch-iOS nightly build and unzip it -mkdir -p ${PROJ_ROOT_IOS}/lib -mkdir -p ${PROJ_ROOT_IOS}/build -mkdir -p ${PROJ_ROOT_IOS}/pytorch -TORCH_ROOT="${PROJ_ROOT_IOS}/pytorch" - -cd ${TORCH_ROOT} -wget https://ossci-ios-build.s3.amazonaws.com/${PYTORCH_IOS_NIGHTLY_NAME} -mkdir -p ./build_ios -unzip -d ./build_ios ./${PYTORCH_IOS_NIGHTLY_NAME} - -LIBTORCH_HEADER_ROOT="${TORCH_ROOT}/build_ios/install/include" -cd ${PROJ_ROOT_IOS} -IOS_ARCH=${IOS_ARCH} LIBTORCH_HEADER_ROOT=${LIBTORCH_HEADER_ROOT} ./build_ios.sh -rm -rf ${TORCH_ROOT} - -# store the binary -DEST_DIR=${WORKSPACE}/ios/${IOS_ARCH} -mkdir -p ${DEST_DIR} -cp ${PROJ_ROOT_IOS}/lib/*.a ${DEST_DIR} diff --git a/.circleci/unittest/ios/scripts/binary_ios_upload.sh b/.circleci/unittest/ios/scripts/binary_ios_upload.sh deleted file mode 100644 index ce56388e5da..00000000000 --- a/.circleci/unittest/ios/scripts/binary_ios_upload.sh +++ /dev/null @@ -1,42 +0,0 @@ -#!/bin/bash -set -ex -o pipefail - -echo "" -echo "DIR: $(pwd)" - -WORKSPACE=/Users/distiller/workspace -PROJ_ROOT=/Users/distiller/project -ARTIFACTS_DIR=${WORKSPACE}/ios -ls ${ARTIFACTS_DIR} -ZIP_DIR=${WORKSPACE}/zip -mkdir -p ${ZIP_DIR}/install/lib - -# build a FAT bianry -cd ${ZIP_DIR}/install/lib -libs=("${ARTIFACTS_DIR}/x86_64/libtorchvision_ops.a" "${ARTIFACTS_DIR}/arm64/libtorchvision_ops.a") -lipo -create "${libs[@]}" -o ${ZIP_DIR}/install/lib/libtorchvision_ops.a -lipo -i ${ZIP_DIR}/install/lib/*.a - -# copy the license -cp ${PROJ_ROOT}/LICENSE ${ZIP_DIR}/ -# zip the library -ZIPFILE=libtorchvision_ops_ios_nightly_build.zip -cd ${ZIP_DIR} -#for testing -touch version.txt -echo $(date +%s) > version.txt -zip -r ${ZIPFILE} install version.txt LICENSE - -# upload to aws -# Install conda then 'conda install' awscli -curl --retry 3 -o ~/conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh -chmod +x ~/conda.sh -/bin/bash ~/conda.sh -b -p ~/anaconda -export PATH="~/anaconda/bin:${PATH}" -source ~/anaconda/bin/activate -conda install -c conda-forge awscli --yes -set +x -export AWS_ACCESS_KEY_ID=${AWS_S3_ACCESS_KEY_FOR_PYTORCH_BINARY_UPLOAD} -export AWS_SECRET_ACCESS_KEY=${AWS_S3_ACCESS_SECRET_FOR_PYTORCH_BINARY_UPLOAD} -set -x -aws s3 cp ${ZIPFILE} s3://ossci-ios-build/ --acl public-read diff --git a/.circleci/unittest/linux/scripts/environment.yml b/.circleci/unittest/linux/scripts/environment.yml deleted file mode 100644 index fae96c5f93c..00000000000 --- a/.circleci/unittest/linux/scripts/environment.yml +++ /dev/null @@ -1,16 +0,0 @@ -channels: - - pytorch - - defaults -dependencies: - - pytest - - pytest-cov - - pytest-mock - - pip - - libpng - - jpeg - - ca-certificates - - h5py - - pip: - - future - - scipy - - av < 10 diff --git a/.circleci/unittest/linux/scripts/install.sh b/.circleci/unittest/linux/scripts/install.sh deleted file mode 100755 index 6375e191072..00000000000 --- a/.circleci/unittest/linux/scripts/install.sh +++ /dev/null @@ -1,45 +0,0 @@ -#!/usr/bin/env bash - -unset PYTORCH_VERSION -# For unittest, nightly PyTorch is used as the following section, -# so no need to set PYTORCH_VERSION. -# In fact, keeping PYTORCH_VERSION forces us to hardcode PyTorch version in config. - -set -ex - -eval "$(./conda/bin/conda shell.bash hook)" -conda activate ./env - -if [ "${CU_VERSION:-}" == cpu ] ; then - cudatoolkit="cpuonly" - version="cpu" -else - if [[ ${#CU_VERSION} -eq 4 ]]; then - CUDA_VERSION="${CU_VERSION:2:1}.${CU_VERSION:3:1}" - elif [[ ${#CU_VERSION} -eq 5 ]]; then - CUDA_VERSION="${CU_VERSION:2:2}.${CU_VERSION:4:1}" - fi - echo "Using CUDA $CUDA_VERSION as determined by CU_VERSION: ${CU_VERSION} " - version="$(python -c "print('.'.join(\"${CUDA_VERSION}\".split('.')[:2]))")" - cudatoolkit="pytorch-cuda=${version}" - - # make sure local cuda is set to required cuda version and not CUDA version by default - rm -f /usr/local/cuda - ln -s /usr/local/cuda-${version} /usr/local/cuda -fi - -case "$(uname -s)" in - Darwin*) os=MacOSX;; - *) os=Linux -esac - -printf "Installing PyTorch with %s\n" "${cudatoolkit}" -if [ "${os}" == "MacOSX" ]; then - conda install -y -c "pytorch-${UPLOAD_CHANNEL}" "pytorch-${UPLOAD_CHANNEL}"::pytorch "${cudatoolkit}" -else - conda install -y -c "pytorch-${UPLOAD_CHANNEL}" -c nvidia "pytorch-${UPLOAD_CHANNEL}"::pytorch[build="*${version}*"] "${cudatoolkit}" -fi - - -printf "* Installing torchvision\n" -python setup.py develop diff --git a/.circleci/unittest/linux/scripts/post_process.sh b/.circleci/unittest/linux/scripts/post_process.sh deleted file mode 100755 index e97bf2a7b1b..00000000000 --- a/.circleci/unittest/linux/scripts/post_process.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/usr/bin/env bash - -set -e - -eval "$(./conda/bin/conda shell.bash hook)" -conda activate ./env diff --git a/.circleci/unittest/linux/scripts/run-clang-format.py b/.circleci/unittest/linux/scripts/run-clang-format.py deleted file mode 100755 index 5c61b2519e0..00000000000 --- a/.circleci/unittest/linux/scripts/run-clang-format.py +++ /dev/null @@ -1,331 +0,0 @@ -#!/usr/bin/env python -""" -MIT License - -Copyright (c) 2017 Guillaume Papin - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. - -A wrapper script around clang-format, suitable for linting multiple files -and to use for continuous integration. - -This is an alternative API for the clang-format command line. -It runs over multiple files and directories in parallel. -A diff output is produced and a sensible exit code is returned. - -""" - -import argparse -import difflib -import fnmatch -import multiprocessing -import os -import signal -import subprocess -import sys -import traceback -from functools import partial - -try: - from subprocess import DEVNULL # py3k -except ImportError: - DEVNULL = open(os.devnull, "wb") - - -DEFAULT_EXTENSIONS = "c,h,C,H,cpp,hpp,cc,hh,c++,h++,cxx,hxx,cu" - - -class ExitStatus: - SUCCESS = 0 - DIFF = 1 - TROUBLE = 2 - - -def list_files(files, recursive=False, extensions=None, exclude=None): - if extensions is None: - extensions = [] - if exclude is None: - exclude = [] - - out = [] - for file in files: - if recursive and os.path.isdir(file): - for dirpath, dnames, fnames in os.walk(file): - fpaths = [os.path.join(dirpath, fname) for fname in fnames] - for pattern in exclude: - # os.walk() supports trimming down the dnames list - # by modifying it in-place, - # to avoid unnecessary directory listings. - dnames[:] = [x for x in dnames if not fnmatch.fnmatch(os.path.join(dirpath, x), pattern)] - fpaths = [x for x in fpaths if not fnmatch.fnmatch(x, pattern)] - for f in fpaths: - ext = os.path.splitext(f)[1][1:] - if ext in extensions: - out.append(f) - else: - out.append(file) - return out - - -def make_diff(file, original, reformatted): - return list( - difflib.unified_diff( - original, reformatted, fromfile=f"{file}\t(original)", tofile=f"{file}\t(reformatted)", n=3 - ) - ) - - -class DiffError(Exception): - def __init__(self, message, errs=None): - super().__init__(message) - self.errs = errs or [] - - -class UnexpectedError(Exception): - def __init__(self, message, exc=None): - super().__init__(message) - self.formatted_traceback = traceback.format_exc() - self.exc = exc - - -def run_clang_format_diff_wrapper(args, file): - try: - ret = run_clang_format_diff(args, file) - return ret - except DiffError: - raise - except Exception as e: - raise UnexpectedError(f"{file}: {e.__class__.__name__}: {e}", e) - - -def run_clang_format_diff(args, file): - try: - with open(file, encoding="utf-8") as f: - original = f.readlines() - except OSError as exc: - raise DiffError(str(exc)) - invocation = [args.clang_format_executable, file] - - # Use of utf-8 to decode the process output. - # - # Hopefully, this is the correct thing to do. - # - # It's done due to the following assumptions (which may be incorrect): - # - clang-format will returns the bytes read from the files as-is, - # without conversion, and it is already assumed that the files use utf-8. - # - if the diagnostics were internationalized, they would use utf-8: - # > Adding Translations to Clang - # > - # > Not possible yet! - # > Diagnostic strings should be written in UTF-8, - # > the client can translate to the relevant code page if needed. - # > Each translation completely replaces the format string - # > for the diagnostic. - # > -- http://clang.llvm.org/docs/InternalsManual.html#internals-diag-translation - - try: - proc = subprocess.Popen( - invocation, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True, encoding="utf-8" - ) - except OSError as exc: - raise DiffError(f"Command '{subprocess.list2cmdline(invocation)}' failed to start: {exc}") - proc_stdout = proc.stdout - proc_stderr = proc.stderr - - # hopefully the stderr pipe won't get full and block the process - outs = list(proc_stdout.readlines()) - errs = list(proc_stderr.readlines()) - proc.wait() - if proc.returncode: - raise DiffError( - "Command '{}' returned non-zero exit status {}".format( - subprocess.list2cmdline(invocation), proc.returncode - ), - errs, - ) - return make_diff(file, original, outs), errs - - -def bold_red(s): - return "\x1b[1m\x1b[31m" + s + "\x1b[0m" - - -def colorize(diff_lines): - def bold(s): - return "\x1b[1m" + s + "\x1b[0m" - - def cyan(s): - return "\x1b[36m" + s + "\x1b[0m" - - def green(s): - return "\x1b[32m" + s + "\x1b[0m" - - def red(s): - return "\x1b[31m" + s + "\x1b[0m" - - for line in diff_lines: - if line[:4] in ["--- ", "+++ "]: - yield bold(line) - elif line.startswith("@@ "): - yield cyan(line) - elif line.startswith("+"): - yield green(line) - elif line.startswith("-"): - yield red(line) - else: - yield line - - -def print_diff(diff_lines, use_color): - if use_color: - diff_lines = colorize(diff_lines) - sys.stdout.writelines(diff_lines) - - -def print_trouble(prog, message, use_colors): - error_text = "error:" - if use_colors: - error_text = bold_red(error_text) - print(f"{prog}: {error_text} {message}", file=sys.stderr) - - -def main(): - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument( - "--clang-format-executable", - metavar="EXECUTABLE", - help="path to the clang-format executable", - default="clang-format", - ) - parser.add_argument( - "--extensions", - help=f"comma separated list of file extensions (default: {DEFAULT_EXTENSIONS})", - default=DEFAULT_EXTENSIONS, - ) - parser.add_argument("-r", "--recursive", action="store_true", help="run recursively over directories") - parser.add_argument("files", metavar="file", nargs="+") - parser.add_argument("-q", "--quiet", action="store_true") - parser.add_argument( - "-j", - metavar="N", - type=int, - default=0, - help="run N clang-format jobs in parallel (default number of cpus + 1)", - ) - parser.add_argument( - "--color", default="auto", choices=["auto", "always", "never"], help="show colored diff (default: auto)" - ) - parser.add_argument( - "-e", - "--exclude", - metavar="PATTERN", - action="append", - default=[], - help="exclude paths matching the given glob-like pattern(s) from recursive search", - ) - - args = parser.parse_args() - - # use default signal handling, like diff return SIGINT value on ^C - # https://bugs.python.org/issue14229#msg156446 - signal.signal(signal.SIGINT, signal.SIG_DFL) - try: - signal.SIGPIPE - except AttributeError: - # compatibility, SIGPIPE does not exist on Windows - pass - else: - signal.signal(signal.SIGPIPE, signal.SIG_DFL) - - colored_stdout = False - colored_stderr = False - if args.color == "always": - colored_stdout = True - colored_stderr = True - elif args.color == "auto": - colored_stdout = sys.stdout.isatty() - colored_stderr = sys.stderr.isatty() - - version_invocation = [args.clang_format_executable, "--version"] - try: - subprocess.check_call(version_invocation, stdout=DEVNULL) - except subprocess.CalledProcessError as e: - print_trouble(parser.prog, str(e), use_colors=colored_stderr) - return ExitStatus.TROUBLE - except OSError as e: - print_trouble( - parser.prog, - f"Command '{subprocess.list2cmdline(version_invocation)}' failed to start: {e}", - use_colors=colored_stderr, - ) - return ExitStatus.TROUBLE - - retcode = ExitStatus.SUCCESS - files = list_files( - args.files, recursive=args.recursive, exclude=args.exclude, extensions=args.extensions.split(",") - ) - - if not files: - return - - njobs = args.j - if njobs == 0: - njobs = multiprocessing.cpu_count() + 1 - njobs = min(len(files), njobs) - - if njobs == 1: - # execute directly instead of in a pool, - # less overhead, simpler stacktraces - it = (run_clang_format_diff_wrapper(args, file) for file in files) - pool = None - else: - pool = multiprocessing.Pool(njobs) - it = pool.imap_unordered(partial(run_clang_format_diff_wrapper, args), files) - while True: - try: - outs, errs = next(it) - except StopIteration: - break - except DiffError as e: - print_trouble(parser.prog, str(e), use_colors=colored_stderr) - retcode = ExitStatus.TROUBLE - sys.stderr.writelines(e.errs) - except UnexpectedError as e: - print_trouble(parser.prog, str(e), use_colors=colored_stderr) - sys.stderr.write(e.formatted_traceback) - retcode = ExitStatus.TROUBLE - # stop at the first unexpected error, - # something could be very wrong, - # don't process all files unnecessarily - if pool: - pool.terminate() - break - else: - sys.stderr.writelines(errs) - if outs == []: - continue - if not args.quiet: - print_diff(outs, use_color=colored_stdout) - if retcode == ExitStatus.SUCCESS: - retcode = ExitStatus.DIFF - return retcode - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/.circleci/unittest/linux/scripts/run_test.sh b/.circleci/unittest/linux/scripts/run_test.sh deleted file mode 100755 index 5348baa71dd..00000000000 --- a/.circleci/unittest/linux/scripts/run_test.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/usr/bin/env bash - -set -e - -eval "$(./conda/bin/conda shell.bash hook)" -conda activate ./env - -python -m torch.utils.collect_env - -case "$(uname -s)" in - Darwin*) - # The largest macOS runner is not able to handle the regular test suite plus the transforms v2 tests at the same - # time due to insufficient resources. Thus, we ignore the transforms v2 tests at first and run them in a separate - # step afterwards. - GLOB='test/test_transforms_v2*' - pytest --junitxml=test-results/junit.xml -v --durations 20 --ignore-glob="${GLOB}" - eval "pytest --junitxml=test-results/junit-transforms-v2.xml -v --durations 20 ${GLOB}" - ;; - *) - pytest --junitxml=test-results/junit.xml -v --durations 20 - ;; -esac diff --git a/.circleci/unittest/linux/scripts/setup_env.sh b/.circleci/unittest/linux/scripts/setup_env.sh deleted file mode 100755 index 8a8a78f1fb2..00000000000 --- a/.circleci/unittest/linux/scripts/setup_env.sh +++ /dev/null @@ -1,47 +0,0 @@ -#!/usr/bin/env bash - -# This script is for setting up environment in which unit test is ran. -# To speed up the CI time, the resulting environment is cached. -# -# Do not install PyTorch and torchvision here, otherwise they also get cached. - -set -ex - -this_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" -# Avoid error: "fatal: unsafe repository" -git config --global --add safe.directory '*' -root_dir="$(git rev-parse --show-toplevel)" -conda_dir="${root_dir}/conda" -env_dir="${root_dir}/env" - -cd "${root_dir}" - -case "$(uname -s)" in - Darwin*) os=MacOSX;; - *) os=Linux -esac - -# 1. Install conda at ./conda -if [ ! -d "${conda_dir}" ]; then - printf "* Installing conda\n" - wget -O miniconda.sh "http://repo.continuum.io/miniconda/Miniconda3-latest-${os}-x86_64.sh" - bash ./miniconda.sh -b -f -p "${conda_dir}" -fi -eval "$(${conda_dir}/bin/conda shell.bash hook)" - -# 2. Create test environment at ./env -if [ ! -d "${env_dir}" ]; then - printf "* Creating a test environment\n" - conda create --prefix "${env_dir}" -y python="$PYTHON_VERSION" -fi -conda activate "${env_dir}" - -# 3. Install Conda dependencies -printf "* Installing dependencies (except PyTorch)\n" -FFMPEG_PIN="=4.2" -if [[ "${PYTHON_VERSION}" == "3.9" ]]; then - FFMPEG_PIN=">=4.2" -fi - -conda install -y -c pytorch "ffmpeg${FFMPEG_PIN}" -conda env update --file "${this_dir}/environment.yml" --prune diff --git a/.circleci/unittest/windows/scripts/environment.yml b/.circleci/unittest/windows/scripts/environment.yml deleted file mode 100644 index d229aafb41a..00000000000 --- a/.circleci/unittest/windows/scripts/environment.yml +++ /dev/null @@ -1,19 +0,0 @@ -channels: - - pytorch - - defaults -dependencies: - - pytest - - pytest-cov - - pytest-mock - - pip - - libpng - - jpeg - - ca-certificates - - hdf5 - - setuptools - - pip: - - future - - scipy - - av !=9.1.1, <10 - - dataclasses - - h5py diff --git a/.circleci/unittest/windows/scripts/install.sh b/.circleci/unittest/windows/scripts/install.sh deleted file mode 100644 index 7c55c8144d9..00000000000 --- a/.circleci/unittest/windows/scripts/install.sh +++ /dev/null @@ -1,53 +0,0 @@ - -#!/usr/bin/env bash - -unset PYTORCH_VERSION -# For unittest, nightly PyTorch is used as the following section, -# so no need to set PYTORCH_VERSION. -# In fact, keeping PYTORCH_VERSION forces us to hardcode PyTorch version in config. - -set -ex - -this_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" - -eval "$(./conda/Scripts/conda.exe 'shell.bash' 'hook')" -conda activate ./env - -# TODO, refactor the below logic to make it easy to understand how to get correct cuda_version. -if [ "${CU_VERSION:-}" == cpu ] ; then - cudatoolkit="cpuonly" - version="cpu" -else - if [[ ${#CU_VERSION} -eq 4 ]]; then - CUDA_VERSION="${CU_VERSION:2:1}.${CU_VERSION:3:1}" - elif [[ ${#CU_VERSION} -eq 5 ]]; then - CUDA_VERSION="${CU_VERSION:2:2}.${CU_VERSION:4:1}" - fi - - cuda_toolkit_pckg="cudatoolkit" - if [[ $CUDA_VERSION == 11.6 || $CUDA_VERSION == 11.7 || $CUDA_VERSION == 11.8 || $CUDA_VERSION == 12.1 ]]; then - cuda_toolkit_pckg="pytorch-cuda" - fi - - echo "Using CUDA $CUDA_VERSION as determined by CU_VERSION" - version="$(python -c "print('.'.join(\"${CUDA_VERSION}\".split('.')[:2]))")" - cudatoolkit="${cuda_toolkit_pckg}=${version}" -fi - -printf "Installing PyTorch with %s\n" "${cudatoolkit}" -conda install -y -c "pytorch-${UPLOAD_CHANNEL}" -c nvidia "pytorch-${UPLOAD_CHANNEL}"::pytorch[build="*${version}*"] "${cudatoolkit}" - -torch_cuda=$(python -c "import torch; print(torch.cuda.is_available())") -echo torch.cuda.is_available is $torch_cuda - -if [ ! -z "${CUDA_VERSION:-}" ] ; then - if [ "$torch_cuda" == "False" ]; then - echo "torch with cuda installed but torch.cuda.is_available() is False" - exit 1 - fi -fi - -source "$this_dir/set_cuda_envs.sh" - -printf "* Installing torchvision\n" -"$this_dir/vc_env_helper.bat" python setup.py develop diff --git a/.circleci/unittest/windows/scripts/install_conda.bat b/.circleci/unittest/windows/scripts/install_conda.bat deleted file mode 100644 index 6052ad08b10..00000000000 --- a/.circleci/unittest/windows/scripts/install_conda.bat +++ /dev/null @@ -1 +0,0 @@ -start /wait "" "%miniconda_exe%" /S /InstallationType=JustMe /RegisterPython=0 /AddToPath=0 /D=%tmp_conda% diff --git a/.circleci/unittest/windows/scripts/post_process.sh b/.circleci/unittest/windows/scripts/post_process.sh deleted file mode 100644 index 5c5cbb758a9..00000000000 --- a/.circleci/unittest/windows/scripts/post_process.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/usr/bin/env bash - -set -e - -eval "$(./conda/Scripts/conda.exe 'shell.bash' 'hook')" -conda activate ./env diff --git a/.circleci/unittest/windows/scripts/run_test.sh b/.circleci/unittest/windows/scripts/run_test.sh deleted file mode 100644 index 802ad37f511..00000000000 --- a/.circleci/unittest/windows/scripts/run_test.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/usr/bin/env bash - -set -e - -eval "$(./conda/Scripts/conda.exe 'shell.bash' 'hook')" -conda activate ./env - -this_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" -source "$this_dir/set_cuda_envs.sh" - -python -m torch.utils.collect_env -pytest --junitxml=test-results/junit.xml -v --durations 20 diff --git a/.circleci/unittest/windows/scripts/set_cuda_envs.sh b/.circleci/unittest/windows/scripts/set_cuda_envs.sh deleted file mode 100644 index 7db3137b594..00000000000 --- a/.circleci/unittest/windows/scripts/set_cuda_envs.sh +++ /dev/null @@ -1,48 +0,0 @@ -#!/usr/bin/env bash -set -ex - -echo CU_VERSION is "${CU_VERSION}" -echo CUDA_VERSION is "${CUDA_VERSION}" - -# Currenly, CU_VERSION and CUDA_VERSION are not consistent. -# to understand this code, see https://github.com/pytorch/vision/issues/4443 -version="cpu" -if [[ ! -z "${CUDA_VERSION}" ]] ; then - version="$CUDA_VERSION" -else - if [[ ${#CU_VERSION} -eq 5 ]]; then - version="${CU_VERSION:2:2}.${CU_VERSION:4:1}" - fi -fi - -# Don't use if [[ "$version" == "cpu" ]]; then exit 0 fi. -# It would exit the shell. One result is cpu tests would not run if the shell exit. -# Unless there's an error, Don't exit. -if [[ "$version" != "cpu" ]]; then - # set cuda envs - export PATH="/c/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v${version}/bin:/c/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v${version}/libnvvp:$PATH" - export CUDA_PATH_V${version/./_}="C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v${version}" - export CUDA_PATH="C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v${version}" - - if [ ! -d "$CUDA_PATH" ]; then - echo "$CUDA_PATH" does not exist - exit 1 - fi - - if [ ! -f "${CUDA_PATH}\include\nvjpeg.h" ]; then - echo "nvjpeg does not exist" - exit 1 - fi - - # check cuda driver version - for path in '/c/Program Files/NVIDIA Corporation/NVSMI/nvidia-smi.exe' /c/Windows/System32/nvidia-smi.exe; do - if [[ -x "$path" ]]; then - "$path" || echo "true"; - break - fi - done - - which nvcc - nvcc --version - env | grep CUDA -fi diff --git a/.circleci/unittest/windows/scripts/setup_env.sh b/.circleci/unittest/windows/scripts/setup_env.sh deleted file mode 100644 index 84697875979..00000000000 --- a/.circleci/unittest/windows/scripts/setup_env.sh +++ /dev/null @@ -1,45 +0,0 @@ -#!/usr/bin/env bash - -# This script is for setting up environment in which unit test is ran. -# To speed up the CI time, the resulting environment is cached. -# -# Do not install PyTorch and torchvision here, otherwise they also get cached. - -set -ex - -this_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" -root_dir="$(git rev-parse --show-toplevel)" -conda_dir="${root_dir}/conda" -env_dir="${root_dir}/env" - -cd "${root_dir}" - -# 1. Install conda at ./conda -if [ ! -d "${conda_dir}" ]; then - printf "* Installing conda\n" - export tmp_conda="$(echo $conda_dir | tr '/' '\\')" - export miniconda_exe="$(echo $root_dir | tr '/' '\\')\\miniconda.exe" - curl --output miniconda.exe https://repo.anaconda.com/miniconda/Miniconda3-latest-Windows-x86_64.exe -O - "$this_dir/install_conda.bat" - unset tmp_conda - unset miniconda_exe -fi - -eval "$(${conda_dir}/Scripts/conda.exe 'shell.bash' 'hook')" - -# 2. Create test environment at ./env -if [ ! -d "${env_dir}" ]; then - printf "* Creating a test environment\n" - conda create --prefix "${env_dir}" -y python="$PYTHON_VERSION" -fi -conda activate "${env_dir}" - -# 3. Install Conda dependencies -printf "* Installing dependencies (except PyTorch)\n" -conda env update --file "${this_dir}/environment.yml" --prune - -# 4. Downgrade setuptools on Python 3.7. -# See https://github.com/pytorch/vision/pull/5868 -if [[ "${PYTHON_VERSION}" == '3.7' ]]; then - pip install --upgrade setuptools==58.0.4 -fi diff --git a/.circleci/unittest/windows/scripts/vc_env_helper.bat b/.circleci/unittest/windows/scripts/vc_env_helper.bat deleted file mode 100644 index 9410135677a..00000000000 --- a/.circleci/unittest/windows/scripts/vc_env_helper.bat +++ /dev/null @@ -1,39 +0,0 @@ -@echo on - -set VC_VERSION_LOWER=16 -set VC_VERSION_UPPER=17 - -for /f "usebackq tokens=*" %%i in (`"%ProgramFiles(x86)%\Microsoft Visual Studio\Installer\vswhere.exe" -legacy -products * -version [%VC_VERSION_LOWER%^,%VC_VERSION_UPPER%^) -property installationPath`) do ( - if exist "%%i" if exist "%%i\VC\Auxiliary\Build\vcvarsall.bat" ( - set "VS15INSTALLDIR=%%i" - set "VS15VCVARSALL=%%i\VC\Auxiliary\Build\vcvarsall.bat" - goto vswhere - ) -) - -:vswhere -if "%VSDEVCMD_ARGS%" == "" ( - call "%VS15VCVARSALL%" x64 || exit /b 1 -) else ( - call "%VS15VCVARSALL%" x64 %VSDEVCMD_ARGS% || exit /b 1 -) - -@echo on - -set DISTUTILS_USE_SDK=1 - -set args=%1 -shift -:start -if [%1] == [] goto done -set args=%args% %1 -shift -goto start - -:done -if "%args%" == "" ( - echo Usage: vc_env_helper.bat [command] [args] - echo e.g. vc_env_helper.bat cl /c test.cpp -) - -%args% || exit /b 1 diff --git a/.github/scripts/unittest.sh b/.github/scripts/unittest.sh index 2a0b7154200..663181517fb 100755 --- a/.github/scripts/unittest.sh +++ b/.github/scripts/unittest.sh @@ -11,4 +11,5 @@ echo '::group::Install testing utilities' pip install --progress-bar=off pytest pytest-mock pytest-cov echo '::endgroup::' -pytest --junit-xml="${RUNNER_TEST_RESULTS_DIR}/test-results.xml" -v --durations=25 +#pytest --junit-xml="${RUNNER_TEST_RESULTS_DIR}/test-results.xml" -v --durations=25 +python test/smoke_test.py diff --git a/.github/workflows/build-cmake.yml b/.github/workflows/build-cmake.yml deleted file mode 100644 index 8fdf99c84d1..00000000000 --- a/.github/workflows/build-cmake.yml +++ /dev/null @@ -1,92 +0,0 @@ -name: CMake - -on: - pull_request: - push: - branches: - - nightly - - main - - release/* - workflow_dispatch: - -jobs: - linux: - strategy: - matrix: - include: - - runner: linux.12xlarge - gpu-arch-type: cpu - - runner: linux.g5.4xlarge.nvidia.gpu - gpu-arch-type: cuda - gpu-arch-version: "11.8" - fail-fast: false - uses: pytorch/test-infra/.github/workflows/linux_job.yml@main - with: - repository: pytorch/vision - runner: ${{ matrix.runner }} - gpu-arch-type: ${{ matrix.gpu-arch-type }} - gpu-arch-version: ${{ matrix.gpu-arch-version }} - script: | - set -euo pipefail - - export PYTHON_VERSION=3.8 - export GPU_ARCH_TYPE=${{ matrix.gpu-arch-type }} - export GPU_ARCH_VERSION=${{ matrix.gpu-arch-version }} - - ./.github/scripts/cmake.sh - - macos: - strategy: - matrix: - include: - - runner: macos-12 - - runner: macos-m1-12 - fail-fast: false - uses: pytorch/test-infra/.github/workflows/macos_job.yml@main - with: - repository: pytorch/vision - runner: ${{ matrix.runner }} - script: | - set -euo pipefail - - export PYTHON_VERSION=3.8 - export GPU_ARCH_TYPE=cpu - export GPU_ARCH_VERSION='' - - ./.github/scripts/cmake.sh - - windows: - strategy: - matrix: - include: - - runner: windows.4xlarge - gpu-arch-type: cpu - - runner: windows.g5.4xlarge.nvidia.gpu - gpu-arch-type: cuda - gpu-arch-version: "11.8" - fail-fast: false - uses: pytorch/test-infra/.github/workflows/windows_job.yml@main - with: - repository: pytorch/vision - runner: ${{ matrix.runner }} - gpu-arch-type: ${{ matrix.gpu-arch-type }} - gpu-arch-version: ${{ matrix.gpu-arch-version }} - script: | - set -euo pipefail - - source packaging/windows/internal/vc_install_helper.sh - - # FIXME: Basically, we are reinstalling CUDA here. We only need this, because we need to copy some files that - # can be extracted from the CUDA installer, but are not available on our Windows AMI. - # See https://github.com/pytorch/test-infra/pull/4189 - if [[ ${{ matrix.gpu-arch-type }} == cuda ]]; then - export CU_VERSION=cu$(echo ${{ matrix.gpu-arch-version }} | sed 's/\.//') - echo CU_VERSION="${CU_VERSION}" - packaging/windows/internal/cuda_install.bat - fi - - export PYTHON_VERSION=3.8 - export GPU_ARCH_TYPE=${{ matrix.gpu-arch-type }} - export GPU_ARCH_VERSION=${{ matrix.gpu-arch-version }} - - ./.github/scripts/cmake.sh diff --git a/.github/workflows/build-conda-linux.yml b/.github/workflows/build-conda-linux.yml deleted file mode 100644 index e44bb877ed6..00000000000 --- a/.github/workflows/build-conda-linux.yml +++ /dev/null @@ -1,51 +0,0 @@ -name: Build Linux Conda - -on: - pull_request: - push: - branches: - - nightly - - main - - release/* - tags: - # NOTE: Binary build pipelines should only get triggered on release candidate builds - # Release candidate tags look like: v1.11.0-rc1 - - v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+ - workflow_dispatch: - -jobs: - generate-matrix: - uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main - with: - package-type: conda - os: linux - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build: - needs: generate-matrix - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/vision - pre-script: "" - post-script: "" - conda-package-directory: packaging/torchvision - smoke-test-script: test/smoke_test.py - package-name: torchvision - name: ${{ matrix.repository }} - uses: pytorch/test-infra/.github/workflows/build_conda_linux.yml@main - with: - conda-package-directory: ${{ matrix.conda-package-directory }} - repository: ${{ matrix.repository }} - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.generate-matrix.outputs.matrix }} - pre-script: ${{ matrix.pre-script }} - post-script: ${{ matrix.post-script }} - package-name: ${{ matrix.package-name }} - smoke-test-script: ${{ matrix.smoke-test-script }} - trigger-event: ${{ github.event_name }} - secrets: - CONDA_PYTORCHBOT_TOKEN: ${{ secrets.CONDA_PYTORCHBOT_TOKEN }} diff --git a/.github/workflows/build-conda-m1.yml b/.github/workflows/build-conda-m1.yml deleted file mode 100644 index 7006b6714aa..00000000000 --- a/.github/workflows/build-conda-m1.yml +++ /dev/null @@ -1,52 +0,0 @@ -name: Build M1 Conda - -on: - pull_request: - push: - branches: - - nightly - - main - - release/* - tags: - # NOTE: Binary build pipelines should only get triggered on release candidate builds - # Release candidate tags look like: v1.11.0-rc1 - - v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+ - workflow_dispatch: - -jobs: - generate-matrix: - uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main - with: - package-type: conda - os: macos-arm64 - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build: - needs: generate-matrix - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/vision - pre-script: "" - post-script: "" - conda-package-directory: packaging/torchvision - smoke-test-script: test/smoke_test.py - package-name: torchvision - name: ${{ matrix.repository }} - uses: pytorch/test-infra/.github/workflows/build_conda_macos.yml@main - with: - conda-package-directory: ${{ matrix.conda-package-directory }} - repository: ${{ matrix.repository }} - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.generate-matrix.outputs.matrix }} - pre-script: ${{ matrix.pre-script }} - post-script: ${{ matrix.post-script }} - package-name: ${{ matrix.package-name }} - smoke-test-script: ${{ matrix.smoke-test-script }} - runner-type: macos-m1-12 - trigger-event: ${{ github.event_name }} - secrets: - CONDA_PYTORCHBOT_TOKEN: ${{ secrets.CONDA_PYTORCHBOT_TOKEN }} diff --git a/.github/workflows/build-conda-macos.yml b/.github/workflows/build-conda-macos.yml deleted file mode 100644 index 39d00534b3a..00000000000 --- a/.github/workflows/build-conda-macos.yml +++ /dev/null @@ -1,52 +0,0 @@ -name: Build Macos Conda - -on: - pull_request: - push: - branches: - - nightly - - main - - release/* - tags: - # NOTE: Binary build pipelines should only get triggered on release candidate builds - # Release candidate tags look like: v1.11.0-rc1 - - v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+ - workflow_dispatch: - -jobs: - generate-matrix: - uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main - with: - package-type: conda - os: macos - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build: - needs: generate-matrix - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/vision - pre-script: "" - post-script: "" - conda-package-directory: packaging/torchvision - smoke-test-script: test/smoke_test.py - package-name: torchvision - name: ${{ matrix.repository }} - uses: pytorch/test-infra/.github/workflows/build_conda_macos.yml@main - with: - conda-package-directory: ${{ matrix.conda-package-directory }} - repository: ${{ matrix.repository }} - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.generate-matrix.outputs.matrix }} - pre-script: ${{ matrix.pre-script }} - post-script: ${{ matrix.post-script }} - package-name: ${{ matrix.package-name }} - smoke-test-script: ${{ matrix.smoke-test-script }} - runner-type: macos-12 - trigger-event: ${{ github.event_name }} - secrets: - CONDA_PYTORCHBOT_TOKEN: ${{ secrets.CONDA_PYTORCHBOT_TOKEN }} diff --git a/.github/workflows/build-conda-windows.yml b/.github/workflows/build-conda-windows.yml deleted file mode 100644 index 954d09d39ec..00000000000 --- a/.github/workflows/build-conda-windows.yml +++ /dev/null @@ -1,51 +0,0 @@ -name: Build Windows Conda - -on: - pull_request: - push: - branches: - - nightly - - main - - release/* - tags: - # NOTE: Binary build pipelines should only get triggered on release candidate builds - # Release candidate tags look like: v1.11.0-rc1 - - v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+ - workflow_dispatch: - -jobs: - generate-matrix: - uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main - with: - package-type: conda - os: windows - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build: - needs: generate-matrix - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/vision - pre-script: packaging/pre_build_script.sh - env-script: packaging/windows/internal/vc_env_helper.bat - post-script: "" - smoke-test-script: test/smoke_test.py - package-name: torchvision - name: ${{ matrix.repository }} - uses: pytorch/test-infra/.github/workflows/build_conda_windows.yml@main - with: - conda-package-directory: ${{ matrix.conda-package-directory }} - repository: ${{ matrix.repository }} - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.generate-matrix.outputs.matrix }} - pre-script: ${{ matrix.pre-script }} - post-script: ${{ matrix.post-script }} - package-name: ${{ matrix.package-name }} - smoke-test-script: ${{ matrix.smoke-test-script }} - trigger-event: ${{ github.event_name }} - secrets: - CONDA_PYTORCHBOT_TOKEN: ${{ secrets.CONDA_PYTORCHBOT_TOKEN }} diff --git a/.github/workflows/build-wheels-linux.yml b/.github/workflows/build-wheels-linux.yml deleted file mode 100644 index e997d648ec2..00000000000 --- a/.github/workflows/build-wheels-linux.yml +++ /dev/null @@ -1,50 +0,0 @@ -name: Build Linux Wheels - -on: - pull_request: - push: - branches: - - nightly - - main - - release/* - tags: - # NOTE: Binary build pipelines should only get triggered on release candidate builds - # Release candidate tags look like: v1.11.0-rc1 - - v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+ - workflow_dispatch: - -jobs: - generate-matrix: - uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main - with: - package-type: wheel - os: linux - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build: - needs: generate-matrix - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/vision - pre-script: packaging/pre_build_script.sh - post-script: packaging/post_build_script.sh - smoke-test-script: test/smoke_test.py - package-name: torchvision - name: ${{ matrix.repository }} - uses: pytorch/test-infra/.github/workflows/build_wheels_linux.yml@main - with: - repository: ${{ matrix.repository }} - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.generate-matrix.outputs.matrix }} - pre-script: ${{ matrix.pre-script }} - post-script: ${{ matrix.post-script }} - package-name: ${{ matrix.package-name }} - smoke-test-script: ${{ matrix.smoke-test-script }} - trigger-event: ${{ github.event_name }} - secrets: - AWS_PYTORCH_UPLOADER_ACCESS_KEY_ID: ${{ secrets.AWS_PYTORCH_UPLOADER_ACCESS_KEY_ID }} - AWS_PYTORCH_UPLOADER_SECRET_ACCESS_KEY: ${{ secrets.AWS_PYTORCH_UPLOADER_SECRET_ACCESS_KEY }} diff --git a/.github/workflows/build-wheels-m1.yml b/.github/workflows/build-wheels-m1.yml deleted file mode 100644 index 66c7687acc9..00000000000 --- a/.github/workflows/build-wheels-m1.yml +++ /dev/null @@ -1,51 +0,0 @@ -name: Build M1 Wheels - -on: - pull_request: - push: - branches: - - nightly - - main - - release/* - tags: - # NOTE: Binary build pipelines should only get triggered on release candidate builds - # Release candidate tags look like: v1.11.0-rc1 - - v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+ - workflow_dispatch: - -jobs: - generate-matrix: - uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main - with: - package-type: wheel - os: macos-arm64 - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build: - needs: generate-matrix - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/vision - pre-script: packaging/pre_build_script.sh - post-script: packaging/post_build_script.sh - smoke-test-script: test/smoke_test.py - package-name: torchvision - name: ${{ matrix.repository }} - uses: pytorch/test-infra/.github/workflows/build_wheels_macos.yml@main - with: - repository: ${{ matrix.repository }} - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.generate-matrix.outputs.matrix }} - pre-script: ${{ matrix.pre-script }} - post-script: ${{ matrix.post-script }} - package-name: ${{ matrix.package-name }} - runner-type: macos-m1-12 - smoke-test-script: ${{ matrix.smoke-test-script }} - trigger-event: ${{ github.event_name }} - secrets: - AWS_PYTORCH_UPLOADER_ACCESS_KEY_ID: ${{ secrets.AWS_PYTORCH_UPLOADER_ACCESS_KEY_ID }} - AWS_PYTORCH_UPLOADER_SECRET_ACCESS_KEY: ${{ secrets.AWS_PYTORCH_UPLOADER_SECRET_ACCESS_KEY }} diff --git a/.github/workflows/build-wheels-macos.yml b/.github/workflows/build-wheels-macos.yml deleted file mode 100644 index 6c5ebc0fc37..00000000000 --- a/.github/workflows/build-wheels-macos.yml +++ /dev/null @@ -1,51 +0,0 @@ -name: Build Macos Wheels - -on: - pull_request: - push: - branches: - - nightly - - main - - release/* - tags: - # NOTE: Binary build pipelines should only get triggered on release candidate builds - # Release candidate tags look like: v1.11.0-rc1 - - v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+ - workflow_dispatch: - -jobs: - generate-matrix: - uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main - with: - package-type: wheel - os: macos - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build: - needs: generate-matrix - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/vision - pre-script: packaging/pre_build_script.sh - post-script: packaging/post_build_script.sh - smoke-test-script: test/smoke_test.py - package-name: torchvision - name: ${{ matrix.repository }} - uses: pytorch/test-infra/.github/workflows/build_wheels_macos.yml@main - with: - repository: ${{ matrix.repository }} - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.generate-matrix.outputs.matrix }} - pre-script: ${{ matrix.pre-script }} - post-script: ${{ matrix.post-script }} - package-name: ${{ matrix.package-name }} - runner-type: macos-12 - smoke-test-script: ${{ matrix.smoke-test-script }} - trigger-event: ${{ github.event_name }} - secrets: - AWS_PYTORCH_UPLOADER_ACCESS_KEY_ID: ${{ secrets.AWS_PYTORCH_UPLOADER_ACCESS_KEY_ID }} - AWS_PYTORCH_UPLOADER_SECRET_ACCESS_KEY: ${{ secrets.AWS_PYTORCH_UPLOADER_SECRET_ACCESS_KEY }} diff --git a/.github/workflows/build-wheels-windows.yml b/.github/workflows/build-wheels-windows.yml deleted file mode 100644 index 3d818ece538..00000000000 --- a/.github/workflows/build-wheels-windows.yml +++ /dev/null @@ -1,52 +0,0 @@ -name: Build Windows Wheels - -on: - pull_request: - push: - branches: - - nightly - - main - - release/* - tags: - # NOTE: Binary build pipelines should only get triggered on release candidate builds - # Release candidate tags look like: v1.11.0-rc1 - - v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+ - workflow_dispatch: - -jobs: - generate-matrix: - uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main - with: - package-type: wheel - os: windows - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build: - needs: generate-matrix - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/vision - pre-script: packaging/pre_build_script.sh - env-script: packaging/windows/internal/vc_env_helper.bat - post-script: "python packaging/wheel/relocate.py" - smoke-test-script: test/smoke_test.py - package-name: torchvision - name: ${{ matrix.repository }} - uses: pytorch/test-infra/.github/workflows/build_wheels_windows.yml@main - with: - repository: ${{ matrix.repository }} - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.generate-matrix.outputs.matrix }} - pre-script: ${{ matrix.pre-script }} - env-script: ${{ matrix.env-script }} - post-script: ${{ matrix.post-script }} - package-name: ${{ matrix.package-name }} - smoke-test-script: ${{ matrix.smoke-test-script }} - trigger-event: ${{ github.event_name }} - secrets: - AWS_PYTORCH_UPLOADER_ACCESS_KEY_ID: ${{ secrets.AWS_PYTORCH_UPLOADER_ACCESS_KEY_ID }} - AWS_PYTORCH_UPLOADER_SECRET_ACCESS_KEY: ${{ secrets.AWS_PYTORCH_UPLOADER_SECRET_ACCESS_KEY }} diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml deleted file mode 100644 index f4cc76db05e..00000000000 --- a/.github/workflows/docs.yml +++ /dev/null @@ -1,115 +0,0 @@ -name: Docs - -on: - pull_request: - push: - branches: - - nightly - - main - - release/* - tags: - - v[0-9]+.[0-9]+.[0-9] - - v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+ - workflow_dispatch: - -jobs: - build: - uses: pytorch/test-infra/.github/workflows/linux_job.yml@main - with: - repository: pytorch/vision - upload-artifact: docs - script: | - set -euo pipefail - - export PYTHON_VERSION=3.8 - export GPU_ARCH_TYPE=cpu - export GPU_ARCH_VERSION='' - ./.github/scripts/setup-env.sh - - # Prepare conda - CONDA_PATH=$(which conda) - eval "$(${CONDA_PATH} shell.bash hook)" - conda activate ci - # FIXME: not sure why we need this. `ldd torchvision/video_reader.so` shows that it - # already links against the one pulled from conda. However, at runtime it pulls from - # /lib64 - # Should we maybe always do this in `./.github/scripts/setup-env.sh` so that we don't - # have to pay attention in all other workflows? - export LD_LIBRARY_PATH="${CONDA_PREFIX}/lib:${LD_LIBRARY_PATH}" - - cd docs - - echo '::group::Install doc requirements' - pip install --progress-bar=off -r requirements.txt - echo '::endgroup::' - - if [[ ${{ github.event_name }} == push && (${{ github.ref_type }} == tag || (${{ github.ref_type }} == branch && ${{ github.ref_name }} == release/*)) ]]; then - echo '::group::Enable version string sanitization' - # This environment variable just has to exist and must not be empty. The actual value is arbitrary. - # See docs/source/conf.py for details - export TORCHVISION_SANITIZE_VERSION_STR_IN_DOCS=1 - echo '::endgroup::' - fi - - # The runner does not have sufficient memory to run with as many processes as there are - # cores (`-j auto`). Thus, we limit to a single process (`-j 1`) here. - sed -i -e 's/-j auto/-j 1/' Makefile - make html - - cp -r build/html "${RUNNER_ARTIFACT_DIR}" - - # On PRs we also want to upload the docs into our S3 bucket for preview. - if [[ ${{ github.event_name == 'pull_request' }} ]]; then - cp -r build/html/* "${RUNNER_DOCS_DIR}" - fi - - upload: - needs: build - if: github.repository == 'pytorch/vision' && github.event_name == 'push' && - ((github.ref_type == 'branch' && github.ref_name == 'main') || github.ref_type == 'tag') - permissions: - contents: write - uses: pytorch/test-infra/.github/workflows/linux_job.yml@main - with: - repository: pytorch/vision - download-artifact: docs - ref: gh-pages - script: | - set -euo pipefail - - REF_TYPE=${{ github.ref_type }} - REF_NAME=${{ github.ref_name }} - - if [[ "${REF_TYPE}" == branch ]]; then - TARGET_FOLDER="${REF_NAME}" - elif [[ "${REF_TYPE}" == tag ]]; then - case "${REF_NAME}" in - *-rc*) - echo "Aborting upload since this is an RC tag: ${REF_NAME}" - exit 0 - ;; - *) - # Strip the leading "v" as well as the trailing patch version. For example: - # 'v0.15.2' -> '0.15' - TARGET_FOLDER=$(echo "${REF_NAME}" | sed 's/v\([0-9]\+\)\.\([0-9]\+\)\.[0-9]\+/\1.\2/') - ;; - esac - fi - echo "Target Folder: ${TARGET_FOLDER}" - - mkdir -p "${TARGET_FOLDER}" - rm -rf "${TARGET_FOLDER}"/* - mv "${RUNNER_ARTIFACT_DIR}"/html/* "${TARGET_FOLDER}" - git add "${TARGET_FOLDER}" || true - - if [[ "${TARGET_FOLDER}" == main ]]; then - mkdir -p _static - rm -rf _static/* - cp -r "${TARGET_FOLDER}"/_static/* _static - git add _static || true - fi - - git config user.name 'pytorchbot' - git config user.email 'soumith+bot@pytorch.org' - git commit -m "auto-generating sphinx docs" || true - git push diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml deleted file mode 100644 index ec8d285c9f3..00000000000 --- a/.github/workflows/lint.yml +++ /dev/null @@ -1,107 +0,0 @@ -name: Lint - -on: - pull_request: - push: - branches: - - nightly - - main - - release/* - workflow_dispatch: - -jobs: - python-source-and-configs: - uses: pytorch/test-infra/.github/workflows/linux_job.yml@main - with: - repository: pytorch/vision - script: | - set -euo pipefail - - echo '::group::Setup environment' - CONDA_PATH=$(which conda) - eval "$(${CONDA_PATH} shell.bash hook)" - conda create --name ci --quiet --yes python=3.8 pip - conda activate ci - echo '::endgroup::' - - echo '::group::Install lint tools' - pip install --progress-bar=off pre-commit - echo '::endgroup::' - - set +e - pre-commit run --all-files - - if [ $? -ne 0 ]; then - git --no-pager diff - exit 1 - fi - - c-source: - uses: pytorch/test-infra/.github/workflows/linux_job.yml@main - with: - repository: pytorch/vision - script: | - set -euo pipefail - - echo '::group::Setup environment' - CONDA_PATH=$(which conda) - eval "$(${CONDA_PATH} shell.bash hook)" - # clang-format needs some shared libraries that conflict with the system ones. Thus, we install them from conda - # and prepend the libraries to linker path to prioritize them. `ncurses=5` is only available on the conda-forge - # channel. Since we are not building or testing here, this is fine. - conda create --name ci --quiet --yes -c conda-forge python=3.8 ncurses=5 libgcc - conda activate ci - export LD_LIBRARY_PATH="${CONDA_PREFIX}/lib:${LD_LIBRARY_PATH}" - echo '::endgroup::' - - echo '::group::Install lint tools' - curl https://oss-clang-format.s3.us-east-2.amazonaws.com/linux64/clang-format-linux64 -o ./clang-format - chmod +x ./clang-format - echo '::endgroup::' - - echo '::group::Lint C source' - set +e - ./.circleci/unittest/linux/scripts/run-clang-format.py -r torchvision/csrc --clang-format-executable ./clang-format - - if [ $? -ne 0 ]; then - git --no-pager diff - exit 1 - fi - echo '::endgroup::' - - - python-types: - uses: pytorch/test-infra/.github/workflows/linux_job.yml@main - with: - repository: pytorch/vision - script: | - set -euo pipefail - - export PYTHON_VERSION=3.8 - export GPU_ARCH_TYPE=cpu - export GPU_ARCH_VERSION='' - - ./.github/scripts/setup-env.sh - - CONDA_PATH=$(which conda) - eval "$(${CONDA_PATH} shell.bash hook)" - conda activate ci - - echo '::group::Install lint tools' - pip install --progress-bar=off mypy - echo '::endgroup::' - - echo '::group::Lint Python types' - mypy --install-types --non-interactive --config-file mypy.ini - echo '::endgroup::' - - bc: - if: github.event.pull_request - runs-on: ubuntu-latest - steps: - - name: Run BC Lint Action - uses: pytorch/test-infra/.github/actions/bc-lint@main - with: - repo: ${{ github.event.pull_request.head.repo.full_name }} - base_sha: ${{ github.event.pull_request.base.sha }} - head_sha: ${{ github.event.pull_request.head.sha }} diff --git a/.github/workflows/pr-labels.yml b/.github/workflows/pr-labels.yml deleted file mode 100644 index 20c37e4fd88..00000000000 --- a/.github/workflows/pr-labels.yml +++ /dev/null @@ -1,35 +0,0 @@ -name: pr-labels - -on: - push: - branches: - - main - -jobs: - is-properly-labeled: - runs-on: ubuntu-latest - - steps: - - name: Set up python - uses: actions/setup-python@v2 - - - name: Install requests - run: pip install requests - - - name: Checkout repository - uses: actions/checkout@v2 - - - name: Process commit and find merger responsible for labeling - id: commit - run: echo "::set-output name=merger::$(python .github/process_commit.py ${{ github.sha }})" - - - name: Ping merger responsible for labeling if necessary - if: ${{ steps.commit.outputs.merger != '' }} - uses: mshick/add-pr-comment@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - message: | - Hey ${{ steps.commit.outputs.merger }}! - - You merged this PR, but no labels were added. The list of valid labels is available at https://github.com/pytorch/vision/blob/main/.github/process_commit.py diff --git a/.github/workflows/prototype-tests-linux-gpu.yml b/.github/workflows/prototype-tests-linux-gpu.yml deleted file mode 100644 index dee425054d5..00000000000 --- a/.github/workflows/prototype-tests-linux-gpu.yml +++ /dev/null @@ -1,54 +0,0 @@ -name: Prototype tests on Linux - -on: - pull_request: - -jobs: - unittests-prototype: - strategy: - matrix: - python-version: - - "3.8" - - "3.9" - - "3.10" - - "3.11" - runner: ["linux.12xlarge"] - gpu-arch-type: ["cpu"] - include: - - python-version: "3.8" - runner: linux.g5.4xlarge.nvidia.gpu - gpu-arch-type: cuda - gpu-arch-version: "11.7" - fail-fast: false - uses: pytorch/test-infra/.github/workflows/linux_job.yml@main - with: - repository: pytorch/vision - runner: ${{ matrix.runner }} - gpu-arch-type: ${{ matrix.gpu-arch-type }} - gpu-arch-version: ${{ matrix.gpu-arch-version }} - timeout: 120 - script: | - set -euo pipefail - - export PYTHON_VERSION=${{ matrix.python-version }} - export GPU_ARCH_TYPE=${{ matrix.gpu-arch-type }} - export GPU_ARCH_VERSION=${{ matrix.gpu-arch-version }} - ./.github/scripts/setup-env.sh - - # Prepare conda - CONDA_PATH=$(which conda) - eval "$(${CONDA_PATH} shell.bash hook)" - conda activate ci - - echo '::group::Install testing utilities' - pip install --progress-bar=off pytest pytest-mock pytest-cov - echo '::endgroup::' - - # We don't want to run the prototype datasets tests. Since the positional glob into `pytest`, i.e. - # `test/test_prototype*.py` takes the highest priority, neither `--ignore` nor `--ignore-glob` can help us here. - rm test/test_prototype_datasets*.py - pytest \ - -v --durations=25 \ - --cov=torchvision/prototype --cov-report=term-missing \ - --junit-xml="${RUNNER_TEST_RESULTS_DIR}/test-results.xml" \ - test/test_prototype_*.py diff --git a/.github/workflows/tests-schedule.yml b/.github/workflows/tests-schedule.yml deleted file mode 100644 index 5426fdc997a..00000000000 --- a/.github/workflows/tests-schedule.yml +++ /dev/null @@ -1,57 +0,0 @@ -name: tests - -on: - pull_request: - paths: - - "test/test_datasets_download.py" - - ".github/failed_schedule_issue_template.md" - - ".github/workflows/tests-schedule.yml" - - schedule: - - cron: "0 9 * * *" - -jobs: - download: - runs-on: ubuntu-latest - - steps: - - name: Set up python - uses: actions/setup-python@v2 - with: - python-version: 3.8 - - - name: Upgrade system packages - run: python -m pip install --upgrade pip setuptools wheel - - - name: SSL - run: python -c 'import ssl; print(ssl.OPENSSL_VERSION)' - - - name: Checkout repository - uses: actions/checkout@v2 - - - name: Install torch nightly build - run: pip install --pre torch -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html - - - name: Install torchvision - run: pip install --no-build-isolation --editable . - - - name: Install all optional dataset requirements - run: pip install scipy pycocotools lmdb requests - - - name: Install tests requirements - run: pip install pytest - - - name: Run tests - run: pytest -ra -v test/test_datasets_download.py - - - uses: JasonEtco/create-an-issue@v2.4.0 - name: Create issue if download tests failed - if: failure() && github.event_name == 'schedule' - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - REPO: ${{ github.repository }} - WORKFLOW: ${{ github.workflow }} - JOB: ${{ github.job }} - ID: ${{ github.run_id }} - with: - filename: .github/failed_schedule_issue_template.md diff --git a/.github/workflows/update-viablestrict.yml b/.github/workflows/update-viablestrict.yml deleted file mode 100644 index 2d9c2265676..00000000000 --- a/.github/workflows/update-viablestrict.yml +++ /dev/null @@ -1,23 +0,0 @@ -name: Update viable/strict - -on: - pull_request: - paths: - - .github/workflows/update-viablestrict.yml - schedule: - - cron: 10,40 * * * * - workflow_dispatch: - -concurrency: - group: ${{ github.workflow }} - cancel-in-progress: false - -jobs: - do_update_viablestrict: - uses: pytorch/test-infra/.github/workflows/update-viablestrict.yml@main - with: - repository: pytorch/vision - required_checks: "Build Linux,Build M1,Build Macos,Tests on Linux,Tests on macOS,Docs,Lint" - secrets: - ROCKSET_API_KEY: ${{ secrets.ROCKSET_API_KEY }} - GITHUB_DEPLOY_KEY : ${{ secrets.VISION_GITHUB_DEPLOY_KEY }}