From 982f07e0fa90161048b322318864c35900490a88 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Tue, 27 Sep 2022 08:42:02 -0500 Subject: [PATCH] Revert "comment out CI" This reverts commit 704aea40c087df9de6568d203c28d03716a87518. --- .github/workflows/cuda.yml | 224 ++++---- .github/workflows/python_package.yml | 152 +++--- .github/workflows/r_package.yml | 488 ++++++++--------- .vsts-ci.yml | 788 +++++++++++++-------------- 4 files changed, 826 insertions(+), 826 deletions(-) diff --git a/.github/workflows/cuda.yml b/.github/workflows/cuda.yml index 4607799ceaf5..54a7aa1e45eb 100644 --- a/.github/workflows/cuda.yml +++ b/.github/workflows/cuda.yml @@ -1,115 +1,115 @@ -# name: CUDA Version +name: CUDA Version -# on: -# push: -# branches: -# - master -# pull_request: -# branches: -# - master +on: + push: + branches: + - master + pull_request: + branches: + - master -# env: -# github_actions: 'true' -# os_name: linux -# conda_env: test-env +env: + github_actions: 'true' + os_name: linux + conda_env: test-env -# jobs: -# test: -# name: ${{ matrix.task }} ${{ matrix.cuda_version }} ${{ matrix.method }} (linux, ${{ matrix.compiler }}, Python ${{ matrix.python_version }}) -# runs-on: [self-hosted, linux] -# timeout-minutes: 60 -# strategy: -# fail-fast: false -# matrix: -# include: -# - method: source -# compiler: gcc -# python_version: "3.8" -# cuda_version: "11.7.1" -# task: cuda -# - method: pip -# compiler: clang -# python_version: "3.9" -# cuda_version: "10.0" -# task: cuda -# - method: wheel -# compiler: gcc -# python_version: "3.10" -# cuda_version: "9.0" -# task: cuda -# - method: source -# compiler: gcc -# python_version: "3.8" -# cuda_version: "11.7.1" -# task: cuda_exp -# - method: pip -# compiler: clang -# python_version: "3.9" -# cuda_version: "10.0" -# task: cuda_exp -# steps: -# - name: Setup or update software on host machine -# run: | -# sudo apt-get update -# sudo apt-get install --no-install-recommends -y \ -# apt-transport-https \ -# ca-certificates \ -# curl \ -# git \ -# gnupg-agent \ -# lsb-release \ -# software-properties-common -# curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - -# sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" -y -# curl -sL https://nvidia.github.io/nvidia-docker/gpgkey | sudo apt-key add - -# curl -sL https://nvidia.github.io/nvidia-docker/$(. /etc/os-release;echo $ID$VERSION_ID)/nvidia-docker.list | sudo tee /etc/apt/sources.list.d/nvidia-docker.list -# sudo apt-get update -# sudo apt-get install --no-install-recommends -y \ -# containerd.io \ -# docker-ce \ -# docker-ce-cli \ -# nvidia-docker2 -# sudo chmod a+rw /var/run/docker.sock -# sudo systemctl restart docker -# - name: Remove old folder with repository -# run: sudo rm -rf $GITHUB_WORKSPACE -# - name: Checkout repository -# uses: actions/checkout@v1 -# with: -# fetch-depth: 5 -# submodules: true -# - name: Setup and run tests -# run: | -# export ROOT_DOCKER_FOLDER=/LightGBM -# cat > docker.env < docker-script.sh < docker.env < docker-script.sh <> tests.log 2>&1 || exit_code=-1 -# cat ./tests.log -# exit ${exit_code} -# test-r-debian-clang: -# name: r-package (debian, R-devel, clang) -# timeout-minutes: 60 -# runs-on: ubuntu-latest -# container: rhub/debian-clang-devel -# steps: -# - name: Install Git before checkout -# shell: bash -# run: | -# apt-get update --allow-releaseinfo-change -# apt-get install --no-install-recommends -y git -# - name: Trust git cloning LightGBM -# run: | -# git config --global --add safe.directory "${GITHUB_WORKSPACE}" -# - name: Checkout repository -# uses: actions/checkout@v2.4.0 -# with: -# fetch-depth: 5 -# submodules: true -# - name: Install packages and run tests -# shell: bash -# run: | -# export PATH=/opt/R-devel/bin/:${PATH} -# Rscript -e "install.packages(c('R6', 'data.table', 'jsonlite', 'knitr', 'Matrix', 'RhpcBLASctl', 'rmarkdown', 'testthat'), repos = 'https://cran.rstudio.com', Ncpus = parallel::detectCores())" -# sh build-cran-package.sh -# R CMD check --as-cran --run-donttest lightgbm_*.tar.gz || exit -1 -# if grep -q -E "NOTE|WARNING|ERROR" lightgbm.Rcheck/00check.log; then -# echo "NOTEs, WARNINGs, or ERRORs have been found by R CMD check" -# exit -1 -# fi -# all-successful: -# # https://github.uint.cloudmunity/t/is-it-possible-to-require-all-github-actions-tasks-to-pass-without-enumerating-them/117957/4?u=graingert -# runs-on: ubuntu-latest -# needs: [test, test-r-sanitizers, test-r-debian-clang] -# steps: -# - name: Note that all tests succeeded -# run: echo "🎉" +jobs: + test: + name: ${{ matrix.task }} (${{ matrix.os }}, ${{ matrix.compiler }}, R ${{ matrix.r_version }}, ${{ matrix.build_type }}) + runs-on: ${{ matrix.os }} + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: + include: + ################ + # CMake builds # + ################ + - os: ubuntu-latest + task: r-package + compiler: gcc + r_version: 3.6 + build_type: cmake + - os: ubuntu-latest + task: r-package + compiler: gcc + r_version: 4.2 + build_type: cmake + - os: ubuntu-latest + task: r-package + compiler: clang + r_version: 3.6 + build_type: cmake + - os: ubuntu-latest + task: r-package + compiler: clang + r_version: 4.2 + build_type: cmake + - os: macOS-latest + task: r-package + compiler: gcc + r_version: 3.6 + build_type: cmake + - os: macOS-latest + task: r-package + compiler: gcc + r_version: 4.2 + build_type: cmake + - os: macOS-latest + task: r-package + compiler: clang + r_version: 3.6 + build_type: cmake + - os: macOS-latest + task: r-package + compiler: clang + r_version: 4.2 + build_type: cmake + - os: windows-latest + task: r-package + compiler: MINGW + toolchain: MINGW + r_version: 3.6 + build_type: cmake + - os: windows-latest + task: r-package + compiler: MINGW + toolchain: MSYS + r_version: 4.1 + build_type: cmake + # Visual Studio 2019 + - os: windows-2019 + task: r-package + compiler: MSVC + toolchain: MSVC + r_version: 3.6 + build_type: cmake + # Visual Studio 2022 + - os: windows-2022 + task: r-package + compiler: MSVC + toolchain: MSVC + r_version: 4.1 + build_type: cmake + ############### + # CRAN builds # + ############### + - os: windows-latest + task: r-package + compiler: MINGW + toolchain: MINGW + r_version: 3.6 + build_type: cran + - os: windows-latest + task: r-package + compiler: MINGW + toolchain: MSYS + r_version: 4.1 + build_type: cran + - os: ubuntu-latest + task: r-package + compiler: gcc + r_version: 4.2 + build_type: cran + - os: macOS-latest + task: r-package + compiler: clang + r_version: 4.2 + build_type: cran + ################ + # Other checks # + ################ + - os: ubuntu-latest + task: r-rchk + compiler: gcc + r_version: 4.2 + build_type: cran + steps: + - name: Prevent conversion of line endings on Windows + if: startsWith(matrix.os, 'windows') + shell: pwsh + run: git config --global core.autocrlf false + - name: Checkout repository + uses: actions/checkout@v2.4.0 + with: + fetch-depth: 5 + submodules: true + - name: Install pandoc + uses: r-lib/actions/setup-pandoc@v1 + - name: Setup and run tests on Linux and macOS + if: matrix.os == 'macOS-latest' || matrix.os == 'ubuntu-latest' + shell: bash + run: | + export TASK="${{ matrix.task }}" + export COMPILER="${{ matrix.compiler }}" + export GITHUB_ACTIONS="true" + if [[ "${{ matrix.os }}" == "macOS-latest" ]]; then + export OS_NAME="macos" + elif [[ "${{ matrix.os }}" == "ubuntu-latest" ]]; then + export OS_NAME="linux" + fi + export BUILD_DIRECTORY="$GITHUB_WORKSPACE" + export R_VERSION="${{ matrix.r_version }}" + export R_BUILD_TYPE="${{ matrix.build_type }}" + $GITHUB_WORKSPACE/.ci/setup.sh + $GITHUB_WORKSPACE/.ci/test.sh + - name: Setup and run tests on Windows + if: startsWith(matrix.os, 'windows') + shell: pwsh -command ". {0}" + run: | + $env:BUILD_SOURCESDIRECTORY = $env:GITHUB_WORKSPACE + $env:TOOLCHAIN = "${{ matrix.toolchain }}" + $env:R_VERSION = "${{ matrix.r_version }}" + $env:R_BUILD_TYPE = "${{ matrix.build_type }}" + $env:COMPILER = "${{ matrix.compiler }}" + $env:GITHUB_ACTIONS = "true" + $env:TASK = "${{ matrix.task }}" + & "$env:GITHUB_WORKSPACE/.ci/test_windows.ps1" + test-r-sanitizers: + name: r-sanitizers (ubuntu-latest, R-devel, ${{ matrix.compiler }} ASAN/UBSAN) + timeout-minutes: 60 + runs-on: ubuntu-latest + container: wch1/r-debug + strategy: + fail-fast: false + matrix: + include: + - r_customization: san + compiler: gcc + - r_customization: csan + compiler: clang + steps: + - name: Trust git cloning LightGBM + run: | + git config --global --add safe.directory "${GITHUB_WORKSPACE}" + - name: Checkout repository + uses: actions/checkout@v2.4.0 + with: + fetch-depth: 5 + submodules: true + - name: Install packages + shell: bash + run: | + RDscript${{ matrix.r_customization }} -e "install.packages(c('R6', 'data.table', 'jsonlite', 'knitr', 'Matrix', 'RhpcBLASctl', 'rmarkdown', 'testthat'), repos = 'https://cran.rstudio.com', Ncpus = parallel::detectCores())" + sh build-cran-package.sh --r-executable=RD${{ matrix.r_customization }} + RD${{ matrix.r_customization }} CMD INSTALL lightgbm_*.tar.gz || exit -1 + - name: Run tests with sanitizers + shell: bash + run: | + cd R-package/tests + exit_code=0 + RDscript${{ matrix.r_customization }} testthat.R >> tests.log 2>&1 || exit_code=-1 + cat ./tests.log + exit ${exit_code} + test-r-debian-clang: + name: r-package (debian, R-devel, clang) + timeout-minutes: 60 + runs-on: ubuntu-latest + container: rhub/debian-clang-devel + steps: + - name: Install Git before checkout + shell: bash + run: | + apt-get update --allow-releaseinfo-change + apt-get install --no-install-recommends -y git + - name: Trust git cloning LightGBM + run: | + git config --global --add safe.directory "${GITHUB_WORKSPACE}" + - name: Checkout repository + uses: actions/checkout@v2.4.0 + with: + fetch-depth: 5 + submodules: true + - name: Install packages and run tests + shell: bash + run: | + export PATH=/opt/R-devel/bin/:${PATH} + Rscript -e "install.packages(c('R6', 'data.table', 'jsonlite', 'knitr', 'Matrix', 'RhpcBLASctl', 'rmarkdown', 'testthat'), repos = 'https://cran.rstudio.com', Ncpus = parallel::detectCores())" + sh build-cran-package.sh + R CMD check --as-cran --run-donttest lightgbm_*.tar.gz || exit -1 + if grep -q -E "NOTE|WARNING|ERROR" lightgbm.Rcheck/00check.log; then + echo "NOTEs, WARNINGs, or ERRORs have been found by R CMD check" + exit -1 + fi + all-successful: + # https://github.uint.cloudmunity/t/is-it-possible-to-require-all-github-actions-tasks-to-pass-without-enumerating-them/117957/4?u=graingert + runs-on: ubuntu-latest + needs: [test, test-r-sanitizers, test-r-debian-clang] + steps: + - name: Note that all tests succeeded + run: echo "🎉" diff --git a/.vsts-ci.yml b/.vsts-ci.yml index 1a407aa73974..139f72061236 100644 --- a/.vsts-ci.yml +++ b/.vsts-ci.yml @@ -1,395 +1,395 @@ -# trigger: -# branches: -# include: -# - master -# tags: -# include: -# - v* -# pr: -# - master -# variables: -# AZURE: 'true' -# PYTHON_VERSION: '3.10' -# CONDA_ENV: test-env -# runCodesignValidationInjection: false -# skipComponentGovernanceDetection: true -# DOTNET_CLI_TELEMETRY_OPTOUT: true -# DOTNET_SKIP_FIRST_TIME_EXPERIENCE: true -# resources: -# containers: -# - container: ubuntu1404 -# image: lightgbm/vsts-agent:ubuntu-14.04 -# - container: ubuntu-latest -# image: 'ubuntu:20.04' -# options: "--name ci-container -v /usr/bin/docker:/tmp/docker:ro" -# - container: rbase -# image: wch1/r-debug -# jobs: -# ########################################### -# - job: Linux -# ########################################### -# variables: -# COMPILER: gcc -# SETUP_CONDA: 'false' -# OS_NAME: 'linux' -# PRODUCES_ARTIFACTS: 'true' -# pool: sh-ubuntu -# container: ubuntu1404 -# strategy: -# matrix: -# regular: -# TASK: regular -# PYTHON_VERSION: '3.9' -# sdist: -# TASK: sdist -# PYTHON_VERSION: '3.7' -# bdist: -# TASK: bdist -# PYTHON_VERSION: '3.8' -# inference: -# TASK: if-else -# mpi_source: -# TASK: mpi -# METHOD: source -# PYTHON_VERSION: '3.8' -# gpu_source: -# TASK: gpu -# METHOD: source -# swig: -# TASK: swig -# steps: -# - script: | -# echo "##vso[task.setvariable variable=BUILD_DIRECTORY]$BUILD_SOURCESDIRECTORY" -# echo "##vso[task.setvariable variable=LGB_VER]$(head -n 1 VERSION.txt)" -# echo "##vso[task.prependpath]$CONDA/bin" -# displayName: 'Set variables' -# - script: | -# echo '$(Build.SourceVersion)' > '$(Build.ArtifactStagingDirectory)/commit.txt' -# displayName: 'Add commit hash to artifacts archive' -# - bash: $(Build.SourcesDirectory)/.ci/setup.sh -# displayName: Setup -# - bash: $(Build.SourcesDirectory)/.ci/test.sh -# displayName: Test -# - task: PublishBuildArtifacts@1 -# condition: and(succeeded(), in(variables['TASK'], 'regular', 'sdist', 'bdist', 'swig'), not(startsWith(variables['Build.SourceBranch'], 'refs/pull/'))) -# inputs: -# pathtoPublish: '$(Build.ArtifactStagingDirectory)' -# artifactName: PackageAssets -# artifactType: container -# ########################################### -# - job: Linux_latest -# ########################################### -# variables: -# COMPILER: clang -# DEBIAN_FRONTEND: 'noninteractive' -# IN_UBUNTU_LATEST_CONTAINER: 'true' -# OS_NAME: 'linux' -# SETUP_CONDA: 'true' -# pool: sh-ubuntu -# container: ubuntu-latest -# strategy: -# matrix: -# regular: -# TASK: regular -# sdist: -# TASK: sdist -# bdist: -# TASK: bdist -# PYTHON_VERSION: '3.8' -# inference: -# TASK: if-else -# mpi_source: -# TASK: mpi -# METHOD: source -# mpi_pip: -# TASK: mpi -# METHOD: pip -# PYTHON_VERSION: '3.9' -# mpi_wheel: -# TASK: mpi -# METHOD: wheel -# PYTHON_VERSION: '3.7' -# gpu_source: -# TASK: gpu -# METHOD: source -# PYTHON_VERSION: '3.9' -# gpu_pip: -# TASK: gpu -# METHOD: pip -# PYTHON_VERSION: '3.8' -# gpu_wheel: -# TASK: gpu -# METHOD: wheel -# PYTHON_VERSION: '3.7' -# cpp_tests: -# TASK: cpp-tests -# METHOD: with-sanitizers -# steps: -# - script: | -# echo "##vso[task.setvariable variable=BUILD_DIRECTORY]$BUILD_SOURCESDIRECTORY" -# echo "##vso[task.setvariable variable=LGB_VER]$(head -n 1 VERSION.txt)" -# CONDA=$HOME/miniforge -# echo "##vso[task.setvariable variable=CONDA]$CONDA" -# echo "##vso[task.prependpath]$CONDA/bin" -# displayName: 'Set variables' -# # https://github.com/microsoft/azure-pipelines-agent/issues/2043#issuecomment-687983301 -# - script: | -# /tmp/docker exec -t -u 0 ci-container \ -# sh -c "apt-get update && apt-get -o Dpkg::Options::="--force-confold" -y install sudo" -# displayName: 'Install sudo' -# - bash: $(Build.SourcesDirectory)/.ci/setup.sh -# displayName: Setup -# - bash: $(Build.SourcesDirectory)/.ci/test.sh -# displayName: Test -# ########################################### -# - job: QEMU_multiarch -# ########################################### -# variables: -# COMPILER: gcc -# OS_NAME: 'linux' -# PRODUCES_ARTIFACTS: 'true' -# pool: -# vmImage: ubuntu-latest -# timeoutInMinutes: 180 -# strategy: -# matrix: -# bdist: -# TASK: bdist -# ARCH: aarch64 -# steps: -# - script: | -# sudo apt-get update -# sudo apt-get install --no-install-recommends -y \ -# binfmt-support \ -# qemu \ -# qemu-user \ -# qemu-user-static -# displayName: 'Install QEMU' -# - script: | -# docker run --rm --privileged multiarch/qemu-user-static --reset -p yes -# displayName: 'Enable Docker multi-architecture support' -# - script: | -# export ROOT_DOCKER_FOLDER=/LightGBM -# cat > docker.env < docker-script.sh < '$(Build.ArtifactStagingDirectory)/commit.txt' + displayName: 'Add commit hash to artifacts archive' + - bash: $(Build.SourcesDirectory)/.ci/setup.sh + displayName: Setup + - bash: $(Build.SourcesDirectory)/.ci/test.sh + displayName: Test + - task: PublishBuildArtifacts@1 + condition: and(succeeded(), in(variables['TASK'], 'regular', 'sdist', 'bdist', 'swig'), not(startsWith(variables['Build.SourceBranch'], 'refs/pull/'))) + inputs: + pathtoPublish: '$(Build.ArtifactStagingDirectory)' + artifactName: PackageAssets + artifactType: container +########################################### +- job: Linux_latest +########################################### + variables: + COMPILER: clang + DEBIAN_FRONTEND: 'noninteractive' + IN_UBUNTU_LATEST_CONTAINER: 'true' + OS_NAME: 'linux' + SETUP_CONDA: 'true' + pool: sh-ubuntu + container: ubuntu-latest + strategy: + matrix: + regular: + TASK: regular + sdist: + TASK: sdist + bdist: + TASK: bdist + PYTHON_VERSION: '3.8' + inference: + TASK: if-else + mpi_source: + TASK: mpi + METHOD: source + mpi_pip: + TASK: mpi + METHOD: pip + PYTHON_VERSION: '3.9' + mpi_wheel: + TASK: mpi + METHOD: wheel + PYTHON_VERSION: '3.7' + gpu_source: + TASK: gpu + METHOD: source + PYTHON_VERSION: '3.9' + gpu_pip: + TASK: gpu + METHOD: pip + PYTHON_VERSION: '3.8' + gpu_wheel: + TASK: gpu + METHOD: wheel + PYTHON_VERSION: '3.7' + cpp_tests: + TASK: cpp-tests + METHOD: with-sanitizers + steps: + - script: | + echo "##vso[task.setvariable variable=BUILD_DIRECTORY]$BUILD_SOURCESDIRECTORY" + echo "##vso[task.setvariable variable=LGB_VER]$(head -n 1 VERSION.txt)" + CONDA=$HOME/miniforge + echo "##vso[task.setvariable variable=CONDA]$CONDA" + echo "##vso[task.prependpath]$CONDA/bin" + displayName: 'Set variables' + # https://github.com/microsoft/azure-pipelines-agent/issues/2043#issuecomment-687983301 + - script: | + /tmp/docker exec -t -u 0 ci-container \ + sh -c "apt-get update && apt-get -o Dpkg::Options::="--force-confold" -y install sudo" + displayName: 'Install sudo' + - bash: $(Build.SourcesDirectory)/.ci/setup.sh + displayName: Setup + - bash: $(Build.SourcesDirectory)/.ci/test.sh + displayName: Test +########################################### +- job: QEMU_multiarch +########################################### + variables: + COMPILER: gcc + OS_NAME: 'linux' + PRODUCES_ARTIFACTS: 'true' + pool: + vmImage: ubuntu-latest + timeoutInMinutes: 180 + strategy: + matrix: + bdist: + TASK: bdist + ARCH: aarch64 + steps: + - script: | + sudo apt-get update + sudo apt-get install --no-install-recommends -y \ + binfmt-support \ + qemu \ + qemu-user \ + qemu-user-static + displayName: 'Install QEMU' + - script: | + docker run --rm --privileged multiarch/qemu-user-static --reset -p yes + displayName: 'Enable Docker multi-architecture support' + - script: | + export ROOT_DOCKER_FOLDER=/LightGBM + cat > docker.env < docker-script.sh <