From 4479de4b4a375e792127ddcbcb5f91e3b58ce237 Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Thu, 10 Oct 2019 17:04:44 +0200 Subject: [PATCH 1/5] ci: extract uploading artifacts into a script --- src/ci/azure-pipelines/steps/run.yml | 44 +++++++++------------------- src/ci/scripts/upload-artifacts.sh | 36 +++++++++++++++++++++++ src/ci/shared.sh | 4 +++ 3 files changed, 54 insertions(+), 30 deletions(-) create mode 100755 src/ci/scripts/upload-artifacts.sh diff --git a/src/ci/azure-pipelines/steps/run.yml b/src/ci/azure-pipelines/steps/run.yml index 01858cca50b1c..c82bc6af738a0 100644 --- a/src/ci/azure-pipelines/steps/run.yml +++ b/src/ci/azure-pipelines/steps/run.yml @@ -201,37 +201,21 @@ steps: condition: and(succeeded(), not(variables.SKIP_JOB)) displayName: Run build -# If we're a deploy builder, use the `aws` command to publish everything to our -# bucket. -- bash: | - set -e - source src/ci/shared.sh - if [ "$AGENT_OS" = "Linux" ]; then - rm -rf obj/build/dist/doc - upload_dir=obj/build/dist - else - rm -rf build/dist/doc - upload_dir=build/dist - fi - ls -la $upload_dir - deploy_dir=rustc-builds - if [ "$DEPLOY_ALT" == "1" ]; then - deploy_dir=rustc-builds-alt - fi - retry aws s3 cp --no-progress --recursive --acl public-read ./$upload_dir s3://$DEPLOY_BUCKET/$deploy_dir/$BUILD_SOURCEVERSION +- bash: src/ci/scripts/upload-artifacts.sh env: AWS_ACCESS_KEY_ID: $(UPLOAD_AWS_ACCESS_KEY_ID) AWS_SECRET_ACCESS_KEY: $(UPLOAD_AWS_SECRET_ACCESS_KEY) - condition: and(succeeded(), not(variables.SKIP_JOB), or(eq(variables.DEPLOY, '1'), eq(variables.DEPLOY_ALT, '1'))) displayName: Upload artifacts - -# Upload CPU usage statistics that we've been gathering this whole time. Always -# execute this step in case we want to inspect failed builds, but don't let -# errors here ever fail the build since this is just informational. -- bash: aws s3 cp --acl public-read cpu-usage.csv s3://$DEPLOY_BUCKET/rustc-builds/$BUILD_SOURCEVERSION/cpu-$CI_JOB_NAME.csv - env: - AWS_ACCESS_KEY_ID: $(UPLOAD_AWS_ACCESS_KEY_ID) - AWS_SECRET_ACCESS_KEY: $(UPLOAD_AWS_SECRET_ACCESS_KEY) - condition: variables['UPLOAD_AWS_SECRET_ACCESS_KEY'] - continueOnError: true - displayName: Upload CPU usage statistics + # Adding a condition on DEPLOY=1 or DEPLOY_ALT=1 is not needed as all deploy + # builders *should* have the AWS credentials available. Still, explicitly + # adding the condition is helpful as this way CI will not silently skip + # deploying artifacts from a dist builder if the variables are misconfigured, + # erroring about invalid credentials instead. + condition: | + and( + succeeded(), not(variables.SKIP_JOB), + or( + variables.UPLOAD_AWS_SECRET_ACCESS_KEY, + eq(variables.DEPLOY, '1'), eq(variables.DEPLOY_ALT, '1') + ) + ) diff --git a/src/ci/scripts/upload-artifacts.sh b/src/ci/scripts/upload-artifacts.sh new file mode 100755 index 0000000000000..2d24fc53a9505 --- /dev/null +++ b/src/ci/scripts/upload-artifacts.sh @@ -0,0 +1,36 @@ +#!/bin/bash +# Upload all the artifacts to our S3 bucket. All the files inside ${upload_dir} +# will be uploaded to the deploy bucket and eventually signed and released in +# static.rust-lang.org. + +set -euo pipefail +IFS=$'\n\t' + +source "$(cd "$(dirname "$0")" && pwd)/../shared.sh" + +upload_dir="$(mktemp -d)" + +# Release tarballs produced by a dist builder. +if [[ "${DEPLOY-0}" = "1" ]] || [[ "${DEPLOY_ALT-0}" = "1" ]]; then + dist_dir=build/dist + if isLinux; then + dist_dir=obj/build/dist + fi + rm -rf "${dist_dir}/doc" + cp -r "${dist_dir}"/* "${upload_dir}" +fi + +# CPU usage statistics. +cp cpu-usage.csv "${upload_dir}/cpu-${CI_JOB_NAME}.csv" + +echo "Files that will be uploaded:" +ls -lah "${upload_dir}" +echo + +deploy_dir="rustc-builds" +if [[ "${DEPLOY_ALT-0}" = "1" ]]; then + deploy_dir="rustc-builds-alt" +fi +deploy_url="s3://${DEPLOY_BUCKET}/${deploy_dir}/$(ciCommit)" + +retry aws s3 cp --no-progress --recursive --acl public-read "${upload_dir}" "${deploy_url}" diff --git a/src/ci/shared.sh b/src/ci/shared.sh index 37e45b5639dc9..718a5379ae558 100644 --- a/src/ci/shared.sh +++ b/src/ci/shared.sh @@ -46,6 +46,10 @@ function getCIBranch { echo "$BUILD_SOURCEBRANCHNAME" } +function ciCommit { + echo "${BUILD_SOURCEVERSION}" +} + function ciCommandAddPath { if [[ $# -ne 1 ]]; then echo "usage: $0 " From 02000505c157350496283c86710d5d314999545a Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Thu, 10 Oct 2019 17:32:50 +0200 Subject: [PATCH 2/5] ci: upload toolstates.json to rust-lang-ci2 Uploading the toolstate data for each commit will help our release tooling understand which components are failing, to possibly skip shipping broken tools to users. --- src/ci/azure-pipelines/auto.yml | 2 ++ src/ci/scripts/upload-artifacts.sh | 5 +++++ 2 files changed, 7 insertions(+) diff --git a/src/ci/azure-pipelines/auto.yml b/src/ci/azure-pipelines/auto.yml index 24b07a1b7c950..651b40cdf297a 100644 --- a/src/ci/azure-pipelines/auto.yml +++ b/src/ci/azure-pipelines/auto.yml @@ -140,6 +140,7 @@ jobs: IMAGE: x86_64-gnu-aux x86_64-gnu-tools: IMAGE: x86_64-gnu-tools + DEPLOY_TOOLSTATES_JSON: toolstates-linux.json x86_64-gnu-debug: IMAGE: x86_64-gnu-debug x86_64-gnu-nopt: @@ -264,6 +265,7 @@ jobs: MSYS_BITS: 64 SCRIPT: src/ci/docker/x86_64-gnu-tools/checktools.sh x.py /tmp/toolstates.json windows RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --save-toolstates=/tmp/toolstates.json + DEPLOY_TOOLSTATES_JSON: toolstates-windows.json # 32/64-bit MinGW builds. # diff --git a/src/ci/scripts/upload-artifacts.sh b/src/ci/scripts/upload-artifacts.sh index 2d24fc53a9505..f060dd28f718c 100755 --- a/src/ci/scripts/upload-artifacts.sh +++ b/src/ci/scripts/upload-artifacts.sh @@ -23,6 +23,11 @@ fi # CPU usage statistics. cp cpu-usage.csv "${upload_dir}/cpu-${CI_JOB_NAME}.csv" +# Toolstate data. +if [[ ! -z "${DEPLOY_TOOLSTATES_JSON+x}" ]]; then + cp /tmp/toolstates.json "${upload_dir}/${DEPLOY_TOOLSTATES_JSON}" +fi + echo "Files that will be uploaded:" ls -lah "${upload_dir}" echo From bdfcde439bcea75d74fe3b181d13332384daf130 Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Tue, 29 Oct 2019 10:14:42 +0100 Subject: [PATCH 3/5] Apply suggestions from lzutao Co-Authored-By: lzutao --- src/ci/scripts/upload-artifacts.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/ci/scripts/upload-artifacts.sh b/src/ci/scripts/upload-artifacts.sh index f060dd28f718c..625cf378a9be3 100755 --- a/src/ci/scripts/upload-artifacts.sh +++ b/src/ci/scripts/upload-artifacts.sh @@ -11,7 +11,7 @@ source "$(cd "$(dirname "$0")" && pwd)/../shared.sh" upload_dir="$(mktemp -d)" # Release tarballs produced by a dist builder. -if [[ "${DEPLOY-0}" = "1" ]] || [[ "${DEPLOY_ALT-0}" = "1" ]]; then +if [[ "${DEPLOY-0}" -eq "1" ]] || [[ "${DEPLOY_ALT-0}" -eq "1" ]]; then dist_dir=build/dist if isLinux; then dist_dir=obj/build/dist @@ -24,7 +24,7 @@ fi cp cpu-usage.csv "${upload_dir}/cpu-${CI_JOB_NAME}.csv" # Toolstate data. -if [[ ! -z "${DEPLOY_TOOLSTATES_JSON+x}" ]]; then +if [[ -n "${DEPLOY_TOOLSTATES_JSON+x}" ]]; then cp /tmp/toolstates.json "${upload_dir}/${DEPLOY_TOOLSTATES_JSON}" fi @@ -33,7 +33,7 @@ ls -lah "${upload_dir}" echo deploy_dir="rustc-builds" -if [[ "${DEPLOY_ALT-0}" = "1" ]]; then +if [[ "${DEPLOY_ALT-0}" -eq "1" ]]; then deploy_dir="rustc-builds-alt" fi deploy_url="s3://${DEPLOY_BUCKET}/${deploy_dir}/$(ciCommit)" From 518497752442ed208d8624e15cbc9febdeaf0dff Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Wed, 30 Oct 2019 10:35:39 +0100 Subject: [PATCH 4/5] ci: move toolstates.json to /tmp/toolstate/ and docker mount it Before this commit toolstates.json was stored in /tmp and it wasn't mounted outside the build container. That caused uploading the file in the upload-artifacts task to fail, as the file was missing on the host. Mounting /tmp/toolstates.json alone is not the best approach: if the file is missing when the container is started the Docker engine will create a *directory* named /tmp/toolstates.json. The Docker issue could be solved by pre-creating an empty file named /tmp/toolstates.json, but doing that could cause problems if bootstrap fails to generate the file and the toolstate scripts receive an empty JSON. The approach I took in this commit is to instead mount a /tmp/toolstate directory inside Docker, and create the toolstates.json file in it. That also required a small bootstrap change to ensure the directory is created if it's missing. --- src/bootstrap/lib.rs | 4 ++++ src/ci/azure-pipelines/auto.yml | 4 ++-- src/ci/docker/run.sh | 2 ++ src/ci/docker/x86_64-gnu-tools/Dockerfile | 4 ++-- src/ci/scripts/upload-artifacts.sh | 2 +- 5 files changed, 11 insertions(+), 5 deletions(-) diff --git a/src/bootstrap/lib.rs b/src/bootstrap/lib.rs index cbdb174c02d97..39d7ea922bced 100644 --- a/src/bootstrap/lib.rs +++ b/src/bootstrap/lib.rs @@ -1080,6 +1080,10 @@ impl Build { /// done. The file is updated immediately after this function completes. pub fn save_toolstate(&self, tool: &str, state: ToolState) { if let Some(ref path) = self.config.save_toolstates { + if let Some(parent) = path.parent() { + // Ensure the parent directory always exists + t!(std::fs::create_dir_all(parent)); + } let mut file = t!(fs::OpenOptions::new() .create(true) .read(true) diff --git a/src/ci/azure-pipelines/auto.yml b/src/ci/azure-pipelines/auto.yml index 651b40cdf297a..271c32585449e 100644 --- a/src/ci/azure-pipelines/auto.yml +++ b/src/ci/azure-pipelines/auto.yml @@ -263,8 +263,8 @@ jobs: # MSVC tools tests x86_64-msvc-tools: MSYS_BITS: 64 - SCRIPT: src/ci/docker/x86_64-gnu-tools/checktools.sh x.py /tmp/toolstates.json windows - RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --save-toolstates=/tmp/toolstates.json + SCRIPT: src/ci/docker/x86_64-gnu-tools/checktools.sh x.py /tmp/toolstate/toolstates.json windows + RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --save-toolstates=/tmp/toolstate/toolstates.json DEPLOY_TOOLSTATES_JSON: toolstates-windows.json # 32/64-bit MinGW builds. diff --git a/src/ci/docker/run.sh b/src/ci/docker/run.sh index 415d6b63eb8dc..cdafcbadc9ec7 100755 --- a/src/ci/docker/run.sh +++ b/src/ci/docker/run.sh @@ -106,6 +106,7 @@ fi mkdir -p $HOME/.cargo mkdir -p $objdir/tmp mkdir -p $objdir/cores +mkdir -p /tmp/toolstate args= if [ "$SCCACHE_BUCKET" != "" ]; then @@ -156,6 +157,7 @@ else args="$args --volume $objdir:/checkout/obj" args="$args --volume $HOME/.cargo:/cargo" args="$args --volume $HOME/rustsrc:$HOME/rustsrc" + args="$args --volume /tmp/toolstate:/tmp/toolstate" args="$args --env LOCAL_USER_ID=`id -u`" fi diff --git a/src/ci/docker/x86_64-gnu-tools/Dockerfile b/src/ci/docker/x86_64-gnu-tools/Dockerfile index 8035195c6ed0a..7687a6ca23e18 100644 --- a/src/ci/docker/x86_64-gnu-tools/Dockerfile +++ b/src/ci/docker/x86_64-gnu-tools/Dockerfile @@ -26,5 +26,5 @@ ENV CHECK_LINKS 1 ENV RUST_CONFIGURE_ARGS \ --build=x86_64-unknown-linux-gnu \ - --save-toolstates=/tmp/toolstates.json -ENV SCRIPT /tmp/checktools.sh ../x.py /tmp/toolstates.json linux + --save-toolstates=/tmp/toolstate/toolstates.json +ENV SCRIPT /tmp/checktools.sh ../x.py /tmp/toolstate/toolstates.json linux diff --git a/src/ci/scripts/upload-artifacts.sh b/src/ci/scripts/upload-artifacts.sh index 625cf378a9be3..312ec9d805012 100755 --- a/src/ci/scripts/upload-artifacts.sh +++ b/src/ci/scripts/upload-artifacts.sh @@ -25,7 +25,7 @@ cp cpu-usage.csv "${upload_dir}/cpu-${CI_JOB_NAME}.csv" # Toolstate data. if [[ -n "${DEPLOY_TOOLSTATES_JSON+x}" ]]; then - cp /tmp/toolstates.json "${upload_dir}/${DEPLOY_TOOLSTATES_JSON}" + cp /tmp/toolstate/toolstates.json "${upload_dir}/${DEPLOY_TOOLSTATES_JSON}" fi echo "Files that will be uploaded:" From 3c6679abd7db0cb4fd57774866503c5117d171c4 Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Wed, 30 Oct 2019 10:52:34 +0100 Subject: [PATCH 5/5] DO NOT MERGE: enable tools builders on try --- src/ci/azure-pipelines/try.yml | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/src/ci/azure-pipelines/try.yml b/src/ci/azure-pipelines/try.yml index c919b1023a0eb..543fec3c21fbb 100644 --- a/src/ci/azure-pipelines/try.yml +++ b/src/ci/azure-pipelines/try.yml @@ -14,13 +14,23 @@ jobs: - template: steps/run.yml strategy: matrix: - dist-x86_64-linux: - IMAGE: dist-x86_64-linux - DEPLOY: 1 + x86_64-gnu-tools: + IMAGE: x86_64-gnu-tools + DEPLOY_TOOLSTATES_JSON: toolstates-linux.json - dist-x86_64-linux-alt: - IMAGE: dist-x86_64-linux - DEPLOY_ALT: 1 +- job: Windows + timeoutInMinutes: 600 + pool: + vmImage: 'vs2017-win2016' + steps: + - template: steps/run.yml + strategy: + matrix: + x86_64-msvc-tools: + MSYS_BITS: 64 + SCRIPT: src/ci/docker/x86_64-gnu-tools/checktools.sh x.py /tmp/toolstate/toolstates.json windows + RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --save-toolstates=/tmp/toolstate/toolstates.json + DEPLOY_TOOLSTATES_JSON: toolstates-windows.json # The macOS and Windows builds here are currently disabled due to them not being # overly necessary on `try` builds. We also don't actually have anything that