rustbuild: Add sccache support

This commit adds support for sccache, a ccache-like compiler which works on MSVC
and stores results into an S3 bucket. This also switches over all Travis and
AppVeyor automation to using sccache to ensure a shared and unified cache over
time which can be shared across builders.

The support for sccache manifests as a new `--enable-sccache` option which
instructs us to configure LLVM differently to use a 'sccache' binary instead of
a 'ccache' binary. All docker images for Travis builds are updated to download
Mozilla's tooltool builds of sccache onto various containers and systems.
Additionally a new `rust-lang-ci-sccache` bucket is configured to hold all of
our ccache goodies.

.travis.yml

@@ -30,25 +30,34 @@ matrix:
         RUST_CONFIGURE_ARGS=--target=x86_64-apple-darwin
         SRC=.
       os: osx
-      install: brew install ccache
+      install: &osx_install_sccache >
+        curl -L https://api.pub.build.mozilla.org/tooltool/sha512/d0025b286468cc5ada83b23d3fafbc936b9f190eaa7d4a981715b18e8e3bf720a7bcee7bfe758cfdeb8268857f6098fd52dcdd8818232692a30ce91039936596 |
+          tar xJf - -C /usr/local/bin --strip-components=1
     - env: >
         RUST_CHECK_TARGET=check
         RUST_CONFIGURE_ARGS=--target=i686-apple-darwin
         SRC=.
       os: osx
-      install: brew install ccache
+      install: *osx_install_sccache
     - env: >
         RUST_CHECK_TARGET=check
         RUST_CONFIGURE_ARGS=--target=x86_64-apple-darwin --disable-rustbuild
         SRC=.
       os: osx
-      install: brew install ccache
+      install: *osx_install_sccache
     - env: >
         RUST_CHECK_TARGET=
         RUST_CONFIGURE_ARGS=--target=aarch64-apple-ios,armv7-apple-ios,armv7s-apple-ios,i386-apple-ios,x86_64-apple-ios
         SRC=.
       os: osx
-      install: brew install ccache
+      install: *osx_install_sccache
+
+env:
+  global:
+    - SCCACHE_BUCKET=rust-lang-ci-sccache
+    - AWS_ACCESS_KEY_ID=AKIAIMX7VLAS3PZAVLUQ
+    # AWS_SECRET_ACCESS_KEY=...
+    - secure: "Pixhh0hXDqGCdOyLtGFjli3J2AtDWIpyb2btIrLe956nCBDRutRoMm6rv5DI9sFZN07Mms7VzNNvhc9wCW1y63JAm414d2Co7Ob8kWMZlz9l9t7ACHuktUiis8yr+S4Quq1Vqd6pqi7pf2J++UxC8R/uLeqVrubzr6+X7AbmEFE="
 
 script:
   - >
@@ -77,5 +86,3 @@ notifications:
 cache:
   directories:
     - $HOME/docker
-    - $HOME/.ccache
-    - $HOME/.cargo

appveyor.yml

@@ -1,4 +1,9 @@
 environment:
+  SCCACHE_BUCKET: rust-lang-ci-sccache
+  AWS_ACCESS_KEY_ID: AKIAIMX7VLAS3PZAVLUQ
+  AWS_SECRET_ACCESS_KEY:
+    secure: 1UkmbiDd15tWtYbMm5O2Uqm0b0Ur8v1MoSlydxl4ojcroPeerRMlUges0l57py8c
+  SCCACHE_DIGEST: f808afabb4a4eb1d7112bcb3fa6be03b61e93412890c88e177c667eb37f46353d7ec294e559b16f9f4b5e894f2185fe7670a0df15fd064889ecbd80f0c34166c
   matrix:
   # 32/64 bit MSVC
   - MSYS_BITS: 64
@@ -84,6 +89,13 @@ install:
   # Otherwise pull in the MinGW installed on appveyor
   - if NOT defined MINGW_URL set PATH=C:\msys64\mingw%MSYS_BITS%\bin;C:\msys64\usr\bin;%PATH%
 
+  # Download and install sccache
+  - appveyor DownloadFile https://api.pub.build.mozilla.org/tooltool/sha512/%SCCACHE_DIGEST%
+  - mv %SCCACHE_DIGEST% sccache.tar.bz2
+  - 7z x -y sccache.tar.bz2 > nul
+  - 7z x -y sccache.tar > nul
+  - set PATH=%PATH%;%CD%\sccache2
+
 test_script:
   - git submodule update --init
   - set SRC=.

configure

@@ -621,6 +621,7 @@ opt llvm-assertions 0 "build LLVM with assertions"
 opt debug-assertions 0 "build with debugging assertions"
 opt fast-make 0 "use .gitmodules as timestamp for submodule deps"
 opt ccache 0 "invoke gcc/clang via ccache to reuse object files between builds"
+opt sccache 0 "invoke gcc/clang via sccache to reuse object files between builds"
 opt local-rust 0 "use an installed rustc rather than downloading a snapshot"
 opt local-rebuild 0 "assume local-rust matches the current version, for rebuilds; implies local-rust, and is implied if local-rust already matches the current version"
 opt llvm-static-stdcpp 0 "statically link to libstdc++ for LLVM"
@@ -1677,11 +1678,23 @@ do
             LLVM_CC_64_ARG1="gcc"
             ;;
         ("gcc")
-            LLVM_CXX_32="g++"
-            LLVM_CC_32="gcc"
+            if [ -z "$CFG_ENABLE_SCCACHE" ]; then
+                LLVM_CXX_32="g++"
+                LLVM_CC_32="gcc"
 
-            LLVM_CXX_64="g++"
-            LLVM_CC_64="gcc"
+                LLVM_CXX_64="g++"
+                LLVM_CC_64="gcc"
+            else
+                LLVM_CXX_32="sccache"
+                LLVM_CC_32="sccache"
+                LLVM_CXX_32_ARG1="g++"
+                LLVM_CC_32_ARG1="gcc"
+
+                LLVM_CXX_64="sccache"
+                LLVM_CC_64="sccache"
+                LLVM_CXX_64_ARG1="g++"
+                LLVM_CC_64_ARG1="gcc"
+            fi
             ;;
 
         (*)

src/bootstrap/config.rs

@@ -38,7 +38,7 @@ use util::push_exe_path;
 /// `src/bootstrap/config.toml.example`.
 #[derive(Default)]
 pub struct Config {
-    pub ccache: bool,
+    pub ccache: Option<String>,
     pub ninja: bool,
     pub verbose: bool,
     pub submodules: bool,
@@ -138,7 +138,7 @@ struct Build {
 /// TOML representation of how the LLVM build is configured.
 #[derive(RustcDecodable, Default)]
 struct Llvm {
-    ccache: Option<bool>,
+    ccache: Option<StringOrBool>,
     ninja: Option<bool>,
     assertions: Option<bool>,
     optimize: Option<bool>,
@@ -147,6 +147,18 @@ struct Llvm {
     static_libstdcpp: Option<bool>,
 }
 
+#[derive(RustcDecodable)]
+enum StringOrBool {
+    String(String),
+    Bool(bool),
+}
+
+impl Default for StringOrBool {
+    fn default() -> StringOrBool {
+        StringOrBool::Bool(false)
+    }
+}
+
 /// TOML representation of how the Rust build is configured.
 #[derive(RustcDecodable, Default)]
 struct Rust {
@@ -247,7 +259,15 @@ impl Config {
         set(&mut config.vendor, build.vendor);
 
         if let Some(ref llvm) = toml.llvm {
-            set(&mut config.ccache, llvm.ccache);
+            match llvm.ccache {
+                Some(StringOrBool::String(ref s)) => {
+                    config.ccache = Some(s.to_string())
+                }
+                Some(StringOrBool::Bool(true)) => {
+                    config.ccache = Some("ccache".to_string());
+                }
+                Some(StringOrBool::Bool(false)) | None => {}
+            }
             set(&mut config.ninja, llvm.ninja);
             set(&mut config.llvm_assertions, llvm.assertions);
             set(&mut config.llvm_optimize, llvm.optimize);
@@ -338,7 +358,6 @@ impl Config {
             }
 
             check! {
-                ("CCACHE", self.ccache),
                 ("MANAGE_SUBMODULES", self.submodules),
                 ("COMPILER_DOCS", self.compiler_docs),
                 ("DOCS", self.docs),
@@ -475,6 +494,12 @@ impl Config {
                     let path = parse_configure_path(value);
                     self.python = Some(path);
                 }
+                "CFG_ENABLE_CCACHE" if value == "1" => {
+                    self.ccache = Some("ccache".to_string());
+                }
+                "CFG_ENABLE_SCCACHE" if value == "1" => {
+                    self.ccache = Some("sccache".to_string());
+                }
                 _ => {}
             }
         }

src/bootstrap/config.toml.example

@@ -25,6 +25,8 @@
 
 # Indicates whether ccache is used when building LLVM
 #ccache = false
+# or alternatively ...
+#ccache = "/path/to/ccache"
 
 # If an external LLVM root is specified, we automatically check the version by
 # default to make sure it's within the range that we're expecting, but setting

src/bootstrap/native.rs

@@ -109,10 +109,10 @@ pub fn llvm(build: &Build, target: &str) {
 
     // MSVC handles compiler business itself
     if !target.contains("msvc") {
-        if build.config.ccache {
-           cfg.define("CMAKE_C_COMPILER", "ccache")
+        if let Some(ref ccache) = build.config.ccache {
+           cfg.define("CMAKE_C_COMPILER", ccache)
               .define("CMAKE_C_COMPILER_ARG1", build.cc(target))
-              .define("CMAKE_CXX_COMPILER", "ccache")
+              .define("CMAKE_CXX_COMPILER", ccache)
               .define("CMAKE_CXX_COMPILER_ARG1", build.cxx(target));
         } else {
            cfg.define("CMAKE_C_COMPILER", build.cc(target))

src/bootstrap/sanity.rs

@@ -223,4 +223,8 @@ $ pacman -R cmake && pacman -S mingw-w64-x86_64-cmake
     if build.lldb_version.is_some() {
         build.lldb_python_dir = run(Command::new("lldb").arg("-P")).ok();
     }
+
+    if let Some(ref s) = build.config.ccache {
+        need_cmd(s.as_ref());
+    }
 }

src/ci/docker/arm-android/Dockerfile

@@ -16,7 +16,8 @@ RUN dpkg --add-architecture i386 && \
   expect \
   openjdk-9-jre \
   sudo \
-  libstdc++6:i386
+  libstdc++6:i386 \
+  xz-utils
 
 WORKDIR /android/
 ENV PATH=$PATH:/android/ndk-arm-9/bin:/android/sdk/tools:/android/sdk/platform-tools
@@ -33,6 +34,10 @@ COPY start-emulator.sh /android/
 
 ENTRYPOINT ["/usr/bin/dumb-init", "--", "/android/start-emulator.sh"]
 
+ENV SCCACHE_DIGEST=7237e38e029342fa27b7ac25412cb9d52554008b12389727320bd533fd7f05b6a96d55485f305caf95e5c8f5f97c3313e10012ccad3e752aba2518f3522ba783
+RUN curl -L https://api.pub.build.mozilla.org/tooltool/sha512/$SCCACHE_DIGEST | \
+      tar xJf - -C /usr/local/bin --strip-components=1
+
 ENV TARGETS=arm-linux-androideabi
 ENV TARGETS=$TARGETS,i686-linux-android
 ENV TARGETS=$TARGETS,aarch64-linux-android

src/ci/docker/cross/Dockerfile

@@ -21,7 +21,12 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
   gcc-powerpc-linux-gnu libc6-dev-powerpc-cross \
   gcc-powerpc64-linux-gnu libc6-dev-ppc64-cross \
   gcc-powerpc64le-linux-gnu libc6-dev-ppc64el-cross \
-  gcc-s390x-linux-gnu libc6-dev-s390x-cross
+  gcc-s390x-linux-gnu libc6-dev-s390x-cross \
+  xz-utils
+
+ENV SCCACHE_DIGEST=7237e38e029342fa27b7ac25412cb9d52554008b12389727320bd533fd7f05b6a96d55485f305caf95e5c8f5f97c3313e10012ccad3e752aba2518f3522ba783
+RUN curl -L https://api.pub.build.mozilla.org/tooltool/sha512/$SCCACHE_DIGEST | \
+      tar xJf - -C /usr/local/bin --strip-components=1
 
 RUN curl -OL https://github.com/Yelp/dumb-init/releases/download/v1.2.0/dumb-init_1.2.0_amd64.deb && \
     dpkg -i dumb-init_*.deb && \

src/ci/docker/i686-gnu-nopt/Dockerfile

@@ -11,7 +11,12 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
   cmake \
   ccache \
   sudo \
-  gdb
+  gdb \
+  xz-utils
+
+ENV SCCACHE_DIGEST=7237e38e029342fa27b7ac25412cb9d52554008b12389727320bd533fd7f05b6a96d55485f305caf95e5c8f5f97c3313e10012ccad3e752aba2518f3522ba783
+RUN curl -L https://api.pub.build.mozilla.org/tooltool/sha512/$SCCACHE_DIGEST | \
+      tar xJf - -C /usr/local/bin --strip-components=1
 
 RUN curl -OL https://github.com/Yelp/dumb-init/releases/download/v1.2.0/dumb-init_1.2.0_amd64.deb && \
     dpkg -i dumb-init_*.deb && \

src/ci/docker/i686-gnu/Dockerfile

@@ -11,7 +11,12 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
   cmake \
   ccache \
   sudo \
-  gdb
+  gdb \
+  xz-utils
+
+ENV SCCACHE_DIGEST=7237e38e029342fa27b7ac25412cb9d52554008b12389727320bd533fd7f05b6a96d55485f305caf95e5c8f5f97c3313e10012ccad3e752aba2518f3522ba783
+RUN curl -L https://api.pub.build.mozilla.org/tooltool/sha512/$SCCACHE_DIGEST | \
+      tar xJf - -C /usr/local/bin --strip-components=1
 
 RUN curl -OL https://github.com/Yelp/dumb-init/releases/download/v1.2.0/dumb-init_1.2.0_amd64.deb && \
     dpkg -i dumb-init_*.deb && \

src/ci/docker/run.sh

@@ -25,7 +25,6 @@ docker \
   -t rust-ci \
   "`dirname "$script"`/$image"
 
-mkdir -p $HOME/.ccache
 mkdir -p $HOME/.cargo
 mkdir -p $root_dir/obj
 
@@ -35,8 +34,9 @@ exec docker \
   --volume "$root_dir/obj:/checkout/obj" \
   --workdir /checkout/obj \
   --env SRC=/checkout \
-  --env CCACHE_DIR=/ccache \
-  --volume "$HOME/.ccache:/ccache" \
+  --env SCCACHE_BUCKET=$SCCACHE_BUCKET \
+  --env AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \
+  --env AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \
   --env CARGO_HOME=/cargo \
   --env LOCAL_USER_ID=`id -u` \
   --volume "$HOME/.cargo:/cargo" \

src/ci/docker/x86_64-freebsd/Dockerfile

@@ -23,6 +23,10 @@ RUN curl -OL https://github.com/Yelp/dumb-init/releases/download/v1.2.0/dumb-ini
     rm dumb-init_*.deb
 ENTRYPOINT ["/usr/bin/dumb-init", "--"]
 
+ENV SCCACHE_DIGEST=7237e38e029342fa27b7ac25412cb9d52554008b12389727320bd533fd7f05b6a96d55485f305caf95e5c8f5f97c3313e10012ccad3e752aba2518f3522ba783
+RUN curl -L https://api.pub.build.mozilla.org/tooltool/sha512/$SCCACHE_DIGEST | \
+      tar xJf - -C /usr/local/bin --strip-components=1
+
 ENV \
     AR_x86_64_unknown_freebsd=x86_64-unknown-freebsd10-ar \
     CC_x86_64_unknown_freebsd=x86_64-unknown-freebsd10-gcc

src/ci/docker/x86_64-gnu-cargotest/Dockerfile

@@ -11,7 +11,12 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
   cmake \
   ccache \
   libssl-dev \
-  sudo
+  sudo \
+  xz-utils
+
+ENV SCCACHE_DIGEST=7237e38e029342fa27b7ac25412cb9d52554008b12389727320bd533fd7f05b6a96d55485f305caf95e5c8f5f97c3313e10012ccad3e752aba2518f3522ba783
+RUN curl -L https://api.pub.build.mozilla.org/tooltool/sha512/$SCCACHE_DIGEST | \
+      tar xJf - -C /usr/local/bin --strip-components=1
 
 RUN curl -OL https://github.com/Yelp/dumb-init/releases/download/v1.2.0/dumb-init_1.2.0_amd64.deb && \
     dpkg -i dumb-init_*.deb && \

src/ci/docker/x86_64-gnu-debug/Dockerfile

@@ -11,7 +11,12 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
   cmake \
   ccache \
   sudo \
-  gdb
+  gdb \
+  xz-utils
+
+ENV SCCACHE_DIGEST=7237e38e029342fa27b7ac25412cb9d52554008b12389727320bd533fd7f05b6a96d55485f305caf95e5c8f5f97c3313e10012ccad3e752aba2518f3522ba783
+RUN curl -L https://api.pub.build.mozilla.org/tooltool/sha512/$SCCACHE_DIGEST | \
+      tar xJf - -C /usr/local/bin --strip-components=1
 
 RUN curl -OL https://github.com/Yelp/dumb-init/releases/download/v1.2.0/dumb-init_1.2.0_amd64.deb && \
     dpkg -i dumb-init_*.deb && \

src/ci/docker/x86_64-gnu-llvm-3.7/Dockerfile

@@ -14,7 +14,12 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
   gdb \
   llvm-3.7-tools \
   libedit-dev \
-  zlib1g-dev
+  zlib1g-dev \
+  xz-utils
+
+ENV SCCACHE_DIGEST=7237e38e029342fa27b7ac25412cb9d52554008b12389727320bd533fd7f05b6a96d55485f305caf95e5c8f5f97c3313e10012ccad3e752aba2518f3522ba783
+RUN curl -L https://api.pub.build.mozilla.org/tooltool/sha512/$SCCACHE_DIGEST | \
+      tar xJf - -C /usr/local/bin --strip-components=1
 
 RUN curl -OL https://github.com/Yelp/dumb-init/releases/download/v1.2.0/dumb-init_1.2.0_amd64.deb && \
     dpkg -i dumb-init_*.deb && \

src/ci/docker/x86_64-gnu-make/Dockerfile

@@ -11,7 +11,12 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
   cmake \
   ccache \
   sudo \
-  gdb
+  gdb \
+  xz-utils
+
+ENV SCCACHE_DIGEST=7237e38e029342fa27b7ac25412cb9d52554008b12389727320bd533fd7f05b6a96d55485f305caf95e5c8f5f97c3313e10012ccad3e752aba2518f3522ba783
+RUN curl -L https://api.pub.build.mozilla.org/tooltool/sha512/$SCCACHE_DIGEST | \
+      tar xJf - -C /usr/local/bin --strip-components=1
 
 RUN curl -OL https://github.com/Yelp/dumb-init/releases/download/v1.2.0/dumb-init_1.2.0_amd64.deb && \
     dpkg -i dumb-init_*.deb && \

src/ci/docker/x86_64-gnu-nopt/Dockerfile

@@ -11,7 +11,12 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
   cmake \
   ccache \
   sudo \
-  gdb
+  gdb \
+  xz-utils
+
+ENV SCCACHE_DIGEST=7237e38e029342fa27b7ac25412cb9d52554008b12389727320bd533fd7f05b6a96d55485f305caf95e5c8f5f97c3313e10012ccad3e752aba2518f3522ba783
+RUN curl -L https://api.pub.build.mozilla.org/tooltool/sha512/$SCCACHE_DIGEST | \
+      tar xJf - -C /usr/local/bin --strip-components=1
 
 RUN curl -OL https://github.com/Yelp/dumb-init/releases/download/v1.2.0/dumb-init_1.2.0_amd64.deb && \
     dpkg -i dumb-init_*.deb && \

src/ci/docker/x86_64-gnu/Dockerfile

@@ -11,7 +11,12 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
   cmake \
   ccache \
   sudo \
-  gdb
+  gdb \
+  xz-utils
+
+ENV SCCACHE_DIGEST=7237e38e029342fa27b7ac25412cb9d52554008b12389727320bd533fd7f05b6a96d55485f305caf95e5c8f5f97c3313e10012ccad3e752aba2518f3522ba783
+RUN curl -L https://api.pub.build.mozilla.org/tooltool/sha512/$SCCACHE_DIGEST | \
+      tar xJf - -C /usr/local/bin --strip-components=1
 
 RUN curl -OL https://github.com/Yelp/dumb-init/releases/download/v1.2.0/dumb-init_1.2.0_amd64.deb && \
     dpkg -i dumb-init_*.deb && \