Rollup merge of rust-lang#40998 - alexcrichton:split-dist, r=TimNN

travis: Split all dist builders in two

Previously we would use one builder on Travis to produce two sets of host
compilers for two different targets. Unfortunately though we've recently
increased how much we're building for each target so this is starting to take
unnecessarily long (rust-lang#40804). This commit splits the dist builders in two by
ensuring that we only dist one target on each builder, which should take a much
shorter amount of time. This should also unblock other work such as landing the
RLS (rust-lang#40584).

.travis.yml

@@ -15,19 +15,27 @@ matrix:
     - env: IMAGE=arm-android
     - env: IMAGE=armhf-gnu
     - env: IMAGE=cross DEPLOY=1
+    - env: IMAGE=dist-aarch64-linux DEPLOY=1
     - env: IMAGE=dist-android DEPLOY=1
     - env: IMAGE=dist-arm-linux DEPLOY=1
-    - env: IMAGE=dist-armv7-aarch64-linux DEPLOY=1
-    - env: IMAGE=dist-freebsd DEPLOY=1
-    - env: IMAGE=dist-i586-gnu-i686-musl DEPLOY=1
+    - env: IMAGE=dist-armhf-linux DEPLOY=1
+    - env: IMAGE=dist-armv7-linux DEPLOY=1
     - env: IMAGE=dist-fuchsia DEPLOY=1
+    - env: IMAGE=dist-i586-gnu-i686-musl DEPLOY=1
+    - env: IMAGE=dist-i686-freebsd DEPLOY=1
+    - env: IMAGE=dist-i686-linux DEPLOY=1
     - env: IMAGE=dist-mips-linux DEPLOY=1
     - env: IMAGE=dist-mips64-linux DEPLOY=1
+    - env: IMAGE=dist-mips64el-linux DEPLOY=1
+    - env: IMAGE=dist-mipsel-linux DEPLOY=1
     - env: IMAGE=dist-powerpc-linux DEPLOY=1
     - env: IMAGE=dist-powerpc64-linux DEPLOY=1
-    - env: IMAGE=dist-s390x-linux-netbsd DEPLOY=1
-    - env: IMAGE=dist-x86-linux DEPLOY=1
+    - env: IMAGE=dist-powerpc64le-linux DEPLOY=1
+    - env: IMAGE=dist-s390x-linux DEPLOY=1
+    - env: IMAGE=dist-x86_64-freebsd DEPLOY=1
+    - env: IMAGE=dist-x86_64-linux DEPLOY=1
     - env: IMAGE=dist-x86_64-musl DEPLOY=1
+    - env: IMAGE=dist-x86_64-netbsd DEPLOY=1
     - env: IMAGE=emscripten
     - env: IMAGE=i686-gnu
     - env: IMAGE=i686-gnu-nopt

src/ci/docker/dist-armv7-aarch64-linux/Dockerfile → src/ci/docker/dist-aarch64-linux/Dockerfile

@@ -56,8 +56,7 @@ RUN mkdir /x-tools && chown rustbuild:rustbuild /x-tools
 USER rustbuild
 WORKDIR /tmp
 
-COPY armv7-linux-gnueabihf.config /tmp/
-COPY armv7-linux-gnueabihf.config aarch64-linux-gnu.config build-toolchains.sh /tmp/
+COPY aarch64-linux-gnu.config build-toolchains.sh /tmp/
 RUN ./build-toolchains.sh
 
 USER root
@@ -67,17 +66,12 @@ RUN curl -o /usr/local/bin/sccache \
       chmod +x /usr/local/bin/sccache
 
 ENV PATH=$PATH:/x-tools/aarch64-unknown-linux-gnueabi/bin
-ENV PATH=$PATH:/x-tools/armv7-unknown-linux-gnueabihf/bin
 
 ENV CC_aarch64_unknown_linux_gnu=aarch64-unknown-linux-gnueabi-gcc \
     AR_aarch64_unknown_linux_gnu=aarch64-unknown-linux-gnueabi-ar \
-    CXX_aarch64_unknown_linux_gnu=aarch64-unknown-linux-gnueabi-g++ \
-    CC_armv7_unknown_linux_gnueabihf=armv7-unknown-linux-gnueabihf-gcc \
-    AR_armv7_unknown_linux_gnueabihf=armv7-unknown-linux-gnueabihf-ar \
-    CXX_armv7_unknown_linux_gnueabihf=armv7-unknown-linux-gnueabihf-g++
+    CXX_aarch64_unknown_linux_gnu=aarch64-unknown-linux-gnueabi-g++
 
-ENV HOSTS=armv7-unknown-linux-gnueabihf
-ENV HOSTS=$HOSTS,aarch64-unknown-linux-gnu
+ENV HOSTS=aarch64-unknown-linux-gnu
 
 ENV RUST_CONFIGURE_ARGS --host=$HOSTS --enable-extended
 ENV SCRIPT python2.7 ../x.py dist --host $HOSTS --target $HOSTS

src/ci/docker/dist-aarch64-linux/build-toolchains.sh

@@ -0,0 +1,37 @@
+#!/bin/bash
+# Copyright 2017 The Rust Project Developers. See the COPYRIGHT
+# file at the top-level directory of this distribution and at
+# http://rust-lang.org/COPYRIGHT.
+#
+# Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+# http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+# <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+# option. This file may not be copied, modified, or distributed
+# except according to those terms.
+
+set -ex
+
+hide_output() {
+  set +x
+  on_err="
+echo ERROR: An error was encountered with the build.
+cat /tmp/build.log
+exit 1
+"
+  trap "$on_err" ERR
+  bash -c "while true; do sleep 30; echo \$(date) - building ...; done" &
+  PING_LOOP_PID=$!
+  $@ &> /tmp/build.log
+  rm /tmp/build.log
+  trap - ERR
+  kill $PING_LOOP_PID
+  set -x
+}
+
+mkdir build
+cd build
+cp ../aarch64-linux-gnu.config .config
+ct-ng oldconfig
+hide_output ct-ng build
+cd ..
+rm -rf build

src/ci/docker/dist-arm-linux/Dockerfile

@@ -56,7 +56,7 @@ RUN mkdir /x-tools && chown rustbuild:rustbuild /x-tools
 USER rustbuild
 WORKDIR /tmp
 
-COPY arm-linux-gnueabihf.config arm-linux-gnueabi.config build-toolchains.sh /tmp/
+COPY arm-linux-gnueabi.config build-toolchains.sh /tmp/
 RUN ./build-toolchains.sh
 
 USER root
@@ -66,17 +66,12 @@ RUN curl -o /usr/local/bin/sccache \
       chmod +x /usr/local/bin/sccache
 
 ENV PATH=$PATH:/x-tools/arm-unknown-linux-gnueabi/bin
-ENV PATH=$PATH:/x-tools/arm-unknown-linux-gnueabihf/bin
 
 ENV CC_arm_unknown_linux_gnueabi=arm-unknown-linux-gnueabi-gcc \
     AR_arm_unknown_linux_gnueabi=arm-unknown-linux-gnueabi-ar \
-    CXX_arm_unknown_linux_gnueabi=arm-unknown-linux-gnueabi-g++ \
-    CC_arm_unknown_linux_gnueabihf=arm-unknown-linux-gnueabihf-gcc \
-    AR_arm_unknown_linux_gnueabihf=arm-unknown-linux-gnueabihf-ar \
-    CXX_arm_unknown_linux_gnueabihf=arm-unknown-linux-gnueabihf-g++
+    CXX_arm_unknown_linux_gnueabi=arm-unknown-linux-gnueabi-g++
 
 ENV HOSTS=arm-unknown-linux-gnueabi
-ENV HOSTS=$HOSTS,arm-unknown-linux-gnueabihf
 
 ENV RUST_CONFIGURE_ARGS --host=$HOSTS --enable-extended
 ENV SCRIPT python2.7 ../x.py dist --host $HOSTS --target $HOSTS

src/ci/docker/dist-arm-linux/build-toolchains.sh

@@ -35,11 +35,3 @@ ct-ng oldconfig
 hide_output ct-ng build
 cd ..
 rm -rf build
-
-mkdir build
-cd build
-cp ../arm-linux-gnueabihf.config .config
-ct-ng oldconfig
-hide_output ct-ng build
-cd ..
-rm -rf build

src/ci/docker/dist-armhf-linux/Dockerfile

@@ -0,0 +1,77 @@
+FROM ubuntu:16.04
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+  automake \
+  bison \
+  bzip2 \
+  ca-certificates \
+  cmake \
+  curl \
+  file \
+  flex \
+  g++ \
+  gawk \
+  gdb \
+  git \
+  gperf \
+  help2man \
+  libncurses-dev \
+  libtool-bin \
+  make \
+  patch \
+  python2.7 \
+  sudo \
+  texinfo \
+  wget \
+  xz-utils \
+  libssl-dev \
+  pkg-config
+
+RUN curl -OL https://github.com/Yelp/dumb-init/releases/download/v1.2.0/dumb-init_1.2.0_amd64.deb && \
+    dpkg -i dumb-init_*.deb && \
+    rm dumb-init_*.deb
+ENTRYPOINT ["/usr/bin/dumb-init", "--"]
+
+# Ubuntu 16.04 (this contianer) ships with make 4, but something in the
+# toolchains we build below chokes on that, so go back to make 3
+RUN curl https://ftp.gnu.org/gnu/make/make-3.81.tar.gz | tar xzf - && \
+      cd make-3.81 && \
+      ./configure --prefix=/usr && \
+      make && \
+      make install && \
+      cd .. && \
+      rm -rf make-3.81
+
+RUN curl http://crosstool-ng.org/download/crosstool-ng/crosstool-ng-1.22.0.tar.bz2 | \
+      tar xjf - && \
+      cd crosstool-ng && \
+      ./configure --prefix=/usr/local && \
+      make -j$(nproc) && \
+      make install && \
+      cd .. && \
+      rm -rf crosstool-ng
+
+RUN groupadd -r rustbuild && useradd -m -r -g rustbuild rustbuild
+RUN mkdir /x-tools && chown rustbuild:rustbuild /x-tools
+USER rustbuild
+WORKDIR /tmp
+
+COPY arm-linux-gnueabihf.config build-toolchains.sh /tmp/
+RUN ./build-toolchains.sh
+
+USER root
+
+RUN curl -o /usr/local/bin/sccache \
+      https://s3.amazonaws.com/rust-lang-ci/rust-ci-mirror/2017-03-24-sccache-x86_64-unknown-linux-musl && \
+      chmod +x /usr/local/bin/sccache
+
+ENV PATH=$PATH:/x-tools/arm-unknown-linux-gnueabihf/bin
+
+ENV CC_arm_unknown_linux_gnueabihf=arm-unknown-linux-gnueabihf-gcc \
+    AR_arm_unknown_linux_gnueabihf=arm-unknown-linux-gnueabihf-ar \
+    CXX_arm_unknown_linux_gnueabihf=arm-unknown-linux-gnueabihf-g++
+
+ENV HOSTS=arm-unknown-linux-gnueabihf
+
+ENV RUST_CONFIGURE_ARGS --host=$HOSTS --enable-extended
+ENV SCRIPT python2.7 ../x.py dist --host $HOSTS --target $HOSTS

src/ci/docker/dist-armhf-linux/build-toolchains.sh

@@ -0,0 +1,37 @@
+#!/bin/bash
+# Copyright 2017 The Rust Project Developers. See the COPYRIGHT
+# file at the top-level directory of this distribution and at
+# http://rust-lang.org/COPYRIGHT.
+#
+# Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+# http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+# <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+# option. This file may not be copied, modified, or distributed
+# except according to those terms.
+
+set -ex
+
+hide_output() {
+  set +x
+  on_err="
+echo ERROR: An error was encountered with the build.
+cat /tmp/build.log
+exit 1
+"
+  trap "$on_err" ERR
+  bash -c "while true; do sleep 30; echo \$(date) - building ...; done" &
+  PING_LOOP_PID=$!
+  $@ &> /tmp/build.log
+  rm /tmp/build.log
+  trap - ERR
+  kill $PING_LOOP_PID
+  set -x
+}
+
+mkdir build
+cd build
+cp ../arm-linux-gnueabihf.config .config
+ct-ng oldconfig
+hide_output ct-ng build
+cd ..
+rm -rf build

src/ci/docker/dist-armv7-linux/Dockerfile

@@ -0,0 +1,77 @@
+FROM ubuntu:16.04
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+  automake \
+  bison \
+  bzip2 \
+  ca-certificates \
+  cmake \
+  curl \
+  file \
+  flex \
+  g++ \
+  gawk \
+  gdb \
+  git \
+  gperf \
+  help2man \
+  libncurses-dev \
+  libtool-bin \
+  make \
+  patch \
+  python2.7 \
+  sudo \
+  texinfo \
+  wget \
+  xz-utils \
+  libssl-dev \
+  pkg-config
+
+RUN curl -OL https://github.com/Yelp/dumb-init/releases/download/v1.2.0/dumb-init_1.2.0_amd64.deb && \
+    dpkg -i dumb-init_*.deb && \
+    rm dumb-init_*.deb
+ENTRYPOINT ["/usr/bin/dumb-init", "--"]
+
+# Ubuntu 16.04 (this contianer) ships with make 4, but something in the
+# toolchains we build below chokes on that, so go back to make 3
+RUN curl https://ftp.gnu.org/gnu/make/make-3.81.tar.gz | tar xzf - && \
+      cd make-3.81 && \
+      ./configure --prefix=/usr && \
+      make && \
+      make install && \
+      cd .. && \
+      rm -rf make-3.81
+
+RUN curl http://crosstool-ng.org/download/crosstool-ng/crosstool-ng-1.22.0.tar.bz2 | \
+      tar xjf - && \
+      cd crosstool-ng && \
+      ./configure --prefix=/usr/local && \
+      make -j$(nproc) && \
+      make install && \
+      cd .. && \
+      rm -rf crosstool-ng
+
+RUN groupadd -r rustbuild && useradd -m -r -g rustbuild rustbuild
+RUN mkdir /x-tools && chown rustbuild:rustbuild /x-tools
+USER rustbuild
+WORKDIR /tmp
+
+COPY build-toolchains.sh armv7-linux-gnueabihf.config /tmp/
+RUN ./build-toolchains.sh
+
+USER root
+
+RUN curl -o /usr/local/bin/sccache \
+      https://s3.amazonaws.com/rust-lang-ci/rust-ci-mirror/2017-03-24-sccache-x86_64-unknown-linux-musl && \
+      chmod +x /usr/local/bin/sccache
+
+ENV PATH=$PATH:/x-tools/armv7-unknown-linux-gnueabihf/bin
+
+ENV CC_armv7_unknown_linux_gnueabihf=armv7-unknown-linux-gnueabihf-gcc \
+    AR_armv7_unknown_linux_gnueabihf=armv7-unknown-linux-gnueabihf-ar \
+    CXX_armv7_unknown_linux_gnueabihf=armv7-unknown-linux-gnueabihf-g++
+
+ENV HOSTS=armv7-unknown-linux-gnueabihf
+
+ENV RUST_CONFIGURE_ARGS --host=$HOSTS --enable-extended
+ENV SCRIPT python2.7 ../x.py dist --host $HOSTS --target $HOSTS

src/ci/docker/dist-armv7-aarch64-linux/build-toolchains.sh → src/ci/docker/dist-armv7-linux/build-toolchains.sh

@@ -35,11 +35,3 @@ ct-ng oldconfig
 hide_output ct-ng build
 cd ..
 rm -rf build
-
-mkdir build
-cd build
-cp ../aarch64-linux-gnu.config .config
-ct-ng oldconfig
-hide_output ct-ng build
-cd ..
-rm -rf build

src/ci/docker/dist-freebsd/Dockerfile → src/ci/docker/dist-i686-freebsd/Dockerfile

@@ -17,7 +17,6 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
   pkg-config
 
 COPY build-toolchain.sh /tmp/
-RUN /tmp/build-toolchain.sh x86_64
 RUN /tmp/build-toolchain.sh i686
 
 RUN curl -OL https://github.com/Yelp/dumb-init/releases/download/v1.2.0/dumb-init_1.2.0_amd64.deb && \
@@ -30,15 +29,11 @@ RUN curl -o /usr/local/bin/sccache \
       chmod +x /usr/local/bin/sccache
 
 ENV \
-    AR_x86_64_unknown_freebsd=x86_64-unknown-freebsd10-ar \
-    CC_x86_64_unknown_freebsd=x86_64-unknown-freebsd10-gcc \
-    CXX_x86_64_unknown_freebsd=x86_64-unknown-freebsd10-g++ \
     AR_i686_unknown_freebsd=i686-unknown-freebsd10-ar \
     CC_i686_unknown_freebsd=i686-unknown-freebsd10-gcc \
     CXX_i686_unknown_freebsd=i686-unknown-freebsd10-g++
 
-ENV HOSTS=x86_64-unknown-freebsd
-ENV HOSTS=$HOSTS,i686-unknown-freebsd
+ENV HOSTS=i686-unknown-freebsd
 
 ENV RUST_CONFIGURE_ARGS --host=$HOSTS --enable-extended
 ENV SCRIPT python2.7 ../x.py dist --host $HOSTS --target $HOSTS

src/ci/docker/dist-x86-linux/Dockerfile → src/ci/docker/dist-i686-linux/Dockerfile

@@ -80,7 +80,6 @@ RUN curl -o /usr/local/bin/sccache \
       chmod +x /usr/local/bin/sccache
 
 ENV HOSTS=i686-unknown-linux-gnu
-ENV HOSTS=$HOSTS,x86_64-unknown-linux-gnu
 
 ENV RUST_CONFIGURE_ARGS \
       --host=$HOSTS \

src/ci/docker/dist-mips-linux/Dockerfile

@@ -13,7 +13,6 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
   gdb \
   xz-utils \
   g++-mips-linux-gnu \
-  g++-mipsel-linux-gnu \
   libssl-dev \
   pkg-config
 
@@ -27,7 +26,6 @@ RUN curl -OL https://github.com/Yelp/dumb-init/releases/download/v1.2.0/dumb-ini
 ENTRYPOINT ["/usr/bin/dumb-init", "--"]
 
 ENV HOSTS=mips-unknown-linux-gnu
-ENV HOSTS=$HOSTS,mipsel-unknown-linux-gnu
 
 ENV RUST_CONFIGURE_ARGS --host=$HOSTS --enable-extended
 ENV SCRIPT python2.7 ../x.py dist --host $HOSTS --target $HOSTS