ci: use a custom android sdk manager with pinning and mirroring

src/ci/docker/arm-android/Dockerfile

@@ -7,23 +7,21 @@ COPY scripts/android-ndk.sh /scripts/
 RUN . /scripts/android-ndk.sh && \
     download_and_make_toolchain android-ndk-r15c-linux-x86_64.zip arm 14
 
-# Note:
-# Do not upgrade to `openjdk-9-jre-headless`, as it will cause certificate error
-# when installing the Android SDK (see PR #45193). This is unfortunate, but
-# every search result suggested either disabling HTTPS or replacing JDK 9 by
-# JDK 8 as the solution (e.g. https://stackoverflow.com/q/41421340). :|
 RUN dpkg --add-architecture i386 && \
     apt-get update && \
     apt-get install -y --no-install-recommends \
   libgl1-mesa-glx \
   libpulse0 \
   libstdc++6:i386 \
-  openjdk-8-jre-headless \
-  tzdata
+  openjdk-9-jre-headless \
+  tzdata \
+  wget \
+  python3
 
 COPY scripts/android-sdk.sh /scripts/
-RUN . /scripts/android-sdk.sh && \
-    download_and_create_avd 4333796 armeabi-v7a 18 5264690
+COPY scripts/android-sdk-manager.py /scripts/
+COPY arm-android/android-sdk.lock /android/sdk/android-sdk.lock
+RUN /scripts/android-sdk.sh
 
 ENV PATH=$PATH:/android/sdk/emulator
 ENV PATH=$PATH:/android/sdk/tools

src/ci/docker/arm-android/android-sdk.lock

@@ -0,0 +1,6 @@
+emulator emulator-linux-5264690.zip 48c1cda2bdf3095d9d9d5c010fbfb3d6d673e3ea
+patcher;v4 3534162-studio.sdk-patcher.zip 046699c5e2716ae11d77e0bad814f7f33fab261e
+platform-tools platform-tools_r28.0.2-linux.zip 46a4c02a9b8e4e2121eddf6025da3c979bf02e28
+platforms;android-18 android-18_r03.zip e6b09b3505754cbbeb4a5622008b907262ee91cb
+system-images;android-18;default;armeabi-v7a sys-img/android/armeabi-v7a-18_r05.zip 580b583720f7de671040d5917c8c9db0c7aa03fd
+tools sdk-tools-linux-4333796.zip 8c7c28554a32318461802c1291d76fccfafde054

src/ci/docker/scripts/android-sdk-manager.py

@@ -0,0 +1,190 @@
+#!/usr/bin/env python3
+# Simpler reimplementation of Android's sdkmanager
+# Extra features of this implementation are pinning and mirroring
+
+# These URLs are the Google repositories containing the list of available
+# packages and their versions. The list has been generated by listing the URLs
+# fetched while executing `tools/bin/sdkmanager --list`
+BASE_REPOSITORY = "https://dl.google.com/android/repository/"
+REPOSITORIES = [
+    "sys-img/android/sys-img2-1.xml",
+    "sys-img/android-wear/sys-img2-1.xml",
+    "sys-img/android-wear-cn/sys-img2-1.xml",
+    "sys-img/android-tv/sys-img2-1.xml",
+    "sys-img/google_apis/sys-img2-1.xml",
+    "sys-img/google_apis_playstore/sys-img2-1.xml",
+    "addon2-1.xml",
+    "glass/addon2-1.xml",
+    "extras/intel/addon2-1.xml",
+    "repository2-1.xml",
+]
+
+# Available hosts: linux, macosx and windows
+HOST_OS = "linux"
+
+# Mirroring options
+MIRROR_BUCKET = "rust-lang-ci2"
+MIRROR_BASE_DIR = "rust-ci-mirror/android/"
+
+import argparse
+import hashlib
+import os
+import subprocess
+import sys
+import tempfile
+import urllib.request
+import xml.etree.ElementTree as ET
+
+class Package:
+    def __init__(self, path, url, sha1, deps=None):
+        if deps is None:
+            deps = []
+        self.path = path.strip()
+        self.url = url.strip()
+        self.sha1 = sha1.strip()
+        self.deps = deps
+
+    def download(self, base_url):
+        _, file = tempfile.mkstemp()
+        url = base_url + self.url
+        subprocess.run(["curl", "-o", file, url], check=True)
+        # Ensure there are no hash mismatches
+        with open(file, "rb") as f:
+            sha1 = hashlib.sha1(f.read()).hexdigest()
+            if sha1 != self.sha1:
+                raise RuntimeError(
+                    "hash mismatch for package " + self.path + ": " +
+                    sha1 + " vs " + self.sha1 + " (known good)"
+                )
+        return file
+
+    def __repr__(self):
+        return "<Package "+self.path+" at "+self.url+" (sha1="+self.sha1+")"
+
+def fetch_url(url):
+    page = urllib.request.urlopen(url)
+    return page.read()
+
+def fetch_repository(base, repo_url):
+    packages = {}
+    root = ET.fromstring(fetch_url(base + repo_url))
+    for package in root:
+        if package.tag != "remotePackage":
+            continue
+        path = package.attrib["path"]
+
+        for archive in package.find("archives"):
+            host_os = archive.find("host-os")
+            if host_os is not None and host_os.text != HOST_OS:
+                continue
+            complete = archive.find("complete")
+            url = os.path.join(os.path.dirname(repo_url), complete.find("url").text)
+            sha1 = complete.find("checksum").text
+
+            deps = []
+            dependencies = package.find("dependencies")
+            if dependencies is not None:
+                for dep in dependencies:
+                    deps.append(dep.attrib["path"])
+
+            packages[path] = Package(path, url, sha1, deps)
+            break
+
+    return packages
+
+def fetch_repositories():
+    packages = {}
+    for repo in REPOSITORIES:
+        packages.update(fetch_repository(BASE_REPOSITORY, repo))
+    return packages
+
+class Lockfile:
+    def __init__(self, path):
+        self.path = path
+        self.packages = {}
+        if os.path.exists(path):
+            with open(path) as f:
+                for line in f:
+                    path, url, sha1 = line.split(" ")
+                    self.packages[path] = Package(path, url, sha1)
+
+    def add(self, packages, name, *, update=True):
+        if name not in packages:
+            raise NameError("package not found: " + name)
+        if not update and name in self.packages:
+            return
+        self.packages[name] = packages[name]
+        for dep in packages[name].deps:
+            self.add(packages, dep, update=False)
+
+    def save(self):
+        packages = list(sorted(self.packages.values(), key=lambda p: p.path))
+        with open(self.path, "w") as f:
+            for package in packages:
+                f.write(package.path + " " + package.url + " " + package.sha1 + "\n")
+
+def cli_add_to_lockfile(args):
+    lockfile = Lockfile(args.lockfile)
+    packages = fetch_repositories()
+    for package in args.packages:
+        lockfile.add(packages, package)
+    lockfile.save()
+
+def cli_update_mirror(args):
+    lockfile = Lockfile(args.lockfile)
+    for package in lockfile.packages.values():
+        path = package.download(BASE_REPOSITORY)
+        subprocess.run([
+            "aws", "s3", "mv", path,
+            "s3://" + MIRROR_BUCKET + "/" + MIRROR_BASE_DIR + package.url,
+            "--profile=" + args.awscli_profile,
+        ], check=True)
+
+def cli_install(args):
+    lockfile = Lockfile(args.lockfile)
+    for package in lockfile.packages.values():
+        # Download the file from the mirror into a temp file
+        url = "https://" + MIRROR_BUCKET + ".s3.amazonaws.com/" + MIRROR_BASE_DIR
+        downloaded = package.download(url)
+        # Extract the file in a temporary directory
+        extract_dir = tempfile.mkdtemp()
+        subprocess.run([
+            "unzip", "-q", downloaded, "-d", extract_dir,
+        ], check=True)
+        # Figure out the prefix used in the zip
+        subdirs = [d for d in os.listdir(extract_dir) if not d.startswith(".")]
+        if len(subdirs) != 1:
+            raise RuntimeError("extracted directory contains more than one dir")
+        # Move the extracted files in the proper directory
+        dest = os.path.join(args.dest, package.path.replace(";", "/"))
+        os.makedirs("/".join(dest.split("/")[:-1]), exist_ok=True)
+        os.rename(os.path.join(extract_dir, subdirs[0]), dest)
+        os.unlink(downloaded)
+
+def cli():
+    parser = argparse.ArgumentParser()
+    subparsers = parser.add_subparsers()
+
+    add_to_lockfile = subparsers.add_parser("add-to-lockfile")
+    add_to_lockfile.add_argument("lockfile")
+    add_to_lockfile.add_argument("packages", nargs="+")
+    add_to_lockfile.set_defaults(func=cli_add_to_lockfile)
+
+    update_mirror = subparsers.add_parser("update-mirror")
+    update_mirror.add_argument("lockfile")
+    update_mirror.add_argument("--awscli-profile", default="default")
+    update_mirror.set_defaults(func=cli_update_mirror)
+
+    install = subparsers.add_parser("install")
+    install.add_argument("lockfile")
+    install.add_argument("dest")
+    install.set_defaults(func=cli_install)
+
+    args = parser.parse_args()
+    if not hasattr(args, "func"):
+        print("error: a subcommand is required (see --help)")
+        exit(1)
+    args.func(args)
+
+if __name__ == "__main__":
+    cli()

src/ci/docker/scripts/android-sdk.sh

@@ -2,66 +2,28 @@ set -ex
 
 export ANDROID_HOME=/android/sdk
 PATH=$PATH:"${ANDROID_HOME}/tools/bin"
+LOCKFILE="${ANDROID_HOME}/android-sdk.lock"
 
-download_sdk() {
-    mkdir -p /android
-    curl -fo sdk.zip "https://dl.google.com/android/repository/sdk-tools-linux-$1.zip"
-    unzip -q sdk.zip -d "$ANDROID_HOME"
-    rm -f sdk.zip
-}
-
-download_sysimage() {
-    abi=$1
-    api=$2
-
-    # See https://developer.android.com/studio/command-line/sdkmanager.html for
-    # usage of `sdkmanager`.
-    #
-    # The output from sdkmanager is so noisy that it will occupy all of the 4 MB
-    # log extremely quickly. Thus we must silence all output.
-    yes | sdkmanager --licenses > /dev/null
-    yes | sdkmanager platform-tools \
-        "platforms;android-$api" \
-        "system-images;android-$api;default;$abi" > /dev/null
-}
-
-download_emulator() {
-    # Download a pinned version of the emulator since upgrades can cause issues
-    curl -fo emulator.zip "https://dl.google.com/android/repository/emulator-linux-$1.zip"
-    rm -rf "${ANDROID_HOME}/emulator"
-    unzip -q emulator.zip -d "${ANDROID_HOME}"
-    rm -f emulator.zip
-}
-
-create_avd() {
-    abi=$1
-    api=$2
+# To add a new packages to the SDK or to update an existing one you need to
+# run the command:
+#
+#    android-sdk-manager.py add-to-lockfile $LOCKFILE <package-name>
+#
+# Then, after every lockfile update the mirror has to be synchronized as well:
+#
+#    android-sdk-manager.py update-mirror $LOCKFILE
+#
+/scripts/android-sdk-manager.py install "${LOCKFILE}" "${ANDROID_HOME}"
 
-    # See https://developer.android.com/studio/command-line/avdmanager.html for
-    # usage of `avdmanager`.
-    echo no | avdmanager create avd \
-        -n "$abi-$api" \
-        -k "system-images;android-$api;default;$abi"
-}
+details=$(cat "${LOCKFILE}" \
+    | grep system-images \
+    | sed 's/^system-images;android-\([0-9]\+\);default;\([a-z0-9-]\+\) /\1 \2 /g')
+api="$(echo "${details}" | awk '{print($1)}')"
+abi="$(echo "${details}" | awk '{print($2)}')"
 
-download_and_create_avd() {
-    download_sdk $1
-    download_sysimage $2 $3
-    create_avd $2 $3
-    download_emulator $4
-}
+# See https://developer.android.com/studio/command-line/avdmanager.html for
+# usage of `avdmanager`.
+echo no | avdmanager create avd \
+    -n "$abi-$api" \
+    -k "system-images;android-$api;default;$abi"
 
-# Usage:
-#
-#       download_and_create_avd 4333796 armeabi-v7a 18 5264690
-#
-# 4333796 =>
-#   SDK tool version.
-#   Copy from https://developer.android.com/studio/index.html#command-tools
-# armeabi-v7a =>
-#   System image ABI
-# 18 =>
-#   Android API Level (18 = Android 4.3 = Jelly Bean MR2)
-# 5264690 =>
-#   Android Emulator version.
-#   Copy from the "build_id" in the `/android/sdk/emulator/emulator -version` output