Merge #730

730: Make FreeBSD builds more resilient. r=Emilgardis a=Alexhuszagh Do not hardcode exact versions of the FreeBSD release, `openssl`, or `sqlite3` versions. Rather, use the FreeBSD ftp server to determine the latest `12.X` release, and use the package list downloaded to determine the latest `openssl` and `sqlite3` versions. Since we use a dynamic version of FreeBSD, and have no good way of detecting this later, we write the version and revision to `/opt/freebsd-version`. Fixes #695. Co-authored-by: Alex Huszagh <ahuszagh@gmail.com>
cross-rs · May 30, 2022 · 7caf557 · 7caf557
2 parents f77cf8f + a89199c
commit 7caf557
Show file tree

Hide file tree

Showing 6 changed files with 95 additions and 40 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -5,6 +5,7 @@ This project adheres to [Semantic Versioning](http://semver.org/).
 
 ## [Unreleased]
 
+- #730 - make FreeBSD builds more resilient.
 - #727 - add `PKG_CONFIG_PATH` to all `*-linux-gnu` images.
 - #725 - support `CROSS_DEBUG` and `CROSS_RUNNER` on android images.
 - #722 - boolean environment variables are evaluated as truthy or falsey.

diff --git a/docker/Dockerfile.i686-unknown-freebsd b/docker/Dockerfile.i686-unknown-freebsd
@@ -10,6 +10,7 @@ RUN /cmake.sh
 COPY xargo.sh /
 RUN /xargo.sh
 
+COPY freebsd-common.sh /
 COPY freebsd.sh /
 RUN /freebsd.sh i686
 

diff --git a/docker/Dockerfile.x86_64-unknown-freebsd b/docker/Dockerfile.x86_64-unknown-freebsd
@@ -10,6 +10,7 @@ RUN /cmake.sh
 COPY xargo.sh /
 RUN /xargo.sh
 
+COPY freebsd-common.sh /
 COPY freebsd.sh /
 RUN /freebsd.sh x86_64
 

diff --git a/docker/freebsd-common.sh b/docker/freebsd-common.sh
@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+
+set -x
+set -euo pipefail
+
+export BSD_ARCH=
+case "${ARCH}" in
+    x86_64)
+        BSD_ARCH=amd64
+        ;;
+    i686)
+        BSD_ARCH=i386
+        ;;
+esac
+export BSD_HOME="ftp.freebsd.org/pub/FreeBSD/releases"
+export BSD_MAJOR=12
diff --git a/docker/freebsd-extras.sh b/docker/freebsd-extras.sh
@@ -3,47 +3,50 @@
 set -x
 set -euo pipefail
 
-main() {
-    local arch="${1}"
+export ARCH="${1}"
+# shellcheck disable=SC1091
+. lib.sh
+# shellcheck disable=SC1091
+. freebsd-common.sh
 
-    local sqlite_ver=3.37.2,1 \
-          openssl_ver=1.1.1n,1 \
-          target="${arch}-unknown-freebsd12"
+main() {
+    local pkg_source="https://pkg.freebsd.org/FreeBSD:${BSD_MAJOR}:${BSD_ARCH}/quarterly"
+    install_packages curl jq xz-utils
 
     local td
     td="$(mktemp -d)"
 
-    mkdir "${td}"/{openssl,sqlite}
+    mkdir "${td}"/{openssl,sqlite,packagesite}
 
     pushd "${td}"
 
-    local bsd_arch=
-    case "${arch}" in
-        x86_64)
-            bsd_arch=amd64
-            ;;
-        i686)
-            bsd_arch=i386
-            ;;
-    esac
+    curl --retry 3 -sSfL "${pkg_source}/packagesite.txz" -O
+    tar -C "${td}/packagesite" -xJf packagesite.txz
+    local openssl_ver
+    local sqlite_ver
+    openssl_ver=$(jq -c '. | select ( .name == "openssl" ) | .version' "${td}/packagesite/packagesite.yaml")
+    sqlite_ver=$(jq -c '. | select ( .name == "sqlite3" ) | .version' "${td}/packagesite/packagesite.yaml")
+    openssl_ver=${openssl_ver//'"'/}
+    sqlite_ver=${sqlite_ver//'"'/}
+
+    local target="${ARCH}-unknown-freebsd${BSD_MAJOR}"
 
     # Adding openssl lib
-    curl --retry 3 -sSfL "https://pkg.freebsd.org/FreeBSD:12:${bsd_arch}/quarterly/All/openssl-${openssl_ver}.txz" -O
-    tar -C "${td}/openssl" -xJf openssl-${openssl_ver}.txz /usr/local/lib /usr/local/include/
+    curl --retry 3 -sSfL "${pkg_source}/All/openssl-${openssl_ver}.txz" -O
+    tar -C "${td}/openssl" -xJf "openssl-${openssl_ver}.txz" /usr/local/lib /usr/local/include/
 
     # Adding sqlite3
-    curl --retry 3 -sSfL "https://pkg.freebsd.org/FreeBSD:12:${bsd_arch}/quarterly/All/sqlite3-${sqlite_ver}.txz" -O
-    tar -C "${td}/sqlite" -xJf sqlite3-${sqlite_ver}.txz /usr/local/lib
+    curl --retry 3 -sSfL "${pkg_source}/All/sqlite3-${sqlite_ver}.txz" -O
+    tar -C "${td}/sqlite" -xJf "sqlite3-${sqlite_ver}.txz" /usr/local/lib
 
     # Copy the linked library
     local destdir="/usr/local/${target}"
     cp -r "${td}/openssl/usr/local/include" "${destdir}"
     cp "${td}/openssl/usr/local/lib"/lib{crypto,ssl}.a "${destdir}/lib"
-    cp "${td}/openssl/usr/local/lib"/lib{crypto,ssl}.so.11 "${destdir}/lib"
-    cp "${td}/openssl/usr/local/lib"/lib{crypto,ssl}.so "${destdir}/lib"
-    cp "${td}/sqlite/usr/local/lib/libsqlite3.so.0.8.6" "${destdir}/lib"
-    cp "${td}/sqlite/usr/local/lib/libsqlite3.so" "${destdir}/lib"
-    cp "${td}/sqlite/usr/local/lib/libsqlite3.so.0" "${destdir}/lib"
+    cp "${td}/openssl/usr/local/lib"/lib{crypto,ssl}.so* "${destdir}/lib"
+    cp "${td}/sqlite/usr/local/lib"/libsqlite3.so* "${destdir}/lib"
+
+    purge_packages
 
     # clean up
     popd

diff --git a/docker/freebsd.sh b/docker/freebsd.sh
@@ -3,13 +3,52 @@
 set -x
 set -euo pipefail
 
-main() {
-    local arch="${1}"
+export ARCH="${1}"
+# shellcheck disable=SC1091
+. freebsd-common.sh
+
+max_freebsd() {
+    local best=
+    local minor=0
+    local version=
+    local release_major=
+    local release_minor=
+    for release in "${@}"; do
+        version=$(echo "${release}" | cut -d '-' -f 1)
+        release_major=$(echo "${version}"| cut -d '.' -f 1)
+        release_minor=$(echo "${version}"| cut -d '.' -f 2)
+        if [ "${release_major}" == "${BSD_MAJOR}" ] && [ "${release_minor}" -gt "${minor}" ]; then
+            best="${release}"
+            minor="${release_minor}"
+        fi
+    done
+    if [[ -z "$best" ]]; then
+        echo "Could not find best release for FreeBSD ${BSD_MAJOR}." 1>&2
+        exit 1
+    fi
+    echo "${best}"
+}
+
+latest_freebsd() {
+    local dirs
+    local releases
+    local max_release
 
-    local base_release=12.3 \
-          binutils=2.32 \
+    dirs=$(curl --silent --list-only "${BSD_HOME}/${BSD_ARCH}/" | grep RELEASE)
+    read -r -a releases <<< "${dirs[@]}"
+    max_release=$(max_freebsd "${releases[@]}")
+
+    echo "${max_release//-RELEASE/}"
+}
+
+base_release="$(latest_freebsd)"
+bsd_ftp="${BSD_HOME}/${BSD_ARCH}/${base_release}-RELEASE"
+bsd_http="http://${bsd_ftp}"
+
+main() {
+    local binutils=2.32 \
           gcc=6.4.0 \
-          target="${arch}-unknown-freebsd12"
+          target="${ARCH}-unknown-freebsd${BSD_MAJOR}"
 
     local dependencies=(
         ca-certificates
@@ -47,17 +86,7 @@ main() {
     ./contrib/download_prerequisites
     cd ..
 
-    local bsd_arch=
-    case "${arch}" in
-        x86_64)
-            bsd_arch=amd64
-            ;;
-        i686)
-            bsd_arch=i386
-            ;;
-    esac
-
-    curl --retry 3 -sSfL "http://ftp.freebsd.org/pub/FreeBSD/releases/${bsd_arch}/${base_release}-RELEASE/base.txz" -O
+    curl --retry 3 -sSfL "${bsd_http}/base.txz" -O
     tar -C "${td}/freebsd" -xJf base.txz ./usr/include ./usr/lib ./lib
 
     cd binutils-build
@@ -119,6 +148,10 @@ main() {
       apt-get purge --assume-yes --auto-remove "${purge_list[@]}"
     fi
 
+    # store the version info for the FreeBSD release
+    bsd_revision=$(curl --retry 3 -sSfL "${bsd_http}/REVISION")
+    echo "${base_release} (${bsd_revision})" > /opt/freebsd-version
+
     rm -rf "${td}"
     rm "${0}"
 }