From 42076f1bf4c3148011a5205b84f04a85a0527753 Mon Sep 17 00:00:00 2001 From: Ed Morley <501702+edmorley@users.noreply.github.com> Date: Wed, 11 Nov 2020 12:39:30 +0000 Subject: [PATCH] Remove deprecated GDAL/GEOS/PROJ support (#1113) The standalone Geo buildpack offers more modern GDAL/GEOS/PROJ library versions, and can be used by apps in all languages, not just Python: https://github.com/heroku/heroku-geo-buildpack As such the Python buildpack's undocumented built-in support was deprecated back in April 2020, with a scheduled removal date of 6th October 2020: https://devcenter.heroku.com/changelog-items/1759 https://help.heroku.com/D5INLB1A/python-s-build_with_geo_libraries-legacy-feature-is-now-deprecated Metrics show very few builds continuing to use the built-in support. Apps with the `BUILD_WITH_GEO_LIBRARIES` env var set will now be shown a warning directing them to the standalone buildpack, as well as apps that hit GDAL related pip install errors but aren't using the env var. This also moves us one step closer to being able to remove the vendored copy of pip-pop (which is partially broken on newer pip). Closes @W-7654424@. --- CHANGELOG.md | 1 + Makefile | 2 +- bin/compile | 16 +------- bin/steps/gdal | 41 -------------------- bin/steps/geo-libs | 58 ----------------------------- bin/warnings | 26 +++++++++++++ builds/libraries/vendor/gdal | 24 ------------ builds/libraries/vendor/geos | 25 ------------- builds/libraries/vendor/proj | 24 ------------ test/fixtures/gdal/requirements.txt | 1 + test/fixtures/geos/requirements.txt | 1 - test/run-deps | 20 +++++----- test/run-features | 1 - 13 files changed, 40 insertions(+), 200 deletions(-) delete mode 100755 bin/steps/gdal delete mode 100755 bin/steps/geo-libs delete mode 100755 builds/libraries/vendor/gdal delete mode 100755 builds/libraries/vendor/geos delete mode 100755 builds/libraries/vendor/proj create mode 100644 test/fixtures/gdal/requirements.txt delete mode 100644 test/fixtures/geos/requirements.txt diff --git a/CHANGELOG.md b/CHANGELOG.md index 5cbcc6089..887e20f66 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ## Unreleased +- Remove deprecated GDAL/GEOS/PROJ support (#1113). - Remove vendored `jq` binary (#1112). - Remove redundant Mercurial install step (#1111). - Remove support for the Cedar-14 stack (#1110). diff --git a/Makefile b/Makefile index 84668a7a9..185fc2fa0 100644 --- a/Makefile +++ b/Makefile @@ -12,7 +12,7 @@ STACK_IMAGE_TAG := heroku/$(subst -,:,$(STACK))-build check: @shellcheck -x bin/compile bin/detect bin/release bin/test-compile bin/utils bin/warnings bin/default_pythons - @shellcheck -x bin/steps/collectstatic bin/steps/eggpath-fix bin/steps/eggpath-fix2 bin/steps/gdal bin/steps/geo-libs bin/steps/nltk bin/steps/pip-install bin/steps/pip-uninstall bin/steps/pipenv bin/steps/pipenv-python-version bin/steps/python + @shellcheck -x bin/steps/collectstatic bin/steps/eggpath-fix bin/steps/eggpath-fix2 bin/steps/nltk bin/steps/pip-install bin/steps/pip-uninstall bin/steps/pipenv bin/steps/pipenv-python-version bin/steps/python @shellcheck -x bin/steps/hooks/* test: diff --git a/bin/compile b/bin/compile index 150dc83bc..6d48b4ddc 100755 --- a/bin/compile +++ b/bin/compile @@ -50,8 +50,6 @@ fi # environment variable mechanism (the ENV_DIR). S3_BASE_URL="${BUILDPACK_S3_BASE_URL:-"https://heroku-buildpack-python.s3.amazonaws.com"}" # This has to be exported since it's used by the geo-libs step which is run in a subshell. -# TODO: Stop exporting once the geo-libs step is removed or no longer uses `sub_env`. -export S3_BASE_URL # Default Python Versions # shellcheck source=bin/default_pythons @@ -137,6 +135,7 @@ export LANG=en_US.UTF-8 # `~/.heroku/vendor` is an place where the buildpack may stick pre-build binaries for known # C dependencies. This section configures Python (GCC, more specifically) # and pip to automatically include these paths when building binaries. +# TODO: Stop adding .heroku/vendor here now that the buildpack no longer vendors anything. export C_INCLUDE_PATH=/app/.heroku/vendor/include:/app/.heroku/python/include:$C_INCLUDE_PATH export CPLUS_INCLUDE_PATH=/app/.heroku/vendor/include:/app/.heroku/python/include:$CPLUS_INCLUDE_PATH export LIBRARY_PATH=/app/.heroku/vendor/lib:/app/.heroku/python/lib:$LIBRARY_PATH @@ -176,8 +175,6 @@ cp -R "$CACHE_DIR/.heroku/python-stack" .heroku/ &> /dev/null || true cp -R "$CACHE_DIR/.heroku/python-version" .heroku/ &> /dev/null || true # A plain text file which contains the current sqlite3 version being used (used for cache busting). cp -R "$CACHE_DIR/.heroku/python-sqlite3-version" .heroku/ &> /dev/null || true -# Any pre-compiled binaries, provided by the buildpack. -cp -R "$CACHE_DIR/.heroku/vendor" .heroku/ &> /dev/null || true # "editable" installations of code repositories, via pip or pipenv. if [[ -d "$CACHE_DIR/.heroku/src" ]]; then cp -R "$CACHE_DIR/.heroku/src" .heroku/ &> /dev/null || true @@ -263,16 +260,6 @@ fi # shellcheck source=bin/steps/eggpath-fix source "$BIN_DIR/steps/eggpath-fix" -# Support for Geo libraries. This is deprecated functionality. -# It is undocumented. -# shellcheck source=bin/steps/geo-libs -sub_env "$BIN_DIR/steps/geo-libs" - -# GDAL support. -# This is part of the Geo support. -# shellcheck source=bin/steps/gdal -source "$BIN_DIR/steps/gdal" - # SQLite3 support. # This sets up and installs sqlite3 dev headers and the sqlite3 binary but not the # libsqlite3-0 library since that exists on the stack image. @@ -370,7 +357,6 @@ mkdir -p "$CACHE_DIR/.heroku" cp -R .heroku/python "$CACHE_DIR/.heroku/" cp -R .heroku/python-version "$CACHE_DIR/.heroku/" cp -R .heroku/python-stack "$CACHE_DIR/.heroku/" &> /dev/null || true -cp -R .heroku/vendor "$CACHE_DIR/.heroku/" &> /dev/null || true if [[ -d .heroku/src ]]; then cp -R .heroku/src "$CACHE_DIR/.heroku/" &> /dev/null || true fi diff --git a/bin/steps/gdal b/bin/steps/gdal deleted file mode 100755 index 6b0fa6fae..000000000 --- a/bin/steps/gdal +++ /dev/null @@ -1,41 +0,0 @@ -#!/usr/bin/env bash - -# This script serves as the GDAL build step of the -# [**Python Buildpack**](https://github.com/heroku/heroku-buildpack-python) -# compiler. -# -# A [buildpack](https://devcenter.heroku.com/articles/buildpacks) is an -# adapter between a Python application and Heroku's runtime. -# -# This script is invoked by [`bin/compile`](/). - -# The location of the pre-compiled cryptography binary. -VENDORED_GDAL="${S3_BASE_URL}/${STACK}/libraries/vendor/gdal.tar.gz" - -PKG_CONFIG_PATH="/app/.heroku/vendor/lib/pkgconfig:$PKG_CONFIG_PATH" - -# Syntax sugar. -# shellcheck source=bin/utils -source "$BIN_DIR/utils" - -# If GDAL exists within requirements, use vendored gdal. -if (pip-grep -s requirements.txt GDAL gdal pygdal &> /dev/null) then - - if [[ ! -f ".heroku/vendor/bin/gdalserver" && "${STACK}" == 'heroku-20' ]]; then - puts-warn "The buildpack's built-in GDAL functonality is not supported on Heroku-20." - puts-warn "Please use this buildpack instead: https://github.com/heroku/heroku-geo-buildpack" - - elif [[ ! -f ".heroku/vendor/bin/gdalserver" ]]; then - puts-warn "The vendored GDAL package in the Heroku Python Buildpack now deprecated." - puts-warn "To enable GDAL use an alternative buildpack is available here - https://github.com/heroku/heroku-geo-buildpack" - - echo "-----> Noticed GDAL. Bootstrapping gdal." - mkdir -p .heroku/vendor - # Download and extract cryptography into target vendor directory. - curl "$VENDORED_GDAL" -s | tar zxv -C .heroku/vendor &> /dev/null - mcount "steps.vendor.gdal" - fi - - GDAL=$(pwd)/vendor - export GDAL -fi diff --git a/bin/steps/geo-libs b/bin/steps/geo-libs deleted file mode 100755 index 2e3a85ad6..000000000 --- a/bin/steps/geo-libs +++ /dev/null @@ -1,58 +0,0 @@ -#!/usr/bin/env bash - -# This script serves as the GDAL build step of the -# [**Python Buildpack**](https://github.com/heroku/heroku-buildpack-python) -# compiler. -# -# A [buildpack](https://devcenter.heroku.com/articles/buildpacks) is an -# adapter between a Python application and Heroku's runtime. -# -# This script is invoked by [`bin/compile`](/). - -# The location of the pre-compiled cryptography binary. -VENDORED_GDAL="${S3_BASE_URL}/${STACK}/libraries/vendor/gdal.tar.gz" -VENDORED_GEOS="${S3_BASE_URL}/${STACK}/libraries/vendor/geos.tar.gz" -VENDORED_PROJ="${S3_BASE_URL}/${STACK}/libraries/vendor/proj.tar.gz" - -PKG_CONFIG_PATH="/app/.heroku/vendor/lib/pkgconfig:$PKG_CONFIG_PATH" - -# Syntax sugar. -# shellcheck source=bin/utils -source "$BIN_DIR/utils" - -if [[ "$BUILD_WITH_GEO_LIBRARIES" && "${STACK}" == 'heroku-20' ]]; then - puts-warn "The GDAL, GEOS and PROJ binaries and BUILD_WITH_GEO_LIBRARIES functonality is not supported on Heroku-20." - puts-warn "Please use this buildpack for GDAL, GEOS and PROJ: https://github.com/heroku/heroku-geo-buildpack" - puts-warn "To hide this message, unset the BUILD_WITH_GEO_LIBRARIES variable using: heroku config:unset BUILD_WITH_GEO_LIBRARIES" -elif [[ "$BUILD_WITH_GEO_LIBRARIES" ]]; then - mcount "buildvar.BUILD_WITH_GEO_LIBRARIES" - - puts-warn "The GDAL, GEOS and PROJ binaries and BUILD_WITH_GEO_LIBRARIES functonality are now deprecated." - puts-warn "An alternative buildpack to enable GDAL, GEOS and PROJ use is available here - https://github.com/heroku/heroku-geo-buildpack" - - if [ ! -f ".heroku/vendor/bin/proj" ]; then - echo "-----> Bootstrapping gdal, geos, proj." - mkdir -p .heroku/vendor - # Download and extract cryptography into target vendor directory. - curl "$VENDORED_GDAL" -s | tar zxv -C .heroku/vendor &> /dev/null - curl "$VENDORED_GEOS" -s | tar zxv -C .heroku/vendor &> /dev/null - curl "$VENDORED_PROJ" -s | tar zxv -C .heroku/vendor &> /dev/null - - mcount "steps.vendor.geo_libs" - # Copy libjasper from build image to slug. - if [[ "$STACK" == "heroku-16" ]]; then - cp /usr/lib/x86_64-linux-gnu/libjasper.so* ".heroku/vendor/lib/." - mcount "steps.vendor.libjasper" - fi - - fi - - GDAL=$(pwd)/vendor - export GDAL - # set path for post_compile hooks - export GDAL_LIBRARY_PATH="$BUILD_DIR/.heroku/vendor/lib/libgdal.so" - export GEOS_LIBRARY_PATH="$BUILD_DIR/.heroku/vendor/lib/libgeos_c.so" - # set path for runtime environmeht - set_env GDAL_LIBRARY_PATH "/app/.heroku/vendor/lib/libgdal.so" - set_env GEOS_LIBRARY_PATH "/app/.heroku/vendor/lib/libgeos_c.so" -fi diff --git a/bin/warnings b/bin/warnings index b987955d1..034d0dd1b 100755 --- a/bin/warnings +++ b/bin/warnings @@ -46,10 +46,36 @@ six-included() { fi } +geo-gdal-support() { + if [[ -f "${ENV_DIR}/BUILD_WITH_GEO_LIBRARIES" ]]; then + mcount "warnings.BUILD_WITH_GEO_LIBRARIES" + echo + puts-warn "The Python buildpack's BUILD_WITH_GEO_LIBRARIES functonality is no longer supported:" + puts-warn "https://help.heroku.com/D5INLB1A/python-s-build_with_geo_libraries-legacy-feature-is-now-deprecated" + puts-warn + puts-warn "For GDAL, GEOS and PROJ support, use the Geo buildpack alongside the Python buildpack:" + puts-warn "https://github.com/heroku/heroku-geo-buildpack" + puts-warn + puts-warn "To hide this message, unset the BUILD_WITH_GEO_LIBRARIES variable using:" + puts-warn "heroku config:unset BUILD_WITH_GEO_LIBRARIES" + echo + return + fi + if grep -qi 'Could not find gdal-config' "$WARNINGS_LOG"; then + mcount 'warnings.gdal' + echo + puts-warn "Hello! Package installation failed since the GDAL library was not found." + puts-warn "For GDAL, GEOS and PROJ support, use the Geo buildpack alongside the Python buildpack:" + puts-warn "https://github.com/heroku/heroku-geo-buildpack" + puts-warn " -- Much Love, Heroku." + fi +} + show-warnings() { old-platform scipy-included distribute-included six-included + geo-gdal-support } diff --git a/builds/libraries/vendor/gdal b/builds/libraries/vendor/gdal deleted file mode 100755 index 7cb110ea8..000000000 --- a/builds/libraries/vendor/gdal +++ /dev/null @@ -1,24 +0,0 @@ -#!/usr/bin/env bash -# Build Path: /app/.heroku/vendor/ - -OUT_PREFIX=$1 - -# Use new path, containing autoconf. -export PATH="/app/.heroku/python/bin/:$PATH" -hash -r - - -echo "Building gdal…" - -VERSION="2.2.1" -SOURCE_TARBALL="http://download.osgeo.org/gdal/${VERSION}/gdal-${VERSION}.tar.gz" - -curl -L $SOURCE_TARBALL | tar zx - -pushd "gdal-${VERSION}" -./configure --prefix=$OUT_PREFIX --enable-static=no && -make -make install - -# Cleanup -popd diff --git a/builds/libraries/vendor/geos b/builds/libraries/vendor/geos deleted file mode 100755 index 3c7bcab27..000000000 --- a/builds/libraries/vendor/geos +++ /dev/null @@ -1,25 +0,0 @@ -#!/usr/bin/env bash -# Build Path: /app/.heroku/vendor/ - -OUT_PREFIX=$1 - -# Use new path, containing autoconf. -export PATH="/app/.heroku/python/bin/:$PATH" -hash -r - - -echo "Building geos…" - -VERSION=3.6.2 - -SOURCE_TARBALL="http://download.osgeo.org/geos/geos-${VERSION}.tar.bz2" - -curl -L $SOURCE_TARBALL | tar xj - -pushd "geos-${VERSION}" -./configure --prefix=$OUT_PREFIX --enable-static=no && -make -make install - -# Cleanup -popd diff --git a/builds/libraries/vendor/proj b/builds/libraries/vendor/proj deleted file mode 100755 index 7f11578bb..000000000 --- a/builds/libraries/vendor/proj +++ /dev/null @@ -1,24 +0,0 @@ -#!/usr/bin/env bash -# Build Path: /app/.heroku/vendor/ - -OUT_PREFIX=$1 - -# Use new path, containing autoconf. -export PATH="/app/.heroku/python/bin/:$PATH" -hash -r - - -echo "Building gdal…" - -VERSION=4.9.3 -SOURCE_TARBALL="http://download.osgeo.org/proj/proj-${VERSION}.tar.gz" - -curl -L $SOURCE_TARBALL | tar zx - -pushd "proj-${VERSION}" -./configure --prefix=$OUT_PREFIX --enable-static=no -make -make install - -# Cleanup -popd diff --git a/test/fixtures/gdal/requirements.txt b/test/fixtures/gdal/requirements.txt new file mode 100644 index 000000000..fe72cdd42 --- /dev/null +++ b/test/fixtures/gdal/requirements.txt @@ -0,0 +1 @@ +gdal diff --git a/test/fixtures/geos/requirements.txt b/test/fixtures/geos/requirements.txt deleted file mode 100644 index 68d357cf8..000000000 --- a/test/fixtures/geos/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -django \ No newline at end of file diff --git a/test/run-deps b/test/run-deps index 4af435f70..5db742753 100755 --- a/test/run-deps +++ b/test/run-deps @@ -17,18 +17,18 @@ testCollectstatic() { assertCaptured "collectstatic" } -testGEOS() { +testBuildWithGeoLibrariesWarning() { local env_dir="$(mktmpdir)" echo '1' > "${env_dir}/BUILD_WITH_GEO_LIBRARIES" - compile 'geos' '' "${env_dir}" - if [[ $STACK == "heroku-20" ]]; then - assertCaptured " ! The GDAL, GEOS and PROJ binaries and BUILD_WITH_GEO_LIBRARIES functonality is not supported on Heroku-20." - else - assertCaptured " ! The GDAL, GEOS and PROJ binaries and BUILD_WITH_GEO_LIBRARIES functonality are now deprecated." - fi - # This should assertCapturedError on Heroku-20, but the test doesn't actually - # install anything that uses GEOS so succeeds (see W-8145375) - assertCapturedSuccess + compile 'gdal' '' "${env_dir}" + assertCaptured " ! The Python buildpack's BUILD_WITH_GEO_LIBRARIES functonality is no longer supported" + assertCapturedError +} + +testGDALWarning() { + compile 'gdal' + assertCaptured " ! Hello! Package installation failed since the GDAL library was not found." + assertCapturedError } testNLTK() { diff --git a/test/run-features b/test/run-features index d2e6a3cee..599940f3b 100755 --- a/test/run-features +++ b/test/run-features @@ -131,7 +131,6 @@ testHooks() { PROFILE_PATH PWD PYTHONUNBUFFERED - S3_BASE_URL SHLVL SOME_APP_CONFIG_VAR STACK