feat: add a cache implementation for downloads (#294)

* refactor: extract constants

* refactor: add a cache for fownloads

* refactor: disable cache cleanup by default

* chore: apply shellcheck formats

* feat: use hash for cache files

src/usr/local/bin/install-tool

@@ -47,5 +47,5 @@ fi
 
 # cleanup
 if [[ $EUID -eq 0 ]]; then
-  rm -rf /var/lib/apt/lists/* /tmp/*
+  rm -rf /var/lib/apt/lists/* "${TEMP_DIR:?}"/*
 fi

src/usr/local/buildpack/util.sh

@@ -4,23 +4,18 @@
 DIR="${BASH_SOURCE%/*}"
 if [[ ! -d "$DIR" ]]; then DIR="$PWD"; fi
 
-# CONSTANTS
-export ENV_FILE=/usr/local/etc/env
-export ROOT_DIR=/usr/local
-# shellcheck disable=SC2153
-export USER_HOME="/home/${USER_NAME}"
-export BASH_RC=/etc/bash.bashrc
-export ROOT_UMASK=750
-export USER_UMASK=770
-
 # source the helper files
 # shellcheck source=/dev/null
+. "${DIR}/utils/constants.sh"
+# shellcheck source=/dev/null
 . "${DIR}/utils/environment.sh"
 # shellcheck source=/dev/null
 . "${DIR}/utils/filesystem.sh"
 # shellcheck source=/dev/null
 . "${DIR}/utils/linking.sh"
 # shellcheck source=/dev/null
+. "${DIR}/utils/cache.sh"
+# shellcheck source=/dev/null
 . "${DIR}/utils/version.sh"
 
 check_debug() {

src/usr/local/buildpack/utils/cache.sh

@@ -0,0 +1,83 @@
+#!/bin/bash
+
+# will attempt to download the file at the given url and stores it in the cache.
+# If this file already exists, will return the cached version
+# The cache will only used if BUILDPACK_CACHE_DIR is set
+# First argument is the url, second one is the filename (optional)
+function get_from_url () {
+  local url=${1}
+  check url true
+
+  local checksum
+  checksum=$(echo "${url}" | sha1sum | awk '{print $1}')
+
+  local name
+  name=${2:-$(basename "${url}")}
+
+  local filename="${checksum}/${name}"
+
+  if [ -n "${BUILDPACK_CACHE_DIR}" ] && [ -e "${BUILDPACK_CACHE_DIR}/${filename}" ]; then
+      # file in cache
+      echo "Found file in cache: ${BUILDPACK_CACHE_DIR}/${filename}" >&2
+      echo "${BUILDPACK_CACHE_DIR}/${filename}"
+  else
+    # cache disabled or not in cache
+    download_file "${url}" "${filename}"
+  fi
+}
+
+# Will download the file into the cache folder and returns the path
+# If the cache is not enabled it will download it to a temp folder
+# The second argument will be the filename if given
+function download_file () {
+  local url=${1}
+  check url true
+
+  local name
+  name=${2:-$(basename "${url}")}
+
+  local temp_folder=${BUILDPACK_CACHE_DIR:-${TEMP_DIR}}
+  curl --create-dirs -sSfLo "${temp_folder}/${name}" "${url}"
+  echo "${temp_folder}/${name}"
+}
+
+# will try to clean up the oldest file in the cache until the cache is empty
+# or unless the threshold is reached
+# When given true as first argument, will only delete a single file
+# If BUILDPACK_CACHE_MAX_ALLOCATED_DISK is not set then the cache will be cleaned
+function cleanup_cache () {
+  local single_file=${1:false}
+  check BUILDPACK_CACHE_DIR true
+
+  local max_fill_level=${BUILDPACK_CACHE_MAX_ALLOCATED_DISK:-100}
+
+  local fill_level
+  local oldest
+  fill_level=$(get_cache_fill_level)
+  oldest=$(get_oldest_file)
+
+  while [ "${fill_level}" -ge "${max_fill_level}" ] && [ -n "${oldest}" ]; do
+    # fill level is greater then threshold and there is a file
+    echo "Cache: ${fill_level}% - Cleaning up: ${oldest}"
+    rm "${oldest}"
+
+    if [ "${single_file}" = "true" ]; then
+      exit 0
+    fi
+
+    fill_level=$(get_cache_fill_level)
+    oldest=$(get_oldest_file)
+  done
+}
+
+# Will get the oldest file in the cache and returns the path to it
+function get_oldest_file () {
+  check BUILDPACK_CACHE_DIR true
+  find "${BUILDPACK_CACHE_DIR}" -type f -printf '%T+ %p\n' | sort | head -n 1 | awk '{ print $2 }'
+}
+
+# Get the current fill level for the cache dir in percent
+function get_cache_fill_level () {
+  check BUILDPACK_CACHE_DIR true
+  df --output=pcent "${BUILDPACK_CACHE_DIR}" | tr -dc '0-9'
+}

src/usr/local/buildpack/utils/constants.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+
+# defines the location of the env file that gets sourced for every command
+export ENV_FILE=/usr/local/etc/env
+# defines the location of the global bashrc
+export BASH_RC=/etc/bash.bashrc
+# defines the root directory where tools will be installed
+export ROOT_DIR=/usr/local
+# defines the directory where user tools will be installed
+# shellcheck disable=SC2153
+export USER_HOME="/home/${USER_NAME}"
+# defines the umask for folders created by the root
+export ROOT_UMASK=750
+# defines the umask fo folders created by the user
+export USER_UMASK=770
+# defines the cache folder for downloaded tools, if empty no cache will be used
+export BUILDPACK_CACHE_DIR=
+# defines the max amount of filled space (in percent from 0-100) that is allowed
+# before the installation tries to free space by cleaning the cache folder
+# If empty, then cache cleanup is disabled
+export BUILDPACK_MAX_ALLOCATED_DISK=
+# defines the temp directory that will be used when the cache is not active
+# it is used for all downloads and will be cleaned up after each install
+export TEMP_DIR=/tmp

test/bash/cache.bats

@@ -0,0 +1,190 @@
+
+setup() {
+  load '../../node_modules/bats-support/load'
+  load '../../node_modules/bats-assert/load'
+
+    TEST_DIR="$(cd "$(dirname "$BATS_TEST_FILENAME")" >/dev/null 2>&1 && pwd)"
+    TEST_ROOT_DIR=$(mktemp -u)
+
+    load "$TEST_DIR/../../src/usr/local/buildpack/util.sh"
+
+    # load test overwrites
+    load "$TEST_DIR/util.sh"
+
+    # set directories for test
+    ROOT_DIR="${TEST_ROOT_DIR}/root"
+    USER_HOME="${TEST_ROOT_DIR}/user"
+    ENV_FILE="${TEST_ROOT_DIR}/env"
+
+    # set default test user
+    TEST_ROOT_USER=1000
+}
+
+teardown() {
+  rm -rf "${TEST_ROOT_DIR}"
+}
+
+@test "get_oldest_file" {
+
+  BUILDPACK_CACHE_DIR= \
+  run get_oldest_file
+  assert_failure
+
+  # create cache dir
+  BUILDPACK_CACHE_DIR="${TEST_ROOT_DIR}/cache"
+  mkdir -p "${BUILDPACK_CACHE_DIR}"
+
+  run get_oldest_file
+  assert_success
+  assert_output ""
+
+  # create files
+  touch "${BUILDPACK_CACHE_DIR}/b"
+  # sleep for a milisecond, otherwise the files have the same age
+  # and then it will get sorted by name
+  sleep 0.01
+  touch "${BUILDPACK_CACHE_DIR}/a"
+
+  run get_oldest_file
+  assert_success
+  assert_output "${BUILDPACK_CACHE_DIR}/b"
+}
+
+@test "get_cache_fill_level" {
+  local TEST_FILL_LEVEL=88
+
+  BUILDPACK_CACHE_DIR= \
+  run get_cache_fill_level
+  assert_failure
+
+  # create cache dir
+  BUILDPACK_CACHE_DIR="${TEST_ROOT_DIR}/cache"
+  mkdir -p "${BUILDPACK_CACHE_DIR}"
+
+  local real_fill_level=$(get_cache_fill_level)
+  assert test "[[ "$real_fill_level" =~ ^[0-9]+$ ]]"
+
+  # overwrite function to verify deletion
+  function get_cache_fill_level () {
+    echo $TEST_FILL_LEVEL
+  }
+
+  run get_cache_fill_level
+  assert_output "88"
+
+  TEST_FILL_LEVEL=12 \
+  run get_cache_fill_level
+  assert_output "12"
+}
+
+@test "cache delete" {
+  local TEST_FILL_LEVEL=88
+  local BUILDPACK_CACHE_MAX_ALLOCATED_DISK=50
+
+  # overwrite function to verify deletion
+  function get_cache_fill_level () {
+    echo $TEST_FILL_LEVEL
+  }
+
+  # create cache dir
+  BUILDPACK_CACHE_DIR="${TEST_ROOT_DIR}/cache"
+  mkdir -p "${BUILDPACK_CACHE_DIR}/b"
+
+  # create files
+  touch "${BUILDPACK_CACHE_DIR}/c"
+  sleep 0.01
+  touch "${BUILDPACK_CACHE_DIR}/b/test"
+  sleep 0.01
+  touch "${BUILDPACK_CACHE_DIR}/a"
+
+  BUILDPACK_CACHE_DIR= \
+  BUILDPACK_CACHE_MAX_ALLOCATED_DISK= \
+  run cleanup_cache
+  assert_failure
+
+  BUILDPACK_CACHE_DIR= \
+  BUILDPACK_CACHE_MAX_ALLOCATED_DISK=20 \
+  run cleanup_cache
+  assert_failure
+
+  BUILDPACK_CACHE_MAX_ALLOCATED_DISK= \
+  run cleanup_cache
+  assert_success
+  assert [ -e "${BUILDPACK_CACHE_DIR}/a" ]
+  assert [ -e "${BUILDPACK_CACHE_DIR}/b/test" ]
+  assert [ -e "${BUILDPACK_CACHE_DIR}/c" ]
+
+  run cleanup_cache true
+  assert_success
+  assert [ -e "${BUILDPACK_CACHE_DIR}/a" ]
+  assert [ -e "${BUILDPACK_CACHE_DIR}/b/test" ]
+  assert [ ! -e "${BUILDPACK_CACHE_DIR}/c" ]
+
+  TEST_FILL_LEVEL=30 \
+  run cleanup_cache
+  assert_success
+  assert [ -e "${BUILDPACK_CACHE_DIR}/a" ]
+  assert [ -e "${BUILDPACK_CACHE_DIR}/b/test" ]
+  assert [ ! -e "${BUILDPACK_CACHE_DIR}/c" ]
+
+  TEST_FILL_LEVEL=90 \
+  run cleanup_cache
+  assert_success
+  assert [ ! -e "${BUILDPACK_CACHE_DIR}/a" ]
+  assert [ ! -e "${BUILDPACK_CACHE_DIR}/b/test" ]
+  assert [ ! -e "${BUILDPACK_CACHE_DIR}/c" ]
+}
+
+@test "download_file" {
+  # create cache dir
+  BUILDPACK_CACHE_DIR="${TEST_ROOT_DIR}/cache"
+  mkdir -p "${BUILDPACK_CACHE_DIR}"
+
+  local file="https://file-examples-com.github.io/uploads/2017/02/file_example_JSON_1kb.json"
+
+  run download_file
+  assert_failure
+
+  run download_file "${file}"
+  assert_success
+  assert_output "${BUILDPACK_CACHE_DIR}/file_example_JSON_1kb.json"
+
+  run download_file "${file}" "foobar"
+  assert_success
+  assert_output "${BUILDPACK_CACHE_DIR}/foobar"
+
+  BUILDPACK_CACHE_DIR= \
+  tmp_file=$(download_file "${file}")
+  rm "${tmp_file}"
+
+  assert test "[[ "${tmp_file}" =~ "\/file_example_JSON_1kb.json" ]]"
+}
+
+@test "get_from_url" {
+  # create cache dir
+  BUILDPACK_CACHE_DIR="${TEST_ROOT_DIR}/cache"
+  mkdir -p "${BUILDPACK_CACHE_DIR}"
+
+  local file="https://file-examples-com.github.io/uploads/2017/02/file_example_JSON_1kb.json"
+
+  run get_from_url "${file}"
+  assert_success
+  assert_output --regexp "^${BUILDPACK_CACHE_DIR}/[0-9a-f]{40}/file_example_JSON_1kb\.json"
+
+  run get_from_url "${file}" test
+  assert_success
+  assert_output --regexp "${BUILDPACK_CACHE_DIR}/[0-9a-f]{40}/test"
+
+  # overwrite donwload function to fail
+  function download_file () {
+    exit 1
+  }
+
+  run get_from_url "${file}"
+  assert_success
+  assert_output --regexp "${BUILDPACK_CACHE_DIR}/[0-9a-f]{40}/file_example_JSON_1kb\.json"
+
+  run get_from_url "${file}" "test"
+  assert_success
+  assert_output --regexp "${BUILDPACK_CACHE_DIR}/[0-9a-f]{40}/test"
+}