From f499495c23dee17cc4ed35f57dc7b53b4126bc8c Mon Sep 17 00:00:00 2001 From: Phil Phillips Date: Fri, 19 Aug 2022 16:31:54 -0500 Subject: [PATCH 1/5] feat: allow config var for slug override --- lib/backends/s3.bash | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/backends/s3.bash b/lib/backends/s3.bash index caf4654..41f4377 100644 --- a/lib/backends/s3.bash +++ b/lib/backends/s3.bash @@ -9,6 +9,7 @@ BK_TAR_ARGS=() BK_TAR_ADDITIONAL_ARGS="--ignore-failed-read" BK_TAR_EXTENSION="tar" BK_TAR_EXTRACT_ARGS="-xf" +BK_PIPELINE_SLUG=${BUILDKITE_PLUGIN_CACHE_SLUG_OVERRIDE:-${BUILDKITE_PIPELINE_SLUG}} if [[ ! "$OSTYPE" == "darwin"* ]]; then shell_exec=$( @@ -65,7 +66,7 @@ fi function restore() { TAR_FILE="${CACHE_KEY}.${BK_TAR_EXTENSION}" - TKEY="${BUILDKITE_ORGANIZATION_SLUG}/${BUILDKITE_PIPELINE_SLUG}" + TKEY="${BUILDKITE_ORGANIZATION_SLUG}/${BK_PIPELINE_SLUG}" BUCKET="${BUILDKITE_PLUGIN_CACHE_S3_BUCKET}/${TKEY}" BK_AWS_FOUND=false @@ -111,7 +112,7 @@ function restore() { function cache() { TAR_FILE="${CACHE_KEY}.${BK_TAR_EXTENSION}" - BUCKET="${BUILDKITE_PLUGIN_CACHE_S3_BUCKET}/${BUILDKITE_ORGANIZATION_SLUG}/${BUILDKITE_PIPELINE_SLUG}" + BUCKET="${BUILDKITE_PLUGIN_CACHE_S3_BUCKET}/${BUILDKITE_ORGANIZATION_SLUG}/${BK_PIPELINE_SLUG}" TAR_TARGETS="" if [ "${#paths[@]}" -eq 1 ]; then From 0d645dce8160bb5745b503dc0608b0abc422bd52 Mon Sep 17 00:00:00 2001 From: Phil Phillips Date: Mon, 14 Nov 2022 11:18:18 -0600 Subject: [PATCH 2/5] chore: add pipeline slug to shared lib --- lib/backends/s3.bash | 5 ++--- lib/shared.bash | 4 ++++ 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/lib/backends/s3.bash b/lib/backends/s3.bash index 41f4377..d510f1e 100644 --- a/lib/backends/s3.bash +++ b/lib/backends/s3.bash @@ -9,7 +9,6 @@ BK_TAR_ARGS=() BK_TAR_ADDITIONAL_ARGS="--ignore-failed-read" BK_TAR_EXTENSION="tar" BK_TAR_EXTRACT_ARGS="-xf" -BK_PIPELINE_SLUG=${BUILDKITE_PLUGIN_CACHE_SLUG_OVERRIDE:-${BUILDKITE_PIPELINE_SLUG}} if [[ ! "$OSTYPE" == "darwin"* ]]; then shell_exec=$( @@ -66,7 +65,7 @@ fi function restore() { TAR_FILE="${CACHE_KEY}.${BK_TAR_EXTENSION}" - TKEY="${BUILDKITE_ORGANIZATION_SLUG}/${BK_PIPELINE_SLUG}" + TKEY="${BUILDKITE_ORGANIZATION_SLUG}/$(pipeline_slug)" BUCKET="${BUILDKITE_PLUGIN_CACHE_S3_BUCKET}/${TKEY}" BK_AWS_FOUND=false @@ -112,7 +111,7 @@ function restore() { function cache() { TAR_FILE="${CACHE_KEY}.${BK_TAR_EXTENSION}" - BUCKET="${BUILDKITE_PLUGIN_CACHE_S3_BUCKET}/${BUILDKITE_ORGANIZATION_SLUG}/${BK_PIPELINE_SLUG}" + BUCKET="${BUILDKITE_PLUGIN_CACHE_S3_BUCKET}/${BUILDKITE_ORGANIZATION_SLUG}/$(pipeline_slug)" TAR_TARGETS="" if [ "${#paths[@]}" -eq 1 ]; then diff --git a/lib/shared.bash b/lib/shared.bash index b9640a5..f5cf384 100644 --- a/lib/shared.bash +++ b/lib/shared.bash @@ -88,3 +88,7 @@ function source_locating() { function cache_locating() { echo -e "${BK_LOG_PREFIX}🔍 Locating cache: $1" } + +function pipeline_slug() { + echo ${BUILDKITE_PLUGIN_CACHE_SLUG_OVERRIDE:-${BUILDKITE_PIPELINE_SLUG}} +} From 9b0aca5a52c47186463f30d3846f5fc4362b8c08 Mon Sep 17 00:00:00 2001 From: Phil Phillips Date: Mon, 14 Nov 2022 11:57:02 -0600 Subject: [PATCH 3/5] feat: extend pipeline slug override to other backends --- lib/backends/rsync.bash | 4 ++-- lib/backends/tarball.bash | 6 +++--- lib/shared.bash | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/backends/rsync.bash b/lib/backends/rsync.bash index 015b4d1..5908d7c 100644 --- a/lib/backends/rsync.bash +++ b/lib/backends/rsync.bash @@ -14,14 +14,14 @@ if [ -n "${BUILDKITE_PLUGIN_CACHE_RSYNC_PATH:-}" ]; then fi function restore() { - CACHE_PREFIX="${BK_BASE_DIR}/${BUILDKITE_ORGANIZATION_SLUG}/${BUILDKITE_PIPELINE_SLUG}" + CACHE_PREFIX="${BK_BASE_DIR}/${BUILDKITE_ORGANIZATION_SLUG}/$(pipeline_slug)" mkdir -p "${CACHE_PREFIX}/${CACHE_KEY}" rsync -a "$RSYNC_ARGS" "${CACHE_PREFIX}/${CACHE_KEY}/" . } function cache() { - CACHE_PREFIX="${BK_BASE_DIR}/${BUILDKITE_ORGANIZATION_SLUG}/${BUILDKITE_PIPELINE_SLUG}" + CACHE_PREFIX="${BK_BASE_DIR}/${BUILDKITE_ORGANIZATION_SLUG}/$(pipeline_slug)" mkdir -p "${CACHE_PREFIX}/${CACHE_KEY}/" if [ "${#paths[@]}" -eq 1 ]; then diff --git a/lib/backends/tarball.bash b/lib/backends/tarball.bash index 9998a33..6ea1b44 100644 --- a/lib/backends/tarball.bash +++ b/lib/backends/tarball.bash @@ -45,8 +45,8 @@ else fi function restore() { - CACHE_PREFIX="${BK_BASE_DIR}/${BUILDKITE_ORGANIZATION_SLUG}/${BUILDKITE_PIPELINE_SLUG}" - mkdir -p "${CACHE_PREFIX}/${BUILDKITE_PIPELINE_SLUG}" + CACHE_PREFIX="${BK_BASE_DIR}/${BUILDKITE_ORGANIZATION_SLUG}/$(pipeline_slug)" + mkdir -p "${CACHE_PREFIX}/$(pipeline_slug)" TAR_FILE="${CACHE_PREFIX}/${CACHE_KEY}.${BK_TAR_EXTENSION}" BK_TAR_FOUND=false @@ -88,7 +88,7 @@ function restore() { } function cache() { - CACHE_PREFIX="${BK_BASE_DIR}/${BUILDKITE_ORGANIZATION_SLUG}/${BUILDKITE_PIPELINE_SLUG}" + CACHE_PREFIX="${BK_BASE_DIR}/${BUILDKITE_ORGANIZATION_SLUG}/$(pipeline_slug)" mkdir -p "${CACHE_PREFIX}" DAYS="${BUILDKITE_PLUGIN_CACHE_TARBALL_MAX:-}" diff --git a/lib/shared.bash b/lib/shared.bash index f5cf384..29d56fa 100644 --- a/lib/shared.bash +++ b/lib/shared.bash @@ -90,5 +90,5 @@ function cache_locating() { } function pipeline_slug() { - echo ${BUILDKITE_PLUGIN_CACHE_SLUG_OVERRIDE:-${BUILDKITE_PIPELINE_SLUG}} + echo ${BUILDKITE_PLUGIN_CACHE_PIPELINE_SLUG_OVERRIDE:-${BUILDKITE_PIPELINE_SLUG}} } From 61e46859cbf53e75ce2cfbc62b52717aac53e3c7 Mon Sep 17 00:00:00 2001 From: Phil Phillips Date: Mon, 14 Nov 2022 12:09:57 -0600 Subject: [PATCH 4/5] docs: add a blurb about pipeline-slug-override --- README.md | 26 ++++++++++++++++++++++++++ lib/shared.bash | 3 +++ 2 files changed, 29 insertions(+) diff --git a/README.md b/README.md index 1ce05d7..5ce3f9f 100644 --- a/README.md +++ b/README.md @@ -34,6 +34,7 @@ Plus, In addition to tarball & rsync, we also do not re-create another tarball f - [Adjust compression level](#adjust-compression-level) - [Continue to cache on failed builds](#continue-to-cache-on-failed-builds) - [Multi-threaded compression](#multi-threaded-compression) + - [Sharing caches between pipelines](#sharing-caches-between-pipelines) - [Auto deletion old caches](#auto-deletion-old-caches) - [Globs on paths](#globs-on-paths) - [Roadmap](#roadmap) @@ -466,6 +467,31 @@ steps: compress-program: pigz # tar will use `pigz` to compress and benefit multithreading... ``` +## Sharing caches between pipelines + +If you have multiple pipelines that can benefit from referencing the same cache, you can use the `pipeline-slug-override` option: + +```yaml +steps: + - name: ':jest: Run tests' + key: jest + command: yarn test --runInBand + plugins: + - gencer/cache#v2.4.11: + id: ruby # or ruby-3.0 + backend: s3 + key: "v1-cache-{{ id }}-{{ runner.os }}-{{ checksum 'Gemfile.lock' }}" + restore-keys: + - 'v1-cache-{{ id }}-{{ runner.os }}-' + - 'v1-cache-{{ id }}-' + compress: 2 # fast compression. + s3: + bucket: s3-bucket + paths: + - bundle/vendor + pipeline-slug-override: "other-pipeline" # other-pipeline references the same Gemfile.lock +``` + ## Auto deletion old caches For tarballs, To keep caches and delete them in _for example_ 7 days, use `max: 7`. diff --git a/lib/shared.bash b/lib/shared.bash index 29d56fa..0ce2793 100644 --- a/lib/shared.bash +++ b/lib/shared.bash @@ -89,6 +89,9 @@ function cache_locating() { echo -e "${BK_LOG_PREFIX}🔍 Locating cache: $1" } +# Value to be used as the pipeline slug +# Returns: +# - String function pipeline_slug() { echo ${BUILDKITE_PLUGIN_CACHE_PIPELINE_SLUG_OVERRIDE:-${BUILDKITE_PIPELINE_SLUG}} } From 57089021564a2c511f313a4ceda6cade8c305014 Mon Sep 17 00:00:00 2001 From: Phil Phillips Date: Mon, 14 Nov 2022 12:22:32 -0600 Subject: [PATCH 5/5] chore: add missing quotes so that spell check linter is happy --- lib/shared.bash | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/shared.bash b/lib/shared.bash index 0ce2793..a8b5160 100644 --- a/lib/shared.bash +++ b/lib/shared.bash @@ -93,5 +93,5 @@ function cache_locating() { # Returns: # - String function pipeline_slug() { - echo ${BUILDKITE_PLUGIN_CACHE_PIPELINE_SLUG_OVERRIDE:-${BUILDKITE_PIPELINE_SLUG}} + echo "${BUILDKITE_PLUGIN_CACHE_PIPELINE_SLUG_OVERRIDE:-${BUILDKITE_PIPELINE_SLUG}}" }