From 54ec5ce187289570b6ce8018a432fad643b37a76 Mon Sep 17 00:00:00 2001 From: Michael Day <108079556+mike-day@users.noreply.github.com> Date: Wed, 8 Jan 2025 10:53:47 -0600 Subject: [PATCH] [CICD-761] Generate dynamic excludes for rsync (#42) * [CICD-761] Explicitly export color globals Resolves a warning about unused variables in the common file. * [CICD-761] Create tests/helpers dir Also moves common.sh to new utils dir. * [CICD-761] Create print_mount_paths util * [CICD-761] Generate excludes list dynamically Outputs the list to stdout, which is called with process substitution in the initial rsync command. Also disables debug mode for the block calling the generate function. This prevents unwanted debug output (specific to the excludes list generation) from being shown. * [CICD-761] Move to a simpler approach for unit tests * [CICD-761] Bump version with patch update * [CICD-761] Glob the current dir instead of checking mount paths Simplifies the logic for determining what files were copied to workdir by globbing for certain dirs instead of parsing proc/self/mountinfo. This is also better for other mounted dir structures (such as when a GHA runner mounts the image); we are simply checking what is in the current dir instead of inferring the structure based off mounted source path. * [CICD-761] Use only REMOTE_PATH for generating excludes The goal of the excludes list is to prevent users from accidentally breaking their site by overwriting or deleting important files on the server. While there are many files that could be deemed important, we explicily want to exclude a handful of files from the site root, some from wp-content, and some from wp-content/mu-plugins. In order to determine if a deploy is at risk of overwriting or deleting an important file on the server, we just need to look at the REMOTE_PATH. If REMOTE_PATH is unset, we know the destination is the site root. If REMOTE_PATH ends in wp-content, we know the destination is wp-content. If REMOTE_PATH ends in wp-content/mu-plugins, we know the destination is wp-content/mu-plugins. All three of these cases carry the risk of overwritting or deleting important files. In order to prevent this, we need to make the exclude rules relative to the REMOTE_PATH. SRC_PATH no longer matters because rsync assumes it to be identical to REMOTE_PATH for the sake of determining transfer/deletion deltas. * [CICD-761] Add additional cases for generated exclude from rules Tests how different REMOTE_PATHS affect the generated exclude rule output. * [CICD-761] nit: update verbiage for batch tests success message --------- Co-authored-by: Andrew Matthews --- .changeset/stale-flowers-hide.md | 5 + Dockerfile | 7 +- Makefile | 3 +- entrypoint.sh | 33 ++-- exclude.txt | 63 ------- tests/common.sh | 5 - .../excludes/exclude_from_mu_plugins.txt | 34 ++++ tests/fixtures/excludes/exclude_from_root.txt | 48 ++++++ .../exclude_from_safe_directories.txt | 21 +++ .../excludes/exclude_from_wp_content.txt | 48 ++++++ tests/helpers/common.sh | 6 + tests/helpers/log_debug.sh | 18 ++ tests/test_flag_formats.sh | 2 +- tests/test_functions.sh | 4 +- tests/test_generate_path_excludes.sh | 77 +++++++++ functions.sh => utils/functions.sh | 0 utils/generate_path_excludes.sh | 158 ++++++++++++++++++ 17 files changed, 443 insertions(+), 89 deletions(-) create mode 100644 .changeset/stale-flowers-hide.md delete mode 100644 exclude.txt delete mode 100644 tests/common.sh create mode 100644 tests/fixtures/excludes/exclude_from_mu_plugins.txt create mode 100644 tests/fixtures/excludes/exclude_from_root.txt create mode 100644 tests/fixtures/excludes/exclude_from_safe_directories.txt create mode 100644 tests/fixtures/excludes/exclude_from_wp_content.txt create mode 100644 tests/helpers/common.sh create mode 100644 tests/helpers/log_debug.sh create mode 100755 tests/test_generate_path_excludes.sh rename functions.sh => utils/functions.sh (100%) create mode 100644 utils/generate_path_excludes.sh diff --git a/.changeset/stale-flowers-hide.md b/.changeset/stale-flowers-hide.md new file mode 100644 index 0000000..2798100 --- /dev/null +++ b/.changeset/stale-flowers-hide.md @@ -0,0 +1,5 @@ +--- +"@wpengine/site-deploy": patch +--- + +Fixes a bug when deploying to dirs other than site root by dynamically generating file exclusion lists on the fly diff --git a/Dockerfile b/Dockerfile index d2939ed..36304af 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,13 +1,12 @@ FROM instrumentisto/rsync-ssh:alpine3.20 -# Intsall dependencies +# Install dependencies RUN apk update \ && apk upgrade \ && apk add --no-cache \ bash \ php \ && rm -rf /var/cache/apk/* -# Add entrypoint and excludes -ADD functions.sh /functions.sh +# Add entrypoint and utils +COPY utils /utils ADD entrypoint.sh /entrypoint.sh -ADD exclude.txt /exclude.txt ENTRYPOINT ["/entrypoint.sh"] diff --git a/Makefile b/Makefile index 6206910..c9ca11a 100644 --- a/Makefile +++ b/Makefile @@ -22,5 +22,6 @@ version: build docker image tag $(IMAGE) $(IMAGE_NAME):v$(MAJOR_VERSION).$(MINOR_VERSION) && \ docker image tag $(IMAGE) $(IMAGE_NAME):v$(MAJOR_VERSION).$(MINOR_VERSION).$(PATCH_VERSION) -test: +test-unit: ./tests/test_functions.sh + ./tests/test_generate_path_excludes.sh diff --git a/entrypoint.sh b/entrypoint.sh index 90edf79..3fd4a82 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -5,8 +5,9 @@ set -e # Get the directory of the current script SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -# Source the functions.sh file relative to the current script's location -source "${SCRIPT_DIR}/functions.sh" +# Source utils relative to the current script's location +source "${SCRIPT_DIR}/utils/functions.sh" +source "${SCRIPT_DIR}/utils/generate_path_excludes.sh" validate() { # mandatory params @@ -27,7 +28,7 @@ setup_env() { WPE_ENV_NAME="${PRD_ENV}"; elif [[ -n ${STG_ENV} ]]; then WPE_ENV_NAME="${STG_ENV}"; - elif [[ -n ${DEV_ENV} ]]; then + elif [[ -n ${DEV_ENV} ]]; then WPE_ENV_NAME="${DEV_ENV}"; else echo "Failure: Missing environment variable..." && exit 1; fi @@ -54,18 +55,18 @@ setup_env() { setup_ssh_dir() { echo "setup ssh path" - if [ ! -d "${HOME}/.ssh" ]; then - mkdir "${HOME}/.ssh" - SSH_PATH="${HOME}/.ssh" + if [ ! -d "${HOME}/.ssh" ]; then + mkdir "${HOME}/.ssh" + SSH_PATH="${HOME}/.ssh" mkdir "${SSH_PATH}/ctl/" - # Set Key Perms + # Set Key Perms chmod -R 700 "$SSH_PATH" - else + else SSH_PATH="${HOME}/.ssh" echo "using established SSH KEY path..."; fi - #Copy secret keys to container + #Copy secret keys to container WPE_SSHG_KEY_PRIVATE_PATH="${SSH_PATH}/wpe_id_rsa" if [ "${CICD_VENDOR}" == "wpe_bb" ]; then @@ -75,7 +76,7 @@ setup_ssh_dir() { fi chmod 600 "${WPE_SSHG_KEY_PRIVATE_PATH}" - #establish knownhosts + #establish knownhosts KNOWN_HOSTS_PATH="${SSH_PATH}/known_hosts" ssh-keyscan -t rsa "${WPE_SSH_HOST}" >> "${KNOWN_HOSTS_PATH}" chmod 644 "${KNOWN_HOSTS_PATH}" @@ -92,7 +93,7 @@ check_lint() { fi done echo "PHP Lint Successful! No errors detected!" - else + else echo "Skipping PHP Linting." fi } @@ -107,14 +108,20 @@ check_cache() { } sync_files() { + # Generate the excludes list before using the output with rsync + local exclude_from; exclude_from="$(generate_exclude_from)" + #create multiplex connection ssh -nNf -v -i "${WPE_SSHG_KEY_PRIVATE_PATH}" -o StrictHostKeyChecking=no -o ControlMaster=yes -o ControlPath="$SSH_PATH/ctl/%C" "$WPE_FULL_HOST" echo "!!! MULTIPLEX SSH CONNECTION ESTABLISHED !!!" set -x - rsync --rsh="ssh -v -p 22 -i ${WPE_SSHG_KEY_PRIVATE_PATH} -o StrictHostKeyChecking=no -o 'ControlPath=$SSH_PATH/ctl/%C'" "${FLAGS_ARRAY[@]}" --exclude-from='/exclude.txt' --chmod=D775,F664 "${SRC_PATH}" "${WPE_DESTINATION}" + rsync --rsh="ssh -v -p 22 -i ${WPE_SSHG_KEY_PRIVATE_PATH} -o StrictHostKeyChecking=no -o 'ControlPath=$SSH_PATH/ctl/%C'" \ + "${FLAGS_ARRAY[@]}" \ + --exclude-from=<( { { set +x; } 2>/dev/null; echo "$exclude_from"; } ) \ + --chmod=D775,F664 "${SRC_PATH}" "${WPE_DESTINATION}" set +x - + if [[ -n ${SCRIPT} || -n ${CACHE_CLEAR} ]]; then if [[ -n ${SCRIPT} ]]; then diff --git a/exclude.txt b/exclude.txt deleted file mode 100644 index 9d376d4..0000000 --- a/exclude.txt +++ /dev/null @@ -1,63 +0,0 @@ -# Version Control -# NOTE: -# WP Engine does not support server side versioning so hosting any version control -# on the server would not be advantageous. - -*~ -.git -.github -.gitignore -.DS_Store -.svn -.cvs -*.bak -*.swp -Thumbs.db - -# WordPress specific files -# NOTE: -# These files are excluded from the deploy so as to prevent unwanted errors from occurring, -# such as accidentally deploying a local version of wp-config.php or accidentally deleting -# wp-content/uploads/ if a --delete flag is passed while deploying root. Most paths here -# are ingnored in the WPE sample .gitignore per best practice. -wp-config.php -wp-content/uploads/ -wp-content/blogs.dir/ -wp-content/upgrade/* -wp-content/backup-db/* -wp-content/advanced-cache.php -wp-content/wp-cache-config.php -wp-content/cache/* -wp-content/cache/supercache/* - -# WP Engine specific files -# NOTE: -# These files are specific to running a WordPress site at WP Engine and would -# likely result in a broken production site if modified in production (in -# fact, permissions would prevent modification for many of these files). While -# some of these files (such as those in /_wpeprivate) would be extremely large -# and completely useless in the context of local WordPress development, others -# (such as some of the WP Engine managed plugins) might be useful in rare -# circumstances to have as a reference for debugging purposes. -.smushit-status -.gitattributes -.wpe-devkit/ -.wpengine-conf/ -_wpeprivate -wp-content/object-cache.php -wp-content/mu-plugins/mu-plugin.php -wp-content/mu-plugins/slt-force-strong-passwords.php -wp-content/mu-plugins/wpengine-security-auditor.php -wp-content/mu-plugins/stop-long-comments.php -wp-content/mu-plugins/force-strong-passwords* -wp-content/mu-plugins/wpengine-common* -wp-content/mu-plugins/wpe-wp-sign-on-plugin* -wp-content/mu-plugins/wpe-elasticpress-autosuggest-logger* -wp-content/mu-plugins/wpe-cache-plugin* -wp-content/mu-plugins/wp-cache-memcached* -wp-content/drop-ins/ -wp-content/drop-ins/wp-cache-memcached* -wp-content/mysql.sql - -# Local specific -wp-content/mu-plugins/local-by-flywheel-live-link-helper.php diff --git a/tests/common.sh b/tests/common.sh deleted file mode 100644 index 154e436..0000000 --- a/tests/common.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash - -GREEN='\033[0;32m' -RED='\033[0;31m' -NC='\033[0m' # No Color diff --git a/tests/fixtures/excludes/exclude_from_mu_plugins.txt b/tests/fixtures/excludes/exclude_from_mu_plugins.txt new file mode 100644 index 0000000..3900655 --- /dev/null +++ b/tests/fixtures/excludes/exclude_from_mu_plugins.txt @@ -0,0 +1,34 @@ +# Version Control +*~ +.git +.github +.gitignore +.DS_Store +.svn +.cvs +*.bak +*.swp +Thumbs.db + +# WordPress specific files +wp-config.php + +# WP Engine specific files and directories +.smushit-status +.gitattributes +.wpe-devkit/ +.wpengine-conf/ +_wpeprivate + +### Dynamic mu-plugin file and directory exclusions +/mu-plugin.php +/slt-force-strong-passwords.php +/wpengine-security-auditor.php +/stop-long-comments.php +/force-strong-passwords* +/wpengine-common* +/wpe-wp-sign-on-plugin* +/wpe-elasticpress-autosuggest-logger* +/wpe-cache-plugin* +/wp-cache-memcached* +/local-by-flywheel-live-link-helper.php diff --git a/tests/fixtures/excludes/exclude_from_root.txt b/tests/fixtures/excludes/exclude_from_root.txt new file mode 100644 index 0000000..69a9d8a --- /dev/null +++ b/tests/fixtures/excludes/exclude_from_root.txt @@ -0,0 +1,48 @@ +# Version Control +*~ +.git +.github +.gitignore +.DS_Store +.svn +.cvs +*.bak +*.swp +Thumbs.db + +# WordPress specific files +wp-config.php + +# WP Engine specific files and directories +.smushit-status +.gitattributes +.wpe-devkit/ +.wpengine-conf/ +_wpeprivate + +### Dynamic file and directory exclusions +/wp-content/uploads/ +/wp-content/blogs.dir/ +/wp-content/upgrade/* +/wp-content/backup-db/* +/wp-content/advanced-cache.php +/wp-content/wp-cache-config.php +/wp-content/cache/* +/wp-content/cache/supercache/* +/wp-content/object-cache.php +/wp-content/drop-ins/ +/wp-content/drop-ins/wp-cache-memcached* +/wp-content/mysql.sql + +### Dynamic mu-plugin file and directory exclusions +/wp-content/mu-plugins/mu-plugin.php +/wp-content/mu-plugins/slt-force-strong-passwords.php +/wp-content/mu-plugins/wpengine-security-auditor.php +/wp-content/mu-plugins/stop-long-comments.php +/wp-content/mu-plugins/force-strong-passwords* +/wp-content/mu-plugins/wpengine-common* +/wp-content/mu-plugins/wpe-wp-sign-on-plugin* +/wp-content/mu-plugins/wpe-elasticpress-autosuggest-logger* +/wp-content/mu-plugins/wpe-cache-plugin* +/wp-content/mu-plugins/wp-cache-memcached* +/wp-content/mu-plugins/local-by-flywheel-live-link-helper.php \ No newline at end of file diff --git a/tests/fixtures/excludes/exclude_from_safe_directories.txt b/tests/fixtures/excludes/exclude_from_safe_directories.txt new file mode 100644 index 0000000..4a73205 --- /dev/null +++ b/tests/fixtures/excludes/exclude_from_safe_directories.txt @@ -0,0 +1,21 @@ +# Version Control +*~ +.git +.github +.gitignore +.DS_Store +.svn +.cvs +*.bak +*.swp +Thumbs.db + +# WordPress specific files +wp-config.php + +# WP Engine specific files and directories +.smushit-status +.gitattributes +.wpe-devkit/ +.wpengine-conf/ +_wpeprivate diff --git a/tests/fixtures/excludes/exclude_from_wp_content.txt b/tests/fixtures/excludes/exclude_from_wp_content.txt new file mode 100644 index 0000000..4f5d899 --- /dev/null +++ b/tests/fixtures/excludes/exclude_from_wp_content.txt @@ -0,0 +1,48 @@ +# Version Control +*~ +.git +.github +.gitignore +.DS_Store +.svn +.cvs +*.bak +*.swp +Thumbs.db + +# WordPress specific files +wp-config.php + +# WP Engine specific files and directories +.smushit-status +.gitattributes +.wpe-devkit/ +.wpengine-conf/ +_wpeprivate + +### Dynamic file and directory exclusions +/uploads/ +/blogs.dir/ +/upgrade/* +/backup-db/* +/advanced-cache.php +/wp-cache-config.php +/cache/* +/cache/supercache/* +/object-cache.php +/drop-ins/ +/drop-ins/wp-cache-memcached* +/mysql.sql + +### Dynamic mu-plugin file and directory exclusions +/mu-plugins/mu-plugin.php +/mu-plugins/slt-force-strong-passwords.php +/mu-plugins/wpengine-security-auditor.php +/mu-plugins/stop-long-comments.php +/mu-plugins/force-strong-passwords* +/mu-plugins/wpengine-common* +/mu-plugins/wpe-wp-sign-on-plugin* +/mu-plugins/wpe-elasticpress-autosuggest-logger* +/mu-plugins/wpe-cache-plugin* +/mu-plugins/wp-cache-memcached* +/mu-plugins/local-by-flywheel-live-link-helper.php diff --git a/tests/helpers/common.sh b/tests/helpers/common.sh new file mode 100644 index 0000000..143fb80 --- /dev/null +++ b/tests/helpers/common.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +export GREEN='\033[0;32m' +export RED='\033[0;31m' +export BLUE='\033[0;34m' +export NC='\033[0m' # No Color diff --git a/tests/helpers/log_debug.sh b/tests/helpers/log_debug.sh new file mode 100644 index 0000000..addc19d --- /dev/null +++ b/tests/helpers/log_debug.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +set -e + +log_debug() { + local subject; subject="$1" + local message; message="$2" + + cat <