[CICD-761] Generate dynamic excludes for rsync (#42)

* [CICD-761] Explicitly export color globals

Resolves a warning about unused variables in the common file.

* [CICD-761] Create tests/helpers dir

Also moves common.sh to new utils dir.

* [CICD-761] Create print_mount_paths util

* [CICD-761] Generate excludes list dynamically

Outputs the list to stdout, which is called with process substitution in the initial rsync command.

Also disables debug mode for the block calling the generate function. This prevents unwanted debug output
(specific to the excludes list generation) from being shown.

* [CICD-761] Move to a simpler approach for unit tests

* [CICD-761] Bump version with patch update

* [CICD-761] Glob the current dir instead of checking  mount paths

Simplifies the logic for determining what files were copied to workdir by globbing for certain dirs instead of parsing proc/self/mountinfo.

This is also better for other mounted dir structures (such as when a GHA runner mounts the image); we are simply checking what is in the current dir instead of inferring the structure based off mounted source path.

* [CICD-761] Use only REMOTE_PATH for generating excludes

The goal of the excludes list is to prevent users from accidentally
breaking their site by overwriting or deleting important files on
the server.

While there are many files that could be deemed important, we explicily
want to exclude a handful of files from the site root, some from wp-content,
and some from wp-content/mu-plugins.

In order to determine if a deploy is at risk of overwriting or deleting an
important file on the server, we just need to look at the REMOTE_PATH.

If REMOTE_PATH is unset, we know the destination is the site root.
If REMOTE_PATH ends in wp-content, we know the destination is wp-content.
If REMOTE_PATH ends in wp-content/mu-plugins, we know the destination is
 wp-content/mu-plugins.

All three of these cases carry the risk of overwritting or deleting
important files. In order to prevent this, we need to make the exclude
rules relative to the REMOTE_PATH.

SRC_PATH no longer matters because rsync assumes it to be identical to
REMOTE_PATH for the sake of determining transfer/deletion deltas.

* [CICD-761] Add additional cases for generated exclude from rules

Tests how different REMOTE_PATHS affect the generated exclude
rule output.

* [CICD-761] nit: update verbiage for batch tests success message

---------

Co-authored-by: Andrew Matthews <andrew.matthews@wpengine.com>

.changeset/stale-flowers-hide.md

@@ -0,0 +1,5 @@
+---
+"@wpengine/site-deploy": patch
+---
+
+Fixes a bug when deploying to dirs other than site root by dynamically generating file exclusion lists on the fly

Dockerfile

@@ -1,13 +1,12 @@
 FROM instrumentisto/rsync-ssh:alpine3.20
-# Intsall dependencies
+# Install dependencies
 RUN apk update \
  && apk upgrade \
  && apk add --no-cache \
             bash \
             php \
  && rm -rf /var/cache/apk/*
-# Add entrypoint and excludes
-ADD functions.sh /functions.sh
+# Add entrypoint and utils
+COPY utils /utils
 ADD entrypoint.sh /entrypoint.sh
-ADD exclude.txt /exclude.txt
 ENTRYPOINT ["/entrypoint.sh"]

Makefile

@@ -22,5 +22,6 @@ version: build
 	docker image tag $(IMAGE) $(IMAGE_NAME):v$(MAJOR_VERSION).$(MINOR_VERSION) && \
 	docker image tag $(IMAGE) $(IMAGE_NAME):v$(MAJOR_VERSION).$(MINOR_VERSION).$(PATCH_VERSION)
 
-test:
+test-unit:
 	./tests/test_functions.sh
+	./tests/test_generate_path_excludes.sh

entrypoint.sh

@@ -5,8 +5,9 @@ set -e
 # Get the directory of the current script
 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 
-# Source the functions.sh file relative to the current script's location
-source "${SCRIPT_DIR}/functions.sh"
+# Source utils relative to the current script's location
+source "${SCRIPT_DIR}/utils/functions.sh"
+source "${SCRIPT_DIR}/utils/generate_path_excludes.sh"
 
 validate() {
   # mandatory params
@@ -27,7 +28,7 @@ setup_env() {
       WPE_ENV_NAME="${PRD_ENV}";
     elif [[ -n ${STG_ENV} ]]; then
       WPE_ENV_NAME="${STG_ENV}";
-    elif [[ -n ${DEV_ENV} ]]; then  
+    elif [[ -n ${DEV_ENV} ]]; then
       WPE_ENV_NAME="${DEV_ENV}";
     else echo "Failure: Missing environment variable..."  && exit 1;
   fi
@@ -54,18 +55,18 @@ setup_env() {
 setup_ssh_dir() {
   echo "setup ssh path"
 
-  if [ ! -d "${HOME}/.ssh" ]; then 
-      mkdir "${HOME}/.ssh" 
-      SSH_PATH="${HOME}/.ssh" 
+  if [ ! -d "${HOME}/.ssh" ]; then
+      mkdir "${HOME}/.ssh"
+      SSH_PATH="${HOME}/.ssh"
       mkdir "${SSH_PATH}/ctl/"
-      # Set Key Perms 
+      # Set Key Perms
       chmod -R 700 "$SSH_PATH"
-    else 
+    else
       SSH_PATH="${HOME}/.ssh"
       echo "using established SSH KEY path...";
   fi
 
-  #Copy secret keys to container 
+  #Copy secret keys to container
   WPE_SSHG_KEY_PRIVATE_PATH="${SSH_PATH}/wpe_id_rsa"
 
   if [ "${CICD_VENDOR}" == "wpe_bb" ]; then
@@ -75,7 +76,7 @@ setup_ssh_dir() {
   fi
 
   chmod 600 "${WPE_SSHG_KEY_PRIVATE_PATH}"
-  #establish knownhosts 
+  #establish knownhosts
   KNOWN_HOSTS_PATH="${SSH_PATH}/known_hosts"
   ssh-keyscan -t rsa "${WPE_SSH_HOST}" >> "${KNOWN_HOSTS_PATH}"
   chmod 644 "${KNOWN_HOSTS_PATH}"
@@ -92,7 +93,7 @@ check_lint() {
           fi
       done
       echo "PHP Lint Successful! No errors detected!"
-  else 
+  else
       echo "Skipping PHP Linting."
   fi
 }
@@ -107,14 +108,20 @@ check_cache() {
 }
 
 sync_files() {
+  # Generate the excludes list before using the output with rsync
+  local exclude_from; exclude_from="$(generate_exclude_from)"
+
   #create multiplex connection 
   ssh -nNf -v -i "${WPE_SSHG_KEY_PRIVATE_PATH}" -o StrictHostKeyChecking=no -o ControlMaster=yes -o ControlPath="$SSH_PATH/ctl/%C" "$WPE_FULL_HOST"
   echo "!!! MULTIPLEX SSH CONNECTION ESTABLISHED !!!"
 
   set -x
-  rsync --rsh="ssh -v -p 22 -i ${WPE_SSHG_KEY_PRIVATE_PATH} -o StrictHostKeyChecking=no -o 'ControlPath=$SSH_PATH/ctl/%C'" "${FLAGS_ARRAY[@]}" --exclude-from='/exclude.txt' --chmod=D775,F664 "${SRC_PATH}" "${WPE_DESTINATION}"
+  rsync --rsh="ssh -v -p 22 -i ${WPE_SSHG_KEY_PRIVATE_PATH} -o StrictHostKeyChecking=no -o 'ControlPath=$SSH_PATH/ctl/%C'" \
+        "${FLAGS_ARRAY[@]}" \
+        --exclude-from=<( { { set +x; } 2>/dev/null; echo "$exclude_from"; } ) \
+        --chmod=D775,F664 "${SRC_PATH}" "${WPE_DESTINATION}"
   set +x
-  
+
   if [[ -n ${SCRIPT} || -n ${CACHE_CLEAR} ]]; then
 
       if [[ -n ${SCRIPT} ]]; then

tests/fixtures/excludes/exclude_from_mu_plugins.txt

@@ -0,0 +1,34 @@
+# Version Control
+*~
+.git
+.github
+.gitignore
+.DS_Store
+.svn
+.cvs
+*.bak
+*.swp
+Thumbs.db
+
+# WordPress specific files
+wp-config.php
+
+# WP Engine specific files and directories
+.smushit-status
+.gitattributes
+.wpe-devkit/
+.wpengine-conf/
+_wpeprivate
+
+### Dynamic mu-plugin file and directory exclusions
+/mu-plugin.php
+/slt-force-strong-passwords.php
+/wpengine-security-auditor.php
+/stop-long-comments.php
+/force-strong-passwords*
+/wpengine-common*
+/wpe-wp-sign-on-plugin*
+/wpe-elasticpress-autosuggest-logger*
+/wpe-cache-plugin*
+/wp-cache-memcached*
+/local-by-flywheel-live-link-helper.php

tests/fixtures/excludes/exclude_from_root.txt

@@ -0,0 +1,48 @@
+# Version Control
+*~
+.git
+.github
+.gitignore
+.DS_Store
+.svn
+.cvs
+*.bak
+*.swp
+Thumbs.db
+
+# WordPress specific files
+wp-config.php
+
+# WP Engine specific files and directories
+.smushit-status
+.gitattributes
+.wpe-devkit/
+.wpengine-conf/
+_wpeprivate
+
+### Dynamic file and directory exclusions
+/wp-content/uploads/
+/wp-content/blogs.dir/
+/wp-content/upgrade/*
+/wp-content/backup-db/*
+/wp-content/advanced-cache.php
+/wp-content/wp-cache-config.php
+/wp-content/cache/*
+/wp-content/cache/supercache/*
+/wp-content/object-cache.php
+/wp-content/drop-ins/
+/wp-content/drop-ins/wp-cache-memcached*
+/wp-content/mysql.sql
+
+### Dynamic mu-plugin file and directory exclusions
+/wp-content/mu-plugins/mu-plugin.php
+/wp-content/mu-plugins/slt-force-strong-passwords.php
+/wp-content/mu-plugins/wpengine-security-auditor.php
+/wp-content/mu-plugins/stop-long-comments.php
+/wp-content/mu-plugins/force-strong-passwords*
+/wp-content/mu-plugins/wpengine-common*
+/wp-content/mu-plugins/wpe-wp-sign-on-plugin*
+/wp-content/mu-plugins/wpe-elasticpress-autosuggest-logger*
+/wp-content/mu-plugins/wpe-cache-plugin*
+/wp-content/mu-plugins/wp-cache-memcached*
+/wp-content/mu-plugins/local-by-flywheel-live-link-helper.php

tests/fixtures/excludes/exclude_from_safe_directories.txt

@@ -0,0 +1,21 @@
+# Version Control
+*~
+.git
+.github
+.gitignore
+.DS_Store
+.svn
+.cvs
+*.bak
+*.swp
+Thumbs.db
+
+# WordPress specific files
+wp-config.php
+
+# WP Engine specific files and directories
+.smushit-status
+.gitattributes
+.wpe-devkit/
+.wpengine-conf/
+_wpeprivate

tests/fixtures/excludes/exclude_from_wp_content.txt

@@ -0,0 +1,48 @@
+# Version Control
+*~
+.git
+.github
+.gitignore
+.DS_Store
+.svn
+.cvs
+*.bak
+*.swp
+Thumbs.db
+
+# WordPress specific files
+wp-config.php
+
+# WP Engine specific files and directories
+.smushit-status
+.gitattributes
+.wpe-devkit/
+.wpengine-conf/
+_wpeprivate
+
+### Dynamic file and directory exclusions
+/uploads/
+/blogs.dir/
+/upgrade/*
+/backup-db/*
+/advanced-cache.php
+/wp-cache-config.php
+/cache/*
+/cache/supercache/*
+/object-cache.php
+/drop-ins/
+/drop-ins/wp-cache-memcached*
+/mysql.sql
+
+### Dynamic mu-plugin file and directory exclusions
+/mu-plugins/mu-plugin.php
+/mu-plugins/slt-force-strong-passwords.php
+/mu-plugins/wpengine-security-auditor.php
+/mu-plugins/stop-long-comments.php
+/mu-plugins/force-strong-passwords*
+/mu-plugins/wpengine-common*
+/mu-plugins/wpe-wp-sign-on-plugin*
+/mu-plugins/wpe-elasticpress-autosuggest-logger*
+/mu-plugins/wpe-cache-plugin*
+/mu-plugins/wp-cache-memcached*
+/mu-plugins/local-by-flywheel-live-link-helper.php

tests/helpers/common.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+export GREEN='\033[0;32m'
+export RED='\033[0;31m'
+export BLUE='\033[0;34m'
+export NC='\033[0m' # No Color

tests/helpers/log_debug.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+set -e
+
+log_debug() {
+  local subject; subject="$1"
+  local message; message="$2"
+
+  cat <<EOF
+
+####################
+$subject
+
+$message
+####################
+
+EOF
+}

tests/test_flag_formats.sh

@@ -12,7 +12,7 @@
 # Get the directory of the current script
 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 
-source "${SCRIPT_DIR}/common.sh"
+source "${SCRIPT_DIR}/helpers/common.sh"
 
 set -x
 

tests/test_functions.sh

@@ -3,8 +3,8 @@
 # Get the directory of the current script
 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 
-source "${SCRIPT_DIR}/common.sh"
-source "${SCRIPT_DIR}/../functions.sh"
+source "${SCRIPT_DIR}/helpers/common.sh"
+source "${SCRIPT_DIR}/../utils/functions.sh"
 
 # First argument represents the value of the FLAGS variable.
 # The rest of the arguments represent the expected values of the FLAGS_ARRAY.