From 632088ca486ff0ac89de1b5497cf022c25992573 Mon Sep 17 00:00:00 2001 From: Andrew Matthews Date: Tue, 7 Jan 2025 17:31:03 -0500 Subject: [PATCH] [CICD-761] Use only REMOTE_PATH for generating excludes The goal of the excludes list is to prevent users from accidentally breaking their site by overwriting or deleting important files on the server. While there are many files that could be deemed important, we explicily want to exclude a handful of files from the site root, some from wp-content, and some from wp-content/mu-plugins. In order to determine if a deploy is at risk of overwriting or deleting an important file on the server, we just need to look at the REMOTE_PATH. If REMOTE_PATH is unset, we know the destination is the site root. If REMOTE_PATH ends in wp-content, we know the destination is wp-content. If REMOTE_PATH ends in wp-content/mu-plugins, we know the destination is wp-content/mu-plugins. All three of these cases carry the risk of overwritting or deleting important files. In order to prevent this, we need to make the exclude rules relative to the REMOTE_PATH. SRC_PATH no longer matters because rsync assumes it to be identical to REMOTE_PATH for the sake of determining transfer/deletion deltas. --- entrypoint.sh | 8 +- ...urce_exclude_from.txt => exclude_from.txt} | 0 tests/fixtures/excludes/remote_excludes.txt | 1 - .../src/wp-content/mu-plugins/.gitkeep | 0 .../fixtures/src/wp-content/plugins/.gitkeep | 0 tests/fixtures/wpengine-common/.gitkeep | 0 tests/helpers/change_working_dir.sh | 17 --- tests/test_exclude_from.sh | 0 tests/test_generate_path_excludes.sh | 108 ++++-------------- utils/generate_path_excludes.sh | 106 ++++------------- 10 files changed, 45 insertions(+), 195 deletions(-) rename tests/fixtures/excludes/{source_exclude_from.txt => exclude_from.txt} (100%) delete mode 100644 tests/fixtures/excludes/remote_excludes.txt delete mode 100644 tests/fixtures/src/wp-content/mu-plugins/.gitkeep delete mode 100644 tests/fixtures/src/wp-content/plugins/.gitkeep delete mode 100644 tests/fixtures/wpengine-common/.gitkeep delete mode 100644 tests/helpers/change_working_dir.sh delete mode 100755 tests/test_exclude_from.sh diff --git a/entrypoint.sh b/entrypoint.sh index 0985cfe..1dc6f03 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -109,16 +109,16 @@ check_cache() { sync_files() { # Generate the excludes list before using the output with rsync - local source_exclude_from; source_exclude_from="$(generate_source_exclude_from)" - local remote_excludes; remote_excludes="$(generate_remote_excludes)" + local exclude_from; exclude_from="$(generate_exclude_from)" #create multiplex connection ssh -nNf -v -i "${WPE_SSHG_KEY_PRIVATE_PATH}" -o StrictHostKeyChecking=no -o ControlMaster=yes -o ControlPath="$SSH_PATH/ctl/%C" "$WPE_FULL_HOST" echo "!!! MULTIPLEX SSH CONNECTION ESTABLISHED !!!" set -x - rsync --rsh="ssh -v -p 22 -i ${WPE_SSHG_KEY_PRIVATE_PATH} -o StrictHostKeyChecking=no -o 'ControlPath=$SSH_PATH/ctl/%C'" "${FLAGS_ARRAY[@]}" \ - --exclude-from=<( { { set +x; } 2>/dev/null; echo "$source_exclude_from"; } ) --rsync-path="rsync $remote_excludes" \ + rsync --rsh="ssh -v -p 22 -i ${WPE_SSHG_KEY_PRIVATE_PATH} -o StrictHostKeyChecking=no -o 'ControlPath=$SSH_PATH/ctl/%C'" \ + "${FLAGS_ARRAY[@]}" \ + --exclude-from=<( { { set +x; } 2>/dev/null; echo "$exclude_from"; } ) \ --chmod=D775,F664 "${SRC_PATH}" "${WPE_DESTINATION}" set +x diff --git a/tests/fixtures/excludes/source_exclude_from.txt b/tests/fixtures/excludes/exclude_from.txt similarity index 100% rename from tests/fixtures/excludes/source_exclude_from.txt rename to tests/fixtures/excludes/exclude_from.txt diff --git a/tests/fixtures/excludes/remote_excludes.txt b/tests/fixtures/excludes/remote_excludes.txt deleted file mode 100644 index 24750a2..0000000 --- a/tests/fixtures/excludes/remote_excludes.txt +++ /dev/null @@ -1 +0,0 @@ ---exclude='wp-config.php' --exclude='.smushit-status' --exclude='.gitattributes' --exclude='.wpe-devkit/' --exclude='.wpengine-conf/' --exclude='_wpeprivate' --exclude='/wp-content/uploads/' --exclude='/wp-content/blogs.dir/' --exclude='/wp-content/upgrade/*' --exclude='/wp-content/backup-db/*' --exclude='/wp-content/advanced-cache.php' --exclude='/wp-content/wp-cache-config.php' --exclude='/wp-content/cache/*' --exclude='/wp-content/cache/supercache/*' --exclude='/wp-content/object-cache.php' --exclude='/wp-content/drop-ins/' --exclude='/wp-content/drop-ins/wp-cache-memcached*' --exclude='/wp-content/mysql.sql' --exclude='/wp-content/mu-plugins/mu-plugin.php' --exclude='/wp-content/mu-plugins/slt-force-strong-passwords.php' --exclude='/wp-content/mu-plugins/wpengine-security-auditor.php' --exclude='/wp-content/mu-plugins/stop-long-comments.php' --exclude='/wp-content/mu-plugins/force-strong-passwords*' --exclude='/wp-content/mu-plugins/wpengine-common*' --exclude='/wp-content/mu-plugins/wpe-wp-sign-on-plugin*' --exclude='/wp-content/mu-plugins/wpe-elasticpress-autosuggest-logger*' --exclude='/wp-content/mu-plugins/wpe-cache-plugin*' --exclude='/wp-content/mu-plugins/wp-cache-memcached*' --exclude='/wp-content/mu-plugins/local-by-flywheel-live-link-helper.php' \ No newline at end of file diff --git a/tests/fixtures/src/wp-content/mu-plugins/.gitkeep b/tests/fixtures/src/wp-content/mu-plugins/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/tests/fixtures/src/wp-content/plugins/.gitkeep b/tests/fixtures/src/wp-content/plugins/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/tests/fixtures/wpengine-common/.gitkeep b/tests/fixtures/wpengine-common/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/tests/helpers/change_working_dir.sh b/tests/helpers/change_working_dir.sh deleted file mode 100644 index 40e32ff..0000000 --- a/tests/helpers/change_working_dir.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/bash - -set -e - -# Change the working directory to the target directory and run the given command -change_working_dir() { - local target_dir=$1 - local orig_dir; orig_dir=$(pwd) - shift - - ( - cd "$target_dir" || exit 1 - # Restore to the original directory after the command is run - trap 'cd "$orig_dir"' EXIT - "$@" - ) -} diff --git a/tests/test_exclude_from.sh b/tests/test_exclude_from.sh deleted file mode 100755 index e69de29..0000000 diff --git a/tests/test_generate_path_excludes.sh b/tests/test_generate_path_excludes.sh index c15a012..38d4031 100755 --- a/tests/test_generate_path_excludes.sh +++ b/tests/test_generate_path_excludes.sh @@ -6,58 +6,10 @@ set -e SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" source "${SCRIPT_DIR}/helpers/common.sh" -source "${SCRIPT_DIR}/helpers/change_working_dir.sh" source "${SCRIPT_DIR}/../utils/generate_path_excludes.sh" -test_determine_source_exclude_paths() { - run_test_determine_source_exclude_paths() { - export SRC_PATH=$1 - local expected_base_path=$2 - local expected_mu_dir_path=$3 - - { - IFS= read -r base_path || base_path="" - IFS= read -r mu_dir_path || mu_dir_path="" - } <<< "$(determine_source_exclude_paths)" - - if [[ "${base_path}" != "${expected_base_path}" || "${mu_dir_path}" != "${expected_mu_dir_path}" ]]; then - echo -e "${RED}Test failed for SRC_PATH='${SRC_PATH}', pwd='$(pwd)': expected '$expected_base_path, $expected_mu_dir_path', got '$base_path, $mu_dir_path'.${NC}" - exit 1 - fi - } - - # Workdir -> src directory - change_working_dir "tests/fixtures/src" run_test_determine_source_exclude_paths "wp-content" "/wp-content/" "/wp-content/mu-plugins/" - change_working_dir "tests/fixtures/src" run_test_determine_source_exclude_paths "wp-content/" "/" "/mu-plugins/" - change_working_dir "tests/fixtures/src" run_test_determine_source_exclude_paths "wp-content/mu-plugins" "/" "/mu-plugins/" - change_working_dir "tests/fixtures/src" run_test_determine_source_exclude_paths "wp-content/mu-plugins/" "" "/" - change_working_dir "tests/fixtures/src" run_test_determine_source_exclude_paths "wp-content/plugins" "" "" - change_working_dir "tests/fixtures/src" run_test_determine_source_exclude_paths "wp-content/plugins/" "" "" - change_working_dir "tests/fixtures/src" run_test_determine_source_exclude_paths "." "/wp-content/" "/wp-content/mu-plugins/" - change_working_dir "tests/fixtures/src" run_test_determine_source_exclude_paths "" "/wp-content/" "/wp-content/mu-plugins/" - - # Workdir -> wp-content directory - change_working_dir "tests/fixtures/src/wp-content" run_test_determine_source_exclude_paths "mu-plugins" "/" "/mu-plugins/" - change_working_dir "tests/fixtures/src/wp-content" run_test_determine_source_exclude_paths "mu-plugins/" "" "/" - change_working_dir "tests/fixtures/src/wp-content" run_test_determine_source_exclude_paths "plugins" "" "" - change_working_dir "tests/fixtures/src/wp-content" run_test_determine_source_exclude_paths "plugins/" "" "" - change_working_dir "tests/fixtures/src/wp-content" run_test_determine_source_exclude_paths "." "/" "/mu-plugins/" - change_working_dir "tests/fixtures/src/wp-content" run_test_determine_source_exclude_paths "" "/" "/mu-plugins/" - - # Workdir -> mu-plugins directory - change_working_dir "tests/fixtures/src/wp-content/mu-plugins" run_test_determine_source_exclude_paths "." "" "/" - - # Workdir -> plugins directory - change_working_dir "tests/fixtures/src/wp-content/plugins" run_test_determine_source_exclude_paths "." "" "" - - # Workdir -> fixtures directory (checks that glob_current_dir is working (wpengine-common should be detected)) - change_working_dir "tests/fixtures" run_test_determine_source_exclude_paths "." "" "/" - - echo -e "${GREEN}All tests passed for determining the source excludes path.${NC}" -} - -test_determine_remote_exclude_paths() { - run_test_determine_remote_exclude_paths() { +test_determine_exclude_paths() { + run_test_determine_exclude_paths() { export REMOTE_PATH=$1 local expected_base_path=$2 local expected_mu_dir_path=$3 @@ -65,54 +17,36 @@ test_determine_remote_exclude_paths() { { IFS= read -r base_path || base_path="" IFS= read -r mu_dir_path || mu_dir_path="" - } <<< "$(determine_remote_exclude_paths)" + } <<< "$(determine_exclude_paths)" if [[ "${base_path}" != "${expected_base_path}" || "${mu_dir_path}" != "${expected_mu_dir_path}" ]]; then - echo -e "${RED}Test failed for REMOTE_PATH='${SRC_PATH}', pwd='$(pwd)': expected '$expected_base_path, $expected_mu_dir_path', got '$base_path, $mu_dir_path'.${NC}" + echo -e "${RED}Test failed for REMOTE_PATH='${REMOTE_PATH}'': expected '$expected_base_path, $expected_mu_dir_path', got '$base_path, $mu_dir_path'.${NC}" exit 1 fi } - run_test_determine_remote_exclude_paths "" "/wp-content/" "/wp-content/mu-plugins/" - run_test_determine_remote_exclude_paths "wp-content" "/" "/mu-plugins/" - run_test_determine_remote_exclude_paths "wp-content/" "/" "/mu-plugins/" - run_test_determine_remote_exclude_paths "wp-content/mu-plugins" "" "/" - run_test_determine_remote_exclude_paths "wp-content/mu-plugins/" "" "/" - run_test_determine_remote_exclude_paths "wp-content/plugins/" "" "" - - echo -e "${GREEN}All tests passed for determining the remote excludes path.${NC}" -} - -test_generate_source_exclude_from() { - export SRC_PATH="." - local output - local expected_output - - output=$(change_working_dir "tests/fixtures/src" generate_source_exclude_from) - expected_output=$(cat "tests/fixtures/excludes/source_exclude_from.txt") - - if [[ "$output" != "$expected_output" ]]; then - echo -e "${RED}Test failed for pwd='$(pwd)': generated output does not match expected output.${NC}" - echo -e "${BLUE}Generated output:${NC}" - echo "$output" - echo -e "${BLUE}Expected output:${NC}" - echo "$expected_output" - exit 1 - fi + run_test_determine_exclude_paths "." "/wp-content/" "/wp-content/mu-plugins/" + run_test_determine_exclude_paths "" "/wp-content/" "/wp-content/mu-plugins/" + run_test_determine_exclude_paths "wp-content" "/" "/mu-plugins/" + run_test_determine_exclude_paths "wp-content/" "/" "/mu-plugins/" + run_test_determine_exclude_paths "wp-content/mu-plugins" "" "/" + run_test_determine_exclude_paths "wp-content/mu-plugins/" "" "/" + run_test_determine_exclude_paths "wp-content/plugins" "" "" + run_test_determine_exclude_paths "wp-content/plugins/" "" "" - echo -e "${GREEN}Test passed for source exclude from: generated output matches expected output.${NC}" + echo -e "${GREEN}All tests passed for determining the excludes path.${NC}" } -test_generate_remote_excludes() { +test_generate_exclude_from() { export REMOTE_PATH="" local output local expected_output - output=$(change_working_dir "tests/fixtures/src" generate_remote_excludes) - expected_output=$(cat "tests/fixtures/excludes/remote_excludes.txt") + output=$(generate_exclude_from) + expected_output=$(cat "tests/fixtures/excludes/exclude_from.txt") if [[ "$output" != "$expected_output" ]]; then - echo -e "${RED}Test failed for pwd='$(pwd)': generated output does not match expected output.${NC}" + echo -e "${RED}Test failed': generated output does not match expected output.${NC}" echo -e "${BLUE}Generated output:${NC}" echo "$output" echo -e "${BLUE}Expected output:${NC}" @@ -120,14 +54,12 @@ test_generate_remote_excludes() { exit 1 fi - echo -e "${GREEN}Test passed for source exclude from: generated output matches expected output.${NC}" + echo -e "${GREEN}Test passed for generating exclude-from rules: generated output matches expected output.${NC}" } main() { - test_determine_source_exclude_paths - test_determine_remote_exclude_paths - test_generate_source_exclude_from - test_generate_remote_excludes + test_determine_exclude_paths + test_generate_exclude_from } main diff --git a/utils/generate_path_excludes.sh b/utils/generate_path_excludes.sh index f937b99..7af65ff 100644 --- a/utils/generate_path_excludes.sh +++ b/utils/generate_path_excludes.sh @@ -5,7 +5,7 @@ shopt -s extglob # Dynamic Excludes # -# This script generates two lists of files to exclude; one for the source, and one for the remote. +# This script generates a list of files to exclude from the rsync deployment. # # Notes about excluded files: # @@ -31,84 +31,37 @@ shopt -s extglob # (such as some of the WP Engine managed plugins) might be useful in rare # circumstances to have as a reference for debugging purposes. -determine_source_exclude_paths() { - local base_path - local mu_dir_path +determine_exclude_paths() { + local base_path; base_path="/wp-content/" + local mu_dir_path; mu_dir_path="/wp-content/mu-plugins/" - parse_src_path() { - case "${SRC_PATH}" in - wp-content ) - base_path="/wp-content/" - mu_dir_path="/wp-content/mu-plugins/" - ;; - wp-content/ | *mu-plugins ) - base_path="/" - mu_dir_path="/mu-plugins/" - ;; - *mu-plugins/ ) - mu_dir_path="/" - ;; - esac + remote_path_is_set() { + [[ -n "${REMOTE_PATH}" && "${REMOTE_PATH}" != '.' ]] } - parse_current_path() { - case "$(pwd)" in - *wp-content ) - base_path="/" - mu_dir_path="/mu-plugins/" - ;; - *mu-plugins ) - mu_dir_path="/" - ;; - esac - } - - glob_current_dir() { - if [[ -d wp-content ]]; then - base_path="/wp-content/" - mu_dir_path="/wp-content/mu-plugins/" - elif [[ -d mu-plugins ]]; then - base_path="/" - mu_dir_path="/mu-plugins/" - elif [[ -d wpengine-common ]]; then - mu_dir_path="/" - fi - } - - # If SRC_PATH is set, use it to determine base and mu-plugins paths. - # Otherwise, use the current directory or check its contents to set the paths. - if [[ -n "${SRC_PATH}" && "${SRC_PATH}" != '.' ]]; then - parse_src_path - elif [[ "$(pwd)" == *wp-content* ]]; then - parse_current_path - else - glob_current_dir - fi - - printf "%s\n%s\n" "$base_path" "$mu_dir_path" -} - -determine_remote_exclude_paths() { - local base_path - local mu_dir_path - - parse_remote_path() { - case "$REMOTE_PATH" in + make_exclude_paths_relative_to_remote() { + case "${REMOTE_PATH}" in wp-content?(/) ) + # REMOTE_PATH is the wp-content directory base_path="/" mu_dir_path="/mu-plugins/" ;; wp-content/mu-plugins?(/) ) + # REMOTE_PATH is the mu-plugins directory + base_path="" mu_dir_path="/" ;; + * ) + # REMOTE_PATH is set, but it's not the site root, wp-content, or mu-plugins. + # No dynamic excludes needed. + base_path="" + mu_dir_path="" + ;; esac } - if [[ -z "${REMOTE_PATH}" ]]; then - base_path="/wp-content/" - mu_dir_path="/wp-content/mu-plugins/" - else - parse_remote_path + if remote_path_is_set; then + make_exclude_paths_relative_to_remote fi printf "%s\n%s\n" "$base_path" "$mu_dir_path" @@ -175,8 +128,8 @@ print_dynamic_excludes() { echo -e "$output" } -generate_source_exclude_from() { - local dynamic_excludes; dynamic_excludes=$(print_dynamic_excludes determine_source_exclude_paths "\n") +generate_exclude_from() { + local dynamic_excludes; dynamic_excludes=$(print_dynamic_excludes determine_exclude_paths "\n") cat << EOF # Version Control @@ -203,20 +156,3 @@ _wpeprivate $(echo -e "$dynamic_excludes") EOF } - -generate_remote_excludes() { - local dynamic_excludes; dynamic_excludes=$(print_dynamic_excludes determine_remote_exclude_paths "\n") - - cat << EOF | grep -Ev '^\s*(#|$)|\s+#' | awk '{printf "--exclude='\''%s'\'' ", $0}' | sed 's/[[:space:]]*$//' -# WordPress specific files -wp-config.php - -# WP Engine specific files and directories -.smushit-status -.gitattributes -.wpe-devkit/ -.wpengine-conf/ -_wpeprivate -$(echo -e "$dynamic_excludes") -EOF -}