From b571eeb8e6425b197b0b2245d3dfb367b1bb5f93 Mon Sep 17 00:00:00 2001 From: DefensiveDepth Date: Wed, 27 Mar 2024 14:58:16 -0400 Subject: [PATCH 1/7] Initial cut of .70 soup changes --- salt/manager/tools/sbin/soup | 74 ++++++++++++++++++++++++++++++++++++ 1 file changed, 74 insertions(+) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index a585f877c7..87d88a57ed 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -357,6 +357,7 @@ preupgrade_changes() { [[ "$INSTALLEDVERSION" == 2.4.30 ]] && up_to_2.4.40 [[ "$INSTALLEDVERSION" == 2.4.40 ]] && up_to_2.4.50 [[ "$INSTALLEDVERSION" == 2.4.50 ]] && up_to_2.4.60 + [[ "$INSTALLEDVERSION" == 2.4.60 ]] && up_to_2.4.70 true } @@ -373,6 +374,7 @@ postupgrade_changes() { [[ "$POSTVERSION" == 2.4.30 ]] && post_to_2.4.40 [[ "$POSTVERSION" == 2.4.40 ]] && post_to_2.4.50 [[ "$POSTVERSION" == 2.4.50 ]] && post_to_2.4.60 + [[ "$POSTVERSION" == 2.4.60 ]] && post_to_2.4.70 true } @@ -435,6 +437,11 @@ post_to_2.4.60() { POSTVERSION=2.4.60 } +post_to_2.4.70() { + echo "Nothing to apply" + POSTVERSION=2.4.70 +} + repo_sync() { echo "Sync the local repo." su socore -c '/usr/sbin/so-repo-sync' || fail "Unable to complete so-repo-sync." @@ -574,6 +581,73 @@ up_to_2.4.60() { INSTALLEDVERSION=2.4.60 } +up_to_2.4.70() { + # Start SOC Detections migration + mkdir -p /nsm/backup/detections-migration/{suricata,sigma/rules,elastalert} + + # Remove cronjobs + crontab -l | grep -v 'so-playbook-sync_cron' | crontab - + crontab -l | grep -v 'so-playbook-ruleupdate_cron' | crontab - + + # Check for active Elastalert rules + active_rules_count=$(find /opt/so/rules/elastalert/playbook/ -type f -name "*.yaml" | wc -l) + + if [[ "$active_rules_count" -gt 0 ]]; then + # Prompt the user to AGREE if active Elastalert rules found + echo + echo "$active_rules_count Active Elastalert/Playbook rules found." + echo "In preparation for the new Detections module, they will be backed up and then disabled." + echo + echo "If you would like to proceed, then type AGREE and press ENTER." + echo + # Read user input + read INPUT + if [ "${INPUT^^}" != 'AGREE' ]; then exit 0; fi + + echo "Backing up the Elastalert rules..." + rsync -av --stats /opt/so/rules/elastalert/playbook/*.yaml /nsm/backup/detections-migration/elastalert/ + + # Verify that rsync completed successfully + if [[ $? -eq 0 ]]; then + # Delete the Elastlaert rules + rm -f /opt/so/rules/elastalert/playbook/*.yaml + echo "Active Elastalert rules have been backed up." + else + echo "Error: rsync failed to copy the files. Active Elastalert rules have not been backed up." + exit 1 + fi + fi + + echo + echo "Exporting Sigma rules from Playbook..." + MYSQLPW=$(lookup_pillar_secret mysql) + + docker exec so-mysql sh -c "exec mysql -uroot -p${MYSQLPW} -D playbook -sN -e \"SELECT id, value FROM custom_values WHERE value LIKE '%View Sigma%'\"" | while read -r id value; do + echo -e "$value" > "/nsm/backup/detections-migration/sigma/rules/$id.yaml" + done + + echo + echo "Exporting Sigma Filters from Playbook..." + docker exec so-mysql sh -c "exec mysql -uroot -p${MYSQLPW} -D playbook -sN -e \"SELECT issues.subject as title, custom_values.value as filter FROM issues JOIN custom_values ON issues.id = custom_values.customized_id WHERE custom_values.value LIKE '%sofilter%'\"" > /nsm/backup/detections-migration/sigma/custom-filters.txt + + echo + echo "Backing up Playbook database..." + docker exec so-mysql sh -c "mysqldump -uroot -p${MYSQLPW} --databases playbook > /tmp/playbook-dump" + docker cp so-mysql:/tmp/playbook-dump /nsm/backup/detections-migration/sigma/playbook-dump.sql + + echo + echo "Stopping Playbook services..." + so-playbook-stop + so-mysql-stop + so-soctopus-stop + + # What about cleaning up various so-utilities like so-playbook-restart? + echo + echo "Playbook Migration is complete...." + + INSTALLEDVERSION=2.4.70 +} + determine_elastic_agent_upgrade() { if [[ $is_airgap -eq 0 ]]; then update_elastic_agent_airgap From ba262ee01a9fada97e21fe82019598ced83529fa Mon Sep 17 00:00:00 2001 From: DefensiveDepth Date: Wed, 27 Mar 2024 15:43:25 -0400 Subject: [PATCH 2/7] Check to see if Playbook is enabled --- salt/manager/tools/sbin/soup | 85 +++++++++++++++++++----------------- 1 file changed, 44 insertions(+), 41 deletions(-) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 87d88a57ed..62a579e181 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -589,51 +589,54 @@ up_to_2.4.70() { crontab -l | grep -v 'so-playbook-sync_cron' | crontab - crontab -l | grep -v 'so-playbook-ruleupdate_cron' | crontab - - # Check for active Elastalert rules - active_rules_count=$(find /opt/so/rules/elastalert/playbook/ -type f -name "*.yaml" | wc -l) - - if [[ "$active_rules_count" -gt 0 ]]; then - # Prompt the user to AGREE if active Elastalert rules found - echo - echo "$active_rules_count Active Elastalert/Playbook rules found." - echo "In preparation for the new Detections module, they will be backed up and then disabled." - echo - echo "If you would like to proceed, then type AGREE and press ENTER." - echo - # Read user input - read INPUT - if [ "${INPUT^^}" != 'AGREE' ]; then exit 0; fi - - echo "Backing up the Elastalert rules..." - rsync -av --stats /opt/so/rules/elastalert/playbook/*.yaml /nsm/backup/detections-migration/elastalert/ - - # Verify that rsync completed successfully - if [[ $? -eq 0 ]]; then - # Delete the Elastlaert rules - rm -f /opt/so/rules/elastalert/playbook/*.yaml - echo "Active Elastalert rules have been backed up." - else - echo "Error: rsync failed to copy the files. Active Elastalert rules have not been backed up." - exit 1 - fi - fi + if grep -A 1 'playbook:' /opt/so/saltstack/local/pillar/minions/* | grep -q 'enabled: True'; then + + # Check for active Elastalert rules + active_rules_count=$(find /opt/so/rules/elastalert/playbook/ -type f -name "*.yaml" | wc -l) + + if [[ "$active_rules_count" -gt 0 ]]; then + # Prompt the user to AGREE if active Elastalert rules found + echo + echo "$active_rules_count Active Elastalert/Playbook rules found." + echo "In preparation for the new Detections module, they will be backed up and then disabled." + echo + echo "If you would like to proceed, then type AGREE and press ENTER." + echo + # Read user input + read INPUT + if [ "${INPUT^^}" != 'AGREE' ]; then exit 0; fi + + echo "Backing up the Elastalert rules..." + rsync -av --stats /opt/so/rules/elastalert/playbook/*.yaml /nsm/backup/detections-migration/elastalert/ + + # Verify that rsync completed successfully + if [[ $? -eq 0 ]]; then + # Delete the Elastlaert rules + rm -f /opt/so/rules/elastalert/playbook/*.yaml + echo "Active Elastalert rules have been backed up." + else + echo "Error: rsync failed to copy the files. Active Elastalert rules have not been backed up." + exit 1 + fi + fi - echo - echo "Exporting Sigma rules from Playbook..." - MYSQLPW=$(lookup_pillar_secret mysql) + echo + echo "Exporting Sigma rules from Playbook..." + MYSQLPW=$(awk '/mysql:/ {print $2}' /opt/so/saltstack/local/pillar/secrets.sls) - docker exec so-mysql sh -c "exec mysql -uroot -p${MYSQLPW} -D playbook -sN -e \"SELECT id, value FROM custom_values WHERE value LIKE '%View Sigma%'\"" | while read -r id value; do - echo -e "$value" > "/nsm/backup/detections-migration/sigma/rules/$id.yaml" - done + docker exec so-mysql sh -c "exec mysql -uroot -p${MYSQLPW} -D playbook -sN -e \"SELECT id, value FROM custom_values WHERE value LIKE '%View Sigma%'\"" | while read -r id value; do + echo -e "$value" > "/nsm/backup/detections-migration/sigma/rules/$id.yaml" + done - echo - echo "Exporting Sigma Filters from Playbook..." - docker exec so-mysql sh -c "exec mysql -uroot -p${MYSQLPW} -D playbook -sN -e \"SELECT issues.subject as title, custom_values.value as filter FROM issues JOIN custom_values ON issues.id = custom_values.customized_id WHERE custom_values.value LIKE '%sofilter%'\"" > /nsm/backup/detections-migration/sigma/custom-filters.txt + echo + echo "Exporting Sigma Filters from Playbook..." + docker exec so-mysql sh -c "exec mysql -uroot -p${MYSQLPW} -D playbook -sN -e \"SELECT issues.subject as title, custom_values.value as filter FROM issues JOIN custom_values ON issues.id = custom_values.customized_id WHERE custom_values.value LIKE '%sofilter%'\"" > /nsm/backup/detections-migration/sigma/custom-filters.txt - echo - echo "Backing up Playbook database..." - docker exec so-mysql sh -c "mysqldump -uroot -p${MYSQLPW} --databases playbook > /tmp/playbook-dump" - docker cp so-mysql:/tmp/playbook-dump /nsm/backup/detections-migration/sigma/playbook-dump.sql + echo + echo "Backing up Playbook database..." + docker exec so-mysql sh -c "mysqldump -uroot -p${MYSQLPW} --databases playbook > /tmp/playbook-dump" + docker cp so-mysql:/tmp/playbook-dump /nsm/backup/detections-migration/sigma/playbook-dump.sql + fi echo echo "Stopping Playbook services..." From ce0c9f846db5d74afa54727d097ecbb8a24fefb5 Mon Sep 17 00:00:00 2001 From: DefensiveDepth Date: Wed, 27 Mar 2024 16:13:52 -0400 Subject: [PATCH 3/7] Remove containers from so-status --- salt/manager/tools/sbin/soup | 2 ++ 1 file changed, 2 insertions(+) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 62a579e181..069a4f345e 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -643,6 +643,8 @@ up_to_2.4.70() { so-playbook-stop so-mysql-stop so-soctopus-stop + sed -i '/so-playbook\|so-soctopus\|so-mysql/d' /opt/so/conf/so-status/so-status.conf + # What about cleaning up various so-utilities like so-playbook-restart? echo From d2c9e0ea4aaa7e7ff7e0aafdf9fa25cbc681eda5 Mon Sep 17 00:00:00 2001 From: DefensiveDepth Date: Thu, 28 Mar 2024 13:04:48 -0400 Subject: [PATCH 4/7] Cleanup --- salt/manager/tools/sbin/soup | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 069a4f345e..822fa05d2d 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -604,7 +604,7 @@ up_to_2.4.70() { echo # Read user input read INPUT - if [ "${INPUT^^}" != 'AGREE' ]; then exit 0; fi + if [ "${INPUT^^}" != 'AGREE' ]; then fail "SOUP canceled."; fi echo "Backing up the Elastalert rules..." rsync -av --stats /opt/so/rules/elastalert/playbook/*.yaml /nsm/backup/detections-migration/elastalert/ @@ -615,8 +615,7 @@ up_to_2.4.70() { rm -f /opt/so/rules/elastalert/playbook/*.yaml echo "Active Elastalert rules have been backed up." else - echo "Error: rsync failed to copy the files. Active Elastalert rules have not been backed up." - exit 1 + fail "Error: rsync failed to copy the files. Active Elastalert rules have not been backed up." fi fi @@ -639,14 +638,13 @@ up_to_2.4.70() { fi echo - echo "Stopping Playbook services..." - so-playbook-stop - so-mysql-stop - so-soctopus-stop + echo "Stopping Playbook services & cleaning up..." + docker stop so-playbook 2>/dev/null + docker stop so-mysql 2>/dev/null + docker stop so-soctopus 2>/dev/null sed -i '/so-playbook\|so-soctopus\|so-mysql/d' /opt/so/conf/so-status/so-status.conf + rm -f /usr/sbin/so-playbook-* /usr/sbin/so-soctopus-* /usr/sbin/so-mysql-* - - # What about cleaning up various so-utilities like so-playbook-restart? echo echo "Playbook Migration is complete...." From 9c5ba92589e583182373acff0a0eefd00573fea2 Mon Sep 17 00:00:00 2001 From: DefensiveDepth Date: Thu, 28 Mar 2024 13:23:40 -0400 Subject: [PATCH 5/7] Check if container is running first --- salt/manager/tools/sbin/soup | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 822fa05d2d..4020ec5314 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -639,9 +639,11 @@ up_to_2.4.70() { echo echo "Stopping Playbook services & cleaning up..." - docker stop so-playbook 2>/dev/null - docker stop so-mysql 2>/dev/null - docker stop so-soctopus 2>/dev/null + for container in so-playbook so-mysql so-soctopus; do + if [ -n "$(docker ps -q -f name=^${container}$)" ]; then + docker stop $container + fi + done sed -i '/so-playbook\|so-soctopus\|so-mysql/d' /opt/so/conf/so-status/so-status.conf rm -f /usr/sbin/so-playbook-* /usr/sbin/so-soctopus-* /usr/sbin/so-mysql-* From 32b8649c77e0698c987b0b4e01fdeb9dfcb406ad Mon Sep 17 00:00:00 2001 From: DefensiveDepth Date: Thu, 28 Mar 2024 14:31:02 -0400 Subject: [PATCH 6/7] Add more error checking --- salt/manager/tools/sbin/soup | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 4020ec5314..d5abda7831 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -625,16 +625,16 @@ up_to_2.4.70() { docker exec so-mysql sh -c "exec mysql -uroot -p${MYSQLPW} -D playbook -sN -e \"SELECT id, value FROM custom_values WHERE value LIKE '%View Sigma%'\"" | while read -r id value; do echo -e "$value" > "/nsm/backup/detections-migration/sigma/rules/$id.yaml" - done + done || fail "Failed to export Sigma rules..." echo echo "Exporting Sigma Filters from Playbook..." - docker exec so-mysql sh -c "exec mysql -uroot -p${MYSQLPW} -D playbook -sN -e \"SELECT issues.subject as title, custom_values.value as filter FROM issues JOIN custom_values ON issues.id = custom_values.customized_id WHERE custom_values.value LIKE '%sofilter%'\"" > /nsm/backup/detections-migration/sigma/custom-filters.txt + docker exec so-mysql sh -c "exec mysql -uroot -p${MYSQLPW} -D playbook -sN -e \"SELECT issues.subject as title, custom_values.value as filter FROM issues JOIN custom_values ON issues.id = custom_values.customized_id WHERE custom_values.value LIKE '%sofilter%'\"" > /nsm/backup/detections-migration/sigma/custom-filters.txt || fail "Failed to export Custom Sigma Filters." echo echo "Backing up Playbook database..." - docker exec so-mysql sh -c "mysqldump -uroot -p${MYSQLPW} --databases playbook > /tmp/playbook-dump" - docker cp so-mysql:/tmp/playbook-dump /nsm/backup/detections-migration/sigma/playbook-dump.sql + docker exec so-mysql sh -c "mysqldump -uroot -p${MYSQLPW} --databases playbook > /tmp/playbook-dump" || fail "Failed to dump Playbook database." + docker cp so-mysql:/tmp/playbook-dump /nsm/backup/detections-migration/sigma/playbook-dump.sql || fail "Failed to backup Playbook database." fi echo From c2f7f7e3a5dbd562c28691722ef4ff07858a2954 Mon Sep 17 00:00:00 2001 From: DefensiveDepth Date: Thu, 4 Apr 2024 08:52:30 -0400 Subject: [PATCH 7/7] Remove dup line --- salt/manager/tools/sbin/soup | 1 - 1 file changed, 1 deletion(-) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 176c290a86..dba3215d1c 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -375,7 +375,6 @@ postupgrade_changes() { [[ "$POSTVERSION" == 2.4.40 ]] && post_to_2.4.50 [[ "$POSTVERSION" == 2.4.50 ]] && post_to_2.4.60 [[ "$POSTVERSION" == 2.4.60 ]] && post_to_2.4.70 - [[ "$POSTVERSION" == 2.4.60 ]] && post_to_2.4.70 true }