From af53dcda1bc064c7dba0a4fb1927007e7280c2ab Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Thu, 11 Apr 2024 15:32:00 -0400 Subject: [PATCH 1/7] Remove references to kafkanode Signed-off-by: reyesj2 <94730068+reyesj2@users.noreply.github.com> --- .../assigned_hostgroups.local.map.yaml | 3 +- pillar/kafka/nodes.sls | 2 +- pillar/logstash/nodes.sls | 2 +- pillar/top.sls | 9 +- salt/allowed_states.map.jinja | 14 +-- salt/firewall/containers.map.jinja | 6 +- salt/firewall/defaults.yaml | 92 +------------------ salt/firewall/soc_firewall.yaml | 62 ------------- salt/kafka/enabled.sls | 2 +- salt/logstash/config.sls | 2 +- salt/logstash/defaults.yaml | 4 - salt/logstash/enabled.sls | 2 +- .../config/so/0800_input_kafka.conf.jinja | 4 +- salt/logstash/soc_logstash.yaml | 2 - salt/manager/tools/sbin/so-firewall-minion | 3 - salt/manager/tools/sbin/so-minion | 5 - salt/ssl/init.sls | 5 +- salt/top.sls | 9 -- setup/so-functions | 4 +- setup/so-whiptail | 3 - 20 files changed, 17 insertions(+), 218 deletions(-) diff --git a/files/firewall/assigned_hostgroups.local.map.yaml b/files/firewall/assigned_hostgroups.local.map.yaml index fca293d3a7..025b32131a 100644 --- a/files/firewall/assigned_hostgroups.local.map.yaml +++ b/files/firewall/assigned_hostgroups.local.map.yaml @@ -19,5 +19,4 @@ role: receiver: standalone: searchnode: - sensor: - kafkanode: \ No newline at end of file + sensor: \ No newline at end of file diff --git a/pillar/kafka/nodes.sls b/pillar/kafka/nodes.sls index b1842834c7..6fe64685d3 100644 --- a/pillar/kafka/nodes.sls +++ b/pillar/kafka/nodes.sls @@ -1,4 +1,4 @@ -{% set current_kafkanodes = salt.saltutil.runner('mine.get', tgt='G@role:so-kafkanode or G@role:so-manager', fun='network.ip_addrs', tgt_type='compound') %} +{% set current_kafkanodes = salt.saltutil.runner('mine.get', tgt='G@role:so-receiver or G@role:so-manager', fun='network.ip_addrs', tgt_type='compound') %} {% set pillar_kafkanodes = salt['pillar.get']('kafka:nodes', default={}, merge=True) %} {% set existing_ids = [] %} diff --git a/pillar/logstash/nodes.sls b/pillar/logstash/nodes.sls index 99fbb857cd..a77978821b 100644 --- a/pillar/logstash/nodes.sls +++ b/pillar/logstash/nodes.sls @@ -2,7 +2,7 @@ {% set cached_grains = salt.saltutil.runner('cache.grains', tgt='*') %} {% for minionid, ip in salt.saltutil.runner( 'mine.get', - tgt='G@role:so-manager or G@role:so-managersearch or G@role:so-standalone or G@role:so-searchnode or G@role:so-heavynode or G@role:so-receiver or G@role:so-fleet or G@role:so-kafkanode ', + tgt='G@role:so-manager or G@role:so-managersearch or G@role:so-standalone or G@role:so-searchnode or G@role:so-heavynode or G@role:so-receiver or G@role:so-fleet ', fun='network.ip_addrs', tgt_type='compound') | dictsort() %} diff --git a/pillar/top.sls b/pillar/top.sls index 61f4f338fb..817767bf7f 100644 --- a/pillar/top.sls +++ b/pillar/top.sls @@ -233,15 +233,8 @@ base: - redis.adv_redis - minions.{{ grains.id }} - minions.adv_{{ grains.id }} - - '*_kafkanode': - - logstash.nodes - - logstash.soc_logstash - - logstash.adv_logstash - - minions.{{ grains.id }} - - minions.adv_{{ grains.id }} - - secrets - kafka.nodes + - secrets '*_import': - secrets diff --git a/salt/allowed_states.map.jinja b/salt/allowed_states.map.jinja index 6fa60c2ea2..0fa9686582 100644 --- a/salt/allowed_states.map.jinja +++ b/salt/allowed_states.map.jinja @@ -188,16 +188,8 @@ 'telegraf', 'firewall', 'schedule', - 'docker_clean' - ], - 'so-kafkanode': [ - 'kafka', - 'logstash', - 'ssl', - 'telegraf', - 'firewall', - 'schedule', - 'docker_clean' + 'docker_clean', + 'kafka' ], 'so-desktop': [ 'ssl', @@ -214,7 +206,7 @@ {% do allowed_states.append('strelka') %} {% endif %} - {% if grains.role in ['so-eval', 'so-manager', 'so-standalone', 'so-searchnode', 'so-managersearch', 'so-heavynode', 'so-import', 'so-kafkanode'] %} + {% if grains.role in ['so-eval', 'so-manager', 'so-standalone', 'so-searchnode', 'so-managersearch', 'so-heavynode', 'so-import'] %} {% do allowed_states.append('elasticsearch') %} {% endif %} diff --git a/salt/firewall/containers.map.jinja b/salt/firewall/containers.map.jinja index 7efb9abab2..02a1b7cac8 100644 --- a/salt/firewall/containers.map.jinja +++ b/salt/firewall/containers.map.jinja @@ -81,11 +81,7 @@ {% set NODE_CONTAINERS = [ 'so-logstash', 'so-redis', -] %} -{% elif GLOBALS.role == 'so-kafkanode' %} -{% set NODE_CONTAINERS = [ - 'so-logstash', - 'so-kafka', + 'so-kafka' ] %} {% elif GLOBALS.role == 'so-idh' %} diff --git a/salt/firewall/defaults.yaml b/salt/firewall/defaults.yaml index e51bf58256..0b6d06eda3 100644 --- a/salt/firewall/defaults.yaml +++ b/salt/firewall/defaults.yaml @@ -19,7 +19,6 @@ firewall: manager: [] managersearch: [] receiver: [] - kafkanode: [] searchnode: [] self: [] sensor: [] @@ -443,15 +442,6 @@ firewall: - elastic_agent_data - elastic_agent_update - sensoroni - kafkanode: - portgroups: - - yum - - docker_registry - - influxdb - - elastic_agent_control - - elastic_agent_data - - elastic_agent_update - - sensoroni analyst: portgroups: - nginx @@ -530,9 +520,6 @@ firewall: receiver: portgroups: - salt_manager - kafkanode: - portgroups: - - salt_manager desktop: portgroups: - salt_manager @@ -647,15 +634,6 @@ firewall: - elastic_agent_data - elastic_agent_update - sensoroni - kafkanode: - portgroups: - - yum - - docker_registry - - influxdb - - elastic_agent_control - - elastic_agent_data - - elastic_agent_update - - sensoroni analyst: portgroups: - nginx @@ -1305,14 +1283,17 @@ firewall: - beats_5044 - beats_5644 - elastic_agent_data + - kafka searchnode: portgroups: - redis - beats_5644 + - kafka managersearch: portgroups: - redis - beats_5644 + - kafka self: portgroups: - redis @@ -1383,73 +1364,6 @@ firewall: portgroups: [] customhostgroup9: portgroups: [] - kafkanode: - chain: - DOCKER-USER: - hostgroups: - searchnode: - portgroups: - - kafka - kafkanode: - portgroups: - - kafka - customhostgroup0: - portgroups: [] - customhostgroup1: - portgroups: [] - customhostgroup2: - portgroups: [] - customhostgroup3: - portgroups: [] - customhostgroup4: - portgroups: [] - customhostgroup5: - portgroups: [] - customhostgroup6: - portgroups: [] - customhostgroup7: - portgroups: [] - customhostgroup8: - portgroups: [] - customhostgroup9: - portgroups: [] - INPUT: - hostgroups: - anywhere: - portgroups: - - ssh - dockernet: - portgroups: - - all - localhost: - portgroups: - - all - self: - portgroups: - - syslog - syslog: - portgroups: - - syslog - customhostgroup0: - portgroups: [] - customhostgroup1: - portgroups: [] - customhostgroup2: - portgroups: [] - customhostgroup3: - portgroups: [] - customhostgroup4: - portgroups: [] - customhostgroup5: - portgroups: [] - customhostgroup6: - portgroups: [] - customhostgroup7: - portgroups: [] - customhostgroup8: - portgroups: [] - customhostgroup9: - portgroups: [] idh: chain: DOCKER-USER: diff --git a/salt/firewall/soc_firewall.yaml b/salt/firewall/soc_firewall.yaml index 3e4c4355f9..28791a705c 100644 --- a/salt/firewall/soc_firewall.yaml +++ b/salt/firewall/soc_firewall.yaml @@ -34,7 +34,6 @@ firewall: heavynode: *hostgroupsettings idh: *hostgroupsettings import: *hostgroupsettings - kafkanode: *hostgroupsettings localhost: *ROhostgroupsettingsadv manager: *hostgroupsettings managersearch: *hostgroupsettings @@ -361,8 +360,6 @@ firewall: portgroups: *portgroupsdocker endgame: portgroups: *portgroupsdocker - kafkanode: - portgroups: *portgroupsdocker analyst: portgroups: *portgroupsdocker desktop: @@ -454,8 +451,6 @@ firewall: portgroups: *portgroupsdocker syslog: portgroups: *portgroupsdocker - kafkanode: - portgroups: *portgroupsdocker analyst: portgroups: *portgroupsdocker desktop: @@ -940,63 +935,6 @@ firewall: portgroups: *portgroupshost customhostgroup9: portgroups: *portgroupshost - kafkanode: - chain: - DOCKER-USER: - hostgroups: - searchnode: - portgroups: *portgroupsdocker - kafkanode: - portgroups: *portgroupsdocker - customhostgroup0: - portgroups: *portgroupsdocker - customhostgroup1: - portgroups: *portgroupsdocker - customhostgroup2: - portgroups: *portgroupsdocker - customhostgroup3: - portgroups: *portgroupsdocker - customhostgroup4: - portgroups: *portgroupsdocker - customhostgroup5: - portgroups: *portgroupsdocker - customhostgroup6: - portgroups: *portgroupsdocker - customhostgroup7: - portgroups: *portgroupsdocker - customhostgroup8: - portgroups: *portgroupsdocker - customhostgroup9: - portgroups: *portgroupsdocker - INPUT: - hostgroups: - anywhere: - portgroups: *portgroupshost - dockernet: - portgroups: *portgroupshost - localhost: - portgroups: *portgroupshost - customhostgroup0: - portgroups: *portgroupshost - customhostgroup1: - portgroups: *portgroupshost - customhostgroup2: - portgroups: *portgroupshost - customhostgroup3: - portgroups: *portgroupshost - customhostgroup4: - portgroups: *portgroupshost - customhostgroup5: - portgroups: *portgroupshost - customhostgroup6: - portgroups: *portgroupshost - customhostgroup7: - portgroups: *portgroupshost - customhostgroup8: - portgroups: *portgroupshost - customhostgroup9: - portgroups: *portgroupshost - idh: chain: DOCKER-USER: diff --git a/salt/kafka/enabled.sls b/salt/kafka/enabled.sls index c2fca70db4..ed26297b3e 100644 --- a/salt/kafka/enabled.sls +++ b/salt/kafka/enabled.sls @@ -7,7 +7,7 @@ {% if sls.split('.')[0] in allowed_states %} {% from 'vars/globals.map.jinja' import GLOBALS %} {% from 'docker/docker.map.jinja' import DOCKER %} -{% set KAFKANODES = salt['pillar.get']('kafka:nodes', {}) %} +{% set KAFKANODES = salt['pillar.get']('kafka:nodes', {}) %} include: - kafka.sostatus diff --git a/salt/logstash/config.sls b/salt/logstash/config.sls index 402d1ef206..8a59c83b7e 100644 --- a/salt/logstash/config.sls +++ b/salt/logstash/config.sls @@ -12,7 +12,7 @@ include: - ssl - {% if GLOBALS.role not in ['so-receiver','so-fleet', 'so-kafkanode'] %} + {% if GLOBALS.role not in ['so-receiver','so-fleet'] %} - elasticsearch {% endif %} diff --git a/salt/logstash/defaults.yaml b/salt/logstash/defaults.yaml index 3ca4570fdb..348acb6226 100644 --- a/salt/logstash/defaults.yaml +++ b/salt/logstash/defaults.yaml @@ -19,8 +19,6 @@ logstash: - search fleet: - fleet - kafkanode: - - kafkanode defined_pipelines: fleet: - so/0012_input_elastic_agent.conf.jinja @@ -39,8 +37,6 @@ logstash: - so/0900_input_redis.conf.jinja - so/9805_output_elastic_agent.conf.jinja - so/9900_output_endgame.conf.jinja - kafkanode: - - so/0899_output_kafka.conf.jinja custom0: [] custom1: [] custom2: [] diff --git a/salt/logstash/enabled.sls b/salt/logstash/enabled.sls index fcc2ec190c..3881ef1f4e 100644 --- a/salt/logstash/enabled.sls +++ b/salt/logstash/enabled.sls @@ -75,7 +75,7 @@ so-logstash: {% else %} - /etc/pki/tls/certs/intca.crt:/usr/share/filebeat/ca.crt:ro {% endif %} - {% if GLOBALS.role in ['so-manager', 'so-managersearch', 'so-standalone', 'so-import', 'so-heavynode', 'so-searchnode', 'so-kafkanode' ] %} + {% if GLOBALS.role in ['so-manager', 'so-managersearch', 'so-standalone', 'so-import', 'so-heavynode', 'so-searchnode' ] %} - /opt/so/conf/ca/cacerts:/etc/pki/ca-trust/extracted/java/cacerts:ro - /opt/so/conf/ca/tls-ca-bundle.pem:/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem:ro - /etc/pki/kafka-logstash.p12:/usr/share/logstash/kafka-logstash.p12:ro diff --git a/salt/logstash/pipelines/config/so/0800_input_kafka.conf.jinja b/salt/logstash/pipelines/config/so/0800_input_kafka.conf.jinja index 1391ce9834..85e6729e23 100644 --- a/salt/logstash/pipelines/config/so/0800_input_kafka.conf.jinja +++ b/salt/logstash/pipelines/config/so/0800_input_kafka.conf.jinja @@ -1,11 +1,9 @@ -{% set kafka_brokers = salt['pillar.get']('logstash:nodes:kafkanode', {}) %} +{% set kafka_brokers = salt['pillar.get']('logstash:nodes:receiver', {}) %} {% set kafka_on_mngr = salt ['pillar.get']('logstash:nodes:manager', {}) %} {% set broker_ips = [] %} {% for node, node_data in kafka_brokers.items() %} {% do broker_ips.append(node_data['ip'] + ":9092") %} {% endfor %} - -{# For testing kafka stuff from manager not dedicated kafkanodes #} {% for node, node_data in kafka_on_mngr.items() %} {% do broker_ips.append(node_data['ip'] + ":9092") %} {% endfor %} diff --git a/salt/logstash/soc_logstash.yaml b/salt/logstash/soc_logstash.yaml index 82fb25becc..3172ff7c58 100644 --- a/salt/logstash/soc_logstash.yaml +++ b/salt/logstash/soc_logstash.yaml @@ -16,7 +16,6 @@ logstash: manager: *assigned_pipelines managersearch: *assigned_pipelines fleet: *assigned_pipelines - kafkanode: *assigned_pipelines defined_pipelines: receiver: &defined_pipelines description: List of pipeline configurations assign to this group. @@ -27,7 +26,6 @@ logstash: fleet: *defined_pipelines manager: *defined_pipelines search: *defined_pipelines - kafkanode: *defined_pipelines custom0: *defined_pipelines custom1: *defined_pipelines custom2: *defined_pipelines diff --git a/salt/manager/tools/sbin/so-firewall-minion b/salt/manager/tools/sbin/so-firewall-minion index 3357e5185a..66a0afcea7 100755 --- a/salt/manager/tools/sbin/so-firewall-minion +++ b/salt/manager/tools/sbin/so-firewall-minion @@ -79,9 +79,6 @@ fi 'RECEIVER') so-firewall includehost receiver "$IP" --apply ;; - 'KAFKANODE') - so-firewall includehost kafkanode "$IP" --apply - ;; 'DESKTOP') so-firewall includehost desktop "$IP" --apply ;; diff --git a/salt/manager/tools/sbin/so-minion b/salt/manager/tools/sbin/so-minion index 7b3e6fd3ed..34e069ece8 100755 --- a/salt/manager/tools/sbin/so-minion +++ b/salt/manager/tools/sbin/so-minion @@ -565,11 +565,6 @@ function createRECEIVER() { add_telegraf_to_minion } -function createKAFKANODE() { - add_logstash_to_minion - # add_telegraf_to_minion -} - function createDESKTOP() { add_desktop_to_minion add_telegraf_to_minion diff --git a/salt/ssl/init.sls b/salt/ssl/init.sls index 90f9cc64f6..f337d62cb0 100644 --- a/salt/ssl/init.sls +++ b/salt/ssl/init.sls @@ -664,10 +664,7 @@ elastickeyperms: {%- endif %} -# Roles will need to be modified. Below is just for testing encrypted kafka pipelines -# Remove so-manager. Just inplace for testing -{% if grains['role'] in ['so-manager', 'so-kafkanode', 'so-searchnode'] %} -# Create a cert for Redis encryption +{% if grains['role'] in ['so-manager', 'so-searchnode', 'so-receiver'] %} kafka_key: x509.private_key_managed: - name: /etc/pki/kafka.key diff --git a/salt/top.sls b/salt/top.sls index 289dd462b0..ec5e4d7386 100644 --- a/salt/top.sls +++ b/salt/top.sls @@ -235,16 +235,7 @@ base: - firewall - logstash - redis - - elasticfleet.install_agent_grid - - '*_kafkanode and G@saltversion:{{saltversion}}': - - match: compound - kafka - - logstash - - ssl - - telegraf - - firewall - - docker_clean - elasticfleet.install_agent_grid '*_idh and G@saltversion:{{saltversion}}': diff --git a/setup/so-functions b/setup/so-functions index 070711d637..a669c52fc4 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1143,7 +1143,7 @@ get_redirect() { get_minion_type() { local minion_type case "$install_type" in - 'EVAL' | 'MANAGERSEARCH' | 'MANAGER' | 'SENSOR' | 'HEAVYNODE' | 'SEARCHNODE' | 'FLEET' | 'IDH' | 'STANDALONE' | 'IMPORT' | 'RECEIVER' | 'DESKTOP' | 'KAFKANODE') + 'EVAL' | 'MANAGERSEARCH' | 'MANAGER' | 'SENSOR' | 'HEAVYNODE' | 'SEARCHNODE' | 'FLEET' | 'IDH' | 'STANDALONE' | 'IMPORT' | 'RECEIVER' | 'DESKTOP') minion_type=$(echo "$install_type" | tr '[:upper:]' '[:lower:]') ;; esac @@ -1505,8 +1505,6 @@ process_installtype() { is_import=true elif [ "$install_type" = 'RECEIVER' ]; then is_receiver=true - elif [ "$install_type" = 'KAFKANODE' ]; then - is_kafka=true elif [ "$install_type" = 'DESKTOP' ]; then is_desktop=true fi diff --git a/setup/so-whiptail b/setup/so-whiptail index a732a9c975..fd9625ec43 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -681,7 +681,6 @@ whiptail_install_type_dist_existing() { "HEAVYNODE" "Sensor + Search Node " \ "IDH" "Intrusion Detection Honeypot Node " \ "RECEIVER" "Receiver Node " \ - "KAFKANODE" "Kafka Broker + Kraft controller" \ 3>&1 1>&2 2>&3 # "HOTNODE" "Add Hot Node (Uses Elastic Clustering)" \ # TODO # "WARMNODE" "Add Warm Node to existing Hot or Search node" \ # TODO @@ -712,8 +711,6 @@ whiptail_install_type_dist_existing() { is_import=true elif [ "$install_type" = 'RECEIVER' ]; then is_receiver=true - elif [ "$install_type" = 'KAFKANODE' ]; then - is_kafka=true elif [ "$install_type" = 'DESKTOP' ]; then is_desktop=true fi From ca7253a5896f2a028021339a04dccd9eabb3b863 Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Thu, 11 Apr 2024 15:38:03 -0400 Subject: [PATCH 2/7] Run kafka-clusterid script when pillar values are missing Signed-off-by: reyesj2 <94730068+reyesj2@users.noreply.github.com> --- salt/manager/tools/sbin/so-kafka-clusterid | 8 +++++--- setup/so-functions | 4 +++- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/salt/manager/tools/sbin/so-kafka-clusterid b/salt/manager/tools/sbin/so-kafka-clusterid index 7199732473..fcbe3ba42f 100644 --- a/salt/manager/tools/sbin/so-kafka-clusterid +++ b/salt/manager/tools/sbin/so-kafka-clusterid @@ -16,7 +16,9 @@ fi if ! grep -q "^ kafka_cluster_id:" $local_salt_dir/pillar/secrets.sls; then kafka_cluster_id=$(get_random_value 22) echo ' kafka_cluster_id: '$kafka_cluster_id >> $local_salt_dir/pillar/secrets.sls -else - echo 'kafka_cluster_id exists' - salt-call pillar.get secrets +fi + +if ! grep -q "^ kafkapass:" $local_salt_dir/pillar/secrets.sls; then + kafkapass=$(get_random_value) + echo ' kafkapass: '$kafkapass >> $local_salt_dir/pillar/secrets.sls fi \ No newline at end of file diff --git a/setup/so-functions b/setup/so-functions index a669c52fc4..176349edb1 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1116,6 +1116,7 @@ generate_passwords(){ SOCSRVKEY=$(get_random_value 64) IMPORTPASS=$(get_random_value) KAFKACLUSTERID=$(get_random_value 22) + KAFKAPASS=$(get_random_value) } generate_interface_vars() { @@ -1947,7 +1948,8 @@ secrets_pillar(){ "secrets:"\ " import_pass: $IMPORTPASS"\ " influx_pass: $INFLUXPASS"\ - " kafka_cluster_id: $KAFKACLUSTERID" > $local_salt_dir/pillar/secrets.sls + " kafka_cluster_id: $KAFKACLUSTERID"\ + " kafka_pass: $KAFKAPASS" > $local_salt_dir/pillar/secrets.sls fi } From 6b28dc72e86ee51476b5e95dedd28361223435ce Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Thu, 11 Apr 2024 15:38:33 -0400 Subject: [PATCH 3/7] Update annotation for global.pipeline Signed-off-by: reyesj2 <94730068+reyesj2@users.noreply.github.com> --- salt/global/soc_global.yaml | 3 +-- salt/kafka/config.sls | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/salt/global/soc_global.yaml b/salt/global/soc_global.yaml index daaf28b286..667bf78262 100644 --- a/salt/global/soc_global.yaml +++ b/salt/global/soc_global.yaml @@ -36,9 +36,8 @@ global: global: True advanced: True pipeline: - description: Sets which pipeline technology for events to use. Currently only Redis is supported. + description: Sets which pipeline technology for events to use. Currently only Redis is fully supported. Kafka is experimental and requires a Security Onion Pro license. global: True - readonly: True advanced: True repo_host: description: Specify the host where operating system packages will be served from. diff --git a/salt/kafka/config.sls b/salt/kafka/config.sls index c856c4f809..c9e028ff59 100644 --- a/salt/kafka/config.sls +++ b/salt/kafka/config.sls @@ -26,7 +26,7 @@ include: - ssl - +g kafka_group: group.present: - name: kafka From 39555873729731bbf79ee3ebbe32d7c4c01e9ad9 Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Thu, 11 Apr 2024 16:20:09 -0400 Subject: [PATCH 4/7] Use global.pipeline for redis / kafka states Signed-off-by: reyesj2 <94730068+reyesj2@users.noreply.github.com> --- salt/global/defaults.yaml | 3 ++- salt/global/soc_global.yaml | 2 ++ salt/kafka/init.sls | 3 ++- salt/manager/tools/sbin/soup | 3 ++- salt/redis/init.sls | 4 ++-- salt/vars/kafkanode.map.jinja | 1 - setup/so-functions | 1 - 7 files changed, 10 insertions(+), 7 deletions(-) delete mode 100644 salt/vars/kafkanode.map.jinja diff --git a/salt/global/defaults.yaml b/salt/global/defaults.yaml index bd7244a587..5daa942c81 100644 --- a/salt/global/defaults.yaml +++ b/salt/global/defaults.yaml @@ -1,2 +1,3 @@ global: - pcapengine: STENO \ No newline at end of file + pcapengine: STENO + pipeline: REDIS \ No newline at end of file diff --git a/salt/global/soc_global.yaml b/salt/global/soc_global.yaml index 667bf78262..5a349a3c34 100644 --- a/salt/global/soc_global.yaml +++ b/salt/global/soc_global.yaml @@ -37,6 +37,8 @@ global: advanced: True pipeline: description: Sets which pipeline technology for events to use. Currently only Redis is fully supported. Kafka is experimental and requires a Security Onion Pro license. + regex: ^(REDIS|KAFKA)$ + regexFailureMessage: You must enter either REDIS or KAFKA. global: True advanced: True repo_host: diff --git a/salt/kafka/init.sls b/salt/kafka/init.sls index b4a6a28b0a..acedba3c36 100644 --- a/salt/kafka/init.sls +++ b/salt/kafka/init.sls @@ -4,9 +4,10 @@ # Elastic License 2.0. {% from 'kafka/map.jinja' import KAFKAMERGED %} +{% from 'vars/globals.map.jinja' import GLOBALS %} include: -{% if KAFKAMERGED.enabled %} +{% if GLOBALS.pipeline == "KAFKA" and KAFKAMERGED.enabled %} - kafka.enabled {% else %} - kafka.disabled diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index fa3c3b5eea..3ca353856c 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -438,7 +438,8 @@ post_to_2.4.60() { } post_to_2.4.70() { - echo "Nothing to apply" + echo "Removing global.pipeline pillar configuration" + sed -i '/pipeline:/d' /opt/so/saltstack/local/pillar/global/soc_global.sls POSTVERSION=2.4.70 } diff --git a/salt/redis/init.sls b/salt/redis/init.sls index 4936c3254b..7142c92c38 100644 --- a/salt/redis/init.sls +++ b/salt/redis/init.sls @@ -4,10 +4,10 @@ # Elastic License 2.0. {% from 'redis/map.jinja' import REDISMERGED %} -{% from 'kafka/map.jinja' import KAFKAMERGED %} +{% from 'vars/globals.map.jinja' import GLOBALS %} include: -{% if REDISMERGED.enabled and not KAFKAMERGED.enabled %} +{% if GLOBALS.pipeline == "REDIS" and REDISMERGED.enabled %} - redis.enabled {% else %} - redis.disabled diff --git a/salt/vars/kafkanode.map.jinja b/salt/vars/kafkanode.map.jinja deleted file mode 100644 index 396cefcc90..0000000000 --- a/salt/vars/kafkanode.map.jinja +++ /dev/null @@ -1 +0,0 @@ -{% set ROLE_GLOBALS = {} %} \ No newline at end of file diff --git a/setup/so-functions b/setup/so-functions index 176349edb1..038a4deb4a 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1327,7 +1327,6 @@ create_global() { # Continue adding other details echo " imagerepo: '$IMAGEREPO'" >> $global_pillar_file - echo " pipeline: 'redis'" >> $global_pillar_file echo " repo_host: '$HOSTNAME'" >> $global_pillar_file echo " influxdb_host: '$HOSTNAME'" >> $global_pillar_file echo " registry_host: '$HOSTNAME'" >> $global_pillar_file From fbd3cff90d6665dce961ed0b56ea6352526e128b Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Fri, 12 Apr 2024 11:21:19 -0400 Subject: [PATCH 5/7] Make global.pipeline use GLOBALMERGED value Signed-off-by: reyesj2 <94730068+reyesj2@users.noreply.github.com> --- salt/vars/globals.map.jinja | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/vars/globals.map.jinja b/salt/vars/globals.map.jinja index ed7129678a..0a4995c0cb 100644 --- a/salt/vars/globals.map.jinja +++ b/salt/vars/globals.map.jinja @@ -23,7 +23,7 @@ 'manager_ip': INIT.PILLAR.global.managerip, 'md_engine': INIT.PILLAR.global.mdengine, 'pcap_engine': GLOBALMERGED.pcapengine, - 'pipeline': INIT.PILLAR.global.pipeline, + 'pipeline': GLOBALMERGED.pipeline, 'so_version': INIT.PILLAR.global.soversion, 'so_docker_gateway': DOCKER.gateway, 'so_docker_range': DOCKER.range, From a6ff92b099faa1c1a43def5e85331d084e18edc8 Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Fri, 12 Apr 2024 12:11:18 -0400 Subject: [PATCH 6/7] Note to remove so-kafka-clusterid. Update soup and setup to generate needed kafka pillar values Signed-off-by: reyesj2 <94730068+reyesj2@users.noreply.github.com> --- salt/manager/tools/sbin/so-kafka-clusterid | 5 +++++ salt/manager/tools/sbin/soup | 17 +++++++++++++++++ setup/so-functions | 13 +++++++++++++ setup/so-variables | 6 ++++++ 4 files changed, 41 insertions(+) diff --git a/salt/manager/tools/sbin/so-kafka-clusterid b/salt/manager/tools/sbin/so-kafka-clusterid index adddfe3cec..7ac0559974 100644 --- a/salt/manager/tools/sbin/so-kafka-clusterid +++ b/salt/manager/tools/sbin/so-kafka-clusterid @@ -5,6 +5,11 @@ # https://securityonion.net/license; you may not use this file except in compliance with the # Elastic License 2.0. + + +### THIS SCRIPT AND SALT STATE REFERENCES TO THIS SCRIPT TO BE REMOVED ONCE INITIAL TESTING IS DONE - THESE VALUES WILL GENERATED IN SETUP AND SOUP + + local_salt_dir=/opt/so/saltstack/local if [[ -f /usr/sbin/so-common ]]; then diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 3ca353856c..a6f9032a5b 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -438,8 +438,25 @@ post_to_2.4.60() { } post_to_2.4.70() { + # Global pipeline changes to REDIS or KAFKA echo "Removing global.pipeline pillar configuration" sed -i '/pipeline:/d' /opt/so/saltstack/local/pillar/global/soc_global.sls + + # Kafka configuration + mkdir -p /opt/so/saltstack/local/pillar/kafka + touch /opt/so/saltstack/local/pillar/kafka/soc_kafka.sls + touch /opt/so/saltstack/local/pillar/kafka/adv_kafka.sls + echo 'kafka: ' > /opt/so/saltstack/local/pillar/kafka/soc_kafka.sls + + if ! grep -q "^ cluster_id:" $local_salt_dir/pillar/kafka/soc_kafka.sls; then + kafka_cluster_id=$(get_random_value 22) + echo ' cluster_id: '$kafka_cluster_id >> $local_salt_dir/pillar/kafka/soc_kafka.sls + + if ! grep -q "^ certpass:" $local_salt_dir/pillar/kafka/soc_kafka.sls; then + kafkapass=$(get_random_value) + echo ' certpass: '$kafkapass >> $local_salt_dir/pillar/kafka/soc_kafka.sls + fi + POSTVERSION=2.4.70 } diff --git a/setup/so-functions b/setup/so-functions index 2332ab94cd..30e8fbfd6a 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -803,6 +803,7 @@ create_manager_pillars() { patch_pillar nginx_pillar kibana_pillar + kafka_pillar } create_repo() { @@ -1191,6 +1192,18 @@ kibana_pillar() { logCmd "touch $kibana_pillar_file" } +kafka_pillar() { + KAFKACLUSTERID=$(get_random_value 22) + KAFKAPASS=$(get_random_value) + logCmd "mkdir -p $local_salt_dir/pillar/kakfa" + logCmd "touch $adv_kafka_pillar_file" + logCmd "touch $kafka_pillar_file" + printf '%s\n'\ + "kafka:"\ + " cluster_id: $KAFKACLUSTERID"\ + " certpass: $KAFKAPASS" > $kafka_pillar_file +} + logrotate_pillar() { logCmd "mkdir -p $local_salt_dir/pillar/logrotate" logCmd "touch $adv_logrotate_pillar_file" diff --git a/setup/so-variables b/setup/so-variables index 42ed8fc5c2..4a2f29c58f 100644 --- a/setup/so-variables +++ b/setup/so-variables @@ -178,6 +178,12 @@ export redis_pillar_file adv_redis_pillar_file="$local_salt_dir/pillar/redis/adv_redis.sls" export adv_redis_pillar_file +kafka_pillar_file="local_salt_dir/pillar/kafka/soc_kafka.sls" +export kafka_pillar_file + +adv_kafka_pillar_file="$local_salt_dir/pillar/kafka/adv_kafka.sls" +export kafka_pillar_file + idh_pillar_file="$local_salt_dir/pillar/idh/soc_idh.sls" export idh_pillar_file From 911ee579a9e8a92b751695d12b7f449358df8260 Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Fri, 12 Apr 2024 12:16:20 -0400 Subject: [PATCH 7/7] Typo Signed-off-by: reyesj2 <94730068+reyesj2@users.noreply.github.com> --- salt/kafka/config.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/kafka/config.sls b/salt/kafka/config.sls index c9e028ff59..c856c4f809 100644 --- a/salt/kafka/config.sls +++ b/salt/kafka/config.sls @@ -26,7 +26,7 @@ include: - ssl -g + kafka_group: group.present: - name: kafka