Remove references to kafkanode

files/firewall/assigned_hostgroups.local.map.yaml

@@ -19,5 +19,4 @@ role:
   receiver:
   standalone:
   searchnode:
-  sensor:
-  kafkanode:
+  sensor:

pillar/logstash/nodes.sls

@@ -2,7 +2,7 @@
 {% set cached_grains = salt.saltutil.runner('cache.grains', tgt='*') %}
 {% for minionid, ip in salt.saltutil.runner(
     'mine.get',
-    tgt='G@role:so-manager or G@role:so-managersearch or G@role:so-standalone or G@role:so-searchnode or G@role:so-heavynode or G@role:so-receiver or G@role:so-fleet or G@role:so-kafkanode ',
+    tgt='G@role:so-manager or G@role:so-managersearch or G@role:so-standalone or G@role:so-searchnode or G@role:so-heavynode or G@role:so-receiver or G@role:so-fleet ',
     fun='network.ip_addrs',
     tgt_type='compound') | dictsort()
 %}

salt/firewall/containers.map.jinja

@@ -81,11 +81,7 @@
 {% set NODE_CONTAINERS = [
      'so-logstash',
      'so-redis',
-] %}
-{% elif GLOBALS.role == 'so-kafkanode' %}
-{% set NODE_CONTAINERS = [
-     'so-logstash',
-     'so-kafka',
+     'so-kafka'
 ] %}
 
 {% elif GLOBALS.role == 'so-idh' %}

salt/firewall/defaults.yaml

@@ -19,7 +19,6 @@ firewall:
     manager: []
     managersearch: []
     receiver: []
-    kafkanode: []
     searchnode: []
     self: []
     sensor: []
@@ -443,15 +442,6 @@ firewall:
                 - elastic_agent_data
                 - elastic_agent_update
                 - sensoroni
-            kafkanode:
-              portgroups:
-                - yum
-                - docker_registry
-                - influxdb
-                - elastic_agent_control
-                - elastic_agent_data
-                - elastic_agent_update
-                - sensoroni
             analyst:
               portgroups:
                 - nginx
@@ -530,9 +520,6 @@ firewall:
             receiver:
               portgroups:
                 - salt_manager
-            kafkanode:
-              portgroups:
-                - salt_manager
             desktop:
               portgroups:
                 - salt_manager
@@ -647,15 +634,6 @@ firewall:
                 - elastic_agent_data
                 - elastic_agent_update
                 - sensoroni
-            kafkanode:
-              portgroups:
-                - yum
-                - docker_registry
-                - influxdb
-                - elastic_agent_control
-                - elastic_agent_data
-                - elastic_agent_update
-                - sensoroni
             analyst:
               portgroups:
                 - nginx
@@ -1305,14 +1283,17 @@ firewall:
                 - beats_5044
                 - beats_5644
                 - elastic_agent_data
+                - kafka
             searchnode:
               portgroups:
                 - redis
                 - beats_5644
+                - kafka
             managersearch:
               portgroups:
                 - redis
                 - beats_5644
+                - kafka
             self:
               portgroups:
                 - redis
@@ -1383,73 +1364,6 @@ firewall:
               portgroups: []
             customhostgroup9:
               portgroups: []
-    kafkanode:
-      chain:
-        DOCKER-USER:
-          hostgroups:
-            searchnode:
-              portgroups:
-                - kafka
-            kafkanode:
-              portgroups:
-                - kafka
-            customhostgroup0:
-              portgroups: []
-            customhostgroup1:
-              portgroups: []
-            customhostgroup2:
-              portgroups: []
-            customhostgroup3:
-              portgroups: []
-            customhostgroup4:
-              portgroups: []
-            customhostgroup5:
-              portgroups: []
-            customhostgroup6:
-              portgroups: []
-            customhostgroup7:
-              portgroups: []
-            customhostgroup8:
-              portgroups: []
-            customhostgroup9:
-              portgroups: []
-        INPUT:
-          hostgroups:
-            anywhere:
-              portgroups:
-                - ssh
-            dockernet:
-              portgroups:
-                - all
-            localhost:
-              portgroups:
-                - all
-            self:
-              portgroups:
-                - syslog
-            syslog:
-              portgroups:
-                - syslog
-            customhostgroup0:
-              portgroups: []
-            customhostgroup1:
-              portgroups: []
-            customhostgroup2:
-              portgroups: []
-            customhostgroup3:
-              portgroups: []
-            customhostgroup4:
-              portgroups: []
-            customhostgroup5:
-              portgroups: []
-            customhostgroup6:
-              portgroups: []
-            customhostgroup7:
-              portgroups: []
-            customhostgroup8:
-              portgroups: []
-            customhostgroup9:
-              portgroups: []
     idh:
       chain:
         DOCKER-USER:

salt/firewall/soc_firewall.yaml

@@ -34,7 +34,6 @@ firewall:
     heavynode: *hostgroupsettings
     idh: *hostgroupsettings
     import: *hostgroupsettings
-    kafkanode: *hostgroupsettings
     localhost: *ROhostgroupsettingsadv
     manager: *hostgroupsettings
     managersearch: *hostgroupsettings
@@ -361,8 +360,6 @@ firewall:
               portgroups: *portgroupsdocker
             endgame:
               portgroups: *portgroupsdocker
-            kafkanode:
-              portgroups: *portgroupsdocker
             analyst:
               portgroups: *portgroupsdocker
             desktop:
@@ -454,8 +451,6 @@ firewall:
               portgroups: *portgroupsdocker
             syslog:
               portgroups: *portgroupsdocker
-            kafkanode:
-              portgroups: *portgroupsdocker
             analyst:
               portgroups: *portgroupsdocker
             desktop:
@@ -940,63 +935,6 @@ firewall:
               portgroups: *portgroupshost
             customhostgroup9: 
               portgroups: *portgroupshost
-    kafkanode:
-      chain:
-        DOCKER-USER:
-          hostgroups:
-            searchnode:
-              portgroups: *portgroupsdocker
-            kafkanode:
-              portgroups: *portgroupsdocker
-            customhostgroup0:
-              portgroups: *portgroupsdocker
-            customhostgroup1: 
-              portgroups: *portgroupsdocker
-            customhostgroup2: 
-              portgroups: *portgroupsdocker
-            customhostgroup3: 
-              portgroups: *portgroupsdocker
-            customhostgroup4: 
-              portgroups: *portgroupsdocker
-            customhostgroup5: 
-              portgroups: *portgroupsdocker
-            customhostgroup6: 
-              portgroups: *portgroupsdocker
-            customhostgroup7: 
-              portgroups: *portgroupsdocker
-            customhostgroup8: 
-              portgroups: *portgroupsdocker
-            customhostgroup9: 
-              portgroups: *portgroupsdocker  
-        INPUT:
-          hostgroups:
-            anywhere:
-              portgroups: *portgroupshost
-            dockernet:
-              portgroups: *portgroupshost
-            localhost:
-              portgroups: *portgroupshost
-            customhostgroup0:
-              portgroups: *portgroupshost
-            customhostgroup1: 
-              portgroups: *portgroupshost
-            customhostgroup2: 
-              portgroups: *portgroupshost
-            customhostgroup3: 
-              portgroups: *portgroupshost
-            customhostgroup4: 
-              portgroups: *portgroupshost
-            customhostgroup5: 
-              portgroups: *portgroupshost
-            customhostgroup6: 
-              portgroups: *portgroupshost
-            customhostgroup7: 
-              portgroups: *portgroupshost
-            customhostgroup8: 
-              portgroups: *portgroupshost
-            customhostgroup9: 
-              portgroups: *portgroupshost
-
     idh:
       chain:
         DOCKER-USER:

salt/global/defaults.yaml

@@ -1,2 +1,3 @@
 global:
-  pcapengine: STENO
+  pcapengine: STENO
+  pipeline: REDIS

salt/global/soc_global.yaml

@@ -36,9 +36,10 @@ global:
     global: True
     advanced: True
   pipeline:
-    description: Sets which pipeline technology for events to use. Currently only Redis is supported.
+    description: Sets which pipeline technology for events to use. Currently only Redis is fully supported. Kafka is experimental and requires a Security Onion Pro license.
+    regex: ^(REDIS|KAFKA)$
+    regexFailureMessage: You must enter either REDIS or KAFKA.
     global: True
-    readonly: True
     advanced: True
   repo_host:
     description: Specify the host where operating system packages will be served from.

salt/kafka/enabled.sls

@@ -7,7 +7,7 @@
 {% if sls.split('.')[0] in allowed_states %}
 {%   from 'vars/globals.map.jinja' import GLOBALS %}
 {%   from 'docker/docker.map.jinja' import DOCKER %}
-{%   set KAFKANODES =  salt['pillar.get']('kafka:nodes', {}) %}
+{%   set KAFKANODES = salt['pillar.get']('kafka:nodes', {}) %}
 
 include:
   - kafka.sostatus

salt/kafka/init.sls

@@ -4,9 +4,10 @@
 # Elastic License 2.0.
 
 {% from 'kafka/map.jinja' import KAFKAMERGED %}
+{% from 'vars/globals.map.jinja' import GLOBALS %}
 
 include:
-{% if KAFKAMERGED.enabled %}
+{% if GLOBALS.pipeline == "KAFKA" and KAFKAMERGED.enabled %}
   - kafka.enabled
 {% else %}
   - kafka.disabled

salt/logstash/config.sls

@@ -12,7 +12,7 @@
 
 include:
   - ssl
-  {% if GLOBALS.role not in ['so-receiver','so-fleet', 'so-kafkanode'] %}
+  {% if GLOBALS.role not in ['so-receiver','so-fleet'] %}
   - elasticsearch
   {% endif %}
 

salt/logstash/defaults.yaml

@@ -19,8 +19,6 @@ logstash:
         - search
       fleet:
         - fleet
-      kafkanode:
-        - kafkanode
   defined_pipelines:
     fleet:
       - so/0012_input_elastic_agent.conf.jinja
@@ -39,8 +37,6 @@ logstash:
       - so/0900_input_redis.conf.jinja
       - so/9805_output_elastic_agent.conf.jinja
       - so/9900_output_endgame.conf.jinja
-    kafkanode:
-      - so/0899_output_kafka.conf.jinja
     custom0: []
     custom1: []
     custom2: []

salt/logstash/enabled.sls

@@ -75,7 +75,7 @@ so-logstash:
       {% else %}
       - /etc/pki/tls/certs/intca.crt:/usr/share/filebeat/ca.crt:ro
       {% endif %}
-      {% if GLOBALS.role in ['so-manager', 'so-managersearch', 'so-standalone', 'so-import', 'so-heavynode', 'so-searchnode', 'so-kafkanode' ] %}
+      {% if GLOBALS.role in ['so-manager', 'so-managersearch', 'so-standalone', 'so-import', 'so-heavynode', 'so-searchnode' ] %}
       - /opt/so/conf/ca/cacerts:/etc/pki/ca-trust/extracted/java/cacerts:ro
       - /opt/so/conf/ca/tls-ca-bundle.pem:/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem:ro
       - /etc/pki/kafka-logstash.p12:/usr/share/logstash/kafka-logstash.p12:ro

salt/logstash/pipelines/config/so/0800_input_kafka.conf.jinja

@@ -1,11 +1,9 @@
-{% set kafka_brokers = salt['pillar.get']('logstash:nodes:kafkanode', {}) %}
+{% set kafka_brokers = salt['pillar.get']('logstash:nodes:receiver', {}) %}
 {% set kafka_on_mngr = salt ['pillar.get']('logstash:nodes:manager', {}) %}
 {% set broker_ips = [] %}
 {% for node, node_data in kafka_brokers.items() %}
   {% do broker_ips.append(node_data['ip'] + ":9092") %}
 {% endfor %}
-
-{# For testing kafka stuff from manager not dedicated kafkanodes #}
 {% for node, node_data in kafka_on_mngr.items() %}
   {% do broker_ips.append(node_data['ip'] + ":9092") %}
 {% endfor %}

salt/logstash/soc_logstash.yaml

@@ -16,7 +16,6 @@ logstash:
       manager: *assigned_pipelines
       managersearch: *assigned_pipelines
       fleet: *assigned_pipelines
-      kafkanode: *assigned_pipelines
   defined_pipelines:
     receiver: &defined_pipelines
       description: List of pipeline configurations assign to this group.
@@ -27,7 +26,6 @@ logstash:
     fleet: *defined_pipelines
     manager: *defined_pipelines
     search: *defined_pipelines
-    kafkanode: *defined_pipelines
     custom0: *defined_pipelines
     custom1: *defined_pipelines
     custom2: *defined_pipelines

salt/manager/tools/sbin/so-firewall-minion

@@ -79,9 +79,6 @@ fi
 		'RECEIVER')
 			so-firewall includehost receiver "$IP" --apply
 			;;
-		'KAFKANODE')
-			so-firewall includehost kafkanode "$IP" --apply
-			;;
 		'DESKTOP')
 			so-firewall includehost desktop "$IP" --apply
 			;;