From 5278601e5d1cd05a613a2486e867c95d87936d8c Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Mon, 7 Aug 2023 11:18:35 -0400 Subject: [PATCH 1/5] manage telegraf scripts with a defaults file assigned per node type --- salt/telegraf/config.sls | 15 ++--- salt/telegraf/defaults.yaml | 79 ++++++++++++++++++++++ salt/telegraf/enabled.sls | 5 +- salt/telegraf/etc/telegraf.conf | 114 +++----------------------------- salt/telegraf/map.jinja | 10 ++- salt/telegraf/soc_telegraf.yaml | 19 +++++- 6 files changed, 127 insertions(+), 115 deletions(-) diff --git a/salt/telegraf/config.sls b/salt/telegraf/config.sls index 1cc7ceed06..0711260b53 100644 --- a/salt/telegraf/config.sls +++ b/salt/telegraf/config.sls @@ -32,17 +32,16 @@ tgrafetsdir: - name: /opt/so/conf/telegraf/scripts - makedirs: True -tgrafsyncscripts: - file.recurse: - - name: /opt/so/conf/telegraf/scripts +{% for script in TELEGRAFMERGED.scripts[GLOBALS.role.split('-')[1]] %} +tgraf_sync_script_{{script}}: + file.managed: + - name: /opt/so/conf/telegraf/scripts/{{script}} - user: root - group: 939 - - file_mode: 770 + - mode: 770 - template: jinja - - source: salt://telegraf/scripts - {% if GLOBALS.md_engine == 'SURICATA' %} - - exclude_pat: zeekcaptureloss.sh - {% endif %} + - source: salt://telegraf/scripts/{{script}} +{% endfor %} telegraf_sbin: file.recurse: diff --git a/salt/telegraf/defaults.yaml b/salt/telegraf/defaults.yaml index 63d437763c..36ef679f05 100644 --- a/salt/telegraf/defaults.yaml +++ b/salt/telegraf/defaults.yaml @@ -9,3 +9,82 @@ telegraf: flush_jitter: '0s' debug: 'false' quiet: 'false' + scripts: + eval: + - beatseps.sh + - checkfiles.sh + - influxdbsize.sh + - oldpcap.sh + - raid.sh + - redis.sh + - sostatus.sh + - stenoloss.sh + - suriloss.sh + - zeekcaptureloss.sh + - zeekloss.sh + standalone: + - beatseps.sh + - checkfiles.sh + - eps.sh + - influxdbsize.sh + - oldpcap.sh + - raid.sh + - redis.sh + - sostatus.sh + - stenoloss.sh + - suriloss.sh + - zeekcaptureloss.sh + - zeekloss.sh + manager: + - beatseps.sh + - influxdbsize.sh + - raid.sh + - redis.sh + - sostatus.sh + managersearch: + - beatseps.sh + - eps.sh + - influxdbsize.sh + - raid.sh + - redis.sh + - sostatus.sh + import: + - sostatus.sh + sensor: + - beatseps.sh + - checkfiles.sh + - oldpcap.sh + - raid.sh + - sostatus.sh + - stenoloss.sh + - suriloss.sh + - zeekcaptureloss.sh + - zeekloss.sh + heavynode: + - beatseps.sh + - checkfiles.sh + - eps.sh + - oldpcap.sh + - raid.sh + - redis.sh + - sostatus.sh + - stenoloss.sh + - suriloss.sh + - zeekcaptureloss.sh + - zeekloss.sh + idh: + - sostatus.sh + searchnode: + - beatseps.sh + - eps.sh + - raid.sh + - sostatus.sh + receiver: + - beatseps.sh + - eps.sh + - raid.sh + - redis.sh + - sostatus.sh + fleet: + - sostatus.sh + desktop: [] diff --git a/salt/telegraf/enabled.sls b/salt/telegraf/enabled.sls index 209c85fb0b..598587e17b 100644 --- a/salt/telegraf/enabled.sls +++ b/salt/telegraf/enabled.sls @@ -7,6 +7,7 @@ {% if sls.split('.')[0] in allowed_states %} {% from 'vars/globals.map.jinja' import GLOBALS %} {% from 'docker/docker.map.jinja' import DOCKER %} +{% from 'telegraf/map.jinja' import TELEGRAFMERGED %} include: @@ -67,8 +68,10 @@ so-telegraf: {% endif %} - watch: - file: tgrafconf - - file: tgrafsyncscripts - file: node_config + {% for script in TELEGRAFMERGED.scripts[GLOBALS.role.split('-')[1]] %} + - file: tgraf_sync_script_{{script}} + {% endfor %} - require: - file: tgrafconf - file: node_config diff --git a/salt/telegraf/etc/telegraf.conf b/salt/telegraf/etc/telegraf.conf index 1a6cdc3111..45b1283e01 100644 --- a/salt/telegraf/etc/telegraf.conf +++ b/salt/telegraf/etc/telegraf.conf @@ -193,7 +193,7 @@ username = "{{ ES_USER }}" password = "{{ ES_PASS }}" insecure_skip_verify = true -{%- elif grains['role'] in ['so-searchnode', 'so-hotnode', 'so-warmnode'] %} +{%- elif grains['role'] in ['so-searchnode'] %} [[inputs.elasticsearch]] servers = ["https://{{ NODEIP }}:9200"] cluster_stats = false @@ -244,6 +244,8 @@ {%- endif %} # # Read metrics from one or more commands that can output to stdout +{%- if 'sostatus.sh' in TELEGRAFMERGED.scripts[GLOBALS.role.split('-')[1]] %} +{%- do TELEGRAFMERGED.scripts[GLOBALS.role.split('-')[1]].remove('sostatus.sh') %} [[inputs.exec]] commands = [ "/scripts/sostatus.sh" @@ -251,122 +253,26 @@ data_format = "influx" timeout = "15s" interval = "60s" +{%- endif %} -# ## Commands array -{% if grains['role'] in ['so-manager'] %} -[[inputs.exec]] - commands = [ - "/scripts/redis.sh", - "/scripts/influxdbsize.sh", - "/scripts/raid.sh", - "/scripts/beatseps.sh" - ] - data_format = "influx" - ## Timeout for each command to complete. - timeout = "15s" -{% elif grains['role'] in ['so-managersearch'] %} -[[inputs.exec]] - commands = [ - "/scripts/redis.sh", - "/scripts/influxdbsize.sh", - "/scripts/eps.sh", - "/scripts/raid.sh", - "/scripts/beatseps.sh" - ] - data_format = "influx" - ## Timeout for each command to complete. - timeout = "15s" -{% elif grains['role'] in ['so-searchnode', 'so-receiver'] %} +{%- if TELEGRAFMERGED.scripts[GLOBALS.role.split('-')[1]] | length > 0 %} [[inputs.exec]] commands = [ - "/scripts/eps.sh", - "/scripts/raid.sh", - {% if grains.role == 'so-receiver' %} - "/scripts/redis.sh", - {% endif %} - "/scripts/beatseps.sh" +{%- for script in TELEGRAFMERGED.scripts[GLOBALS.role.split('-')[1]] %} + "/scripts/{{script}}"{% if not loop.last %},{% endif %} +{%- endfor %} ] data_format = "influx" ## Timeout for each command to complete. timeout = "15s" -{% elif grains['role'] == 'so-sensor' %} -[[inputs.exec]] - commands = [ - "/scripts/stenoloss.sh", - "/scripts/suriloss.sh", - "/scripts/checkfiles.sh", - {%- if MDENGINE == 'ZEEK' and ZEEK_ENABLED %} - "/scripts/zeekloss.sh", - "/scripts/zeekcaptureloss.sh", - {%- endif %} - "/scripts/oldpcap.sh", - "/scripts/raid.sh", - "/scripts/beatseps.sh" - ] - data_format = "influx" - timeout = "15s" -{% elif grains['role'] == 'so-heavynode' %} -[[inputs.exec]] - commands = [ - "/scripts/redis.sh", - "/scripts/stenoloss.sh", - "/scripts/suriloss.sh", - "/scripts/checkfiles.sh", - {%- if MDENGINE == 'ZEEK' and ZEEK_ENABLED %} - "/scripts/zeekloss.sh", - "/scripts/zeekcaptureloss.sh", - {%- endif %} - "/scripts/oldpcap.sh", - "/scripts/eps.sh", - "/scripts/raid.sh", - "/scripts/beatseps.sh" - ] - data_format = "influx" - timeout = "15s" -{% elif grains['role'] == 'so-standalone' %} -[[inputs.exec]] - commands = [ - "/scripts/redis.sh", - "/scripts/influxdbsize.sh", - "/scripts/stenoloss.sh", - "/scripts/suriloss.sh", - "/scripts/checkfiles.sh", - {%- if MDENGINE == 'ZEEK' and ZEEK_ENABLED %} - "/scripts/zeekloss.sh", - "/scripts/zeekcaptureloss.sh", - {%- endif %} - "/scripts/oldpcap.sh", - "/scripts/eps.sh", - "/scripts/raid.sh", - "/scripts/beatseps.sh" - ] - data_format = "influx" - timeout = "15s" -{% elif grains['role'] == 'so-eval' %} -[[inputs.exec]] - commands = [ - "/scripts/redis.sh", - "/scripts/stenoloss.sh", - "/scripts/suriloss.sh", - "/scripts/checkfiles.sh", - {%- if MDENGINE == 'ZEEK' and ZEEK_ENABLED %} - "/scripts/zeekloss.sh", - "/scripts/zeekcaptureloss.sh", - {%- endif %} - "/scripts/oldpcap.sh", - "/scripts/influxdbsize.sh", - "/scripts/raid.sh", - "/scripts/beatseps.sh" - ] - data_format = "influx" - timeout = "15s" -{% endif %} +{%- endif %} {%- if salt['pillar.get']('healthcheck:enabled', False) %} [[inputs.file]] files = ["/host/nsm/zeek/logs/zeek_restart.log"] data_format = "influx" {%- endif %} + [[inputs.file]] files = ["/etc/telegraf/node_config.json"] name_override = "node_config" diff --git a/salt/telegraf/map.jinja b/salt/telegraf/map.jinja index f1412d3ac2..0f197a7b10 100644 --- a/salt/telegraf/map.jinja +++ b/salt/telegraf/map.jinja @@ -2,6 +2,14 @@ or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at https://securityonion.net/license; you may not use this file except in compliance with the Elastic License 2.0. #} - + +{% from 'vars/globals.map.jinja' import GLOBALS %} {% import_yaml 'telegraf/defaults.yaml' as TELEGRAFDEFAULTS %} {% set TELEGRAFMERGED = salt['pillar.get']('telegraf', TELEGRAFDEFAULTS.telegraf, merge=True) %} +{% from 'zeek/config.map.jinja' import ZEEKMERGED %} + +{# if the md engine isn't zeek or zeek is disabled, dont run the zeek scripts for telegraf #} +{% if (GLOBALS.role in ['so-eval', 'so-standalone', 'so-sensor', 'so-heavynode']) and (GLOBALS.md_engine != 'ZEEK' or not ZEEKMERGED.enabled) %} +{% do TELEGRAFMERGED.scripts[GLOBALS.role.split('-')[1]].remove('zeekloss.sh') %} +{% do TELEGRAFMERGED.scripts[GLOBALS.role.split('-')[1]].remove('zeekcaptureloss.sh') %} +{% endif %} diff --git a/salt/telegraf/soc_telegraf.yaml b/salt/telegraf/soc_telegraf.yaml index a688ea2a3d..1550c66cb7 100644 --- a/salt/telegraf/soc_telegraf.yaml +++ b/salt/telegraf/soc_telegraf.yaml @@ -42,4 +42,21 @@ telegraf: global: True advanced: True helpLink: telegraf.html - \ No newline at end of file + scripts: + eval: &telegrafscripts + description: List of input.exec scripts to run for this node type. The script must be present in salt/telegraf/scripts. + forcedType: "[]string" + multiline: True + advanced: True + helpLink: telegraf.html + standalone: *telegrafscripts + manager: *telegrafscripts + managersearch: *telegrafscripts + import: *telegrafscripts + sensor: *telegrafscripts + heavynode: *telegrafscripts + idh: *telegrafscripts + searchnode: *telegrafscripts + receiver: *telegrafscripts + fleet: *telegrafscripts + desktop: *telegrafscripts From 609a2bf32e7085b18c0d039b0de51501800e468c Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 8 Aug 2023 09:27:03 -0400 Subject: [PATCH 2/5] only import ZEEKMERGED if a sensor type node --- salt/telegraf/map.jinja | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/salt/telegraf/map.jinja b/salt/telegraf/map.jinja index 0f197a7b10..e6d3460d67 100644 --- a/salt/telegraf/map.jinja +++ b/salt/telegraf/map.jinja @@ -6,10 +6,12 @@ {% from 'vars/globals.map.jinja' import GLOBALS %} {% import_yaml 'telegraf/defaults.yaml' as TELEGRAFDEFAULTS %} {% set TELEGRAFMERGED = salt['pillar.get']('telegraf', TELEGRAFDEFAULTS.telegraf, merge=True) %} -{% from 'zeek/config.map.jinja' import ZEEKMERGED %} -{# if the md engine isn't zeek or zeek is disabled, dont run the zeek scripts for telegraf #} -{% if (GLOBALS.role in ['so-eval', 'so-standalone', 'so-sensor', 'so-heavynode']) and (GLOBALS.md_engine != 'ZEEK' or not ZEEKMERGED.enabled) %} -{% do TELEGRAFMERGED.scripts[GLOBALS.role.split('-')[1]].remove('zeekloss.sh') %} -{% do TELEGRAFMERGED.scripts[GLOBALS.role.split('-')[1]].remove('zeekcaptureloss.sh') %} +{% if GLOBALS.role in ['so-eval', 'so-standalone', 'so-sensor', 'so-heavynode'] %} +{% from 'zeek/config.map.jinja' import ZEEKMERGED %} +{# if the md engine isn't zeek or zeek is disabled, dont run the zeek scripts for telegraf #} +{% if GLOBALS.md_engine != 'ZEEK' or not ZEEKMERGED.enabled %} +{% do TELEGRAFMERGED.scripts[GLOBALS.role.split('-')[1]].remove('zeekloss.sh') %} +{% do TELEGRAFMERGED.scripts[GLOBALS.role.split('-')[1]].remove('zeekcaptureloss.sh') %} +{% endif %} {% endif %} From 69553f9017b3ffe5ffa4ae776bf5006eddfec894 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 8 Aug 2023 09:34:59 -0400 Subject: [PATCH 3/5] removes spaces from zeekcaptureloss script --- salt/telegraf/scripts/zeekcaptureloss.sh | 3 --- 1 file changed, 3 deletions(-) diff --git a/salt/telegraf/scripts/zeekcaptureloss.sh b/salt/telegraf/scripts/zeekcaptureloss.sh index e254ada321..f2c3fcd2d5 100644 --- a/salt/telegraf/scripts/zeekcaptureloss.sh +++ b/salt/telegraf/scripts/zeekcaptureloss.sh @@ -5,9 +5,6 @@ # https://securityonion.net/license; you may not use this file except in compliance with the # Elastic License 2.0. - - - # This script returns the average of all the workers average capture loss to telegraf / influxdb in influx format include nanosecond precision timestamp # if this script isn't already running From a06040c035e6db035426fb001afc1887276cc55a Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 8 Aug 2023 09:37:37 -0400 Subject: [PATCH 4/5] add WORKERS calculation back to zeekcaptureloss script --- salt/telegraf/scripts/zeekcaptureloss.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/salt/telegraf/scripts/zeekcaptureloss.sh b/salt/telegraf/scripts/zeekcaptureloss.sh index 6e1785237d..ddb6cd1287 100644 --- a/salt/telegraf/scripts/zeekcaptureloss.sh +++ b/salt/telegraf/scripts/zeekcaptureloss.sh @@ -11,7 +11,11 @@ if [[ ! "`pidof -x $(basename $0) -o %PPID`" ]]; then if [ -d "/host/nsm/zeek/spool/logger" ]; then - WORKERS={{ salt['pillar.get']('sensor:zeek_lbprocs', salt['pillar.get']('sensor:zeek_pins') | length) }} +{%- if ZEEKMERGED.config.node.pins %} + WORKERS={{ ZEEKMERGED.config.node.pins | length }} +{%- else %} + WORKERS={{ ZEEKMERGED.config.node.lb_procs }} +{%- endif %} ZEEKLOG=/host/nsm/zeek/spool/logger/capture_loss.log elif [ -d "/host/nsm/zeek/spool/zeeksa" ]; then WORKERS=1 From 673b45af092b6a655bf115edffc4d88ef0ba393a Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 8 Aug 2023 09:41:42 -0400 Subject: [PATCH 5/5] import ZEEKMERGED --- salt/telegraf/scripts/zeekcaptureloss.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/telegraf/scripts/zeekcaptureloss.sh b/salt/telegraf/scripts/zeekcaptureloss.sh index ddb6cd1287..f2c3fcd2d5 100644 --- a/salt/telegraf/scripts/zeekcaptureloss.sh +++ b/salt/telegraf/scripts/zeekcaptureloss.sh @@ -8,6 +8,7 @@ # This script returns the average of all the workers average capture loss to telegraf / influxdb in influx format include nanosecond precision timestamp # if this script isn't already running +{%- from 'zeek/config.map.jinja' import ZEEKMERGED %} if [[ ! "`pidof -x $(basename $0) -o %PPID`" ]]; then if [ -d "/host/nsm/zeek/spool/logger" ]; then