From d203aec44a47f61e7d75fa19250fad01ebd12821 Mon Sep 17 00:00:00 2001 From: Wes Date: Fri, 8 Dec 2023 19:37:06 +0000 Subject: [PATCH 1/5] Remove Curator --- pillar/top.sls | 10 --- salt/allowed_states.map.jinja | 4 - salt/common/tools/sbin/so-image-common | 1 - salt/curator/config.sls | 81 ------------------- salt/curator/defaults.yaml | 34 -------- salt/curator/disabled.sls | 35 -------- salt/curator/enabled.sls | 81 ------------------- salt/curator/files/action/delete.yml | 32 -------- salt/curator/files/action/so-beats-delete.yml | 27 ------- .../files/action/so-elasticsearch-delete.yml | 27 ------- .../files/action/so-firewall-delete.yml | 28 ------- salt/curator/files/action/so-ids-delete.yml | 28 ------- .../curator/files/action/so-import-delete.yml | 27 ------- .../curator/files/action/so-kibana-delete.yml | 27 ------- .../curator/files/action/so-kratos-delete.yml | 27 ------- .../files/action/so-logstash-delete.yml | 27 ------- .../files/action/so-netflow-delete.yml | 27 ------- .../files/action/so-osquery-delete.yml | 27 ------- salt/curator/files/action/so-ossec-delete.yml | 27 ------- salt/curator/files/action/so-redis-delete.yml | 27 ------- .../files/action/so-strelka-delete.yml | 27 ------- .../curator/files/action/so-syslog-delete.yml | 27 ------- salt/curator/files/action/so-zeek-delete.yml | 27 ------- salt/curator/files/curator.yml | 40 --------- salt/curator/init.sls | 13 --- salt/curator/map.jinja | 7 -- salt/curator/soc_curator.yaml | 39 --------- salt/curator/sostatus.sls | 21 ----- salt/curator/tools/sbin/so-curator-delete | 17 ---- salt/curator/tools/sbin/so-curator-restart | 12 --- salt/curator/tools/sbin/so-curator-start | 12 --- salt/curator/tools/sbin/so-curator-stop | 12 --- salt/docker/defaults.yaml | 5 -- salt/docker/soc_docker.yaml | 1 - salt/elasticsearch/disabled.sls | 4 + salt/elasticsearch/enabled.sls | 20 +++++ .../sbin/so-elasticsearch-indices-delete} | 2 +- .../so-elasticsearch-indices-delete-delete} | 4 +- salt/firewall/containers.map.jinja | 3 - salt/logrotate/defaults.yaml | 2 +- salt/logrotate/soc_logrotate.yaml | 12 +-- .../tools/sbin/so-elastic-auth-password-reset | 2 - salt/manager/tools/sbin/so-minion | 14 +--- salt/manager/tools/sbin/soup | 5 +- salt/top.sls | 5 -- setup/so-functions | 3 +- setup/so-variables | 8 +- 47 files changed, 39 insertions(+), 909 deletions(-) delete mode 100644 salt/curator/config.sls delete mode 100644 salt/curator/defaults.yaml delete mode 100644 salt/curator/disabled.sls delete mode 100644 salt/curator/enabled.sls delete mode 100644 salt/curator/files/action/delete.yml delete mode 100644 salt/curator/files/action/so-beats-delete.yml delete mode 100644 salt/curator/files/action/so-elasticsearch-delete.yml delete mode 100644 salt/curator/files/action/so-firewall-delete.yml delete mode 100644 salt/curator/files/action/so-ids-delete.yml delete mode 100644 salt/curator/files/action/so-import-delete.yml delete mode 100644 salt/curator/files/action/so-kibana-delete.yml delete mode 100644 salt/curator/files/action/so-kratos-delete.yml delete mode 100644 salt/curator/files/action/so-logstash-delete.yml delete mode 100644 salt/curator/files/action/so-netflow-delete.yml delete mode 100644 salt/curator/files/action/so-osquery-delete.yml delete mode 100644 salt/curator/files/action/so-ossec-delete.yml delete mode 100644 salt/curator/files/action/so-redis-delete.yml delete mode 100644 salt/curator/files/action/so-strelka-delete.yml delete mode 100644 salt/curator/files/action/so-syslog-delete.yml delete mode 100644 salt/curator/files/action/so-zeek-delete.yml delete mode 100644 salt/curator/files/curator.yml delete mode 100644 salt/curator/init.sls delete mode 100644 salt/curator/map.jinja delete mode 100644 salt/curator/soc_curator.yaml delete mode 100644 salt/curator/sostatus.sls delete mode 100644 salt/curator/tools/sbin/so-curator-delete delete mode 100644 salt/curator/tools/sbin/so-curator-restart delete mode 100644 salt/curator/tools/sbin/so-curator-start delete mode 100644 salt/curator/tools/sbin/so-curator-stop rename salt/{curator/tools/sbin/so-curator-cluster-delete => elasticsearch/tools/sbin/so-elasticsearch-indices-delete} (92%) rename salt/{curator/tools/sbin_jinja/so-curator-cluster-delete-delete => elasticsearch/tools/sbin_jinja/so-elasticsearch-indices-delete-delete} (96%) diff --git a/pillar/top.sls b/pillar/top.sls index 53ec8a3306..ed3e492543 100644 --- a/pillar/top.sls +++ b/pillar/top.sls @@ -61,8 +61,6 @@ base: - elastalert.adv_elastalert - backup.soc_backup - backup.adv_backup - - curator.soc_curator - - curator.adv_curator - soctopus.soc_soctopus - soctopus.adv_soctopus - minions.{{ grains.id }} @@ -113,8 +111,6 @@ base: - kibana.adv_kibana - strelka.soc_strelka - strelka.adv_strelka - - curator.soc_curator - - curator.adv_curator - kratos.soc_kratos - kratos.adv_kratos - redis.soc_redis @@ -172,8 +168,6 @@ base: - kibana.adv_kibana - strelka.soc_strelka - strelka.adv_strelka - - curator.soc_curator - - curator.adv_curator - backup.soc_backup - backup.adv_backup - zeek.soc_zeek @@ -194,8 +188,6 @@ base: - logstash.adv_logstash - elasticsearch.soc_elasticsearch - elasticsearch.adv_elasticsearch - - curator.soc_curator - - curator.adv_curator - redis.soc_redis - redis.adv_redis - zeek.soc_zeek @@ -268,8 +260,6 @@ base: - soctopus.adv_soctopus - kibana.soc_kibana - kibana.adv_kibana - - curator.soc_curator - - curator.adv_curator - backup.soc_backup - backup.adv_backup - kratos.soc_kratos diff --git a/salt/allowed_states.map.jinja b/salt/allowed_states.map.jinja index 4e3e57f9c8..6585bd96c2 100644 --- a/salt/allowed_states.map.jinja +++ b/salt/allowed_states.map.jinja @@ -219,10 +219,6 @@ {% do allowed_states.append('kibana.secrets') %} {% endif %} - {% if grains.role in ['so-eval', 'so-standalone', 'so-managersearch', 'so-heavynode', 'so-manager'] %} - {% do allowed_states.append('curator') %} - {% endif %} - {% if grains.role in ['so-eval', 'so-manager', 'so-standalone', 'so-managersearch'] %} {% do allowed_states.append('elastalert') %} {% endif %} diff --git a/salt/common/tools/sbin/so-image-common b/salt/common/tools/sbin/so-image-common index 7e510e3ad6..7900b3c529 100755 --- a/salt/common/tools/sbin/so-image-common +++ b/salt/common/tools/sbin/so-image-common @@ -42,7 +42,6 @@ container_list() { ) elif [ $MANAGERCHECK != 'so-helix' ]; then TRUSTED_CONTAINERS=( - "so-curator" "so-elastalert" "so-elastic-agent" "so-elastic-agent-builder" diff --git a/salt/curator/config.sls b/salt/curator/config.sls deleted file mode 100644 index 89ff53b2a5..0000000000 --- a/salt/curator/config.sls +++ /dev/null @@ -1,81 +0,0 @@ -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - -{% from 'allowed_states.map.jinja' import allowed_states %} -{% if sls.split('.')[0] in allowed_states %} -{% from "curator/map.jinja" import CURATORMERGED %} - -# Create the group -curatorgroup: - group.present: - - name: curator - - gid: 934 - -# Add user -curator: - user.present: - - uid: 934 - - gid: 934 - - home: /opt/so/conf/curator - - createhome: False - -# Create the log directory -curlogdir: - file.directory: - - name: /opt/so/log/curator - - user: 934 - - group: 939 - -curactiondir: - file.directory: - - name: /opt/so/conf/curator/action - - user: 934 - - group: 939 - - makedirs: True - -actionconfs: - file.recurse: - - name: /opt/so/conf/curator/action - - source: salt://curator/files/action - - user: 934 - - group: 939 - - template: jinja - - defaults: - CURATORMERGED: {{ CURATORMERGED.elasticsearch.index_settings }} - -curconf: - file.managed: - - name: /opt/so/conf/curator/curator.yml - - source: salt://curator/files/curator.yml - - user: 934 - - group: 939 - - mode: 660 - - template: jinja - - show_changes: False - -curator_sbin: - file.recurse: - - name: /usr/sbin - - source: salt://curator/tools/sbin - - user: 934 - - group: 939 - - file_mode: 755 - -curator_sbin_jinja: - file.recurse: - - name: /usr/sbin - - source: salt://curator/tools/sbin_jinja - - user: 934 - - group: 939 - - file_mode: 755 - - template: jinja - -{% else %} - -{{sls}}_state_not_allowed: - test.fail_without_changes: - - name: {{sls}}_state_not_allowed - -{% endif %} diff --git a/salt/curator/defaults.yaml b/salt/curator/defaults.yaml deleted file mode 100644 index b2be8d5edf..0000000000 --- a/salt/curator/defaults.yaml +++ /dev/null @@ -1,34 +0,0 @@ -curator: - enabled: False - elasticsearch: - index_settings: - so-beats: - delete: 365 - so-elasticsearch: - delete: 365 - so-firewall: - delete: 365 - so-ids: - delete: 365 - so-import: - delete: 73001 - so-kratos: - delete: 365 - so-kibana: - delete: 365 - so-logstash: - delete: 365 - so-netflow: - delete: 365 - so-osquery: - delete: 365 - so-ossec: - delete: 365 - so-redis: - delete: 365 - so-strelka: - delete: 365 - so-syslog: - delete: 365 - so-zeek: - delete: 365 diff --git a/salt/curator/disabled.sls b/salt/curator/disabled.sls deleted file mode 100644 index acf9e37011..0000000000 --- a/salt/curator/disabled.sls +++ /dev/null @@ -1,35 +0,0 @@ -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - -{% from 'allowed_states.map.jinja' import allowed_states %} -{% if sls.split('.')[0] in allowed_states %} - -include: - - curator.sostatus - -so-curator: - docker_container.absent: - - force: True - -so-curator_so-status.disabled: - file.comment: - - name: /opt/so/conf/so-status/so-status.conf - - regex: ^so-curator$ - -so-curator-cluster-close: - cron.absent: - - identifier: so-curator-cluster-close - -so-curator-cluster-delete: - cron.absent: - - identifier: so-curator-cluster-delete - -{% else %} - -{{sls}}_state_not_allowed: - test.fail_without_changes: - - name: {{sls}}_state_not_allowed - -{% endif %} diff --git a/salt/curator/enabled.sls b/salt/curator/enabled.sls deleted file mode 100644 index 916aa920d5..0000000000 --- a/salt/curator/enabled.sls +++ /dev/null @@ -1,81 +0,0 @@ -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - -{% from 'allowed_states.map.jinja' import allowed_states %} -{% if sls.split('.')[0] in allowed_states %} -{% from 'vars/globals.map.jinja' import GLOBALS %} -{% from 'docker/docker.map.jinja' import DOCKER %} - -include: - - curator.config - - curator.sostatus - -so-curator: - docker_container.running: - - image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-curator:{{ GLOBALS.so_version }} - - start: True - - hostname: curator - - name: so-curator - - user: curator - - networks: - - sobridge: - - ipv4_address: {{ DOCKER.containers['so-curator'].ip }} - - interactive: True - - tty: True - - binds: - - /opt/so/conf/curator/curator.yml:/etc/curator/config/curator.yml:ro - - /opt/so/conf/curator/action/:/etc/curator/action:ro - - /opt/so/log/curator:/var/log/curator:rw - {% if DOCKER.containers['so-curator'].custom_bind_mounts %} - {% for BIND in DOCKER.containers['so-curator'].custom_bind_mounts %} - - {{ BIND }} - {% endfor %} - {% endif %} - {% if DOCKER.containers['so-curator'].extra_hosts %} - - extra_hosts: - {% for XTRAHOST in DOCKER.containers['so-curator'].extra_hosts %} - - {{ XTRAHOST }} - {% endfor %} - {% endif %} - {% if DOCKER.containers['so-curator'].extra_env %} - - environment: - {% for XTRAENV in DOCKER.containers['so-curator'].extra_env %} - - {{ XTRAENV }} - {% endfor %} - {% endif %} - - require: - - file: actionconfs - - file: curconf - - file: curlogdir - - watch: - - file: curconf - -delete_so-curator_so-status.disabled: - file.uncomment: - - name: /opt/so/conf/so-status/so-status.conf - - regex: ^so-curator$ - -so-curator-cluster-close: - cron.absent: - - identifier: so-curator-cluster-close - -so-curator-cluster-delete: - cron.present: - - name: /usr/sbin/so-curator-cluster-delete > /opt/so/log/curator/cron-cluster-delete.log 2>&1 - - identifier: so-curator-cluster-delete - - user: root - - minute: '*/5' - - hour: '*' - - daymonth: '*' - - month: '*' - - dayweek: '*' - -{% else %} - -{{sls}}_state_not_allowed: - test.fail_without_changes: - - name: {{sls}}_state_not_allowed - -{% endif %} diff --git a/salt/curator/files/action/delete.yml b/salt/curator/files/action/delete.yml deleted file mode 100644 index 253c6fd674..0000000000 --- a/salt/curator/files/action/delete.yml +++ /dev/null @@ -1,32 +0,0 @@ -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - -{% import_yaml 'elasticsearch/defaults.yaml' as ELASTICDEFAULTS %} -{% set ELASTICMERGED = salt['pillar.get']('elasticsearch:retention', ELASTICDEFAULTS.elasticsearch.retention, merge=true) %} - -{{ ELASTICMERGED.retention_pct }} - -{%- set log_size_limit = salt['pillar.get']('elasticsearch:log_size_limit') %} -actions: - 1: - action: delete_indices - description: >- - Delete indices when {{log_size_limit}}(GB) is exceeded. - options: - allow_ilm_indices: True - ignore_empty_list: True - disable_action: False - filters: - - filtertype: pattern - kind: regex - value: '^(logstash-.*|so-.*|.ds-logs-.*-so.*)$' - - filtertype: pattern - kind: regex - value: '^(so-case.*)$' - exclude: True - - filtertype: space - source: creation_date - use_age: True - disk_space: {{log_size_limit}} diff --git a/salt/curator/files/action/so-beats-delete.yml b/salt/curator/files/action/so-beats-delete.yml deleted file mode 100644 index c4e1f8b4e9..0000000000 --- a/salt/curator/files/action/so-beats-delete.yml +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - -{%- set DELETE_DAYS = CURATORMERGED['so-beats'].delete %} -actions: - 1: - action: delete_indices - description: >- - Delete beats indices when older than {{ DELETE_DAYS }} days. - options: - ignore_empty_list: True - disable_action: False - filters: - - filtertype: pattern - kind: regex - value: '^(logstash-beats.*|so-beats.*)$' - - filtertype: age - source: name - direction: older - timestring: '%Y.%m.%d' - unit: days - unit_count: {{ DELETE_DAYS }} - exclude: - - \ No newline at end of file diff --git a/salt/curator/files/action/so-elasticsearch-delete.yml b/salt/curator/files/action/so-elasticsearch-delete.yml deleted file mode 100644 index 3c6bf4aacf..0000000000 --- a/salt/curator/files/action/so-elasticsearch-delete.yml +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - -{%- set DELETE_DAYS = CURATORMERGED['so-elasticsearch'].delete %} -actions: - 1: - action: delete_indices - description: >- - Delete elasticsearch indices when older than {{ DELETE_DAYS }} days. - options: - ignore_empty_list: True - disable_action: False - filters: - - filtertype: pattern - kind: regex - value: '^(logstash-elasticsearch.*|so-elasticsearch.*)$' - - filtertype: age - source: name - direction: older - timestring: '%Y.%m.%d' - unit: days - unit_count: {{ DELETE_DAYS }} - exclude: - - \ No newline at end of file diff --git a/salt/curator/files/action/so-firewall-delete.yml b/salt/curator/files/action/so-firewall-delete.yml deleted file mode 100644 index 5143e2fe95..0000000000 --- a/salt/curator/files/action/so-firewall-delete.yml +++ /dev/null @@ -1,28 +0,0 @@ -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - - -{%- set DELETE_DAYS = CURATORMERGED['so-firewall'].delete %} -actions: - 1: - action: delete_indices - description: >- - Delete firewall indices when older than {{ DELETE_DAYS }} days. - options: - ignore_empty_list: True - disable_action: False - filters: - - filtertype: pattern - kind: regex - value: '^(logstash-firewall.*|so-firewall.*)$' - - filtertype: age - source: name - direction: older - timestring: '%Y.%m.%d' - unit: days - unit_count: {{ DELETE_DAYS }} - exclude: - - \ No newline at end of file diff --git a/salt/curator/files/action/so-ids-delete.yml b/salt/curator/files/action/so-ids-delete.yml deleted file mode 100644 index 6cf120fef3..0000000000 --- a/salt/curator/files/action/so-ids-delete.yml +++ /dev/null @@ -1,28 +0,0 @@ -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - - -{%- set DELETE_DAYS = CURATORMERGED['so-ids'].delete %} -actions: - 1: - action: delete_indices - description: >- - Delete IDS indices when older than {{ DELETE_DAYS }} days. - options: - ignore_empty_list: True - disable_action: False - filters: - - filtertype: pattern - kind: regex - value: '^(logstash-ids.*|so-ids.*)$' - - filtertype: age - source: name - direction: older - timestring: '%Y.%m.%d' - unit: days - unit_count: {{ DELETE_DAYS }} - exclude: - - \ No newline at end of file diff --git a/salt/curator/files/action/so-import-delete.yml b/salt/curator/files/action/so-import-delete.yml deleted file mode 100644 index 36e213b26f..0000000000 --- a/salt/curator/files/action/so-import-delete.yml +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - -{%- set DELETE_DAYS = CURATORMERGED['so-import'].delete %} -actions: - 1: - action: delete_indices - description: >- - Delete import indices when older than {{ DELETE_DAYS }} days. - options: - ignore_empty_list: True - disable_action: False - filters: - - filtertype: pattern - kind: regex - value: '^(logstash-import.*|so-import.*)$' - - filtertype: age - source: name - direction: older - timestring: '%Y.%m.%d' - unit: days - unit_count: {{ DELETE_DAYS }} - exclude: - - \ No newline at end of file diff --git a/salt/curator/files/action/so-kibana-delete.yml b/salt/curator/files/action/so-kibana-delete.yml deleted file mode 100644 index 971a178fef..0000000000 --- a/salt/curator/files/action/so-kibana-delete.yml +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - -{%- set DELETE_DAYS = CURATORMERGED['so-kibana'].delete %} -actions: - 1: - action: delete_indices - description: >- - Delete kibana indices when older than {{ DELETE_DAYS }} days. - options: - ignore_empty_list: True - disable_action: False - filters: - - filtertype: pattern - kind: regex - value: '^(logstash-kibana.*|so-kibana.*)$' - - filtertype: age - source: name - direction: older - timestring: '%Y.%m.%d' - unit: days - unit_count: {{ DELETE_DAYS }} - exclude: - - \ No newline at end of file diff --git a/salt/curator/files/action/so-kratos-delete.yml b/salt/curator/files/action/so-kratos-delete.yml deleted file mode 100644 index d7cb2c4adf..0000000000 --- a/salt/curator/files/action/so-kratos-delete.yml +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - -{%- set DELETE_DAYS = CURATORMERGED['so-kratos'].delete %} -actions: - 1: - action: delete_indices - description: >- - Delete kratos indices when older than {{ DELETE_DAYS }} days. - options: - ignore_empty_list: True - disable_action: False - filters: - - filtertype: pattern - kind: regex - value: '^(logstash-kratos.*|so-kratos.*)$' - - filtertype: age - source: name - direction: older - timestring: '%Y.%m.%d' - unit: days - unit_count: {{ DELETE_DAYS }} - exclude: - - \ No newline at end of file diff --git a/salt/curator/files/action/so-logstash-delete.yml b/salt/curator/files/action/so-logstash-delete.yml deleted file mode 100644 index 1ca1a6f6c6..0000000000 --- a/salt/curator/files/action/so-logstash-delete.yml +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - -{%- set DELETE_DAYS = CURATORMERGED['so-logstash'].delete %} -actions: - 1: - action: delete_indices - description: >- - Delete logstash indices when older than {{ DELETE_DAYS }} days. - options: - ignore_empty_list: True - disable_action: False - filters: - - filtertype: pattern - kind: regex - value: '^(logstash-logstash.*|so-logstash.*)$' - - filtertype: age - source: name - direction: older - timestring: '%Y.%m.%d' - unit: days - unit_count: {{ DELETE_DAYS }} - exclude: - - \ No newline at end of file diff --git a/salt/curator/files/action/so-netflow-delete.yml b/salt/curator/files/action/so-netflow-delete.yml deleted file mode 100644 index 63adaa393e..0000000000 --- a/salt/curator/files/action/so-netflow-delete.yml +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - -{%- set DELETE_DAYS = CURATORMERGED['so-netflow'].delete %} -actions: - 1: - action: delete_indices - description: >- - Delete netflow indices when older than {{ DELETE_DAYS }} days. - options: - ignore_empty_list: True - disable_action: False - filters: - - filtertype: pattern - kind: regex - value: '^(logstash-netflow.*|so-netflow.*)$' - - filtertype: age - source: name - direction: older - timestring: '%Y.%m.%d' - unit: days - unit_count: {{ DELETE_DAYS }} - exclude: - - \ No newline at end of file diff --git a/salt/curator/files/action/so-osquery-delete.yml b/salt/curator/files/action/so-osquery-delete.yml deleted file mode 100644 index b6263b0e84..0000000000 --- a/salt/curator/files/action/so-osquery-delete.yml +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - -{%- set DELETE_DAYS = CURATORMERGED['so-osquery'].delete %} -actions: - 1: - action: delete_indices - description: >- - Delete import indices when older than {{ DELETE_DAYS }} days. - options: - ignore_empty_list: True - disable_action: False - filters: - - filtertype: pattern - kind: regex - value: '^(logstash-osquery.*|so-osquery.*)$' - - filtertype: age - source: name - direction: older - timestring: '%Y.%m.%d' - unit: days - unit_count: {{ DELETE_DAYS }} - exclude: - - diff --git a/salt/curator/files/action/so-ossec-delete.yml b/salt/curator/files/action/so-ossec-delete.yml deleted file mode 100644 index e24fe38191..0000000000 --- a/salt/curator/files/action/so-ossec-delete.yml +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - -{%- set DELETE_DAYS = CURATORMERGED['so-ossec'].delete %} -actions: - 1: - action: delete_indices - description: >- - Delete ossec indices when older than {{ DELETE_DAYS }} days. - options: - ignore_empty_list: True - disable_action: False - filters: - - filtertype: pattern - kind: regex - value: '^(logstash-ossec.*|so-ossec.*)$' - - filtertype: age - source: name - direction: older - timestring: '%Y.%m.%d' - unit: days - unit_count: {{ DELETE_DAYS }} - exclude: - - \ No newline at end of file diff --git a/salt/curator/files/action/so-redis-delete.yml b/salt/curator/files/action/so-redis-delete.yml deleted file mode 100644 index 1c7f95ded4..0000000000 --- a/salt/curator/files/action/so-redis-delete.yml +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - -{%- set DELETE_DAYS = CURATORMERGED['so-redis'].delete %} -actions: - 1: - action: delete_indices - description: >- - Delete redis indices when older than {{ DELETE_DAYS }} days. - options: - ignore_empty_list: True - disable_action: False - filters: - - filtertype: pattern - kind: regex - value: '^(logstash-redis.*|so-redis.*)$' - - filtertype: age - source: name - direction: older - timestring: '%Y.%m.%d' - unit: days - unit_count: {{ DELETE_DAYS }} - exclude: - - \ No newline at end of file diff --git a/salt/curator/files/action/so-strelka-delete.yml b/salt/curator/files/action/so-strelka-delete.yml deleted file mode 100644 index 90cf88e46f..0000000000 --- a/salt/curator/files/action/so-strelka-delete.yml +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - -{%- set DELETE_DAYS = CURATORMERGED['so-strelka'].delete %} -actions: - 1: - action: delete_indices - description: >- - Delete Strelka indices when older than {{ DELETE_DAYS }} days. - options: - ignore_empty_list: True - disable_action: False - filters: - - filtertype: pattern - kind: regex - value: '^(logstash-strelka.*|so-strelka.*)$' - - filtertype: age - source: name - direction: older - timestring: '%Y.%m.%d' - unit: days - unit_count: {{ DELETE_DAYS }} - exclude: - - \ No newline at end of file diff --git a/salt/curator/files/action/so-syslog-delete.yml b/salt/curator/files/action/so-syslog-delete.yml deleted file mode 100644 index c11d2ef5aa..0000000000 --- a/salt/curator/files/action/so-syslog-delete.yml +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - -{%- set DELETE_DAYS = CURATORMERGED['so-syslog'].delete %} -actions: - 1: - action: delete_indices - description: >- - Delete syslog indices when older than {{ DELETE_DAYS }} days. - options: - ignore_empty_list: True - disable_action: False - filters: - - filtertype: pattern - kind: regex - value: '^(logstash-syslog.*|so-syslog.*)$' - - filtertype: age - source: name - direction: older - timestring: '%Y.%m.%d' - unit: days - unit_count: {{ DELETE_DAYS }} - exclude: - - \ No newline at end of file diff --git a/salt/curator/files/action/so-zeek-delete.yml b/salt/curator/files/action/so-zeek-delete.yml deleted file mode 100644 index 1f85226961..0000000000 --- a/salt/curator/files/action/so-zeek-delete.yml +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - -{%- set DELETE_DAYS = CURATORMERGED['so-zeek'].delete %} -actions: - 1: - action: delete_indices - description: >- - Delete Zeek indices when older than {{ DELETE_DAYS }} days. - options: - ignore_empty_list: True - disable_action: False - filters: - - filtertype: pattern - kind: regex - value: '^(logstash-zeek.*|so-zeek.*)$' - - filtertype: age - source: name - direction: older - timestring: '%Y.%m.%d' - unit: days - unit_count: {{ DELETE_DAYS }} - exclude: - - \ No newline at end of file diff --git a/salt/curator/files/curator.yml b/salt/curator/files/curator.yml deleted file mode 100644 index 4ea1dddf70..0000000000 --- a/salt/curator/files/curator.yml +++ /dev/null @@ -1,40 +0,0 @@ -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - -{% from 'vars/globals.map.jinja' import GLOBALS %} -{% if GLOBALS.role in ['so-searchnode', 'so-heavynode'] %} - {%- set elasticsearch = GLOBALS.node_ip -%} -{% elif GLOBALS.role in ['so-eval', 'so-managersearch', 'so-standalone', 'so-manager'] %} - {%- set elasticsearch = GLOBALS.manager_ip -%} -{%- endif %} -{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') %} -{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') %} - ---- -# Remember, leave a key empty if there is no value. None will be a string, -# not a Python "NoneType" -elasticsearch: - client: - hosts: - - https://{{elasticsearch}}:9200 - cloud_id: - ca_certs: - client_cert: - client_key: - verify_certs: False - request_timeout: 30 - other_settings: - api_key: - id: - api_key: - master_only: False - username: "{{ ES_USER }}" - password: "{{ ES_PASS }}" - -logging: - loglevel: INFO - logfile: '/var/log/curator/curator.log' - logformat: default - blacklist: ['elasticsearch', 'urllib3'] diff --git a/salt/curator/init.sls b/salt/curator/init.sls deleted file mode 100644 index 201195b609..0000000000 --- a/salt/curator/init.sls +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - -{% from 'curator/map.jinja' import CURATORMERGED %} - -include: -{% if CURATORMERGED.enabled %} - - curator.enabled -{% else %} - - curator.disabled -{% endif %} diff --git a/salt/curator/map.jinja b/salt/curator/map.jinja deleted file mode 100644 index 517209635b..0000000000 --- a/salt/curator/map.jinja +++ /dev/null @@ -1,7 +0,0 @@ -{# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one - or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at - https://securityonion.net/license; you may not use this file except in compliance with the - Elastic License 2.0. #} - -{% import_yaml 'curator/defaults.yaml' as CURATORDEFAULTS %} -{% set CURATORMERGED = salt['pillar.get']('curator', CURATORDEFAULTS.curator, merge=true) %} diff --git a/salt/curator/soc_curator.yaml b/salt/curator/soc_curator.yaml deleted file mode 100644 index acfba7f85d..0000000000 --- a/salt/curator/soc_curator.yaml +++ /dev/null @@ -1,39 +0,0 @@ -curator: - enabled: - description: You can enable or disable Curator. - helpLink: curator.html - elasticsearch: - index_settings: - so-beats: - delete: &delete - description: Age, in days, when Curator deletes the index. - helpLink: curator.html - forcedType: int - so-elasticsearch: - delete: *delete - so-firewall: - delete: *delete - so-ids: - delete: *delete - so-import: - delete: *delete - so-kratos: - delete: *delete - so-kibana: - delete: *delete - so-logstash: - delete: *delete - so-netflow: - delete: *delete - so-osquery: - delete: *delete - so-ossec: - delete: *delete - so-redis: - delete: *delete - so-strelka: - delete: *delete - so-syslog: - delete: *delete - so-zeek: - delete: *delete diff --git a/salt/curator/sostatus.sls b/salt/curator/sostatus.sls deleted file mode 100644 index de6459a6d7..0000000000 --- a/salt/curator/sostatus.sls +++ /dev/null @@ -1,21 +0,0 @@ -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - -{% from 'allowed_states.map.jinja' import allowed_states %} -{% if sls.split('.')[0] in allowed_states %} - -append_so-curator_so-status.conf: - file.append: - - name: /opt/so/conf/so-status/so-status.conf - - text: so-curator - - unless: grep -q so-curator /opt/so/conf/so-status/so-status.conf - -{% else %} - -{{sls}}_state_not_allowed: - test.fail_without_changes: - - name: {{sls}}_state_not_allowed - -{% endif %} diff --git a/salt/curator/tools/sbin/so-curator-delete b/salt/curator/tools/sbin/so-curator-delete deleted file mode 100644 index 2d128bfdfb..0000000000 --- a/salt/curator/tools/sbin/so-curator-delete +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/bash -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - - -APP=delete -lf=/tmp/$APP-pidLockFile -# create empty lock file if none exists -cat /dev/null >> $lf -read lastPID < $lf -# if lastPID is not null and a process with that pid exists , exit -[ ! -z "$lastPID" -a -d /proc/$lastPID ] && exit -echo $$ > $lf - -docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/delete.yml > /dev/null 2>&1 diff --git a/salt/curator/tools/sbin/so-curator-restart b/salt/curator/tools/sbin/so-curator-restart deleted file mode 100644 index f57e7b22ee..0000000000 --- a/salt/curator/tools/sbin/so-curator-restart +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash - -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - - - -. /usr/sbin/so-common - -/usr/sbin/so-restart curator $1 diff --git a/salt/curator/tools/sbin/so-curator-start b/salt/curator/tools/sbin/so-curator-start deleted file mode 100644 index c5f0fc4d1f..0000000000 --- a/salt/curator/tools/sbin/so-curator-start +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash - -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - - - -. /usr/sbin/so-common - -/usr/sbin/so-start curator $1 diff --git a/salt/curator/tools/sbin/so-curator-stop b/salt/curator/tools/sbin/so-curator-stop deleted file mode 100644 index 30fb07e4bd..0000000000 --- a/salt/curator/tools/sbin/so-curator-stop +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash - -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - - - -. /usr/sbin/so-common - -/usr/sbin/so-stop curator $1 diff --git a/salt/docker/defaults.yaml b/salt/docker/defaults.yaml index a5d6c5d6d5..9a27843ae1 100644 --- a/salt/docker/defaults.yaml +++ b/salt/docker/defaults.yaml @@ -159,11 +159,6 @@ docker: custom_bind_mounts: [] extra_hosts: [] extra_env: [] - 'so-curator': - final_octet: 43 - custom_bind_mounts: [] - extra_hosts: [] - extra_env: [] 'so-elastic-fleet-package-registry': final_octet: 44 port_bindings: diff --git a/salt/docker/soc_docker.yaml b/salt/docker/soc_docker.yaml index d227a3e85b..850324a9e8 100644 --- a/salt/docker/soc_docker.yaml +++ b/salt/docker/soc_docker.yaml @@ -60,7 +60,6 @@ docker: so-strelka-gatekeeper: *dockerOptions so-strelka-coordinator: *dockerOptions so-elastalert: *dockerOptions - so-curator: *dockerOptions so-elastic-fleet-package-registry: *dockerOptions so-idh: *dockerOptions so-elastic-agent: *dockerOptions diff --git a/salt/elasticsearch/disabled.sls b/salt/elasticsearch/disabled.sls index 210ad59a4e..2453f9f773 100644 --- a/salt/elasticsearch/disabled.sls +++ b/salt/elasticsearch/disabled.sls @@ -18,6 +18,10 @@ so-elasticsearch_so-status.disabled: - name: /opt/so/conf/so-status/so-status.conf - regex: ^so-elasticsearch$ +so-elasticsearch-indices-delete: + cron.absent: + - identifier: so-elasticsearch-indices-delete + {% else %} {{sls}}_state_not_allowed: diff --git a/salt/elasticsearch/enabled.sls b/salt/elasticsearch/enabled.sls index f7ab7749f0..349fe64c43 100644 --- a/salt/elasticsearch/enabled.sls +++ b/salt/elasticsearch/enabled.sls @@ -195,6 +195,26 @@ so-elasticsearch-roles-load: - require: - docker_container: so-elasticsearch - file: elasticsearch_sbin_jinja +{% if grains.role in ['so-eval', 'so-standalone', 'so-managersearch', 'so-heavynode', 'so-manager'] %} +so-curator-cluster-close: + cron.absent: + - identifier: so-curator-cluster-close + +so-curator-cluster-delete: + cron.absent: + - identifier: so-curator-cluster-delete + +so-elasticsearch-indices-delete: + cron.present: + - name: /usr/sbin/so-elasticsearch-indices-delete > /opt/so/log/curator/cron-elasticsearch-indices-delete.log 2>&1 + - identifier: so-elasticsearch-indices-delete + - user: root + - minute: '*/5' + - hour: '*' + - daymonth: '*' + - month: '*' + - dayweek: '*' +{% endif %} {% endif %} {% else %} diff --git a/salt/curator/tools/sbin/so-curator-cluster-delete b/salt/elasticsearch/tools/sbin/so-elasticsearch-indices-delete similarity index 92% rename from salt/curator/tools/sbin/so-curator-cluster-delete rename to salt/elasticsearch/tools/sbin/so-elasticsearch-indices-delete index 0f7945b78a..036ff844fc 100755 --- a/salt/curator/tools/sbin/so-curator-cluster-delete +++ b/salt/elasticsearch/tools/sbin/so-elasticsearch-indices-delete @@ -14,4 +14,4 @@ read lastPID < $lf [ ! -z "$lastPID" -a -d /proc/$lastPID ] && exit echo $$ > $lf -/usr/sbin/so-curator-cluster-delete-delete +/usr/sbin/so-elasticsearch-indices-delete-delete diff --git a/salt/curator/tools/sbin_jinja/so-curator-cluster-delete-delete b/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-indices-delete-delete similarity index 96% rename from salt/curator/tools/sbin_jinja/so-curator-cluster-delete-delete rename to salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-indices-delete-delete index e0c5144bca..292265095d 100755 --- a/salt/curator/tools/sbin_jinja/so-curator-cluster-delete-delete +++ b/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-indices-delete-delete @@ -9,8 +9,8 @@ {%- set ELASTICSEARCH_HOST = GLOBALS.node_ip -%} {%- set RETENTION = salt['pillar.get']('elasticsearch:retention', ELASTICDEFAULTS.elasticsearch.retention, merge=true) -%} -LOG="/opt/so/log/curator/so-curator-cluster-delete.log" -ALERT_LOG="/opt/so/log/curator/alert.log" +LOG="/opt/so/log/elasticsearch/so-elasticsearch-indices-delete.log" +ALERT_LOG="/opt/so/log/elasticsearch/indices-delete-alert.log" LOG_SIZE_LIMIT_GB=$(/usr/sbin/so-elasticsearch-cluster-space-total {{ RETENTION.retention_pct}}) LOG_SIZE_LIMIT=$(( "$LOG_SIZE_LIMIT_GB" * 1000 * 1000 * 1000 )) ITERATION=0 diff --git a/salt/firewall/containers.map.jinja b/salt/firewall/containers.map.jinja index 02e8a46446..0ba2389e91 100644 --- a/salt/firewall/containers.map.jinja +++ b/salt/firewall/containers.map.jinja @@ -2,7 +2,6 @@ {% if GLOBALS.role == 'so-eval' %} {% set NODE_CONTAINERS = [ - 'so-curator', 'so-dockerregistry', 'so-elasticsearch', 'so-elastic-fleet', @@ -25,7 +24,6 @@ {% elif GLOBALS.role == 'so-manager' or GLOBALS.role == 'so-standalone' or GLOBALS.role == 'so-managersearch' %} {% set NODE_CONTAINERS = [ - 'so-curator', 'so-dockerregistry', 'so-elasticsearch', 'so-elastic-fleet', @@ -56,7 +54,6 @@ {% elif GLOBALS.role == 'so-heavynode' %} {% set NODE_CONTAINERS = [ - 'so-curator', 'so-elasticsearch', 'so-elastic-agent', 'so-logstash', diff --git a/salt/logrotate/defaults.yaml b/salt/logrotate/defaults.yaml index 4d6a688e45..7333c78e98 100644 --- a/salt/logrotate/defaults.yaml +++ b/salt/logrotate/defaults.yaml @@ -80,7 +80,7 @@ logrotate: - extension .log - dateext - dateyesterday - /opt/so/log/curator/*_x_log: + /opt/so/log/elasticsearch/*indices-delete*_x_log: - daily - rotate 14 - missingok diff --git a/salt/logrotate/soc_logrotate.yaml b/salt/logrotate/soc_logrotate.yaml index 5e6c78fcc3..62aa935c96 100644 --- a/salt/logrotate/soc_logrotate.yaml +++ b/salt/logrotate/soc_logrotate.yaml @@ -56,23 +56,23 @@ logrotate: multiline: True global: True forcedType: "[]string" - "/opt/so/log/curator/*_x_log": + "/opt/so/log/elasticfleet/*_x_log": description: List of logrotate options for this file. - title: /opt/so/log/curator/*.log + title: /opt/so/log/elasticfleet/*.log advanced: True multiline: True global: True forcedType: "[]string" - "/opt/so/log/elasticfleet/*_x_log": + "/opt/so/log/elasticfleet/*_x_ndjson": description: List of logrotate options for this file. - title: /opt/so/log/elasticfleet/*.log + title: /opt/so/log/elasticfleet/*.ndjson advanced: True multiline: True global: True forcedType: "[]string" - "/opt/so/log/elasticfleet/*_x_ndjson": + "/opt/so/log/elasticsearch/*indices-delete*_x_log": description: List of logrotate options for this file. - title: /opt/so/log/elasticfleet/*.ndjson + title: /opt/so/log/elasticsearch/*indices-delete.log advanced: True multiline: True global: True diff --git a/salt/manager/tools/sbin/so-elastic-auth-password-reset b/salt/manager/tools/sbin/so-elastic-auth-password-reset index 5bc5230c3e..b6bfd87213 100755 --- a/salt/manager/tools/sbin/so-elastic-auth-password-reset +++ b/salt/manager/tools/sbin/so-elastic-auth-password-reset @@ -97,8 +97,6 @@ function soUserSync() { salt -C 'G@role:so-standalone or G@role:so-eval or G@role:so-import or G@role:so-manager or G@role:so-managersearch or G@role:so-searchnode or G@role:so-heavynode' state.apply logstash queue=True printf "\nApplying kibana state to the appropriate nodes.\n\n" salt -C 'G@role:so-standalone or G@role:so-eval or G@role:so-import or G@role:so-manager or G@role:so-managersearch' state.apply kibana queue=True - printf "\nApplying curator state to the appropriate nodes.\n\n" - salt -C 'G@role:so-standalone or G@role:so-eval or G@role:so-import or G@role:so-manager or G@role:so-managersearch or G@role:so-searchnode or G@role:so-heavynode' state.apply curator queue=True } function highstateManager() { diff --git a/salt/manager/tools/sbin/so-minion b/salt/manager/tools/sbin/so-minion index ada56dc2a0..d5225cc827 100755 --- a/salt/manager/tools/sbin/so-minion +++ b/salt/manager/tools/sbin/so-minion @@ -302,13 +302,6 @@ function add_strelka_to_minion() { " " >> $PILLARFILE } -function add_curator_to_minion() { - printf '%s\n'\ - "curator:"\ - " enabled: True"\ - " " >> $PILLARFILE -} - function add_telegraf_to_minion() { printf '%s\n'\ "telegraf:"\ @@ -435,7 +428,6 @@ function createEVAL() { add_playbook_to_minion add_elastalert_to_minion add_kibana_to_minion - add_curator_to_minion add_telegraf_to_minion add_influxdb_to_minion add_nginx_to_minion @@ -458,7 +450,6 @@ function createSTANDALONE() { add_elastalert_to_minion add_kibana_to_minion add_redis_to_minion - add_curator_to_minion add_telegraf_to_minion add_influxdb_to_minion add_nginx_to_minion @@ -478,7 +469,6 @@ function createMANAGER() { add_elastalert_to_minion add_kibana_to_minion add_redis_to_minion - add_curator_to_minion add_telegraf_to_minion add_influxdb_to_minion add_nginx_to_minion @@ -498,7 +488,6 @@ function createMANAGERSEARCH() { add_elastalert_to_minion add_kibana_to_minion add_redis_to_minion - add_curator_to_minion add_telegraf_to_minion add_influxdb_to_minion add_nginx_to_minion @@ -548,7 +537,6 @@ function createHEAVYNODE() { add_sensor_to_minion add_strelka_to_minion add_redis_to_minion - add_curator_to_minion add_telegraf_to_minion } @@ -679,4 +667,4 @@ case "$OPERATION" in *) usage ;; -esac \ No newline at end of file +esac diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 35c9347729..2485900c5c 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -465,9 +465,8 @@ post_to_2.4.30() { } post_to_2.4.40() { - echo "Removing Curator close and delete files" - rm -f /opt/so/conf/curator/action/*-close.y*ml - rm -f /opt/so/conf/curator/action/logs*-delete.y*ml + echo "Removing Curator configuration" + rm -rf /opt/so/conf/curator/ } repo_sync() { diff --git a/salt/top.sls b/salt/top.sls index 4f84e17ac7..5e7d13532b 100644 --- a/salt/top.sls +++ b/salt/top.sls @@ -74,7 +74,6 @@ base: - suricata - zeek - strelka - - curator - elastalert - utility - soctopus @@ -104,7 +103,6 @@ base: - redis - elastic-fleet-package-registry - kibana - - curator - elastalert - utility - soctopus @@ -140,7 +138,6 @@ base: - suricata - zeek - strelka - - curator - elastalert - utility - soctopus @@ -179,7 +176,6 @@ base: - elasticsearch - logstash - redis - - curator - elastic-fleet-package-registry - kibana - elastalert @@ -199,7 +195,6 @@ base: - elasticsearch - logstash - redis - - curator - strelka - pcap - suricata diff --git a/setup/so-functions b/setup/so-functions index 6ad0947d16..e1faa275e6 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1412,7 +1412,7 @@ make_some_dirs() { mkdir -p $local_salt_dir/salt/firewall/portgroups mkdir -p $local_salt_dir/salt/firewall/ports - for THEDIR in bpf pcap elasticsearch ntp firewall redis backup influxdb strelka sensoroni curator soc soctopus docker zeek suricata nginx telegraf logstash soc manager kratos idstools idh elastalert global;do + for THEDIR in bpf pcap elasticsearch ntp firewall redis backup influxdb strelka sensoroni soc soctopus docker zeek suricata nginx telegraf logstash soc manager kratos idstools idh elastalert global;do mkdir -p $local_salt_dir/pillar/$THEDIR touch $local_salt_dir/pillar/$THEDIR/adv_$THEDIR.sls touch $local_salt_dir/pillar/$THEDIR/soc_$THEDIR.sls @@ -1555,7 +1555,6 @@ reserve_group_ids() { logCmd "groupadd -g 931 logstash" logCmd "groupadd -g 932 kibana" logCmd "groupadd -g 933 elastalert" - logCmd "groupadd -g 934 curator" logCmd "groupadd -g 937 zeek" logCmd "groupadd -g 940 suricata" logCmd "groupadd -g 941 stenographer" diff --git a/setup/so-variables b/setup/so-variables index 7f65224877..4d3cbb7390 100644 --- a/setup/so-variables +++ b/setup/so-variables @@ -1,4 +1,4 @@ -#!/bin/bash +a!/bin/bash SOVERSION=$(cat ../VERSION) mkdir -p /nsm @@ -112,12 +112,6 @@ export sensoroni_pillar_file adv_sensoroni_pillar_file="$local_salt_dir/pillar/sensoroni/adv_sensoroni.sls" export adv_sensoroni_pillar_file -curator_pillar_file="$local_salt_dir/pillar/curator/soc_curator.sls" -export curator_pillar_file - -adv_curator_pillar_file="$local_salt_dir/pillar/curator/adv_curator.sls" -export adv_curator_pillar_file - soctopus_pillar_file="$local_salt_dir/pillar/soctopus/soc_soctopus.sls" export soctopus_pillar_file From f7373ed79c241ea62bf16a070daaa18eb202713f Mon Sep 17 00:00:00 2001 From: Wes Date: Mon, 11 Dec 2023 19:20:52 +0000 Subject: [PATCH 2/5] Stop Curator, remove scripts and status --- salt/manager/tools/sbin/soup | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 2485900c5c..4a0adbc989 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -465,8 +465,12 @@ post_to_2.4.30() { } post_to_2.4.40() { + echo "Stopping Curator" + so-curator-stop echo "Removing Curator configuration" rm -rf /opt/so/conf/curator/ + rm -f /usr/sbin/so-curator-* + sed -i '/so-curator/d' /opt/so/conf/so-status/so-status.conf } repo_sync() { From b1721b646749d4b390ed5018e5e2dbfcb9586022 Mon Sep 17 00:00:00 2001 From: Wes Date: Mon, 11 Dec 2023 21:43:25 +0000 Subject: [PATCH 3/5] Fix directory --- salt/elasticsearch/enabled.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/elasticsearch/enabled.sls b/salt/elasticsearch/enabled.sls index 349fe64c43..6fe4635a30 100644 --- a/salt/elasticsearch/enabled.sls +++ b/salt/elasticsearch/enabled.sls @@ -206,7 +206,7 @@ so-curator-cluster-delete: so-elasticsearch-indices-delete: cron.present: - - name: /usr/sbin/so-elasticsearch-indices-delete > /opt/so/log/curator/cron-elasticsearch-indices-delete.log 2>&1 + - name: /usr/sbin/so-elasticsearch-indices-delete > /opt/so/log/elasticsearch/cron-elasticsearch-indices-delete.log 2>&1 - identifier: so-elasticsearch-indices-delete - user: root - minute: '*/5' From 54c3167b104f2c8d2694b370437b6665055ebe2f Mon Sep 17 00:00:00 2001 From: Wes Date: Tue, 12 Dec 2023 05:25:50 +0000 Subject: [PATCH 4/5] Delete data streams when necessary --- .../tools/sbin_jinja/so-elasticsearch-indices-delete-delete | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-indices-delete-delete b/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-indices-delete-delete index 292265095d..07feb36bdf 100755 --- a/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-indices-delete-delete +++ b/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-indices-delete-delete @@ -51,8 +51,11 @@ while overlimit && [[ $ITERATION -lt $MAX_ITERATIONS ]]; do if [ "${INDEX}" != "${CURRENT_WRITE_INDEX}" ]; then # This should not be a write index, so we should be allowed to delete it printf "\n$(date) - Used disk space exceeds LOG_SIZE_LIMIT (${LOG_SIZE_LIMIT_GB} GB) - Deleting ${INDEX} index...\n" >> ${LOG} - /usr/sbin/so-elasticsearch-query ${INDEX} -XDELETE >> ${LOG} 2>&1 + /usr/sbin/so-elasticsearch-query ${INDEX} -XDELETE >> ${LOG} 2>&1 fi + else + printf "\n$(date) - Used disk space exceeds LOG_SIZE_LIMIT (${LOG_SIZE_LIMIT_GB} GB) - There is only one backing index (${INDEX}). Deleting ${DATASTREAM} data stream...\n" >> ${LOG} + /usr/sbin/so-elasticsearch-query _data_stream/$DATASTREAM -XDELETE >> ${LOG} 2>&1 fi if ! overlimit ; then exit From bbe091fa1433b05a5cc744232e5281a4e80bec24 Mon Sep 17 00:00:00 2001 From: Wes Date: Tue, 12 Dec 2023 15:08:47 +0000 Subject: [PATCH 5/5] Fix accidental change --- setup/so-variables | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/so-variables b/setup/so-variables index 4d3cbb7390..511dfc43bd 100644 --- a/setup/so-variables +++ b/setup/so-variables @@ -1,4 +1,4 @@ -a!/bin/bash +#!/bin/bash SOVERSION=$(cat ../VERSION) mkdir -p /nsm