From dfe916d7c8a996c5070fc89fb1ade0d957480bfd Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 9 Aug 2023 15:19:17 -0400 Subject: [PATCH 1/3] add annotation for so-logs index --- salt/elasticsearch/soc_elasticsearch.yaml | 107 ++++++++++++++++------ 1 file changed, 77 insertions(+), 30 deletions(-) diff --git a/salt/elasticsearch/soc_elasticsearch.yaml b/salt/elasticsearch/soc_elasticsearch.yaml index da22268f69..2228eccf63 100644 --- a/salt/elasticsearch/soc_elasticsearch.yaml +++ b/salt/elasticsearch/soc_elasticsearch.yaml @@ -46,28 +46,26 @@ elasticsearch: description: Max number of boolean clauses per query. global: True helpLink: elasticsearch.html - index_settings: - so-elasticsearch: &indexSettings - warm: - description: Age (in days) of this index before it will move to warm storage, if warm nodes are present. Once moved, events on this index can take longer to fetch. - global: True - helpLink: elasticsearch.html - close: - description: Age (in days) of this index before it will be closed. Once closed, events on this index cannot be retrieved without first re-opening the index. - global: True - helpLink: elasticsearch.html - delete: - description: Age (in days) of this index before it will be deleted. Once deleted, events are permanently unrecoverable. - global: True - helpLink: elasticsearch.html + index_settings: + so-logs: &indexSettings index_sorting: description: Sorts the index by event time, at the cost of additional processing resource consumption. global: True helpLink: elasticsearch.html index_template: + index_patterns: + description: Patterns for matching multiple indices or tables. + forceType: "[]string" + multiline: True + global: True + helpLink: elasticsearch.html template: settings: index: + number_of_replicas: + description: Number of replicas required for this index. Multiple replicas protects against data loss, but also increases storage costs. + global: True + helpLink: elasticsearch.html mapping: total_fields: limit: @@ -75,17 +73,59 @@ elasticsearch: global: True helpLink: elasticsearch.html refresh_interval: - description: Seconds between index refreshes. Shorter intervals can cause query performance to suffer since this is a synchronous and resource-intensive operation. + description: Seconds between index refreshes. Shorter intervals can cause query performance to suffer since this is a synchronous and resource-intensive operation. + global: True + helpLink: elasticsearch.html + number_of_shards: + description: Number of shards required for this index. Using multiple shards increases fault tolerance, but also increases storage and network costs. + global: True + helpLink: elasticsearch.html + sort: + field: + description: The field to sort by. Must set index_sorting to True. global: True helpLink: elasticsearch.html - number_of_shards: - description: Number of shards required for this index. Using multiple shards increases fault tolerance, but also increases storage and network costs. + order: + description: The order to sort by. Must set index_sorting to True. global: True helpLink: elasticsearch.html - number_of_replicas: - description: Number of replicas required for this index. Multiple replicas protects against data loss, but also increases storage costs. + mappings: + _meta: + package: + name: + description: Meta settings for the mapping. + global: True + helpLink: elasticsearch.html + managed_by: + description: Meta settings for the mapping. + global: True + helpLink: elasticsearch.html + managed: + description: Meta settings for the mapping. + forcedType: bool global: True helpLink: elasticsearch.html + composed_of: + description: The index template is composed of these component templates. + forcedType: "[]string" + global: True + helpLink: elasticsearch.html + priority: + description: The priority of the index template. + forcedType: int + global: True + helpLink: elasticsearch.html + data_stream: + hidden: + description: Hide the data stream. + forcedType: bool + global: True + helpLink: elasticsearch.html + allow_custom_routing: + description: Allow custom routing for the data stream. + forcedType: bool + global: True + helpLink: elasticsearch.html policy: phases: hot: @@ -97,6 +137,7 @@ elasticsearch: set_priority: priority: description: Priority of index. This is used for recovery after a node restart. Indices with higher priorities are recovered before indices with lower priorities. + forcedType: int global: True helpLink: elasticsearch.html rollover: @@ -117,20 +158,26 @@ elasticsearch: set_priority: priority: description: Used for index recovery after a node restart. Indices with higher priorities are recovered before indices with lower priorities. + forcedType: int global: True helpLink: elasticsearch.html delete: min_age: description: Minimum age of index. This determines when the index should be deleted. global: True - helpLink: elastic - so-endgame: *indexSettings - so-firewall: *indexSettings - so-import: *indexSettings - so-kibana: *indexSettings - so-logstash: *indexSettings - so-osquery: *indexSettings - so-redis: *indexSettings - so-strelka: *indexSettings - so-syslog: *indexSettings - so-zeek: *indexSettings + helpLink: elasticsearch.html + _meta: + package: + name: + description: Meta settings for the mapping. + global: True + helpLink: elasticsearch.html + managed_by: + description: Meta settings for the mapping. + global: True + helpLink: elasticsearch.html + managed: + description: Meta settings for the mapping. + forcedType: bool + global: True + helpLink: elasticsearch.html From f9e272dd8f07613c748fee6540c88e4bea59b145 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 9 Aug 2023 16:09:23 -0400 Subject: [PATCH 2/3] add additional annotations for elasticsearch index settings --- salt/elasticsearch/soc_elasticsearch.yaml | 86 +++++++++++++++++++++++ 1 file changed, 86 insertions(+) diff --git a/salt/elasticsearch/soc_elasticsearch.yaml b/salt/elasticsearch/soc_elasticsearch.yaml index 2228eccf63..89d347b427 100644 --- a/salt/elasticsearch/soc_elasticsearch.yaml +++ b/salt/elasticsearch/soc_elasticsearch.yaml @@ -181,3 +181,89 @@ elasticsearch: forcedType: bool global: True helpLink: elasticsearch.html + so-logs-system.auth: *indexSettings + so-logs-system.syslog: *indexSettings + so-logs-system.system: *indexSettings + so-logs-system.application: *indexSettings + so-logs-system.security: *indexSettings + so-logs-windows.forwarded: *indexSettings + so-logs-windows.powershell: *indexSettings + so-logs-windows.powershell_operational: *indexSettings + so-logs-windows.sysmon_operational: *indexSettings + so-logs-aws.cloudtrail: *indexSettings + so-logs-aws.cloudwatch_logs: *indexSettings + so-logs-aws.ec2_logs: *indexSettings + so-logs-aws.elb_logs: *indexSettings + so-logs-aws.firewall_logs: *indexSettings + so-logs-aws.route53_public_logs: *indexSettings + so-logs-aws.route53_resolver_logs: *indexSettings + so-logs-aws.s3access: *indexSettings + so-logs-aws.vpcflow: *indexSettings + so-logs-aws.waf: *indexSettings + so-logs-azure.activitylogs: *indexSettings + so-logs-azure.application_gateway: *indexSettings + so-logs-azure.auditlogs: *indexSettings + so-logs-azure.eventhub: *indexSettings + so-logs-azure.firewall_logs: *indexSettings + so-logs-azure.identity_protection: *indexSettings + so-logs-azure.platformlogs: *indexSettings + so-logs-azure.provisioning: *indexSettings + so-logs-azure.signinlogs: *indexSettings + so-logs-azure.springcloudlogs: *indexSettings + so-logs-cloudflare.audit: *indexSettings + so-logs-cloudflare.logpull: *indexSettings + so-logs-fim.event: *indexSettings + so-logs-github.audit: *indexSettings + so-logs-github.code_scanning: *indexSettings + so-logs-github.dependabot: *indexSettings + so-logs-github.issues: *indexSettings + so-logs-github.secret_scanning: *indexSettings + so-logs-google_workspace.access_transparency: *indexSettings + so-logs-google_workspace.admin: *indexSettings + so-logs-google_workspace.alert: *indexSettings + so-logs-google_workspace.context_aware_access: *indexSettings + so-logs-google_workspace.device: *indexSettings + so-logs-google_workspace.drive: *indexSettings + so-logs-google_workspace.gcp: *indexSettings + so-logs-google_workspace.group_enterprise: *indexSettings + so-logs-google_workspace.groups: *indexSettings + so-logs-google_workspace.login: *indexSettings + so-logs-google_workspace.rules: *indexSettings + so-logs-google_workspace.saml: *indexSettings + so-logs-google_workspace.token: *indexSettings + so-logs-google_workspace.user_accounts: *indexSettings + so-logs-1password.item_usages: *indexSettings + so-logs-1password.signin_attempts: *indexSettings + so-logs-osquery-manager-actions: *indexSettings + so-logs-osquery-manager-action.responses: *indexSettings + so-logs-elastic_agent.apm_server: *indexSettings + so-logs-elastic_agent.auditbeat: *indexSettings + so-logs-elastic_agent.cloudbeat: *indexSettings + so-logs-elastic_agent.endpoint_security: *indexSettings + so-logs-endpoint.alerts: *indexSettings + so-logs-endpoint.events.api: *indexSettings + so-logs-endpoint.events.file: *indexSettings + so-logs-endpoint.events.library: *indexSettings + so-logs-endpoint.events.network: *indexSettings + so-logs-endpoint.events.process: *indexSettings + so-logs-endpoint.events.registry: *indexSettings + so-logs-endpoint.events.security: *indexSettings + so-logs-elastic_agent.filebeat: *indexSettings + so-logs-elastic_agent.fleet_server: *indexSettings + so-logs-elastic_agent.heartbeat: *indexSettings + so-logs-elastic_agent: *indexSettings + so-logs-elastic_agent.metricbeat: *indexSettings + so-logs-elastic_agent.osquerybeat: *indexSettings + so-logs-elastic_agent.packetbeat: *indexSettings + so-case: *indexSettings + so-common: *indexSettings + so-endgame: *indexSettings + so-idh: *indexSettings + so-suricata: *indexSettings + so-import: *indexSettings + so-kratos: *indexSettings + so-logstash: *indexSettings + so-redis: *indexSettings + so-strelka: *indexSettings + so-syslog: *indexSettings + so-zeek: *indexSettings From 4d497022dbcd9730b2e0903d93a388bc48c7c564 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 10 Aug 2023 09:52:18 -0400 Subject: [PATCH 3/3] replace . with _x_ for soc ui compat --- salt/elasticsearch/defaults.yaml | 144 +++++++++--------- salt/elasticsearch/soc_elasticsearch.yaml | 144 +++++++++--------- salt/elasticsearch/template.map.jinja | 6 +- .../so-elasticsearch-ilm-policy-load | 3 +- 4 files changed, 149 insertions(+), 148 deletions(-) diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml index 035079f542..5791970402 100644 --- a/salt/elasticsearch/defaults.yaml +++ b/salt/elasticsearch/defaults.yaml @@ -113,7 +113,7 @@ elasticsearch: name: elastic_agent managed_by: security_onion managed: true - so-logs-system.auth: + so-logs-system_x_auth: index_sorting: False index_template: index_patterns: @@ -132,7 +132,7 @@ elasticsearch: data_stream: hidden: false allow_custom_routing: false - so-logs-system.syslog: + so-logs-system_x_syslog: index_sorting: False index_template: index_patterns: @@ -151,7 +151,7 @@ elasticsearch: data_stream: hidden: false allow_custom_routing: false - so-logs-system.system: + so-logs-system_x_system: index_sorting: False index_template: index_patterns: @@ -170,7 +170,7 @@ elasticsearch: data_stream: hidden: false allow_custom_routing: false - so-logs-system.application: + so-logs-system_x_application: index_sorting: False index_template: index_patterns: @@ -189,7 +189,7 @@ elasticsearch: data_stream: hidden: false allow_custom_routing: false - so-logs-system.security: + so-logs-system_x_security: index_sorting: False index_template: index_patterns: @@ -208,7 +208,7 @@ elasticsearch: data_stream: hidden: false allow_custom_routing: false - so-logs-windows.forwarded: + so-logs-windows_x_forwarded: index_sorting: False index_template: index_patterns: @@ -226,7 +226,7 @@ elasticsearch: data_stream: hidden: false allow_custom_routing: false - so-logs-windows.powershell: + so-logs-windows_x_powershell: index_sorting: False index_template: index_patterns: @@ -244,7 +244,7 @@ elasticsearch: data_stream: hidden: false allow_custom_routing: false - so-logs-windows.powershell_operational: + so-logs-windows_x_powershell_operational: index_sorting: False index_template: index_patterns: @@ -262,7 +262,7 @@ elasticsearch: data_stream: hidden: false allow_custom_routing: false - so-logs-windows.sysmon_operational: + so-logs-windows_x_sysmon_operational: index_sorting: False index_template: index_patterns: @@ -280,7 +280,7 @@ elasticsearch: data_stream: hidden: false allow_custom_routing: false - so-logs-aws.cloudtrail: + so-logs-aws_x_cloudtrail: index_sorting: False index_template: index_patterns: @@ -298,7 +298,7 @@ elasticsearch: data_stream: hidden: false allow_custom_routing: false - so-logs-aws.cloudwatch_logs: + so-logs-aws_x_cloudwatch_logs: index_sorting: False index_template: index_patterns: @@ -316,7 +316,7 @@ elasticsearch: data_stream: hidden: false allow_custom_routing: false - so-logs-aws.ec2_logs: + so-logs-aws_x_ec2_logs: index_sorting: False index_template: index_patterns: @@ -334,7 +334,7 @@ elasticsearch: data_stream: hidden: false allow_custom_routing: false - so-logs-aws.elb_logs: + so-logs-aws_x_elb_logs: index_sorting: False index_template: index_patterns: @@ -352,7 +352,7 @@ elasticsearch: data_stream: hidden: false allow_custom_routing: false - so-logs-aws.firewall_logs: + so-logs-aws_x_firewall_logs: index_sorting: False index_template: index_patterns: @@ -370,7 +370,7 @@ elasticsearch: data_stream: hidden: false allow_custom_routing: false - so-logs-aws.route53_public_logs: + so-logs-aws_x_route53_public_logs: index_sorting: False index_template: index_patterns: @@ -388,7 +388,7 @@ elasticsearch: data_stream: hidden: false allow_custom_routing: false - so-logs-aws.route53_resolver_logs: + so-logs-aws_x_route53_resolver_logs: index_sorting: False index_template: index_patterns: @@ -406,7 +406,7 @@ elasticsearch: data_stream: hidden: false allow_custom_routing: false - so-logs-aws.s3access: + so-logs-aws_x_s3access: index_sorting: False index_template: index_patterns: @@ -424,7 +424,7 @@ elasticsearch: data_stream: hidden: false allow_custom_routing: false - so-logs-aws.vpcflow: + so-logs-aws_x_vpcflow: index_sorting: False index_template: index_patterns: @@ -442,7 +442,7 @@ elasticsearch: data_stream: hidden: false allow_custom_routing: false - so-logs-aws.waf: + so-logs-aws_x_waf: index_sorting: False index_template: index_patterns: @@ -460,7 +460,7 @@ elasticsearch: data_stream: hidden: false allow_custom_routing: false - so-logs-azure.activitylogs: + so-logs-azure_x_activitylogs: index_sorting: False index_template: index_patterns: @@ -478,7 +478,7 @@ elasticsearch: data_stream: hidden: false allow_custom_routing: false - so-logs-azure.application_gateway: + so-logs-azure_x_application_gateway: index_sorting: False index_template: index_patterns: @@ -496,7 +496,7 @@ elasticsearch: data_stream: hidden: false allow_custom_routing: false - so-logs-azure.auditlogs: + so-logs-azure_x_auditlogs: index_sorting: False index_template: index_patterns: @@ -514,7 +514,7 @@ elasticsearch: data_stream: hidden: false allow_custom_routing: false - so-logs-azure.eventhub: + so-logs-azure_x_eventhub: index_sorting: False index_template: index_patterns: @@ -532,7 +532,7 @@ elasticsearch: data_stream: hidden: false allow_custom_routing: false - so-logs-azure.firewall_logs: + so-logs-azure_x_firewall_logs: index_sorting: False index_template: index_patterns: @@ -550,7 +550,7 @@ elasticsearch: data_stream: hidden: false allow_custom_routing: false - so-logs-azure.identity_protection: + so-logs-azure_x_identity_protection: index_sorting: False index_template: index_patterns: @@ -568,7 +568,7 @@ elasticsearch: data_stream: hidden: false allow_custom_routing: false - so-logs-azure.platformlogs: + so-logs-azure_x_platformlogs: index_sorting: False index_template: index_patterns: @@ -586,7 +586,7 @@ elasticsearch: data_stream: hidden: false allow_custom_routing: false - so-logs-azure.provisioning: + so-logs-azure_x_provisioning: index_sorting: False index_template: index_patterns: @@ -604,7 +604,7 @@ elasticsearch: data_stream: hidden: false allow_custom_routing: false - so-logs-azure.signinlogs: + so-logs-azure_x_signinlogs: index_sorting: False index_template: index_patterns: @@ -622,7 +622,7 @@ elasticsearch: data_stream: hidden: false allow_custom_routing: false - so-logs-azure.springcloudlogs: + so-logs-azure_x_springcloudlogs: index_sorting: False index_template: index_patterns: @@ -640,7 +640,7 @@ elasticsearch: data_stream: hidden: false allow_custom_routing: false - so-logs-cloudflare.audit: + so-logs-cloudflare_x_audit: index_sorting: False index_template: index_patterns: @@ -658,7 +658,7 @@ elasticsearch: data_stream: hidden: false allow_custom_routing: false - so-logs-cloudflare.logpull: + so-logs-cloudflare_x_logpull: index_sorting: False index_template: index_patterns: @@ -676,7 +676,7 @@ elasticsearch: data_stream: hidden: false allow_custom_routing: false - so-logs-fim.event: + so-logs-fim_x_event: index_sorting: False index_template: index_patterns: @@ -694,7 +694,7 @@ elasticsearch: data_stream: hidden: false allow_custom_routing: false - so-logs-github.audit: + so-logs-github_x_audit: index_sorting: False index_template: index_patterns: @@ -712,7 +712,7 @@ elasticsearch: data_stream: hidden: false allow_custom_routing: false - so-logs-github.code_scanning: + so-logs-github_x_code_scanning: index_sorting: False index_template: index_patterns: @@ -730,7 +730,7 @@ elasticsearch: data_stream: hidden: false allow_custom_routing: false - so-logs-github.dependabot: + so-logs-github_x_dependabot: index_sorting: False index_template: index_patterns: @@ -748,7 +748,7 @@ elasticsearch: data_stream: hidden: false allow_custom_routing: false - so-logs-github.issues: + so-logs-github_x_issues: index_sorting: False index_template: index_patterns: @@ -766,7 +766,7 @@ elasticsearch: data_stream: hidden: false allow_custom_routing: false - so-logs-github.secret_scanning: + so-logs-github_x_secret_scanning: index_sorting: False index_template: index_patterns: @@ -784,7 +784,7 @@ elasticsearch: data_stream: hidden: false allow_custom_routing: false - so-logs-google_workspace.access_transparency: + so-logs-google_workspace_x_access_transparency: index_sorting: False index_template: index_patterns: @@ -802,7 +802,7 @@ elasticsearch: data_stream: hidden: false allow_custom_routing: false - so-logs-google_workspace.admin: + so-logs-google_workspace_x_admin: index_sorting: False index_template: index_patterns: @@ -820,7 +820,7 @@ elasticsearch: data_stream: hidden: false allow_custom_routing: false - so-logs-google_workspace.alert: + so-logs-google_workspace_x_alert: index_sorting: False index_template: index_patterns: @@ -838,7 +838,7 @@ elasticsearch: data_stream: hidden: false allow_custom_routing: false - so-logs-google_workspace.context_aware_access: + so-logs-google_workspace_x_context_aware_access: index_sorting: False index_template: index_patterns: @@ -856,7 +856,7 @@ elasticsearch: data_stream: hidden: false allow_custom_routing: false - so-logs-google_workspace.device: + so-logs-google_workspace_x_device: index_sorting: False index_template: index_patterns: @@ -874,7 +874,7 @@ elasticsearch: data_stream: hidden: false allow_custom_routing: false - so-logs-google_workspace.drive: + so-logs-google_workspace_x_drive: index_sorting: False index_template: index_patterns: @@ -892,7 +892,7 @@ elasticsearch: data_stream: hidden: false allow_custom_routing: false - so-logs-google_workspace.gcp: + so-logs-google_workspace_x_gcp: index_sorting: False index_template: index_patterns: @@ -910,7 +910,7 @@ elasticsearch: data_stream: hidden: false allow_custom_routing: false - so-logs-google_workspace.group_enterprise: + so-logs-google_workspace_x_group_enterprise: index_sorting: False index_template: index_patterns: @@ -928,7 +928,7 @@ elasticsearch: data_stream: hidden: false allow_custom_routing: false - so-logs-google_workspace.groups: + so-logs-google_workspace_x_groups: index_sorting: False index_template: index_patterns: @@ -946,7 +946,7 @@ elasticsearch: data_stream: hidden: false allow_custom_routing: false - so-logs-google_workspace.login: + so-logs-google_workspace_x_login: index_sorting: False index_template: index_patterns: @@ -964,7 +964,7 @@ elasticsearch: data_stream: hidden: false allow_custom_routing: false - so-logs-google_workspace.rules: + so-logs-google_workspace_x_rules: index_sorting: False index_template: index_patterns: @@ -982,7 +982,7 @@ elasticsearch: data_stream: hidden: false allow_custom_routing: false - so-logs-google_workspace.saml: + so-logs-google_workspace_x_saml: index_sorting: False index_template: index_patterns: @@ -1000,7 +1000,7 @@ elasticsearch: data_stream: hidden: false allow_custom_routing: false - so-logs-google_workspace.token: + so-logs-google_workspace_x_token: index_sorting: False index_template: index_patterns: @@ -1018,7 +1018,7 @@ elasticsearch: data_stream: hidden: false allow_custom_routing: false - so-logs-google_workspace.user_accounts: + so-logs-google_workspace_x_user_accounts: index_sorting: False index_template: index_patterns: @@ -1036,7 +1036,7 @@ elasticsearch: data_stream: hidden: false allow_custom_routing: false - so-logs-1password.item_usages: + so-logs-1password_x_item_usages: index_sorting: False index_template: index_patterns: @@ -1054,7 +1054,7 @@ elasticsearch: data_stream: hidden: false allow_custom_routing: false - so-logs-1password.signin_attempts: + so-logs-1password_x_signin_attempts: index_sorting: False index_template: index_patterns: @@ -1089,7 +1089,7 @@ elasticsearch: name: elastic_agent managed_by: security_onion managed: true - so-logs-osquery-manager-action.responses: + so-logs-osquery-manager-action_x_responses: index_sorting: False index_template: index_patterns: @@ -1106,7 +1106,7 @@ elasticsearch: name: elastic_agent managed_by: security_onion managed: true - so-logs-elastic_agent.apm_server: + so-logs-elastic_agent_x_apm_server: index_sorting: False index_template: index_patterns: @@ -1160,7 +1160,7 @@ elasticsearch: name: elastic_agent managed_by: security_onion managed: true - so-logs-elastic_agent.auditbeat: + so-logs-elastic_agent_x_auditbeat: index_sorting: False index_template: index_patterns: @@ -1214,7 +1214,7 @@ elasticsearch: name: elastic_agent managed_by: security_onion managed: true - so-logs-elastic_agent.cloudbeat: + so-logs-elastic_agent_x_cloudbeat: index_sorting: False index_template: index_patterns: @@ -1265,7 +1265,7 @@ elasticsearch: name: elastic_agent managed_by: security_onion managed: true - so-logs-elastic_agent.endpoint_security: + so-logs-elastic_agent_x_endpoint_security: index_sorting: False index_template: index_patterns: @@ -1314,7 +1314,7 @@ elasticsearch: name: elastic_agent managed_by: security_onion managed: true - so-logs-endpoint.alerts: + so-logs-endpoint_x_alerts: index_sorting: False index_template: index_patterns: @@ -1363,7 +1363,7 @@ elasticsearch: name: elastic_agent managed_by: security_onion managed: true - so-logs-endpoint.events.api: + so-logs-endpoint_x_events_x_api: index_sorting: False index_template: index_patterns: @@ -1412,7 +1412,7 @@ elasticsearch: name: elastic_agent managed_by: security_onion managed: true - so-logs-endpoint.events.file: + so-logs-endpoint_x_events_x_file: index_sorting: False index_template: index_patterns: @@ -1461,7 +1461,7 @@ elasticsearch: name: elastic_agent managed_by: security_onion managed: true - so-logs-endpoint.events.library: + so-logs-endpoint_x_events_x_library: index_sorting: False index_template: index_patterns: @@ -1510,7 +1510,7 @@ elasticsearch: name: elastic_agent managed_by: security_onion managed: true - so-logs-endpoint.events.network: + so-logs-endpoint_x_events_x_network: index_sorting: False index_template: index_patterns: @@ -1559,7 +1559,7 @@ elasticsearch: name: elastic_agent managed_by: security_onion managed: true - so-logs-endpoint.events.process: + so-logs-endpoint_x_events_x_process: index_sorting: False index_template: index_patterns: @@ -1608,7 +1608,7 @@ elasticsearch: name: elastic_agent managed_by: security_onion managed: true - so-logs-endpoint.events.registry: + so-logs-endpoint_x_events_x_registry: index_sorting: False index_template: index_patterns: @@ -1657,7 +1657,7 @@ elasticsearch: name: elastic_agent managed_by: security_onion managed: true - so-logs-endpoint.events.security: + so-logs-endpoint_x_events_x_security: index_sorting: False index_template: index_patterns: @@ -1706,7 +1706,7 @@ elasticsearch: name: elastic_agent managed_by: security_onion managed: true - so-logs-elastic_agent.filebeat: + so-logs-elastic_agent_x_filebeat: index_sorting: False index_template: index_patterns: @@ -1755,7 +1755,7 @@ elasticsearch: name: elastic_agent managed_by: security_onion managed: true - so-logs-elastic_agent.fleet_server: + so-logs-elastic_agent_x_fleet_server: index_sorting: False index_template: index_patterns: @@ -1801,7 +1801,7 @@ elasticsearch: name: elastic_agent managed_by: security_onion managed: true - so-logs-elastic_agent.heartbeat: + so-logs-elastic_agent_x_heartbeat: index_sorting: False index_template: index_patterns: @@ -1907,7 +1907,7 @@ elasticsearch: name: elastic_agent managed_by: security_onion managed: true - so-logs-elastic_agent.metricbeat: + so-logs-elastic_agent_x_metricbeat: index_sorting: False index_template: index_patterns: @@ -1956,7 +1956,7 @@ elasticsearch: name: elastic_agent managed_by: security_onion managed: true - so-logs-elastic_agent.osquerybeat: + so-logs-elastic_agent_x_osquerybeat: index_sorting: False index_template: index_patterns: @@ -2005,7 +2005,7 @@ elasticsearch: name: elastic_agent managed_by: security_onion managed: true - so-logs-elastic_agent.packetbeat: + so-logs-elastic_agent_x_packetbeat: index_sorting: False index_template: index_patterns: diff --git a/salt/elasticsearch/soc_elasticsearch.yaml b/salt/elasticsearch/soc_elasticsearch.yaml index 89d347b427..889e9f6a4d 100644 --- a/salt/elasticsearch/soc_elasticsearch.yaml +++ b/salt/elasticsearch/soc_elasticsearch.yaml @@ -181,80 +181,80 @@ elasticsearch: forcedType: bool global: True helpLink: elasticsearch.html - so-logs-system.auth: *indexSettings - so-logs-system.syslog: *indexSettings - so-logs-system.system: *indexSettings - so-logs-system.application: *indexSettings - so-logs-system.security: *indexSettings - so-logs-windows.forwarded: *indexSettings - so-logs-windows.powershell: *indexSettings - so-logs-windows.powershell_operational: *indexSettings - so-logs-windows.sysmon_operational: *indexSettings - so-logs-aws.cloudtrail: *indexSettings - so-logs-aws.cloudwatch_logs: *indexSettings - so-logs-aws.ec2_logs: *indexSettings - so-logs-aws.elb_logs: *indexSettings - so-logs-aws.firewall_logs: *indexSettings - so-logs-aws.route53_public_logs: *indexSettings - so-logs-aws.route53_resolver_logs: *indexSettings - so-logs-aws.s3access: *indexSettings - so-logs-aws.vpcflow: *indexSettings - so-logs-aws.waf: *indexSettings - so-logs-azure.activitylogs: *indexSettings - so-logs-azure.application_gateway: *indexSettings - so-logs-azure.auditlogs: *indexSettings - so-logs-azure.eventhub: *indexSettings - so-logs-azure.firewall_logs: *indexSettings - so-logs-azure.identity_protection: *indexSettings - so-logs-azure.platformlogs: *indexSettings - so-logs-azure.provisioning: *indexSettings - so-logs-azure.signinlogs: *indexSettings - so-logs-azure.springcloudlogs: *indexSettings - so-logs-cloudflare.audit: *indexSettings - so-logs-cloudflare.logpull: *indexSettings - so-logs-fim.event: *indexSettings - so-logs-github.audit: *indexSettings - so-logs-github.code_scanning: *indexSettings - so-logs-github.dependabot: *indexSettings - so-logs-github.issues: *indexSettings - so-logs-github.secret_scanning: *indexSettings - so-logs-google_workspace.access_transparency: *indexSettings - so-logs-google_workspace.admin: *indexSettings - so-logs-google_workspace.alert: *indexSettings - so-logs-google_workspace.context_aware_access: *indexSettings - so-logs-google_workspace.device: *indexSettings - so-logs-google_workspace.drive: *indexSettings - so-logs-google_workspace.gcp: *indexSettings - so-logs-google_workspace.group_enterprise: *indexSettings - so-logs-google_workspace.groups: *indexSettings - so-logs-google_workspace.login: *indexSettings - so-logs-google_workspace.rules: *indexSettings - so-logs-google_workspace.saml: *indexSettings - so-logs-google_workspace.token: *indexSettings - so-logs-google_workspace.user_accounts: *indexSettings - so-logs-1password.item_usages: *indexSettings - so-logs-1password.signin_attempts: *indexSettings + so-logs-system_x_auth: *indexSettings + so-logs-system_x_syslog: *indexSettings + so-logs-system_x_system: *indexSettings + so-logs-system_x_application: *indexSettings + so-logs-system_x_security: *indexSettings + so-logs-windows_x_forwarded: *indexSettings + so-logs-windows_x_powershell: *indexSettings + so-logs-windows_x_powershell_operational: *indexSettings + so-logs-windows_x_sysmon_operational: *indexSettings + so-logs-aws_x_cloudtrail: *indexSettings + so-logs-aws_x_cloudwatch_logs: *indexSettings + so-logs-aws_x_ec2_logs: *indexSettings + so-logs-aws_x_elb_logs: *indexSettings + so-logs-aws_x_firewall_logs: *indexSettings + so-logs-aws_x_route53_public_logs: *indexSettings + so-logs-aws_x_route53_resolver_logs: *indexSettings + so-logs-aws_x_s3access: *indexSettings + so-logs-aws_x_vpcflow: *indexSettings + so-logs-aws_x_waf: *indexSettings + so-logs-azure_x_activitylogs: *indexSettings + so-logs-azure_x_application_gateway: *indexSettings + so-logs-azure_x_auditlogs: *indexSettings + so-logs-azure_x_eventhub: *indexSettings + so-logs-azure_x_firewall_logs: *indexSettings + so-logs-azure_x_identity_protection: *indexSettings + so-logs-azure_x_platformlogs: *indexSettings + so-logs-azure_x_provisioning: *indexSettings + so-logs-azure_x_signinlogs: *indexSettings + so-logs-azure_x_springcloudlogs: *indexSettings + so-logs-cloudflare_x_audit: *indexSettings + so-logs-cloudflare_x_logpull: *indexSettings + so-logs-fim_x_event: *indexSettings + so-logs-github_x_audit: *indexSettings + so-logs-github_x_code_scanning: *indexSettings + so-logs-github_x_dependabot: *indexSettings + so-logs-github_x_issues: *indexSettings + so-logs-github_x_secret_scanning: *indexSettings + so-logs-google_workspace_x_access_transparency: *indexSettings + so-logs-google_workspace_x_admin: *indexSettings + so-logs-google_workspace_x_alert: *indexSettings + so-logs-google_workspace_x_context_aware_access: *indexSettings + so-logs-google_workspace_x_device: *indexSettings + so-logs-google_workspace_x_drive: *indexSettings + so-logs-google_workspace_x_gcp: *indexSettings + so-logs-google_workspace_x_group_enterprise: *indexSettings + so-logs-google_workspace_x_groups: *indexSettings + so-logs-google_workspace_x_login: *indexSettings + so-logs-google_workspace_x_rules: *indexSettings + so-logs-google_workspace_x_saml: *indexSettings + so-logs-google_workspace_x_token: *indexSettings + so-logs-google_workspace_x_user_accounts: *indexSettings + so-logs-1password_x_item_usages: *indexSettings + so-logs-1password_x_signin_attempts: *indexSettings so-logs-osquery-manager-actions: *indexSettings - so-logs-osquery-manager-action.responses: *indexSettings - so-logs-elastic_agent.apm_server: *indexSettings - so-logs-elastic_agent.auditbeat: *indexSettings - so-logs-elastic_agent.cloudbeat: *indexSettings - so-logs-elastic_agent.endpoint_security: *indexSettings - so-logs-endpoint.alerts: *indexSettings - so-logs-endpoint.events.api: *indexSettings - so-logs-endpoint.events.file: *indexSettings - so-logs-endpoint.events.library: *indexSettings - so-logs-endpoint.events.network: *indexSettings - so-logs-endpoint.events.process: *indexSettings - so-logs-endpoint.events.registry: *indexSettings - so-logs-endpoint.events.security: *indexSettings - so-logs-elastic_agent.filebeat: *indexSettings - so-logs-elastic_agent.fleet_server: *indexSettings - so-logs-elastic_agent.heartbeat: *indexSettings + so-logs-osquery-manager-action_x_responses: *indexSettings + so-logs-elastic_agent_x_apm_server: *indexSettings + so-logs-elastic_agent_x_auditbeat: *indexSettings + so-logs-elastic_agent_x_cloudbeat: *indexSettings + so-logs-elastic_agent_x_endpoint_security: *indexSettings + so-logs-endpoint_x_alerts: *indexSettings + so-logs-endpoint_x_events_x_api: *indexSettings + so-logs-endpoint_x_events_x_file: *indexSettings + so-logs-endpoint_x_events_x_library: *indexSettings + so-logs-endpoint_x_events_x_network: *indexSettings + so-logs-endpoint_x_events_x_process: *indexSettings + so-logs-endpoint_x_events_x_registry: *indexSettings + so-logs-endpoint_x_events_x_security: *indexSettings + so-logs-elastic_agent_x_filebeat: *indexSettings + so-logs-elastic_agent_x_fleet_server: *indexSettings + so-logs-elastic_agent_x_heartbeat: *indexSettings so-logs-elastic_agent: *indexSettings - so-logs-elastic_agent.metricbeat: *indexSettings - so-logs-elastic_agent.osquerybeat: *indexSettings - so-logs-elastic_agent.packetbeat: *indexSettings + so-logs-elastic_agent_x_metricbeat: *indexSettings + so-logs-elastic_agent_x_osquerybeat: *indexSettings + so-logs-elastic_agent_x_packetbeat: *indexSettings so-case: *indexSettings so-common: *indexSettings so-endgame: *indexSettings diff --git a/salt/elasticsearch/template.map.jinja b/salt/elasticsearch/template.map.jinja index 49d86d1870..5fe0ed3039 100644 --- a/salt/elasticsearch/template.map.jinja +++ b/salt/elasticsearch/template.map.jinja @@ -1,9 +1,11 @@ {% import_yaml 'elasticsearch/defaults.yaml' as ELASTICSEARCHDEFAULTS with context %} -{%- set ES_INDEX_SETTINGS = salt['pillar.get']('elasticsearch:index_settings', default=ELASTICSEARCHDEFAULTS.elasticsearch.index_settings, merge=True) %} -{% for index, settings in ES_INDEX_SETTINGS.items() %} +{%- set ES_INDEX_SETTINGS_ORIG = salt['pillar.get']('elasticsearch:index_settings', default=ELASTICSEARCHDEFAULTS.elasticsearch.index_settings, merge=True) %} +{% set ES_INDEX_SETTINGS = {} %} +{% for index, settings in ES_INDEX_SETTINGS_ORIG.items() %} {% if settings.index_template is defined %} {% if not settings.get('index_sorting', False) | to_bool and settings.index_template.template.settings.index.sort is defined %} {% do settings.index_template.template.settings.index.pop('sort') %} {% endif %} {% endif %} + {% do ES_INDEX_SETTINGS.update({index | replace("_x_", "."): ES_INDEX_SETTINGS_ORIG[index]}) %} {% endfor %} diff --git a/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-ilm-policy-load b/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-ilm-policy-load index afb8bdc67a..b00fcbedfd 100755 --- a/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-ilm-policy-load +++ b/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-ilm-policy-load @@ -6,8 +6,7 @@ . /usr/sbin/so-common -{%- import_yaml 'elasticsearch/defaults.yaml' as ELASTICSEARCHDEFAULTS %} -{%- set ES_INDEX_SETTINGS = salt['pillar.get']('elasticsearch:index_settings', default=ELASTICSEARCHDEFAULTS.elasticsearch.index_settings, merge=True) %} +{%- from 'elasticsearch/template.map.jinja' import ES_INDEX_SETTINGS %} {%- for index, settings in ES_INDEX_SETTINGS.items() %} {%- if settings.policy is defined %}