From 968fee3488eee120dcf1fc1e403539e09e93c459 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Tue, 1 Aug 2023 13:10:41 -0400 Subject: [PATCH 1/8] Regen Agent Installers when Fleet URLs change --- .../tools/sbin_jinja/so-elastic-agent-gen-installers | 6 ++++++ .../tools/sbin_jinja/so-elastic-fleet-urls-update | 3 ++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-gen-installers b/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-gen-installers index 2a19dcbd9c..d7d6458c9b 100755 --- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-gen-installers +++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-gen-installers @@ -11,6 +11,12 @@ . /usr/sbin/so-common . /usr/sbin/so-elastic-fleet-common +LOG="/opt/so/log/elasticfleet/so-elastic-agent-gen-installers.log" + +# Check to see if we are already running +NUM_RUNNING=$(pgrep -cf "/bin/bash /sbin/so-elastic-agent-gen-installers") +[ "$NUM_RUNNING" -gt 1 ] && echo "$(date) - $NUM_RUNNING gen installers script processes running...exiting." >>$LOG && exit 0 + for i in {1..30} do ENROLLMENTOKEN=$(curl -K /opt/so/conf/elasticsearch/curl.config -L "localhost:5601/api/fleet/enrollment_api_keys" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' | jq .list | jq -r -c '.[] | select(.policy_id | contains("endpoints-initial")) | .api_key') diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-urls-update b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-urls-update index 24c5dabed3..4a744665a5 100644 --- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-urls-update +++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-urls-update @@ -62,7 +62,7 @@ fi NEW_LIST_JSON=$(jq --compact-output --null-input '$ARGS.positional' --args -- "${NEW_LIST[@]}") NEW_HASH=$(sha1sum <<< "$NEW_LIST_JSON" | awk '{print $1}') -# Compare the current & new list of URLs - if different, update the Fleet Server URLs +# Compare the current & new list of URLs - if different, update the Fleet Server URLs & regenerate the agent installer if [ "$NEW_HASH" = "$CURRENT_HASH" ]; then printf "\nHashes match - no update needed.\n" printf "Current List: $CURRENT_LIST\nNew List: $NEW_LIST_JSON\n" @@ -71,4 +71,5 @@ else printf "\nHashes don't match - update needed.\n" printf "Current List: $CURRENT_LIST\nNew List: $NEW_LIST_JSON\n" update_fleet_urls + /sbin/so-elastic-agent-gen-installers & fi From b520c1abb777a479df05e7e033edfa7b57b37d77 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Wed, 2 Aug 2023 10:36:40 -0400 Subject: [PATCH 2/8] Allow multiple Custom Fleet FQDN --- salt/elasticfleet/defaults.yaml | 3 ++- salt/elasticfleet/soc_elasticfleet.yaml | 2 +- .../sbin_jinja/so-elastic-fleet-outputs-update | 14 +++++++++----- .../sbin_jinja/so-elastic-fleet-urls-update | 16 ++++++++++------ 4 files changed, 22 insertions(+), 13 deletions(-) diff --git a/salt/elasticfleet/defaults.yaml b/salt/elasticfleet/defaults.yaml index 3d806d63f2..62a1302c1d 100644 --- a/salt/elasticfleet/defaults.yaml +++ b/salt/elasticfleet/defaults.yaml @@ -2,7 +2,8 @@ elasticfleet: enabled: False config: server: - custom_fqdn: '' + custom_fqdn: + - '' enable_auto_configuration: True endpoints_enrollment: '' es_token: '' diff --git a/salt/elasticfleet/soc_elasticfleet.yaml b/salt/elasticfleet/soc_elasticfleet.yaml index 9b918f0aca..772e681816 100644 --- a/salt/elasticfleet/soc_elasticfleet.yaml +++ b/salt/elasticfleet/soc_elasticfleet.yaml @@ -12,7 +12,7 @@ elasticfleet: config: server: custom_fqdn: - description: Custom FQDN for Agents to connect to. + description: Custom FQDN for Agents to connect to. One per line. global: True helpLink: elastic-fleet.html advanced: True diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-outputs-update b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-outputs-update index 042084d84f..400a6224f4 100644 --- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-outputs-update +++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-outputs-update @@ -2,7 +2,6 @@ # or more contributor license agreements. Licensed under the Elastic License 2.0; you may not use # this file except in compliance with the Elastic License 2.0. {% from 'vars/globals.map.jinja' import GLOBALS %} -{% set CUSTOMFQDN = salt['pillar.get']('elasticfleet:config:server:custom_fqdn') %} . /usr/sbin/so-common @@ -41,10 +40,15 @@ else NEW_LIST=("{{ GLOBALS.url_base }}:5055" "{{ GLOBALS.hostname }}:5055") fi -{% if CUSTOMFQDN != "" %} -# Add Custom Hostname to list -NEW_LIST+=("{{ CUSTOMFQDN }}:5055") -{% endif %} +# Query for FQDN entries & add them to the list +CUSTOMFQDNLIST=$( salt-call --out=json pillar.get elasticfleet:config:server:custom_fqdn | jq -r '.local | .[]') +if [ -n "$CUSTOMFQDNLIST" ]; then + readarray -t CUSTOMFQDN <<< $CUSTOMFQDNLIST + for CUSTOMNAME in "${CUSTOMFQDN[@]}" + do + NEW_LIST+=("$CUSTOMNAME:5055") + done +fi # Query for the current Grid Nodes that are running Logstash LOGSTASHNODES=$(salt-call --out=json pillar.get logstash:nodes | jq '.local') diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-urls-update b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-urls-update index 4a744665a5..52727780d7 100644 --- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-urls-update +++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-urls-update @@ -2,7 +2,6 @@ # or more contributor license agreements. Licensed under the Elastic License 2.0; you may not use # this file except in compliance with the Elastic License 2.0. {% from 'vars/globals.map.jinja' import GLOBALS %} -{% set CUSTOMFQDN = salt['pillar.get']('elasticfleet:config:server:custom_fqdn') %} . /usr/sbin/so-common @@ -41,10 +40,15 @@ else NEW_LIST=("https://{{ GLOBALS.url_base }}:8220" "https://{{ GLOBALS.hostname }}:8220") fi -{% if CUSTOMFQDN != "" %} -# Add Custom Hostname to list -NEW_LIST+=("https://{{ CUSTOMFQDN }}:8220") -{% endif %} +# Query for FQDN entries & add them to the list +CUSTOMFQDNLIST=$( salt-call --out=json pillar.get elasticfleet:config:server:custom_fqdn | jq -r '.local | .[]') +if [ -n "$CUSTOMFQDNLIST" ]; then + readarray -t CUSTOMFQDN <<< $CUSTOMFQDNLIST + for CUSTOMNAME in "${CUSTOMFQDN[@]}" + do + NEW_LIST+=("https://$CUSTOMNAME:8220") + done +fi # Query for the current Grid Nodes that are running Logstash (which includes Fleet Nodes) LOGSTASHNODES=$(salt-call --out=json pillar.get logstash:nodes | jq '.local') @@ -71,5 +75,5 @@ else printf "\nHashes don't match - update needed.\n" printf "Current List: $CURRENT_LIST\nNew List: $NEW_LIST_JSON\n" update_fleet_urls - /sbin/so-elastic-agent-gen-installers & + /sbin/so-elastic-agent-gen-installers >> /opt/so/log/elasticfleet/so-elastic-agent-gen-installers.log & fi From f153c1125d9dba74b5358c298936fbd0b873c2f8 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Wed, 2 Aug 2023 15:23:18 -0400 Subject: [PATCH 3/8] Allow multiple Custom Fleet FQDN --- salt/elasticfleet/defaults.yaml | 3 +-- salt/elasticfleet/enabled.sls | 13 ++++++++++--- salt/elasticfleet/soc_elasticfleet.yaml | 1 + 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/salt/elasticfleet/defaults.yaml b/salt/elasticfleet/defaults.yaml index 62a1302c1d..0ae7a5176b 100644 --- a/salt/elasticfleet/defaults.yaml +++ b/salt/elasticfleet/defaults.yaml @@ -2,8 +2,7 @@ elasticfleet: enabled: False config: server: - custom_fqdn: - - '' + custom_fqdn: [] enable_auto_configuration: True endpoints_enrollment: '' es_token: '' diff --git a/salt/elasticfleet/enabled.sls b/salt/elasticfleet/enabled.sls index 025a87e14a..bb6410f2cb 100644 --- a/salt/elasticfleet/enabled.sls +++ b/salt/elasticfleet/enabled.sls @@ -15,6 +15,7 @@ include: - elasticfleet.config - elasticfleet.sostatus + - ssl # If enabled, automatically update Fleet Logstash Outputs {% if ELASTICFLEETMERGED.config.server.enable_auto_configuration and grains.role not in ['so-import', 'so-eval', 'so-fleet'] %} @@ -61,11 +62,14 @@ so-elastic-fleet: - {{ BINDING }} {% endfor %} - binds: - - /etc/pki:/etc/pki:ro + - /etc/pki/elasticfleet-server.crt:/etc/pki/elasticfleet-server.crt:ro + - /etc/pki/elasticfleet-server.key:/etc/pki/elasticfleet-server.key:ro + - /etc/pki/tls/certs/intca.crt:/etc/pki/tls/certs/intca.crt:ro {% if GLOBALS.os_family == 'Debian' %} - - /etc/ssl:/etc/ssl:ro + - /etc/ssl/elasticfleet-server.crt:/etc/ssl/elasticfleet-server.crt:ro + - /etc/ssl/elasticfleet-server.key:/etc/ssl/elasticfleet-server.key:ro + - /etc/ssl/tls/certs/intca.crt:/etc/ssl/tls/certs/intca.crt:ro {% endif %} - #- /opt/so/conf/elastic-fleet/state:/usr/share/elastic-agent/state:rw - /opt/so/log/elasticfleet:/usr/share/elastic-agent/logs {% if DOCKER.containers['so-elastic-fleet'].custom_bind_mounts %} {% for BIND in DOCKER.containers['so-elastic-fleet'].custom_bind_mounts %} @@ -93,6 +97,9 @@ so-elastic-fleet: - {{ XTRAENV }} {% endfor %} {% endif %} + - watch: + - x509: etc_elasticfleet_key + - x509: etc_elasticfleet_crt {% endif %} {% if GLOBALS.role != "so-fleet" %} diff --git a/salt/elasticfleet/soc_elasticfleet.yaml b/salt/elasticfleet/soc_elasticfleet.yaml index 772e681816..af660358a6 100644 --- a/salt/elasticfleet/soc_elasticfleet.yaml +++ b/salt/elasticfleet/soc_elasticfleet.yaml @@ -16,6 +16,7 @@ elasticfleet: global: True helpLink: elastic-fleet.html advanced: True + forcedType: "[]string" enable_auto_configuration: description: Enable auto-configuration of Logstash Outputs & Fleet Host URLs. global: True From ab28cee7cf3041ac6276b120956e8d117a1323b4 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Wed, 2 Aug 2023 17:45:37 -0400 Subject: [PATCH 4/8] Allow multiple Custom Fleet FQDN --- salt/ssl/init.sls | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/salt/ssl/init.sls b/salt/ssl/init.sls index 97e971b83f..15c29791f1 100644 --- a/salt/ssl/init.sls +++ b/salt/ssl/init.sls @@ -7,7 +7,7 @@ {% if sls in allowed_states %} {% from 'vars/globals.map.jinja' import GLOBALS %} -{% set CUSTOMFQDN = salt['pillar.get']('elasticfleet:config:server:custom_fqdn') %} +{% from 'elasticfleet/map.jinja' import ELASTICFLEETMERGED %} {% set global_ca_text = [] %} {% set global_ca_server = [] %} @@ -154,7 +154,7 @@ etc_elasticfleet_crt: - signing_policy: elasticfleet - private_key: /etc/pki/elasticfleet-server.key - CN: {{ GLOBALS.url_base }} - - subjectAltName: DNS:{{ GLOBALS.hostname }},IP:{{ GLOBALS.node_ip }} {% if CUSTOMFQDN != "" %},DNS:{{ CUSTOMFQDN }}{% endif %} + - subjectAltName: DNS:{{ GLOBALS.hostname }},IP:{{ GLOBALS.node_ip }}{% if ELASTICFLEETMERGED.config.server.custom_fqdn[0] != "" %},DNS:{{ ELASTICFLEETMERGED.config.server.custom_fqdn | join(',DNS:') }}{% endif %} - days_remaining: 0 - days_valid: 820 - backup: True @@ -211,7 +211,7 @@ etc_elasticfleet_logstash_crt: - signing_policy: elasticfleet - private_key: /etc/pki/elasticfleet-logstash.key - CN: {{ GLOBALS.url_base }} - - subjectAltName: DNS:{{ GLOBALS.hostname }},IP:{{ GLOBALS.node_ip }} {% if CUSTOMFQDN != "" %},DNS:{{ CUSTOMFQDN }}{% endif %} + - subjectAltName: DNS:{{ GLOBALS.hostname }},IP:{{ GLOBALS.node_ip }}{% if ELASTICFLEETMERGED.config.server.custom_fqdn[0] != "" %},DNS:{{ ELASTICFLEETMERGED.config.server.custom_fqdn | join(',DNS:') }}{% endif %} - days_remaining: 0 - days_valid: 820 - backup: True From 1c8a8c460c90572cbeea725a88a60dc358c5b5f9 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Wed, 2 Aug 2023 17:53:29 -0400 Subject: [PATCH 5/8] Restart logstash when certs change --- salt/logstash/enabled.sls | 3 +++ 1 file changed, 3 insertions(+) diff --git a/salt/logstash/enabled.sls b/salt/logstash/enabled.sls index cd9d6dd7e6..a33080f8dc 100644 --- a/salt/logstash/enabled.sls +++ b/salt/logstash/enabled.sls @@ -22,6 +22,7 @@ include: {% endif %} - logstash.config - logstash.sostatus + - ssl so-logstash: docker_container.running: @@ -90,6 +91,8 @@ so-logstash: {% endfor %} {% endif %} - watch: + - x509: etc_elasticfleet_logstash_key + - x509: etc_elasticfleet_logstash_crt - file: lsetcsync {% for assigned_pipeline in LOGSTASH_MERGED.assigned_pipelines.roles[GLOBALS.role.split('-')[1]] %} - file: ls_pipeline_{{assigned_pipeline}} From 1bc7bbc76efe8dcead09867a34358f64ea71e2c5 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Wed, 2 Aug 2023 20:02:37 -0400 Subject: [PATCH 6/8] Refactor custom_fqdn --- salt/ssl/init.sls | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/ssl/init.sls b/salt/ssl/init.sls index 15c29791f1..063172e008 100644 --- a/salt/ssl/init.sls +++ b/salt/ssl/init.sls @@ -154,7 +154,7 @@ etc_elasticfleet_crt: - signing_policy: elasticfleet - private_key: /etc/pki/elasticfleet-server.key - CN: {{ GLOBALS.url_base }} - - subjectAltName: DNS:{{ GLOBALS.hostname }},IP:{{ GLOBALS.node_ip }}{% if ELASTICFLEETMERGED.config.server.custom_fqdn[0] != "" %},DNS:{{ ELASTICFLEETMERGED.config.server.custom_fqdn | join(',DNS:') }}{% endif %} + - subjectAltName: DNS:{{ GLOBALS.hostname }},IP:{{ GLOBALS.node_ip }}{% if ELASTICFLEETMERGED.config.server.custom_fqdn | length > 0 %},DNS:{{ ELASTICFLEETMERGED.config.server.custom_fqdn | join(',DNS:') }}{% endif %} - days_remaining: 0 - days_valid: 820 - backup: True @@ -211,7 +211,7 @@ etc_elasticfleet_logstash_crt: - signing_policy: elasticfleet - private_key: /etc/pki/elasticfleet-logstash.key - CN: {{ GLOBALS.url_base }} - - subjectAltName: DNS:{{ GLOBALS.hostname }},IP:{{ GLOBALS.node_ip }}{% if ELASTICFLEETMERGED.config.server.custom_fqdn[0] != "" %},DNS:{{ ELASTICFLEETMERGED.config.server.custom_fqdn | join(',DNS:') }}{% endif %} + - subjectAltName: DNS:{{ GLOBALS.hostname }},IP:{{ GLOBALS.node_ip }}{% if ELASTICFLEETMERGED.config.server.custom_fqdn | length > 0 %},DNS:{{ ELASTICFLEETMERGED.config.server.custom_fqdn | join(',DNS:') }}{% endif %} - days_remaining: 0 - days_valid: 820 - backup: True From 27b70cbf6891d021981d5c798332f602f8612b25 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Thu, 3 Aug 2023 15:21:20 -0400 Subject: [PATCH 7/8] Use jinja instead --- .../tools/sbin_jinja/so-elastic-fleet-urls-update | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-urls-update b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-urls-update index 52727780d7..c484fa704e 100644 --- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-urls-update +++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-urls-update @@ -2,6 +2,7 @@ # or more contributor license agreements. Licensed under the Elastic License 2.0; you may not use # this file except in compliance with the Elastic License 2.0. {% from 'vars/globals.map.jinja' import GLOBALS %} +{% from 'elasticfleet/map.jinja' import ELASTICFLEETMERGED %} . /usr/sbin/so-common @@ -41,7 +42,8 @@ else fi # Query for FQDN entries & add them to the list -CUSTOMFQDNLIST=$( salt-call --out=json pillar.get elasticfleet:config:server:custom_fqdn | jq -r '.local | .[]') +{% if ELASTICFLEETMERGED.config.server.custom_fqdn | length > 0 %} +CUSTOMFQDNLIST=({{ ELASTICFLEETMERGED.config.server.custom_fqdn | join(' ') }}) if [ -n "$CUSTOMFQDNLIST" ]; then readarray -t CUSTOMFQDN <<< $CUSTOMFQDNLIST for CUSTOMNAME in "${CUSTOMFQDN[@]}" @@ -49,6 +51,7 @@ if [ -n "$CUSTOMFQDNLIST" ]; then NEW_LIST+=("https://$CUSTOMNAME:8220") done fi +{% endif %} # Query for the current Grid Nodes that are running Logstash (which includes Fleet Nodes) LOGSTASHNODES=$(salt-call --out=json pillar.get logstash:nodes | jq '.local') From e78fcbc6cbc2fa4362e45e378eebfcc80a0d8fc9 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Thu, 3 Aug 2023 15:25:11 -0400 Subject: [PATCH 8/8] Refactor for Jinja instead --- .../tools/sbin_jinja/so-elastic-fleet-outputs-update | 7 +++++-- .../tools/sbin_jinja/so-elastic-fleet-urls-update | 2 +- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-outputs-update b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-outputs-update index 400a6224f4..17c867c07f 100644 --- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-outputs-update +++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-outputs-update @@ -2,6 +2,7 @@ # or more contributor license agreements. Licensed under the Elastic License 2.0; you may not use # this file except in compliance with the Elastic License 2.0. {% from 'vars/globals.map.jinja' import GLOBALS %} +{% from 'elasticfleet/map.jinja' import ELASTICFLEETMERGED %} . /usr/sbin/so-common @@ -41,14 +42,16 @@ else fi # Query for FQDN entries & add them to the list -CUSTOMFQDNLIST=$( salt-call --out=json pillar.get elasticfleet:config:server:custom_fqdn | jq -r '.local | .[]') +{% if ELASTICFLEETMERGED.config.server.custom_fqdn | length > 0 %} +CUSTOMFQDNLIST=({{ ELASTICFLEETMERGED.config.server.custom_fqdn | join(' ') }}) if [ -n "$CUSTOMFQDNLIST" ]; then readarray -t CUSTOMFQDN <<< $CUSTOMFQDNLIST for CUSTOMNAME in "${CUSTOMFQDN[@]}" do - NEW_LIST+=("$CUSTOMNAME:5055") + NEW_LIST+=("https://$CUSTOMNAME:8220") done fi +{% endif %} # Query for the current Grid Nodes that are running Logstash LOGSTASHNODES=$(salt-call --out=json pillar.get logstash:nodes | jq '.local') diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-urls-update b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-urls-update index c484fa704e..7d29fe0808 100644 --- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-urls-update +++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-urls-update @@ -2,7 +2,7 @@ # or more contributor license agreements. Licensed under the Elastic License 2.0; you may not use # this file except in compliance with the Elastic License 2.0. {% from 'vars/globals.map.jinja' import GLOBALS %} -{% from 'elasticfleet/map.jinja' import ELASTICFLEETMERGED %} +{% from 'elasticfleet/map.jinja' import ELASTICFLEETMERGED %} . /usr/sbin/so-common