From ea145f59a381b03d826d41851a6445a07e5d0af0 Mon Sep 17 00:00:00 2001
From: Felix Hennig <felix.hennig@stackable.tech>
Date: Mon, 3 Jun 2024 15:52:18 +0200
Subject: [PATCH 01/23] replaced first sed instance

---
 rust/crd/src/lib.rs                          | 18 +++---------------
 rust/operator-binary/src/druid_controller.rs |  3 +--
 2 files changed, 4 insertions(+), 17 deletions(-)

diff --git a/rust/crd/src/lib.rs b/rust/crd/src/lib.rs
index ce476623..8c4f589e 100644
--- a/rust/crd/src/lib.rs
+++ b/rust/crd/src/lib.rs
@@ -136,9 +136,7 @@ pub const SC_VOLUME_NAME: &str = "segment-cache";
 
 pub const ENV_INTERNAL_SECRET: &str = "INTERNAL_SECRET";
 
-// DB credentials
-pub const DB_USERNAME_PLACEHOLDER: &str = "xxx_db_username_xxx";
-pub const DB_PASSWORD_PLACEHOLDER: &str = "xxx_db_password_xxx";
+// DB credentials - both of these are read from an env var by Druid with the ${env:...} syntax
 pub const DB_USERNAME_ENV: &str = "DB_USERNAME_ENV";
 pub const DB_PASSWORD_ENV: &str = "DB_PASSWORD_ENV";
 
@@ -518,7 +516,6 @@ impl DruidRole {
     pub fn main_container_prepare_commands(
         &self,
         s3_connection: Option<&S3ConnectionSpec>,
-        credentials_secret: Option<&String>,
     ) -> Vec<String> {
         let mut commands = vec![];
 
@@ -560,15 +557,6 @@ impl DruidRole {
             rw_conf = RW_CONFIG_DIRECTORY,
         ));
 
-        // db credentials
-        if credentials_secret.is_some() {
-            commands.extend([
-                format!("echo replacing {DB_USERNAME_PLACEHOLDER} and {DB_PASSWORD_PLACEHOLDER} with secret values."),
-                format!("sed -i \"s|{DB_USERNAME_PLACEHOLDER}|${DB_USERNAME_ENV}|g\" {RW_CONFIG_DIRECTORY}/{RUNTIME_PROPS}"),
-                format!("sed -i \"s|{DB_PASSWORD_PLACEHOLDER}|${DB_PASSWORD_ENV}|g\" {RW_CONFIG_DIRECTORY}/{RUNTIME_PROPS}"),
-            ]);
-        }
-
         commands
     }
 
@@ -631,11 +619,11 @@ impl DruidCluster {
                 if mds.credentials_secret.is_some() {
                     result.insert(
                         METADATA_STORAGE_USER.to_string(),
-                        Some(DB_USERNAME_PLACEHOLDER.into()),
+                        Some(format!("${{env:{DB_USERNAME_ENV}}}").into()),
                     );
                     result.insert(
                         METADATA_STORAGE_PASSWORD.to_string(),
-                        Some(DB_PASSWORD_PLACEHOLDER.into()),
+                        Some(format!("${{env:{DB_PASSWORD_ENV}}}").into()),
                     );
                 }
 
diff --git a/rust/operator-binary/src/druid_controller.rs b/rust/operator-binary/src/druid_controller.rs
index 53e2c208..f8196355 100644
--- a/rust/operator-binary/src/druid_controller.rs
+++ b/rust/operator-binary/src/druid_controller.rs
@@ -915,8 +915,7 @@ fn build_rolegroup_statefulset(
         .metadata_storage_database
         .credentials_secret
         .as_ref();
-    let mut main_container_commands =
-        role.main_container_prepare_commands(s3_conn, credentials_secret);
+    let mut main_container_commands = role.main_container_prepare_commands(s3_conn);
     let mut prepare_container_commands = vec![];
     if let Some(ContainerLogConfig {
         choice: Some(ContainerLogConfigChoice::Automatic(log_config)),

From 1e144d183f8f650716dc88031540f597dc376076 Mon Sep 17 00:00:00 2001
From: Felix Hennig <felix.hennig@stackable.tech>
Date: Mon, 3 Jun 2024 16:13:45 +0200
Subject: [PATCH 02/23] removed second instance of sed - still needs testing

---
 rust/crd/src/authentication/ldap.rs | 34 ++++++++++-------------------
 rust/crd/src/lib.rs                 |  4 ++--
 2 files changed, 13 insertions(+), 25 deletions(-)

diff --git a/rust/crd/src/authentication/ldap.rs b/rust/crd/src/authentication/ldap.rs
index 6ba1e0cc..0a88d01d 100644
--- a/rust/crd/src/authentication/ldap.rs
+++ b/rust/crd/src/authentication/ldap.rs
@@ -8,7 +8,7 @@ use stackable_operator::kube::ResourceExt;
 use crate::authentication::ResolvedAuthenticationClasses;
 use crate::{
     security::{add_cert_to_trust_store_cmd, STACKABLE_TLS_DIR, TLS_STORE_PASSWORD},
-    ENV_INTERNAL_SECRET, RUNTIME_PROPS, RW_CONFIG_DIRECTORY,
+    ENV_INTERNAL_SECRET,
 };
 
 #[derive(Snafu, Debug)]
@@ -25,10 +25,8 @@ pub struct DruidLdapSettings {
     pub authentication_class_name: String,
 }
 
-pub const PLACEHOLDER_INTERNAL_CLIENT_PASSWORD: &str =
-    "xxx_druid_system_internal_client_password_xxx";
-pub const PLACEHOLDER_LDAP_BIND_PASSWORD: &str = "xxx_ldap_bind_password_xxx";
-pub const PLACEHOLDER_LDAP_BIND_USER: &str = "xxx_ldap_bind_user_xxx";
+pub const ENV_LDAP_BIND_USER: &str = "LDAP_BIND_USER";
+pub const ENV_LDAP_BIND_PASSWORD: &str = "LDAP_BIND_PASSWORD";
 
 impl DruidLdapSettings {
     pub fn new_from(
@@ -63,7 +61,7 @@ impl DruidLdapSettings {
 
         config.insert(
             format!("{PREFIX}.initialInternalClientPassword"),
-            Some(PLACEHOLDER_INTERNAL_CLIENT_PASSWORD.to_string()),
+            Some(format!("${{env:{ENV_INTERNAL_SECRET}}}").to_string()),
         );
         config.insert(
             format!("{PREFIX}.authorizerName"),
@@ -100,11 +98,11 @@ impl DruidLdapSettings {
         if self.ldap.bind_credentials_mount_paths().is_some() {
             config.insert(
                 format!("{PREFIX}.credentialsValidator.bindUser"),
-                Some(PLACEHOLDER_LDAP_BIND_USER.to_string()), // NOTE: this placeholder will be replaced from a mounted secret operator volume on container startup
+                Some(format!("${{env:{ENV_LDAP_BIND_USER}}}").to_string()), // NOTE: this placeholder will be replaced from a mounted secret operator volume on container startup
             );
             config.insert(
                 format!("{PREFIX}.credentialsValidator.bindPassword"),
-                Some(PLACEHOLDER_LDAP_BIND_PASSWORD.to_string()), // NOTE: this placeholder will be replaced from a mounted secret operator volume on container startup
+                Some(format!("${{env:{ENV_LDAP_BIND_PASSWORD}}}").to_string()), // NOTE: this placeholder will be replaced from a mounted secret operator volume on container startup
             );
         }
 
@@ -139,7 +137,7 @@ impl DruidLdapSettings {
         );
         config.insert(
             "druid.escalator.internalClientPassword".to_string(),
-            Some(PLACEHOLDER_INTERNAL_CLIENT_PASSWORD.to_string()),
+            Some(format!("${{env:{ENV_INTERNAL_SECRET}}}").to_string()),
         );
         config.insert(
             "druid.escalator.authorizerName".to_string(),
@@ -183,27 +181,17 @@ impl DruidLdapSettings {
     pub fn main_container_commands(&self) -> Vec<String> {
         let mut commands = Vec::new();
 
-        let runtime_properties_file: String = format!("{RW_CONFIG_DIRECTORY}/{RUNTIME_PROPS}");
-        let internal_client_password = format!("$(echo ${ENV_INTERNAL_SECRET})");
-
-        commands
-                .push(format!("echo \"Replacing LDAP placeholders with their proper values in {runtime_properties_file}\""));
-        commands.push(format!(
-            r#"sed "s|{PLACEHOLDER_INTERNAL_CLIENT_PASSWORD}|{internal_client_password}|g" -i {runtime_properties_file}"# // using another delimiter (|) here because of base64 string
-        ));
-
         if let Some((ldap_bind_user_path, ldap_bind_password_path)) =
             self.ldap.bind_credentials_mount_paths()
         {
             let ldap_bind_user = format!("$(cat {ldap_bind_user_path})");
             let ldap_bind_password = format!("$(cat {ldap_bind_password_path})");
 
+            commands.push(format!("export {ENV_LDAP_BIND_USER}={ldap_bind_user}"));
             commands.push(format!(
-                    r#"sed "s/{PLACEHOLDER_LDAP_BIND_USER}/{ldap_bind_user}/g" -i {runtime_properties_file}"#
-                ));
-            commands.push(format!(
-                    r#"sed "s/{PLACEHOLDER_LDAP_BIND_PASSWORD}/{ldap_bind_password}/g" -i {runtime_properties_file}"#
-                ));
+                "export {ENV_LDAP_BIND_PASSWORD}={ldap_bind_password}"
+            ));
+            // TODO test if it works - need to run the LDAP test
         }
 
         commands
diff --git a/rust/crd/src/lib.rs b/rust/crd/src/lib.rs
index 8c4f589e..98cb13c2 100644
--- a/rust/crd/src/lib.rs
+++ b/rust/crd/src/lib.rs
@@ -619,11 +619,11 @@ impl DruidCluster {
                 if mds.credentials_secret.is_some() {
                     result.insert(
                         METADATA_STORAGE_USER.to_string(),
-                        Some(format!("${{env:{DB_USERNAME_ENV}}}").into()),
+                        Some(format!("${{env:{DB_USERNAME_ENV}}}")),
                     );
                     result.insert(
                         METADATA_STORAGE_PASSWORD.to_string(),
-                        Some(format!("${{env:{DB_PASSWORD_ENV}}}").into()),
+                        Some(format!("${{env:{DB_PASSWORD_ENV}}}")),
                     );
                 }
 

From b75aed05b44b047c6d17b5a11eb37766065ab636 Mon Sep 17 00:00:00 2001
From: Felix Hennig <felix.hennig@stackable.tech>
Date: Tue, 4 Jun 2024 14:42:51 +0200
Subject: [PATCH 03/23] ldap test worked

---
 rust/crd/src/authentication/ldap.rs | 1 -
 1 file changed, 1 deletion(-)

diff --git a/rust/crd/src/authentication/ldap.rs b/rust/crd/src/authentication/ldap.rs
index 0a88d01d..bbf1b4b3 100644
--- a/rust/crd/src/authentication/ldap.rs
+++ b/rust/crd/src/authentication/ldap.rs
@@ -191,7 +191,6 @@ impl DruidLdapSettings {
             commands.push(format!(
                 "export {ENV_LDAP_BIND_PASSWORD}={ldap_bind_password}"
             ));
-            // TODO test if it works - need to run the LDAP test
         }
 
         commands

From 06852b40a2ddeff0f3798c2cae02682576d0e100 Mon Sep 17 00:00:00 2001
From: Felix Hennig <felix.hennig@stackable.tech>
Date: Tue, 4 Jun 2024 14:45:29 +0200
Subject: [PATCH 04/23] remove outdated comment

---
 rust/crd/src/authentication/ldap.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/rust/crd/src/authentication/ldap.rs b/rust/crd/src/authentication/ldap.rs
index bbf1b4b3..2f4e5912 100644
--- a/rust/crd/src/authentication/ldap.rs
+++ b/rust/crd/src/authentication/ldap.rs
@@ -98,11 +98,11 @@ impl DruidLdapSettings {
         if self.ldap.bind_credentials_mount_paths().is_some() {
             config.insert(
                 format!("{PREFIX}.credentialsValidator.bindUser"),
-                Some(format!("${{env:{ENV_LDAP_BIND_USER}}}").to_string()), // NOTE: this placeholder will be replaced from a mounted secret operator volume on container startup
+                Some(format!("${{env:{ENV_LDAP_BIND_USER}}}").to_string()),
             );
             config.insert(
                 format!("{PREFIX}.credentialsValidator.bindPassword"),
-                Some(format!("${{env:{ENV_LDAP_BIND_PASSWORD}}}").to_string()), // NOTE: this placeholder will be replaced from a mounted secret operator volume on container startup
+                Some(format!("${{env:{ENV_LDAP_BIND_PASSWORD}}}").to_string()),
             );
         }
 

From adc870f205d44becf97310da2df057452373b4bd Mon Sep 17 00:00:00 2001
From: Felix Hennig <felix.hennig@stackable.tech>
Date: Thu, 6 Jun 2024 11:07:58 +0200
Subject: [PATCH 05/23] simplify and add "

---
 rust/crd/src/authentication/ldap.rs | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/rust/crd/src/authentication/ldap.rs b/rust/crd/src/authentication/ldap.rs
index 2f4e5912..59401373 100644
--- a/rust/crd/src/authentication/ldap.rs
+++ b/rust/crd/src/authentication/ldap.rs
@@ -184,12 +184,11 @@ impl DruidLdapSettings {
         if let Some((ldap_bind_user_path, ldap_bind_password_path)) =
             self.ldap.bind_credentials_mount_paths()
         {
-            let ldap_bind_user = format!("$(cat {ldap_bind_user_path})");
-            let ldap_bind_password = format!("$(cat {ldap_bind_password_path})");
-
-            commands.push(format!("export {ENV_LDAP_BIND_USER}={ldap_bind_user}"));
             commands.push(format!(
-                "export {ENV_LDAP_BIND_PASSWORD}={ldap_bind_password}"
+                "export {ENV_LDAP_BIND_USER}=\"$(cat {ldap_bind_user_path})\""
+            ));
+            commands.push(format!(
+                "export {ENV_LDAP_BIND_PASSWORD}=\"$(cat {ldap_bind_password_path})\""
             ));
         }
 

From 34353672fcdd2138ee6874518162034d8b949d70 Mon Sep 17 00:00:00 2001
From: Felix Hennig <felix.hennig@stackable.tech>
Date: Thu, 6 Jun 2024 11:15:45 +0200
Subject: [PATCH 06/23] make ldap admin password more complicated

---
 .../kuttl/ldap-authentication/04-assert-ldap-user.yaml        | 4 ++--
 .../ldap-authentication/create-authentication-classes.yaml.j2 | 2 +-
 .../kuttl/ldap-authentication/install-openldap.yaml.j2        | 2 +-
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/tests/templates/kuttl/ldap-authentication/04-assert-ldap-user.yaml b/tests/templates/kuttl/ldap-authentication/04-assert-ldap-user.yaml
index 663a6f96..a0bf8cbf 100644
--- a/tests/templates/kuttl/ldap-authentication/04-assert-ldap-user.yaml
+++ b/tests/templates/kuttl/ldap-authentication/04-assert-ldap-user.yaml
@@ -2,5 +2,5 @@
 apiVersion: kuttl.dev/v1beta1
 kind: TestAssert
 commands:
-  - script: kubectl exec -n $NAMESPACE openldap-0 -- ldapsearch -H ldap://localhost:1389 -D cn=admin,dc=example,dc=org -w admin -b ou=users,dc=example,dc=org > /dev/null
-  - script: kubectl exec -n $NAMESPACE openldap-0 -- bash -c LDAPTLS_CACERT=/tls/ca.crt ldapsearch -Z -H ldaps://localhost:1636 -D cn=admin,dc=example,dc=org -w admin -b ou=users,dc=example,dc=org > /dev/null
+  - script: kubectl exec -n $NAMESPACE openldap-0 -- ldapsearch -H ldap://localhost:1389 -D cn=admin,dc=example,dc=org -w "admin-pw withSpace$%\" &&} §" -b ou=users,dc=example,dc=org > /dev/null
+  - script: kubectl exec -n $NAMESPACE openldap-0 -- bash -c LDAPTLS_CACERT=/tls/ca.crt ldapsearch -Z -H ldaps://localhost:1636 -D cn=admin,dc=example,dc=org -w "admin-pw withSpace$%\" &&} §" -b ou=users,dc=example,dc=org > /dev/null
diff --git a/tests/templates/kuttl/ldap-authentication/create-authentication-classes.yaml.j2 b/tests/templates/kuttl/ldap-authentication/create-authentication-classes.yaml.j2
index f41d2fa8..8fe5c3ee 100644
--- a/tests/templates/kuttl/ldap-authentication/create-authentication-classes.yaml.j2
+++ b/tests/templates/kuttl/ldap-authentication/create-authentication-classes.yaml.j2
@@ -46,4 +46,4 @@ metadata:
     secrets.stackable.tech/class: druid-ldap-secret
 stringData:
   user: cn=admin,dc=example,dc=org
-  password: admin
+  password: "admin-pw withSpace$%\" &&} §"
diff --git a/tests/templates/kuttl/ldap-authentication/install-openldap.yaml.j2 b/tests/templates/kuttl/ldap-authentication/install-openldap.yaml.j2
index 0561e4a1..a952a68d 100644
--- a/tests/templates/kuttl/ldap-authentication/install-openldap.yaml.j2
+++ b/tests/templates/kuttl/ldap-authentication/install-openldap.yaml.j2
@@ -44,7 +44,7 @@ spec:
             - name: LDAP_ADMIN_USERNAME
               value: admin
             - name: LDAP_ADMIN_PASSWORD
-              value: admin
+              value: "admin-pw withSpace$%\" &&} §"  # use spaces and special characters to make sure it doesn't break anything
             - name: LDAP_ENABLE_TLS
               value: \"yes\"
             - name: LDAP_TLS_CERT_FILE

From 9144aa2f9155c3a3833e4cd8544a0b249b27a2d5 Mon Sep 17 00:00:00 2001
From: Felix Hennig <felix.hennig@stackable.tech>
Date: Thu, 6 Jun 2024 13:47:27 +0200
Subject: [PATCH 07/23] use the feature to load secret directly from file

---
 rust/crd/src/authentication/ldap.rs          | 28 ++++----------------
 rust/operator-binary/src/druid_controller.rs |  1 -
 2 files changed, 5 insertions(+), 24 deletions(-)

diff --git a/rust/crd/src/authentication/ldap.rs b/rust/crd/src/authentication/ldap.rs
index 59401373..24a1a4fc 100644
--- a/rust/crd/src/authentication/ldap.rs
+++ b/rust/crd/src/authentication/ldap.rs
@@ -25,9 +25,6 @@ pub struct DruidLdapSettings {
     pub authentication_class_name: String,
 }
 
-pub const ENV_LDAP_BIND_USER: &str = "LDAP_BIND_USER";
-pub const ENV_LDAP_BIND_PASSWORD: &str = "LDAP_BIND_PASSWORD";
-
 impl DruidLdapSettings {
     pub fn new_from(
         resolved_authentication_config: &ResolvedAuthenticationClasses,
@@ -95,14 +92,16 @@ impl DruidLdapSettings {
             ),
         );
 
-        if self.ldap.bind_credentials_mount_paths().is_some() {
+        if let Some((ldap_bind_user_path, ldap_bind_password_path)) =
+            self.ldap.bind_credentials_mount_paths()
+        {
             config.insert(
                 format!("{PREFIX}.credentialsValidator.bindUser"),
-                Some(format!("${{env:{ENV_LDAP_BIND_USER}}}").to_string()),
+                Some(format!("${{file:UTF-8:{ldap_bind_user_path}}}").to_string()),
             );
             config.insert(
                 format!("{PREFIX}.credentialsValidator.bindPassword"),
-                Some(format!("${{env:{ENV_LDAP_BIND_PASSWORD}}}").to_string()),
+                Some(format!("${{file:UTF-8:{ldap_bind_password_path}}}").to_string()),
             );
         }
 
@@ -178,23 +177,6 @@ impl DruidLdapSettings {
         Ok(config)
     }
 
-    pub fn main_container_commands(&self) -> Vec<String> {
-        let mut commands = Vec::new();
-
-        if let Some((ldap_bind_user_path, ldap_bind_password_path)) =
-            self.ldap.bind_credentials_mount_paths()
-        {
-            commands.push(format!(
-                "export {ENV_LDAP_BIND_USER}=\"$(cat {ldap_bind_user_path})\""
-            ));
-            commands.push(format!(
-                "export {ENV_LDAP_BIND_PASSWORD}=\"$(cat {ldap_bind_password_path})\""
-            ));
-        }
-
-        commands
-    }
-
     pub fn prepare_container_commands(&self) -> Vec<String> {
         let mut command = vec![];
         if let Some(tls_ca_cert_mount_path) = self.ldap.tls.tls_ca_cert_mount_path() {
diff --git a/rust/operator-binary/src/druid_controller.rs b/rust/operator-binary/src/druid_controller.rs
index f8196355..ba92a8b6 100644
--- a/rust/operator-binary/src/druid_controller.rs
+++ b/rust/operator-binary/src/druid_controller.rs
@@ -948,7 +948,6 @@ fn build_rolegroup_statefulset(
             .context(AddLdapVolumesSnafu)?;
 
         prepare_container_commands.extend(ldap_settings.prepare_container_commands());
-        main_container_commands.extend(ldap_settings.main_container_commands());
     }
 
     // volume and volume mounts

From 6f8f24177e2a83228ff9a729bd6072b87c950c31 Mon Sep 17 00:00:00 2001
From: Felix Hennig <felix.hennig@stackable.tech>
Date: Thu, 6 Jun 2024 14:01:09 +0200
Subject: [PATCH 08/23] Load S3 keys from file instead of from env

---
 rust/crd/src/lib.rs                          | 13 ++++---------
 rust/operator-binary/src/druid_controller.rs | 20 ++++++++++++++++++--
 2 files changed, 22 insertions(+), 11 deletions(-)

diff --git a/rust/crd/src/lib.rs b/rust/crd/src/lib.rs
index 98cb13c2..0ccbe3aa 100644
--- a/rust/crd/src/lib.rs
+++ b/rust/crd/src/lib.rs
@@ -93,6 +93,8 @@ pub const DS_DIRECTORY: &str = "druid.storage.storageDirectory";
 pub const DS_BUCKET: &str = "druid.storage.bucket";
 pub const DS_BASE_KEY: &str = "druid.storage.baseKey";
 pub const S3_ENDPOINT_URL: &str = "druid.s3.endpoint.url";
+pub const S3_ACCESS_KEY: &str = "druid.s3.accessKey";
+pub const S3_SECRET_KEY: &str = "druid.s3.secretKey";
 pub const S3_PATH_STYLE_ACCESS: &str = "druid.s3.enablePathStyleAccess";
 // OPA
 pub const AUTH_AUTHORIZERS: &str = "druid.auth.authorizers";
@@ -125,10 +127,8 @@ pub const PROMETHEUS_PORT: &str = "druid.emitter.prometheus.port";
 pub const METRICS_PORT: u16 = 9090;
 // container locations
 pub const S3_SECRET_DIR_NAME: &str = "/stackable/secrets";
-const ENV_S3_ACCESS_KEY: &str = "AWS_ACCESS_KEY_ID";
-const ENV_S3_SECRET_KEY: &str = "AWS_SECRET_ACCESS_KEY";
-const SECRET_KEY_S3_ACCESS_KEY: &str = "accessKey";
-const SECRET_KEY_S3_SECRET_KEY: &str = "secretKey";
+pub const SECRET_KEY_S3_ACCESS_KEY: &str = "accessKey";
+pub const SECRET_KEY_S3_SECRET_KEY: &str = "secretKey";
 // segment storage
 pub const SC_LOCATIONS: &str = "druid.segmentCache.locations";
 pub const SC_DIRECTORY: &str = "/stackable/var/druid/segment-cache";
@@ -529,11 +529,6 @@ impl DruidRole {
             {
                 commands.push(format!("keytool -importcert -file {CERTS_DIR}/{secret_class}-tls-certificate/ca.crt -alias stackable-{secret_class} -keystore {STACKABLE_TRUST_STORE} -storepass {STACKABLE_TRUST_STORE_PASSWORD} -noprompt"));
             }
-
-            if s3_connection.credentials.is_some() {
-                commands.push(format!("export {ENV_S3_ACCESS_KEY}=$(cat {S3_SECRET_DIR_NAME}/{SECRET_KEY_S3_ACCESS_KEY})"));
-                commands.push(format!("export {ENV_S3_SECRET_KEY}=$(cat {S3_SECRET_DIR_NAME}/{SECRET_KEY_S3_SECRET_KEY})"));
-            }
         }
 
         // copy druid config to rw config
diff --git a/rust/operator-binary/src/druid_controller.rs b/rust/operator-binary/src/druid_controller.rs
index ba92a8b6..a9a81f0f 100644
--- a/rust/operator-binary/src/druid_controller.rs
+++ b/rust/operator-binary/src/druid_controller.rs
@@ -21,8 +21,9 @@ use stackable_druid_crd::{
     APP_NAME, AUTH_AUTHORIZER_OPA_URI, CERTS_DIR, CREDENTIALS_SECRET_PROPERTY, DB_PASSWORD_ENV,
     DB_USERNAME_ENV, DRUID_CONFIG_DIRECTORY, DS_BUCKET, ENV_INTERNAL_SECRET, EXTENSIONS_LOADLIST,
     HDFS_CONFIG_DIRECTORY, JVM_CONFIG, JVM_SECURITY_PROPERTIES_FILE, LOG_CONFIG_DIRECTORY, LOG_DIR,
-    MAX_DRUID_LOG_FILES_SIZE, RUNTIME_PROPS, RW_CONFIG_DIRECTORY, S3_ENDPOINT_URL,
-    S3_PATH_STYLE_ACCESS, S3_SECRET_DIR_NAME, ZOOKEEPER_CONNECTION_STRING,
+    MAX_DRUID_LOG_FILES_SIZE, RUNTIME_PROPS, RW_CONFIG_DIRECTORY, S3_ACCESS_KEY, S3_ENDPOINT_URL,
+    S3_PATH_STYLE_ACCESS, S3_SECRET_DIR_NAME, S3_SECRET_KEY, SECRET_KEY_S3_ACCESS_KEY,
+    SECRET_KEY_S3_SECRET_KEY, ZOOKEEPER_CONNECTION_STRING,
 };
 use stackable_operator::{
     builder::{
@@ -711,6 +712,21 @@ fn build_rolegroup_config_map(
                         conf.insert(S3_ENDPOINT_URL.to_string(), Some(endpoint));
                     }
 
+                    if conn.credentials.is_some() {
+                        conf.insert(
+                            S3_ACCESS_KEY.to_string(),
+                            Some(format!(
+                                "${{file:UTF-8:{S3_SECRET_DIR_NAME}/{SECRET_KEY_S3_ACCESS_KEY}}}"
+                            )),
+                        );
+                        conf.insert(
+                            S3_SECRET_KEY.to_string(),
+                            Some(format!(
+                                "${{file:UTF-8:{S3_SECRET_DIR_NAME}/{SECRET_KEY_S3_SECRET_KEY}]]"
+                            )),
+                        );
+                    }
+
                     // We did choose a match statement here to detect new access styles in the future
                     let path_style_access = match conn.access_style.clone().unwrap_or_default() {
                         S3AccessStyle::Path => true,

From 603e24c2474445e8adcf245a506509e6a5f69498 Mon Sep 17 00:00:00 2001
From: Sebastian Bernauer <sebastian.bernauer@stackable.tech>
Date: Tue, 25 Jun 2024 08:26:36 +0200
Subject: [PATCH 09/23] Split INTERNAL_INITIAL_CLIENT_PASSWORD_ENV and
 ESCALATOR_INTERNAL_CLIENT_PASSWORD_ENV

---
 rust/crd/src/authentication/ldap.rs          | 10 +++++-----
 rust/crd/src/lib.rs                          |  7 +++++--
 rust/crd/src/security.rs                     |  4 ++++
 rust/operator-binary/src/druid_controller.rs | 14 ++++++++++----
 rust/operator-binary/src/internal_secret.rs  |  8 ++++++--
 5 files changed, 30 insertions(+), 13 deletions(-)

diff --git a/rust/crd/src/authentication/ldap.rs b/rust/crd/src/authentication/ldap.rs
index 24a1a4fc..65a07838 100644
--- a/rust/crd/src/authentication/ldap.rs
+++ b/rust/crd/src/authentication/ldap.rs
@@ -6,9 +6,9 @@ use stackable_operator::commons::authentication::AuthenticationClassProvider;
 use stackable_operator::kube::ResourceExt;
 
 use crate::authentication::ResolvedAuthenticationClasses;
-use crate::{
-    security::{add_cert_to_trust_store_cmd, STACKABLE_TLS_DIR, TLS_STORE_PASSWORD},
-    ENV_INTERNAL_SECRET,
+use crate::security::{
+    add_cert_to_trust_store_cmd, ESCALATOR_INTERNAL_CLIENT_PASSWORD_ENV,
+    INTERNAL_INITIAL_CLIENT_PASSWORD_ENV, STACKABLE_TLS_DIR, TLS_STORE_PASSWORD,
 };
 
 #[derive(Snafu, Debug)]
@@ -58,7 +58,7 @@ impl DruidLdapSettings {
 
         config.insert(
             format!("{PREFIX}.initialInternalClientPassword"),
-            Some(format!("${{env:{ENV_INTERNAL_SECRET}}}").to_string()),
+            Some(format!("${{env:{INTERNAL_INITIAL_CLIENT_PASSWORD_ENV}}}").to_string()),
         );
         config.insert(
             format!("{PREFIX}.authorizerName"),
@@ -136,7 +136,7 @@ impl DruidLdapSettings {
         );
         config.insert(
             "druid.escalator.internalClientPassword".to_string(),
-            Some(format!("${{env:{ENV_INTERNAL_SECRET}}}").to_string()),
+            Some(format!("${{env:{ESCALATOR_INTERNAL_CLIENT_PASSWORD_ENV}}}").to_string()),
         );
         config.insert(
             "druid.escalator.authorizerName".to_string(),
diff --git a/rust/crd/src/lib.rs b/rust/crd/src/lib.rs
index 0ccbe3aa..42e8ddb4 100644
--- a/rust/crd/src/lib.rs
+++ b/rust/crd/src/lib.rs
@@ -134,8 +134,6 @@ pub const SC_LOCATIONS: &str = "druid.segmentCache.locations";
 pub const SC_DIRECTORY: &str = "/stackable/var/druid/segment-cache";
 pub const SC_VOLUME_NAME: &str = "segment-cache";
 
-pub const ENV_INTERNAL_SECRET: &str = "INTERNAL_SECRET";
-
 // DB credentials - both of these are read from an env var by Druid with the ${env:...} syntax
 pub const DB_USERNAME_ENV: &str = "DB_USERNAME_ENV";
 pub const DB_PASSWORD_ENV: &str = "DB_PASSWORD_ENV";
@@ -156,16 +154,21 @@ pub enum Error {
     ResolveS3Connection {
         source: stackable_operator::commons::s3::Error,
     },
+
     #[snafu(display("failed to resolve S3 bucket"))]
     ResolveS3Bucket {
         source: stackable_operator::commons::s3::Error,
     },
+
     #[snafu(display("2 differing s3 connections were given, this is unsupported by Druid"))]
     IncompatibleS3Connections,
+
     #[snafu(display("the role group {rolegroup_name} is not defined"))]
     CannotRetrieveRoleGroup { rolegroup_name: String },
+
     #[snafu(display("missing namespace for resource {name}"))]
     MissingNamespace { name: String },
+
     #[snafu(display("fragment validation failure"))]
     FragmentValidationFailure { source: ValidationError },
 }
diff --git a/rust/crd/src/security.rs b/rust/crd/src/security.rs
index be8b6bab..20b30065 100644
--- a/rust/crd/src/security.rs
+++ b/rust/crd/src/security.rs
@@ -101,6 +101,10 @@ pub const STACKABLE_TLS_DIR: &str = "/stackable/tls";
 const TLS_VOLUME_NAME: &str = "tls";
 const TLS_MOUNT_VOLUME_NAME: &str = "tls-mount";
 
+pub const INTERNAL_INITIAL_CLIENT_PASSWORD_ENV: &str = "INTERNAL_INITAL_CLIENT_PASSWORD";
+// Let's just re-use the secret we generated above.
+pub const ESCALATOR_INTERNAL_CLIENT_PASSWORD_ENV: &str = INTERNAL_INITIAL_CLIENT_PASSWORD_ENV;
+
 impl DruidTlsSecurity {
     pub fn new(
         resolved_authentication_classes: ResolvedAuthenticationClasses,
diff --git a/rust/operator-binary/src/druid_controller.rs b/rust/operator-binary/src/druid_controller.rs
index a9a81f0f..c3105ff2 100644
--- a/rust/operator-binary/src/druid_controller.rs
+++ b/rust/operator-binary/src/druid_controller.rs
@@ -16,11 +16,13 @@ use stackable_druid_crd::{
     authentication::ldap::DruidLdapSettings,
     authorization::DruidAuthorization,
     build_recommended_labels, build_string_list,
-    security::{resolve_authentication_classes, DruidTlsSecurity},
+    security::{
+        resolve_authentication_classes, DruidTlsSecurity, INTERNAL_INITIAL_CLIENT_PASSWORD_ENV,
+    },
     CommonRoleGroupConfig, Container, DeepStorageSpec, DruidCluster, DruidClusterStatus, DruidRole,
     APP_NAME, AUTH_AUTHORIZER_OPA_URI, CERTS_DIR, CREDENTIALS_SECRET_PROPERTY, DB_PASSWORD_ENV,
-    DB_USERNAME_ENV, DRUID_CONFIG_DIRECTORY, DS_BUCKET, ENV_INTERNAL_SECRET, EXTENSIONS_LOADLIST,
-    HDFS_CONFIG_DIRECTORY, JVM_CONFIG, JVM_SECURITY_PROPERTIES_FILE, LOG_CONFIG_DIRECTORY, LOG_DIR,
+    DB_USERNAME_ENV, DRUID_CONFIG_DIRECTORY, DS_BUCKET, EXTENSIONS_LOADLIST, HDFS_CONFIG_DIRECTORY,
+    JVM_CONFIG, JVM_SECURITY_PROPERTIES_FILE, LOG_CONFIG_DIRECTORY, LOG_DIR,
     MAX_DRUID_LOG_FILES_SIZE, RUNTIME_PROPS, RW_CONFIG_DIRECTORY, S3_ACCESS_KEY, S3_ENDPOINT_URL,
     S3_PATH_STYLE_ACCESS, S3_SECRET_DIR_NAME, S3_SECRET_KEY, SECRET_KEY_S3_ACCESS_KEY,
     SECRET_KEY_S3_SECRET_KEY, ZOOKEEPER_CONNECTION_STRING,
@@ -1021,7 +1023,11 @@ fn build_rolegroup_statefulset(
         .collect::<Vec<_>>();
 
     let secret_name = build_shared_internal_secret_name(druid);
-    rest_env.push(env_var_from_secret(&secret_name, None, ENV_INTERNAL_SECRET));
+    rest_env.push(env_var_from_secret(
+        &secret_name,
+        None,
+        INTERNAL_INITIAL_CLIENT_PASSWORD_ENV,
+    ));
 
     // load database credentials to environment variables: these will be used to replace
     // the placeholders in runtime.properties so that the operator does not "touch" the secret.
diff --git a/rust/operator-binary/src/internal_secret.rs b/rust/operator-binary/src/internal_secret.rs
index a47471c6..51ec3808 100644
--- a/rust/operator-binary/src/internal_secret.rs
+++ b/rust/operator-binary/src/internal_secret.rs
@@ -1,5 +1,6 @@
 use snafu::{OptionExt, ResultExt, Snafu};
-use stackable_druid_crd::{DruidCluster, ENV_INTERNAL_SECRET};
+use stackable_druid_crd::security::INTERNAL_INITIAL_CLIENT_PASSWORD_ENV;
+use stackable_druid_crd::DruidCluster;
 use stackable_operator::k8s_openapi::api::core::v1::{EnvVar, EnvVarSource, SecretKeySelector};
 use stackable_operator::kube::ResourceExt;
 use stackable_operator::{
@@ -57,7 +58,10 @@ pub async fn create_shared_internal_secret(
 
 pub fn build_shared_internal_secret(druid: &DruidCluster) -> Result<Secret, Error> {
     let mut internal_secret = BTreeMap::new();
-    internal_secret.insert(ENV_INTERNAL_SECRET.to_string(), get_random_base64());
+    internal_secret.insert(
+        INTERNAL_INITIAL_CLIENT_PASSWORD_ENV.to_string(),
+        get_random_base64(),
+    );
 
     Ok(Secret {
         immutable: Some(true),

From 550bd69123b5f02d9aa0d707e24d95149aadc7ee Mon Sep 17 00:00:00 2001
From: Sebastian Bernauer <sebastian.bernauer@stackable.tech>
Date: Tue, 25 Jun 2024 08:32:39 +0200
Subject: [PATCH 10/23] fix typo

---
 rust/crd/src/lib.rs                          | 2 ++
 rust/operator-binary/src/druid_controller.rs | 2 +-
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/rust/crd/src/lib.rs b/rust/crd/src/lib.rs
index 42e8ddb4..16a2c46e 100644
--- a/rust/crd/src/lib.rs
+++ b/rust/crd/src/lib.rs
@@ -555,6 +555,8 @@ impl DruidRole {
             rw_conf = RW_CONFIG_DIRECTORY,
         ));
 
+        // TODO: Run config-utils
+
         commands
     }
 
diff --git a/rust/operator-binary/src/druid_controller.rs b/rust/operator-binary/src/druid_controller.rs
index c3105ff2..903f9f97 100644
--- a/rust/operator-binary/src/druid_controller.rs
+++ b/rust/operator-binary/src/druid_controller.rs
@@ -724,7 +724,7 @@ fn build_rolegroup_config_map(
                         conf.insert(
                             S3_SECRET_KEY.to_string(),
                             Some(format!(
-                                "${{file:UTF-8:{S3_SECRET_DIR_NAME}/{SECRET_KEY_S3_SECRET_KEY}]]"
+                                "${{file:UTF-8:{S3_SECRET_DIR_NAME}/{SECRET_KEY_S3_SECRET_KEY}}}"
                             )),
                         );
                     }

From 88fb6dcdc6eeff138b942c20033a48866d7e45e5 Mon Sep 17 00:00:00 2001
From: Sebastian Bernauer <sebastian.bernauer@stackable.tech>
Date: Tue, 25 Jun 2024 09:14:05 +0200
Subject: [PATCH 11/23] fix: Re-create internal secret if needed

---
 rust/operator-binary/src/internal_secret.rs | 17 +++++++++++++++--
 1 file changed, 15 insertions(+), 2 deletions(-)

diff --git a/rust/operator-binary/src/internal_secret.rs b/rust/operator-binary/src/internal_secret.rs
index 51ec3808..1fd31544 100644
--- a/rust/operator-binary/src/internal_secret.rs
+++ b/rust/operator-binary/src/internal_secret.rs
@@ -35,7 +35,7 @@ pub async fn create_shared_internal_secret(
     controller_name: &str,
 ) -> Result<(), Error> {
     let secret = build_shared_internal_secret(druid)?;
-    if client
+    if !client
         .get_opt::<Secret>(
             &secret.name_any(),
             secret
@@ -45,12 +45,25 @@ pub async fn create_shared_internal_secret(
         )
         .await
         .context(FailedToRetrieveInternalSecretSnafu)?
-        .is_none()
+        .and_then(|s| s.data)
+        // It might be the case that the Secret already exists but has the wrong content, so let's check that the
+        // required keys are there
+        .map(|d| d.contains_key(INTERNAL_INITIAL_CLIENT_PASSWORD_ENV))
+        .unwrap_or_default()
     {
+        tracing::info!(
+            secret_name = secret.name_any(),
+            "Did not found a shared internal secret with the necesarry data, creating on"
+        );
         client
             .apply_patch(controller_name, &secret, &secret)
             .await
             .context(ApplyInternalSecretSnafu)?;
+    } else {
+        tracing::debug!(
+            secret_name = secret.name_any(),
+            "Found existing shared internal secret which contains the data I'd expect"
+        );
     }
 
     Ok(())

From 755a97192e3ab633d6f6b6d53672acb24ded4db8 Mon Sep 17 00:00:00 2001
From: Sebastian Bernauer <sebastian.bernauer@stackable.tech>
Date: Tue, 25 Jun 2024 09:21:48 +0200
Subject: [PATCH 12/23] typo

---
 rust/crd/src/security.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/rust/crd/src/security.rs b/rust/crd/src/security.rs
index 20b30065..00710d17 100644
--- a/rust/crd/src/security.rs
+++ b/rust/crd/src/security.rs
@@ -101,7 +101,7 @@ pub const STACKABLE_TLS_DIR: &str = "/stackable/tls";
 const TLS_VOLUME_NAME: &str = "tls";
 const TLS_MOUNT_VOLUME_NAME: &str = "tls-mount";
 
-pub const INTERNAL_INITIAL_CLIENT_PASSWORD_ENV: &str = "INTERNAL_INITAL_CLIENT_PASSWORD";
+pub const INTERNAL_INITIAL_CLIENT_PASSWORD_ENV: &str = "INTERNAL_INITIAL_CLIENT_PASSWORD";
 // Let's just re-use the secret we generated above.
 pub const ESCALATOR_INTERNAL_CLIENT_PASSWORD_ENV: &str = INTERNAL_INITIAL_CLIENT_PASSWORD_ENV;
 

From dc7b78348649ffac40975c2f783d27f2b54a6689 Mon Sep 17 00:00:00 2001
From: Sebastian Bernauer <sebastian.bernauer@stackable.tech>
Date: Tue, 25 Jun 2024 10:30:02 +0200
Subject: [PATCH 13/23] Improve internal secret management

---
 rust/crd/src/security.rs                    |   2 +-
 rust/operator-binary/src/internal_secret.rs | 103 +++++++++++++++-----
 2 files changed, 81 insertions(+), 24 deletions(-)

diff --git a/rust/crd/src/security.rs b/rust/crd/src/security.rs
index 00710d17..2b88019e 100644
--- a/rust/crd/src/security.rs
+++ b/rust/crd/src/security.rs
@@ -102,7 +102,7 @@ const TLS_VOLUME_NAME: &str = "tls";
 const TLS_MOUNT_VOLUME_NAME: &str = "tls-mount";
 
 pub const INTERNAL_INITIAL_CLIENT_PASSWORD_ENV: &str = "INTERNAL_INITIAL_CLIENT_PASSWORD";
-// Let's just re-use the secret we generated above.
+// It seems this needs to be the same password for Druid to work, so we re-use the existing env variable from above.
 pub const ESCALATOR_INTERNAL_CLIENT_PASSWORD_ENV: &str = INTERNAL_INITIAL_CLIENT_PASSWORD_ENV;
 
 impl DruidTlsSecurity {
diff --git a/rust/operator-binary/src/internal_secret.rs b/rust/operator-binary/src/internal_secret.rs
index 1fd31544..3260bf8f 100644
--- a/rust/operator-binary/src/internal_secret.rs
+++ b/rust/operator-binary/src/internal_secret.rs
@@ -6,7 +6,7 @@ use stackable_operator::kube::ResourceExt;
 use stackable_operator::{
     builder::meta::ObjectMetaBuilder, client::Client, k8s_openapi::api::core::v1::Secret,
 };
-use std::collections::BTreeMap;
+use std::collections::{BTreeMap, HashSet};
 use strum::{EnumDiscriminants, IntoStaticStr};
 
 #[derive(Snafu, Debug, EnumDiscriminants)]
@@ -17,12 +17,20 @@ pub enum Error {
     ApplyInternalSecret {
         source: stackable_operator::client::Error,
     },
+
+    #[snafu(display("failed to delete internal secret"))]
+    DeleteInternalSecret {
+        source: stackable_operator::client::Error,
+    },
+
     #[snafu(display("failed to retrieve secret for internal communications"))]
     FailedToRetrieveInternalSecret {
         source: stackable_operator::client::Error,
     },
+
     #[snafu(display("object defines no namespace"))]
     ObjectHasNoNamespace,
+
     #[snafu(display("object is missing metadata to build owner reference"))]
     ObjectMissingMetadataForOwnerRef {
         source: stackable_operator::builder::meta::Error,
@@ -35,7 +43,7 @@ pub async fn create_shared_internal_secret(
     controller_name: &str,
 ) -> Result<(), Error> {
     let secret = build_shared_internal_secret(druid)?;
-    if !client
+    let existing_secret = client
         .get_opt::<Secret>(
             &secret.name_any(),
             secret
@@ -44,26 +52,76 @@ pub async fn create_shared_internal_secret(
                 .context(ObjectHasNoNamespaceSnafu)?,
         )
         .await
-        .context(FailedToRetrieveInternalSecretSnafu)?
-        .and_then(|s| s.data)
-        // It might be the case that the Secret already exists but has the wrong content, so let's check that the
-        // required keys are there
-        .map(|d| d.contains_key(INTERNAL_INITIAL_CLIENT_PASSWORD_ENV))
-        .unwrap_or_default()
-    {
-        tracing::info!(
-            secret_name = secret.name_any(),
-            "Did not found a shared internal secret with the necesarry data, creating on"
-        );
-        client
-            .apply_patch(controller_name, &secret, &secret)
-            .await
-            .context(ApplyInternalSecretSnafu)?;
-    } else {
-        tracing::debug!(
-            secret_name = secret.name_any(),
-            "Found existing shared internal secret which contains the data I'd expect"
-        );
+        .context(FailedToRetrieveInternalSecretSnafu)?;
+
+    match existing_secret {
+        None => {
+            tracing::info!(
+                secret_name = secret.name_any(),
+                "Did not found a shared internal secret with the necesarry data, creating one"
+            );
+            client
+                .apply_patch(controller_name, &secret, &secret)
+                .await
+                .context(ApplyInternalSecretSnafu)?;
+        }
+        Some(existing_secret) => {
+            if existing_secret.immutable == Some(true) {
+                // Before 2024-06-25 we did set `spec.immutable` to avoid accidentially chaning the contents. Which was
+                // great back than, *but* we now need something more flexible. AFAIK we can not make the Secret mutable,
+                // so there seems to be no other way than to re-create it. We *could* read in the contents and use them
+                // during the re-creation (so we don't change the contents to avoid downtime), but we strive that our
+                // operators don't read Secret contents and it's a one time migration thing.
+
+                tracing::warn!(
+                    secret_name = secret.name_any(),
+                    "Shared internal secret found, which is immutable. Re-creating it, as we can not modify it. This \
+                    should only happen once and will change the contents of the Secret. This might cause a short \
+                    downtime of Druid, as the changed internal Secrets need to propagate through all Druid nodes"
+                );
+
+                client
+                    .delete(&secret)
+                    .await
+                    .context(DeleteInternalSecretSnafu)?;
+
+                client
+                    .apply_patch(controller_name, &secret, &secret)
+                    .await
+                    .context(ApplyInternalSecretSnafu)?;
+                return Ok(());
+            }
+
+            let current_secret_keys = existing_secret
+                .data
+                .unwrap_or_default()
+                .into_keys()
+                .collect::<HashSet<_>>();
+            for required in secret
+                .string_data
+                .as_ref()
+                .expect("Secret data must be set by the `build_shared_internal_secret` function")
+                .keys()
+            {
+                if !current_secret_keys.contains(required) {
+                    tracing::info!(
+                        secret_name = secret.name_any(),
+                        "Found shared internal secret, which is missing the key {required}, patching it"
+                    );
+                    tracing::warn!(
+                        secret_name = secret.name_any(),
+                        "Found shared internal secret, which is missing the key {required}, patching it. This \
+                        should only happen once and will change the contents of the Secret. This might cause a short \
+                        downtime of Druid, as the changed internal Secrets need to propagate through all Druid nodes"
+                    );
+                    client
+                        .apply_patch(controller_name, &secret, &secret)
+                        .await
+                        .context(ApplyInternalSecretSnafu)?;
+                    return Ok(());
+                }
+            }
+        }
     }
 
     Ok(())
@@ -77,7 +135,6 @@ pub fn build_shared_internal_secret(druid: &DruidCluster) -> Result<Secret, Erro
     );
 
     Ok(Secret {
-        immutable: Some(true),
         metadata: ObjectMetaBuilder::new()
             .name(build_shared_internal_secret_name(druid))
             .namespace_opt(druid.namespace())

From 6fcd7dc3c9c3c249b72dad4a2e79349d2cb7974b Mon Sep 17 00:00:00 2001
From: Sebastian Bernauer <sebastian.bernauer@stackable.tech>
Date: Tue, 25 Jun 2024 10:33:16 +0200
Subject: [PATCH 14/23] typo

---
 rust/operator-binary/src/internal_secret.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/rust/operator-binary/src/internal_secret.rs b/rust/operator-binary/src/internal_secret.rs
index 3260bf8f..fae3248d 100644
--- a/rust/operator-binary/src/internal_secret.rs
+++ b/rust/operator-binary/src/internal_secret.rs
@@ -58,7 +58,7 @@ pub async fn create_shared_internal_secret(
         None => {
             tracing::info!(
                 secret_name = secret.name_any(),
-                "Did not found a shared internal secret with the necesarry data, creating one"
+                "Did not found a shared internal secret with the necessary data, creating one"
             );
             client
                 .apply_patch(controller_name, &secret, &secret)
@@ -67,7 +67,7 @@ pub async fn create_shared_internal_secret(
         }
         Some(existing_secret) => {
             if existing_secret.immutable == Some(true) {
-                // Before 2024-06-25 we did set `spec.immutable` to avoid accidentially chaning the contents. Which was
+                // Before 2024-06-25 we did set `spec.immutable` to avoid accidentally changing the contents. Which was
                 // great back than, *but* we now need something more flexible. AFAIK we can not make the Secret mutable,
                 // so there seems to be no other way than to re-create it. We *could* read in the contents and use them
                 // during the re-creation (so we don't change the contents to avoid downtime), but we strive that our

From 55aff2578b606b6b049e2e38a18acd3696984f89 Mon Sep 17 00:00:00 2001
From: Sebastian Bernauer <sebastian.bernauer@stackable.tech>
Date: Tue, 25 Jun 2024 10:36:34 +0200
Subject: [PATCH 15/23] typo

---
 rust/operator-binary/src/internal_secret.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/rust/operator-binary/src/internal_secret.rs b/rust/operator-binary/src/internal_secret.rs
index fae3248d..a2d050ee 100644
--- a/rust/operator-binary/src/internal_secret.rs
+++ b/rust/operator-binary/src/internal_secret.rs
@@ -71,7 +71,7 @@ pub async fn create_shared_internal_secret(
                 // great back than, *but* we now need something more flexible. AFAIK we can not make the Secret mutable,
                 // so there seems to be no other way than to re-create it. We *could* read in the contents and use them
                 // during the re-creation (so we don't change the contents to avoid downtime), but we strive that our
-                // operators don't read Secret contents and it's a one time migration thing.
+                // operators don't handle Secret contents and it's a one time migration thing.
 
                 tracing::warn!(
                     secret_name = secret.name_any(),

From 5cb553dedbb8780926b017bc329da4c8b434b11b Mon Sep 17 00:00:00 2001
From: Sebastian Bernauer <sebastian.bernauer@stackable.tech>
Date: Tue, 25 Jun 2024 10:41:10 +0200
Subject: [PATCH 16/23] revert test changes

---
 tests/templates/kuttl/ldap-authentication/03_openldap.yaml.j2 | 2 +-
 .../kuttl/ldap-authentication/04-assert-ldap-user.yaml        | 4 ++--
 .../kuttl/ldap-authentication/05_authentication-class.yaml.j2 | 2 +-
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/tests/templates/kuttl/ldap-authentication/03_openldap.yaml.j2 b/tests/templates/kuttl/ldap-authentication/03_openldap.yaml.j2
index a952a68d..0561e4a1 100644
--- a/tests/templates/kuttl/ldap-authentication/03_openldap.yaml.j2
+++ b/tests/templates/kuttl/ldap-authentication/03_openldap.yaml.j2
@@ -44,7 +44,7 @@ spec:
             - name: LDAP_ADMIN_USERNAME
               value: admin
             - name: LDAP_ADMIN_PASSWORD
-              value: "admin-pw withSpace$%\" &&} §"  # use spaces and special characters to make sure it doesn't break anything
+              value: admin
             - name: LDAP_ENABLE_TLS
               value: \"yes\"
             - name: LDAP_TLS_CERT_FILE
diff --git a/tests/templates/kuttl/ldap-authentication/04-assert-ldap-user.yaml b/tests/templates/kuttl/ldap-authentication/04-assert-ldap-user.yaml
index a0bf8cbf..663a6f96 100644
--- a/tests/templates/kuttl/ldap-authentication/04-assert-ldap-user.yaml
+++ b/tests/templates/kuttl/ldap-authentication/04-assert-ldap-user.yaml
@@ -2,5 +2,5 @@
 apiVersion: kuttl.dev/v1beta1
 kind: TestAssert
 commands:
-  - script: kubectl exec -n $NAMESPACE openldap-0 -- ldapsearch -H ldap://localhost:1389 -D cn=admin,dc=example,dc=org -w "admin-pw withSpace$%\" &&} §" -b ou=users,dc=example,dc=org > /dev/null
-  - script: kubectl exec -n $NAMESPACE openldap-0 -- bash -c LDAPTLS_CACERT=/tls/ca.crt ldapsearch -Z -H ldaps://localhost:1636 -D cn=admin,dc=example,dc=org -w "admin-pw withSpace$%\" &&} §" -b ou=users,dc=example,dc=org > /dev/null
+  - script: kubectl exec -n $NAMESPACE openldap-0 -- ldapsearch -H ldap://localhost:1389 -D cn=admin,dc=example,dc=org -w admin -b ou=users,dc=example,dc=org > /dev/null
+  - script: kubectl exec -n $NAMESPACE openldap-0 -- bash -c LDAPTLS_CACERT=/tls/ca.crt ldapsearch -Z -H ldaps://localhost:1636 -D cn=admin,dc=example,dc=org -w admin -b ou=users,dc=example,dc=org > /dev/null
diff --git a/tests/templates/kuttl/ldap-authentication/05_authentication-class.yaml.j2 b/tests/templates/kuttl/ldap-authentication/05_authentication-class.yaml.j2
index 8fe5c3ee..f41d2fa8 100644
--- a/tests/templates/kuttl/ldap-authentication/05_authentication-class.yaml.j2
+++ b/tests/templates/kuttl/ldap-authentication/05_authentication-class.yaml.j2
@@ -46,4 +46,4 @@ metadata:
     secrets.stackable.tech/class: druid-ldap-secret
 stringData:
   user: cn=admin,dc=example,dc=org
-  password: "admin-pw withSpace$%\" &&} §"
+  password: admin

From 46c7eaa8d81db5235014dba3ca40c45f030dc19d Mon Sep 17 00:00:00 2001
From: Sebastian Bernauer <sebastian.bernauer@stackable.tech>
Date: Tue, 25 Jun 2024 13:27:33 +0200
Subject: [PATCH 17/23] Actually run config-utils

---
 rust/crd/src/lib.rs | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/rust/crd/src/lib.rs b/rust/crd/src/lib.rs
index 16a2c46e..f36e2cd0 100644
--- a/rust/crd/src/lib.rs
+++ b/rust/crd/src/lib.rs
@@ -555,7 +555,11 @@ impl DruidRole {
             rw_conf = RW_CONFIG_DIRECTORY,
         ));
 
-        // TODO: Run config-utils
+        commands.extend([
+            format!("config-utils template {RW_CONFIG_DIRECTORY}/runtime.properties",),
+            format!("if test -f {RW_CONFIG_DIRECTORY}/core-site.xml; then config-utils template {RW_CONFIG_DIRECTORY}/core-site.xml; fi",),
+            format!("if test -f {RW_CONFIG_DIRECTORY}/hdfs-site.xml; then config-utils template {RW_CONFIG_DIRECTORY}/hdfs-site.xml; fi",),
+        ]);
 
         commands
     }

From 3e6ee68b384271412cc6aa1176ef8e96a28c15af Mon Sep 17 00:00:00 2001
From: Benedikt Labrenz <benedikt@labrenz.org>
Date: Tue, 25 Jun 2024 14:55:40 +0200
Subject: [PATCH 18/23] refactor ldap test

---
 .../04-assert-ldap-user.yaml                  |  6 ----
 .../kuttl/ldap-authentication/20-assert.yaml  |  6 ----
 .../ldap-authentication/create_ldap_user.sh   | 15 --------
 .../00-assert.yaml.j2                         |  0
 ...tor-aggregator-discovery-configmap.yaml.j2 |  0
 .../00-patch-ns.yaml.j2                       |  0
 .../01-assert.yaml}                           |  0
 .../01-install-openldap.yaml}                 |  2 +-
 .../01_openldap.yaml.j2}                      | 23 +++++-------
 tests/templates/kuttl/ldap/02-assert.yaml     |  6 ++++
 .../02-create-ldap-user.yaml}                 |  0
 .../11-assert.yaml => ldap/03-assert.yaml}    |  2 +-
 .../03-install-test-druid.yaml}               | 11 +++---
 .../01-assert.yaml => ldap/10-assert.yaml}    |  7 +---
 .../10-install-zk.yaml.j2}                    |  6 ++--
 .../11-create-authentication-classes.yaml.j2} |  4 +--
 .../11_authentication-classes.yaml.j2}        | 27 +++-----------
 .../02-assert.yaml => ldap/12-assert.yaml}    |  0
 .../12-install-hdfs.yaml.j2}                  |  0
 .../10-assert.yaml => ldap/13-assert.yaml}    |  0
 .../13-install-druid.yaml.j2}                 | 16 +++++++++
 tests/templates/kuttl/ldap/20-assert.yaml     |  6 ++++
 .../20-authcheck.yaml                         |  2 +-
 .../authcheck.py                              |  6 ++--
 .../templates/kuttl/ldap/create_ldap_user.sh  | 36 +++++++++++++++++++
 25 files changed, 97 insertions(+), 84 deletions(-)
 delete mode 100644 tests/templates/kuttl/ldap-authentication/04-assert-ldap-user.yaml
 delete mode 100644 tests/templates/kuttl/ldap-authentication/20-assert.yaml
 delete mode 100644 tests/templates/kuttl/ldap-authentication/create_ldap_user.sh
 rename tests/templates/kuttl/{ldap-authentication => ldap}/00-assert.yaml.j2 (100%)
 rename tests/templates/kuttl/{ldap-authentication => ldap}/00-install-vector-aggregator-discovery-configmap.yaml.j2 (100%)
 rename tests/templates/kuttl/{ldap-authentication => ldap}/00-patch-ns.yaml.j2 (100%)
 rename tests/templates/kuttl/{ldap-authentication/03-assert-openldap.yaml => ldap/01-assert.yaml} (100%)
 rename tests/templates/kuttl/{ldap-authentication/03-install-openldap.yaml => ldap/01-install-openldap.yaml} (63%)
 rename tests/templates/kuttl/{ldap-authentication/03_openldap.yaml.j2 => ldap/01_openldap.yaml.j2} (85%)
 create mode 100644 tests/templates/kuttl/ldap/02-assert.yaml
 rename tests/templates/kuttl/{ldap-authentication/04-ldap-user.yaml => ldap/02-create-ldap-user.yaml} (100%)
 rename tests/templates/kuttl/{ldap-authentication/11-assert.yaml => ldap/03-assert.yaml} (89%)
 rename tests/templates/kuttl/{ldap-authentication/11-checks-container.yaml => ldap/03-install-test-druid.yaml} (67%)
 rename tests/templates/kuttl/{ldap-authentication/01-assert.yaml => ldap/10-assert.yaml} (62%)
 rename tests/templates/kuttl/{ldap-authentication/01-install-zk.yaml.j2 => ldap/10-install-zk.yaml.j2} (93%)
 rename tests/templates/kuttl/{ldap-authentication/05-authentication-class.yaml.j2 => ldap/11-create-authentication-classes.yaml.j2} (56%)
 rename tests/templates/kuttl/{ldap-authentication/05_authentication-class.yaml.j2 => ldap/11_authentication-classes.yaml.j2} (51%)
 rename tests/templates/kuttl/{ldap-authentication/02-assert.yaml => ldap/12-assert.yaml} (100%)
 rename tests/templates/kuttl/{ldap-authentication/02-install-hdfs.yaml.j2 => ldap/12-install-hdfs.yaml.j2} (100%)
 rename tests/templates/kuttl/{ldap-authentication/10-assert.yaml => ldap/13-assert.yaml} (100%)
 rename tests/templates/kuttl/{ldap-authentication/10-install-druid.yaml.j2 => ldap/13-install-druid.yaml.j2} (77%)
 create mode 100644 tests/templates/kuttl/ldap/20-assert.yaml
 rename tests/templates/kuttl/{ldap-authentication => ldap}/20-authcheck.yaml (50%)
 rename tests/templates/kuttl/{ldap-authentication => ldap}/authcheck.py (85%)
 create mode 100755 tests/templates/kuttl/ldap/create_ldap_user.sh

diff --git a/tests/templates/kuttl/ldap-authentication/04-assert-ldap-user.yaml b/tests/templates/kuttl/ldap-authentication/04-assert-ldap-user.yaml
deleted file mode 100644
index 663a6f96..00000000
--- a/tests/templates/kuttl/ldap-authentication/04-assert-ldap-user.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
----
-apiVersion: kuttl.dev/v1beta1
-kind: TestAssert
-commands:
-  - script: kubectl exec -n $NAMESPACE openldap-0 -- ldapsearch -H ldap://localhost:1389 -D cn=admin,dc=example,dc=org -w admin -b ou=users,dc=example,dc=org > /dev/null
-  - script: kubectl exec -n $NAMESPACE openldap-0 -- bash -c LDAPTLS_CACERT=/tls/ca.crt ldapsearch -Z -H ldaps://localhost:1636 -D cn=admin,dc=example,dc=org -w admin -b ou=users,dc=example,dc=org > /dev/null
diff --git a/tests/templates/kuttl/ldap-authentication/20-assert.yaml b/tests/templates/kuttl/ldap-authentication/20-assert.yaml
deleted file mode 100644
index 48cc7f14..00000000
--- a/tests/templates/kuttl/ldap-authentication/20-assert.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
----
-apiVersion: kuttl.dev/v1beta1
-kind: TestAssert
-commands:
-  - script: kubectl exec -n $NAMESPACE checks-0 -- python /tmp/authcheck.py
-timeout: 60
diff --git a/tests/templates/kuttl/ldap-authentication/create_ldap_user.sh b/tests/templates/kuttl/ldap-authentication/create_ldap_user.sh
deleted file mode 100644
index ef0c6ead..00000000
--- a/tests/templates/kuttl/ldap-authentication/create_ldap_user.sh
+++ /dev/null
@@ -1,15 +0,0 @@
-#!/bin/sh
-
-cat << 'EOF' | ldapadd -H ldap://localhost:1389 -D cn=admin,dc=example,dc=org -w admin
-dn: uid=alice,ou=users,dc=example,dc=org
-uid: alice
-cn: alice
-sn: alice
-objectClass: top
-objectClass: posixAccount
-objectClass: inetOrgPerson
-homeDirectory: /home/alice
-uidNumber: 3
-gidNumber: 3
-userPassword: alice
-EOF
diff --git a/tests/templates/kuttl/ldap-authentication/00-assert.yaml.j2 b/tests/templates/kuttl/ldap/00-assert.yaml.j2
similarity index 100%
rename from tests/templates/kuttl/ldap-authentication/00-assert.yaml.j2
rename to tests/templates/kuttl/ldap/00-assert.yaml.j2
diff --git a/tests/templates/kuttl/ldap-authentication/00-install-vector-aggregator-discovery-configmap.yaml.j2 b/tests/templates/kuttl/ldap/00-install-vector-aggregator-discovery-configmap.yaml.j2
similarity index 100%
rename from tests/templates/kuttl/ldap-authentication/00-install-vector-aggregator-discovery-configmap.yaml.j2
rename to tests/templates/kuttl/ldap/00-install-vector-aggregator-discovery-configmap.yaml.j2
diff --git a/tests/templates/kuttl/ldap-authentication/00-patch-ns.yaml.j2 b/tests/templates/kuttl/ldap/00-patch-ns.yaml.j2
similarity index 100%
rename from tests/templates/kuttl/ldap-authentication/00-patch-ns.yaml.j2
rename to tests/templates/kuttl/ldap/00-patch-ns.yaml.j2
diff --git a/tests/templates/kuttl/ldap-authentication/03-assert-openldap.yaml b/tests/templates/kuttl/ldap/01-assert.yaml
similarity index 100%
rename from tests/templates/kuttl/ldap-authentication/03-assert-openldap.yaml
rename to tests/templates/kuttl/ldap/01-assert.yaml
diff --git a/tests/templates/kuttl/ldap-authentication/03-install-openldap.yaml b/tests/templates/kuttl/ldap/01-install-openldap.yaml
similarity index 63%
rename from tests/templates/kuttl/ldap-authentication/03-install-openldap.yaml
rename to tests/templates/kuttl/ldap/01-install-openldap.yaml
index e593a84a..4508fa66 100644
--- a/tests/templates/kuttl/ldap-authentication/03-install-openldap.yaml
+++ b/tests/templates/kuttl/ldap/01-install-openldap.yaml
@@ -3,4 +3,4 @@ apiVersion: kuttl.dev/v1beta1
 kind: TestStep
 commands:
   # We need to replace $NAMESPACE (by KUTTL) in the install-openldap.yaml
-  - script: eval "echo \"$(cat 03_openldap.yaml)\"" | kubectl apply -f -
+  - script: envsubst < 01_openldap.yaml | kubectl apply -n $NAMESPACE -f -
diff --git a/tests/templates/kuttl/ldap-authentication/03_openldap.yaml.j2 b/tests/templates/kuttl/ldap/01_openldap.yaml.j2
similarity index 85%
rename from tests/templates/kuttl/ldap-authentication/03_openldap.yaml.j2
rename to tests/templates/kuttl/ldap/01_openldap.yaml.j2
index 0561e4a1..ef0f2e35 100644
--- a/tests/templates/kuttl/ldap-authentication/03_openldap.yaml.j2
+++ b/tests/templates/kuttl/ldap/01_openldap.yaml.j2
@@ -2,7 +2,7 @@
 apiVersion: secrets.stackable.tech/v1alpha1
 kind: SecretClass
 metadata:
-  name: openldap-tls-$NAMESPACE
+  name: openldap-tls
 spec:
   backend:
     autoTls:
@@ -30,7 +30,7 @@ spec:
       labels:
         app.kubernetes.io/name: openldap
     spec:
-      serviceAccountName: "druid-ldap-sa"
+      serviceAccountName: "ldap-sa"
       #
       # The security context below is necessary to avoid the following error on OpenShift:
       #    /opt/bitnami/scripts/openldap/setup.sh: line 102: /opt/bitnami/openldap/sbin/slappasswd: Operation not permitted
@@ -46,17 +46,13 @@ spec:
             - name: LDAP_ADMIN_PASSWORD
               value: admin
             - name: LDAP_ENABLE_TLS
-              value: \"yes\"
+              value: "yes"
             - name: LDAP_TLS_CERT_FILE
               value: /tls/tls.crt
             - name: LDAP_TLS_KEY_FILE
               value: /tls/tls.key
             - name: LDAP_TLS_CA_FILE
               value: /tls/ca.crt
-{% if test_scenario['values']['ldap-no-bind-credentials'] == 'true' %}
-            - name: LDAP_ALLOW_ANON_BINDING
-              value: \"yes\"
-{% endif %}
           ports:
             - name: ldap
               containerPort: 1389
@@ -76,7 +72,7 @@ spec:
           csi:
             driver: secrets.stackable.tech
             volumeAttributes:
-              secrets.stackable.tech/class: openldap-tls-$NAMESPACE
+              secrets.stackable.tech/class: openldap-tls
               secrets.stackable.tech/scope: pod
 ---
 apiVersion: v1
@@ -101,15 +97,14 @@ spec:
 apiVersion: v1
 kind: ServiceAccount
 metadata:
-  name: druid-ldap-sa
+  name: ldap-sa
   namespace: $NAMESPACE
-
 {% if test_scenario['values']['openshift'] == 'true' %}
 ---
 kind: Role
 apiVersion: rbac.authorization.k8s.io/v1
 metadata:
-  name: druid-ldap-tests-role
+  name: use-integration-tests-scc
   namespace: $NAMESPACE
 rules:
 - apiGroups: ["security.openshift.io"]
@@ -120,13 +115,13 @@ rules:
 kind: RoleBinding
 apiVersion: rbac.authorization.k8s.io/v1
 metadata:
-  name: druid-ldap-tests-rolebinding
+  name: use-integration-tests-scc
   namespace: $NAMESPACE
 subjects:
   - kind: ServiceAccount
-    name: druid-ldap-sa
+    name: ldap-sa
 roleRef:
   kind: Role
-  name: druid-ldap-tests-role
+  name: use-integration-tests-scc
   apiGroup: rbac.authorization.k8s.io
 {% endif %}
diff --git a/tests/templates/kuttl/ldap/02-assert.yaml b/tests/templates/kuttl/ldap/02-assert.yaml
new file mode 100644
index 00000000..46bea753
--- /dev/null
+++ b/tests/templates/kuttl/ldap/02-assert.yaml
@@ -0,0 +1,6 @@
+---
+apiVersion: kuttl.dev/v1beta1
+kind: TestAssert
+commands:
+  - script: kubectl exec -n $NAMESPACE openldap-0 -- ldapsearch -H ldap://localhost:1389 -D "cn=integrationtest,ou=my users,dc=example,dc=org" -w 'bindPasswordWithSpecialCharacter\@<&>"'"'" -b "ou=my users,dc=example,dc=org" > /dev/null
+  - script: kubectl exec -n $NAMESPACE openldap-0 -- bash -c LDAPTLS_CACERT=/tls/ca.crt ldapsearch -Z -H ldaps://localhost:1636 -D "cn=integrationtest,ou=my users,dc=example,dc=org" -w 'bindPasswordWithSpecialCharacter\@<&>"'"'" -b "ou=my users,dc=example,dc=org" > /dev/null
diff --git a/tests/templates/kuttl/ldap-authentication/04-ldap-user.yaml b/tests/templates/kuttl/ldap/02-create-ldap-user.yaml
similarity index 100%
rename from tests/templates/kuttl/ldap-authentication/04-ldap-user.yaml
rename to tests/templates/kuttl/ldap/02-create-ldap-user.yaml
diff --git a/tests/templates/kuttl/ldap-authentication/11-assert.yaml b/tests/templates/kuttl/ldap/03-assert.yaml
similarity index 89%
rename from tests/templates/kuttl/ldap-authentication/11-assert.yaml
rename to tests/templates/kuttl/ldap/03-assert.yaml
index dc085bb1..a4f62208 100644
--- a/tests/templates/kuttl/ldap-authentication/11-assert.yaml
+++ b/tests/templates/kuttl/ldap/03-assert.yaml
@@ -6,7 +6,7 @@ timeout: 300
 apiVersion: apps/v1
 kind: StatefulSet
 metadata:
-  name: checks
+  name: test-druid
 status:
   readyReplicas: 1
   replicas: 1
diff --git a/tests/templates/kuttl/ldap-authentication/11-checks-container.yaml b/tests/templates/kuttl/ldap/03-install-test-druid.yaml
similarity index 67%
rename from tests/templates/kuttl/ldap-authentication/11-checks-container.yaml
rename to tests/templates/kuttl/ldap/03-install-test-druid.yaml
index 24734226..450becea 100644
--- a/tests/templates/kuttl/ldap-authentication/11-checks-container.yaml
+++ b/tests/templates/kuttl/ldap/03-install-test-druid.yaml
@@ -2,20 +2,21 @@
 apiVersion: apps/v1
 kind: StatefulSet
 metadata:
-  name: checks
+  name: test-druid
   labels:
-    app: checks
+    app: test-druid
 spec:
   replicas: 1
   selector:
     matchLabels:
-      app: checks
+      app: test-druid
   template:
     metadata:
       labels:
-        app: checks
+        app: test-druid
     spec:
       containers:
-        - name: checks
+        - name: test-druid
           image: docker.stackable.tech/stackable/testing-tools:0.2.0-stackable0.0.0-dev
           command: ["sleep", "infinity"]
+      terminationGracePeriodSeconds: 1
diff --git a/tests/templates/kuttl/ldap-authentication/01-assert.yaml b/tests/templates/kuttl/ldap/10-assert.yaml
similarity index 62%
rename from tests/templates/kuttl/ldap-authentication/01-assert.yaml
rename to tests/templates/kuttl/ldap/10-assert.yaml
index f21094ef..0a9d9e7b 100644
--- a/tests/templates/kuttl/ldap-authentication/01-assert.yaml
+++ b/tests/templates/kuttl/ldap/10-assert.yaml
@@ -6,12 +6,7 @@ timeout: 600
 apiVersion: apps/v1
 kind: StatefulSet
 metadata:
-  name: druid-zk-server-default
+  name: zk-server-default
 status:
   readyReplicas: 1
   replicas: 1
----
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: hdfs-znode
diff --git a/tests/templates/kuttl/ldap-authentication/01-install-zk.yaml.j2 b/tests/templates/kuttl/ldap/10-install-zk.yaml.j2
similarity index 93%
rename from tests/templates/kuttl/ldap-authentication/01-install-zk.yaml.j2
rename to tests/templates/kuttl/ldap/10-install-zk.yaml.j2
index 1b307f07..ea59f09e 100644
--- a/tests/templates/kuttl/ldap-authentication/01-install-zk.yaml.j2
+++ b/tests/templates/kuttl/ldap/10-install-zk.yaml.j2
@@ -2,7 +2,7 @@
 apiVersion: zookeeper.stackable.tech/v1alpha1
 kind: ZookeeperCluster
 metadata:
-  name: druid-zk
+  name: zk
 spec:
   image:
     productVersion: "{{ test_scenario['values']['zookeeper-latest'] }}"
@@ -25,7 +25,7 @@ metadata:
   name: druid-znode
 spec:
   clusterRef:
-    name: druid-zk
+    name: zk
 ---
 apiVersion: zookeeper.stackable.tech/v1alpha1
 kind: ZookeeperZnode
@@ -33,4 +33,4 @@ metadata:
   name: hdfs-znode
 spec:
   clusterRef:
-    name: druid-zk
+    name: zk
diff --git a/tests/templates/kuttl/ldap-authentication/05-authentication-class.yaml.j2 b/tests/templates/kuttl/ldap/11-create-authentication-classes.yaml.j2
similarity index 56%
rename from tests/templates/kuttl/ldap-authentication/05-authentication-class.yaml.j2
rename to tests/templates/kuttl/ldap/11-create-authentication-classes.yaml.j2
index 7c35481c..9af5805a 100644
--- a/tests/templates/kuttl/ldap-authentication/05-authentication-class.yaml.j2
+++ b/tests/templates/kuttl/ldap/11-create-authentication-classes.yaml.j2
@@ -2,7 +2,7 @@
 apiVersion: kuttl.dev/v1beta1
 kind: TestStep
 metadata:
-  name: create-authentication-classes
+  name: create-ldap-user
 commands:
   # We need to replace $NAMESPACE (by KUTTL) in the create-authentication-classes.yaml(.j2)
-  - script: eval "echo \"$(cat 05_authentication-class.yaml)\"" | kubectl apply -f -
+  - script: envsubst < 11_authentication-classes.yaml | kubectl apply -n $NAMESPACE -f -
diff --git a/tests/templates/kuttl/ldap-authentication/05_authentication-class.yaml.j2 b/tests/templates/kuttl/ldap/11_authentication-classes.yaml.j2
similarity index 51%
rename from tests/templates/kuttl/ldap-authentication/05_authentication-class.yaml.j2
rename to tests/templates/kuttl/ldap/11_authentication-classes.yaml.j2
index f41d2fa8..cf326491 100644
--- a/tests/templates/kuttl/ldap-authentication/05_authentication-class.yaml.j2
+++ b/tests/templates/kuttl/ldap/11_authentication-classes.yaml.j2
@@ -7,43 +7,26 @@ spec:
   provider:
     ldap:
       hostname: openldap.$NAMESPACE.svc.cluster.local
-      searchBase: ou=users,dc=example,dc=org
-      searchFilter: (uid=%s)
+      searchBase: ou=my users,dc=example,dc=org
+      bindCredentials:
+        secretClass: druid-with-ldap-bind
 {% if test_scenario['values']['ldap-use-tls'] == 'false' %}
       port: 1389
-      tls: null
 {% else %}
       port: 1636
       tls:
         verification:
           server:
             caCert:
-              secretClass: openldap-tls-$NAMESPACE
-{% endif %}
-{% if test_scenario['values']['ldap-no-bind-credentials'] == 'true' %}
-      bindCredentials: null
-{% else %}
-      bindCredentials:
-        secretClass: druid-ldap-secret
+              secretClass: openldap-tls
 {% endif %}
 ---
 apiVersion: secrets.stackable.tech/v1alpha1
 kind: SecretClass
 metadata:
-  name: druid-ldap-secret
+  name: druid-with-ldap-bind
 spec:
   backend:
     k8sSearch:
       searchNamespace:
         pod: {}
----
-apiVersion: v1
-kind: Secret
-metadata:
-  name: druid-ldap-secret
-  namespace: $NAMESPACE
-  labels:
-    secrets.stackable.tech/class: druid-ldap-secret
-stringData:
-  user: cn=admin,dc=example,dc=org
-  password: admin
diff --git a/tests/templates/kuttl/ldap-authentication/02-assert.yaml b/tests/templates/kuttl/ldap/12-assert.yaml
similarity index 100%
rename from tests/templates/kuttl/ldap-authentication/02-assert.yaml
rename to tests/templates/kuttl/ldap/12-assert.yaml
diff --git a/tests/templates/kuttl/ldap-authentication/02-install-hdfs.yaml.j2 b/tests/templates/kuttl/ldap/12-install-hdfs.yaml.j2
similarity index 100%
rename from tests/templates/kuttl/ldap-authentication/02-install-hdfs.yaml.j2
rename to tests/templates/kuttl/ldap/12-install-hdfs.yaml.j2
diff --git a/tests/templates/kuttl/ldap-authentication/10-assert.yaml b/tests/templates/kuttl/ldap/13-assert.yaml
similarity index 100%
rename from tests/templates/kuttl/ldap-authentication/10-assert.yaml
rename to tests/templates/kuttl/ldap/13-assert.yaml
diff --git a/tests/templates/kuttl/ldap-authentication/10-install-druid.yaml.j2 b/tests/templates/kuttl/ldap/13-install-druid.yaml.j2
similarity index 77%
rename from tests/templates/kuttl/ldap-authentication/10-install-druid.yaml.j2
rename to tests/templates/kuttl/ldap/13-install-druid.yaml.j2
index 6be8a87f..d615e173 100644
--- a/tests/templates/kuttl/ldap-authentication/10-install-druid.yaml.j2
+++ b/tests/templates/kuttl/ldap/13-install-druid.yaml.j2
@@ -33,6 +33,7 @@ commands:
 {% endif %}
         brokers:
           config:
+            gracefulShutdownTimeout: 1s # Let the test run faster
             logging:
               enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }}
           roleGroups:
@@ -40,6 +41,7 @@ commands:
               replicas: 1
         coordinators:
           config:
+            gracefulShutdownTimeout: 1s # Let the test run faster
             logging:
               enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }}
           roleGroups:
@@ -47,6 +49,7 @@ commands:
               replicas: 1
         historicals:
           config:
+            gracefulShutdownTimeout: 1s # Let the test run faster
             logging:
               enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }}
           roleGroups:
@@ -54,6 +57,7 @@ commands:
               replicas: 1
         middleManagers:
           config:
+            gracefulShutdownTimeout: 1s # Let the test run faster
             logging:
               enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }}
           roleGroups:
@@ -61,8 +65,20 @@ commands:
               replicas: 1
         routers:
           config:
+            gracefulShutdownTimeout: 1s # Let the test run faster
             logging:
               enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }}
           roleGroups:
             default:
               replicas: 1
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: druid-with-ldap-bind-secret
+  labels:
+    secrets.stackable.tech/class: druid-with-ldap-bind
+stringData:
+  user: cn=integrationtest,ou=my users,dc=example,dc=org
+  password: >
+    bindPasswordWithSpecialCharacter\@<&>"'
diff --git a/tests/templates/kuttl/ldap/20-assert.yaml b/tests/templates/kuttl/ldap/20-assert.yaml
new file mode 100644
index 00000000..c947cc92
--- /dev/null
+++ b/tests/templates/kuttl/ldap/20-assert.yaml
@@ -0,0 +1,6 @@
+---
+apiVersion: kuttl.dev/v1beta1
+kind: TestAssert
+commands:
+  - script: kubectl exec -n $NAMESPACE test-druid-0 -- python /tmp/authcheck.py
+timeout: 60
diff --git a/tests/templates/kuttl/ldap-authentication/20-authcheck.yaml b/tests/templates/kuttl/ldap/20-authcheck.yaml
similarity index 50%
rename from tests/templates/kuttl/ldap-authentication/20-authcheck.yaml
rename to tests/templates/kuttl/ldap/20-authcheck.yaml
index 58d7d1bc..01967066 100644
--- a/tests/templates/kuttl/ldap-authentication/20-authcheck.yaml
+++ b/tests/templates/kuttl/ldap/20-authcheck.yaml
@@ -3,4 +3,4 @@ apiVersion: kuttl.dev/v1beta1
 kind: TestStep
 timeout: 60
 commands:
-  - script: kubectl cp -n $NAMESPACE ./authcheck.py  checks-0:/tmp
+  - script: kubectl cp -n $NAMESPACE ./authcheck.py test-druid-0:/tmp
diff --git a/tests/templates/kuttl/ldap-authentication/authcheck.py b/tests/templates/kuttl/ldap/authcheck.py
similarity index 85%
rename from tests/templates/kuttl/ldap-authentication/authcheck.py
rename to tests/templates/kuttl/ldap/authcheck.py
index be28d4e0..8d4149b0 100755
--- a/tests/templates/kuttl/ldap-authentication/authcheck.py
+++ b/tests/templates/kuttl/ldap/authcheck.py
@@ -2,6 +2,8 @@
 import sys
 import logging
 
+USER_NAME = "cn=integrationtest,ou=my users,dc=example,dc=org"
+USER_PASSWORD = "bindPasswordWithSpecialCharacter\\@<&>\"'"
 
 def main():
     result = 0
@@ -30,8 +32,8 @@ def main():
         else:
             logging.info("success")
         # make an authorized request -> return 200 expected
-        logging.info(f"making request as LDAP user [alice] to {role}")
-        res = requests.get(url, auth=("alice", "alice"), verify=False)
+        logging.info(f"making request as LDAP user [{USER_NAME}] to {role}")
+        res = requests.get(url, auth=(USER_NAME, USER_PASSWORD), verify=False)
         if res.status_code != 200:
             logging.error(f"expected 200 but got {res.status_code}")
             result = 1
diff --git a/tests/templates/kuttl/ldap/create_ldap_user.sh b/tests/templates/kuttl/ldap/create_ldap_user.sh
new file mode 100755
index 00000000..6c51cd03
--- /dev/null
+++ b/tests/templates/kuttl/ldap/create_ldap_user.sh
@@ -0,0 +1,36 @@
+#!/bin/sh
+
+# To check the existing users
+# ldapsearch -H ldap://localhost:1389 -D "cn=admin,dc=example,dc=org" -w admin -b "ou=my users,dc=example,dc=org"
+
+# To check the new user
+# ldapsearch -H ldap://localhost:1389 -D "cn=integrationtest,ou=my users,dc=example,dc=org" -w 'bindPasswordWithSpecialCharacter\@<&>"'"'" -b "ou=my users,dc=example,dc=org"
+
+cat << 'EOF' | ldapadd -H ldap://localhost:1389 -D "cn=admin,dc=example,dc=org" -w admin
+dn: ou=my users,dc=example,dc=org
+ou: my users
+objectclass: top
+objectclass: organizationalUnit
+EOF
+
+cat << 'EOF' | ldapadd -H ldap://localhost:1389 -D "cn=admin,dc=example,dc=org" -w admin
+dn: cn=integrationtest,ou=my users,dc=example,dc=org
+objectClass: inetOrgPerson
+objectClass: posixAccount
+objectClass: shadowAccount
+cn: integrationtest
+uid: integrationtest
+givenName: Stackable
+sn: Integration-Test
+mail: integrationtest@stackable.de
+uidNumber: 16842
+gidNumber: 100
+homeDirectory: /home/integrationtest
+loginShell: /bin/bash
+userPassword: {crypt}x
+shadowLastChange: 0
+shadowMax: 0
+shadowWarning: 0
+EOF
+
+ldappasswd -H ldap://localhost:1389 -D "cn=admin,dc=example,dc=org" -w admin -s 'bindPasswordWithSpecialCharacter\@<&>"'"'" "cn=integrationtest,ou=my users,dc=example,dc=org"

From c9b4ec262a25849f4fc1dd372f80070fa28a1b7c Mon Sep 17 00:00:00 2001
From: Sebastian Bernauer <sebastian.bernauer@stackable.tech>
Date: Tue, 25 Jun 2024 15:32:11 +0200
Subject: [PATCH 19/23] Adopt ldap test name

---
 tests/test-definition.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/test-definition.yaml b/tests/test-definition.yaml
index 73d037dc..7036f737 100644
--- a/tests/test-definition.yaml
+++ b/tests/test-definition.yaml
@@ -113,7 +113,7 @@ tests:
       - zookeeper-latest
       - tls-mode
       - openshift
-  - name: ldap-authentication
+  - name: ldap
     dimensions:
       - druid
       - zookeeper-latest

From 07f055ed81696d2f647d5ae80945359144e8379e Mon Sep 17 00:00:00 2001
From: Sebastian Bernauer <sebastian.bernauer@stackable.tech>
Date: Tue, 25 Jun 2024 15:59:31 +0200
Subject: [PATCH 20/23] Fix test

---
 tests/templates/kuttl/ldap/11_authentication-classes.yaml.j2 | 5 +++++
 tests/templates/kuttl/ldap/authcheck.py                      | 2 +-
 2 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/tests/templates/kuttl/ldap/11_authentication-classes.yaml.j2 b/tests/templates/kuttl/ldap/11_authentication-classes.yaml.j2
index cf326491..75f83998 100644
--- a/tests/templates/kuttl/ldap/11_authentication-classes.yaml.j2
+++ b/tests/templates/kuttl/ldap/11_authentication-classes.yaml.j2
@@ -8,8 +8,13 @@ spec:
     ldap:
       hostname: openldap.$NAMESPACE.svc.cluster.local
       searchBase: ou=my users,dc=example,dc=org
+      searchFilter: (uid=%s)
+{% if test_scenario['values']['ldap-no-bind-credentials'] == 'true' %}
+      bindCredentials: null
+{% else %}
       bindCredentials:
         secretClass: druid-with-ldap-bind
+{% endif %}
 {% if test_scenario['values']['ldap-use-tls'] == 'false' %}
       port: 1389
 {% else %}
diff --git a/tests/templates/kuttl/ldap/authcheck.py b/tests/templates/kuttl/ldap/authcheck.py
index 8d4149b0..a1de4b8e 100755
--- a/tests/templates/kuttl/ldap/authcheck.py
+++ b/tests/templates/kuttl/ldap/authcheck.py
@@ -2,7 +2,7 @@
 import sys
 import logging
 
-USER_NAME = "cn=integrationtest,ou=my users,dc=example,dc=org"
+USER_NAME = "integrationtest"
 USER_PASSWORD = "bindPasswordWithSpecialCharacter\\@<&>\"'"
 
 def main():

From 238ca01f92b933077ec1a842cb68ad2c13c49f41 Mon Sep 17 00:00:00 2001
From: Benedikt Labrenz <benedikt@labrenz.org>
Date: Wed, 26 Jun 2024 12:55:46 +0200
Subject: [PATCH 21/23] fix tests by adding --skip-tests option

---
 scripts/run-tests | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/scripts/run-tests b/scripts/run-tests
index 3768dec1..428ae32d 100755
--- a/scripts/run-tests
+++ b/scripts/run-tests
@@ -78,6 +78,12 @@ def parse_args(argv: list[str]) -> argparse.Namespace:
         action="store_true",
     )
 
+    parser.add_argument(
+        "--skip-tests",
+        help="Do not actually run the tests.",
+        action="store_true",
+    )
+
     parser.add_argument(
         "--parallel",
         help="How many tests to run in parallel. Default 2.",
@@ -349,7 +355,10 @@ def main(argv) -> int:
         gen_tests(opts.test_suite)
         with release_file(opts.operator) as f:
             maybe_install_release(opts.skip_release, f)
-        run_tests(opts.test, opts.parallel, opts.namespace, opts.skip_delete)
+        if opts.skip_tests:
+            logging.info("Skip running tests.")
+        else:
+            run_tests(opts.test, opts.parallel, opts.namespace, opts.skip_delete)
     except TestRunnerException:
         ret = 1
     return ret

From 36f31699c97225ff1ee4f3f647e78f8cb2a89d7e Mon Sep 17 00:00:00 2001
From: Sebastian Bernauer <sebastian.bernauer@stackable.tech>
Date: Wed, 26 Jun 2024 14:23:01 +0200
Subject: [PATCH 22/23] Revert "fix tests by adding --skip-tests option"

This reverts commit 238ca01f92b933077ec1a842cb68ad2c13c49f41.
---
 scripts/run-tests | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/scripts/run-tests b/scripts/run-tests
index dbc06807..c331043b 100755
--- a/scripts/run-tests
+++ b/scripts/run-tests
@@ -84,12 +84,6 @@ def parse_args(argv: list[str]) -> argparse.Namespace:
         action="store_true",
     )
 
-    parser.add_argument(
-        "--skip-tests",
-        help="Do not actually run the tests.",
-        action="store_true",
-    )
-
     parser.add_argument(
         "--parallel",
         help="How many tests to run in parallel. Default 2.",

From a0d3f6ba078c9bc92ff849da6c8a2a46918b5390 Mon Sep 17 00:00:00 2001
From: Sebastian Bernauer <sebastian.bernauer@stackable.tech>
Date: Wed, 26 Jun 2024 14:43:18 +0200
Subject: [PATCH 23/23] Set terminationGracePeriodSeconds on all test
 containers

---
 tests/templates/kuttl/authorizer/05-checks-container.yaml        | 1 +
 tests/templates/kuttl/hdfs-deep-storage/04-checks-container.yaml | 1 +
 .../templates/kuttl/ingestion-no-s3-ext/04-checks-container.yaml | 1 +
 tests/templates/kuttl/ingestion-s3-ext/04-checks-container.yaml  | 1 +
 tests/templates/kuttl/logging/06-install-druid-test-runner.yaml  | 1 +
 tests/templates/kuttl/oidc/51-install-test-container.yaml.j2     | 1 +
 tests/templates/kuttl/s3-deep-storage/07-checks-container.yaml   | 1 +
 tests/templates/kuttl/smoke/60-checks-container.yaml             | 1 +
 tests/templates/kuttl/tls/05-install-checks.yaml.j2              | 1 +
 9 files changed, 9 insertions(+)

diff --git a/tests/templates/kuttl/authorizer/05-checks-container.yaml b/tests/templates/kuttl/authorizer/05-checks-container.yaml
index 24734226..dd9ecb77 100644
--- a/tests/templates/kuttl/authorizer/05-checks-container.yaml
+++ b/tests/templates/kuttl/authorizer/05-checks-container.yaml
@@ -19,3 +19,4 @@ spec:
         - name: checks
           image: docker.stackable.tech/stackable/testing-tools:0.2.0-stackable0.0.0-dev
           command: ["sleep", "infinity"]
+      terminationGracePeriodSeconds: 1
diff --git a/tests/templates/kuttl/hdfs-deep-storage/04-checks-container.yaml b/tests/templates/kuttl/hdfs-deep-storage/04-checks-container.yaml
index 24734226..dd9ecb77 100644
--- a/tests/templates/kuttl/hdfs-deep-storage/04-checks-container.yaml
+++ b/tests/templates/kuttl/hdfs-deep-storage/04-checks-container.yaml
@@ -19,3 +19,4 @@ spec:
         - name: checks
           image: docker.stackable.tech/stackable/testing-tools:0.2.0-stackable0.0.0-dev
           command: ["sleep", "infinity"]
+      terminationGracePeriodSeconds: 1
diff --git a/tests/templates/kuttl/ingestion-no-s3-ext/04-checks-container.yaml b/tests/templates/kuttl/ingestion-no-s3-ext/04-checks-container.yaml
index 24734226..dd9ecb77 100644
--- a/tests/templates/kuttl/ingestion-no-s3-ext/04-checks-container.yaml
+++ b/tests/templates/kuttl/ingestion-no-s3-ext/04-checks-container.yaml
@@ -19,3 +19,4 @@ spec:
         - name: checks
           image: docker.stackable.tech/stackable/testing-tools:0.2.0-stackable0.0.0-dev
           command: ["sleep", "infinity"]
+      terminationGracePeriodSeconds: 1
diff --git a/tests/templates/kuttl/ingestion-s3-ext/04-checks-container.yaml b/tests/templates/kuttl/ingestion-s3-ext/04-checks-container.yaml
index 24734226..dd9ecb77 100644
--- a/tests/templates/kuttl/ingestion-s3-ext/04-checks-container.yaml
+++ b/tests/templates/kuttl/ingestion-s3-ext/04-checks-container.yaml
@@ -19,3 +19,4 @@ spec:
         - name: checks
           image: docker.stackable.tech/stackable/testing-tools:0.2.0-stackable0.0.0-dev
           command: ["sleep", "infinity"]
+      terminationGracePeriodSeconds: 1
diff --git a/tests/templates/kuttl/logging/06-install-druid-test-runner.yaml b/tests/templates/kuttl/logging/06-install-druid-test-runner.yaml
index c7e1097f..f1657554 100644
--- a/tests/templates/kuttl/logging/06-install-druid-test-runner.yaml
+++ b/tests/templates/kuttl/logging/06-install-druid-test-runner.yaml
@@ -20,3 +20,4 @@ spec:
           image: docker.stackable.tech/stackable/testing-tools:0.2.0-stackable0.0.0-dev
           stdin: true
           tty: true
+      terminationGracePeriodSeconds: 1
diff --git a/tests/templates/kuttl/oidc/51-install-test-container.yaml.j2 b/tests/templates/kuttl/oidc/51-install-test-container.yaml.j2
index 1ae86a1d..3bec5cb8 100644
--- a/tests/templates/kuttl/oidc/51-install-test-container.yaml.j2
+++ b/tests/templates/kuttl/oidc/51-install-test-container.yaml.j2
@@ -94,3 +94,4 @@ spec:
         - name: ingestion-check
           configMap:
             name: ingestion-check
+      terminationGracePeriodSeconds: 1
diff --git a/tests/templates/kuttl/s3-deep-storage/07-checks-container.yaml b/tests/templates/kuttl/s3-deep-storage/07-checks-container.yaml
index 24734226..dd9ecb77 100644
--- a/tests/templates/kuttl/s3-deep-storage/07-checks-container.yaml
+++ b/tests/templates/kuttl/s3-deep-storage/07-checks-container.yaml
@@ -19,3 +19,4 @@ spec:
         - name: checks
           image: docker.stackable.tech/stackable/testing-tools:0.2.0-stackable0.0.0-dev
           command: ["sleep", "infinity"]
+      terminationGracePeriodSeconds: 1
diff --git a/tests/templates/kuttl/smoke/60-checks-container.yaml b/tests/templates/kuttl/smoke/60-checks-container.yaml
index 98d3ab13..4aafaf71 100644
--- a/tests/templates/kuttl/smoke/60-checks-container.yaml
+++ b/tests/templates/kuttl/smoke/60-checks-container.yaml
@@ -26,3 +26,4 @@ spec:
             limits:
               memory: "128Mi"
               cpu: "1"
+      terminationGracePeriodSeconds: 1
diff --git a/tests/templates/kuttl/tls/05-install-checks.yaml.j2 b/tests/templates/kuttl/tls/05-install-checks.yaml.j2
index 5c153768..1700f687 100644
--- a/tests/templates/kuttl/tls/05-install-checks.yaml.j2
+++ b/tests/templates/kuttl/tls/05-install-checks.yaml.j2
@@ -91,3 +91,4 @@ spec:
                   requests:
                     storage: "1"
 {% endif %}
+      terminationGracePeriodSeconds: 1