diff --git a/CHANGELOG.md b/CHANGELOG.md index 7f4d031a..0ce8fdff 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,12 +4,6 @@ All notable changes to this project will be documented in this file. ## [Unreleased] -### Changed - -- Fixed the RoleGroup `selector`. It was not used before. ([#611]) - -[#611]: https://github.com/stackabletech/zookeeper-operator/pull/611 - ### Added - Log aggregation added ([#588]). @@ -23,12 +17,16 @@ All notable changes to this project will be documented in this file. - Operator-rs: 0.25.3 -> 0.27.1 ([#591]). - Fixed bug where ZNode ConfigMaps were not created due to labeling issues ([#592]). - Don't run init container as root and avoid chmod and chowning ([#603]). +- Fixed the RoleGroup `selector`. It was not used before. ([#611]). +- [BREAKING] Moved `spec.authentication`, `spec.tls` and `spec.logging` to `spec.clusterConfig`. Consolidated sub field names like `tls.client.secretClass` to `tls.serverSecretClass` ([#612]). [#586]: https://github.com/stackabletech/zookeeper-operator/pull/586 [#591]: https://github.com/stackabletech/zookeeper-operator/pull/591 [#592]: https://github.com/stackabletech/zookeeper-operator/pull/592 [#599]: https://github.com/stackabletech/zookeeper-operator/pull/599 [#603]: https://github.com/stackabletech/zookeeper-operator/pull/603 +[#611]: https://github.com/stackabletech/zookeeper-operator/pull/611 +[#612]: https://github.com/stackabletech/zookeeper-operator/pull/612 ## [0.12.0] - 2022-11-07 diff --git a/deploy/crd/zookeepercluster.crd.yaml b/deploy/crd/zookeepercluster.crd.yaml index b29e2cec..d51a40e8 100644 --- a/deploy/crd/zookeepercluster.crd.yaml +++ b/deploy/crd/zookeepercluster.crd.yaml @@ -23,35 +23,56 @@ spec: spec: description: A cluster of ZooKeeper nodes properties: - config: + clusterConfig: default: + authentication: [] tls: - secretClass: tls - quorumTlsSecretClass: tls + quorumSecretClass: tls + serverSecretClass: tls + description: Global ZooKeeper cluster configuration that applies to all roles and role groups. properties: - clientAuthentication: - description: 'Only affects client connections. This setting controls: - If clients need to authenticate themselves against the server via TLS - Which ca.crt to use when validating the provided client certs Defaults to `None`' + authentication: + default: [] + description: Authentication class settings for ZooKeeper like mTLS authentication. + items: + properties: + authenticationClass: + description: |- + The AuthenticationClass to use. + + ## mTLS + + Only affects client connections. This setting controls: - If clients need to authenticate themselves against the server via TLS - Which ca.crt to use when validating the provided client certs This will override the server TLS settings (if set) in `spec.clusterConfig.tls.serverSecretClass`. + type: string + required: + - authenticationClass + type: object + type: array + logging: + description: Logging options for ZooKeeper. nullable: true properties: - authenticationClass: + vectorAggregatorConfigMapName: + description: Name of the Vector discovery ConfigMap. It must contain the key `ADDRESS` with the address of the Vector aggregator. + nullable: true type: string - required: - - authenticationClass type: object - quorumTlsSecretClass: - default: tls - description: 'Only affects quorum communication. Use mutual verification between Zookeeper Nodes (mandatory). This setting controls: - Which cert the servers should use to authenticate themselves against other servers - Which ca.crt to use when validating the other server' - type: string tls: default: - secretClass: tls - description: 'Only affects client connections. This setting controls: - If TLS encryption is used at all - Which cert the servers should use to authenticate themselves against the client Defaults to `TlsSecretClass` { secret_class: "tls".to_string() }.' + quorumSecretClass: tls + serverSecretClass: tls + description: TLS encryption settings for ZooKeeper (server, quorum). nullable: true properties: - secretClass: + quorumSecretClass: + default: tls + description: 'The to use for internal quorum communication. Use mutual verification between Zookeeper Nodes (mandatory). This setting controls: - Which cert the servers should use to authenticate themselves against other servers - Which ca.crt to use when validating the other server Defaults to `tls`' + type: string + serverSecretClass: + default: tls + description: 'The to use for client connections. This setting controls: - If TLS encryption is used at all - Which cert the servers should use to authenticate themselves against the client Defaults to `tls`.' + nullable: true type: string - required: - - secretClass type: object type: object image: @@ -62,7 +83,7 @@ spec: - required: - productVersion - stackableVersion - description: Desired ZooKeeper version + description: Desired ZooKeeper image to use. properties: custom: description: Overwrite the docker image. Specify the full docker image name, e.g. `docker.stackable.tech/stackable/superset:1.4.1-stackable2.1.0` @@ -98,6 +119,7 @@ spec: type: string type: object servers: + description: ZooKeeper server configuration. nullable: true properties: cliOverrides: @@ -791,13 +813,9 @@ spec: - roleGroups type: object stopped: - description: Emergency stop button, if `true` then all pods are stopped without affecting configuration (as setting `replicas` to `0` would) + description: Emergency stop button, if `true` then all pods are stopped without affecting configuration (as setting `replicas` to `0` would). nullable: true type: boolean - vectorAggregatorConfigMapName: - description: Name of the Vector discovery ConfigMap. It must contain the key `ADDRESS` with the address of the Vector aggregator. - nullable: true - type: string required: - image type: object diff --git a/deploy/helm/zookeeper-operator/crds/crds.yaml b/deploy/helm/zookeeper-operator/crds/crds.yaml index 9bc64dc2..2c3f8981 100644 --- a/deploy/helm/zookeeper-operator/crds/crds.yaml +++ b/deploy/helm/zookeeper-operator/crds/crds.yaml @@ -25,35 +25,56 @@ spec: spec: description: A cluster of ZooKeeper nodes properties: - config: + clusterConfig: default: + authentication: [] tls: - secretClass: tls - quorumTlsSecretClass: tls + quorumSecretClass: tls + serverSecretClass: tls + description: Global ZooKeeper cluster configuration that applies to all roles and role groups. properties: - clientAuthentication: - description: 'Only affects client connections. This setting controls: - If clients need to authenticate themselves against the server via TLS - Which ca.crt to use when validating the provided client certs Defaults to `None`' + authentication: + default: [] + description: Authentication class settings for ZooKeeper like mTLS authentication. + items: + properties: + authenticationClass: + description: |- + The AuthenticationClass to use. + + ## mTLS + + Only affects client connections. This setting controls: - If clients need to authenticate themselves against the server via TLS - Which ca.crt to use when validating the provided client certs This will override the server TLS settings (if set) in `spec.clusterConfig.tls.serverSecretClass`. + type: string + required: + - authenticationClass + type: object + type: array + logging: + description: Logging options for ZooKeeper. nullable: true properties: - authenticationClass: + vectorAggregatorConfigMapName: + description: Name of the Vector discovery ConfigMap. It must contain the key `ADDRESS` with the address of the Vector aggregator. + nullable: true type: string - required: - - authenticationClass type: object - quorumTlsSecretClass: - default: tls - description: 'Only affects quorum communication. Use mutual verification between Zookeeper Nodes (mandatory). This setting controls: - Which cert the servers should use to authenticate themselves against other servers - Which ca.crt to use when validating the other server' - type: string tls: default: - secretClass: tls - description: 'Only affects client connections. This setting controls: - If TLS encryption is used at all - Which cert the servers should use to authenticate themselves against the client Defaults to `TlsSecretClass` { secret_class: "tls".to_string() }.' + quorumSecretClass: tls + serverSecretClass: tls + description: TLS encryption settings for ZooKeeper (server, quorum). nullable: true properties: - secretClass: + quorumSecretClass: + default: tls + description: 'The to use for internal quorum communication. Use mutual verification between Zookeeper Nodes (mandatory). This setting controls: - Which cert the servers should use to authenticate themselves against other servers - Which ca.crt to use when validating the other server Defaults to `tls`' + type: string + serverSecretClass: + default: tls + description: 'The to use for client connections. This setting controls: - If TLS encryption is used at all - Which cert the servers should use to authenticate themselves against the client Defaults to `tls`.' + nullable: true type: string - required: - - secretClass type: object type: object image: @@ -64,7 +85,7 @@ spec: - required: - productVersion - stackableVersion - description: Desired ZooKeeper version + description: Desired ZooKeeper image to use. properties: custom: description: Overwrite the docker image. Specify the full docker image name, e.g. `docker.stackable.tech/stackable/superset:1.4.1-stackable2.1.0` @@ -100,6 +121,7 @@ spec: type: string type: object servers: + description: ZooKeeper server configuration. nullable: true properties: cliOverrides: @@ -793,13 +815,9 @@ spec: - roleGroups type: object stopped: - description: Emergency stop button, if `true` then all pods are stopped without affecting configuration (as setting `replicas` to `0` would) + description: Emergency stop button, if `true` then all pods are stopped without affecting configuration (as setting `replicas` to `0` would). nullable: true type: boolean - vectorAggregatorConfigMapName: - description: Name of the Vector discovery ConfigMap. It must contain the key `ADDRESS` with the address of the Vector aggregator. - nullable: true - type: string required: - image type: object diff --git a/deploy/manifests/crds.yaml b/deploy/manifests/crds.yaml index 372eacc8..1f24fe0e 100644 --- a/deploy/manifests/crds.yaml +++ b/deploy/manifests/crds.yaml @@ -26,35 +26,56 @@ spec: spec: description: A cluster of ZooKeeper nodes properties: - config: + clusterConfig: default: + authentication: [] tls: - secretClass: tls - quorumTlsSecretClass: tls + quorumSecretClass: tls + serverSecretClass: tls + description: Global ZooKeeper cluster configuration that applies to all roles and role groups. properties: - clientAuthentication: - description: 'Only affects client connections. This setting controls: - If clients need to authenticate themselves against the server via TLS - Which ca.crt to use when validating the provided client certs Defaults to `None`' + authentication: + default: [] + description: Authentication class settings for ZooKeeper like mTLS authentication. + items: + properties: + authenticationClass: + description: |- + The AuthenticationClass to use. + + ## mTLS + + Only affects client connections. This setting controls: - If clients need to authenticate themselves against the server via TLS - Which ca.crt to use when validating the provided client certs This will override the server TLS settings (if set) in `spec.clusterConfig.tls.serverSecretClass`. + type: string + required: + - authenticationClass + type: object + type: array + logging: + description: Logging options for ZooKeeper. nullable: true properties: - authenticationClass: + vectorAggregatorConfigMapName: + description: Name of the Vector discovery ConfigMap. It must contain the key `ADDRESS` with the address of the Vector aggregator. + nullable: true type: string - required: - - authenticationClass type: object - quorumTlsSecretClass: - default: tls - description: 'Only affects quorum communication. Use mutual verification between Zookeeper Nodes (mandatory). This setting controls: - Which cert the servers should use to authenticate themselves against other servers - Which ca.crt to use when validating the other server' - type: string tls: default: - secretClass: tls - description: 'Only affects client connections. This setting controls: - If TLS encryption is used at all - Which cert the servers should use to authenticate themselves against the client Defaults to `TlsSecretClass` { secret_class: "tls".to_string() }.' + quorumSecretClass: tls + serverSecretClass: tls + description: TLS encryption settings for ZooKeeper (server, quorum). nullable: true properties: - secretClass: + quorumSecretClass: + default: tls + description: 'The to use for internal quorum communication. Use mutual verification between Zookeeper Nodes (mandatory). This setting controls: - Which cert the servers should use to authenticate themselves against other servers - Which ca.crt to use when validating the other server Defaults to `tls`' + type: string + serverSecretClass: + default: tls + description: 'The to use for client connections. This setting controls: - If TLS encryption is used at all - Which cert the servers should use to authenticate themselves against the client Defaults to `tls`.' + nullable: true type: string - required: - - secretClass type: object type: object image: @@ -65,7 +86,7 @@ spec: - required: - productVersion - stackableVersion - description: Desired ZooKeeper version + description: Desired ZooKeeper image to use. properties: custom: description: Overwrite the docker image. Specify the full docker image name, e.g. `docker.stackable.tech/stackable/superset:1.4.1-stackable2.1.0` @@ -101,6 +122,7 @@ spec: type: string type: object servers: + description: ZooKeeper server configuration. nullable: true properties: cliOverrides: @@ -794,13 +816,9 @@ spec: - roleGroups type: object stopped: - description: Emergency stop button, if `true` then all pods are stopped without affecting configuration (as setting `replicas` to `0` would) + description: Emergency stop button, if `true` then all pods are stopped without affecting configuration (as setting `replicas` to `0` would). nullable: true type: boolean - vectorAggregatorConfigMapName: - description: Name of the Vector discovery ConfigMap. It must contain the key `ADDRESS` with the address of the Vector aggregator. - nullable: true - type: string required: - image type: object diff --git a/docs/modules/usage_guide/examples/example-cluster-tls-authentication.yaml b/docs/modules/usage_guide/examples/example-cluster-tls-authentication.yaml index e3477960..6e6d14b8 100644 --- a/docs/modules/usage_guide/examples/example-cluster-tls-authentication.yaml +++ b/docs/modules/usage_guide/examples/example-cluster-tls-authentication.yaml @@ -7,10 +7,9 @@ spec: image: productVersion: 3.8.0 stackableVersion: 0.8.0 - config: - clientAuthentication: - authenticationClass: zk-client-tls # <1> - quorumTlsSecretClass: tls + clusterConfig: + authentication: + - authenticationClass: zk-client-tls # <1> servers: roleGroups: default: diff --git a/docs/modules/usage_guide/examples/example-cluster-tls-encryption.yaml b/docs/modules/usage_guide/examples/example-cluster-tls-encryption.yaml index 06ee82b9..88b537d4 100644 --- a/docs/modules/usage_guide/examples/example-cluster-tls-encryption.yaml +++ b/docs/modules/usage_guide/examples/example-cluster-tls-encryption.yaml @@ -7,10 +7,10 @@ spec: image: productVersion: 3.8.0 stackableVersion: 0.8.0 - config: + clusterConfig: tls: - secretClass: tls # <1> - quorumTlsSecretClass: tls # <2> + serverSecretClass: tls # <1> + quorumSecretClass: tls # <2> servers: roleGroups: default: diff --git a/docs/modules/usage_guide/pages/authentication.adoc b/docs/modules/usage_guide/pages/authentication.adoc index e45753dc..5ac01549 100644 --- a/docs/modules/usage_guide/pages/authentication.adoc +++ b/docs/modules/usage_guide/pages/authentication.adoc @@ -1,6 +1,8 @@ = Authentication -The quorum or server-to-server communication is authenticated via TLS per default. In order to enforce TLS authentication for client-to-server communication, you can set an `AuthenticationClass` reference in the custom resource provided by the xref:commons-operator::index.adoc[Commons Operator]. +The communication between nodes (server to server) is encrypted TLS by default. In order to enforce TLS authentication for client-to-server communication, you can set an `AuthenticationClass` reference in the custom resource provided by the xref:commons-operator::index.adoc[Commons Operator]. + +Currently it is possible to configure a single form of authentication (of type TLS) by adding one (and only one) entry in the `authentication` sequence as shown in the example below. Additional authentication methods, such as Kerberos, are not yet supported. [source,yaml] ---- @@ -8,11 +10,11 @@ include::example$example-cluster-tls-authentication.yaml[] include::example$example-cluster-tls-authentication-class.yaml[] include::example$example-cluster-tls-authentication-secret.yaml[] ---- -<1> The `config.clientAuthentication.authenticationClass` can be set to use TLS for authentication. This is optional. +<1> The `clusterConfig.authentication.authenticationClass` can be set to use TLS for authentication. This is optional. <2> The referenced `AuthenticationClass` that references a `SecretClass` to provide certificates. <3> The reference to a `SecretClass`. <4> The `SecretClass` that is referenced by the `AuthenticationClass` in order to provide certificates. -If both `spec.config.tls.secretClass` and `spec.config.clientAuthentication.authenticationClass` are set, the authentication class will take precedence over the secret class. The cluster will be encrypted and authenticate only against the authentication class. +If both `spec.clusterConfig.tls.server.secretClass` and `spec.clusterConfig.authentication.authenticationClass` are set, the authentication class will take precedence over the secret class. The cluster will be encrypted and authenticate only against the authentication class. WARNING: Due to a https://issues.apache.org/jira/browse/ZOOKEEPER-4276[bug] in ZooKeeper, the `clientPort` property in combination with `client.portUnification=true` is used instead of the `secureClientPort`. This means that unencrypted and unauthenticated access to the ZooKeeper cluster is still possible. diff --git a/docs/modules/usage_guide/pages/encryption.adoc b/docs/modules/usage_guide/pages/encryption.adoc index 07851ce0..4258dd17 100644 --- a/docs/modules/usage_guide/pages/encryption.adoc +++ b/docs/modules/usage_guide/pages/encryption.adoc @@ -6,8 +6,8 @@ The quorum and client communication are encrypted by default via TLS. This requi ---- include::example$example-cluster-tls-encryption.yaml[] ---- -<1> The `tls.secretClass` refers to the client-to-server encryption. Defaults to the `tls` secret. -<2> The `quorumTlsSecretClass` refers to the server-to-server quorum encryption. Defaults to the `tls` secret. +<1> The `tls.server.secretClass` refers to the client-to-server encryption. Defaults to the `tls` secret. +<2> The `tls.quorum.secretClass` refers to the server-to-server quorum encryption. Defaults to the `tls` secret. The `tls` secret is deployed from the xref:secret-operator::index.adoc[Secret Operator] and looks like this: diff --git a/docs/modules/usage_guide/pages/isolating_clients_with_znodes.adoc b/docs/modules/usage_guide/pages/isolating_clients_with_znodes.adoc index f1c3ea08..533f5912 100644 --- a/docs/modules/usage_guide/pages/isolating_clients_with_znodes.adoc +++ b/docs/modules/usage_guide/pages/isolating_clients_with_znodes.adoc @@ -43,7 +43,7 @@ include::example$znode/example-znode-kafka.yaml[] <2> The namespace where the ZNode should be created. Since Kafka is running in the same namespace as ZooKeeper, this is the namespace of `my-zookeeper`. <3> The ZooKeeper cluster reference. The namespace is omitted here because the ZooKeeper is in the same namespace as the ZNode object. -The Stackable Operator for ZooKeeper watches for ZookeeperZnode objects. If one is found it creates the ZNode _inside_ of the ZooKeeper cluster and also creates a xref:home:concepts:service_discovery.adoc[discovery ConfigMap] in the same namespace as the ZookeeperZnode with the same name as the ZookeeperZnode. +The Stackable Operator for ZooKeeper watches for ZookeeperZnode objects. If one is found it creates the ZNode _inside_ the ZooKeeper cluster and also creates a xref:home:concepts:service_discovery.adoc[discovery ConfigMap] in the same namespace as the ZookeeperZnode with the same name as the ZookeeperZnode. In this example, two ConfigMaps are created: diff --git a/examples/simple-zookeeper-tls-cluster.yaml b/examples/simple-zookeeper-tls-cluster.yaml index 4105a688..ebd51c5d 100644 --- a/examples/simple-zookeeper-tls-cluster.yaml +++ b/examples/simple-zookeeper-tls-cluster.yaml @@ -7,12 +7,12 @@ spec: image: productVersion: 3.8.0 stackableVersion: 0.9.0 - config: + clusterConfig: + authentication: + - authenticationClass: zk-client-tls tls: - secretClass: tls - clientAuthentication: - authenticationClass: zk-client-tls - quorumTlsSecretClass: tls + serverSecretClass: tls + quorumSecretClass: tls servers: roleGroups: default: diff --git a/rust/crd/src/authentication.rs b/rust/crd/src/authentication.rs new file mode 100644 index 00000000..387c7fd4 --- /dev/null +++ b/rust/crd/src/authentication.rs @@ -0,0 +1,112 @@ +use crate::ObjectRef; + +use serde::{Deserialize, Serialize}; +use snafu::{ResultExt, Snafu}; +use stackable_operator::commons::authentication::AuthenticationClassProvider; +use stackable_operator::{ + client::Client, + commons::authentication::AuthenticationClass, + schemars::{self, JsonSchema}, +}; + +const SUPPORTED_AUTHENTICATION_CLASS: [&str; 1] = ["TLS"]; + +#[derive(Snafu, Debug)] +pub enum Error { + #[snafu(display("failed to retrieve AuthenticationClass [{}]", authentication_class))] + AuthenticationClassRetrieval { + source: stackable_operator::error::Error, + authentication_class: ObjectRef, + }, + // TODO: Adapt message if multiple authentication classes are supported + #[snafu(display("only one authentication class is currently supported. Possible Authentication classes are {SUPPORTED_AUTHENTICATION_CLASS:?}"))] + MultipleAuthenticationClassesProvided, + #[snafu(display( + "failed to use authentication method [{method}] for authentication class [{authentication_class}] - supported mechanisms: {SUPPORTED_AUTHENTICATION_CLASS:?}", + ))] + AuthenticationMethodNotSupported { + authentication_class: ObjectRef, + method: String, + }, +} + +#[derive(Clone, Deserialize, Debug, Eq, JsonSchema, PartialEq, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct ZookeeperAuthentication { + /// The AuthenticationClass to use. + /// + /// ## mTLS + /// + /// Only affects client connections. This setting controls: + /// - If clients need to authenticate themselves against the server via TLS + /// - Which ca.crt to use when validating the provided client certs + /// This will override the server TLS settings (if set) in `spec.clusterConfig.tls.serverSecretClass`. + pub authentication_class: String, +} + +#[derive(Clone, Debug)] +/// Helper struct that contains resolved AuthenticationClasses to reduce network API calls. +pub struct ResolvedAuthenticationClasses { + resolved_authentication_classes: Vec, +} + +impl ResolvedAuthenticationClasses { + /// Return the (first) TLS `AuthenticationClass` if available + pub fn get_tls_authentication_class(&self) -> Option<&AuthenticationClass> { + self.resolved_authentication_classes + .iter() + .find(|auth| matches!(auth.spec.provider, AuthenticationClassProvider::Tls(_))) + } + + /// Validates the resolved AuthenticationClasses. + /// Currently errors out if: + /// - More than one AuthenticationClass was provided + /// - AuthenticationClass mechanism was not supported + pub fn validate(&self) -> Result { + if self.resolved_authentication_classes.len() > 1 { + return Err(Error::MultipleAuthenticationClassesProvided); + } + + for auth_class in &self.resolved_authentication_classes { + match &auth_class.spec.provider { + AuthenticationClassProvider::Tls(_) => {} + _ => { + return Err(Error::AuthenticationMethodNotSupported { + authentication_class: ObjectRef::from_obj(auth_class), + method: auth_class.spec.provider.to_string(), + }) + } + } + } + + Ok(self.clone()) + } +} + +/// Resolve provided AuthenticationClasses via API calls and validate the contents. +/// Currently errors out if: +/// - AuthenticationClass could not be resolved +/// - Validation failed +pub async fn resolve_authentication_classes( + client: &Client, + auth_classes: &Vec, +) -> Result { + let mut resolved_authentication_classes: Vec = vec![]; + + for auth_class in auth_classes { + resolved_authentication_classes.push( + AuthenticationClass::resolve(client, &auth_class.authentication_class) + .await + .context(AuthenticationClassRetrievalSnafu { + authentication_class: ObjectRef::::new( + &auth_class.authentication_class, + ), + })?, + ); + } + + ResolvedAuthenticationClasses { + resolved_authentication_classes, + } + .validate() +} diff --git a/rust/crd/src/lib.rs b/rust/crd/src/lib.rs index 5a736639..98c830ff 100644 --- a/rust/crd/src/lib.rs +++ b/rust/crd/src/lib.rs @@ -1,3 +1,10 @@ +pub mod authentication; +pub mod security; +pub mod tls; + +use crate::authentication::ZookeeperAuthentication; +use crate::tls::ZookeeperTls; + use serde::{Deserialize, Serialize}; use snafu::{OptionExt, ResultExt, Snafu}; use stackable_operator::{ @@ -28,8 +35,6 @@ use strum::{Display, EnumIter, EnumString}; pub const ZOOKEEPER_PROPERTIES_FILE: &str = "zoo.cfg"; -pub const CLIENT_PORT: u16 = 2181; -pub const SECURE_CLIENT_PORT: u16 = 2282; pub const METRICS_PORT: u16 = 9505; pub const STACKABLE_DATA_DIR: &str = "/stackable/data"; @@ -38,12 +43,6 @@ pub const STACKABLE_LOG_CONFIG_DIR: &str = "/stackable/log_config"; pub const STACKABLE_LOG_DIR: &str = "/stackable/log"; pub const STACKABLE_RW_CONFIG_DIR: &str = "/stackable/rwconfig"; -pub const QUORUM_TLS_DIR: &str = "/stackable/quorum_tls"; -pub const QUORUM_TLS_MOUNT_DIR: &str = "/stackable/quorum_tls_mount"; -pub const CLIENT_TLS_DIR: &str = "/stackable/client_tls"; -pub const CLIENT_TLS_MOUNT_DIR: &str = "/stackable/client_tls_mount"; -pub const SYSTEM_TRUST_STORE_DIR: &str = "/etc/pki/java/cacerts"; - pub const LOGBACK_CONFIG_FILE: &str = "logback.xml"; pub const LOG4J_CONFIG_FILE: &str = "log4j.properties"; @@ -56,7 +55,6 @@ const MAX_PREPARE_LOG_FILE_SIZE_IN_MIB: u32 = 1; pub const LOG_VOLUME_SIZE_IN_MIB: u32 = MAX_ZK_LOG_FILES_SIZE_IN_MIB + MAX_PREPARE_LOG_FILE_SIZE_IN_MIB; const JVM_HEAP_FACTOR: f32 = 0.8; -const TLS_DEFAULT_SECRET_CLASS: &str = "tls"; pub const DOCKER_IMAGE_BASE_NAME: &str = "zookeeper"; @@ -88,75 +86,51 @@ pub enum Error { )] #[serde(rename_all = "camelCase")] pub struct ZookeeperClusterSpec { - /// Emergency stop button, if `true` then all pods are stopped without affecting configuration (as setting `replicas` to `0` would) - #[serde(skip_serializing_if = "Option::is_none")] - pub stopped: Option, - /// Desired ZooKeeper version + /// Global ZooKeeper cluster configuration that applies to all roles and role groups. + #[serde(default = "cluster_config_default")] + pub cluster_config: ZookeeperClusterConfig, + /// Desired ZooKeeper image to use. pub image: ProductImage, - #[serde(default = "global_config_default")] - pub config: GlobalZookeeperConfig, + /// ZooKeeper server configuration. #[serde(skip_serializing_if = "Option::is_none")] pub servers: Option>, - /// Name of the Vector discovery ConfigMap. - /// It must contain the key `ADDRESS` with the address of the Vector aggregator. + /// Emergency stop button, if `true` then all pods are stopped without affecting configuration (as setting `replicas` to `0` would). #[serde(skip_serializing_if = "Option::is_none")] - pub vector_aggregator_config_map_name: Option, + pub stopped: Option, } #[derive(Clone, Deserialize, Debug, Eq, JsonSchema, PartialEq, Serialize)] #[serde(rename_all = "camelCase")] -pub struct GlobalZookeeperConfig { - /// Only affects client connections. This setting controls: - /// - If TLS encryption is used at all - /// - Which cert the servers should use to authenticate themselves against the client - /// Defaults to `TlsSecretClass` { secret_class: "tls".to_string() }. - #[serde(default = "tls_default", skip_serializing_if = "Option::is_none")] - pub tls: Option, - /// Only affects client connections. This setting controls: - /// - If clients need to authenticate themselves against the server via TLS - /// - Which ca.crt to use when validating the provided client certs - /// Defaults to `None` +pub struct ZookeeperClusterConfig { + /// Authentication class settings for ZooKeeper like mTLS authentication. + #[serde(default)] + pub authentication: Vec, + /// Logging options for ZooKeeper. #[serde(skip_serializing_if = "Option::is_none")] - pub client_authentication: Option, - /// Only affects quorum communication. Use mutual verification between Zookeeper Nodes - /// (mandatory). This setting controls: - /// - Which cert the servers should use to authenticate themselves against other servers - /// - Which ca.crt to use when validating the other server - #[serde(default = "quorum_tls_default")] - pub quorum_tls_secret_class: String, + pub logging: Option, + /// TLS encryption settings for ZooKeeper (server, quorum). + #[serde( + default = "tls::default_zookeeper_tls", + skip_serializing_if = "Option::is_none" + )] + pub tls: Option, } -/// This is to have the GlobalZookeeperConfig.tls default if e.g. only client_authentication is set -fn tls_default() -> Option { - Some(TlsSecretClass { - secret_class: TLS_DEFAULT_SECRET_CLASS.into(), - }) -} - -/// This is to set the quorum if e.g. only GlobalZookeeperConfig.tls is set -fn quorum_tls_default() -> String { - TLS_DEFAULT_SECRET_CLASS.to_string() -} - -/// This is to set defaults if the config is left out completely -fn global_config_default() -> GlobalZookeeperConfig { - GlobalZookeeperConfig { - tls: tls_default(), - client_authentication: None, - quorum_tls_secret_class: quorum_tls_default(), +fn cluster_config_default() -> ZookeeperClusterConfig { + ZookeeperClusterConfig { + authentication: vec![], + logging: None, + tls: tls::default_zookeeper_tls(), } } -#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)] -#[serde(rename_all = "camelCase")] -pub struct TlsSecretClass { - pub secret_class: String, -} - -#[derive(Clone, Default, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)] +#[derive(Clone, Deserialize, Debug, Eq, JsonSchema, PartialEq, Serialize)] #[serde(rename_all = "camelCase")] -pub struct ClientAuthenticationClass { - pub authentication_class: String, +pub struct ZookeeperLogging { + /// Name of the Vector discovery ConfigMap. + /// It must contain the key `ADDRESS` with the address of the Vector aggregator. + #[serde(skip_serializing_if = "Option::is_none")] + pub vector_aggregator_config_map_name: Option, } #[derive(Clone, Debug, Default, Fragment, JsonSchema, PartialEq)] @@ -232,27 +206,6 @@ impl ZookeeperConfig { pub const MYID_OFFSET: &'static str = "MYID_OFFSET"; pub const SERVER_JVMFLAGS: &'static str = "SERVER_JVMFLAGS"; pub const ZK_SERVER_HEAP: &'static str = "ZK_SERVER_HEAP"; - pub const CLIENT_PORT: &'static str = "clientPort"; - - // Quorum TLS - pub const SSL_QUORUM: &'static str = "sslQuorum"; - pub const SSL_QUORUM_CLIENT_AUTH: &'static str = "ssl.quorum.clientAuth"; - pub const SSL_QUORUM_HOST_NAME_VERIFICATION: &'static str = "ssl.quorum.hostnameVerification"; - pub const SSL_QUORUM_KEY_STORE_LOCATION: &'static str = "ssl.quorum.keyStore.location"; - pub const SSL_QUORUM_KEY_STORE_PASSWORD: &'static str = "ssl.quorum.keyStore.password"; - pub const SSL_QUORUM_TRUST_STORE_LOCATION: &'static str = "ssl.quorum.trustStore.location"; - pub const SSL_QUORUM_TRUST_STORE_PASSWORD: &'static str = "ssl.quorum.trustStore.password"; - // Client TLS - pub const SECURE_CLIENT_PORT: &'static str = "secureClientPort"; - pub const SSL_CLIENT_AUTH: &'static str = "ssl.clientAuth"; - pub const SSL_HOST_NAME_VERIFICATION: &'static str = "ssl.hostnameVerification"; - pub const SSL_KEY_STORE_LOCATION: &'static str = "ssl.keyStore.location"; - pub const SSL_KEY_STORE_PASSWORD: &'static str = "ssl.keyStore.password"; - pub const SSL_TRUST_STORE_LOCATION: &'static str = "ssl.trustStore.location"; - pub const SSL_TRUST_STORE_PASSWORD: &'static str = "ssl.trustStore.password"; - // Common TLS - pub const SSL_AUTH_PROVIDER_X509: &'static str = "authProvider.x509"; - pub const SERVER_CNXN_FACTORY: &'static str = "serverCnxnFactory"; fn default_server_config() -> ZookeeperConfigFragment { ZookeeperConfigFragment { @@ -328,7 +281,7 @@ impl Configuration for ZookeeperConfigFragment { fn compute_files( &self, - resource: &Self::Configurable, + _resource: &Self::Configurable, _role_name: &str, _file: &str, ) -> Result>, ConfigError> { @@ -356,107 +309,6 @@ impl Configuration for ZookeeperConfigFragment { Some(STACKABLE_DATA_DIR.to_string()), ); - // Quorum TLS - result.insert( - ZookeeperConfig::SSL_QUORUM.to_string(), - Some("true".to_string()), - ); - result.insert( - ZookeeperConfig::SSL_QUORUM_HOST_NAME_VERIFICATION.to_string(), - Some("true".to_string()), - ); - result.insert( - ZookeeperConfig::SSL_QUORUM_CLIENT_AUTH.to_string(), - Some("need".to_string()), - ); - result.insert( - ZookeeperConfig::SERVER_CNXN_FACTORY.to_string(), - Some("org.apache.zookeeper.server.NettyServerCnxnFactory".to_string()), - ); - result.insert( - ZookeeperConfig::SSL_AUTH_PROVIDER_X509.to_string(), - Some("org.apache.zookeeper.server.auth.X509AuthenticationProvider".to_string()), - ); - // The keystore and truststore passwords should not be in the configmap and are generated - // and written later via script in the init container - result.insert( - ZookeeperConfig::SSL_QUORUM_KEY_STORE_LOCATION.to_string(), - Some(format!("{dir}/keystore.p12", dir = QUORUM_TLS_DIR)), - ); - result.insert( - ZookeeperConfig::SSL_QUORUM_TRUST_STORE_LOCATION.to_string(), - Some(format!("{dir}/truststore.p12", dir = QUORUM_TLS_DIR)), - ); - - // Client TLS - if resource.client_tls_enabled() { - // We set only the clientPort and portUnification here because otherwise there is a port bind exception - // See: https://issues.apache.org/jira/browse/ZOOKEEPER-4276 - // --> Normally we would like to only set the secureClientPort (check out commented code below) - // What we tried: - // 1) Set clientPort and secureClientPort will fail with - // "static.config different from dynamic config .. " - // result.insert( - // ZookeeperConfig::CLIENT_PORT.to_string(), - // Some(CLIENT_PORT.to_string()), - // ); - // result.insert( - // ZookeeperConfig::SECURE_CLIENT_PORT.to_string(), - // Some(SECURE_CLIENT_PORT.to_string()), - // ); - - // 2) Setting only secureClientPort will result in the above mentioned bind exception. - // The NettyFactory tries to bind multiple times on the secureClientPort. - // result.insert( - // ZookeeperConfig::SECURE_CLIENT_PORT.to_string(), - // Some(resource.client_port().to_string()), - // ); - - // 3) Using the clientPort and portUnification still allows plaintext connection without - // authentication, but at least TLS and authentication works when connecting securely. - result.insert( - ZookeeperConfig::CLIENT_PORT.to_string(), - Some(resource.client_port().to_string()), - ); - result.insert( - "client.portUnification".to_string(), - Some("true".to_string()), - ); - // TODO: Remove clientPort and portUnification (above) in favor of secureClientPort once the bug is fixed - // result.insert( - // ZookeeperConfig::SECURE_CLIENT_PORT.to_string(), - // Some(resource.client_port().to_string()), - // ); - // END TICKET - - result.insert( - ZookeeperConfig::SSL_HOST_NAME_VERIFICATION.to_string(), - Some("true".to_string()), - ); - // The keystore and truststore passwords should not be in the configmap and are generated - // and written later via script in the init container - result.insert( - ZookeeperConfig::SSL_KEY_STORE_LOCATION.to_string(), - Some(format!("{dir}/keystore.p12", dir = CLIENT_TLS_DIR)), - ); - result.insert( - ZookeeperConfig::SSL_TRUST_STORE_LOCATION.to_string(), - Some(format!("{dir}/truststore.p12", dir = CLIENT_TLS_DIR)), - ); - // Check if we need to enable authentication - if resource.client_tls_authentication_class().is_some() { - result.insert( - ZookeeperConfig::SSL_CLIENT_AUTH.to_string(), - Some("need".to_string()), - ); - } - } else { - result.insert( - ZookeeperConfig::CLIENT_PORT.to_string(), - Some(resource.client_port().to_string()), - ); - } - Ok(result) } } @@ -566,39 +418,6 @@ impl ZookeeperCluster { Ok(pod_refs.into_iter()) } - pub fn client_port(&self) -> u16 { - if self.client_tls_enabled() { - SECURE_CLIENT_PORT - } else { - CLIENT_PORT - } - } - - /// Returns the secret class for client connection encryption. Defaults to `tls`. - pub fn client_tls_secret_class(&self) -> Option<&TlsSecretClass> { - let spec: &ZookeeperClusterSpec = &self.spec; - spec.config.tls.as_ref() - } - - /// Checks if we should use TLS to encrypt client connections. - pub fn client_tls_enabled(&self) -> bool { - self.client_tls_secret_class().is_some() || self.client_tls_authentication_class().is_some() - } - - /// Returns the authentication class used for client authentication - pub fn client_tls_authentication_class(&self) -> Option<&str> { - let spec: &ZookeeperClusterSpec = &self.spec; - spec.config - .client_authentication - .as_ref() - .map(|auth| auth.authentication_class.as_ref()) - } - - /// Returns the secret class for internal server encryption - pub fn quorum_tls_secret_class(&self) -> &str { - &self.spec.config.quorum_tls_secret_class - } - pub fn merged_config( &self, role: &ZookeeperRole, @@ -735,6 +554,24 @@ pub struct ZookeeperZnodeSpec { mod tests { use super::*; + fn get_server_secret_class(zk: &ZookeeperCluster) -> Option<&str> { + zk.spec + .cluster_config + .tls + .as_ref() + .and_then(|tls| tls.server_secret_class.as_deref()) + } + + fn get_quorum_secret_class(zk: &ZookeeperCluster) -> &str { + zk.spec + .cluster_config + .tls + .as_ref() + .unwrap() + .quorum_secret_class + .as_str() + } + #[test] fn test_client_tls() { let input = r#" @@ -749,12 +586,12 @@ mod tests { "#; let zookeeper: ZookeeperCluster = serde_yaml::from_str(input).expect("illegal test input"); assert_eq!( - zookeeper.client_tls_secret_class().unwrap().secret_class, - TLS_DEFAULT_SECRET_CLASS.to_string() + get_server_secret_class(&zookeeper), + tls::server_tls_default().as_deref() ); assert_eq!( - zookeeper.quorum_tls_secret_class(), - TLS_DEFAULT_SECRET_CLASS.to_string() + get_quorum_secret_class(&zookeeper), + tls::quorum_tls_default().as_str() ); let input = r#" @@ -766,19 +603,19 @@ mod tests { image: productVersion: "3.8.0" stackableVersion: "0.8.0" - config: + clusterConfig: tls: - secretClass: simple-zookeeper-client-tls + serverSecretClass: simple-zookeeper-client-tls "#; let zookeeper: ZookeeperCluster = serde_yaml::from_str(input).expect("illegal test input"); assert_eq!( - zookeeper.client_tls_secret_class().unwrap().secret_class, - "simple-zookeeper-client-tls".to_string() + get_server_secret_class(&zookeeper), + Some("simple-zookeeper-client-tls") ); assert_eq!( - zookeeper.quorum_tls_secret_class(), - TLS_DEFAULT_SECRET_CLASS + get_quorum_secret_class(&zookeeper), + tls::quorum_tls_default().as_str() ); let input = r#" @@ -790,14 +627,15 @@ mod tests { image: productVersion: "3.8.0" stackableVersion: "0.8.0" - config: - tls: null + clusterConfig: + tls: + serverSecretClass: null "#; let zookeeper: ZookeeperCluster = serde_yaml::from_str(input).expect("illegal test input"); - assert_eq!(zookeeper.client_tls_secret_class(), None); + assert_eq!(get_server_secret_class(&zookeeper), None); assert_eq!( - zookeeper.quorum_tls_secret_class(), - TLS_DEFAULT_SECRET_CLASS.to_string() + get_quorum_secret_class(&zookeeper), + tls::quorum_tls_default().as_str() ); let input = r#" @@ -809,16 +647,17 @@ mod tests { image: productVersion: "3.8.0" stackableVersion: "0.8.0" - config: - quorumTlsSecretClass: simple-zookeeper-quorum-tls + clusterConfig: + tls: + quorumSecretClass: simple-zookeeper-quorum-tls "#; let zookeeper: ZookeeperCluster = serde_yaml::from_str(input).expect("illegal test input"); assert_eq!( - zookeeper.client_tls_secret_class().unwrap().secret_class, - TLS_DEFAULT_SECRET_CLASS.to_string() + get_server_secret_class(&zookeeper), + tls::server_tls_default().as_deref() ); assert_eq!( - zookeeper.quorum_tls_secret_class(), + get_quorum_secret_class(&zookeeper), "simple-zookeeper-quorum-tls" ); } @@ -836,13 +675,14 @@ mod tests { stackableVersion: "0.8.0" "#; let zookeeper: ZookeeperCluster = serde_yaml::from_str(input).expect("illegal test input"); + assert_eq!( - zookeeper.quorum_tls_secret_class(), - TLS_DEFAULT_SECRET_CLASS.to_string() + get_server_secret_class(&zookeeper), + tls::server_tls_default().as_deref() ); assert_eq!( - zookeeper.client_tls_secret_class().unwrap().secret_class, - TLS_DEFAULT_SECRET_CLASS + get_quorum_secret_class(&zookeeper), + tls::quorum_tls_default() ); let input = r#" @@ -854,17 +694,18 @@ mod tests { image: productVersion: "3.8.0" stackableVersion: "0.8.0" - config: - quorumTlsSecretClass: simple-zookeeper-quorum-tls + clusterConfig: + tls: + quorumSecretClass: simple-zookeeper-quorum-tls "#; let zookeeper: ZookeeperCluster = serde_yaml::from_str(input).expect("illegal test input"); assert_eq!( - zookeeper.quorum_tls_secret_class(), - "simple-zookeeper-quorum-tls".to_string() + get_server_secret_class(&zookeeper), + tls::server_tls_default().as_deref() ); assert_eq!( - zookeeper.client_tls_secret_class().unwrap().secret_class, - TLS_DEFAULT_SECRET_CLASS + get_quorum_secret_class(&zookeeper), + "simple-zookeeper-quorum-tls" ); let input = r#" @@ -876,18 +717,18 @@ mod tests { image: productVersion: "3.8.0" stackableVersion: "0.8.0" - config: + clusterConfig: tls: - secretClass: simple-zookeeper-client-tls + serverSecretClass: simple-zookeeper-server-tls "#; let zookeeper: ZookeeperCluster = serde_yaml::from_str(input).expect("illegal test input"); assert_eq!( - zookeeper.quorum_tls_secret_class(), - TLS_DEFAULT_SECRET_CLASS.to_string() + get_server_secret_class(&zookeeper), + Some("simple-zookeeper-server-tls") ); assert_eq!( - zookeeper.client_tls_secret_class().unwrap().secret_class, - "simple-zookeeper-client-tls" + get_quorum_secret_class(&zookeeper), + tls::quorum_tls_default().as_str() ); } } diff --git a/rust/crd/src/security.rs b/rust/crd/src/security.rs new file mode 100644 index 00000000..3f117605 --- /dev/null +++ b/rust/crd/src/security.rs @@ -0,0 +1,377 @@ +//! A helper module to process Apache ZooKeeper security configuration +//! +//! This module merges the `tls` and `authentication` module and offers better accessibility +//! and helper functions +//! +//! This is required due to overlaps between TLS encryption and e.g. mTLS authentication or Kerberos + +use crate::{ + authentication, authentication::ResolvedAuthenticationClasses, tls, ZookeeperCluster, + STACKABLE_RW_CONFIG_DIR, ZOOKEEPER_PROPERTIES_FILE, +}; + +use snafu::{ResultExt, Snafu}; +use stackable_operator::{ + builder::{ContainerBuilder, PodBuilder, SecretOperatorVolumeSourceBuilder, VolumeBuilder}, + client::Client, + commons::authentication::AuthenticationClassProvider, + k8s_openapi::api::core::v1::Volume, +}; +use std::collections::BTreeMap; + +#[derive(Snafu, Debug)] +pub enum Error { + #[snafu(display("failed to process authentication class"))] + InvalidAuthenticationClassConfiguration { source: authentication::Error }, +} + +/// Helper struct combining TLS settings for server and quorum with the resolved AuthenticationClasses +pub struct ZookeeperSecurity { + resolved_authentication_classes: ResolvedAuthenticationClasses, + server_secret_class: Option, + quorum_secret_class: String, +} + +impl ZookeeperSecurity { + // ports + pub const CLIENT_PORT: u16 = 2181; + pub const CLIENT_PORT_NAME: &'static str = "clientPort"; + pub const SECURE_CLIENT_PORT: u16 = 2282; + pub const SECURE_CLIENT_PORT_NAME: &'static str = "secureClientPort"; + // directories + pub const QUORUM_TLS_DIR: &'static str = "/stackable/quorum_tls"; + pub const QUORUM_TLS_MOUNT_DIR: &'static str = "/stackable/quorum_tls_mount"; + pub const SERVER_TLS_DIR: &'static str = "/stackable/server_tls"; + pub const SERVER_TLS_MOUNT_DIR: &'static str = "/stackable/server_tls_mount"; + pub const SYSTEM_TRUST_STORE_DIR: &'static str = "/etc/pki/java/cacerts"; + // Quorum TLS + pub const SSL_QUORUM: &'static str = "sslQuorum"; + pub const SSL_QUORUM_CLIENT_AUTH: &'static str = "ssl.quorum.clientAuth"; + pub const SSL_QUORUM_HOST_NAME_VERIFICATION: &'static str = "ssl.quorum.hostnameVerification"; + pub const SSL_QUORUM_KEY_STORE_LOCATION: &'static str = "ssl.quorum.keyStore.location"; + pub const SSL_QUORUM_KEY_STORE_PASSWORD: &'static str = "ssl.quorum.keyStore.password"; + pub const SSL_QUORUM_TRUST_STORE_LOCATION: &'static str = "ssl.quorum.trustStore.location"; + pub const SSL_QUORUM_TRUST_STORE_PASSWORD: &'static str = "ssl.quorum.trustStore.password"; + // Client TLS + pub const SSL_CLIENT_AUTH: &'static str = "ssl.clientAuth"; + pub const SSL_HOST_NAME_VERIFICATION: &'static str = "ssl.hostnameVerification"; + pub const SSL_KEY_STORE_LOCATION: &'static str = "ssl.keyStore.location"; + pub const SSL_KEY_STORE_PASSWORD: &'static str = "ssl.keyStore.password"; + pub const SSL_TRUST_STORE_LOCATION: &'static str = "ssl.trustStore.location"; + pub const SSL_TRUST_STORE_PASSWORD: &'static str = "ssl.trustStore.password"; + // Common TLS + pub const SSL_AUTH_PROVIDER_X509: &'static str = "authProvider.x509"; + pub const SERVER_CNXN_FACTORY: &'static str = "serverCnxnFactory"; + // Mis + pub const STORE_PASSWORD_ENV: &'static str = "STORE_PASSWORD"; + + /// Create a `ZookeeperSecurity` struct from the Zookeeper custom resource and resolve + /// all provided `AuthenticationClass` references. + pub async fn new_from_zookeeper_cluster( + client: &Client, + zk: &ZookeeperCluster, + ) -> Result { + Ok(ZookeeperSecurity { + resolved_authentication_classes: authentication::resolve_authentication_classes( + client, + &zk.spec.cluster_config.authentication, + ) + .await + .context(InvalidAuthenticationClassConfigurationSnafu)?, + server_secret_class: zk + .spec + .cluster_config + .tls + .as_ref() + .and_then(|tls| tls.server_secret_class.clone()), + quorum_secret_class: zk + .spec + .cluster_config + .tls + .as_ref() + .map(|tls| tls.quorum_secret_class.clone()) + .unwrap_or_else(tls::quorum_tls_default), + }) + } + + /// Check if TLS encryption is enabled. This could be due to: + /// - A provided server `SecretClass` + /// - A provided client `AuthenticationClass` + /// This affects init container commands, ZooKeeper configuration, volume mounts and + /// the ZooKeeper client port + pub fn tls_enabled(&self) -> bool { + // TODO: This must be adapted if other authentication methods are supported and require TLS + self.server_secret_class.is_some() + || self + .resolved_authentication_classes + .get_tls_authentication_class() + .is_some() + } + + /// Return the ZooKeeper (secure) client port depending on tls or authentication settings. + pub fn client_port(&self) -> u16 { + if self.tls_enabled() { + Self::SECURE_CLIENT_PORT + } else { + Self::CLIENT_PORT + } + } + + /// Returns required (init) container commands to generate keystores and truststores + /// depending on the tls and authentication settings. + pub fn commands(&self) -> Vec { + let mut args = vec![]; + // Quorum + args.push(Self::generate_password(Self::STORE_PASSWORD_ENV)); + args.extend(Self::create_key_and_trust_store_cmd( + Self::QUORUM_TLS_MOUNT_DIR, + Self::QUORUM_TLS_DIR, + "quorum-tls", + Self::STORE_PASSWORD_ENV, + )); + args.extend(vec![ + Self::write_store_password_to_config( + Self::SSL_QUORUM_KEY_STORE_PASSWORD, + STACKABLE_RW_CONFIG_DIR, + Self::STORE_PASSWORD_ENV, + ), + Self::write_store_password_to_config( + Self::SSL_QUORUM_TRUST_STORE_PASSWORD, + STACKABLE_RW_CONFIG_DIR, + Self::STORE_PASSWORD_ENV, + ), + ]); + + // server-tls or client-auth-tls (only the certificates specified are accepted) + if self.tls_enabled() { + args.push(Self::generate_password(Self::STORE_PASSWORD_ENV)); + + args.extend(Self::create_key_and_trust_store_cmd( + Self::SERVER_TLS_MOUNT_DIR, + Self::SERVER_TLS_DIR, + "server-tls", + Self::STORE_PASSWORD_ENV, + )); + + args.extend(vec![ + Self::write_store_password_to_config( + Self::SSL_KEY_STORE_PASSWORD, + STACKABLE_RW_CONFIG_DIR, + Self::STORE_PASSWORD_ENV, + ), + Self::write_store_password_to_config( + Self::SSL_TRUST_STORE_PASSWORD, + STACKABLE_RW_CONFIG_DIR, + Self::STORE_PASSWORD_ENV, + ), + ]); + } + + args + } + + /// Adds required volumes and volume mounts to the pod and container builders + /// depending on the tls and authentication settings. + pub fn add_volume_mounts( + &self, + pod_builder: &mut PodBuilder, + cb_prepare: &mut ContainerBuilder, + cb_zookeeper: &mut ContainerBuilder, + ) { + let tls_secret_class = self.get_tls_secret_class(); + + if let Some(secret_class) = tls_secret_class { + // mounts for secret volume + cb_prepare.add_volume_mount("server-tls-mount", Self::SERVER_TLS_MOUNT_DIR); + cb_zookeeper.add_volume_mount("server-tls-mount", Self::SERVER_TLS_MOUNT_DIR); + pod_builder.add_volume(Self::create_tls_volume("server-tls-mount", secret_class)); + // empty mount for trust and keystore + cb_prepare.add_volume_mount("server-tls", Self::SERVER_TLS_DIR); + cb_zookeeper.add_volume_mount("server-tls", Self::SERVER_TLS_DIR); + pod_builder.add_volume( + VolumeBuilder::new("server-tls") + .with_empty_dir(Some(""), None) + .build(), + ); + } + + // quorum + // mounts for secret volume + cb_prepare.add_volume_mount("quorum-tls-mount", Self::QUORUM_TLS_MOUNT_DIR); + cb_zookeeper.add_volume_mount("quorum-tls-mount", Self::QUORUM_TLS_MOUNT_DIR); + pod_builder.add_volume(Self::create_tls_volume( + "quorum-tls-mount", + &self.quorum_secret_class, + )); + // empty mount for trust and keystore + cb_prepare.add_volume_mount("quorum-tls", Self::QUORUM_TLS_DIR); + cb_zookeeper.add_volume_mount("quorum-tls", Self::QUORUM_TLS_DIR); + pod_builder.add_volume( + VolumeBuilder::new("quorum-tls") + .with_empty_dir(Some(""), None) + .build(), + ); + } + + /// Returns required ZooKeeper configuration settings for the `zoo.cfg` properties file + /// depending on the tls and authentication settings. + pub fn config_settings(&self) -> BTreeMap { + let mut config = BTreeMap::new(); + // Quorum TLS + config.insert(Self::SSL_QUORUM.to_string(), "true".to_string()); + config.insert( + Self::SSL_QUORUM_HOST_NAME_VERIFICATION.to_string(), + "true".to_string(), + ); + config.insert(Self::SSL_QUORUM_CLIENT_AUTH.to_string(), "need".to_string()); + config.insert( + Self::SERVER_CNXN_FACTORY.to_string(), + "org.apache.zookeeper.server.NettyServerCnxnFactory".to_string(), + ); + config.insert( + Self::SSL_AUTH_PROVIDER_X509.to_string(), + "org.apache.zookeeper.server.auth.X509AuthenticationProvider".to_string(), + ); + // The keystore and truststore passwords should not be in the configmap and are generated + // and written later via script in the init container + config.insert( + Self::SSL_QUORUM_KEY_STORE_LOCATION.to_string(), + format!("{dir}/keystore.p12", dir = Self::QUORUM_TLS_DIR), + ); + config.insert( + Self::SSL_QUORUM_TRUST_STORE_LOCATION.to_string(), + format!("{dir}/truststore.p12", dir = Self::QUORUM_TLS_DIR), + ); + + // Server TLS + if self.tls_enabled() { + // We set only the clientPort and portUnification here because otherwise there is a port bind exception + // See: https://issues.apache.org/jira/browse/ZOOKEEPER-4276 + // --> Normally we would like to only set the secureClientPort (check out commented code below) + // What we tried: + // 1) Set clientPort and secureClientPort will fail with + // "static.config different from dynamic config .. " + // config.insert( + // Self::CLIENT_PORT_NAME.to_string(), + // CLIENT_PORT.to_string(), + // ); + // config.insert( + // Self::SECURE_CLIENT_PORT_NAME.to_string(), + // SECURE_CLIENT_PORT.to_string(), + // ); + + // 2) Setting only secureClientPort will config in the above mentioned bind exception. + // The NettyFactory tries to bind multiple times on the secureClientPort. + // config.insert( + // Self::SECURE_CLIENT_PORT_NAME.to_string(), + // self.client_port(.to_string()), + // ); + + // 3) Using the clientPort and portUnification still allows plaintext connection without + // authentication, but at least TLS and authentication works when connecting securely. + config.insert( + Self::CLIENT_PORT_NAME.to_string(), + self.client_port().to_string(), + ); + config.insert("client.portUnification".to_string(), "true".to_string()); + // TODO: Remove clientPort and portUnification (above) in favor of secureClientPort once the bug is fixed + // config.insert( + // Self::SECURE_CLIENT_PORT_NAME.to_string(), + // self.client_port(.to_string()), + // ); + // END TICKET + + config.insert( + Self::SSL_HOST_NAME_VERIFICATION.to_string(), + "true".to_string(), + ); + // The keystore and truststore passwords should not be in the configmap and are generated + // and written later via script in the init container + config.insert( + Self::SSL_KEY_STORE_LOCATION.to_string(), + format!("{dir}/keystore.p12", dir = Self::SERVER_TLS_DIR), + ); + config.insert( + Self::SSL_TRUST_STORE_LOCATION.to_string(), + format!("{dir}/truststore.p12", dir = Self::SERVER_TLS_DIR), + ); + // Check if we need to enable client TLS authentication + if self + .resolved_authentication_classes + .get_tls_authentication_class() + .is_some() + { + config.insert(Self::SSL_CLIENT_AUTH.to_string(), "need".to_string()); + } + } else { + config.insert( + Self::CLIENT_PORT_NAME.to_string(), + self.client_port().to_string(), + ); + } + + config + } + + /// Returns the `SecretClass` provided in a `AuthenticationClass` for TLS. + fn get_tls_secret_class(&self) -> Option<&String> { + self.resolved_authentication_classes + .get_tls_authentication_class() + .and_then(|auth_class| match &auth_class.spec.provider { + AuthenticationClassProvider::Tls(tls) => tls.client_cert_secret_class.as_ref(), + _ => None, + }) + .or(self.server_secret_class.as_ref()) + } + + /// Creates ephemeral volumes to mount the `SecretClass` into the Pods + fn create_tls_volume(volume_name: &str, secret_class_name: &str) -> Volume { + VolumeBuilder::new(volume_name) + .ephemeral( + SecretOperatorVolumeSourceBuilder::new(secret_class_name) + .with_pod_scope() + .with_node_scope() + .build(), + ) + .build() + } + + /// Generates the shell script to retrieve a random 20 character password + fn generate_password(store_password_env_var: &str) -> String { + format!( + "export {store_password_env_var}=$(tr -dc A-Za-z0-9 String { + format!( + "echo {property}=${store_password_env_var} >> {rw_conf_dir}/{ZOOKEEPER_PROPERTIES_FILE}", + ) + } + + /// Generates the shell script to create key and trust stores from the certificates provided + /// by the secret operator + fn create_key_and_trust_store_cmd( + mount_directory: &str, + stackable_directory: &str, + alias_name: &str, + store_password_env_var: &str, + ) -> Vec { + vec![ + format!("echo [{stackable_directory}] Cleaning up truststore - just in case"), + format!("rm -f {stackable_directory}/truststore.p12"), + format!("echo [{stackable_directory}] Creating truststore"), + format!("keytool -importcert -file {mount_directory}/ca.crt -keystore {stackable_directory}/truststore.p12 -storetype pkcs12 -noprompt -alias {alias_name} -storepass ${store_password_env_var}"), + format!("echo [{stackable_directory}] Creating certificate chain"), + format!("cat {mount_directory}/ca.crt {mount_directory}/tls.crt > {stackable_directory}/chain.crt"), + format!("echo [{stackable_directory}] Creating keystore"), + format!("openssl pkcs12 -export -in {stackable_directory}/chain.crt -inkey {mount_directory}/tls.key -out {stackable_directory}/keystore.p12 --passout pass:${store_password_env_var}"), + ] + } +} diff --git a/rust/crd/src/tls.rs b/rust/crd/src/tls.rs new file mode 100644 index 00000000..22e9a9e8 --- /dev/null +++ b/rust/crd/src/tls.rs @@ -0,0 +1,45 @@ +use serde::{Deserialize, Serialize}; +use stackable_operator::schemars::{self, JsonSchema}; + +const TLS_DEFAULT_SECRET_CLASS: &str = "tls"; + +#[derive(Clone, Deserialize, Debug, Eq, JsonSchema, PartialEq, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct ZookeeperTls { + /// The to use for + /// internal quorum communication. Use mutual verification between Zookeeper Nodes + /// (mandatory). This setting controls: + /// - Which cert the servers should use to authenticate themselves against other servers + /// - Which ca.crt to use when validating the other server + /// Defaults to `tls` + #[serde(default = "quorum_tls_default")] + pub quorum_secret_class: String, + /// The to use for + /// client connections. This setting controls: + /// - If TLS encryption is used at all + /// - Which cert the servers should use to authenticate themselves against the client + /// Defaults to `tls`. + #[serde( + default = "server_tls_default", + skip_serializing_if = "Option::is_none" + )] + pub server_secret_class: Option, +} + +/// Default TLS settings. Internal and server communication default to "tls" secret class. +pub fn default_zookeeper_tls() -> Option { + Some(ZookeeperTls { + quorum_secret_class: quorum_tls_default(), + server_secret_class: server_tls_default(), + }) +} + +/// Helper methods to provide defaults in the CRDs and tests +pub fn server_tls_default() -> Option { + Some(TLS_DEFAULT_SECRET_CLASS.into()) +} + +/// Helper methods to provide defaults in the CRDs and tests +pub fn quorum_tls_default() -> String { + TLS_DEFAULT_SECRET_CLASS.into() +} diff --git a/rust/operator-binary/src/command.rs b/rust/operator-binary/src/command.rs index bc5e52d8..09c871f6 100644 --- a/rust/operator-binary/src/command.rs +++ b/rust/operator-binary/src/command.rs @@ -1,13 +1,7 @@ -use stackable_zookeeper_crd::{ - ZookeeperCluster, ZookeeperConfig, CLIENT_TLS_DIR, CLIENT_TLS_MOUNT_DIR, QUORUM_TLS_DIR, - QUORUM_TLS_MOUNT_DIR, STACKABLE_CONFIG_DIR, STACKABLE_DATA_DIR, STACKABLE_RW_CONFIG_DIR, - ZOOKEEPER_PROPERTIES_FILE, -}; +use stackable_zookeeper_crd::{STACKABLE_CONFIG_DIR, STACKABLE_DATA_DIR, STACKABLE_RW_CONFIG_DIR}; -const STORE_PASSWORD_ENV: &str = "STORE_PASSWORD"; - -pub fn create_init_container_command_args(zk: &ZookeeperCluster) -> String { - let mut args = vec![ +pub fn create_init_container_command_args() -> Vec { + vec![ // copy config files to a writeable empty folder in order to set key and // truststore passwords in the init container via script format!( @@ -20,74 +14,9 @@ pub fn create_init_container_command_args(zk: &ZookeeperCluster) -> String { conf = STACKABLE_CONFIG_DIR, rw_conf = STACKABLE_RW_CONFIG_DIR ), - ]; - - // Quorum - args.push(generate_password()); - args.extend(create_key_and_trust_store_cmd( - QUORUM_TLS_MOUNT_DIR, - QUORUM_TLS_DIR, - "quorum-tls", - )); - args.extend(vec![ - write_store_password_to_config(ZookeeperConfig::SSL_QUORUM_KEY_STORE_PASSWORD), - write_store_password_to_config(ZookeeperConfig::SSL_QUORUM_TRUST_STORE_PASSWORD), - ]); - - // client-tls and client-auth-tls (only the certificates specified are accepted) - if zk.client_tls_enabled() { - args.push(generate_password()); - - args.extend(create_key_and_trust_store_cmd( - CLIENT_TLS_MOUNT_DIR, - CLIENT_TLS_DIR, - "client-tls", - )); - - args.extend(vec![ - write_store_password_to_config(ZookeeperConfig::SSL_KEY_STORE_PASSWORD), - write_store_password_to_config(ZookeeperConfig::SSL_TRUST_STORE_PASSWORD), - ]); - } - - args.push(format!( - "expr $MYID_OFFSET + $(echo $POD_NAME | sed 's/.*-//') > {dir}/myid", - dir = STACKABLE_DATA_DIR - )); - - args.join(" && ") -} - -/// Generates the shell script to retrieve a random 20 character password -fn generate_password() -> String { - format!("export {STORE_PASSWORD_ENV}=$(tr -dc A-Za-z0-9 String { - format!( - "echo {property}=${STORE_PASSWORD_ENV} >> {rwconf}/{ZOOKEEPER_PROPERTIES_FILE}", - property = property, - rwconf = STACKABLE_RW_CONFIG_DIR - ) -} - -/// Generates the shell script to create key and trust stores from the certificates provided -/// by the secret operator -fn create_key_and_trust_store_cmd( - mount_directory: &str, - stackable_directory: &str, - alias_name: &str, -) -> Vec { - vec![ - format!("echo [{stackable_directory}] Cleaning up truststore - just in case"), - format!("rm -f {stackable_directory}/truststore.p12"), - format!("echo [{stackable_directory}] Creating truststore"), - format!("keytool -importcert -file {mount_directory}/ca.crt -keystore {stackable_directory}/truststore.p12 -storetype pkcs12 -noprompt -alias {alias_name} -storepass ${STORE_PASSWORD_ENV}"), - format!("echo [{stackable_directory}] Creating certificate chain"), - format!("cat {mount_directory}/ca.crt {mount_directory}/tls.crt > {stackable_directory}/chain.crt"), - format!("echo [{stackable_directory}] Creating keystore"), - format!("openssl pkcs12 -export -in {stackable_directory}/chain.crt -inkey {mount_directory}/tls.key -out {stackable_directory}/keystore.p12 --passout pass:${STORE_PASSWORD_ENV}"), + format!( + "expr $MYID_OFFSET + $(echo $POD_NAME | sed 's/.*-//') > {dir}/myid", + dir = STACKABLE_DATA_DIR + ), ] } diff --git a/rust/operator-binary/src/discovery.rs b/rust/operator-binary/src/discovery.rs index 55deff06..907a1e7a 100644 --- a/rust/operator-binary/src/discovery.rs +++ b/rust/operator-binary/src/discovery.rs @@ -7,6 +7,7 @@ use stackable_operator::{ k8s_openapi::api::core::v1::{ConfigMap, Endpoints, Service}, kube::{runtime::reflector::ObjectRef, Resource}, }; +use stackable_zookeeper_crd::security::ZookeeperSecurity; use stackable_zookeeper_crd::{ZookeeperCluster, ZookeeperRole}; use std::{collections::BTreeSet, num::TryFromIntError}; @@ -45,6 +46,7 @@ pub enum Error { } /// Builds discovery [`ConfigMap`]s for connecting to a [`ZookeeperCluster`] for all expected scenarios +#[allow(clippy::too_many_arguments)] pub async fn build_discovery_configmaps( zk: &ZookeeperCluster, owner: &impl Resource, @@ -53,6 +55,7 @@ pub async fn build_discovery_configmaps( svc: &Service, chroot: Option<&str>, resolved_product_image: &ResolvedProductImage, + zookeeper_security: &ZookeeperSecurity, ) -> Result, Error> { let name = owner.meta().name.as_deref().context(NoNameSnafu)?; Ok(vec![ @@ -62,7 +65,7 @@ pub async fn build_discovery_configmaps( name, controller_name, chroot, - pod_hosts(zk)?, + pod_hosts(zk, zookeeper_security)?, resolved_product_image, )?, build_discovery_configmap( @@ -131,12 +134,15 @@ fn build_discovery_configmap( } /// Lists all Pods FQDNs expected to host the [`ZookeeperCluster`] -fn pod_hosts(zk: &ZookeeperCluster) -> Result + '_, Error> { +fn pod_hosts<'a>( + zk: &'a ZookeeperCluster, + zookeeper_security: &'a ZookeeperSecurity, +) -> Result + 'a, Error> { Ok(zk .pods() .context(ExpectedPodsSnafu)? .into_iter() - .map(|pod_ref| (pod_ref.fqdn(), zk.client_port()))) + .map(|pod_ref| (pod_ref.fqdn(), zookeeper_security.client_port()))) } /// Lists all nodes currently hosting Pods participating in the [`Service`] diff --git a/rust/operator-binary/src/product_logging.rs b/rust/operator-binary/src/product_logging.rs index 6855a312..195416a6 100644 --- a/rust/operator-binary/src/product_logging.rs +++ b/rust/operator-binary/src/product_logging.rs @@ -48,8 +48,12 @@ pub async fn resolve_vector_aggregator_address( zk: &ZookeeperCluster, client: &Client, ) -> Result> { - let vector_aggregator_address = if let Some(vector_aggregator_config_map_name) = - &zk.spec.vector_aggregator_config_map_name + let vector_aggregator_address = if let Some(vector_aggregator_config_map_name) = &zk + .spec + .cluster_config + .logging + .as_ref() + .and_then(|logging| logging.vector_aggregator_config_map_name.as_ref()) { let vector_aggregator_address = client .get::( diff --git a/rust/operator-binary/src/zk_controller.rs b/rust/operator-binary/src/zk_controller.rs index 0912daed..53ed663b 100644 --- a/rust/operator-binary/src/zk_controller.rs +++ b/rust/operator-binary/src/zk_controller.rs @@ -10,16 +10,9 @@ use crate::{ use fnv::FnvHasher; use snafu::{OptionExt, ResultExt, Snafu}; use stackable_operator::{ - builder::{ - ConfigMapBuilder, ContainerBuilder, ObjectMetaBuilder, PodBuilder, - SecretOperatorVolumeSourceBuilder, VolumeBuilder, - }, + builder::{ConfigMapBuilder, ContainerBuilder, ObjectMetaBuilder, PodBuilder}, cluster_resources::ClusterResources, - commons::{ - authentication::{AuthenticationClass, AuthenticationClassProvider}, - product_image_selection::ResolvedProductImage, - tls::TlsAuthenticationProvider, - }, + commons::product_image_selection::ResolvedProductImage, k8s_openapi::{ api::{ apps::v1::{StatefulSet, StatefulSetSpec}, @@ -49,11 +42,10 @@ use stackable_operator::{ role_utils::RoleGroupRef, }; use stackable_zookeeper_crd::{ - Container, ZookeeperCluster, ZookeeperClusterStatus, ZookeeperConfig, ZookeeperRole, - CLIENT_TLS_DIR, CLIENT_TLS_MOUNT_DIR, DOCKER_IMAGE_BASE_NAME, LOG_VOLUME_SIZE_IN_MIB, - QUORUM_TLS_DIR, QUORUM_TLS_MOUNT_DIR, STACKABLE_CONFIG_DIR, STACKABLE_DATA_DIR, - STACKABLE_LOG_CONFIG_DIR, STACKABLE_LOG_DIR, STACKABLE_RW_CONFIG_DIR, - ZOOKEEPER_PROPERTIES_FILE, + security::ZookeeperSecurity, Container, ZookeeperCluster, ZookeeperClusterStatus, + ZookeeperConfig, ZookeeperRole, DOCKER_IMAGE_BASE_NAME, LOG_VOLUME_SIZE_IN_MIB, + STACKABLE_CONFIG_DIR, STACKABLE_DATA_DIR, STACKABLE_LOG_CONFIG_DIR, STACKABLE_LOG_DIR, + STACKABLE_RW_CONFIG_DIR, ZOOKEEPER_PROPERTIES_FILE, }; use std::{ borrow::Cow, @@ -145,21 +137,6 @@ pub enum Error { ApplyStatus { source: stackable_operator::error::Error, }, - #[snafu(display("failed to retrieve {}", authentication_class))] - AuthenticationClassRetrieval { - source: stackable_operator::error::Error, - authentication_class: ObjectRef, - }, - #[snafu(display( - "failed to use authentication mechanism {} - supported methods: {:?}", - method, - supported - ))] - AuthenticationMethodNotSupported { - authentication_class: ObjectRef, - supported: Vec, - method: String, - }, #[snafu(display("invalid java heap config"))] InvalidJavaHeapConfig { source: stackable_operator::error::Error, @@ -185,6 +162,10 @@ pub enum Error { source: crate::product_logging::Error, cm_name: String, }, + #[snafu(display("failed to initialize security context"))] + FailedToInitializeSecurityContext { + source: stackable_zookeeper_crd::security::Error, + }, } type Result = std::result::Result; @@ -211,20 +192,13 @@ impl ReconcilerError for Error { Error::BuildDiscoveryConfig { .. } => None, Error::ApplyDiscoveryConfig { .. } => None, Error::ApplyStatus { .. } => None, - Error::AuthenticationClassRetrieval { - authentication_class, - .. - } => Some(authentication_class.clone().erase()), - Error::AuthenticationMethodNotSupported { - authentication_class, - .. - } => Some(authentication_class.clone().erase()), Error::InvalidJavaHeapConfig { .. } => None, Error::ApplyServiceAccount { .. } => None, Error::ApplyRoleBinding { .. } => None, Error::DeleteOrphans { .. } => None, Error::ResolveVectorAggregatorAddress { .. } => None, Error::InvalidLoggingConfig { .. } => None, + Error::FailedToInitializeSecurityContext { .. } => None, } } } @@ -275,18 +249,9 @@ pub async fn reconcile_zk(zk: Arc, ctx: Arc) -> Result::new(auth_class), - })?, - ) - } else { - None - }; + let zookeeper_security = ZookeeperSecurity::new_from_zookeeper_cluster(client, &zk) + .await + .context(FailedToInitializeSecurityContextSnafu)?; let (rbac_sa, rbac_rolebinding) = build_zk_rbac_resources(&zk, &resolved_product_image)?; cluster_resources @@ -301,7 +266,7 @@ pub async fn reconcile_zk(zk: Arc, ctx: Arc) -> Result, ctx: Arc) -> Result, ctx: Arc) -> Result Result { let role_name = ZookeeperRole::Server.to_string(); let role_svc_name = zk @@ -462,7 +435,7 @@ pub fn build_server_role_service( spec: Some(ServiceSpec { ports: Some(vec![ServicePort { name: Some("zk".to_string()), - port: zk.client_port().into(), + port: zookeeper_security.client_port().into(), protocol: Some("TCP".to_string()), ..ServicePort::default() }]), @@ -481,6 +454,7 @@ fn build_server_rolegroup_config_map( server_config: &HashMap>, resolved_product_image: &ResolvedProductImage, vector_aggregator_address: Option<&str>, + zookeeper_security: &ZookeeperSecurity, ) -> Result { let mut zoo_cfg = server_config .get(&PropertyNameKind::File( @@ -491,21 +465,26 @@ fn build_server_rolegroup_config_map( zoo_cfg.extend(zk.pods().into_iter().flatten().map(|pod| { ( format!("server.{}", pod.zookeeper_myid), - format!("{}:2888:3888;{}", pod.fqdn(), zk.client_port()), + format!( + "{}:2888:3888;{}", + pod.fqdn(), + zookeeper_security.client_port() + ), ) })); + zoo_cfg.extend(zookeeper_security.config_settings()); + let role = ZookeeperRole::from_str(&rolegroup.role).with_context(|_| RoleParseFailureSnafu { role: rolegroup.role.to_string(), })?; - let zoo_cfg = zoo_cfg - .into_iter() - .map(|(k, v)| (k, Some(v))) - .collect::>(); let mut cm_builder = ConfigMapBuilder::new(); + let zk_data: BTreeMap> = + zoo_cfg.into_iter().map(|(k, v)| (k, Some(v))).collect(); + cm_builder .metadata( ObjectMetaBuilder::new() @@ -524,10 +503,8 @@ fn build_server_rolegroup_config_map( ) .add_data( ZOOKEEPER_PROPERTIES_FILE, - to_java_properties_string(zoo_cfg.iter().map(|(k, v)| (k, v))).with_context(|_| { - SerializeZooCfgSnafu { - rolegroup: rolegroup.clone(), - } + to_java_properties_string(zk_data.iter()).with_context(|_| SerializeZooCfgSnafu { + rolegroup: rolegroup.clone(), })?, ); @@ -556,6 +533,7 @@ fn build_server_rolegroup_service( zk: &ZookeeperCluster, rolegroup: &RoleGroupRef, resolved_product_image: &ResolvedProductImage, + zookeeper_security: &ZookeeperSecurity, ) -> Result { Ok(Service { metadata: ObjectMetaBuilder::new() @@ -577,7 +555,7 @@ fn build_server_rolegroup_service( ports: Some(vec![ ServicePort { name: Some("zk".to_string()), - port: zk.client_port().into(), + port: zookeeper_security.client_port().into(), protocol: Some("TCP".to_string()), ..ServicePort::default() }, @@ -608,7 +586,7 @@ fn build_server_rolegroup_statefulset( zk: &ZookeeperCluster, rolegroup_ref: &RoleGroupRef, server_config: &HashMap>, - client_authentication_class: Option<&AuthenticationClass>, + zookeeper_security: &ZookeeperSecurity, resolved_product_image: &ResolvedProductImage, ) -> Result { let zk_role = @@ -660,13 +638,7 @@ fn build_server_rolegroup_statefulset( let mut pod_builder = PodBuilder::new(); // add volumes and mounts depending on tls / auth settings - tls_volume_mounts( - zk, - &mut pod_builder, - &mut cb_prepare, - &mut cb_zookeeper, - client_authentication_class, - )?; + zookeeper_security.add_volume_mounts(&mut pod_builder, &mut cb_prepare, &mut cb_zookeeper); let mut args = Vec::new(); @@ -680,7 +652,8 @@ fn build_server_rolegroup_statefulset( log_config, )); } - args.push(create_init_container_command_args(zk)); + args.extend(create_init_container_command_args()); + args.extend(zookeeper_security.commands()); let container_prepare = cb_prepare .image_from_product_image(resolved_product_image) @@ -723,14 +696,14 @@ fn build_server_rolegroup_statefulset( // we can use Bash's virtual /dev/tcp filesystem to accomplish the same thing format!( "exec 3<>/dev/tcp/127.0.0.1/{} && echo srvr >&3 && grep '^Mode: ' <&3", - zk.client_port() + zookeeper_security.client_port() ), ]), }), period_seconds: Some(1), ..Probe::default() }) - .add_container_port("zk", zk.client_port().into()) + .add_container_port("zk", zookeeper_security.client_port().into()) .add_container_port("zk-leader", 2888) .add_container_port("zk-election", 3888) .add_container_port("metrics", 9505) @@ -867,77 +840,6 @@ fn build_server_rolegroup_statefulset( }) } -fn tls_volume_mounts( - zk: &ZookeeperCluster, - pod_builder: &mut PodBuilder, - cb_prepare: &mut ContainerBuilder, - cb_zookeeper: &mut ContainerBuilder, - client_authentication_class: Option<&AuthenticationClass>, -) -> Result<()> { - let tls_secret_class = if let Some(auth_class) = client_authentication_class { - match &auth_class.spec.provider { - AuthenticationClassProvider::Tls(TlsAuthenticationProvider { - client_cert_secret_class: Some(secret_class), - }) => Some(secret_class), - _ => { - return Err(Error::AuthenticationMethodNotSupported { - authentication_class: ObjectRef::from_obj(auth_class), - supported: vec!["tls".to_string()], - method: auth_class.spec.provider.to_string(), - }) - } - } - } else { - zk.client_tls_secret_class() - .map(|client_tls| &client_tls.secret_class) - }; - - if let Some(secret_class) = tls_secret_class { - // mounts for secret volume - cb_prepare.add_volume_mount("client-tls-mount", CLIENT_TLS_MOUNT_DIR); - cb_zookeeper.add_volume_mount("client-tls-mount", CLIENT_TLS_MOUNT_DIR); - pod_builder.add_volume(create_tls_volume("client-tls-mount", secret_class)); - // empty mount for trust and keystore - cb_prepare.add_volume_mount("client-tls", CLIENT_TLS_DIR); - cb_zookeeper.add_volume_mount("client-tls", CLIENT_TLS_DIR); - pod_builder.add_volume( - VolumeBuilder::new("client-tls") - .with_empty_dir(Some(""), None) - .build(), - ); - } - - // quorum - // mounts for secret volume - cb_prepare.add_volume_mount("quorum-tls-mount", QUORUM_TLS_MOUNT_DIR); - cb_zookeeper.add_volume_mount("quorum-tls-mount", QUORUM_TLS_MOUNT_DIR); - pod_builder.add_volume(create_tls_volume( - "quorum-tls-mount", - zk.quorum_tls_secret_class(), - )); - // empty mount for trust and keystore - cb_prepare.add_volume_mount("quorum-tls", QUORUM_TLS_DIR); - cb_zookeeper.add_volume_mount("quorum-tls", QUORUM_TLS_DIR); - pod_builder.add_volume( - VolumeBuilder::new("quorum-tls") - .with_empty_dir(Some(""), None) - .build(), - ); - - Ok(()) -} - -fn create_tls_volume(volume_name: &str, secret_class_name: &str) -> Volume { - VolumeBuilder::new(volume_name) - .ephemeral( - SecretOperatorVolumeSourceBuilder::new(secret_class_name) - .with_pod_scope() - .with_node_scope() - .build(), - ) - .build() -} - pub fn error_policy( _obj: Arc, _error: &Error, diff --git a/rust/operator-binary/src/znode_controller.rs b/rust/operator-binary/src/znode_controller.rs index 45a24955..fa9a6c63 100644 --- a/rust/operator-binary/src/znode_controller.rs +++ b/rust/operator-binary/src/znode_controller.rs @@ -22,6 +22,7 @@ use stackable_operator::{ }, logging::controller::ReconcilerError, }; +use stackable_zookeeper_crd::security::ZookeeperSecurity; use stackable_zookeeper_crd::{ZookeeperCluster, ZookeeperZnode, DOCKER_IMAGE_BASE_NAME}; use strum::{EnumDiscriminants, IntoStaticStr}; @@ -92,6 +93,10 @@ pub enum Error { }, #[snafu(display("object has no namespace"))] ObjectHasNoNamespace, + #[snafu(display("failed to initialize security context"))] + FailedToInitializeSecurityContext { + source: stackable_zookeeper_crd::security::Error, + }, } type Result = std::result::Result; @@ -135,6 +140,7 @@ impl ReconcilerError for Error { Error::ObjectMissingMetadataForOwnerRef { source: _ } => None, Error::DeleteOrphans { source: _ } => None, Error::ObjectHasNoNamespace => None, + Error::FailedToInitializeSecurityContext { source: _ } => None, } } } @@ -173,7 +179,9 @@ pub async fn reconcile_znode( reconcile_apply(client, &znode, Ok(zk), &znode_path, &resolved_product_image) .await } - finalizer::Event::Cleanup(_znode) => reconcile_cleanup(zk, &znode_path).await, + finalizer::Event::Cleanup(_znode) => { + reconcile_cleanup(client, zk, &znode_path).await + } } }, ) @@ -189,6 +197,11 @@ async fn reconcile_apply( resolved_product_image: &ResolvedProductImage, ) -> Result { let zk = zk?; + + let zookeeper_security = ZookeeperSecurity::new_from_zookeeper_cluster(client, &zk) + .await + .context(FailedToInitializeSecurityContextSnafu)?; + let mut cluster_resources = ClusterResources::new( APP_NAME, OPERATOR_NAME, @@ -197,7 +210,7 @@ async fn reconcile_apply( ) .unwrap(); - znode_mgmt::ensure_znode_exists(&zk_mgmt_addr(&zk)?, znode_path) + znode_mgmt::ensure_znode_exists(&zk_mgmt_addr(&zk, &zookeeper_security)?, znode_path) .await .with_context(|_| EnsureZnodeSnafu { zk: ObjectRef::from_obj(&zk), @@ -227,6 +240,7 @@ async fn reconcile_apply( &server_role_service, Some(znode_path), resolved_product_image, + &zookeeper_security, ) .await .context(BuildDiscoveryConfigMapSnafu)? @@ -247,6 +261,7 @@ async fn reconcile_apply( } async fn reconcile_cleanup( + client: &stackable_operator::client::Client, zk: Result, znode_path: &str, ) -> Result { @@ -257,8 +272,13 @@ async fn reconcile_cleanup( } res => res?, }; + + let zookeeper_security = ZookeeperSecurity::new_from_zookeeper_cluster(client, &zk) + .await + .context(FailedToInitializeSecurityContextSnafu)?; + // Clean up znode from the ZooKeeper cluster before letting Kubernetes delete the object - znode_mgmt::ensure_znode_missing(&zk_mgmt_addr(&zk)?, znode_path) + znode_mgmt::ensure_znode_missing(&zk_mgmt_addr(&zk, &zookeeper_security)?, znode_path) .await .with_context(|_| EnsureZnodeMissingSnafu { zk: ObjectRef::from_obj(&zk), @@ -268,7 +288,7 @@ async fn reconcile_cleanup( Ok(controller::Action::await_change()) } -fn zk_mgmt_addr(zk: &ZookeeperCluster) -> Result { +fn zk_mgmt_addr(zk: &ZookeeperCluster, zookeeper_security: &ZookeeperSecurity) -> Result { // Rust ZooKeeper client does not support client-side load-balancing, so use // (load-balanced) global service instead. Ok(format!( @@ -277,7 +297,7 @@ fn zk_mgmt_addr(zk: &ZookeeperCluster) -> Result { .with_context(|| NoZkFqdnSnafu { zk: ObjectRef::from_obj(zk), })?, - zk.client_port(), + zookeeper_security.client_port(), )) } diff --git a/tests/templates/kuttl/delete-rolegroup/00-install-zookeeper.yaml.j2 b/tests/templates/kuttl/delete-rolegroup/00-install-zookeeper.yaml.j2 index fb1f80ea..3475e1dd 100644 --- a/tests/templates/kuttl/delete-rolegroup/00-install-zookeeper.yaml.j2 +++ b/tests/templates/kuttl/delete-rolegroup/00-install-zookeeper.yaml.j2 @@ -16,12 +16,13 @@ spec: image: productVersion: "{{ test_scenario['values']['zookeeper'].split('-stackable')[0] }}" stackableVersion: "{{ test_scenario['values']['zookeeper'].split('-stackable')[1] }}" + clusterConfig: + tls: + serverSecretClass: null {% if lookup('env', 'VECTOR_AGGREGATOR') %} - vectorAggregatorConfigMapName: vector-aggregator-discovery + logging: + vectorAggregatorConfigMapName: vector-aggregator-discovery {% endif %} - config: - tls: null - quorumTlsSecretClass: tls servers: config: logging: diff --git a/tests/templates/kuttl/delete-rolegroup/01-remove-secondary.yaml.j2 b/tests/templates/kuttl/delete-rolegroup/01-remove-secondary.yaml.j2 index a239f38b..ce133ad6 100644 --- a/tests/templates/kuttl/delete-rolegroup/01-remove-secondary.yaml.j2 +++ b/tests/templates/kuttl/delete-rolegroup/01-remove-secondary.yaml.j2 @@ -7,9 +7,9 @@ spec: image: productVersion: "{{ test_scenario['values']['zookeeper'].split('-stackable')[0] }}" stackableVersion: "{{ test_scenario['values']['zookeeper'].split('-stackable')[1] }}" - config: - tls: null - quorumTlsSecretClass: tls + clusterConfig: + tls: + serverSecretClass: null servers: config: resources: diff --git a/tests/templates/kuttl/logging/01-install-zookeeper.yaml.j2 b/tests/templates/kuttl/logging/01-install-zookeeper.yaml.j2 index 87fc974b..57f685c3 100644 --- a/tests/templates/kuttl/logging/01-install-zookeeper.yaml.j2 +++ b/tests/templates/kuttl/logging/01-install-zookeeper.yaml.j2 @@ -30,7 +30,9 @@ spec: image: productVersion: "{{ test_scenario['values']['zookeeper'].split('-stackable')[0] }}" stackableVersion: "{{ test_scenario['values']['zookeeper'].split('-stackable')[1] }}" - vectorAggregatorConfigMapName: vector-aggregator-discovery + clusterConfig: + logging: + vectorAggregatorConfigMapName: vector-aggregator-discovery servers: roleGroups: automatic-log-config: diff --git a/tests/templates/kuttl/smoke/00-install-zookeeper.yaml.j2 b/tests/templates/kuttl/smoke/00-install-zookeeper.yaml.j2 index e0c6ca7c..cebedee3 100644 --- a/tests/templates/kuttl/smoke/00-install-zookeeper.yaml.j2 +++ b/tests/templates/kuttl/smoke/00-install-zookeeper.yaml.j2 @@ -12,26 +12,26 @@ apiVersion: zookeeper.stackable.tech/v1alpha1 kind: ZookeeperCluster metadata: name: test-zk -spec: spec: image: productVersion: "{{ test_scenario['values']['zookeeper'].split('-stackable')[0] }}" stackableVersion: "{{ test_scenario['values']['zookeeper'].split('-stackable')[1] }}" -{% if lookup('env', 'VECTOR_AGGREGATOR') %} - vectorAggregatorConfigMapName: vector-aggregator-discovery -{% endif %} - config: -{% if test_scenario['values']['use-client-tls'] == 'true' %} + clusterConfig: +{% if test_scenario['values']['use-server-tls'] == 'true' %} tls: - secretClass: zk-client-secret + serverSecretClass: zk-client-secret {% else %} - tls: null + tls: + serverSecretClass: null {% endif %} {% if test_scenario['values']['use-client-auth-tls'] == 'true' %} - clientAuthentication: - authenticationClass: zk-client-auth-tls + authentication: + - authenticationClass: zk-client-auth-tls +{% endif %} +{% if lookup('env', 'VECTOR_AGGREGATOR') %} + logging: + vectorAggregatorConfigMapName: vector-aggregator-discovery {% endif %} - quorumTlsSecretClass: tls servers: config: logging: @@ -82,7 +82,7 @@ spec: namespace: default autoGenerate: true {% endif %} -{% if test_scenario['values']['use-client-tls'] == 'true' %} +{% if test_scenario['values']['use-server-tls'] == 'true' %} --- apiVersion: secrets.stackable.tech/v1alpha1 kind: SecretClass diff --git a/tests/templates/kuttl/smoke/02-assert.yaml.j2 b/tests/templates/kuttl/smoke/02-assert.yaml.j2 index ee6ab5e5..5e2d0108 100644 --- a/tests/templates/kuttl/smoke/02-assert.yaml.j2 +++ b/tests/templates/kuttl/smoke/02-assert.yaml.j2 @@ -6,6 +6,6 @@ metadata: commands: - script: kubectl exec -n $NAMESPACE zk-test-helper-0 -- python /tmp/test_zookeeper.py -n $NAMESPACE - script: kubectl exec -n $NAMESPACE test-zk-server-primary-0 --container='zookeeper' -- /tmp/test_heap.sh -{% if test_scenario['values']['use-client-auth-tls'] == 'true' or test_scenario['values']['use-client-tls'] == 'true' %} +{% if test_scenario['values']['use-client-auth-tls'] == 'true' or test_scenario['values']['use-server-tls'] == 'true' %} - script: kubectl exec -n $NAMESPACE test-zk-server-primary-0 --container='zookeeper' -- /tmp/test_tls.sh $NAMESPACE {% endif %} diff --git a/tests/templates/kuttl/smoke/02-prepare-test-zookeeper.yaml.j2 b/tests/templates/kuttl/smoke/02-prepare-test-zookeeper.yaml.j2 index 618d3d11..967d259e 100644 --- a/tests/templates/kuttl/smoke/02-prepare-test-zookeeper.yaml.j2 +++ b/tests/templates/kuttl/smoke/02-prepare-test-zookeeper.yaml.j2 @@ -4,6 +4,6 @@ kind: TestStep commands: - script: kubectl cp -n $NAMESPACE ./test_zookeeper.py zk-test-helper-0:/tmp - script: kubectl cp -n $NAMESPACE ./test_heap.sh test-zk-server-primary-0:/tmp --container='zookeeper' -{% if test_scenario['values']['use-client-auth-tls'] == 'true' or test_scenario['values']['use-client-tls'] == 'true' %} +{% if test_scenario['values']['use-client-auth-tls'] == 'true' or test_scenario['values']['use-server-tls'] == 'true' %} - script: kubectl cp -n $NAMESPACE ./test_tls.sh test-zk-server-primary-0:/tmp --container='zookeeper' {% endif %} diff --git a/tests/templates/kuttl/smoke/test_tls.sh.j2 b/tests/templates/kuttl/smoke/test_tls.sh.j2 index 7be8e86a..2d87370e 100755 --- a/tests/templates/kuttl/smoke/test_tls.sh.j2 +++ b/tests/templates/kuttl/smoke/test_tls.sh.j2 @@ -3,7 +3,7 @@ NAMESPACE=$1 -{% if test_scenario['values']['use-client-auth-tls'] == 'true' or test_scenario['values']['use-client-tls'] == 'true' %} +{% if test_scenario['values']['use-client-auth-tls'] == 'true' or test_scenario['values']['use-server-tls'] == 'true' %} SERVER="test-zk-server-primary-1.test-zk-server-primary.${NAMESPACE}.svc.cluster.local:2282" {% else %} SERVER="test-zk-server-primary-1.test-zk-server-primary.${NAMESPACE}.svc.cluster.local:2181" @@ -34,9 +34,9 @@ export CLIENT_JVMFLAGS=" -Dzookeeper.authProvider.x509=org.apache.zookeeper.server.auth.X509AuthenticationProvider -Dzookeeper.clientCnxnSocket=org.apache.zookeeper.ClientCnxnSocketNetty -Dzookeeper.client.secure=true --Dzookeeper.ssl.keyStore.location=/stackable/client_tls/keystore.p12 +-Dzookeeper.ssl.keyStore.location=/stackable/server_tls/keystore.p12 -Dzookeeper.ssl.keyStore.password=${CLIENT_STORE_SECRET} --Dzookeeper.ssl.trustStore.location=/stackable/client_tls/truststore.p12 +-Dzookeeper.ssl.trustStore.location=/stackable/server_tls/truststore.p12 -Dzookeeper.ssl.trustStore.password=${CLIENT_STORE_SECRET}" if ! /stackable/zookeeper/bin/zkCli.sh -server "${SERVER}" ls / &> /dev/null; diff --git a/tests/templates/kuttl/znode/00-install-zookeeper.yaml.j2 b/tests/templates/kuttl/znode/00-install-zookeeper.yaml.j2 index 46a72272..91519882 100644 --- a/tests/templates/kuttl/znode/00-install-zookeeper.yaml.j2 +++ b/tests/templates/kuttl/znode/00-install-zookeeper.yaml.j2 @@ -12,13 +12,14 @@ apiVersion: zookeeper.stackable.tech/v1alpha1 kind: ZookeeperCluster metadata: name: test-zk -spec: spec: image: productVersion: "{{ test_scenario['values']['zookeeper-latest'].split('-stackable')[0] }}" stackableVersion: "{{ test_scenario['values']['zookeeper-latest'].split('-stackable')[1] }}" {% if lookup('env', 'VECTOR_AGGREGATOR') %} - vectorAggregatorConfigMapName: vector-aggregator-discovery + clusterConfig: + logging: + vectorAggregatorConfigMapName: vector-aggregator-discovery {% endif %} servers: config: diff --git a/tests/test-definition.yaml b/tests/test-definition.yaml index 26bac5fa..c48f2134 100644 --- a/tests/test-definition.yaml +++ b/tests/test-definition.yaml @@ -9,7 +9,7 @@ dimensions: - name: zookeeper-latest values: - 3.8.0-stackable0.9.0 - - name: use-client-tls + - name: use-server-tls values: - "true" - "false" @@ -21,7 +21,7 @@ tests: - name: smoke dimensions: - zookeeper - - use-client-tls + - use-server-tls - use-client-auth-tls - name: delete-rolegroup dimensions: