From d4618fd367c7638f8390e6a15cb53710d8480a50 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Mon, 6 Jun 2022 16:19:50 +0200 Subject: [PATCH 1/9] adapted crds to crd review, fixed tests (added timeouts) --- CHANGELOG.md | 3 + deploy/crd/nificluster.crd.yaml | 94 +++++++++---------- deploy/helm/nifi-operator/crds/crds.yaml | 94 +++++++++---------- deploy/manifests/crds.yaml | 94 +++++++++---------- docs/modules/ROOT/pages/usage.adoc | 16 ++-- examples/simple-nifi-cluster.yaml | 18 ++-- rust/crd/src/authentication.rs | 31 ++---- rust/crd/src/lib.rs | 11 ++- rust/operator-binary/src/config.rs | 5 +- rust/operator-binary/src/controller.rs | 54 +++++++---- tests/templates/kuttl/smoke/00-assert.yaml | 6 ++ .../kuttl/smoke/01-install-nifi.yaml.j2 | 14 +-- tests/templates/kuttl/smoke/02-assert.yaml | 6 ++ .../kuttl/smoke/02-scale-up-nifi.yaml.j2 | 14 +-- tests/templates/kuttl/smoke/04-assert.yaml | 1 + tests/templates/kuttl/smoke/05-assert.yaml | 6 ++ .../kuttl/smoke/05-enable-anonymous.yaml.j2 | 16 ++-- .../kuttl/smoke/test_nifi_metrics.py | 4 +- 18 files changed, 257 insertions(+), 230 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b96724e7..edc3efec 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,8 @@ All notable changes to this project will be documented in this file. - `operator-rs` `0.10.0` -> `0.15.0` ([#218], [#223], [#230]). - [BREAKING] Specifying the product version has been changed to adhere to [ADR018](https://docs.stackable.tech/home/contributor/adr/ADR018-product_image_versioning.html) instead of just specifying the product version you will now have to add the Stackable image version as well, so `version: 3.5.8` becomes (for example) `version: 3.5.8-stackable0.1.0` ([#270]) +- [BREAKING] CRD overhaul: Moved `authenticationConfig` to top level `config.authentication`. `SingleUser` now proper camelCase `singleUser`. `adminCredentialsSecret` now takes a String instead of `SecretReference` ([#xxx]). +- [BREAKING] CRD overhaul: Moved `sensitivePropertiesConfig` to top level `config.sensitiveProperties` ([#xxx]). ### Removed @@ -25,6 +27,7 @@ All notable changes to this project will be documented in this file. [#223]: https://github.com/stackabletech/nifi-operator/pull/223 [#230]: https://github.com/stackabletech/nifi-operator/pull/230 [#270]: https://github.com/stackabletech/nifi-operator/pull/270 +[#xxx]: https://github.com/stackabletech/nifi-operator/pull/xxx ## [0.5.0] - 2022-02-14 diff --git a/deploy/crd/nificluster.crd.yaml b/deploy/crd/nificluster.crd.yaml index dd652d70..bd69848b 100644 --- a/deploy/crd/nificluster.crd.yaml +++ b/deploy/crd/nificluster.crd.yaml @@ -22,38 +22,60 @@ spec: properties: spec: properties: - authenticationConfig: - description: A reference to a Secret containing username/password for the initial admin user + config: + description: Global Nifi config for e.g. authentication or sensitive properties properties: - allowAnonymousAccess: - nullable: true - type: boolean - method: - oneOf: - - required: - - SingleUser + authentication: + description: A reference to a Secret containing username/password for the initial admin user properties: - SingleUser: + allowAnonymousAccess: + nullable: true + type: boolean + method: + oneOf: + - required: + - singleUser properties: - adminCredentialsSecret: - description: SecretReference represents a Secret Reference. It has enough information to retrieve secret in any namespace + singleUser: properties: - name: - description: Name is unique within a namespace to reference a secret resource. - type: string - namespace: - description: Namespace defines the space within which the secret name must be unique. + adminCredentialsSecret: type: string + autoGenerate: + default: false + type: boolean + required: + - adminCredentialsSecret type: object - autoGenerate: - default: false - type: boolean - required: - - adminCredentialsSecret type: object + required: + - method + type: object + sensitiveProperties: + description: Configuration options for how NiFi encrypts sensitive properties on disk + properties: + algorithm: + enum: + - nifiArgon2AesGcm128 + - nifiArgon2AesGcm256 + - nifiBcryptAesGcm128 + - nifiBcryptAesGcm256 + - nifiPbkdf2AesGcm128 + - nifiPbkdf2AesGcm256 + - nifiScryptAesGcm128 + - nifiScryptAesGcm256 + nullable: true + type: string + autoGenerate: + default: false + type: boolean + keySecret: + type: string + required: + - keySecret type: object required: - - method + - authentication + - sensitiveProperties type: object nodes: description: Available NiFi roles @@ -171,29 +193,6 @@ spec: required: - roleGroups type: object - sensitivePropertiesConfig: - description: Configuration options for how NiFi encrypts sensitive properties on disk - properties: - algorithm: - enum: - - nifiArgon2AesGcm128 - - nifiArgon2AesGcm256 - - nifiBcryptAesGcm128 - - nifiBcryptAesGcm256 - - nifiPbkdf2AesGcm128 - - nifiPbkdf2AesGcm256 - - nifiScryptAesGcm128 - - nifiScryptAesGcm256 - nullable: true - type: string - autoGenerate: - default: false - type: boolean - keySecret: - type: string - required: - - keySecret - type: object stopped: description: "Emergency stop button, if `true` then all pods are stopped without affecting configuration (as setting `replicas` to `0` would)" nullable: true @@ -206,8 +205,7 @@ spec: description: The reference to the ZooKeeper cluster type: string required: - - authenticationConfig - - sensitivePropertiesConfig + - config - zookeeperConfigMapName type: object status: diff --git a/deploy/helm/nifi-operator/crds/crds.yaml b/deploy/helm/nifi-operator/crds/crds.yaml index fd70348a..b477b361 100644 --- a/deploy/helm/nifi-operator/crds/crds.yaml +++ b/deploy/helm/nifi-operator/crds/crds.yaml @@ -24,38 +24,60 @@ spec: properties: spec: properties: - authenticationConfig: - description: A reference to a Secret containing username/password for the initial admin user + config: + description: Global Nifi config for e.g. authentication or sensitive properties properties: - allowAnonymousAccess: - nullable: true - type: boolean - method: - oneOf: - - required: - - SingleUser + authentication: + description: A reference to a Secret containing username/password for the initial admin user properties: - SingleUser: + allowAnonymousAccess: + nullable: true + type: boolean + method: + oneOf: + - required: + - singleUser properties: - adminCredentialsSecret: - description: SecretReference represents a Secret Reference. It has enough information to retrieve secret in any namespace + singleUser: properties: - name: - description: Name is unique within a namespace to reference a secret resource. - type: string - namespace: - description: Namespace defines the space within which the secret name must be unique. + adminCredentialsSecret: type: string + autoGenerate: + default: false + type: boolean + required: + - adminCredentialsSecret type: object - autoGenerate: - default: false - type: boolean - required: - - adminCredentialsSecret type: object + required: + - method + type: object + sensitiveProperties: + description: Configuration options for how NiFi encrypts sensitive properties on disk + properties: + algorithm: + enum: + - nifiArgon2AesGcm128 + - nifiArgon2AesGcm256 + - nifiBcryptAesGcm128 + - nifiBcryptAesGcm256 + - nifiPbkdf2AesGcm128 + - nifiPbkdf2AesGcm256 + - nifiScryptAesGcm128 + - nifiScryptAesGcm256 + nullable: true + type: string + autoGenerate: + default: false + type: boolean + keySecret: + type: string + required: + - keySecret type: object required: - - method + - authentication + - sensitiveProperties type: object nodes: description: Available NiFi roles @@ -173,29 +195,6 @@ spec: required: - roleGroups type: object - sensitivePropertiesConfig: - description: Configuration options for how NiFi encrypts sensitive properties on disk - properties: - algorithm: - enum: - - nifiArgon2AesGcm128 - - nifiArgon2AesGcm256 - - nifiBcryptAesGcm128 - - nifiBcryptAesGcm256 - - nifiPbkdf2AesGcm128 - - nifiPbkdf2AesGcm256 - - nifiScryptAesGcm128 - - nifiScryptAesGcm256 - nullable: true - type: string - autoGenerate: - default: false - type: boolean - keySecret: - type: string - required: - - keySecret - type: object stopped: description: "Emergency stop button, if `true` then all pods are stopped without affecting configuration (as setting `replicas` to `0` would)" nullable: true @@ -208,8 +207,7 @@ spec: description: The reference to the ZooKeeper cluster type: string required: - - authenticationConfig - - sensitivePropertiesConfig + - config - zookeeperConfigMapName type: object status: diff --git a/deploy/manifests/crds.yaml b/deploy/manifests/crds.yaml index 788d95e6..26e5b6f0 100644 --- a/deploy/manifests/crds.yaml +++ b/deploy/manifests/crds.yaml @@ -25,38 +25,60 @@ spec: properties: spec: properties: - authenticationConfig: - description: A reference to a Secret containing username/password for the initial admin user + config: + description: Global Nifi config for e.g. authentication or sensitive properties properties: - allowAnonymousAccess: - nullable: true - type: boolean - method: - oneOf: - - required: - - SingleUser + authentication: + description: A reference to a Secret containing username/password for the initial admin user properties: - SingleUser: + allowAnonymousAccess: + nullable: true + type: boolean + method: + oneOf: + - required: + - singleUser properties: - adminCredentialsSecret: - description: SecretReference represents a Secret Reference. It has enough information to retrieve secret in any namespace + singleUser: properties: - name: - description: Name is unique within a namespace to reference a secret resource. - type: string - namespace: - description: Namespace defines the space within which the secret name must be unique. + adminCredentialsSecret: type: string + autoGenerate: + default: false + type: boolean + required: + - adminCredentialsSecret type: object - autoGenerate: - default: false - type: boolean - required: - - adminCredentialsSecret type: object + required: + - method + type: object + sensitiveProperties: + description: Configuration options for how NiFi encrypts sensitive properties on disk + properties: + algorithm: + enum: + - nifiArgon2AesGcm128 + - nifiArgon2AesGcm256 + - nifiBcryptAesGcm128 + - nifiBcryptAesGcm256 + - nifiPbkdf2AesGcm128 + - nifiPbkdf2AesGcm256 + - nifiScryptAesGcm128 + - nifiScryptAesGcm256 + nullable: true + type: string + autoGenerate: + default: false + type: boolean + keySecret: + type: string + required: + - keySecret type: object required: - - method + - authentication + - sensitiveProperties type: object nodes: description: Available NiFi roles @@ -174,29 +196,6 @@ spec: required: - roleGroups type: object - sensitivePropertiesConfig: - description: Configuration options for how NiFi encrypts sensitive properties on disk - properties: - algorithm: - enum: - - nifiArgon2AesGcm128 - - nifiArgon2AesGcm256 - - nifiBcryptAesGcm128 - - nifiBcryptAesGcm256 - - nifiPbkdf2AesGcm128 - - nifiPbkdf2AesGcm256 - - nifiScryptAesGcm128 - - nifiScryptAesGcm256 - nullable: true - type: string - autoGenerate: - default: false - type: boolean - keySecret: - type: string - required: - - keySecret - type: object stopped: description: "Emergency stop button, if `true` then all pods are stopped without affecting configuration (as setting `replicas` to `0` would)" nullable: true @@ -209,8 +208,7 @@ spec: description: The reference to the ZooKeeper cluster type: string required: - - authenticationConfig - - sensitivePropertiesConfig + - config - zookeeperConfigMapName type: object status: diff --git a/docs/modules/ROOT/pages/usage.adoc b/docs/modules/ROOT/pages/usage.adoc index 0b44c162..5d8d7916 100644 --- a/docs/modules/ROOT/pages/usage.adoc +++ b/docs/modules/ROOT/pages/usage.adoc @@ -29,14 +29,13 @@ metadata: spec: version: "1.15.0-stackable0.4.0" zookeeperConfigMapName: simple-nifi-znode - authenticationConfig: - method: - SingleUser: - adminCredentialsSecret: - name: nifi-admin-credentials-simple - namespace: default - aut - allowAnonymousAccess: true + config: + authentication: + method: + singleUser: + adminCredentialsSecret: nifi-admin-credentials-simple + sensitiveProperties: + keySecret: nifi-sensitive-property-key nodes: roleGroups: default: @@ -44,7 +43,6 @@ spec: matchLabels: kubernetes.io/os: linux config: - sensitivePropertyKeySecret: nifi-sensitive-property-key log: rootLogLevel: INFO replicas: 3 diff --git a/examples/simple-nifi-cluster.yaml b/examples/simple-nifi-cluster.yaml index 24319e18..a4566d70 100644 --- a/examples/simple-nifi-cluster.yaml +++ b/examples/simple-nifi-cluster.yaml @@ -39,15 +39,15 @@ metadata: spec: version: "1.15.0-stackable0.4.0" zookeeperConfigMapName: simple-nifi-znode - authenticationConfig: - method: - SingleUser: - adminCredentialsSecret: - name: nifi-admin-credentials-simple - autoGenerate: true - sensitivePropertiesConfig: - keySecret: nifi-sensitive-property-key - autoGenerate: true + config: + authentication: + method: + singleUser: + adminCredentialsSecret: nifi-admin-credentials-simple + autoGenerate: true + sensitivePropertiesConfig: + keySecret: nifi-sensitive-property-key + autoGenerate: true nodes: roleGroups: default: diff --git a/rust/crd/src/authentication.rs b/rust/crd/src/authentication.rs index 1c5b5445..7b994458 100644 --- a/rust/crd/src/authentication.rs +++ b/rust/crd/src/authentication.rs @@ -1,12 +1,10 @@ use rand::distributions::Alphanumeric; use rand::Rng; use serde::{Deserialize, Serialize}; -use snafu::{OptionExt, ResultExt, Snafu}; +use snafu::{ResultExt, Snafu}; use stackable_operator::builder::ObjectMetaBuilder; use stackable_operator::client::Client; -use stackable_operator::k8s_openapi::api::core::v1::{ - Secret, SecretReference, SecretVolumeSource, Volume, -}; +use stackable_operator::k8s_openapi::api::core::v1::{Secret, SecretVolumeSource, Volume}; use stackable_operator::kube::runtime::reflector::ObjectRef; use stackable_operator::schemars::{self, JsonSchema}; use std::collections::BTreeMap; @@ -59,11 +57,11 @@ pub struct NifiAuthenticationConfig { } #[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize, strum::Display)] -#[strum(serialize_all = "camelCase")] +#[serde(rename_all = "camelCase")] pub enum NifiAuthenticationMethod { #[serde(rename_all = "camelCase")] SingleUser { - admin_credentials_secret: SecretReference, + admin_credentials_secret: String, #[serde(default)] auto_generate: bool, }, @@ -85,23 +83,12 @@ pub async fn get_login_identity_provider_xml( admin_credentials_secret, auto_generate, } => { - let secret_name = admin_credentials_secret.name.clone().with_context(|| { - MissingSecretReferenceSnafu { - secret: "admin_credentials_secret".to_string(), - } - })?; - // If no namespace was specified the namespace of the NifiCluster object is assumed - let secret_namespace = admin_credentials_secret - .namespace - .clone() - .unwrap_or_else(|| current_namespace.to_string()); - // Check if the referenced secret exists and contains all necessary keys, otherwise // generate random password and default user check_or_generate_admin_credentials( client, - &secret_name, - &secret_namespace, + admin_credentials_secret, + current_namespace, auto_generate, ) .await?; @@ -126,11 +113,7 @@ pub fn get_auth_volumes( let admin_volume = Volume { name: AUTH_VOLUME_NAME.to_string(), secret: Some(SecretVolumeSource { - secret_name: Some(admin_credentials_secret.name.clone().with_context( - || MissingRequiredValueSnafu { - value: "name".to_string(), - }, - )?), + secret_name: Some(admin_credentials_secret.to_string()), ..SecretVolumeSource::default() }), ..Volume::default() diff --git a/rust/crd/src/lib.rs b/rust/crd/src/lib.rs index 47c8395e..8097495a 100644 --- a/rust/crd/src/lib.rs +++ b/rust/crd/src/lib.rs @@ -51,10 +51,17 @@ pub struct NifiSpec { pub nodes: Option>, /// The reference to the ZooKeeper cluster pub zookeeper_config_map_name: String, + /// Global Nifi config for e.g. authentication or sensitive properties + pub config: NifiGlobalConfig, +} + +#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct NifiGlobalConfig { /// A reference to a Secret containing username/password for the initial admin user - pub authentication_config: NifiAuthenticationConfig, + pub authentication: NifiAuthenticationConfig, /// Configuration options for how NiFi encrypts sensitive properties on disk - pub sensitive_properties_config: NifiSensitivePropertiesConfig, + pub sensitive_properties: NifiSensitivePropertiesConfig, } #[derive(Clone, Debug, Default, Deserialize, JsonSchema, PartialEq, Serialize)] diff --git a/rust/operator-binary/src/config.rs b/rust/operator-binary/src/config.rs index 18d25289..093e0ff8 100644 --- a/rust/operator-binary/src/config.rs +++ b/rust/operator-binary/src/config.rs @@ -441,7 +441,8 @@ pub fn build_nifi_properties( ); let algorithm = &spec - .sensitive_properties_config + .config + .sensitive_properties .algorithm .clone() .unwrap_or_default(); @@ -487,7 +488,7 @@ pub fn build_nifi_properties( ); properties.insert( "nifi.security.allow.anonymous.authentication".to_string(), - spec.authentication_config.allow_anonymous().to_string(), + spec.config.authentication.allow_anonymous().to_string(), ); properties.insert( "nifi.cluster.protocol.is.secure".to_string(), diff --git a/rust/operator-binary/src/controller.rs b/rust/operator-binary/src/controller.rs index a95fcde3..ee9a7c55 100644 --- a/rust/operator-binary/src/controller.rs +++ b/rust/operator-binary/src/controller.rs @@ -151,9 +151,10 @@ pub enum Error { }, #[snafu(display("Failed to find an external port to use for proxy hosts"))] ExternalPort, - #[snafu(display("Could not build role service fqdn"))] NoRoleServiceFqdn, + #[snafu(display("Could not extract NiFi product version (x.x.x) from image: [{version}]. Expected format e.g. x.x.x-stackable0.1.0"))] + FailedNifiProductVersionRetrieval { version: String }, } type Result = std::result::Result; @@ -167,7 +168,7 @@ impl ReconcilerError for Error { pub async fn reconcile_nifi(nifi: Arc, ctx: Context) -> Result { tracing::info!("Starting reconcile"); let client = &ctx.get_ref().client; - let nifi_version = nifi_version(&nifi)?; + let nifi_product_version = nifi_product_version(&nifi)?; let namespace = &nifi .metadata .namespace @@ -179,7 +180,7 @@ pub async fn reconcile_nifi(nifi: Arc, ctx: Context) -> Result let validated_config = validated_product_config( &nifi, - nifi_version, + nifi_product_version, nifi.spec.nodes.as_ref().context(NoNodeRoleSnafu)?, &ctx.get_ref().product_config, ) @@ -283,7 +284,13 @@ pub fn build_node_role_service(nifi: &NifiCluster) -> Result { .name(&role_svc_name) .ownerreference_from_resource(nifi, None, Some(true)) .context(ObjectMissingMetadataForOwnerRefSnafu)? - .with_recommended_labels(nifi, APP_NAME, nifi_version(nifi)?, &role_name, "global") + .with_recommended_labels( + nifi, + APP_NAME, + nifi_image_version(nifi)?, + &role_name, + "global", + ) .build(), spec: Some(ServiceSpec { ports: Some(vec![ServicePort { @@ -328,7 +335,7 @@ fn build_node_rolegroup_log_config_map( .with_recommended_labels( nifi, APP_NAME, - nifi_version(nifi)?, + nifi_image_version(nifi)?, &rolegroup.role, &rolegroup.role_group, ) @@ -368,7 +375,7 @@ async fn build_node_rolegroup_config_map( .with_recommended_labels( nifi, APP_NAME, - nifi_version(nifi)?, + nifi_image_version(nifi)?, &rolegroup.role, &rolegroup.role_group, ) @@ -403,7 +410,7 @@ async fn build_node_rolegroup_config_map( "login-identity-providers.xml", stackable_nifi_crd::authentication::get_login_identity_provider_xml( client, - &nifi.spec.authentication_config, + &nifi.spec.config.authentication, namespace, ) .await @@ -432,7 +439,7 @@ fn build_node_rolegroup_service( .with_recommended_labels( nifi, APP_NAME, - nifi_version(nifi)?, + nifi_image_version(nifi)?, &rolegroup.role, &rolegroup.role_group, ) @@ -526,7 +533,7 @@ fn build_node_rolegroup_statefulset( .role_groups .get(&rolegroup_ref.role_group); - let nifi_version = nifi_version(nifi)?; + let nifi_version = nifi_image_version(nifi)?; let image = format!("docker.stackable.tech/stackable/nifi:{}", nifi_version); let node_address = format!( @@ -540,9 +547,9 @@ fn build_node_rolegroup_statefulset( .with_context(|| ObjectHasNoNamespaceSnafu {})? ); - let sensitive_key_secret = &nifi.spec.sensitive_properties_config.key_secret; + let sensitive_key_secret = &nifi.spec.config.sensitive_properties.key_secret; - let auth_volumes = get_auth_volumes(&nifi.spec.authentication_config.method) + let auth_volumes = get_auth_volumes(&nifi.spec.config.authentication.method) .context(MaterializeAuthConfigSnafu)?; let mut container_prepare = ContainerBuilder::new("prepare") @@ -864,6 +871,7 @@ fn build_reporting_task_job( ) -> Result { let rolegroup_obj_name = rolegroup_ref.object_name(); let namespace: &str = &nifi.namespace().context(ObjectHasNoNamespaceSnafu)?; + let product_version = nifi_product_version(nifi)?; let nifi_connect_url = format!( "https://{rolegroup}-0.{rolegroup}.{namespace}.svc.cluster.local:{port}/nifi-api", rolegroup = rolegroup_obj_name, @@ -879,7 +887,7 @@ fn build_reporting_task_job( &format!("-n {}", nifi_connect_url), &format!("-u $(cat {}/username)", AUTH_VOLUME_MOUNT_PATH), &format!("-p $(cat {}/password)", AUTH_VOLUME_MOUNT_PATH), - &format!("-v {}", nifi_version(nifi)?), + &format!("-v {}", product_version), &format!("-m {}", METRICS_PORT), &format!("-c {}/ca.crt", KEYSTORE_REPORTING_TASK_MOUNT), ] @@ -896,7 +904,7 @@ fn build_reporting_task_job( let mut volumes = vec![build_keystore_volume(KEYSTORE_VOLUME_NAME)]; // Volume and Volume mounts for the authentication secret - let auth_volumes = get_auth_volumes(&nifi.spec.authentication_config.method) + let auth_volumes = get_auth_volumes(&nifi.spec.config.authentication.method) .context(MaterializeAuthConfigSnafu)?; for (name, (mount_path, volume)) in auth_volumes { @@ -914,7 +922,7 @@ fn build_reporting_task_job( let job_name = format!( "{}-create-reporting-task-{}", nifi.name(), - nifi_version(nifi)?.replace('.', "-") + product_version.replace('.', "-") ); let pod = PodTemplateSpec { @@ -958,7 +966,7 @@ async fn check_or_generate_sensitive_key( client: &Client, nifi: &NifiCluster, ) -> Result { - let sensitive_config = &nifi.spec.sensitive_properties_config; + let sensitive_config = &nifi.spec.config.sensitive_properties; let namespace: &str = &nifi.namespace().context(ObjectHasNoNamespaceSnafu)?; match client @@ -1131,13 +1139,27 @@ async fn get_proxy_hosts( Ok(proxy_setting.join(",")) } -pub fn nifi_version(nifi: &NifiCluster) -> Result<&str> { +/// Returns the provided docker image e.g. 1.15.0-stackable0 +fn nifi_image_version(nifi: &NifiCluster) -> Result<&str> { nifi.spec .version .as_deref() .context(ObjectHasNoVersionSnafu) } +/// Returns our semver representation for product config e.g. 1.15.0 +fn nifi_product_version(nifi: &NifiCluster) -> Result<&str> { + let image_version = nifi_image_version(nifi)?; + image_version + .split('-') + .collect::>() + .first() + .cloned() + .with_context(|| FailedNifiProductVersionRetrievalSnafu { + version: image_version.to_string(), + }) +} + pub fn error_policy(_error: &Error, _ctx: Context) -> Action { Action::requeue(Duration::from_secs(10)) } diff --git a/tests/templates/kuttl/smoke/00-assert.yaml b/tests/templates/kuttl/smoke/00-assert.yaml index 22642c22..a1e216b4 100644 --- a/tests/templates/kuttl/smoke/00-assert.yaml +++ b/tests/templates/kuttl/smoke/00-assert.yaml @@ -1,4 +1,10 @@ --- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +metadata: + name: install-zk +timeout: 600 +--- apiVersion: apps/v1 kind: StatefulSet metadata: diff --git a/tests/templates/kuttl/smoke/01-install-nifi.yaml.j2 b/tests/templates/kuttl/smoke/01-install-nifi.yaml.j2 index 37e58921..f75f7135 100644 --- a/tests/templates/kuttl/smoke/01-install-nifi.yaml.j2 +++ b/tests/templates/kuttl/smoke/01-install-nifi.yaml.j2 @@ -21,13 +21,13 @@ metadata: spec: version: {{ test_scenario['values']['nifi'] }} zookeeperConfigMapName: test-nifi-znode - authenticationConfig: - method: - SingleUser: - adminCredentialsSecret: - name: nifi-admin-credentials-simple - sensitivePropertiesConfig: - keySecret: nifi-sensitive-property-key + config: + authentication: + method: + singleUser: + adminCredentialsSecret: nifi-admin-credentials-simple + sensitiveProperties: + keySecret: nifi-sensitive-property-key nodes: roleGroups: default: diff --git a/tests/templates/kuttl/smoke/02-assert.yaml b/tests/templates/kuttl/smoke/02-assert.yaml index 3fb72f54..9dc8f1ef 100644 --- a/tests/templates/kuttl/smoke/02-assert.yaml +++ b/tests/templates/kuttl/smoke/02-assert.yaml @@ -1,4 +1,10 @@ --- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +metadata: + name: scale-up-nifi +timeout: 1200 +--- apiVersion: apps/v1 kind: StatefulSet metadata: diff --git a/tests/templates/kuttl/smoke/02-scale-up-nifi.yaml.j2 b/tests/templates/kuttl/smoke/02-scale-up-nifi.yaml.j2 index be8afadd..d036b02d 100644 --- a/tests/templates/kuttl/smoke/02-scale-up-nifi.yaml.j2 +++ b/tests/templates/kuttl/smoke/02-scale-up-nifi.yaml.j2 @@ -6,13 +6,13 @@ metadata: spec: version: {{ test_scenario['values']['nifi'] }} zookeeperConfigMapName: test-nifi-znode - authenticationConfig: - method: - SingleUser: - adminCredentialsSecret: - name: nifi-admin-credentials-simple - sensitivePropertiesConfig: - keySecret: nifi-sensitive-property-key + config: + authentication: + method: + singleUser: + adminCredentialsSecret: nifi-admin-credentials-simple + sensitiveProperties: + keySecret: nifi-sensitive-property-key nodes: roleGroups: default: diff --git a/tests/templates/kuttl/smoke/04-assert.yaml b/tests/templates/kuttl/smoke/04-assert.yaml index 62ebacc2..4c13659c 100644 --- a/tests/templates/kuttl/smoke/04-assert.yaml +++ b/tests/templates/kuttl/smoke/04-assert.yaml @@ -3,6 +3,7 @@ apiVersion: kuttl.dev/v1beta1 kind: TestAssert metadata: name: test-nifi +timeout: 300 commands: - script: kubectl exec -n $NAMESPACE test-nifi-0 -- python /tmp/test_nifi.py -u admin -p supersecretpassword -n $NAMESPACE - script: kubectl exec -n $NAMESPACE test-nifi-0 -- python /tmp/test_nifi_metrics.py -n $NAMESPACE diff --git a/tests/templates/kuttl/smoke/05-assert.yaml b/tests/templates/kuttl/smoke/05-assert.yaml index 3a857787..3e00bab1 100644 --- a/tests/templates/kuttl/smoke/05-assert.yaml +++ b/tests/templates/kuttl/smoke/05-assert.yaml @@ -1,4 +1,10 @@ --- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +metadata: + name: test-nifi-anonymous +timeout: 600 +--- apiVersion: v1 kind: Event reason: Started diff --git a/tests/templates/kuttl/smoke/05-enable-anonymous.yaml.j2 b/tests/templates/kuttl/smoke/05-enable-anonymous.yaml.j2 index f61680bd..c9f82109 100644 --- a/tests/templates/kuttl/smoke/05-enable-anonymous.yaml.j2 +++ b/tests/templates/kuttl/smoke/05-enable-anonymous.yaml.j2 @@ -6,20 +6,20 @@ metadata: spec: version: {{ test_scenario['values']['nifi'] }} zookeeperConfigMapName: test-nifi-znode - authenticationConfig: - method: - SingleUser: - adminCredentialsSecret: - name: nifi-admin-credentials-simple - allowAnonymousAccess: true + config: + authentication: + method: + singleUser: + adminCredentialsSecret: nifi-admin-credentials-simple + allowAnonymousAccess: true + sensitiveProperties: + keySecret: nifi-sensitive-property-key nodes: roleGroups: default: selector: matchLabels: kubernetes.io/os: linux - config: - sensitivePropertyKeySecret: nifi-sensitive-property-key log: rootLogLevel: INFO replicas: 2 diff --git a/tests/templates/kuttl/smoke/test_nifi_metrics.py b/tests/templates/kuttl/smoke/test_nifi_metrics.py index 510b146f..d4bc187d 100755 --- a/tests/templates/kuttl/smoke/test_nifi_metrics.py +++ b/tests/templates/kuttl/smoke/test_nifi_metrics.py @@ -14,7 +14,7 @@ help="The namespace the test is running in") all_args.add_argument("-p", "--port", required=False, default="8081", help="The port where metrics are exposed") - all_args.add_argument("-t", "--timeout", required=False, default="60", + all_args.add_argument("-t", "--timeout", required=False, default="120", help="The timeout in seconds to wait for the metrics port to be opened") args = vars(all_args.parse_args()) @@ -39,6 +39,6 @@ time.sleep(timeout) except Exception as ex: print(f"Failed to connect to [{url}]:\n {str(ex)}") - time.sleep(timeout) + time.sleep(10) exit(-1) From 6f8e1b603b620efabe30fcc138c79f243035c9bb Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Mon, 6 Jun 2022 16:22:49 +0200 Subject: [PATCH 2/9] Adapted changelog --- CHANGELOG.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index edc3efec..53d11817 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,8 +16,8 @@ All notable changes to this project will be documented in this file. - `operator-rs` `0.10.0` -> `0.15.0` ([#218], [#223], [#230]). - [BREAKING] Specifying the product version has been changed to adhere to [ADR018](https://docs.stackable.tech/home/contributor/adr/ADR018-product_image_versioning.html) instead of just specifying the product version you will now have to add the Stackable image version as well, so `version: 3.5.8` becomes (for example) `version: 3.5.8-stackable0.1.0` ([#270]) -- [BREAKING] CRD overhaul: Moved `authenticationConfig` to top level `config.authentication`. `SingleUser` now proper camelCase `singleUser`. `adminCredentialsSecret` now takes a String instead of `SecretReference` ([#xxx]). -- [BREAKING] CRD overhaul: Moved `sensitivePropertiesConfig` to top level `config.sensitiveProperties` ([#xxx]). +- [BREAKING] CRD overhaul: Moved `authenticationConfig` to top level `config.authentication`. `SingleUser` now proper camelCase `singleUser`. `adminCredentialsSecret` now takes a String instead of `SecretReference` ([#277]). +- [BREAKING] CRD overhaul: Moved `sensitivePropertiesConfig` to top level `config.sensitiveProperties` ([#277]). ### Removed @@ -27,7 +27,7 @@ All notable changes to this project will be documented in this file. [#223]: https://github.com/stackabletech/nifi-operator/pull/223 [#230]: https://github.com/stackabletech/nifi-operator/pull/230 [#270]: https://github.com/stackabletech/nifi-operator/pull/270 -[#xxx]: https://github.com/stackabletech/nifi-operator/pull/xxx +[#277]: https://github.com/stackabletech/nifi-operator/pull/277 ## [0.5.0] - 2022-02-14 From 87a9b79c912d871c6ab195189aa282b0f002002b Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Mon, 6 Jun 2022 16:24:43 +0200 Subject: [PATCH 3/9] fixed md linter --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 53d11817..849ad22f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,7 +16,7 @@ All notable changes to this project will be documented in this file. - `operator-rs` `0.10.0` -> `0.15.0` ([#218], [#223], [#230]). - [BREAKING] Specifying the product version has been changed to adhere to [ADR018](https://docs.stackable.tech/home/contributor/adr/ADR018-product_image_versioning.html) instead of just specifying the product version you will now have to add the Stackable image version as well, so `version: 3.5.8` becomes (for example) `version: 3.5.8-stackable0.1.0` ([#270]) -- [BREAKING] CRD overhaul: Moved `authenticationConfig` to top level `config.authentication`. `SingleUser` now proper camelCase `singleUser`. `adminCredentialsSecret` now takes a String instead of `SecretReference` ([#277]). +- [BREAKING] CRD overhaul: Moved `authenticationConfig` to top level `config.authentication`. `SingleUser` now proper camelCase `singleUser`. `adminCredentialsSecret` now takes a String instead of `SecretReference` ([#277]). - [BREAKING] CRD overhaul: Moved `sensitivePropertiesConfig` to top level `config.sensitiveProperties` ([#277]). ### Removed From c00139e2b4038d0f647b0acc153294458706f758 Mon Sep 17 00:00:00 2001 From: Nikolaus Winter Date: Tue, 7 Jun 2022 12:35:52 +0200 Subject: [PATCH 4/9] added CI config files for test migration --- .ci/integration-tests/aws-eks/cluster.yaml | 15 +++++++++++++++ .ci/integration-tests/aws-eks/test.sh | 7 +++++++ .ci/integration-tests/azure-aks/cluster.yaml | 13 +++++++++++++ .ci/integration-tests/azure-aks/test.sh | 7 +++++++ .../hcloud-centos-8/cluster.yaml | 19 +++++++++++++++++++ .ci/integration-tests/hcloud-centos-8/test.sh | 7 +++++++ .ci/integration-tests/ionos-k8s/cluster.yaml | 15 +++++++++++++++ .ci/integration-tests/ionos-k8s/test.sh | 7 +++++++ 8 files changed, 90 insertions(+) create mode 100644 .ci/integration-tests/aws-eks/cluster.yaml create mode 100644 .ci/integration-tests/aws-eks/test.sh create mode 100644 .ci/integration-tests/azure-aks/cluster.yaml create mode 100644 .ci/integration-tests/azure-aks/test.sh create mode 100644 .ci/integration-tests/hcloud-centos-8/cluster.yaml create mode 100644 .ci/integration-tests/hcloud-centos-8/test.sh create mode 100644 .ci/integration-tests/ionos-k8s/cluster.yaml create mode 100644 .ci/integration-tests/ionos-k8s/test.sh diff --git a/.ci/integration-tests/aws-eks/cluster.yaml b/.ci/integration-tests/aws-eks/cluster.yaml new file mode 100644 index 00000000..3e1495cc --- /dev/null +++ b/.ci/integration-tests/aws-eks/cluster.yaml @@ -0,0 +1,15 @@ +--- +apiVersion: t2.stackable.tech/v1 +kind: Infra +template: aws-eks +metadata: + name: nifi-operator-integration-tests + description: "Cluster for NiFi Operator Integration Tests (AWS EKS)" +publicKeys: [] +spec: + region: "eu-central-1" + awsInstanceType: "t2.medium" + versions: + _-operator: NIGHTLY + nifi-operator: "$NIFI_OPERATOR_VERSION" + node_count: 3 diff --git a/.ci/integration-tests/aws-eks/test.sh b/.ci/integration-tests/aws-eks/test.sh new file mode 100644 index 00000000..56474c82 --- /dev/null +++ b/.ci/integration-tests/aws-eks/test.sh @@ -0,0 +1,7 @@ +#!/bin/bash +git clone -b "$GIT_BRANCH" https://github.com/stackabletech/nifi-operator.git +(cd nifi-operator/ && ./scripts/run_tests.sh --parallel 1) +exit_code=$? +./operator-logs.sh nifi > /target/nifi-operator.log +./operator-logs.sh zookeeper > /target/zookeeper-operator.log +exit $exit_code diff --git a/.ci/integration-tests/azure-aks/cluster.yaml b/.ci/integration-tests/azure-aks/cluster.yaml new file mode 100644 index 00000000..59b9fa42 --- /dev/null +++ b/.ci/integration-tests/azure-aks/cluster.yaml @@ -0,0 +1,13 @@ +--- +apiVersion: t2.stackable.tech/v1 +kind: Infra +template: azure-aks +metadata: + name: nifi-operator-integration-tests + description: "Cluster for NiFi Operator Integration Tests (Azure AKS)" +publicKeys: [] +spec: + versions: + _-operator: NIGHTLY + nifi-operator: "$NIFI_OPERATOR_VERSION" + node_count: 3 diff --git a/.ci/integration-tests/azure-aks/test.sh b/.ci/integration-tests/azure-aks/test.sh new file mode 100644 index 00000000..56474c82 --- /dev/null +++ b/.ci/integration-tests/azure-aks/test.sh @@ -0,0 +1,7 @@ +#!/bin/bash +git clone -b "$GIT_BRANCH" https://github.com/stackabletech/nifi-operator.git +(cd nifi-operator/ && ./scripts/run_tests.sh --parallel 1) +exit_code=$? +./operator-logs.sh nifi > /target/nifi-operator.log +./operator-logs.sh zookeeper > /target/zookeeper-operator.log +exit $exit_code diff --git a/.ci/integration-tests/hcloud-centos-8/cluster.yaml b/.ci/integration-tests/hcloud-centos-8/cluster.yaml new file mode 100644 index 00000000..ffb527eb --- /dev/null +++ b/.ci/integration-tests/hcloud-centos-8/cluster.yaml @@ -0,0 +1,19 @@ +--- +apiVersion: t2.stackable.tech/v1 +kind: Infra +template: hcloud-centos-8 +metadata: + name: nifi-operator-integration-tests + description: "Cluster for NiFi Operator Integration Tests (Hetzner Cloud / CentOS 8)" +domain: stackable.test +publicKeys: [] +spec: + location: "hel1" + k8sVersion: "$K8S_VERSION" + wireguard: false + versions: + _-operator: NIGHTLY + nifi-operator: "$NIFI_OPERATOR_VERSION" + nodes: + main: + numberOfNodes: 3 diff --git a/.ci/integration-tests/hcloud-centos-8/test.sh b/.ci/integration-tests/hcloud-centos-8/test.sh new file mode 100644 index 00000000..56474c82 --- /dev/null +++ b/.ci/integration-tests/hcloud-centos-8/test.sh @@ -0,0 +1,7 @@ +#!/bin/bash +git clone -b "$GIT_BRANCH" https://github.com/stackabletech/nifi-operator.git +(cd nifi-operator/ && ./scripts/run_tests.sh --parallel 1) +exit_code=$? +./operator-logs.sh nifi > /target/nifi-operator.log +./operator-logs.sh zookeeper > /target/zookeeper-operator.log +exit $exit_code diff --git a/.ci/integration-tests/ionos-k8s/cluster.yaml b/.ci/integration-tests/ionos-k8s/cluster.yaml new file mode 100644 index 00000000..5a9a01d5 --- /dev/null +++ b/.ci/integration-tests/ionos-k8s/cluster.yaml @@ -0,0 +1,15 @@ +--- +apiVersion: t2.stackable.tech/v1 +kind: Infra +template: ionos-k8s +metadata: + name: nifi-operator-integration-tests + description: "Cluster for NiFi Operator Integration Tests (IONOS Cloud managed K8s)" +domain: stackable.test +publicKeys: [] +spec: + region: de/fra + versions: + _-operator: NIGHTLY + nifi-operator: "$NIFI_OPERATOR_VERSION" + node_count: 3 diff --git a/.ci/integration-tests/ionos-k8s/test.sh b/.ci/integration-tests/ionos-k8s/test.sh new file mode 100644 index 00000000..56474c82 --- /dev/null +++ b/.ci/integration-tests/ionos-k8s/test.sh @@ -0,0 +1,7 @@ +#!/bin/bash +git clone -b "$GIT_BRANCH" https://github.com/stackabletech/nifi-operator.git +(cd nifi-operator/ && ./scripts/run_tests.sh --parallel 1) +exit_code=$? +./operator-logs.sh nifi > /target/nifi-operator.log +./operator-logs.sh zookeeper > /target/zookeeper-operator.log +exit $exit_code From 32b54559c805ca54c1079831a72c9ce00c05b8b3 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Tue, 7 Jun 2022 16:08:01 +0200 Subject: [PATCH 5/9] moved image/version methods to NifiCluster impl --- rust/crd/src/lib.rs | 37 ++++++++++++++++++--- rust/operator-binary/src/controller.rs | 45 ++++++++------------------ 2 files changed, 45 insertions(+), 37 deletions(-) diff --git a/rust/crd/src/lib.rs b/rust/crd/src/lib.rs index 8097495a..3d09f3f8 100644 --- a/rust/crd/src/lib.rs +++ b/rust/crd/src/lib.rs @@ -23,6 +23,16 @@ pub const BALANCE_PORT: u16 = 6243; pub const METRICS_PORT_NAME: &str = "metrics"; pub const METRICS_PORT: u16 = 8081; +#[derive(Snafu, Debug)] +pub enum Error { + #[snafu(display("could not parse product version from image: [{image_version}]. Expected format e.g. [1.15.0-stackable0.1.0]"))] + NifiProductVersion { image_version: String }, + #[snafu(display("object has no namespace associated"))] + NoNamespace, + #[snafu(display("object defines no version"))] + ObjectHasNoVersion, +} + #[derive(Clone, CustomResource, Debug, Deserialize, JsonSchema, PartialEq, Serialize)] #[kube( group = "nifi.stackable.tech", @@ -179,10 +189,6 @@ pub enum LogLevel { FATAL, } -#[derive(Debug, Snafu)] -#[snafu(display("object has no namespace associated"))] -pub struct NoNamespaceError; - impl NifiCluster { /// The name of the role-level load-balanced Kubernetes `Service` pub fn node_role_service_name(&self) -> Option { @@ -211,7 +217,7 @@ impl NifiCluster { /// /// We try to predict the pods here rather than looking at the current cluster state in order to /// avoid instance churn. - pub fn pods(&self) -> Result + '_, NoNamespaceError> { + pub fn pods(&self) -> Result + '_, Error> { let ns = self.metadata.namespace.clone().context(NoNamespaceSnafu)?; Ok(self .spec @@ -231,6 +237,27 @@ impl NifiCluster { }) })) } + + /// Returns the provided docker image e.g. 1.15.0-stackable0 + pub fn image_version(&self) -> Result<&str, Error> { + self.spec + .version + .as_deref() + .context(ObjectHasNoVersionSnafu) + } + + /// Returns our semver representation for product config e.g. 1.15.0 + pub fn product_version(&self) -> Result<&str, Error> { + let image_version = self.image_version()?; + image_version + .split('-') + .collect::>() + .first() + .cloned() + .with_context(|| NifiProductVersionSnafu { + image_version: image_version.to_string(), + }) + } } /// Reference to a single `Pod` that is a component of a [`NifiCluster`] diff --git a/rust/operator-binary/src/controller.rs b/rust/operator-binary/src/controller.rs index ee9a7c55..c5669da1 100644 --- a/rust/operator-binary/src/controller.rs +++ b/rust/operator-binary/src/controller.rs @@ -69,8 +69,6 @@ pub struct Ctx { #[strum_discriminants(derive(IntoStaticStr))] #[allow(clippy::enum_variant_names)] pub enum Error { - #[snafu(display("object defines no version"))] - ObjectHasNoVersion, #[snafu(display("object defines no name"))] ObjectHasNoName, #[snafu(display("object defines no spec"))] @@ -153,8 +151,8 @@ pub enum Error { ExternalPort, #[snafu(display("Could not build role service fqdn"))] NoRoleServiceFqdn, - #[snafu(display("Could not extract NiFi product version (x.x.x) from image: [{version}]. Expected format e.g. x.x.x-stackable0.1.0"))] - FailedNifiProductVersionRetrieval { version: String }, + #[snafu(display("failed to parse NiFi version"))] + NifiVersionParseFailure { source: stackable_nifi_crd::Error }, } type Result = std::result::Result; @@ -168,7 +166,9 @@ impl ReconcilerError for Error { pub async fn reconcile_nifi(nifi: Arc, ctx: Context) -> Result { tracing::info!("Starting reconcile"); let client = &ctx.get_ref().client; - let nifi_product_version = nifi_product_version(&nifi)?; + let nifi_product_version = nifi + .product_version() + .context(NifiVersionParseFailureSnafu)?; let namespace = &nifi .metadata .namespace @@ -287,7 +287,7 @@ pub fn build_node_role_service(nifi: &NifiCluster) -> Result { .with_recommended_labels( nifi, APP_NAME, - nifi_image_version(nifi)?, + nifi.image_version().context(NifiVersionParseFailureSnafu)?, &role_name, "global", ) @@ -335,7 +335,7 @@ fn build_node_rolegroup_log_config_map( .with_recommended_labels( nifi, APP_NAME, - nifi_image_version(nifi)?, + nifi.image_version().context(NifiVersionParseFailureSnafu)?, &rolegroup.role, &rolegroup.role_group, ) @@ -375,7 +375,7 @@ async fn build_node_rolegroup_config_map( .with_recommended_labels( nifi, APP_NAME, - nifi_image_version(nifi)?, + nifi.image_version().context(NifiVersionParseFailureSnafu)?, &rolegroup.role, &rolegroup.role_group, ) @@ -439,7 +439,7 @@ fn build_node_rolegroup_service( .with_recommended_labels( nifi, APP_NAME, - nifi_image_version(nifi)?, + nifi.image_version().context(NifiVersionParseFailureSnafu)?, &rolegroup.role, &rolegroup.role_group, ) @@ -533,7 +533,7 @@ fn build_node_rolegroup_statefulset( .role_groups .get(&rolegroup_ref.role_group); - let nifi_version = nifi_image_version(nifi)?; + let nifi_version = nifi.image_version().context(NifiVersionParseFailureSnafu)?; let image = format!("docker.stackable.tech/stackable/nifi:{}", nifi_version); let node_address = format!( @@ -871,7 +871,9 @@ fn build_reporting_task_job( ) -> Result { let rolegroup_obj_name = rolegroup_ref.object_name(); let namespace: &str = &nifi.namespace().context(ObjectHasNoNamespaceSnafu)?; - let product_version = nifi_product_version(nifi)?; + let product_version = nifi + .product_version() + .context(NifiVersionParseFailureSnafu)?; let nifi_connect_url = format!( "https://{rolegroup}-0.{rolegroup}.{namespace}.svc.cluster.local:{port}/nifi-api", rolegroup = rolegroup_obj_name, @@ -1139,27 +1141,6 @@ async fn get_proxy_hosts( Ok(proxy_setting.join(",")) } -/// Returns the provided docker image e.g. 1.15.0-stackable0 -fn nifi_image_version(nifi: &NifiCluster) -> Result<&str> { - nifi.spec - .version - .as_deref() - .context(ObjectHasNoVersionSnafu) -} - -/// Returns our semver representation for product config e.g. 1.15.0 -fn nifi_product_version(nifi: &NifiCluster) -> Result<&str> { - let image_version = nifi_image_version(nifi)?; - image_version - .split('-') - .collect::>() - .first() - .cloned() - .with_context(|| FailedNifiProductVersionRetrievalSnafu { - version: image_version.to_string(), - }) -} - pub fn error_policy(_error: &Error, _ctx: Context) -> Action { Action::requeue(Duration::from_secs(10)) } From b855907f0664ada2d4ad9ac8b63020c82e25df5e Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Tue, 7 Jun 2022 16:28:33 +0200 Subject: [PATCH 6/9] ran pre commit for all files --- .pre-commit-config.yaml | 5 ----- CHANGELOG.md | 8 ++++++-- Cargo.toml | 2 +- docs/modules/ROOT/pages/env_var_args.adoc | 1 - docs/modules/ROOT/partials/supported-versions.adoc | 2 +- rust/crd/Cargo.toml | 1 - .../resources/singleuser-login-identity-providers.xml | 2 +- rust/operator-binary/src/controller.rs | 2 +- tests/templates/kuttl/smoke/00-install-zk.yaml.j2 | 2 +- tests/templates/kuttl/smoke/01-install-nifi.yaml.j2 | 1 - tests/templates/kuttl/smoke/requirements.txt | 2 +- 11 files changed, 12 insertions(+), 16 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index eda50813..7153b386 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -33,8 +33,3 @@ repos: rev: 4.0.1 hooks: - id: flake8 - - - repo: https://github.com/PyCQA/pylint - rev: v2.13.9 - hooks: - - id: pylint diff --git a/CHANGELOG.md b/CHANGELOG.md index 849ad22f..26a3341f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,7 +17,7 @@ All notable changes to this project will be documented in this file. - `operator-rs` `0.10.0` -> `0.15.0` ([#218], [#223], [#230]). - [BREAKING] Specifying the product version has been changed to adhere to [ADR018](https://docs.stackable.tech/home/contributor/adr/ADR018-product_image_versioning.html) instead of just specifying the product version you will now have to add the Stackable image version as well, so `version: 3.5.8` becomes (for example) `version: 3.5.8-stackable0.1.0` ([#270]) - [BREAKING] CRD overhaul: Moved `authenticationConfig` to top level `config.authentication`. `SingleUser` now proper camelCase `singleUser`. `adminCredentialsSecret` now takes a String instead of `SecretReference` ([#277]). -- [BREAKING] CRD overhaul: Moved `sensitivePropertiesConfig` to top level `config.sensitiveProperties` ([#277]). +- [BREAKING] CRD overhaul: Moved `sensitivePropertiesConfig` to top level `config.sensitiveProperties` ([#277]). ### Removed @@ -41,6 +41,7 @@ All notable changes to this project will be documented in this file. ## [0.4.0] - 2021-12-06 ### Added + - Support for 1.15.0 ([#125]) - Sensitive property key is setable via a secret ([#125]) @@ -59,6 +60,7 @@ All notable changes to this project will be documented in this file. ## [0.3.0] - 2021-10-27 ### Added + - Added versioning code from operator-rs for up and downgrades ([#81]). - Added `ProductVersion` to status ([#81]). - Added `Condition` to status ([#81]). @@ -71,8 +73,9 @@ All notable changes to this project will be documented in this file. - `kube-rs`: `0.58` → `0.60` ([#83]). - `k8s-openapi` `0.12` → `0.13` and features: `v1_21` → `v1_22` ([#83]). - `operator-rs` `0.2.1` → `0.2.2` ([#83]). - + ### Fixed + - Fixed a bug where `wait_until_crds_present` only reacted to the main CRD, not the commands ([#92]). [#92]: https://github.com/stackabletech/nifi-operator/pull/92 @@ -83,6 +86,7 @@ All notable changes to this project will be documented in this file. ## [0.2.0] - 2021-09-14 ### Changed + - **Breaking:** Repository structure was changed and the -server crate renamed to -binary. As part of this change the -server suffix was removed from both the package name for os packages and the name of the executable ([#72]). [#72]: https://github.com/stackabletech/nifi-operator/pull/72 diff --git a/Cargo.toml b/Cargo.toml index 2182c5f7..9915447f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,4 +2,4 @@ members = [ "rust/crd", "rust/operator-binary" -] \ No newline at end of file +] diff --git a/docs/modules/ROOT/pages/env_var_args.adoc b/docs/modules/ROOT/pages/env_var_args.adoc index 9f8b3111..30756b69 100644 --- a/docs/modules/ROOT/pages/env_var_args.adoc +++ b/docs/modules/ROOT/pages/env_var_args.adoc @@ -53,4 +53,3 @@ docker run \ --mount type=bind,source="$HOME/.kube/config",target="/home/stackable/.kube/config" \ docker.stackable.tech/stackable/nifi-operator:latest ---- - diff --git a/docs/modules/ROOT/partials/supported-versions.adoc b/docs/modules/ROOT/partials/supported-versions.adoc index d31e0d23..cd030d5b 100644 --- a/docs/modules/ROOT/partials/supported-versions.adoc +++ b/docs/modules/ROOT/partials/supported-versions.adoc @@ -2,4 +2,4 @@ // This is a separate file, since it is used by both the direct NiFi-Operator documentation, and the overarching // Stackable Platform documentation. -- 1.15.0 \ No newline at end of file +- 1.15.0 diff --git a/rust/crd/Cargo.toml b/rust/crd/Cargo.toml index e67dc521..f233aa5d 100644 --- a/rust/crd/Cargo.toml +++ b/rust/crd/Cargo.toml @@ -16,4 +16,3 @@ rand = "0.8" stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "0.15.0" } strum = { version = "0.24", features = ["derive"] } tracing = "0.1" - diff --git a/rust/operator-binary/resources/singleuser-login-identity-providers.xml b/rust/operator-binary/resources/singleuser-login-identity-providers.xml index aa80c5ec..9ec0c362 100644 --- a/rust/operator-binary/resources/singleuser-login-identity-providers.xml +++ b/rust/operator-binary/resources/singleuser-login-identity-providers.xml @@ -6,4 +6,4 @@ xxx yyy - \ No newline at end of file + diff --git a/rust/operator-binary/src/controller.rs b/rust/operator-binary/src/controller.rs index c5669da1..239cfdfd 100644 --- a/rust/operator-binary/src/controller.rs +++ b/rust/operator-binary/src/controller.rs @@ -858,7 +858,7 @@ fn build_node_rolegroup_statefulset( /// docker image and more specifically the `create_nifi_reporting_task.py` Python script. /// /// This script uses the [`nipyapi`](https://nipyapi.readthedocs.io/en/latest/readme.html) -/// library to authenticate and run the required REST calls to the NiFi REST API. +/// library to authenticate and run the required REST calls to the NiFi REST API. /// /// In order to authenticate we need the `username` and `password` from the /// [`NifiAuthenticationConfig`](`stackable_nifi_crd::authentication::NifiAuthenticationConfig`) diff --git a/tests/templates/kuttl/smoke/00-install-zk.yaml.j2 b/tests/templates/kuttl/smoke/00-install-zk.yaml.j2 index 0c75b592..4d6a2a81 100644 --- a/tests/templates/kuttl/smoke/00-install-zk.yaml.j2 +++ b/tests/templates/kuttl/smoke/00-install-zk.yaml.j2 @@ -19,4 +19,4 @@ metadata: name: test-nifi-znode spec: clusterRef: - name: test-zk \ No newline at end of file + name: test-zk diff --git a/tests/templates/kuttl/smoke/01-install-nifi.yaml.j2 b/tests/templates/kuttl/smoke/01-install-nifi.yaml.j2 index f75f7135..b0c54542 100644 --- a/tests/templates/kuttl/smoke/01-install-nifi.yaml.j2 +++ b/tests/templates/kuttl/smoke/01-install-nifi.yaml.j2 @@ -38,4 +38,3 @@ spec: log: rootLogLevel: INFO replicas: 2 - diff --git a/tests/templates/kuttl/smoke/requirements.txt b/tests/templates/kuttl/smoke/requirements.txt index 9cdfca0c..a743bbe3 100644 --- a/tests/templates/kuttl/smoke/requirements.txt +++ b/tests/templates/kuttl/smoke/requirements.txt @@ -1 +1 @@ -requests==2.27.1 \ No newline at end of file +requests==2.27.1 From 6d9b024f68ec52d620176ea24d48b7d7901142e2 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Wed, 8 Jun 2022 10:06:07 +0200 Subject: [PATCH 7/9] adapted to pr review --- rust/crd/src/lib.rs | 4 +--- rust/operator-binary/src/controller.rs | 8 ++++---- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/rust/crd/src/lib.rs b/rust/crd/src/lib.rs index 3d09f3f8..0063f195 100644 --- a/rust/crd/src/lib.rs +++ b/rust/crd/src/lib.rs @@ -251,9 +251,7 @@ impl NifiCluster { let image_version = self.image_version()?; image_version .split('-') - .collect::>() - .first() - .cloned() + .next() .with_context(|| NifiProductVersionSnafu { image_version: image_version.to_string(), }) diff --git a/rust/operator-binary/src/controller.rs b/rust/operator-binary/src/controller.rs index 239cfdfd..5034c461 100644 --- a/rust/operator-binary/src/controller.rs +++ b/rust/operator-binary/src/controller.rs @@ -533,8 +533,8 @@ fn build_node_rolegroup_statefulset( .role_groups .get(&rolegroup_ref.role_group); - let nifi_version = nifi.image_version().context(NifiVersionParseFailureSnafu)?; - let image = format!("docker.stackable.tech/stackable/nifi:{}", nifi_version); + let image_version = nifi.image_version().context(NifiVersionParseFailureSnafu)?; + let image = format!("docker.stackable.tech/stackable/nifi:{}", image_version); let node_address = format!( "$POD_NAME.{}-node-{}.{}.svc.cluster.local", @@ -699,7 +699,7 @@ fn build_node_rolegroup_statefulset( m.with_recommended_labels( nifi, APP_NAME, - nifi_version, + image_version, &rolegroup_ref.role, &rolegroup_ref.role_group, ) @@ -796,7 +796,7 @@ fn build_node_rolegroup_statefulset( .with_recommended_labels( nifi, APP_NAME, - nifi_version, + image_version, &rolegroup_ref.role, &rolegroup_ref.role_group, ) From 0ca99ec18e335b0d010b18741ebed159d8579bbe Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Thu, 9 Jun 2022 16:08:23 +0200 Subject: [PATCH 8/9] removed redundant error --- rust/crd/src/lib.rs | 4 ---- 1 file changed, 4 deletions(-) diff --git a/rust/crd/src/lib.rs b/rust/crd/src/lib.rs index d9ee9aa3..d537b813 100644 --- a/rust/crd/src/lib.rs +++ b/rust/crd/src/lib.rs @@ -252,10 +252,6 @@ pub struct NifiStorageConfig { pub state_repo: PvcConfig, } -#[derive(Debug, Snafu)] -#[snafu(display("object has no namespace associated"))] -pub struct NoNamespaceError; - impl NifiCluster { /// The name of the role-level load-balanced Kubernetes `Service` pub fn node_role_service_name(&self) -> Option { From 85babdbffbb082323e849c270c84e8d4425f19aa Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Mon, 13 Jun 2022 12:47:01 +0200 Subject: [PATCH 9/9] fix resources test --- .../kuttl/resources/01-install-nifi.yaml.j2 | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/templates/kuttl/resources/01-install-nifi.yaml.j2 b/tests/templates/kuttl/resources/01-install-nifi.yaml.j2 index 41223e09..d44a74d7 100644 --- a/tests/templates/kuttl/resources/01-install-nifi.yaml.j2 +++ b/tests/templates/kuttl/resources/01-install-nifi.yaml.j2 @@ -21,13 +21,13 @@ metadata: spec: version: {{ test_scenario['values']['nifi'] }} zookeeperConfigMapName: test-nifi-znode - authenticationConfig: - method: - SingleUser: - adminCredentialsSecret: - name: nifi-admin-credentials-simple - sensitivePropertiesConfig: - keySecret: nifi-sensitive-property-key + config: + authentication: + method: + singleUser: + adminCredentialsSecret: nifi-admin-credentials-simple + sensitiveProperties: + keySecret: nifi-sensitive-property-key nodes: config: resources: