Skip to content

Commit

Permalink
e2e for OADP-460 noDefaultBackupLocation, check B/R is successful usi…
Browse files Browse the repository at this point in the history 
…ng BSL created manually (#690)

* e2e for OADP-460

* return err from BSL delete/create

* Add WithBackupImages

* Add backupOpts

* make test go fmt

* get dpaCR.Name from testSuiteInstanceName
  • Loading branch information
@kaovilai
kaovilai committed Jun 6, 2022
1 parent bee7a37 commit 77de96e
Show file tree
Hide file tree
Showing 5 changed files with 152 additions and 26 deletions.
1 change: 0 additions & 1 deletion controllers/velero.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -815,4 +815,3 @@ func (r DPAReconciler) noDefaultCredentials(dpa oadpv1alpha1.DataProtectionAppli
return providerNeedsDefaultCreds, hasCloudStorage, nil

}

63 changes: 52 additions & 11 deletions tests/e2e/backup_restore_suite_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,12 +9,14 @@ import (
"github.com/google/uuid"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
"github.com/openshift/oadp-operator/api/v1alpha1"
. "github.com/openshift/oadp-operator/tests/e2e/lib"
utils "github.com/openshift/oadp-operator/tests/e2e/utils"
"sigs.k8s.io/controller-runtime/pkg/client"
velerov1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

type VerificationFunction func(client.Client, string) error
type VerificationFunction func(*DpaCustomResource, string) error

var _ = Describe("AWS backup restore tests", func() {

Expand Down Expand Up @@ -53,18 +55,20 @@ var _ = Describe("AWS backup restore tests", func() {
PostRestoreVerify VerificationFunction
MaxK8SVersion *K8sVersion
MinK8SVersion *K8sVersion
dpaCrOpts []DpaCROption
backupOpts []BackupOpts
}

mongoReady := VerificationFunction(func(ocClient client.Client, namespace string) error {
Eventually(IsDCReady(ocClient, "mongo-persistent", "todolist"), timeoutMultiplier*time.Minute*10, time.Second*10).Should(BeTrue())
mongoReady := VerificationFunction(func(dpaCR *DpaCustomResource, namespace string) error {
Eventually(IsDCReady(dpaCR.Client, "mongo-persistent", "todolist"), timeoutMultiplier*time.Minute*10, time.Second*10).Should(BeTrue())
// err := VerifyBackupRestoreData(artifact_dir, namespace, "restify", "parks-app") // TODO: VERIFY PARKS APP DATA
return nil
})
mysqlReady := VerificationFunction(func(ocClient client.Client, namespace string) error {
mysqlReady := VerificationFunction(func(dpaCR *DpaCustomResource, namespace string) error {
// This test confirms that SCC restore logic in our plugin is working
//Eventually(IsDCReady(ocClient, "mssql-persistent", "mysql"), timeoutMultiplier*time.Minute*10, time.Second*10).Should(BeTrue())
Eventually(IsDeploymentReady(ocClient, "mysql-persistent", "mysql"), timeoutMultiplier*time.Minute*10, time.Second*10).Should(BeTrue())
exists, err := DoesSCCExist(ocClient, "mysql-persistent-scc")
Eventually(IsDeploymentReady(dpaCR.Client, "mysql-persistent", "mysql"), timeoutMultiplier*time.Minute*10, time.Second*10).Should(BeTrue())
exists, err := DoesSCCExist(dpaCR.Client, "mysql-persistent-scc")
if err != nil {
return err
}
Expand All @@ -86,7 +90,7 @@ var _ = Describe("AWS backup restore tests", func() {
Skip(reason)
}

err := dpaCR.Build(brCase.BackupRestoreType)
err := dpaCR.Build(brCase.BackupRestoreType, brCase.dpaCrOpts...)
Expect(err).NotTo(HaveOccurred())

updateLastInstallingNamespace(dpaCR.Namespace)
Expand Down Expand Up @@ -130,14 +134,14 @@ var _ = Describe("AWS backup restore tests", func() {

// Run optional custom verification
log.Printf("Running pre-backup function for case %s", brCase.Name)
err = brCase.PreBackupVerify(dpaCR.Client, brCase.ApplicationNamespace)
err = brCase.PreBackupVerify(dpaCR, brCase.ApplicationNamespace)
Expect(err).ToNot(HaveOccurred())

nsRequiresResticDCWorkaround, err := NamespaceRequiresResticDCWorkaround(dpaCR.Client, brCase.ApplicationNamespace)
Expect(err).ToNot(HaveOccurred())
// create backup
log.Printf("Creating backup %s for case %s", backupName, brCase.Name)
backup, err := CreateBackupForNamespaces(dpaCR.Client, namespace, backupName, []string{brCase.ApplicationNamespace})
backup, err := CreateBackupForNamespaces(dpaCR.Client, namespace, backupName, []string{brCase.ApplicationNamespace}, brCase.backupOpts...)
Expect(err).ToNot(HaveOccurred())

// wait for backup to not be running
Expand Down Expand Up @@ -219,7 +223,7 @@ var _ = Describe("AWS backup restore tests", func() {

// Run optional custom verification
log.Printf("Running post-restore function for case %s", brCase.Name)
err = brCase.PostRestoreVerify(dpaCR.Client, brCase.ApplicationNamespace)
err = brCase.PostRestoreVerify(dpaCR, brCase.ApplicationNamespace)
Expect(err).ToNot(HaveOccurred())

// Test is successful, clean up everything
Expand Down Expand Up @@ -262,5 +266,42 @@ var _ = Describe("AWS backup restore tests", func() {
PreBackupVerify: mysqlReady,
PostRestoreVerify: mysqlReady,
}, nil),
Entry("MySQL application NoDefaultBackupStorageLocation", BackupRestoreCase{
ApplicationTemplate: "./sample-applications/mysql-persistent/mysql-persistent-template.yaml",
ApplicationNamespace: "mysql-persistent",
Name: "mysql-e2e",
BackupRestoreType: RESTIC,
PreBackupVerify: VerificationFunction(func(dpaCR *DpaCustomResource, namespace string) error {
// create BSL
err := CreateBackupStorageLocation(dpaCR.Client, velerov1.BackupStorageLocation{
ObjectMeta: metav1.ObjectMeta{
Name: dpaCR.Name + "nobsl-1",
Namespace: dpaCR.Namespace,
},
Spec: *dpaCR.VeleroBSL(),
})
if err != nil {
return err
}
return mysqlReady(dpaCR, namespace)
}),
PostRestoreVerify: VerificationFunction(func(dpaCR *DpaCustomResource, namespace string) error {
// delete BSL
err := DeleteBackupStorageLocation(dpaCR.Client, velerov1.BackupStorageLocation{
ObjectMeta: metav1.ObjectMeta{
Name: dpaCR.Name + "nobsl-1",
Namespace: dpaCR.Namespace,
}})
if err != nil {
return err
}
return mysqlReady(dpaCR, namespace)
}),
dpaCrOpts: []DpaCROption{
WithVeleroConfig(&v1alpha1.VeleroConfig{NoDefaultBackupLocation: true}),
WithBackupImages(false),
},
backupOpts: []BackupOpts{WithBackupStorageLocation("ts-" + instanceName + "nobsl-1")}, // e2e_sute_test.go: dpaCR.name = "ts-" + instanceName
}, nil),
)
})
17 changes: 16 additions & 1 deletion tests/e2e/lib/backup.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,16 @@ import (
"sigs.k8s.io/controller-runtime/pkg/client"
)

func CreateBackupForNamespaces(ocClient client.Client, veleroNamespace, backupName string, namespaces []string) (velero.Backup, error) {
type BackupOpts func(*velero.Backup) error

func WithBackupStorageLocation(name string) BackupOpts {
return func(backup *velero.Backup) error {
backup.Spec.StorageLocation = name
return nil
}
}

func CreateBackupForNamespaces(ocClient client.Client, veleroNamespace, backupName string, namespaces []string, backupOpts ...BackupOpts) (velero.Backup, error) {

backup := velero.Backup{
ObjectMeta: metav1.ObjectMeta{
Expand All @@ -22,6 +31,12 @@ func CreateBackupForNamespaces(ocClient client.Client, veleroNamespace, backupNa
IncludedNamespaces: namespaces,
},
}
for _, opt := range backupOpts {
err := opt(&backup)
if err != nil {
return velero.Backup{}, err
}
}
err := ocClient.Create(context.Background(), &backup)
return backup, err
}
Expand Down
66 changes: 53 additions & 13 deletions tests/e2e/lib/dpa_helpers.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,10 +53,55 @@ type DpaCustomResource struct {
Provider string
}

type DpaCROption func(*oadpv1alpha1.DataProtectionApplication) error

func WithConfiguration(configuration *oadpv1alpha1.ApplicationConfig) DpaCROption {
return func(cr *oadpv1alpha1.DataProtectionApplication) error {
cr.Spec.Configuration = configuration
return nil
}
}

func WithVeleroConfig(config *oadpv1alpha1.VeleroConfig) DpaCROption {
return func(cr *oadpv1alpha1.DataProtectionApplication) error {
cr.Spec.Configuration.Velero = config
return nil
}
}

func WithResticConfig(config *oadpv1alpha1.ResticConfig) DpaCROption {
return func(cr *oadpv1alpha1.DataProtectionApplication) error {
cr.Spec.Configuration.Restic = config
return nil
}
}

func WithBackupImages(backupImage bool) DpaCROption {
return func(cr *oadpv1alpha1.DataProtectionApplication) error {
cr.Spec.BackupImages = pointer.Bool(backupImage)
return nil
}
}

var VeleroPrefix = "velero-e2e-" + string(uuid.NewUUID())
var Dpa *oadpv1alpha1.DataProtectionApplication

func (v *DpaCustomResource) Build(backupRestoreType BackupRestoreType) error {
func (v *DpaCustomResource) VeleroBSL() *velero.BackupStorageLocationSpec {
return &velero.BackupStorageLocationSpec{
Provider: v.CustomResource.Spec.BackupLocations[0].Velero.Provider,
Default: true,
Config: v.CustomResource.Spec.BackupLocations[0].Velero.Config,
Credential: v.CustomResource.Spec.BackupLocations[0].Velero.Credential,
StorageType: velero.StorageType{
ObjectStorage: &velero.ObjectStorageLocation{
Bucket: v.CustomResource.Spec.BackupLocations[0].Velero.ObjectStorage.Bucket,
Prefix: VeleroPrefix,
},
},
}
}

func (v *DpaCustomResource) Build(backupRestoreType BackupRestoreType, dpaCrOpts ...DpaCROption) error {
// Velero Instance creation spec with backupstorage location default to AWS. Would need to parameterize this later on to support multiple plugins.
dpaInstance := oadpv1alpha1.DataProtectionApplication{
ObjectMeta: metav1.ObjectMeta{
Expand All @@ -75,18 +120,7 @@ func (v *DpaCustomResource) Build(backupRestoreType BackupRestoreType) error {
SnapshotLocations: v.CustomResource.Spec.SnapshotLocations,
BackupLocations: []oadpv1alpha1.BackupLocation{
{
Velero: &velero.BackupStorageLocationSpec{
Provider: v.CustomResource.Spec.BackupLocations[0].Velero.Provider,
Default: true,
Config: v.CustomResource.Spec.BackupLocations[0].Velero.Config,
Credential: v.CustomResource.Spec.BackupLocations[0].Velero.Credential,
StorageType: velero.StorageType{
ObjectStorage: &velero.ObjectStorageLocation{
Bucket: v.CustomResource.Spec.BackupLocations[0].Velero.ObjectStorage.Bucket,
Prefix: VeleroPrefix,
},
},
},
Velero: v.VeleroBSL(),
},
},
},
Expand All @@ -103,6 +137,12 @@ func (v *DpaCustomResource) Build(backupRestoreType BackupRestoreType) error {
dpaInstance.Spec.Configuration.Velero.DefaultPlugins = append(dpaInstance.Spec.Configuration.Velero.DefaultPlugins, oadpv1alpha1.DefaultPluginCSI)
dpaInstance.Spec.Configuration.Velero.FeatureFlags = append(dpaInstance.Spec.Configuration.Velero.FeatureFlags, "EnableCSI")
}

for _, opt := range dpaCrOpts {
if err := opt(&dpaInstance); err != nil {
return err
}
}
v.CustomResource = &dpaInstance
return nil
}
Expand Down
31 changes: 31 additions & 0 deletions tests/e2e/lib/velero_helpers.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ import (
"github.com/vmware-tanzu/velero/pkg/label"
"github.com/vmware-tanzu/velero/pkg/restic"
appsv1 "k8s.io/api/apps/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"sigs.k8s.io/controller-runtime/pkg/client"
)
Expand Down Expand Up @@ -158,6 +159,36 @@ func RestoreErrorLogs(ocClient client.Client, restore velero.Restore) []string {
return logLines
}

func CreateBackupStorageLocation(ocClient client.Client, backupStorageLocation velero.BackupStorageLocation) error {
veleroClient, err := GetVeleroClient()
if err != nil {
return err
}
_, err = veleroClient.VeleroV1().BackupStorageLocations(backupStorageLocation.Namespace).Create(context.TODO(), &backupStorageLocation, metav1.CreateOptions{})
if err != nil {
if apierrors.IsAlreadyExists(err) {
return nil
}
return err
}
return nil
}

func DeleteBackupStorageLocation(ocClient client.Client, backupStorageLocation velero.BackupStorageLocation) error {
veleroClient, err := GetVeleroClient()
if err != nil {
return err
}
err = veleroClient.VeleroV1().BackupStorageLocations(backupStorageLocation.Namespace).Delete(context.TODO(), backupStorageLocation.Name, metav1.DeleteOptions{})
if err != nil {
if apierrors.IsNotFound(err) {
return nil
}
return err
}
return nil
}

func GetVeleroDeploymentList(namespace string) (*appsv1.DeploymentList, error) {
client, err := setUpClient()
if err != nil {
Expand Down

0 comments on commit 77de96e

Please sign in to comment.