fix(system-backup): handle snapshot of last backup removed

The snapshot of the last backup of the volume might be removed.
Therefore, volume backup will be not up-to-date and it needs to
create a new backup.

ref: longhorn/longhorn 10215

Signed-off-by: James Lu <james.lu@suse.com>

controller/system_backup_controller.go

@@ -915,11 +915,19 @@ func (c *SystemBackupController) isVolumeBackupUpToDate(volume *longhorn.Volume,
 	// Retrieve last backup and its snapshot.
 	lastBackup, err := c.ds.GetBackup(volume.Status.LastBackup)
 	if err != nil {
+		if apierrors.IsNotFound(err) {
+			log.Warnf("Last Backup %v not found for volume %v", volume.Status.LastBackup, volume.Name)
+			return false, nil
+		}
 		return false, err
 	}
 
 	lastBackupSnapshot, err := c.ds.GetSnapshot(lastBackup.Status.SnapshotName)
 	if err != nil {
+		if apierrors.IsNotFound(err) {
+			log.Warnf("Snapshot %v not found for backup %v", lastBackup.Status.SnapshotName, lastBackup.Name)
+			return false, nil
+		}
 		return false, err
 	}
 

controller/system_backup_controller_test.go

@@ -46,6 +46,7 @@ type SystemBackupTestCase struct {
 	existPersistentVolumes map[SystemRolloutCRName]*corev1.PersistentVolume
 	existVolumes           map[SystemRolloutCRName]*longhorn.Volume
 	existBackingImages     map[SystemRolloutCRName]*longhorn.BackingImage
+	existBackups           map[string]*longhorn.Backup
 
 	expectError                 bool
 	expectErrorConditionMessage string
@@ -102,6 +103,15 @@ func (s *TestSuite) TestReconcileSystemBackup(c *C) {
 					},
 				},
 			},
+			existBackups: map[string]*longhorn.Backup{
+				"exists": {
+					Status: longhorn.BackupStatus{
+						State:        longhorn.BackupStateCompleted,
+						SnapshotName: "exists",
+						VolumeName:   TestVolumeName,
+					},
+				},
+			},
 			expectState:               longhorn.SystemBackupStateBackingImageBackup,
 			expectNewVolumBackupCount: 0,
 		},
@@ -115,6 +125,15 @@ func (s *TestSuite) TestReconcileSystemBackup(c *C) {
 					},
 				},
 			},
+			existBackups: map[string]*longhorn.Backup{
+				"exists": {
+					Status: longhorn.BackupStatus{
+						State:        longhorn.BackupStateCompleted,
+						SnapshotName: "exists",
+						VolumeName:   TestVolumeName,
+					},
+				},
+			},
 			expectState:               longhorn.SystemBackupStateBackingImageBackup,
 			expectNewVolumBackupCount: 1,
 		},
@@ -235,6 +254,7 @@ func (s *TestSuite) TestReconcileSystemBackup(c *C) {
 		fakeSystemRolloutStorageClassesDefault(c, informerFactories.KubeInformerFactory, kubeClient)
 
 		fakeSystemRolloutVolumes(tc.existVolumes, c, informerFactories.LhInformerFactory, lhClient)
+		fakeSystemRolloutBackups(tc.existBackups, c, informerFactories.LhInformerFactory, lhClient)
 		fakeSystemRolloutBackingImages(tc.existBackingImages, c, informerFactories.LhInformerFactory, lhClient)
 		fakeSystemRolloutPersistentVolumes(tc.existPersistentVolumes, c, informerFactories.KubeInformerFactory, kubeClient)
 
@@ -255,6 +275,17 @@ func (s *TestSuite) TestReconcileSystemBackup(c *C) {
 
 		switch systemBackup.Status.State {
 		case longhorn.SystemBackupStateVolumeBackup:
+			if tc.existBackups != nil {
+				existBackupSnap := &longhorn.Snapshot{
+					ObjectMeta: metav1.ObjectMeta{Name: "exists"},
+					Spec:       longhorn.SnapshotSpec{Volume: TestVolumeName},
+					Status: longhorn.SnapshotStatus{
+						ReadyToUse:   true,
+						CreationTime: metav1.Now().Time.Format(time.RFC3339),
+					},
+				}
+				fakeSystemRolloutSnapshot(existBackupSnap, c, informerFactories.LhInformerFactory, lhClient)
+			}
 			backups, _ := systemBackupController.BackupVolumes(systemBackup)
 
 			for _, backup := range backups {