Skip to content

Commit

Permalink
Merge remote-tracking branch 'origin/master' into 2238-auto-eviction
Browse files Browse the repository at this point in the history 
Longhorn 2238

Signed-off-by: Eric Weber <eric.weber@suse.com>
  • Loading branch information
@ejweber
ejweber committed Nov 20, 2023
2 parents 4593f9a + 483ef28 commit aa126ad
Show file tree
Hide file tree
Showing 54 changed files with 1,011 additions and 631 deletions.
2 changes: 1 addition & 1 deletion api/forwarder.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ func NodeHasDefaultEngineImage(m *manager.VolumeManager) func(req *http.Request)
if err != nil {
return "", err
}
nodes, err := m.ListReadyNodesWithEngineImage(engineImage)
nodes, err := m.ListReadyNodesContainingEngineImageRO(engineImage)
if err != nil {
return "", err
}
Expand Down
6 changes: 3 additions & 3 deletions controller/backing_image_data_source_controller.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -630,7 +630,7 @@ func (c *BackingImageDataSourceController) generateBackingImageDataSourcePodMani
return nil, err
}

priorityClass, err := c.ds.GetSetting(types.SettingNamePriorityClass)
priorityClass, err := c.ds.GetSettingWithAutoFillingRO(types.SettingNamePriorityClass)
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -735,7 +735,7 @@ func (c *BackingImageDataSourceController) generateBackingImageDataSourcePodMani
},
}

registrySecretSetting, err := c.ds.GetSetting(types.SettingNameRegistrySecret)
registrySecretSetting, err := c.ds.GetSettingWithAutoFillingRO(types.SettingNameRegistrySecret)
if err != nil {
return nil, err
}
Expand All @@ -747,7 +747,7 @@ func (c *BackingImageDataSourceController) generateBackingImageDataSourcePodMani
}
}

storageNetwork, err := c.ds.GetSetting(types.SettingNameStorageNetwork)
storageNetwork, err := c.ds.GetSettingWithAutoFillingRO(types.SettingNameStorageNetwork)
if err != nil {
return nil, err
}
Expand Down
114 changes: 47 additions & 67 deletions controller/backing_image_manager_controller.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -378,14 +378,14 @@ func (c *BackingImageManagerController) updateForUnknownBackingImageManager(bim
if _, ok := bim.Status.BackingImageFileMap[biName]; ok {
continue
}
bi, err := c.ds.GetBackingImage(biName)
biRO, err := c.ds.GetBackingImageRO(biName)
if err != nil {
log.Warnf("Failed to get backing image %v before marking the empty file record in an unavailable disk as unknown", biName)
continue
}
info := longhorn.BackingImageFileInfo{
Name: bi.Name,
UUID: bi.Status.UUID,
Name: biRO.Name,
UUID: biRO.Status.UUID,
State: longhorn.BackingImageStateUnknown,
}
bim.Status.BackingImageFileMap[biName] = info
Expand Down Expand Up @@ -572,15 +572,15 @@ func (c *BackingImageManagerController) deleteInvalidBackingImages(bim *longhorn
for biName, biFileInfo := range bim.Status.BackingImageFileMap {
deleteRequired := false

bi, err := c.ds.GetBackingImage(biName)
biRO, err := c.ds.GetBackingImageRO(biName)
if err != nil {
if !apierrors.IsNotFound(err) {
return err
}
deleteRequired = true
log.Warnf("Failed to find backing image %v during invalid backing image cleanup, will skip it", biName)
}
if bi != nil && bi.Status.UUID == "" {
if biRO != nil && biRO.Status.UUID == "" {
continue
}

Expand All @@ -590,18 +590,18 @@ func (c *BackingImageManagerController) deleteInvalidBackingImages(bim *longhorn
// 2. The status record does not match the current backing image.
// 3. The file state recorded in the current backing image is failed
// and there are available files in other backing image managers.
deleteRequired = deleteRequired || (bi != nil && bim.Spec.BackingImages[biName] != bi.Status.UUID)
deleteRequired = deleteRequired || (bi != nil && biFileInfo.UUID != "" && biFileInfo.UUID != bi.Status.UUID)
if !deleteRequired && bi != nil {
deleteRequired = deleteRequired || (biRO != nil && bim.Spec.BackingImages[biName] != biRO.Status.UUID)
deleteRequired = deleteRequired || (biRO != nil && biFileInfo.UUID != "" && biFileInfo.UUID != biRO.Status.UUID)
if !deleteRequired && biRO != nil {
// Prefer to check the file state in BackingImage.Status,
// which is synced from BackingImageManager.Status with some
// adjustments.
fileState := biFileInfo.State
if bi.Status.DiskFileStatusMap[bim.Spec.DiskUUID] != nil {
fileState = bi.Status.DiskFileStatusMap[bim.Spec.DiskUUID].State
if biRO.Status.DiskFileStatusMap[bim.Spec.DiskUUID] != nil {
fileState = biRO.Status.DiskFileStatusMap[bim.Spec.DiskUUID].State
}
if fileState == longhorn.BackingImageStateFailed {
for _, biFileInfo := range bi.Status.DiskFileStatusMap {
for _, biFileInfo := range biRO.Status.DiskFileStatusMap {
if biFileInfo.State == longhorn.BackingImageStateFailed {
continue
}
Expand Down Expand Up @@ -631,14 +631,14 @@ func (c *BackingImageManagerController) prepareBackingImageFiles(currentBIM *lon
err = errors.Wrap(err, "failed to prepare backing image files")
}()

bims, err := c.ds.ListBackingImageManagers()
bimsRO, err := c.ds.ListBackingImageManagersRO()
if err != nil {
return err
}
for biName := range currentBIM.Spec.BackingImages {
log := bimLog.WithFields(logrus.Fields{"backingImage": biName})

bi, err := c.ds.GetBackingImage(biName)
biRO, err := c.ds.GetBackingImageRO(biName)
if err != nil {
if !apierrors.IsNotFound(err) {
log.WithError(err).Warn("Failed to get backing image before preparing files, will skip handling this backing image")
Expand Down Expand Up @@ -667,7 +667,7 @@ func (c *BackingImageManagerController) prepareBackingImageFiles(currentBIM *lon
// If bids is failed and not transferred, orphan tmp file might be left on the host.
// Clean up and set the state to failed-and-cleanup
if bids.Status.CurrentState == longhorn.BackingImageStateFailed {
if err := cli.Delete(bi.Name, bi.Status.UUID); err != nil {
if err := cli.Delete(biRO.Name, biRO.Status.UUID); err != nil {
return err
}
bids.Status.CurrentState = longhorn.BackingImageStateFailedAndCleanUp
Expand All @@ -688,29 +688,29 @@ func (c *BackingImageManagerController) prepareBackingImageFiles(currentBIM *lon
continue
}
log.Infof("Starting to fetch the data source file from the backing image data source work directory %v", bimtypes.DataSourceDirectoryName)
if _, err := cli.Fetch(bi.Name, bi.Status.UUID, bids.Status.Checksum, fmt.Sprintf("%s:%d", bids.Status.StorageIP, engineapi.BackingImageDataSourceDefaultPort), bids.Status.Size); err != nil {
if _, err := cli.Fetch(biRO.Name, biRO.Status.UUID, bids.Status.Checksum, fmt.Sprintf("%s:%d", bids.Status.StorageIP, engineapi.BackingImageDataSourceDefaultPort), bids.Status.Size); err != nil {
if types.ErrorAlreadyExists(err) {
continue
}
return err
}
// No backoff when fetching the 1st file.
c.eventRecorder.Eventf(currentBIM, corev1.EventTypeNormal, constant.EventReasonFetching, "Fetched the first file for backing image %v in disk %v on node %v", bi.Name, currentBIM.Spec.DiskUUID, currentBIM.Spec.NodeID)
c.eventRecorder.Eventf(currentBIM, corev1.EventTypeNormal, constant.EventReasonFetching, "Fetched the first file for backing image %v in disk %v on node %v", biRO.Name, currentBIM.Spec.DiskUUID, currentBIM.Spec.NodeID)
continue
}

if backoff.IsInBackOffSinceUpdate(bi.Name, time.Now()) {
log.Debugf("Failed to re-fetch or re-sync backing image file %v immediately since it is still in the backoff window", bi.Name)
if backoff.IsInBackOffSinceUpdate(biRO.Name, time.Now()) {
log.Debugf("Failed to re-fetch or re-sync backing image file %v immediately since it is still in the backoff window", biRO.Name)
continue
}

noReadyFile := true
var senderCandidate *longhorn.BackingImageManager
for _, bim := range bims {
if bim.Status.CurrentState != longhorn.BackingImageManagerStateRunning || bim.Spec.Image != c.bimImageName {
var senderCandidateRO *longhorn.BackingImageManager
for _, bimRO := range bimsRO {
if bimRO.Status.CurrentState != longhorn.BackingImageManagerStateRunning || bimRO.Spec.Image != c.bimImageName {
continue
}
info, exists := bim.Status.BackingImageFileMap[biName]
info, exists := bimRO.Status.BackingImageFileMap[biName]
if !exists {
continue
}
Expand All @@ -721,44 +721,44 @@ func (c *BackingImageManagerController) prepareBackingImageFiles(currentBIM *lon
if info.SendingReference >= bimtypes.SendingLimit {
continue
}
senderCandidate = bim
senderCandidateRO = bimRO
break
}

// Due to cases like upgrade, there is no ready record among all default backing image manager.
// Then Longhorn will ask managers to check then reuse existing files.
if noReadyFile {
size := bi.Status.Size
size := biRO.Status.Size
if size == 0 {
size = bids.Status.Size
}
// Empty source file name means trying to find and reuse the file in the work directory.
if _, err := cli.Fetch(bi.Name, bi.Status.UUID, bi.Status.Checksum, "", size); err != nil {
if _, err := cli.Fetch(biRO.Name, biRO.Status.UUID, biRO.Status.Checksum, "", size); err != nil {
if types.ErrorAlreadyExists(err) {
log.Warn("Backing image already exists, no need to check and reuse file")
continue
}
backoff.Next(bi.Name, time.Now())
backoff.Next(biRO.Name, time.Now())
return err
}
backoff.Next(bi.Name, time.Now())
backoff.Next(biRO.Name, time.Now())
log.Info("Reusing the existing file in the work directory")
c.eventRecorder.Eventf(currentBIM, corev1.EventTypeNormal, constant.EventReasonFetching, "Reuse the existing file for backing image %v in disk %v on node %v", bi.Name, currentBIM.Spec.DiskUUID, currentBIM.Spec.NodeID)
c.eventRecorder.Eventf(currentBIM, corev1.EventTypeNormal, constant.EventReasonFetching, "Reuse the existing file for backing image %v in disk %v on node %v", biRO.Name, currentBIM.Spec.DiskUUID, currentBIM.Spec.NodeID)
continue
}

if senderCandidate != nil {
log.WithFields(logrus.Fields{"fromHost": senderCandidate.Status.StorageIP, "size": bi.Status.Size}).Info("Requesting syncing backing image")
if _, err := cli.Sync(biName, bi.Status.UUID, bi.Status.Checksum, senderCandidate.Status.StorageIP, bi.Status.Size); err != nil {
if senderCandidateRO != nil {
log.WithFields(logrus.Fields{"fromHost": senderCandidateRO.Status.StorageIP, "size": biRO.Status.Size}).Info("Requesting syncing backing image")
if _, err := cli.Sync(biName, biRO.Status.UUID, biRO.Status.Checksum, senderCandidateRO.Status.StorageIP, biRO.Status.Size); err != nil {
if types.ErrorAlreadyExists(err) {
log.WithFields(logrus.Fields{"fromHost": senderCandidate.Status.StorageIP, "size": bi.Status.Size}).Warn("Backing image already exists, no need to sync from others")
log.WithFields(logrus.Fields{"fromHost": senderCandidateRO.Status.StorageIP, "size": biRO.Status.Size}).Warn("Backing image already exists, no need to sync from others")
continue
}
backoff.Next(bi.Name, time.Now())
backoff.Next(biRO.Name, time.Now())
return err
}
backoff.Next(bi.Name, time.Now())
c.eventRecorder.Eventf(currentBIM, corev1.EventTypeNormal, constant.EventReasonSyncing, "Syncing backing image %v in disk %v on node %v from %v(%v)", bi.Name, currentBIM.Spec.DiskUUID, currentBIM.Spec.NodeID, senderCandidate.Name, senderCandidate.Status.StorageIP)
backoff.Next(biRO.Name, time.Now())
c.eventRecorder.Eventf(currentBIM, corev1.EventTypeNormal, constant.EventReasonSyncing, "Syncing backing image %v in disk %v on node %v from %v(%v)", biRO.Name, currentBIM.Spec.DiskUUID, currentBIM.Spec.NodeID, senderCandidateRO.Name, senderCandidateRO.Status.StorageIP)
continue
}
}
Expand All @@ -779,7 +779,7 @@ func (c *BackingImageManagerController) createBackingImageManagerPod(bim *longho
if err != nil {
return err
}
registrySecretSetting, err := c.ds.GetSetting(types.SettingNameRegistrySecret)
registrySecretSetting, err := c.ds.GetSettingWithAutoFillingRO(types.SettingNameRegistrySecret)
if err != nil {
return err
}
Expand All @@ -802,7 +802,7 @@ func (c *BackingImageManagerController) generateBackingImageManagerPodManifest(b
return nil, err
}

priorityClass, err := c.ds.GetSetting(types.SettingNamePriorityClass)
priorityClass, err := c.ds.GetSettingWithAutoFillingRO(types.SettingNamePriorityClass)
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -892,7 +892,7 @@ func (c *BackingImageManagerController) generateBackingImageManagerPodManifest(b
}
}

storageNetwork, err := c.ds.GetSetting(types.SettingNameStorageNetwork)
storageNetwork, err := c.ds.GetSettingWithAutoFillingRO(types.SettingNameStorageNetwork)
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -953,16 +953,7 @@ func (c *BackingImageManagerController) enqueueForBackingImage(obj interface{})
}
}

backingImage, err := c.ds.GetBackingImage(backingImage.Name)
if err != nil {
if apierrors.IsNotFound(err) {
return
}
utilruntime.HandleError(fmt.Errorf("failed to get backing image %v: %v ", backingImage.Name, err))
return
}

bims, err := c.ds.ListBackingImageManagers()
bimsRO, err := c.ds.ListBackingImageManagersRO()
if err != nil {
if apierrors.IsNotFound(err) {
c.logger.WithField("backingImage", backingImage.Name).Warn("Failed to list backing image managers for a backing image, may be deleted")
Expand All @@ -972,9 +963,9 @@ func (c *BackingImageManagerController) enqueueForBackingImage(obj interface{})
return
}

for _, bim := range bims {
if _, exists := bim.Spec.BackingImages[backingImage.Name]; exists {
c.enqueueBackingImageManager(bim)
for _, bimRO := range bimsRO {
if _, exists := bimRO.Spec.BackingImages[backingImage.Name]; exists {
c.enqueueBackingImageManager(bimRO)
}
}
}
Expand All @@ -996,18 +987,7 @@ func (c *BackingImageManagerController) enqueueForLonghornNode(obj interface{})
}
}

node, err := c.ds.GetNode(node.Name)
if err != nil {
if apierrors.IsNotFound(err) {
// there is no Longhorn node created for the Kubernetes
// node (e.g. controller/etcd node). Skip it
return
}
utilruntime.HandleError(fmt.Errorf("failed to get node %v: %v ", node.Name, err))
return
}

bims, err := c.ds.ListBackingImageManagersByNode(node.Name)
bimsRO, err := c.ds.ListBackingImageManagersByNodeRO(node.Name)
if err != nil {
if apierrors.IsNotFound(err) {
c.logger.WithField("node", node.Name).Warn("Failed to list backing image managers for a node, may be deleted")
Expand All @@ -1017,8 +997,8 @@ func (c *BackingImageManagerController) enqueueForLonghornNode(obj interface{})
return
}

for _, bim := range bims {
c.enqueueBackingImageManager(bim)
for _, bimRO := range bimsRO {
c.enqueueBackingImageManager(bimRO)
}
}

Expand All @@ -1039,7 +1019,7 @@ func (c *BackingImageManagerController) enqueueForBackingImageManagerPod(obj int
}
}

bim, err := c.ds.GetBackingImageManager(pod.Name)
bimRO, err := c.ds.GetBackingImageManagerRO(pod.Name)
if err != nil {
if apierrors.IsNotFound(err) {
c.logger.WithField("pod", pod.Name).Warn("Failed to find backing image manager for pod, may be deleted")
Expand All @@ -1048,7 +1028,7 @@ func (c *BackingImageManagerController) enqueueForBackingImageManagerPod(obj int
utilruntime.HandleError(fmt.Errorf("couldn't get backing image manager: %v", err))
return
}
c.enqueueBackingImageManager(bim)
c.enqueueBackingImageManager(bimRO)
}

func (c *BackingImageManagerController) startMonitoring(bim *longhorn.BackingImageManager, backoff *flowcontrol.Backoff) {
Expand Down
8 changes: 4 additions & 4 deletions controller/backup_controller.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -197,7 +197,7 @@ func (bc *BackupController) isBackupNotBeingUsedForVolumeRestore(backupName, bac
if !v.Status.RestoreRequired {
continue
}
engines, err := bc.ds.ListVolumeEngines(v.Name)
engines, err := bc.ds.ListVolumeEnginesRO(v.Name)
if err != nil {
return false, errors.Wrapf(err, "failed to list engines for volume %v for checking restore status", v.Name)
}
Expand Down Expand Up @@ -479,7 +479,7 @@ func (bc *BackupController) handleAttachmentTicketCreation(backup *longhorn.Back
err = errors.Wrap(err, "handleAttachmentTicketCreation: failed to create/update attachment")
}()

vol, err := bc.ds.GetVolume(volumeName)
vol, err := bc.ds.GetVolumeRO(volumeName)
if err != nil {
return err
}
Expand Down Expand Up @@ -516,7 +516,7 @@ func (bc *BackupController) VerifyAttachment(backup *longhorn.Backup, volumeName
err = errors.Wrap(err, "VerifyAttachment: failed to verify attachment")
}()

vol, err := bc.ds.GetVolume(volumeName)
vol, err := bc.ds.GetVolumeRO(volumeName)
if err != nil {
return false, err
}
Expand Down Expand Up @@ -580,7 +580,7 @@ func (bc *BackupController) validateBackingImageChecksum(volName, biName string)
return "", nil
}

bi, err := bc.ds.GetBackingImage(biName)
bi, err := bc.ds.GetBackingImageRO(biName)
if err != nil {
return "", err
}
Expand Down
2 changes: 1 addition & 1 deletion controller/backup_target_controller.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -214,7 +214,7 @@ func getLoggerForBackupTarget(logger logrus.FieldLogger, backupTarget *longhorn.
}

func getBackupTarget(controllerID string, backupTarget *longhorn.BackupTarget, ds *datastore.DataStore, log logrus.FieldLogger, proxyConnCounter util.Counter) (engineClientProxy engineapi.EngineClientProxy, backupTargetClient *engineapi.BackupTargetClient, err error) {
instanceManager, err := ds.GetDefaultInstanceManagerByNode(controllerID)
instanceManager, err := ds.GetDefaultInstanceManagerByNodeRO(controllerID)
if err != nil {
return nil, nil, errors.Wrap(err, "failed to get default engine instance manager for proxy client")
}
Expand Down
4 changes: 2 additions & 2 deletions controller/controller_manager.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -142,15 +142,15 @@ func GetInstanceManagerCPURequirement(ds *datastore.DataStore, imName string) (*
if err != nil {
return nil, err
}
kubeNode, err := ds.GetKubernetesNode(im.Spec.NodeID)
kubeNode, err := ds.GetKubernetesNodeRO(im.Spec.NodeID)
if err != nil {
return nil, err
}

cpuRequest := lhNode.Spec.InstanceManagerCPURequest
guaranteedCPUSettingName := types.SettingNameGuaranteedInstanceManagerCPU
if cpuRequest == 0 {
guaranteedCPUSetting, err := ds.GetSetting(guaranteedCPUSettingName)
guaranteedCPUSetting, err := ds.GetSettingWithAutoFillingRO(guaranteedCPUSettingName)
if err != nil {
return nil, err
}
Expand Down
Loading

0 comments on commit aa126ad

Please sign in to comment.