Skip to content

Commit

Permalink
wip
Browse files Browse the repository at this point in the history
  • Loading branch information
Jorres committed Jan 21, 2025
1 parent b455160 commit e254df6
Show file tree
Hide file tree
Showing 3 changed files with 20 additions and 5 deletions.
2 changes: 1 addition & 1 deletion pkg/rolling/options.go
Original file line number Diff line number Diff line change
Expand Up @@ -98,7 +98,7 @@ ydbops will try to figure out if you broke this rule by comparing before\after o
`The limit on the number of simultaneous node restarts`)

fs.DurationVar(&o.DelayBetweenRestarts, "delay-between-restarts", DefaultDelayBetweenRestarts,
`Delay between two consecutive restarts in seconds. The number of simultaneous is set by 'max-concurrent-restarts'`)
`Delay between two consecutive restarts in seconds. The number of simultaneous is set by 'nodes-inflight'`)
}

func (o *RestartOptions) GetRestartDuration() *durationpb.Duration {
Expand Down
10 changes: 9 additions & 1 deletion pkg/rolling/restart_handler.go
Original file line number Diff line number Diff line change
Expand Up @@ -77,8 +77,10 @@ func (rh *restartHandler) run() {

select {
case <-rh.done:
rh.logger.Debug("received from rh.done")
return
case <-time.After(rh.delayBetweenRestarts):
rh.logger.Debugf("successfully waited for %s", rh.delayBetweenRestarts.String())
continue
}
}
Expand All @@ -87,9 +89,15 @@ func (rh *restartHandler) run() {
}
}

func (rh *restartHandler) stop() {
func (rh *restartHandler) stopWithDelay() {
close(rh.queue)
rh.wg.Wait()
close(rh.done)
}

func (rh *restartHandler) stopRightNow() {
close(rh.queue)
close(rh.done)
}

func newRestartHandler(
Expand Down
13 changes: 10 additions & 3 deletions pkg/rolling/rolling.go
Original file line number Diff line number Diff line change
Expand Up @@ -362,7 +362,7 @@ func (r *Rolling) processActionGroupStates(actions []*Ydb_Maintenance.ActionGrou
}

<-done
restartHandler.stop()

result, err := r.cms.CompleteAction(r.completedActions)
if err != nil {
r.logger.Warnf("Failed to complete action: %+v", err)
Expand All @@ -371,8 +371,15 @@ func (r *Rolling) processActionGroupStates(actions []*Ydb_Maintenance.ActionGrou
r.logCompleteResult(result)
r.state.unreportedButFinishedActionIds = []string{}

// completed when all actions marked as completed
return len(actions) == len(result.ActionStatuses)
isRestartCompleted := len(actions) == len(result.ActionStatuses)

if !isRestartCompleted {
restartHandler.stopWithDelay()
} else {
restartHandler.stopRightNow()
}

return isRestartCompleted
}

func (r *Rolling) atomicHasActionInUnreported(actionID string) bool {
Expand Down

0 comments on commit e254df6

Please sign in to comment.