Skip to content

Commit

Permalink
Updater: fix stuck updater process
Browse files Browse the repository at this point in the history
There are several goroutines in updateWhileRenewingLock() which were
blocking updater process because they were waiting for event which never
happened when update() function was successful.

This commit adds context which unblock other goroutines when update
pass.
  • Loading branch information
Allda committed May 6, 2019
1 parent 5fef44d commit a14b372
Showing 1 changed file with 13 additions and 2 deletions.
15 changes: 13 additions & 2 deletions updater.go
Original file line number Diff line number Diff line change
Expand Up @@ -132,14 +132,16 @@ func RunUpdater(config *UpdaterConfig, datastore database.Datastore, st *stopper
}

if acquiredLock {
sleepDuration, err = updateWhileRenewingLock(datastore, whoAmI, isFirstUpdate, st)
err = updateWhileRenewingLock(datastore, whoAmI, isFirstUpdate, st)
if err != nil {
if err == errReceivedStopSignal {
log.Debug("updater received stop signal")
return
}
log.WithError(err).Debug("failed to acquired lock")
sleepDuration = timeutil.ExpBackoff(sleepDuration, config.Interval)
} else {
sleepDuration = config.Interval
}
} else {
sleepDuration = updaterSleepBetweenLoopsDuration
Expand All @@ -156,9 +158,13 @@ func RunUpdater(config *UpdaterConfig, datastore database.Datastore, st *stopper

var errReceivedStopSignal = errors.New("stopped")

func updateWhileRenewingLock(datastore database.Datastore, whoAmI string, isFirstUpdate bool, st *stopper.Stopper) (sleepDuration time.Duration, err error) {
func updateWhileRenewingLock(datastore database.Datastore, whoAmI string, isFirstUpdate bool, st *stopper.Stopper) (err error) {
g, ctx := errgroup.WithContext(context.Background())
// done context is used when updater finishes and all other
// go rutines in group should finish too
doneCtx, done := context.WithCancel(context.Background())
g.Go(func() error {
defer done()
return update(ctx, datastore, isFirstUpdate)
})

Expand All @@ -175,6 +181,9 @@ func updateWhileRenewingLock(datastore database.Datastore, whoAmI string, isFirs
case <-ctx.Done():
database.ReleaseLock(datastore, updaterLockName, whoAmI)
return ctx.Err()
case <-doneCtx.Done():
database.ReleaseLock(datastore, updaterLockName, whoAmI)
return nil
}
}
})
Expand All @@ -185,6 +194,8 @@ func updateWhileRenewingLock(datastore database.Datastore, whoAmI string, isFirs
return errReceivedStopSignal
case <-ctx.Done():
return ctx.Err()
case <-doneCtx.Done():
return nil
}
})

Expand Down

0 comments on commit a14b372

Please sign in to comment.