Thanks to visit codestin.com
Credit goes to github.com

Skip to content

fix: retry middleware corrupts exponential backoff state #690

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Abzaek wants to merge 1 commit into
base: master
Choose a base branch
from
Open

fix: retry middleware corrupts exponential backoff state #690

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
36 changes: 34 additions & 2 deletions message/router/middleware/retry.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -125,10 +125,22 @@ func (r Retry) Middleware(h message.HandlerFunc) message.HandlerFunc {
return producedMessages, nil
}

// Compute the expected delay without calling NextBackOff().
// Calling expBackoff.NextBackOff() inside this function would
// advance the backoff's internal state, making backoff.Retry's
// own NextBackOff() call read a future interval.
// See https://github.com/ThreeDotsLabs/watermill/issues/XXX
approxDelay := computeBackoffDelay_atRetryNum(
retryNum,
expBackoff.InitialInterval,
expBackoff.MaxInterval,
expBackoff.Multiplier,
)

if r.ShouldRetry != nil && !r.ShouldRetry(RetryParams{
RetryNum: retryNum,
Err: err,
Delay: expBackoff.NextBackOff(),
Delay: approxDelay,
}) {
// backoff.Permanent will stop the retry attempts
stoppedByPermanent = true
Expand All @@ -137,7 +149,7 @@ func (r Retry) Middleware(h message.HandlerFunc) message.HandlerFunc {

if r.OnRetryHook != nil && retryNum > 0 {
// call RetryHook function on each retry attempt.
r.OnRetryHook(retryNum, expBackoff.NextBackOff())
r.OnRetryHook(retryNum, approxDelay)
}
retryNum++
return producedMessages, err
Expand Down Expand Up @@ -170,3 +182,23 @@ func (r Retry) Middleware(h message.HandlerFunc) message.HandlerFunc {
return producedMessages, nil
}
}

// computeBackoffDelay_atRetryNum computes the expected backoff delay for the
// given retry attempt number (1-based: retry 1 is the first retry, delay =
// InitialInterval × Multiplier^(retry-1)), capped to MaxInterval. This does
// NOT mutate any state, making it safe to use without corrupting the backoff
// state machine maintained by backoff.Retry.
func computeBackoffDelay_atRetryNum(retryNum int, initialInterval, maxInterval time.Duration, multiplier float64) time.Duration {
if retryNum <= 1 {
return initialInterval
}
delay := initialInterval
for i := 1; i < retryNum; i++ {
delay = time.Duration(float64(delay) * multiplier)
if maxInterval > 0 && delay > maxInterval {
delay = maxInterval
break
}
}
return delay
}