[filebeat][streaming] - Improved websocket exponential backoff logic to produce a smoother backoff curve (#44069)

ShourieG · web-flow · commit a684f006214f · 2025-05-26T16:10:56.000+05:30
The previous waitTime calculation for the exponential backoff strategy
produced an extremely sharp curve, where depending on the values of waitMin and
waitMax and number of attempts, the waitMax (cap) would be easily reached
after the initial couple of attempts thereby limiting the growth in the wait
time compared to the total number of attempts. Simply the waitTime growth
curve would hit the cap and flatten out after 1-2 retry attempts because of
uncapped jitter. This new change makes it such the waitTime growth curve
increases more smoothly with the number of attempts providing a smoother backoff
function.
diff --git a/CHANGELOG-developer.next.asciidoc b/CHANGELOG-developer.next.asciidoc
@@ -230,6 +230,7 @@ client will aggregate metrics by input on this registry. It'll collect
 - Add new API to libbeat/monitoring/inputmon. The API allows to register and
 unregister input metrics without relaying on the global 'dataset' namespace.{pull}42618[42618] {issue}42761[42761]
 - Added exponential backoff retry logic for vSphere client logout operation in metricbeat. {issue}43449[43449]
+- Improved Filebeat Websocket input exponential backoff logic to produce a smoother backoff curve. {pull}44069[44069]
 
 ==== Deprecated
 
diff --git a/x-pack/filebeat/input/streaming/websocket.go b/x-pack/filebeat/input/streaming/websocket.go
@@ -30,6 +30,13 @@ import (
 	"github.com/elastic/elastic-agent-libs/transport/tlscommon"
 )
 
+const (
+	// spread is the induced jitter for the backoff logic. This is not user configurable
+	// as it is not a common use case. The value of 1.0 is chosen as it produces a relatively
+	// smooth backoff curve.
+	spread = 1.0
+)
+
 type websocketStream struct {
 	processor
 
@@ -422,7 +429,7 @@ func connectWebSocket(ctx context.Context, cfg config, url string, log *logp.Log
 				} else {
 					log.Errorf("attempt %d: webSocket connection failed with error %v and no response, retrying...\n", attempt, err)
 				}
-				waitTime := calculateWaitTime(retryConfig.WaitMin, retryConfig.WaitMax, attempt)
+				waitTime := calculateWaitTime(retryConfig.WaitMin, retryConfig.WaitMax, attempt, retryConfig.MaxAttempts)
 				time.Sleep(waitTime)
 			}
 			if response == nil {
@@ -446,7 +453,7 @@ func connectWebSocket(ctx context.Context, cfg config, url string, log *logp.Log
 				} else {
 					log.Errorf("attempt %d: webSocket connection failed with error %v and no response, retrying...\n", attempt, err)
 				}
-				waitTime := calculateWaitTime(retryConfig.WaitMin, retryConfig.WaitMax, attempt)
+				waitTime := calculateWaitTime(retryConfig.WaitMin, retryConfig.WaitMax, attempt, retryConfig.MaxAttempts)
 				time.Sleep(waitTime)
 			}
 		}
@@ -456,16 +463,9 @@ func connectWebSocket(ctx context.Context, cfg config, url string, log *logp.Log
 }
 
 // calculateWaitTime calculates the wait time for the next attempt based on the exponential backoff algorithm.
-func calculateWaitTime(waitMin, waitMax time.Duration, attempt int) time.Duration {
+func calculateWaitTime(waitMin, waitMax time.Duration, attempt, maxAttempts int) time.Duration {
 	// calculate exponential backoff
-	base := float64(waitMin)
-	backoff := base * math.Pow(2, float64(attempt-1))
-
-	// calculate jitter proportional to the backoff
-	maxJitter := float64(waitMax-waitMin) * math.Pow(2, float64(attempt-1))
-	jitter := rand.Float64() * maxJitter
-
-	waitTime := time.Duration(backoff + jitter)
+	waitTime := wait(waitMin, waitMax, attempt, maxAttempts, spread)
 	// caps the wait time to the maximum wait time
 	if waitTime > waitMax {
 		waitTime = waitMax
@@ -474,6 +474,28 @@ func calculateWaitTime(waitMin, waitMax time.Duration, attempt int) time.Duratio
 	return waitTime
 }
 
+// wait returns a logistic backoff duration with jitter. The duration increases
+// from min to max in n steps, with i indicating the step. Jitter is added around
+// the logistic based on the value of spread. Zero spread results in no jitter,
+// and unit spread is maximal. Spread values above one may result in durations
+// outside [min, max]. min must not be greater than max.
+func wait(min, max time.Duration, i, n int, spread float64) time.Duration {
+	l := logistic(i, n-1) // n-1 because of fence posts.
+	return min + time.Duration(float64(max-min)*(l+spread*jitter(l)))
+}
+
+// logistic returns the ith value of n of the logistic function shifted
+// n/2 right. The returned value is in (0, 1) for all sensible values.
+func logistic(i, n int) float64 {
+	return 1 / (1 + math.Exp(float64(n)/2-float64(i)))
+}
+
+// jitter returns a jittered value around f, f±eps, where eps is f(1-f).
+// f must be in [0, 1].
+func jitter(f float64) float64 {
+	return (rand.Float64() - 0.5) * f * (1 - f)
+}
+
 // now is time.Now with a modifiable time source.
 func (s *websocketStream) now() time.Time {
 	if s.time == nil {
