[Messenger] Add jitter parameter to MultiplierRetryStrategy

rmikalkenas · rmikalkenas · commit 96e3c84b6926 · 2023-12-31T07:14:05.000+02:00
diff --git a/src/Symfony/Bundle/FrameworkBundle/DependencyInjection/Configuration.php b/src/Symfony/Bundle/FrameworkBundle/DependencyInjection/Configuration.php
@@ -1586,6 +1586,7 @@ function ($a) {
                                             ->integerNode('delay')->defaultValue(1000)->min(0)->info('Time in ms to delay (or the initial value when multiplier is used)')->end()
                                             ->floatNode('multiplier')->defaultValue(2)->min(1)->info('If greater than 1, delay will grow exponentially for each retry: this delay = (delay * (multiple ^ retries))')->end()
                                             ->integerNode('max_delay')->defaultValue(0)->min(0)->info('Max time in ms that a retry should ever be delayed (0 = infinite)')->end()
+                                            ->floatNode('jitter')->defaultValue(0.1)->min(0)->max(1)->info('Randomness in percent (between 0 and 1) to apply to the delay')->end()
                                         ->end()
                                     ->end()
                                     ->scalarNode('rate_limiter')
diff --git a/src/Symfony/Bundle/FrameworkBundle/DependencyInjection/FrameworkExtension.php b/src/Symfony/Bundle/FrameworkBundle/DependencyInjection/FrameworkExtension.php
@@ -2195,7 +2195,8 @@ private function registerMessengerConfiguration(array $config, ContainerBuilder
                     ->replaceArgument(0, $transport['retry_strategy']['max_retries'])
                     ->replaceArgument(1, $transport['retry_strategy']['delay'])
                     ->replaceArgument(2, $transport['retry_strategy']['multiplier'])
-                    ->replaceArgument(3, $transport['retry_strategy']['max_delay']);
+                    ->replaceArgument(3, $transport['retry_strategy']['max_delay'])
+                    ->replaceArgument(4, $transport['retry_strategy']['jitter']);
                 $container->setDefinition($retryServiceId, $retryDefinition);
 
                 $transportRetryReferences[$name] = new Reference($retryServiceId);
diff --git a/src/Symfony/Bundle/FrameworkBundle/Resources/config/messenger.php b/src/Symfony/Bundle/FrameworkBundle/Resources/config/messenger.php
@@ -160,6 +160,7 @@
                 abstract_arg('delay ms'),
                 abstract_arg('multiplier'),
                 abstract_arg('max delay ms'),
+                abstract_arg('jitter'),
             ])
 
         // rate limiter
diff --git a/src/Symfony/Component/Messenger/CHANGELOG.md b/src/Symfony/Component/Messenger/CHANGELOG.md
@@ -7,6 +7,7 @@ CHANGELOG
  * Add option `redis_sentinel` as an alias for `sentinel_master`
  * Add `--all` option to the `messenger:consume` command
  * Make `#[AsMessageHandler]` final
+ * Add parameter `$jitter` to `MultiplierRetryStrategy` in order to randomize delay and prevent the thundering herd effect.
 
 7.0
 ---
diff --git a/src/Symfony/Component/Messenger/Retry/MultiplierRetryStrategy.php b/src/Symfony/Component/Messenger/Retry/MultiplierRetryStrategy.php
@@ -36,14 +36,16 @@ class MultiplierRetryStrategy implements RetryStrategyInterface
     private int $delayMilliseconds;
     private float $multiplier;
     private int $maxDelayMilliseconds;
+    private float $jitter;
 
     /**
      * @param int   $maxRetries           The maximum number of times to retry
      * @param int   $delayMilliseconds    Amount of time to delay (or the initial value when multiplier is used)
      * @param float $multiplier           Multiplier to apply to the delay each time a retry occurs
      * @param int   $maxDelayMilliseconds Maximum delay to allow (0 means no maximum)
+     * @param float $jitter               Probability of randomness int delay (0 = none, 1 = 100% random)
      */
-    public function __construct(int $maxRetries = 3, int $delayMilliseconds = 1000, float $multiplier = 1, int $maxDelayMilliseconds = 0)
+    public function __construct(int $maxRetries = 3, int $delayMilliseconds = 1000, float $multiplier = 1, int $maxDelayMilliseconds = 0, float $jitter = 0.1)
     {
         $this->maxRetries = $maxRetries;
 
@@ -61,6 +63,11 @@ public function __construct(int $maxRetries = 3, int $delayMilliseconds = 1000,
             throw new InvalidArgumentException(sprintf('Max delay must be greater than or equal to zero: "%s" given.', $maxDelayMilliseconds));
         }
         $this->maxDelayMilliseconds = $maxDelayMilliseconds;
+
+        if ($jitter < 0 || $jitter > 1) {
+            throw new InvalidArgumentException(sprintf('Jitter must be between 0 and 1: "%s" given.', $jitter));
+        }
+        $this->jitter = $jitter;
     }
 
     /**
@@ -82,6 +89,11 @@ public function getWaitingTime(Envelope $message, \Throwable $throwable = null):
 
         $delay = $this->delayMilliseconds * $this->multiplier ** $retries;
 
+        if ($this->jitter > 0) {
+            $randomness = (int) ($delay * $this->jitter);
+            $delay += random_int(-$randomness, +$randomness);
+        }
+
         if ($delay > $this->maxDelayMilliseconds && 0 !== $this->maxDelayMilliseconds) {
             return $this->maxDelayMilliseconds;
         }
diff --git a/src/Symfony/Component/Messenger/Tests/Retry/MultiplierRetryStrategyTest.php b/src/Symfony/Component/Messenger/Tests/Retry/MultiplierRetryStrategyTest.php
@@ -54,7 +54,7 @@ public function testIsRetryableWithNoStamp()
      */
     public function testGetWaitTime(int $delay, float $multiplier, int $maxDelay, int $previousRetries, int $expectedDelay)
     {
-        $strategy = new MultiplierRetryStrategy(10, $delay, $multiplier, $maxDelay);
+        $strategy = new MultiplierRetryStrategy(10, $delay, $multiplier, $maxDelay, 0);
         $envelope = new Envelope(new \stdClass(), [new RedeliveryStamp($previousRetries)]);
 
         $this->assertSame($expectedDelay, $strategy->getWaitingTime($envelope));
@@ -89,4 +89,39 @@ public static function getWaitTimeTests(): iterable
         yield [1000, 1.5555, 5000, 1, 1556];
         yield [1000, 1.5555, 5000, 2, 2420];
     }
+
+    /**
+     * @dataProvider getJitterTest
+     */
+    public function testJitter(float $jitter, int $maxMin, int $maxMax)
+    {
+        $strategy = new MultiplierRetryStrategy(3, 1000, 1, 0, $jitter);
+        $envelope = new Envelope(new \stdClass());
+
+        $min = 1000;
+        $max = 1000;
+        for ($i = 0; $i < 100; ++$i) {
+            $delay = $strategy->getWaitingTime($envelope);
+            $min = min($min, $delay);
+            $max = max($max, $delay);
+        }
+
+        $this->assertGreaterThanOrEqual($maxMin, $min);
+        $this->assertLessThanOrEqual($maxMax, $max);
+    }
+
+    public static function getJitterTest(): iterable
+    {
+        yield [1.0, 0, 2000];
+        yield [0.9, 100, 1900];
+        yield [0.8, 200, 1800];
+        yield [0.7, 300, 1700];
+        yield [0.6, 400, 1600];
+        yield [0.5, 500, 1500];
+        yield [0.4, 600, 1400];
+        yield [0.3, 700, 1300];
+        yield [0.2, 800, 1200];
+        yield [0.1, 900, 1100];
+        yield [0.0, 1000, 1000];
+    }
 }
