symfony · tunterreitmeier · Jul 8, 2024 · Jul 7, 2024 · Jul 7, 2024 · tunterreitmeier
diff --git a/src/Symfony/Bundle/FrameworkBundle/Resources/config/messenger.php b/src/Symfony/Bundle/FrameworkBundle/Resources/config/messenger.php
@@ -186,6 +186,7 @@
             ->args([
                 abstract_arg('failure transports'),
                 service('logger')->ignoreOnInvalid(),
+                service('messenger.retry_strategy_locator'),
             ])
             ->tag('kernel.event_subscriber')
             ->tag('monolog.logger', ['channel' => 'messenger'])

diff --git a/src/Symfony/Component/Messenger/Bridge/Amqp/Tests/Transport/AmqpSenderTest.php b/src/Symfony/Component/Messenger/Bridge/Amqp/Tests/Transport/AmqpSenderTest.php
@@ -13,11 +13,14 @@
 
 use PHPUnit\Framework\TestCase;
 use Symfony\Component\Messenger\Bridge\Amqp\Tests\Fixtures\DummyMessage;
+use Symfony\Component\Messenger\Bridge\Amqp\Transport\AmqpReceivedStamp;
 use Symfony\Component\Messenger\Bridge\Amqp\Transport\AmqpSender;
 use Symfony\Component\Messenger\Bridge\Amqp\Transport\AmqpStamp;
 use Symfony\Component\Messenger\Bridge\Amqp\Transport\Connection;
 use Symfony\Component\Messenger\Envelope;
 use Symfony\Component\Messenger\Exception\TransportException;
+use Symfony\Component\Messenger\Stamp\RedeliveryStamp;
+use Symfony\Component\Messenger\Stamp\SentToFailureTransportStamp;
 use Symfony\Component\Messenger\Transport\Serialization\SerializerInterface;
 
 /**
@@ -55,6 +58,29 @@ public function testItSendsTheEncodedMessageUsingARoutingKey()
         $sender->send($envelope);
     }
 
+    public function testItDoesNotUseRetryRoutingKeyWhenSendingToFailureTransport()
+    {
+        $envelope = (new Envelope(new DummyMessage('Oy')))->with()
+            ->with(new AmqpReceivedStamp(
+                $this->createStub(\AMQPEnvelope::class),
+                'original_receiver'
+            ))
+            ->with(new RedeliveryStamp(1))
+            ->with(new SentToFailureTransportStamp('original_receiver'));
+        $encoded = ['body' => '...', 'headers' => ['type' => DummyMessage::class]];
+
+        $serializer = $this->createStub(SerializerInterface::class);
+        $serializer->method('encode')->with($envelope)->willReturn($encoded);
+
+        $connection = $this->createMock(Connection::class);
+        $connection->expects($this->once())->method('publish')->with($encoded['body'], $encoded['headers'], 0, $this->callback(
+            static fn (AmqpStamp $stamp) => '' === $stamp->getRoutingKey()
+        ));
+
+        $sender = new AmqpSender($connection, $serializer);
+        $sender->send($envelope);
+    }
+
     public function testItSendsTheEncodedMessageWithoutHeaders()
     {
         $envelope = new Envelope(new DummyMessage('Oy'));

diff --git a/...mfony/Component/Messenger/Bridge/Amqp/Tests/Transport/FailureTransportIntegrationTest.php b/...mfony/Component/Messenger/Bridge/Amqp/Tests/Transport/FailureTransportIntegrationTest.php
@@ -0,0 +1,163 @@
+<?php
+
+/*
+ * This file is part of the Symfony package.
+ *
+ * (c) Fabien Potencier <[email protected]>
+ *
+ * For the full copyright and license information, please view the LICENSE
+ * file that was distributed with this source code.
+ */
+
+namespace Symfony\Component\Messenger\Bridge\Amqp\Tests\Transport;
+
+use PHPUnit\Framework\TestCase;
+use Psr\Container\ContainerInterface;
+use Symfony\Component\DependencyInjection\ServiceLocator;
+use Symfony\Component\EventDispatcher\EventDispatcher;
+use Symfony\Component\Messenger\Bridge\Amqp\Tests\Fixtures\DummyMessage;
+use Symfony\Component\Messenger\Bridge\Amqp\Transport\AmqpTransport;
+use Symfony\Component\Messenger\Bridge\Amqp\Transport\Connection;
+use Symfony\Component\Messenger\Envelope;
+use Symfony\Component\Messenger\EventListener\SendFailedMessageForRetryListener;
+use Symfony\Component\Messenger\EventListener\SendFailedMessageToFailureTransportListener;
+use Symfony\Component\Messenger\EventListener\StopWorkerOnMessageLimitListener;
+use Symfony\Component\Messenger\EventListener\StopWorkerOnTimeLimitListener;
+use Symfony\Component\Messenger\Handler\HandlerDescriptor;
+use Symfony\Component\Messenger\Handler\HandlersLocator;
+use Symfony\Component\Messenger\MessageBus;
+use Symfony\Component\Messenger\Middleware\FailedMessageProcessingMiddleware;
+use Symfony\Component\Messenger\Middleware\HandleMessageMiddleware;
+use Symfony\Component\Messenger\Middleware\SendMessageMiddleware;
+use Symfony\Component\Messenger\Retry\MultiplierRetryStrategy;
+use Symfony\Component\Messenger\Stamp\DelayStamp;
+use Symfony\Component\Messenger\Stamp\RedeliveryStamp;
+use Symfony\Component\Messenger\Stamp\SentToFailureTransportStamp;
+use Symfony\Component\Messenger\Transport\Sender\SendersLocator;
+use Symfony\Component\Messenger\Worker;
+
+/**
+ * @requires extension amqp
+ *
+ * @group integration
+ */
+class FailureTransportIntegrationTest extends TestCase
+{
+    protected function setUp(): void
+    {
+        parent::setUp();
+
+        if (!getenv('MESSENGER_AMQP_DSN')) {
+            $this->markTestSkipped('The "MESSENGER_AMQP_DSN" environment variable is required.');
+        }
+    }
+
+    public function testItDoesNotLoseMessagesFromTheFailedTransport()
+    {
+        $connection = Connection::fromDsn(getenv('MESSENGER_AMQP_DSN'));
+        $connection->setup();
+        $connection->purgeQueues();
+
+        $failureConnection = Connection::fromDsn(getenv('MESSENGER_AMQP_DSN'),
+            ['exchange' => [
+                'name' => 'failed',
+                'type' => 'fanout',
+            ], 'queues' => ['failed' => []]]
+        );
+        $failureConnection->setup();
+        $failureConnection->purgeQueues();
+
+        $originalTransport = new AmqpTransport($connection);
+        $failureTransport = new AmqpTransport($failureConnection);
+
+        $retryStrategy = new MultiplierRetryStrategy(1, 100, 2);
+        $retryStrategyLocator = $this->createStub(ContainerInterface::class);
+        $retryStrategyLocator->method('has')->willReturn(true);
+        $retryStrategyLocator->method('get')->willReturn($retryStrategy);
+
+        $sendersLocatorFailureTransport = new ServiceLocator([
+            'original' => static fn () => $failureTransport,
+        ]);
+
+        $transports = [
+            'original' => $originalTransport,
+            'failed' => $failureTransport,
+        ];
+
+        $locator = $this->createStub(ContainerInterface::class);
+        $locator->method('has')->willReturn(true);
+        $locator->method('get')->willReturnCallback(static fn ($transportName) => $transports[$transportName]);
+        $senderLocator = new SendersLocator(
+            [DummyMessage::class => ['original']],
+            $locator
+        );
+
+        $timesHandled = 0;
+
+        $handler = static function () use (&$timesHandled) {
+            ++$timesHandled;
+            throw new \Exception('Handler failed');
+        };
+
+        $handlerLocator = new HandlersLocator([
+            DummyMessage::class => [new HandlerDescriptor($handler, ['from_transport' => 'original'])],
+        ]);
+
+        $bus = new MessageBus([
+            new FailedMessageProcessingMiddleware(),
+            new SendMessageMiddleware($senderLocator),
+            new HandleMessageMiddleware($handlerLocator),
+        ]);
+
+        $dispatcher = new EventDispatcher();
+        $dispatcher->addSubscriber(new SendFailedMessageForRetryListener($locator, $retryStrategyLocator));
+        $dispatcher->addSubscriber(new SendFailedMessageToFailureTransportListener(
+            $sendersLocatorFailureTransport, null, $retryStrategyLocator
+        ));
+        $dispatcher->addSubscriber(new StopWorkerOnMessageLimitListener(1));
+        $dispatcher->addSubscriber(new StopWorkerOnTimeLimitListener(2));
+
+        $originalTransport->send(Envelope::wrap(new DummyMessage('dummy')));
+
+        $runWorker = static function (string $transportName) use ($bus, $dispatcher, $transports): void {
+            (new Worker(
+                [$transportName => $transports[$transportName]],
+                $bus,
+                $dispatcher,
+            ))->run();
+        };
+
+        $runWorker('original');
+        $runWorker('original');
+        $runWorker('failed');
+        $runWorker('failed');
+
+        $this->assertSame(4, $timesHandled);
+        $failedMessage = $this->waitForFailedMessage($failureTransport, 2);
+        // 100 delay * 2 multiplier ^ 3 retries = 800 expected delay
+        $this->assertSame(800, $failedMessage->last(DelayStamp::class)->getDelay());
+        $this->assertSame(0, $failedMessage->last(RedeliveryStamp::class)->getRetryCount());
+        $this->assertCount(4, $failedMessage->all(RedeliveryStamp::class));
+        $this->assertCount(2, $failedMessage->all(SentToFailureTransportStamp::class));
+        foreach ($failedMessage->all(SentToFailureTransportStamp::class) as $stamp) {
+            $this->assertSame('original', $stamp->getOriginalReceiverName());
+        }
+    }
+
+    private function waitForFailedMessage(AmqpTransport $failureTransport, int $timeOutInS): Envelope
+    {
+        $start = microtime(true);
+        while (microtime(true) - $start < $timeOutInS) {
+            $envelopes = iterator_to_array($failureTransport->get());
+            if (\count($envelopes) > 0) {
+                foreach ($envelopes as $envelope) {
+                    $failureTransport->reject($envelope);
+                }
+
+                return $envelopes[0];
+            }
+            usleep(100 * 1000);
+        }
+        throw new \RuntimeException('Message was not received from failure transport within expected timeframe.');
+    }
+}
diff --git a/src/Symfony/Component/Messenger/Bridge/Amqp/Transport/AmqpSender.php b/src/Symfony/Component/Messenger/Bridge/Amqp/Transport/AmqpSender.php
@@ -15,6 +15,7 @@
 use Symfony\Component\Messenger\Exception\TransportException;
 use Symfony\Component\Messenger\Stamp\DelayStamp;
 use Symfony\Component\Messenger\Stamp\RedeliveryStamp;
+use Symfony\Component\Messenger\Stamp\SentToFailureTransportStamp;
 use Symfony\Component\Messenger\Transport\Sender\SenderInterface;
 use Symfony\Component\Messenger\Transport\Serialization\PhpSerializer;
 use Symfony\Component\Messenger\Transport\Serialization\SerializerInterface;
@@ -59,7 +60,7 @@ public function send(Envelope $envelope): Envelope
             $amqpStamp = AmqpStamp::createFromAmqpEnvelope(
                 $amqpReceivedStamp->getAmqpEnvelope(),
                 $amqpStamp,
-                $envelope->last(RedeliveryStamp::class) ? $amqpReceivedStamp->getQueueName() : null
+                $this->getRetryRoutingKey($envelope, $amqpReceivedStamp)
             );
         }
 
@@ -76,4 +77,13 @@ public function send(Envelope $envelope): Envelope
 
         return $envelope;
     }
+
+    private function getRetryRoutingKey(Envelope $envelope, AmqpReceivedStamp $amqpReceivedStamp): ?string
+    {
+        if (1 === \count($envelope->all(SentToFailureTransportStamp::class))) {
+            return null;
+        }
+
+        return $envelope->last(RedeliveryStamp::class) ? $amqpReceivedStamp->getQueueName() : null;
+    }
 }
diff --git a/...Symfony/Component/Messenger/EventListener/SendFailedMessageToFailureTransportListener.php b/...Symfony/Component/Messenger/EventListener/SendFailedMessageToFailureTransportListener.php
@@ -15,7 +15,9 @@
 use Psr\Log\LoggerInterface;
 use Symfony\Component\EventDispatcher\EventSubscriberInterface;
 use Symfony\Component\Messenger\Event\WorkerMessageFailedEvent;
+use Symfony\Component\Messenger\Retry\RetryStrategyInterface;
 use Symfony\Component\Messenger\Stamp\DelayStamp;
+use Symfony\Component\Messenger\Stamp\ReceivedStamp;
 use Symfony\Component\Messenger\Stamp\RedeliveryStamp;
 use Symfony\Component\Messenger\Stamp\SentToFailureTransportStamp;
 
@@ -28,11 +30,13 @@ class SendFailedMessageToFailureTransportListener implements EventSubscriberInte
 {
     private ContainerInterface $failureSenders;
     private ?LoggerInterface $logger;
+    private ?ContainerInterface $retryStrategyLocator;
 
-    public function __construct(ContainerInterface $failureSenders, ?LoggerInterface $logger = null)
+    public function __construct(ContainerInterface $failureSenders, ?LoggerInterface $logger = null, ?ContainerInterface $retryStrategyLocator = null)
     {
         $this->failureSenders = $failureSenders;
         $this->logger = $logger;
+        $this->retryStrategyLocator = $retryStrategyLocator;
     }
 
     /**
@@ -44,28 +48,30 @@ public function onMessageFailed(WorkerMessageFailedEvent $event)
             return;
         }
 
-        if (!$this->failureSenders->has($event->getReceiverName())) {
+        $originalTransportName = $event->getEnvelope()->last(ReceivedStamp::class)
+            ?->getTransportName() ?? $event->getReceiverName();
+
+        if (!$this->failureSenders->has($originalTransportName)) {
             return;
         }
 
-        $failureSender = $this->failureSenders->get($event->getReceiverName());
+        $failureSender = $this->failureSenders->get($originalTransportName);
 
         $envelope = $event->getEnvelope();
 
-        // avoid re-sending to the failed sender
-        if (null !== $envelope->last(SentToFailureTransportStamp::class)) {
-            return;
-        }
+        $delay = $this->getRetryStrategyForTransport($event->getReceiverName())
+            ?->getWaitingTime($envelope, $event->getThrowable()) ?? 0;
 
         $envelope = $envelope->with(
-            new SentToFailureTransportStamp($event->getReceiverName()),
-            new DelayStamp(0),
+            new SentToFailureTransportStamp($originalTransportName),
+            new DelayStamp($delay),
             new RedeliveryStamp(0)
         );
 
-        $this->logger?->info('Rejected message {class} will be sent to the failure transport {transport}.', [
+        $this->logger?->info('Rejected message {class} will be sent to the failure transport {transport} using {delay} ms delay.', [
             'class' => $envelope->getMessage()::class,
             'transport' => $failureSender::class,
+            'delay' => $delay,
         ]);
 
         $failureSender->send($envelope);
@@ -77,4 +83,13 @@ public static function getSubscribedEvents(): array
             WorkerMessageFailedEvent::class => ['onMessageFailed', -100],
         ];
     }
+
+    private function getRetryStrategyForTransport(string $transportName): ?RetryStrategyInterface
+    {
+        if (null === $this->retryStrategyLocator || !$this->retryStrategyLocator->has($transportName)) {
+            return null;
+        }
+
+        return $this->retryStrategyLocator->get($transportName);
+    }
 }
diff --git a/src/Symfony/Component/Messenger/Retry/MultiplierRetryStrategy.php b/src/Symfony/Component/Messenger/Retry/MultiplierRetryStrategy.php
@@ -78,7 +78,7 @@ public function isRetryable(Envelope $message, ?\Throwable $throwable = null): b
      */
     public function getWaitingTime(Envelope $message, ?\Throwable $throwable = null): int
     {
-        $retries = RedeliveryStamp::getRetryCountFromEnvelope($message);
+        $retries = \count($message->all(RedeliveryStamp::class));
 
         $delay = $this->delayMilliseconds * $this->multiplier ** $retries;