From c79d4cbd1c97dad238eb12fa23cc45f12df4c11a Mon Sep 17 00:00:00 2001 From: Thomas Unterreitmeier Date: Mon, 8 Jul 2024 19:22:12 +0200 Subject: [PATCH 1/3] [Messenger] Resend failed messages to failure transport Use retry strategy with an increasing delay to prevent handling failed messages too fast/often --- ...ailedMessageToFailureTransportListener.php | 35 ++++++--- .../Retry/MultiplierRetryStrategy.php | 2 +- ...dMessageToFailureTransportListenerTest.php | 71 +++++++++++++++++-- .../Tests/FailureIntegrationTest.php | 40 +++++++---- .../Retry/MultiplierRetryStrategyTest.php | 5 +- 5 files changed, 122 insertions(+), 31 deletions(-) diff --git a/src/Symfony/Component/Messenger/EventListener/SendFailedMessageToFailureTransportListener.php b/src/Symfony/Component/Messenger/EventListener/SendFailedMessageToFailureTransportListener.php index 18b255190ee19..9878a13ff8b45 100644 --- a/src/Symfony/Component/Messenger/EventListener/SendFailedMessageToFailureTransportListener.php +++ b/src/Symfony/Component/Messenger/EventListener/SendFailedMessageToFailureTransportListener.php @@ -15,7 +15,9 @@ use Psr\Log\LoggerInterface; use Symfony\Component\EventDispatcher\EventSubscriberInterface; use Symfony\Component\Messenger\Event\WorkerMessageFailedEvent; +use Symfony\Component\Messenger\Retry\RetryStrategyInterface; use Symfony\Component\Messenger\Stamp\DelayStamp; +use Symfony\Component\Messenger\Stamp\ReceivedStamp; use Symfony\Component\Messenger\Stamp\RedeliveryStamp; use Symfony\Component\Messenger\Stamp\SentToFailureTransportStamp; @@ -28,11 +30,13 @@ class SendFailedMessageToFailureTransportListener implements EventSubscriberInte { private ContainerInterface $failureSenders; private ?LoggerInterface $logger; + private ?ContainerInterface $retryStrategyLocator; - public function __construct(ContainerInterface $failureSenders, ?LoggerInterface $logger = null) + public function __construct(ContainerInterface $failureSenders, ?LoggerInterface $logger = null, ?ContainerInterface $retryStrategyLocator = null) { $this->failureSenders = $failureSenders; $this->logger = $logger; + $this->retryStrategyLocator = $retryStrategyLocator; } /** @@ -44,28 +48,30 @@ public function onMessageFailed(WorkerMessageFailedEvent $event) return; } - if (!$this->failureSenders->has($event->getReceiverName())) { + $originalTransportName = $event->getEnvelope()->last(ReceivedStamp::class) + ?->getTransportName() ?? $event->getReceiverName(); + + if (!$this->failureSenders->has($originalTransportName)) { return; } - $failureSender = $this->failureSenders->get($event->getReceiverName()); + $failureSender = $this->failureSenders->get($originalTransportName); $envelope = $event->getEnvelope(); - // avoid re-sending to the failed sender - if (null !== $envelope->last(SentToFailureTransportStamp::class)) { - return; - } + $delay = $this->getRetryStrategyForTransport($event->getReceiverName()) + ?->getWaitingTime($envelope, $event->getThrowable()) ?? 0; $envelope = $envelope->with( - new SentToFailureTransportStamp($event->getReceiverName()), - new DelayStamp(0), + new SentToFailureTransportStamp($originalTransportName), + new DelayStamp($delay), new RedeliveryStamp(0) ); - $this->logger?->info('Rejected message {class} will be sent to the failure transport {transport}.', [ + $this->logger?->info('Rejected message {class} will be sent to the failure transport {transport} using {delay} ms delay.', [ 'class' => $envelope->getMessage()::class, 'transport' => $failureSender::class, + 'delay' => $delay, ]); $failureSender->send($envelope); @@ -77,4 +83,13 @@ public static function getSubscribedEvents(): array WorkerMessageFailedEvent::class => ['onMessageFailed', -100], ]; } + + private function getRetryStrategyForTransport(string $transportName): ?RetryStrategyInterface + { + if (null === $this->retryStrategyLocator || !$this->retryStrategyLocator->has($transportName)) { + return null; + } + + return $this->retryStrategyLocator->get($transportName); + } } diff --git a/src/Symfony/Component/Messenger/Retry/MultiplierRetryStrategy.php b/src/Symfony/Component/Messenger/Retry/MultiplierRetryStrategy.php index 16bc3b9325f7d..5d248a23f645e 100644 --- a/src/Symfony/Component/Messenger/Retry/MultiplierRetryStrategy.php +++ b/src/Symfony/Component/Messenger/Retry/MultiplierRetryStrategy.php @@ -78,7 +78,7 @@ public function isRetryable(Envelope $message, ?\Throwable $throwable = null): b */ public function getWaitingTime(Envelope $message, ?\Throwable $throwable = null): int { - $retries = RedeliveryStamp::getRetryCountFromEnvelope($message); + $retries = \count($message->all(RedeliveryStamp::class)); $delay = $this->delayMilliseconds * $this->multiplier ** $retries; diff --git a/src/Symfony/Component/Messenger/Tests/EventListener/SendFailedMessageToFailureTransportListenerTest.php b/src/Symfony/Component/Messenger/Tests/EventListener/SendFailedMessageToFailureTransportListenerTest.php index 9060ff515ed84..d019c4438bee9 100644 --- a/src/Symfony/Component/Messenger/Tests/EventListener/SendFailedMessageToFailureTransportListenerTest.php +++ b/src/Symfony/Component/Messenger/Tests/EventListener/SendFailedMessageToFailureTransportListenerTest.php @@ -16,6 +16,9 @@ use Symfony\Component\Messenger\Envelope; use Symfony\Component\Messenger\Event\WorkerMessageFailedEvent; use Symfony\Component\Messenger\EventListener\SendFailedMessageToFailureTransportListener; +use Symfony\Component\Messenger\Retry\RetryStrategyInterface; +use Symfony\Component\Messenger\Stamp\DelayStamp; +use Symfony\Component\Messenger\Stamp\ReceivedStamp; use Symfony\Component\Messenger\Stamp\SentToFailureTransportStamp; use Symfony\Component\Messenger\Transport\Sender\SenderInterface; @@ -29,6 +32,10 @@ public function testItSendsToTheFailureTransportWithSenderLocator() /* @var Envelope $envelope */ $this->assertInstanceOf(Envelope::class, $envelope); + $delayStamp = $envelope->last(DelayStamp::class); + $this->assertNotNull($delayStamp); + $this->assertSame(5000, $delayStamp->getDelay()); + /** @var SentToFailureTransportStamp $sentToFailureTransportStamp */ $sentToFailureTransportStamp = $envelope->last(SentToFailureTransportStamp::class); $this->assertNotNull($sentToFailureTransportStamp); @@ -40,6 +47,36 @@ public function testItSendsToTheFailureTransportWithSenderLocator() $serviceLocator = $this->createMock(ServiceLocator::class); $serviceLocator->expects($this->once())->method('has')->willReturn(true); $serviceLocator->expects($this->once())->method('get')->with($receiverName)->willReturn($sender); + + $retryStrategy = $this->createMock(RetryStrategyInterface::class); + $retryStrategy->expects($this->once())->method('getWaitingTime')->willReturn(5000); + + $retryStrategyLocator = $this->createMock(ServiceLocator::class); + $retryStrategyLocator->expects($this->once())->method('has')->with($receiverName)->willReturn(true); + $retryStrategyLocator->expects($this->once())->method('get')->with($receiverName)->willReturn($retryStrategy); + + $listener = new SendFailedMessageToFailureTransportListener($serviceLocator, null, $retryStrategyLocator); + + $exception = new \Exception('no!'); + $envelope = new Envelope(new \stdClass()); + $event = new WorkerMessageFailedEvent($envelope, 'my_receiver', $exception); + + $listener->onMessageFailed($event); + } + + public function testItSendsToTheFailureTransportWithoutRetryStrategyLocator() + { + $sender = $this->createMock(SenderInterface::class); + $sender->expects($this->once())->method('send')->with($this->callback(function (Envelope $envelope) { + $this->assertSame(0, $envelope->last(DelayStamp::class)->getDelay()); + + return true; + }))->willReturnArgument(0); + + $serviceLocator = $this->createStub(ServiceLocator::class); + $serviceLocator->method('has')->willReturn(true); + $serviceLocator->method('get')->willReturn($sender); + $listener = new SendFailedMessageToFailureTransportListener($serviceLocator); $exception = new \Exception('no!'); @@ -55,7 +92,8 @@ public function testDoNothingOnRetryWithServiceLocator() $sender->expects($this->never())->method('send'); $serviceLocator = $this->createMock(ServiceLocator::class); - $listener = new SendFailedMessageToFailureTransportListener($serviceLocator); + $retryStrategyLocator = $this->createStub(ServiceLocator::class); + $listener = new SendFailedMessageToFailureTransportListener($serviceLocator, null, $retryStrategyLocator); $envelope = new Envelope(new \stdClass()); $event = new WorkerMessageFailedEvent($envelope, 'my_receiver', new \Exception()); @@ -64,19 +102,40 @@ public function testDoNothingOnRetryWithServiceLocator() $listener->onMessageFailed($event); } - public function testDoNotRedeliverToFailedWithServiceLocator() + public function testDoRedeliverToFailedWithServiceLocator() { $receiverName = 'my_receiver'; + $failedReceiver = 'failed_receiver'; $sender = $this->createMock(SenderInterface::class); - $sender->expects($this->never())->method('send'); + $sender->expects($this->once())->method('send')->with($this->callback(function (Envelope $envelope) use ($receiverName) { + $delayStamp = $envelope->last(DelayStamp::class); + $this->assertNotNull($delayStamp); + $this->assertSame(1000, $delayStamp->getDelay()); + + $sentToFailureTransportStamp = $envelope->last(SentToFailureTransportStamp::class); + $this->assertNotNull($sentToFailureTransportStamp); + $this->assertSame($receiverName, $sentToFailureTransportStamp->getOriginalReceiverName()); + + return true; + }))->willReturnArgument(0); $serviceLocator = $this->createMock(ServiceLocator::class); + $serviceLocator->expects($this->once())->method('has')->willReturn(true); + $serviceLocator->expects($this->once())->method('get')->with($receiverName)->willReturn($sender); - $listener = new SendFailedMessageToFailureTransportListener($serviceLocator); + $retryStrategy = $this->createStub(RetryStrategyInterface::class); + $retryStrategy->method('getWaitingTime')->willReturn(1000); + $retryStrategyLocator = $this->createMock(ServiceLocator::class); + $retryStrategyLocator->expects($this->once())->method('has')->willReturn(true); + $retryStrategyLocator->expects($this->once())->method('get')->with($failedReceiver)->willReturn($retryStrategy); + + $listener = new SendFailedMessageToFailureTransportListener($serviceLocator, null, $retryStrategyLocator); $envelope = new Envelope(new \stdClass(), [ + // the received stamp is assumed to be added by the FailedMessageProcessingMiddleware + new ReceivedStamp($receiverName), new SentToFailureTransportStamp($receiverName), ]); - $event = new WorkerMessageFailedEvent($envelope, $receiverName, new \Exception()); + $event = new WorkerMessageFailedEvent($envelope, $failedReceiver, new \Exception()); $listener->onMessageFailed($event); } @@ -87,7 +146,7 @@ public function testDoNothingIfFailureTransportIsNotDefined() $sender->expects($this->never())->method('send'); $serviceLocator = $this->createMock(ServiceLocator::class); - $listener = new SendFailedMessageToFailureTransportListener($serviceLocator, null); + $listener = new SendFailedMessageToFailureTransportListener($serviceLocator); $exception = new \Exception('no!'); $envelope = new Envelope(new \stdClass()); diff --git a/src/Symfony/Component/Messenger/Tests/FailureIntegrationTest.php b/src/Symfony/Component/Messenger/Tests/FailureIntegrationTest.php index d711097ee21d4..1524e6961848f 100644 --- a/src/Symfony/Component/Messenger/Tests/FailureIntegrationTest.php +++ b/src/Symfony/Component/Messenger/Tests/FailureIntegrationTest.php @@ -13,7 +13,6 @@ use PHPUnit\Framework\TestCase; use Psr\Container\ContainerInterface; -use Psr\Log\NullLogger; use Symfony\Component\DependencyInjection\ServiceLocator; use Symfony\Component\EventDispatcher\EventDispatcher; use Symfony\Component\Messenger\Envelope; @@ -115,7 +114,11 @@ public function testRequeueMechanism() $dispatcher->addSubscriber(new AddErrorDetailsStampListener()); $dispatcher->addSubscriber(new SendFailedMessageForRetryListener($locator, $retryStrategyLocator)); - $dispatcher->addSubscriber(new SendFailedMessageToFailureTransportListener($sendersLocatorFailureTransport)); + $dispatcher->addSubscriber(new SendFailedMessageToFailureTransportListener( + $sendersLocatorFailureTransport, + null, + $retryStrategyLocator + )); $dispatcher->addSubscriber(new StopWorkerOnMessageLimitListener(1)); $runWorker = function (string $transportName) use ($transports, $bus, $dispatcher): ?\Throwable { @@ -195,14 +198,14 @@ public function testRequeueMechanism() $this->assertCount(1, $transport2->getMessagesWaitingToBeReceived()); /* - * Message is retried on failure transport then discarded + * Message is retried on failure transport then re-queued */ $runWorker('the_failure_transport'); // only the "failed" handler is called a 4th time $this->assertSame(4, $transport1HandlerThatFails->getTimesCalled()); $this->assertSame(1, $allTransportHandlerThatWorks->getTimesCalled()); - // handling fails again, message is discarded - $this->assertCount(0, $failureTransport->getMessagesWaitingToBeReceived()); + // handling fails again, message is re-queued with a delay + $this->assertCount(1, $failureTransport->getMessagesWaitingToBeReceived()); /* * Execute handlers on transport2 @@ -214,10 +217,10 @@ public function testRequeueMechanism() $this->assertSame(2, $allTransportHandlerThatWorks->getTimesCalled()); // transport1 handler called for the first time $this->assertSame(1, $transport2HandlerThatWorks->getTimesCalled()); - // all transport should be empty + // all original transports should be empty - failed queue still holds the message $this->assertEmpty($transport1->getMessagesWaitingToBeReceived()); $this->assertEmpty($transport2->getMessagesWaitingToBeReceived()); - $this->assertEmpty($failureTransport->getMessagesWaitingToBeReceived()); + $this->assertCount(1, $failureTransport->getMessagesWaitingToBeReceived()); /* * Dispatch the original message again @@ -226,9 +229,18 @@ public function testRequeueMechanism() // handle the failing message so it goes into the failure transport $runWorker('transport1'); $runWorker('transport1'); + $this->assertCount(2, $failureTransport->getMessagesWaitingToBeReceived()); // now make the handler work! $transport1HandlerThatFails->setShouldThrow(false); $runWorker('the_failure_transport'); + $runWorker('the_failure_transport'); + + // the message is now handled and the failure transport is empty + // transport1Handler is called 4 more times - 2 retries and twice successfully from failure transport + $this->assertSame(8, $transport1HandlerThatFails->getTimesCalled()); + $this->assertSame(3, $allTransportHandlerThatWorks->getTimesCalled()); + $this->assertSame(1, $transport2HandlerThatWorks->getTimesCalled()); + // the failure transport is empty because it worked $this->assertEmpty($failureTransport->getMessagesWaitingToBeReceived()); } @@ -297,7 +309,8 @@ public function testMultipleFailedTransportsWithoutGlobalFailureTransport() $dispatcher->addSubscriber(new SendFailedMessageForRetryListener($locator, $retryStrategyLocator)); $dispatcher->addSubscriber(new SendFailedMessageToFailureTransportListener( $sendersLocatorFailureTransport, - new NullLogger() + null, + $retryStrategyLocator, )); $dispatcher->addSubscriber(new StopWorkerOnMessageLimitListener(1)); @@ -342,7 +355,8 @@ public function testMultipleFailedTransportsWithoutGlobalFailureTransport() // "transport1" handler is called again from the "the_failed_transport1" and it fails $this->assertSame(2, $transport1HandlerThatFails->getTimesCalled()); $this->assertSame(0, $transport2HandlerThatFails->getTimesCalled()); - $this->assertCount(0, $failureTransport1->getMessagesWaitingToBeReceived()); + // message is not discarded but remains in failed transport + $this->assertCount(1, $failureTransport1->getMessagesWaitingToBeReceived()); $this->assertCount(0, $failureTransport2->getMessagesWaitingToBeReceived()); // Receive the message from "transport2" @@ -351,7 +365,7 @@ public function testMultipleFailedTransportsWithoutGlobalFailureTransport() $this->assertSame(2, $transport1HandlerThatFails->getTimesCalled()); // handler for "transport2" is called $this->assertSame(1, $transport2HandlerThatFails->getTimesCalled()); - $this->assertCount(0, $failureTransport1->getMessagesWaitingToBeReceived()); + $this->assertCount(1, $failureTransport1->getMessagesWaitingToBeReceived()); // the failure transport "the_failure_transport2" has 1 new message failed from "transport2" $this->assertCount(1, $failureTransport2->getMessagesWaitingToBeReceived()); @@ -360,9 +374,9 @@ public function testMultipleFailedTransportsWithoutGlobalFailureTransport() $this->assertSame(2, $transport1HandlerThatFails->getTimesCalled()); // "transport2" handler is called again from the "the_failed_transport2" and it fails $this->assertSame(2, $transport2HandlerThatFails->getTimesCalled()); - $this->assertCount(0, $failureTransport1->getMessagesWaitingToBeReceived()); - // After the message fails again, the message is discarded from the "the_failure_transport2" - $this->assertCount(0, $failureTransport2->getMessagesWaitingToBeReceived()); + $this->assertCount(1, $failureTransport1->getMessagesWaitingToBeReceived()); + // After the message fails again, the message is re-queued to the "the_failure_transport2" + $this->assertCount(1, $failureTransport2->getMessagesWaitingToBeReceived()); } public function testStampsAddedByMiddlewaresDontDisappearWhenDelayedMessageFails() diff --git a/src/Symfony/Component/Messenger/Tests/Retry/MultiplierRetryStrategyTest.php b/src/Symfony/Component/Messenger/Tests/Retry/MultiplierRetryStrategyTest.php index 5c37aa6bf547c..29b30ec79f043 100644 --- a/src/Symfony/Component/Messenger/Tests/Retry/MultiplierRetryStrategyTest.php +++ b/src/Symfony/Component/Messenger/Tests/Retry/MultiplierRetryStrategyTest.php @@ -55,7 +55,10 @@ public function testIsRetryableWithNoStamp() public function testGetWaitTime(int $delay, float $multiplier, int $maxDelay, int $previousRetries, int $expectedDelay) { $strategy = new MultiplierRetryStrategy(10, $delay, $multiplier, $maxDelay); - $envelope = new Envelope(new \stdClass(), [new RedeliveryStamp($previousRetries)]); + $envelope = Envelope::wrap(new \stdClass()); + for ($i = 0; $i < $previousRetries; ++$i) { + $envelope = $envelope->with(new RedeliveryStamp($i)); + } $this->assertSame($expectedDelay, $strategy->getWaitingTime($envelope)); } From 6acfab5deb227522075eb6619419249c4fb76eaf Mon Sep 17 00:00:00 2001 From: Thomas Unterreitmeier Date: Sun, 7 Jul 2024 05:44:20 +0200 Subject: [PATCH 2/3] [FrameworkBundle] Inject default retry strategy locator into messenger failure listener --- .../Bundle/FrameworkBundle/Resources/config/messenger.php | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Symfony/Bundle/FrameworkBundle/Resources/config/messenger.php b/src/Symfony/Bundle/FrameworkBundle/Resources/config/messenger.php index 5e4726265db3f..b7e813d45a678 100644 --- a/src/Symfony/Bundle/FrameworkBundle/Resources/config/messenger.php +++ b/src/Symfony/Bundle/FrameworkBundle/Resources/config/messenger.php @@ -186,6 +186,7 @@ ->args([ abstract_arg('failure transports'), service('logger')->ignoreOnInvalid(), + service('messenger.retry_strategy_locator'), ]) ->tag('kernel.event_subscriber') ->tag('monolog.logger', ['channel' => 'messenger']) From ea61fcb0e3dcf4fd3e78d045007266b7c572ff62 Mon Sep 17 00:00:00 2001 From: Thomas Unterreitmeier Date: Sun, 7 Jul 2024 05:47:45 +0200 Subject: [PATCH 3/3] [Messenger][Amqp] Do not use redelivery routing key when sending to failure transport The failure transport uses a delay - the retry routing key from the previous stamp would interfere with publishing to the failure exchange/queue --- .../Amqp/Tests/Transport/AmqpSenderTest.php | 26 +++ .../FailureTransportIntegrationTest.php | 163 ++++++++++++++++++ .../Bridge/Amqp/Transport/AmqpSender.php | 12 +- 3 files changed, 200 insertions(+), 1 deletion(-) create mode 100644 src/Symfony/Component/Messenger/Bridge/Amqp/Tests/Transport/FailureTransportIntegrationTest.php diff --git a/src/Symfony/Component/Messenger/Bridge/Amqp/Tests/Transport/AmqpSenderTest.php b/src/Symfony/Component/Messenger/Bridge/Amqp/Tests/Transport/AmqpSenderTest.php index b1dda969fb49b..1cef9677da355 100644 --- a/src/Symfony/Component/Messenger/Bridge/Amqp/Tests/Transport/AmqpSenderTest.php +++ b/src/Symfony/Component/Messenger/Bridge/Amqp/Tests/Transport/AmqpSenderTest.php @@ -13,11 +13,14 @@ use PHPUnit\Framework\TestCase; use Symfony\Component\Messenger\Bridge\Amqp\Tests\Fixtures\DummyMessage; +use Symfony\Component\Messenger\Bridge\Amqp\Transport\AmqpReceivedStamp; use Symfony\Component\Messenger\Bridge\Amqp\Transport\AmqpSender; use Symfony\Component\Messenger\Bridge\Amqp\Transport\AmqpStamp; use Symfony\Component\Messenger\Bridge\Amqp\Transport\Connection; use Symfony\Component\Messenger\Envelope; use Symfony\Component\Messenger\Exception\TransportException; +use Symfony\Component\Messenger\Stamp\RedeliveryStamp; +use Symfony\Component\Messenger\Stamp\SentToFailureTransportStamp; use Symfony\Component\Messenger\Transport\Serialization\SerializerInterface; /** @@ -55,6 +58,29 @@ public function testItSendsTheEncodedMessageUsingARoutingKey() $sender->send($envelope); } + public function testItDoesNotUseRetryRoutingKeyWhenSendingToFailureTransport() + { + $envelope = (new Envelope(new DummyMessage('Oy')))->with() + ->with(new AmqpReceivedStamp( + $this->createStub(\AMQPEnvelope::class), + 'original_receiver' + )) + ->with(new RedeliveryStamp(1)) + ->with(new SentToFailureTransportStamp('original_receiver')); + $encoded = ['body' => '...', 'headers' => ['type' => DummyMessage::class]]; + + $serializer = $this->createStub(SerializerInterface::class); + $serializer->method('encode')->with($envelope)->willReturn($encoded); + + $connection = $this->createMock(Connection::class); + $connection->expects($this->once())->method('publish')->with($encoded['body'], $encoded['headers'], 0, $this->callback( + static fn (AmqpStamp $stamp) => '' === $stamp->getRoutingKey() + )); + + $sender = new AmqpSender($connection, $serializer); + $sender->send($envelope); + } + public function testItSendsTheEncodedMessageWithoutHeaders() { $envelope = new Envelope(new DummyMessage('Oy')); diff --git a/src/Symfony/Component/Messenger/Bridge/Amqp/Tests/Transport/FailureTransportIntegrationTest.php b/src/Symfony/Component/Messenger/Bridge/Amqp/Tests/Transport/FailureTransportIntegrationTest.php new file mode 100644 index 0000000000000..beba4ef02ee5b --- /dev/null +++ b/src/Symfony/Component/Messenger/Bridge/Amqp/Tests/Transport/FailureTransportIntegrationTest.php @@ -0,0 +1,163 @@ + + * + * For the full copyright and license information, please view the LICENSE + * file that was distributed with this source code. + */ + +namespace Symfony\Component\Messenger\Bridge\Amqp\Tests\Transport; + +use PHPUnit\Framework\TestCase; +use Psr\Container\ContainerInterface; +use Symfony\Component\DependencyInjection\ServiceLocator; +use Symfony\Component\EventDispatcher\EventDispatcher; +use Symfony\Component\Messenger\Bridge\Amqp\Tests\Fixtures\DummyMessage; +use Symfony\Component\Messenger\Bridge\Amqp\Transport\AmqpTransport; +use Symfony\Component\Messenger\Bridge\Amqp\Transport\Connection; +use Symfony\Component\Messenger\Envelope; +use Symfony\Component\Messenger\EventListener\SendFailedMessageForRetryListener; +use Symfony\Component\Messenger\EventListener\SendFailedMessageToFailureTransportListener; +use Symfony\Component\Messenger\EventListener\StopWorkerOnMessageLimitListener; +use Symfony\Component\Messenger\EventListener\StopWorkerOnTimeLimitListener; +use Symfony\Component\Messenger\Handler\HandlerDescriptor; +use Symfony\Component\Messenger\Handler\HandlersLocator; +use Symfony\Component\Messenger\MessageBus; +use Symfony\Component\Messenger\Middleware\FailedMessageProcessingMiddleware; +use Symfony\Component\Messenger\Middleware\HandleMessageMiddleware; +use Symfony\Component\Messenger\Middleware\SendMessageMiddleware; +use Symfony\Component\Messenger\Retry\MultiplierRetryStrategy; +use Symfony\Component\Messenger\Stamp\DelayStamp; +use Symfony\Component\Messenger\Stamp\RedeliveryStamp; +use Symfony\Component\Messenger\Stamp\SentToFailureTransportStamp; +use Symfony\Component\Messenger\Transport\Sender\SendersLocator; +use Symfony\Component\Messenger\Worker; + +/** + * @requires extension amqp + * + * @group integration + */ +class FailureTransportIntegrationTest extends TestCase +{ + protected function setUp(): void + { + parent::setUp(); + + if (!getenv('MESSENGER_AMQP_DSN')) { + $this->markTestSkipped('The "MESSENGER_AMQP_DSN" environment variable is required.'); + } + } + + public function testItDoesNotLoseMessagesFromTheFailedTransport() + { + $connection = Connection::fromDsn(getenv('MESSENGER_AMQP_DSN')); + $connection->setup(); + $connection->purgeQueues(); + + $failureConnection = Connection::fromDsn(getenv('MESSENGER_AMQP_DSN'), + ['exchange' => [ + 'name' => 'failed', + 'type' => 'fanout', + ], 'queues' => ['failed' => []]] + ); + $failureConnection->setup(); + $failureConnection->purgeQueues(); + + $originalTransport = new AmqpTransport($connection); + $failureTransport = new AmqpTransport($failureConnection); + + $retryStrategy = new MultiplierRetryStrategy(1, 100, 2); + $retryStrategyLocator = $this->createStub(ContainerInterface::class); + $retryStrategyLocator->method('has')->willReturn(true); + $retryStrategyLocator->method('get')->willReturn($retryStrategy); + + $sendersLocatorFailureTransport = new ServiceLocator([ + 'original' => static fn () => $failureTransport, + ]); + + $transports = [ + 'original' => $originalTransport, + 'failed' => $failureTransport, + ]; + + $locator = $this->createStub(ContainerInterface::class); + $locator->method('has')->willReturn(true); + $locator->method('get')->willReturnCallback(static fn ($transportName) => $transports[$transportName]); + $senderLocator = new SendersLocator( + [DummyMessage::class => ['original']], + $locator + ); + + $timesHandled = 0; + + $handler = static function () use (&$timesHandled) { + ++$timesHandled; + throw new \Exception('Handler failed'); + }; + + $handlerLocator = new HandlersLocator([ + DummyMessage::class => [new HandlerDescriptor($handler, ['from_transport' => 'original'])], + ]); + + $bus = new MessageBus([ + new FailedMessageProcessingMiddleware(), + new SendMessageMiddleware($senderLocator), + new HandleMessageMiddleware($handlerLocator), + ]); + + $dispatcher = new EventDispatcher(); + $dispatcher->addSubscriber(new SendFailedMessageForRetryListener($locator, $retryStrategyLocator)); + $dispatcher->addSubscriber(new SendFailedMessageToFailureTransportListener( + $sendersLocatorFailureTransport, null, $retryStrategyLocator + )); + $dispatcher->addSubscriber(new StopWorkerOnMessageLimitListener(1)); + $dispatcher->addSubscriber(new StopWorkerOnTimeLimitListener(2)); + + $originalTransport->send(Envelope::wrap(new DummyMessage('dummy'))); + + $runWorker = static function (string $transportName) use ($bus, $dispatcher, $transports): void { + (new Worker( + [$transportName => $transports[$transportName]], + $bus, + $dispatcher, + ))->run(); + }; + + $runWorker('original'); + $runWorker('original'); + $runWorker('failed'); + $runWorker('failed'); + + $this->assertSame(4, $timesHandled); + $failedMessage = $this->waitForFailedMessage($failureTransport, 2); + // 100 delay * 2 multiplier ^ 3 retries = 800 expected delay + $this->assertSame(800, $failedMessage->last(DelayStamp::class)->getDelay()); + $this->assertSame(0, $failedMessage->last(RedeliveryStamp::class)->getRetryCount()); + $this->assertCount(4, $failedMessage->all(RedeliveryStamp::class)); + $this->assertCount(2, $failedMessage->all(SentToFailureTransportStamp::class)); + foreach ($failedMessage->all(SentToFailureTransportStamp::class) as $stamp) { + $this->assertSame('original', $stamp->getOriginalReceiverName()); + } + } + + private function waitForFailedMessage(AmqpTransport $failureTransport, int $timeOutInS): Envelope + { + $start = microtime(true); + while (microtime(true) - $start < $timeOutInS) { + $envelopes = iterator_to_array($failureTransport->get()); + if (\count($envelopes) > 0) { + foreach ($envelopes as $envelope) { + $failureTransport->reject($envelope); + } + + return $envelopes[0]; + } + usleep(100 * 1000); + } + throw new \RuntimeException('Message was not received from failure transport within expected timeframe.'); + } +} diff --git a/src/Symfony/Component/Messenger/Bridge/Amqp/Transport/AmqpSender.php b/src/Symfony/Component/Messenger/Bridge/Amqp/Transport/AmqpSender.php index 4f7caaa71635f..7df1d897b5ae8 100644 --- a/src/Symfony/Component/Messenger/Bridge/Amqp/Transport/AmqpSender.php +++ b/src/Symfony/Component/Messenger/Bridge/Amqp/Transport/AmqpSender.php @@ -15,6 +15,7 @@ use Symfony\Component\Messenger\Exception\TransportException; use Symfony\Component\Messenger\Stamp\DelayStamp; use Symfony\Component\Messenger\Stamp\RedeliveryStamp; +use Symfony\Component\Messenger\Stamp\SentToFailureTransportStamp; use Symfony\Component\Messenger\Transport\Sender\SenderInterface; use Symfony\Component\Messenger\Transport\Serialization\PhpSerializer; use Symfony\Component\Messenger\Transport\Serialization\SerializerInterface; @@ -59,7 +60,7 @@ public function send(Envelope $envelope): Envelope $amqpStamp = AmqpStamp::createFromAmqpEnvelope( $amqpReceivedStamp->getAmqpEnvelope(), $amqpStamp, - $envelope->last(RedeliveryStamp::class) ? $amqpReceivedStamp->getQueueName() : null + $this->getRetryRoutingKey($envelope, $amqpReceivedStamp) ); } @@ -76,4 +77,13 @@ public function send(Envelope $envelope): Envelope return $envelope; } + + private function getRetryRoutingKey(Envelope $envelope, AmqpReceivedStamp $amqpReceivedStamp): ?string + { + if (1 === \count($envelope->all(SentToFailureTransportStamp::class))) { + return null; + } + + return $envelope->last(RedeliveryStamp::class) ? $amqpReceivedStamp->getQueueName() : null; + } }