From 79ae82d1b5ab7e3426189b8c75a8f3fda062a646 Mon Sep 17 00:00:00 2001 From: Adam Katz Date: Wed, 18 Jan 2023 20:55:23 +0300 Subject: [PATCH] [Messenger] allow Redis transport to claim multiple messages sequentially --- .../Redis/Tests/Transport/ConnectionTest.php | 164 ++++++++++++++---- .../Bridge/Redis/Transport/Connection.php | 33 ++-- 2 files changed, 138 insertions(+), 59 deletions(-) diff --git a/src/Symfony/Component/Messenger/Bridge/Redis/Tests/Transport/ConnectionTest.php b/src/Symfony/Component/Messenger/Bridge/Redis/Tests/Transport/ConnectionTest.php index fb98baf70b610..23e884d97d022 100644 --- a/src/Symfony/Component/Messenger/Bridge/Redis/Tests/Transport/ConnectionTest.php +++ b/src/Symfony/Component/Messenger/Bridge/Redis/Tests/Transport/ConnectionTest.php @@ -157,6 +157,14 @@ public function testKeepGettingPendingMessages() ->with('symfony', 'consumer', ['queue' => 0], 1, null) ->willReturn(['queue' => [['message' => json_encode(['body' => 'Test', 'headers' => []])]]]); + $redis->expects($this->exactly(3))->method('xpending')->willReturn( + [[ + 0 => 'redisid-123', // message-id + 1 => 'consumer', // consumer-name + 2 => 0, // idle + ]] + ); + $connection = Connection::fromDsn('redis://localhost/queue', ['delete_after_ack' => true], $redis); $this->assertNotNull($connection->get()); $this->assertNotNull($connection->get()); @@ -253,6 +261,14 @@ public function testGetPendingMessageFirst() ->with('symfony', 'consumer', ['queue' => '0'], 1, null) ->willReturn(['queue' => [['message' => '{"body":"1","headers":[]}']]]); + $redis->expects($this->once())->method('xpending')->willReturn( + [[ + 0 => 'redisid-123', // message-id + 1 => 'consumer', // consumer-name + 2 => 0, // idle + ]] + ); + $connection = Connection::fromDsn('redis://localhost/queue', ['delete_after_ack' => true], $redis); $message = $connection->get(); @@ -271,57 +287,90 @@ public function testClaimAbandonedMessageWithRaceCondition() { $redis = $this->createMock(\Redis::class); - $redis->expects($this->exactly(3))->method('xreadgroup') - ->willReturnCallback(function (...$args) { - static $series = [ - // first call for pending messages - [['symfony', 'consumer', ['queue' => '0'], 1, null], []], - // second call because of claimed message (redisid-123) - [['symfony', 'consumer', ['queue' => '0'], 1, null], []], - // third call because of no result (other consumer claimed message redisid-123) - [['symfony', 'consumer', ['queue' => '>'], 1, null], []], - ]; - - [$expectedArgs, $return] = array_shift($series); - $this->assertSame($expectedArgs, $args); - - return $return; - }) - ; + // xpending -> xclaim -> xpending -> xreadgroup + $redis->expects($this->once())->method('xreadgroup') + ->with('symfony', 'consumer', ['queue' => '>'], 1, null) // weren't able to claim the message, asking for new ones + ->willReturn([]); - $redis->expects($this->once())->method('xpending')->willReturn([[ - 0 => 'redisid-123', // message-id - 1 => 'consumer-2', // consumer-name - 2 => 3600001, // idle - ]]); + $redis->expects($this->exactly(2))->method('xpending')->willReturnOnConsecutiveCalls( + [[ + 0 => 'redisid-123', // message-id + 1 => 'consumer-2', // consumer-name + 2 => 3600001, // idle + ]], + [] + ); $redis->expects($this->exactly(1))->method('xclaim') ->with('queue', 'symfony', 'consumer', 3600000, ['redisid-123'], ['JUSTID']) + ->willReturn([]); // other consumer claimed the message first + + $connection = Connection::fromDsn('redis://localhost/queue', ['delete_after_ack' => true], $redis); + $connection->get(); + } + + public function testClaimAbandonedMessageWithRaceConditionExtremeContention() + { + $redis = $this->createMock(\Redis::class); + + // (xpending -> xclaim) x 3 -> xreadgroup + $redis->expects($this->once())->method('xreadgroup') + ->with('symfony', 'consumer', ['queue' => '>'], 1, null) // weren't able to claim a message, asking for new ones ->willReturn([]); + $redis->expects($this->exactly(3))->method('xpending')->willReturnOnConsecutiveCalls( + [[ + 0 => 'redisid-123', // message-id + 1 => 'consumer-2', // consumer-name + 2 => 3600001, // idle + ]], + [[ + 0 => 'redisid-234', // message-id + 1 => 'consumer-2', // consumer-name + 2 => 3600001, // idle + ]], + [[ + 0 => 'redisid-345', // message-id + 1 => 'consumer-2', // consumer-name + 2 => 3600001, // idle + ]] + ); + + $redis->expects($this->exactly(3))->method('xclaim') + ->withConsecutive( + ['queue', 'symfony', 'consumer', 3600000, ['redisid-123'], ['JUSTID']], // claim and get message redisid-123 + ['queue', 'symfony', 'consumer', 3600000, ['redisid-234'], ['JUSTID']], // claim and get message redisid-234 + ['queue', 'symfony', 'consumer', 3600000, ['redisid-345'], ['JUSTID']] // claim and get message redisid-345 + ) + ->willReturn([]); // other consumers claimed all messages first + $connection = Connection::fromDsn('redis://localhost/queue', ['delete_after_ack' => true], $redis); $connection->get(); } - public function testClaimAbandonedMessage() + public function testExitClaimMode() { $redis = $this->createMock(\Redis::class); - $redis->expects($this->exactly(2))->method('xreadgroup') - ->willReturnCallback(function (...$args) { - static $series = [ - // first call for pending messages - [['symfony', 'consumer', ['queue' => '0'], 1, null], []], - // second call because of claimed message (redisid-123) - [['symfony', 'consumer', ['queue' => '0'], 1, null], ['queue' => [['message' => '{"body":"1","headers":[]}']]]], - ]; + // xpending -> xreadgroup + $redis->expects($this->once())->method('xreadgroup') + ->with('symfony', 'consumer', ['queue' => '>'], 1, null) // no pending messages, asking for new ones straightaway + ->willReturn([]); + + $redis->expects($this->once())->method('xpending')->willReturn([]); + $redis->expects($this->never())->method('xclaim'); + + $connection = Connection::fromDsn('redis://localhost/queue', ['delete_after_ack' => true], $redis); + $connection->get(); + } - [$expectedArgs, $return] = array_shift($series); - $this->assertSame($expectedArgs, $args); + public function testClaimAbandonedMessage() + { + $redis = $this->createMock(\Redis::class); - return $return; - }) - ; + $redis->expects($this->once())->method('xreadgroup') + ->with('symfony', 'consumer', ['queue' => '0'], 1, null) // claim and get the message (redisid-123) + ->willReturn(['queue' => [['message' => '{"body":"1","headers":[]}']]]); $redis->expects($this->once())->method('xpending')->willReturn([[ 0 => 'redisid-123', // message-id @@ -331,17 +380,58 @@ public function testClaimAbandonedMessage() $redis->expects($this->exactly(1))->method('xclaim') ->with('queue', 'symfony', 'consumer', 3600000, ['redisid-123'], ['JUSTID']) - ->willReturn([]); + ->willReturn(['something']); $connection = Connection::fromDsn('redis://localhost/queue', ['delete_after_ack' => true], $redis); $connection->get(); } + public function testClaimTwoAbandonedMessagesInSequence() + { + $redis = $this->createMock(\Redis::class); + + // twice: xpending -> xclaim -> xreadgroup + $redis->expects($this->exactly(2))->method('xreadgroup') + ->withConsecutive( + ['symfony', 'consumer', ['queue' => '0'], 1, null], // claim and get message redisid-123 + ['symfony', 'consumer', ['queue' => '0'], 1, null] // claim and get message redisid-234 + ) + ->willReturnOnConsecutiveCalls( + ['queue' => [['message' => '{"body":"1","headers":[]}']]], + ['queue' => [['message' => '{"body":"2","headers":[]}']]] + ); + + $redis->expects($this->exactly(2))->method('xpending')->willReturnOnConsecutiveCalls( + [[ + 0 => 'redisid-123', // message-id + 1 => 'consumer-2', // consumer-name + 2 => 3600001, // idle + ]], + [[ + 0 => 'redisid-234', // message-id + 1 => 'consumer-2', // consumer-name + 2 => 3600001, // idle + ]] + ); + + $redis->expects($this->exactly(2))->method('xclaim') + ->withConsecutive( + ['queue', 'symfony', 'consumer', 3600000, ['redisid-123'], ['JUSTID']], + ['queue', 'symfony', 'consumer', 3600000, ['redisid-234'], ['JUSTID']] + ) + ->willReturn(['something']); + + $connection = Connection::fromDsn('redis://localhost/queue', ['delete_after_ack' => true], $redis); + $connection->get(); + $connection->get(); + } + public function testUnexpectedRedisError() { $this->expectException(TransportException::class); $this->expectExceptionMessage('Redis error happens'); $redis = $this->createMock(\Redis::class); + $redis->expects($this->once())->method('xpending')->willReturn([]); $redis->expects($this->once())->method('xreadgroup')->willReturn(false); $redis->expects($this->once())->method('getLastError')->willReturn('Redis error happens'); diff --git a/src/Symfony/Component/Messenger/Bridge/Redis/Transport/Connection.php b/src/Symfony/Component/Messenger/Bridge/Redis/Transport/Connection.php index 064c939ba4a1b..dbf5b4f7a1f24 100644 --- a/src/Symfony/Component/Messenger/Bridge/Redis/Transport/Connection.php +++ b/src/Symfony/Component/Messenger/Bridge/Redis/Transport/Connection.php @@ -57,7 +57,6 @@ class Connection private $claimInterval; private $deleteAfterAck; private $deleteAfterReject; - private $couldHavePendingMessages = true; /** * @param \Redis|\RedisCluster|null $redis @@ -295,8 +294,9 @@ private static function validateOptions(array $options): void } } - private function claimOldPendingMessages() + private function claimOldPendingMessages(int $try = 1): bool { + $couldHavePendingMessages = false; try { // This could soon be optimized with https://github.com/antirez/redis/issues/5212 or // https://github.com/antirez/redis/issues/6256 @@ -308,19 +308,15 @@ private function claimOldPendingMessages() $claimableIds = []; foreach ($pendingMessages as $pendingMessage) { if ($pendingMessage[1] === $this->consumer) { - $this->couldHavePendingMessages = true; - - return; - } - - if ($pendingMessage[2] >= $this->redeliverTimeout) { + $couldHavePendingMessages = true; + } elseif ($pendingMessage[2] >= $this->redeliverTimeout) { $claimableIds[] = $pendingMessage[0]; } } - if (\count($claimableIds) > 0) { + if ($claimableIds) { try { - $this->connection->xclaim( + $claimResult = $this->connection->xclaim( $this->stream, $this->group, $this->consumer, @@ -329,13 +325,13 @@ private function claimOldPendingMessages() ['JUSTID'] ); - $this->couldHavePendingMessages = true; + $couldHavePendingMessages = $couldHavePendingMessages || $claimResult || $try < 3 && $this->claimOldPendingMessages(++$try); } catch (\RedisException $e) { throw new TransportException($e->getMessage(), 0, $e); } } - $this->nextClaim = microtime(true) + $this->claimInterval; + return $couldHavePendingMessages; } public function get(): ?array @@ -370,13 +366,13 @@ public function get(): ?array $this->add(\array_key_exists('body', $decodedQueuedMessage) ? $decodedQueuedMessage['body'] : $queuedMessage, $decodedQueuedMessage['headers'] ?? [], 0); } - if (!$this->couldHavePendingMessages && $this->nextClaim <= microtime(true)) { - $this->claimOldPendingMessages(); + if (!$couldHavePendingMessages = $this->nextClaim <= microtime(true) && $this->claimOldPendingMessages()) { + $this->nextClaim = microtime(true) + $this->claimInterval; } $messageId = '>'; // will receive new messages - if ($this->couldHavePendingMessages) { + if ($couldHavePendingMessages) { $messageId = '0'; // will receive consumers pending messages } @@ -399,13 +395,6 @@ public function get(): ?array throw new TransportException($error ?? 'Could not read messages from the redis stream.'); } - if ($this->couldHavePendingMessages && empty($messages[$this->stream])) { - $this->couldHavePendingMessages = false; - - // No pending messages so get a new one - return $this->get(); - } - foreach ($messages[$this->stream] ?? [] as $key => $message) { return [ 'id' => $key,