From 2f5375fbfb971c6a8e18f84a0e72c3392b75969c Mon Sep 17 00:00:00 2001 From: Herberto Graca Date: Sun, 9 Jul 2023 22:05:00 +0200 Subject: [PATCH 01/11] Ensure messages flagged for deletion have clear flag --- .../Messenger/Bridge/Doctrine/Transport/Connection.php | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/Symfony/Component/Messenger/Bridge/Doctrine/Transport/Connection.php b/src/Symfony/Component/Messenger/Bridge/Doctrine/Transport/Connection.php index 85f95e088ffe5..d3adc22265093 100644 --- a/src/Symfony/Component/Messenger/Bridge/Doctrine/Transport/Connection.php +++ b/src/Symfony/Component/Messenger/Bridge/Doctrine/Transport/Connection.php @@ -38,6 +38,7 @@ * * @author Vincent Touzet * @author Kévin Dunglas + * @author Herberto Graca */ class Connection implements ResetInterface { @@ -50,6 +51,8 @@ class Connection implements ResetInterface 'auto_setup' => true, ]; + public const FLAGGED_FOR_DELETION = '9999-12-31 23:59:59'; + /** * Configuration of the connection. * @@ -159,7 +162,7 @@ public function get(): ?array { if ($this->driverConnection->getDatabasePlatform() instanceof MySQLPlatform) { try { - $this->driverConnection->delete($this->configuration['table_name'], ['delivered_at' => '9999-12-31 23:59:59']); + $this->driverConnection->delete($this->configuration['table_name'], ['delivered_at' => self::FLAGGED_FOR_DELETION]); } catch (DriverException $e) { // Ignore the exception } @@ -248,7 +251,7 @@ public function ack(string $id): bool { try { if ($this->driverConnection->getDatabasePlatform() instanceof MySQLPlatform) { - return $this->driverConnection->update($this->configuration['table_name'], ['delivered_at' => '9999-12-31 23:59:59'], ['id' => $id]) > 0; + return $this->driverConnection->update($this->configuration['table_name'], ['delivered_at' => self::FLAGGED_FOR_DELETION], ['id' => $id]) > 0; } return $this->driverConnection->delete($this->configuration['table_name'], ['id' => $id]) > 0; @@ -261,7 +264,7 @@ public function reject(string $id): bool { try { if ($this->driverConnection->getDatabasePlatform() instanceof MySQLPlatform) { - return $this->driverConnection->update($this->configuration['table_name'], ['delivered_at' => '9999-12-31 23:59:59'], ['id' => $id]) > 0; + return $this->driverConnection->update($this->configuration['table_name'], ['delivered_at' => self::FLAGGED_FOR_DELETION], ['id' => $id]) > 0; } return $this->driverConnection->delete($this->configuration['table_name'], ['id' => $id]) > 0; From a2f274131b4de7759c8f06654983485f5a3bde2e Mon Sep 17 00:00:00 2001 From: Herberto Graca Date: Sun, 9 Jul 2023 22:44:43 +0200 Subject: [PATCH 02/11] Add type hint for $driverConnection in DoctrineTransportFactory --- .../Bridge/Doctrine/Transport/DoctrineTransportFactory.php | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/Symfony/Component/Messenger/Bridge/Doctrine/Transport/DoctrineTransportFactory.php b/src/Symfony/Component/Messenger/Bridge/Doctrine/Transport/DoctrineTransportFactory.php index cff92bf11c19f..9bfceda9e2775 100644 --- a/src/Symfony/Component/Messenger/Bridge/Doctrine/Transport/DoctrineTransportFactory.php +++ b/src/Symfony/Component/Messenger/Bridge/Doctrine/Transport/DoctrineTransportFactory.php @@ -11,6 +11,7 @@ namespace Symfony\Component\Messenger\Bridge\Doctrine\Transport; +use Doctrine\DBAL\Connection as DbalConnection; use Doctrine\DBAL\Platforms\PostgreSQLPlatform; use Doctrine\Persistence\ConnectionRegistry; use Symfony\Component\Messenger\Exception\TransportException; @@ -20,6 +21,7 @@ /** * @author Vincent Touzet + * @author Herberto Graca */ class DoctrineTransportFactory implements TransportFactoryInterface { @@ -38,6 +40,7 @@ public function createTransport(#[\SensitiveParameter] string $dsn, array $optio $configuration = PostgreSqlConnection::buildConfiguration($dsn, $options); try { + /** @var DbalConnection $driverConnection */ $driverConnection = $this->registry->getConnection($configuration['connection']); } catch (\InvalidArgumentException $e) { throw new TransportException('Could not find Doctrine connection from Messenger DSN.', 0, $e); From 84cc536d73579b6dc348427a05b731575b2e6117 Mon Sep 17 00:00:00 2001 From: Herberto Graca Date: Sun, 9 Jul 2023 22:47:12 +0200 Subject: [PATCH 03/11] Add DoctrineTransportFactory DSN_PREFIX constant This allows for projects to set up the messenger config using this constant, making it deterministic, and therefore less prone to errors. --- .../Bridge/Doctrine/Transport/DoctrineTransportFactory.php | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/Symfony/Component/Messenger/Bridge/Doctrine/Transport/DoctrineTransportFactory.php b/src/Symfony/Component/Messenger/Bridge/Doctrine/Transport/DoctrineTransportFactory.php index 9bfceda9e2775..80b89fa084cb6 100644 --- a/src/Symfony/Component/Messenger/Bridge/Doctrine/Transport/DoctrineTransportFactory.php +++ b/src/Symfony/Component/Messenger/Bridge/Doctrine/Transport/DoctrineTransportFactory.php @@ -25,6 +25,8 @@ */ class DoctrineTransportFactory implements TransportFactoryInterface { + public const DSN_PREFIX = 'doctrine://'; + private ConnectionRegistry $registry; public function __construct(ConnectionRegistry $registry) @@ -57,6 +59,6 @@ public function createTransport(#[\SensitiveParameter] string $dsn, array $optio public function supports(#[\SensitiveParameter] string $dsn, array $options): bool { - return str_starts_with($dsn, 'doctrine://'); + return str_starts_with($dsn, self::DSN_PREFIX); } } From 1863a80fa8f55b0b8078eaaec0a576b1570cb4bd Mon Sep 17 00:00:00 2001 From: Herberto Graca Date: Sun, 9 Jul 2023 22:56:32 +0200 Subject: [PATCH 04/11] Replace usage of DoctrineReceivedStamp We already have the TransportMessageIdStamp, so the DoctrineReceivedStamp is redundant, as it contains the same data and its usage is semantically more correct by using the TransportMessageIdStamp. --- .../Tests/Transport/DoctrineReceiverTest.php | 5 ----- .../Doctrine/Transport/DoctrineReceiver.php | 17 +++++++++-------- 2 files changed, 9 insertions(+), 13 deletions(-) diff --git a/src/Symfony/Component/Messenger/Bridge/Doctrine/Tests/Transport/DoctrineReceiverTest.php b/src/Symfony/Component/Messenger/Bridge/Doctrine/Tests/Transport/DoctrineReceiverTest.php index 43a0772371a97..d1e56dff17a44 100644 --- a/src/Symfony/Component/Messenger/Bridge/Doctrine/Tests/Transport/DoctrineReceiverTest.php +++ b/src/Symfony/Component/Messenger/Bridge/Doctrine/Tests/Transport/DoctrineReceiverTest.php @@ -47,11 +47,6 @@ public function testItReturnsTheDecodedMessageToTheHandler() $actualEnvelope = $actualEnvelopes[0]; $this->assertEquals(new DummyMessage('Hi'), $actualEnvelopes[0]->getMessage()); - /** @var DoctrineReceivedStamp $doctrineReceivedStamp */ - $doctrineReceivedStamp = $actualEnvelope->last(DoctrineReceivedStamp::class); - $this->assertNotNull($doctrineReceivedStamp); - $this->assertSame('1', $doctrineReceivedStamp->getId()); - /** @var TransportMessageIdStamp $transportMessageIdStamp */ $transportMessageIdStamp = $actualEnvelope->last(TransportMessageIdStamp::class); $this->assertNotNull($transportMessageIdStamp); diff --git a/src/Symfony/Component/Messenger/Bridge/Doctrine/Transport/DoctrineReceiver.php b/src/Symfony/Component/Messenger/Bridge/Doctrine/Transport/DoctrineReceiver.php index 79310f35a9a9c..bc3ef46b2e8a9 100644 --- a/src/Symfony/Component/Messenger/Bridge/Doctrine/Transport/DoctrineReceiver.php +++ b/src/Symfony/Component/Messenger/Bridge/Doctrine/Transport/DoctrineReceiver.php @@ -25,6 +25,7 @@ /** * @author Vincent Touzet + * @author Herberto Graca */ class DoctrineReceiver implements ListableReceiverInterface, MessageCountAwareInterface { @@ -68,7 +69,7 @@ public function get(): iterable public function ack(Envelope $envelope): void { try { - $this->connection->ack($this->findDoctrineReceivedStamp($envelope)->getId()); + $this->connection->ack($this->findMessageIdStamp($envelope)->getId()); } catch (DBALException $exception) { throw new TransportException($exception->getMessage(), 0, $exception); } @@ -77,7 +78,7 @@ public function ack(Envelope $envelope): void public function reject(Envelope $envelope): void { try { - $this->connection->reject($this->findDoctrineReceivedStamp($envelope)->getId()); + $this->connection->reject($this->findMessageIdStamp($envelope)->getId()); } catch (DBALException $exception) { throw new TransportException($exception->getMessage(), 0, $exception); } @@ -120,16 +121,16 @@ public function find(mixed $id): ?Envelope return $this->createEnvelopeFromData($doctrineEnvelope); } - private function findDoctrineReceivedStamp(Envelope $envelope): DoctrineReceivedStamp + private function findMessageIdStamp(Envelope $envelope): TransportMessageIdStamp { - /** @var DoctrineReceivedStamp|null $doctrineReceivedStamp */ - $doctrineReceivedStamp = $envelope->last(DoctrineReceivedStamp::class); + /** @var TransportMessageIdStamp|null $transportMessageIdStamp */ + $transportMessageIdStamp = $envelope->last(TransportMessageIdStamp::class); - if (null === $doctrineReceivedStamp) { - throw new LogicException('No DoctrineReceivedStamp found on the Envelope.'); + if (null === $transportMessageIdStamp) { + throw new LogicException('No TransportMessageIdStamp found on the Envelope.'); } - return $doctrineReceivedStamp; + return $transportMessageIdStamp; } private function createEnvelopeFromData(array $data): Envelope From ab468d1c04d39daab152ae504f4fdd3c54f457a8 Mon Sep 17 00:00:00 2001 From: Herberto Graca Date: Sun, 9 Jul 2023 22:59:38 +0200 Subject: [PATCH 05/11] Remove the unused DoctrineReceivedStamp --- .../Transport/DoctrineReceivedStamp.php | 32 ------------------- .../Doctrine/Transport/DoctrineReceiver.php | 1 - 2 files changed, 33 deletions(-) delete mode 100644 src/Symfony/Component/Messenger/Bridge/Doctrine/Transport/DoctrineReceivedStamp.php diff --git a/src/Symfony/Component/Messenger/Bridge/Doctrine/Transport/DoctrineReceivedStamp.php b/src/Symfony/Component/Messenger/Bridge/Doctrine/Transport/DoctrineReceivedStamp.php deleted file mode 100644 index 513f4b9f01c4c..0000000000000 --- a/src/Symfony/Component/Messenger/Bridge/Doctrine/Transport/DoctrineReceivedStamp.php +++ /dev/null @@ -1,32 +0,0 @@ - - * - * For the full copyright and license information, please view the LICENSE - * file that was distributed with this source code. - */ - -namespace Symfony\Component\Messenger\Bridge\Doctrine\Transport; - -use Symfony\Component\Messenger\Stamp\NonSendableStampInterface; - -/** - * @author Vincent Touzet - */ -class DoctrineReceivedStamp implements NonSendableStampInterface -{ - private string $id; - - public function __construct(string $id) - { - $this->id = $id; - } - - public function getId(): string - { - return $this->id; - } -} diff --git a/src/Symfony/Component/Messenger/Bridge/Doctrine/Transport/DoctrineReceiver.php b/src/Symfony/Component/Messenger/Bridge/Doctrine/Transport/DoctrineReceiver.php index bc3ef46b2e8a9..d7ab930b0a987 100644 --- a/src/Symfony/Component/Messenger/Bridge/Doctrine/Transport/DoctrineReceiver.php +++ b/src/Symfony/Component/Messenger/Bridge/Doctrine/Transport/DoctrineReceiver.php @@ -147,7 +147,6 @@ private function createEnvelopeFromData(array $data): Envelope } return $envelope->with( - new DoctrineReceivedStamp($data['id']), new TransportMessageIdStamp($data['id']) ); } From 66a8bb710075b4612e3370707d65f90ea45a3ecd Mon Sep 17 00:00:00 2001 From: Herberto Graca Date: Sun, 9 Jul 2023 22:25:32 +0200 Subject: [PATCH 06/11] Add "SKIP LOCKED" to the query that retrieves messages The current `SELECT ... FOR UPDATE` retrieves rows, using the index to find and lock the retrieved rows. This means that when we have more than one consumer, while one consumer is locking those rows, the whole index is locked thus other queries (consumers) will be put on hold. While with a small table this might not be noticeable, as the table grows the meddling with the index becomes slower and the other consumers have to wait more time, eventually making the MQ inoperable. The `SKIP LOCKED` addition will allow other consumers to query the table and get messages immediately, ignoring the rows that other consumers are locking. Co-authored-by: Alexander Malyk --- .../Tests/Transport/ConnectionTest.php | 16 ++++++++++-- .../Bridge/Doctrine/Transport/Connection.php | 25 ++++++++++++++++++- src/Symfony/Component/Messenger/CHANGELOG.md | 5 ++++ 3 files changed, 43 insertions(+), 3 deletions(-) diff --git a/src/Symfony/Component/Messenger/Bridge/Doctrine/Tests/Transport/ConnectionTest.php b/src/Symfony/Component/Messenger/Bridge/Doctrine/Tests/Transport/ConnectionTest.php index 3dae29a4cbd66..4de01c8b88fef 100644 --- a/src/Symfony/Component/Messenger/Bridge/Doctrine/Tests/Transport/ConnectionTest.php +++ b/src/Symfony/Component/Messenger/Bridge/Doctrine/Tests/Transport/ConnectionTest.php @@ -17,7 +17,9 @@ use Doctrine\DBAL\Platforms\AbstractPlatform; use Doctrine\DBAL\Platforms\MariaDBPlatform; use Doctrine\DBAL\Platforms\MySQL57Platform; +use Doctrine\DBAL\Platforms\MySQL80Platform; use Doctrine\DBAL\Platforms\OraclePlatform; +use Doctrine\DBAL\Platforms\PostgreSQLPlatform; use Doctrine\DBAL\Platforms\SQLServer2012Platform; use Doctrine\DBAL\Query\QueryBuilder; use Doctrine\DBAL\Result; @@ -384,11 +386,21 @@ public function testGeneratedSql(AbstractPlatform $platform, string $expectedSql public static function providePlatformSql(): iterable { - yield 'MySQL' => [ + yield 'MySQL57' => [ new MySQL57Platform(), 'SELECT m.* FROM messenger_messages m WHERE (m.delivered_at is null OR m.delivered_at < ?) AND (m.available_at <= ?) AND (m.queue_name = ?) ORDER BY available_at ASC LIMIT 1 FOR UPDATE', ]; + yield 'MySQL8' => [ + new MySQL80Platform(), + 'SELECT m.* FROM messenger_messages m WHERE (m.delivered_at is null OR m.delivered_at < ?) AND (m.available_at <= ?) AND (m.queue_name = ?) ORDER BY available_at ASC LIMIT 1 FOR UPDATE SKIP LOCKED', + ]; + + yield 'Postgres' => [ + new PostgreSQLPlatform(), + 'SELECT m.* FROM messenger_messages m WHERE (m.delivered_at is null OR m.delivered_at < ?) AND (m.available_at <= ?) AND (m.queue_name = ?) ORDER BY available_at ASC LIMIT 1 FOR UPDATE SKIP LOCKED', + ]; + if (class_exists(MariaDBPlatform::class)) { yield 'MariaDB' => [ new MariaDBPlatform(), @@ -398,7 +410,7 @@ public static function providePlatformSql(): iterable yield 'SQL Server' => [ new SQLServer2012Platform(), - 'SELECT m.* FROM messenger_messages m WITH (UPDLOCK, ROWLOCK) WHERE (m.delivered_at is null OR m.delivered_at < ?) AND (m.available_at <= ?) AND (m.queue_name = ?) ORDER BY available_at ASC OFFSET 0 ROWS FETCH NEXT 1 ROWS ONLY ', + 'SELECT m.* FROM messenger_messages m WITH (UPDLOCK, ROWLOCK, READPAST) WHERE (m.delivered_at is null OR m.delivered_at < ?) AND (m.available_at <= ?) AND (m.queue_name = ?) ORDER BY available_at ASC OFFSET 0 ROWS FETCH NEXT 1 ROWS ONLY ', ]; yield 'Oracle' => [ diff --git a/src/Symfony/Component/Messenger/Bridge/Doctrine/Transport/Connection.php b/src/Symfony/Component/Messenger/Bridge/Doctrine/Transport/Connection.php index d3adc22265093..73c38e30aa3df 100644 --- a/src/Symfony/Component/Messenger/Bridge/Doctrine/Transport/Connection.php +++ b/src/Symfony/Component/Messenger/Bridge/Doctrine/Transport/Connection.php @@ -18,8 +18,11 @@ use Doctrine\DBAL\Exception as DBALException; use Doctrine\DBAL\Exception\TableNotFoundException; use Doctrine\DBAL\LockMode; +use Doctrine\DBAL\Platforms\MySQL80Platform; use Doctrine\DBAL\Platforms\MySQLPlatform; use Doctrine\DBAL\Platforms\OraclePlatform; +use Doctrine\DBAL\Platforms\PostgreSQLPlatform; +use Doctrine\DBAL\Platforms\SQLServerPlatform; use Doctrine\DBAL\Query\QueryBuilder; use Doctrine\DBAL\Result; use Doctrine\DBAL\Schema\AbstractSchemaManager; @@ -39,6 +42,7 @@ * @author Vincent Touzet * @author Kévin Dunglas * @author Herberto Graca + * @author Alexander Malyk */ class Connection implements ResetInterface { @@ -201,8 +205,9 @@ public function get(): ?array } // use SELECT ... FOR UPDATE to lock table + $sql .= ' '.$this->driverConnection->getDatabasePlatform()->getWriteLockSQL(); $stmt = $this->executeQuery( - $sql.' '.$this->driverConnection->getDatabasePlatform()->getWriteLockSQL(), + $this->addSkipLocked($sql),// use SELECT ... FOR UPDATE SKIP LOCKED to lock only the relevant rows $query->getParameters(), $query->getParameterTypes() ); @@ -540,4 +545,22 @@ private function compareSchemas(Comparator $comparator, Schema $from, Schema $to ? $comparator->compareSchemas($from, $to) : $comparator->compare($from, $to); } + + /** + * @throws DBALException + */ + public function addSkipLocked(string $sql): string + { + switch (true) { + case $this->driverConnection->getDatabasePlatform() instanceof MySQL80Platform: + case $this->driverConnection->getDatabasePlatform() instanceof PostgreSQLPlatform: + $sql .= ' SKIP LOCKED'; // use skip locked so workers pulling in parallel don't wait on each other + break; + case $this->driverConnection->getDatabasePlatform() instanceof SQLServerPlatform: + $sql = str_replace('(UPDLOCK, ROWLOCK)', '(UPDLOCK, ROWLOCK, READPAST)', $sql); + break; + } + + return $sql; + } } diff --git a/src/Symfony/Component/Messenger/CHANGELOG.md b/src/Symfony/Component/Messenger/CHANGELOG.md index aabf140f6ce73..8bb52272dd9ad 100644 --- a/src/Symfony/Component/Messenger/CHANGELOG.md +++ b/src/Symfony/Component/Messenger/CHANGELOG.md @@ -1,6 +1,11 @@ CHANGELOG ========= +6.4 +--- + + * Use `SKIP LOCKED` in the doctrine transport for MySQL, PostgreSQL and MSSQL + 6.3 --- From 0dae97fb2934a386c620f262e3bfcadfa57a8845 Mon Sep 17 00:00:00 2001 From: Herberto Graca Date: Sun, 9 Jul 2023 22:32:03 +0200 Subject: [PATCH 07/11] Add batch delivery Currently, the doctrine transport only delivers one message at a time, resulting in low performance and a very high amount of hits to the DB. (one hit per message) With batch delivery, the transport will retrieve several messages in a single query. Co-authored-by: Alexander Malyk --- .../Tests/Transport/ConnectionTest.php | 121 ++++++++++++++---- .../Transport/DoctrineIntegrationTest.php | 16 ++- .../Tests/Transport/DoctrineReceiverTest.php | 4 +- .../Tests/Transport/DoctrineTransportTest.php | 2 +- .../Bridge/Doctrine/Transport/Connection.php | 109 +++++++++++++--- .../Doctrine/Transport/DoctrineReceiver.php | 17 ++- .../Doctrine/Transport/DoctrineTransport.php | 14 ++ .../Transport/PostgreSqlConnection.php | 4 +- src/Symfony/Component/Messenger/CHANGELOG.md | 1 + 9 files changed, 232 insertions(+), 56 deletions(-) diff --git a/src/Symfony/Component/Messenger/Bridge/Doctrine/Tests/Transport/ConnectionTest.php b/src/Symfony/Component/Messenger/Bridge/Doctrine/Tests/Transport/ConnectionTest.php index 4de01c8b88fef..db89c201fda0b 100644 --- a/src/Symfony/Component/Messenger/Bridge/Doctrine/Tests/Transport/ConnectionTest.php +++ b/src/Symfony/Component/Messenger/Bridge/Doctrine/Tests/Transport/ConnectionTest.php @@ -21,6 +21,7 @@ use Doctrine\DBAL\Platforms\OraclePlatform; use Doctrine\DBAL\Platforms\PostgreSQLPlatform; use Doctrine\DBAL\Platforms\SQLServer2012Platform; +use Doctrine\DBAL\Query\Expression\ExpressionBuilder; use Doctrine\DBAL\Query\QueryBuilder; use Doctrine\DBAL\Result; use Doctrine\DBAL\Schema\AbstractSchemaManager; @@ -38,11 +39,19 @@ public function testGetAMessageWillChangeItsStatus() { $queryBuilder = $this->getQueryBuilderMock(); $driverConnection = $this->getDBALConnectionMock(); - $stmt = $this->getResultMock([ - 'id' => 1, - 'body' => '{"message":"Hi"}', - 'headers' => json_encode(['type' => DummyMessage::class]), - ]); + $resultMock = [ + [ + 'id' => 1, + 'body' => '{"message":"Hi"}', + 'headers' => json_encode(['type' => DummyMessage::class]), + ], + [ + 'id' => 2, + 'body' => '{"message":"There"}', + 'headers' => json_encode(['type' => DummyMessage::class]), + ] + ]; + $stmt = $this->getResultMockForGet($resultMock); $driverConnection ->method('createQueryBuilder') @@ -64,17 +73,19 @@ public function testGetAMessageWillChangeItsStatus() ->willReturn(1); $connection = new Connection([], $driverConnection); - $doctrineEnvelope = $connection->get(); - $this->assertEquals(1, $doctrineEnvelope['id']); - $this->assertEquals('{"message":"Hi"}', $doctrineEnvelope['body']); - $this->assertEquals(['type' => DummyMessage::class], $doctrineEnvelope['headers']); + $doctrineEnvelopeList = $connection->get(); + foreach ($doctrineEnvelopeList as $key => $doctrineEnvelope) { + $this->assertEquals($resultMock[$key]['id'], $doctrineEnvelope['id']); + $this->assertEquals($resultMock[$key]['body'], $doctrineEnvelope['body']); + $this->assertEquals(json_decode($resultMock[$key]['headers'], true), $doctrineEnvelope['headers']); + } } - public function testGetWithNoPendingMessageWillReturnNull() + public function testGetWithNoPendingMessageWillReturnEmptyList() { $queryBuilder = $this->getQueryBuilderMock(); $driverConnection = $this->getDBALConnectionMock(); - $stmt = $this->getResultMock(false); + $stmt = $this->getResultMockForGet([]); $queryBuilder ->method('getParameters') @@ -92,8 +103,9 @@ public function testGetWithNoPendingMessageWillReturnNull() ->willReturn($stmt); $connection = new Connection([], $driverConnection); - $doctrineEnvelope = $connection->get(); - $this->assertNull($doctrineEnvelope); + $doctrineEnvelopeList = $connection->get(); + $this->assertIsArray($doctrineEnvelopeList); + $this->assertEmpty($doctrineEnvelopeList); } public function testItThrowsATransportExceptionIfItCannotAcknowledgeMessage() @@ -153,11 +165,27 @@ private function getQueryBuilderMock() $queryBuilder->method('setMaxResults')->willReturn($queryBuilder); $queryBuilder->method('setParameter')->willReturn($queryBuilder); $queryBuilder->method('setParameters')->willReturn($queryBuilder); + $queryBuilder->method('addOrderBy')->willReturn($queryBuilder); + + $expressionBuilderMock = $this->createMock(ExpressionBuilder::class); + $expressionBuilderMock->method('in')->willReturn(''); + $queryBuilder->method('expr')->willReturn($expressionBuilderMock); return $queryBuilder; } - private function getResultMock($expectedResult) + private function getResultMockForGet($expectedResult) + { + $stmt = $this->createMock(class_exists(Result::class) ? Result::class : ResultStatement::class); + + $stmt->expects($this->once()) + ->method(class_exists(Result::class) ? 'fetchAllAssociative' : 'fetchAll') + ->willReturn($expectedResult); + + return $stmt; + } + + private function getResultMockForFind($expectedResult) { $stmt = $this->createMock(class_exists(Result::class) ? Result::class : ResultStatement::class); @@ -257,14 +285,14 @@ public static function buildConfigurationProvider(): iterable public function testItThrowsAnExceptionIfAnExtraOptionsInDefined() { $this->expectException(InvalidArgumentException::class); - $this->expectExceptionMessage('Unknown option found: [new_option]. Allowed options are [table_name, queue_name, redeliver_timeout, auto_setup]'); + $this->expectExceptionMessage('Unknown option found: [new_option]. Allowed options are [table_name, queue_name, batch_size, redeliver_timeout, auto_setup]'); Connection::buildConfiguration('doctrine://default', ['new_option' => 'woops']); } public function testItThrowsAnExceptionIfAnExtraOptionsInDefinedInDSN() { $this->expectException(InvalidArgumentException::class); - $this->expectExceptionMessage('Unknown option found in DSN: [new_option]. Allowed options are [table_name, queue_name, redeliver_timeout, auto_setup]'); + $this->expectExceptionMessage('Unknown option found in DSN: [new_option]. Allowed options are [table_name, queue_name, batch_size, redeliver_timeout, auto_setup]'); Connection::buildConfiguration('doctrine://default?new_option=woops'); } @@ -273,7 +301,7 @@ public function testFind() $queryBuilder = $this->getQueryBuilderMock(); $driverConnection = $this->getDBALConnectionMock(); $id = 1; - $stmt = $this->getResultMock([ + $stmt = $this->getResultMockForFind([ 'id' => $id, 'body' => '{"message":"Hi"}', 'headers' => json_encode(['type' => DummyMessage::class]), @@ -357,7 +385,7 @@ public function testFindAll() /** * @dataProvider providePlatformSql */ - public function testGeneratedSql(AbstractPlatform $platform, string $expectedSql) + public function testGeneratedSql(AbstractPlatform $platform, int $batchSize, string $expectedSql) { $driverConnection = $this->createMock(DBALConnection::class); $driverConnection->method('getDatabasePlatform')->willReturn($platform); @@ -380,43 +408,84 @@ public function testGeneratedSql(AbstractPlatform $platform, string $expectedSql ; $driverConnection->expects($this->once())->method('commit'); - $connection = new Connection([], $driverConnection); + $connection = new Connection(['batch_size' => $batchSize], $driverConnection); $connection->get(); } public static function providePlatformSql(): iterable { - yield 'MySQL57' => [ + yield 'MySQL57|batch_size=1' => [ new MySQL57Platform(), + 1, 'SELECT m.* FROM messenger_messages m WHERE (m.delivered_at is null OR m.delivered_at < ?) AND (m.available_at <= ?) AND (m.queue_name = ?) ORDER BY available_at ASC LIMIT 1 FOR UPDATE', ]; - yield 'MySQL8' => [ + yield 'MySQL57|batch_size=50' => [ + new MySQL57Platform(), + 50, + 'SELECT m.* FROM messenger_messages m WHERE (m.delivered_at is null OR m.delivered_at < ?) AND (m.available_at <= ?) AND (m.queue_name = ?) ORDER BY available_at ASC LIMIT 50 FOR UPDATE', + ]; + + yield 'MySQL8|batch_size=1' => [ new MySQL80Platform(), + 1, 'SELECT m.* FROM messenger_messages m WHERE (m.delivered_at is null OR m.delivered_at < ?) AND (m.available_at <= ?) AND (m.queue_name = ?) ORDER BY available_at ASC LIMIT 1 FOR UPDATE SKIP LOCKED', ]; - yield 'Postgres' => [ + yield 'MySQL8|batch_size=50' => [ + new MySQL80Platform(), + 50, + 'SELECT m.* FROM messenger_messages m WHERE (m.delivered_at is null OR m.delivered_at < ?) AND (m.available_at <= ?) AND (m.queue_name = ?) ORDER BY available_at ASC LIMIT 50 FOR UPDATE SKIP LOCKED', + ]; + + yield 'Postgres|batch_size=1' => [ new PostgreSQLPlatform(), + 1, 'SELECT m.* FROM messenger_messages m WHERE (m.delivered_at is null OR m.delivered_at < ?) AND (m.available_at <= ?) AND (m.queue_name = ?) ORDER BY available_at ASC LIMIT 1 FOR UPDATE SKIP LOCKED', ]; + yield 'Postgres|batch_size=50' => [ + new PostgreSQLPlatform(), + 50, + 'SELECT m.* FROM messenger_messages m WHERE (m.delivered_at is null OR m.delivered_at < ?) AND (m.available_at <= ?) AND (m.queue_name = ?) ORDER BY available_at ASC LIMIT 50 FOR UPDATE SKIP LOCKED', + ]; + if (class_exists(MariaDBPlatform::class)) { - yield 'MariaDB' => [ + yield 'MariaDB|batch_size=1' => [ new MariaDBPlatform(), + 1, 'SELECT m.* FROM messenger_messages m WHERE (m.delivered_at is null OR m.delivered_at < ?) AND (m.available_at <= ?) AND (m.queue_name = ?) ORDER BY available_at ASC LIMIT 1 FOR UPDATE', ]; + yield 'MariaDB|batch_size=50' => [ + new MariaDBPlatform(), + 50, + 'SELECT m.* FROM messenger_messages m WHERE (m.delivered_at is null OR m.delivered_at < ?) AND (m.available_at <= ?) AND (m.queue_name = ?) ORDER BY available_at ASC LIMIT 50 FOR UPDATE', + ]; } - yield 'SQL Server' => [ + yield 'SQL Server|batch_size=1' => [ new SQLServer2012Platform(), - 'SELECT m.* FROM messenger_messages m WITH (UPDLOCK, ROWLOCK, READPAST) WHERE (m.delivered_at is null OR m.delivered_at < ?) AND (m.available_at <= ?) AND (m.queue_name = ?) ORDER BY available_at ASC OFFSET 0 ROWS FETCH NEXT 1 ROWS ONLY ', + 1, + 'SELECT m.* FROM messenger_messages m WITH (UPDLOCK, ROWLOCK, READPAST) WHERE (m.delivered_at is null OR m.delivered_at < ?) AND (m.available_at <= ?) AND (m.queue_name = ?) ORDER BY available_at ASC OFFSET 0 ROWS FETCH NEXT 1 ROWS ONLY ', ]; - yield 'Oracle' => [ + yield 'SQL Server|batch_size=50' => [ + new SQLServer2012Platform(), + 50, + 'SELECT m.* FROM messenger_messages m WITH (UPDLOCK, ROWLOCK, READPAST) WHERE (m.delivered_at is null OR m.delivered_at < ?) AND (m.available_at <= ?) AND (m.queue_name = ?) ORDER BY available_at ASC OFFSET 0 ROWS FETCH NEXT 50 ROWS ONLY ', + ]; + + yield 'Oracle|batch_size=1' => [ new OraclePlatform(), + 1, 'SELECT w.id AS "id", w.body AS "body", w.headers AS "headers", w.queue_name AS "queue_name", w.created_at AS "created_at", w.available_at AS "available_at", w.delivered_at AS "delivered_at" FROM messenger_messages w WHERE w.id IN (SELECT a.id FROM (SELECT m.id FROM messenger_messages m WHERE (m.delivered_at is null OR m.delivered_at < ?) AND (m.available_at <= ?) AND (m.queue_name = ?) ORDER BY available_at ASC) a WHERE ROWNUM <= 1) FOR UPDATE', ]; + + yield 'Oracle|batch_size=50' => [ + new OraclePlatform(), + 50, + 'SELECT w.id AS "id", w.body AS "body", w.headers AS "headers", w.queue_name AS "queue_name", w.created_at AS "created_at", w.available_at AS "available_at", w.delivered_at AS "delivered_at" FROM messenger_messages w WHERE w.id IN (SELECT a.id FROM (SELECT m.id FROM messenger_messages m WHERE (m.delivered_at is null OR m.delivered_at < ?) AND (m.available_at <= ?) AND (m.queue_name = ?) ORDER BY available_at ASC) a WHERE ROWNUM <= 50) FOR UPDATE', + ]; } public function testConfigureSchema() diff --git a/src/Symfony/Component/Messenger/Bridge/Doctrine/Tests/Transport/DoctrineIntegrationTest.php b/src/Symfony/Component/Messenger/Bridge/Doctrine/Tests/Transport/DoctrineIntegrationTest.php index c9ff7c851e845..043e8db3b9ba0 100644 --- a/src/Symfony/Component/Messenger/Bridge/Doctrine/Tests/Transport/DoctrineIntegrationTest.php +++ b/src/Symfony/Component/Messenger/Bridge/Doctrine/Tests/Transport/DoctrineIntegrationTest.php @@ -52,9 +52,9 @@ protected function tearDown(): void public function testConnectionSendAndGet() { $this->connection->send('{"message": "Hi"}', ['type' => DummyMessage::class]); - $encoded = $this->connection->get(); - $this->assertEquals('{"message": "Hi"}', $encoded['body']); - $this->assertEquals(['type' => DummyMessage::class], $encoded['headers']); + $encodedList = $this->connection->get()[0]; + $this->assertEquals('{"message": "Hi"}', $encodedList['body']); + $this->assertEquals(['type' => DummyMessage::class], $encodedList['headers']); } public function testSendWithDelay() @@ -108,7 +108,7 @@ public function testItRetrieveTheFirstAvailableMessage() 'available_at' => $this->formatDateTime(new \DateTimeImmutable('2019-03-15 12:30:00', new \DateTimeZone('UTC'))), ]); - $encoded = $this->connection->get(); + $encoded = $this->connection->get()[0]; $this->assertEquals('{"message": "Hi available"}', $encoded['body']); } @@ -173,7 +173,7 @@ public function testItRetrieveTheMessageThatIsOlderThanRedeliverTimeout() 'available_at' => $this->formatDateTime(new \DateTimeImmutable('2019-03-15 12:30:00', new \DateTimeZone('UTC'))), ]); - $next = $this->connection->get(); + $next = $this->connection->get()[0]; $this->assertEquals('{"message": "Hi requeued"}', $next['body']); $this->connection->reject($next['id']); } @@ -181,10 +181,12 @@ public function testItRetrieveTheMessageThatIsOlderThanRedeliverTimeout() public function testTheTransportIsSetupOnGet() { $this->assertFalse($this->createSchemaManager()->tablesExist(['messenger_messages'])); - $this->assertNull($this->connection->get()); + $result = $this->connection->get(); + $this->assertIsArray($result); + $this->assertEmpty($result); $this->connection->send('the body', ['my' => 'header']); - $envelope = $this->connection->get(); + $envelope = $this->connection->get()[0]; $this->assertEquals('the body', $envelope['body']); } diff --git a/src/Symfony/Component/Messenger/Bridge/Doctrine/Tests/Transport/DoctrineReceiverTest.php b/src/Symfony/Component/Messenger/Bridge/Doctrine/Tests/Transport/DoctrineReceiverTest.php index d1e56dff17a44..01410708bd370 100644 --- a/src/Symfony/Component/Messenger/Bridge/Doctrine/Tests/Transport/DoctrineReceiverTest.php +++ b/src/Symfony/Component/Messenger/Bridge/Doctrine/Tests/Transport/DoctrineReceiverTest.php @@ -38,7 +38,7 @@ public function testItReturnsTheDecodedMessageToTheHandler() $doctrineEnvelope = $this->createDoctrineEnvelope(); $connection = $this->createMock(Connection::class); - $connection->method('get')->willReturn($doctrineEnvelope); + $connection->method('get')->willReturn([$doctrineEnvelope]); $receiver = new DoctrineReceiver($connection, $serializer); $actualEnvelopes = $receiver->get(); @@ -61,7 +61,7 @@ public function testItRejectTheMessageIfThereIsAMessageDecodingFailedException() $doctrineEnvelop = $this->createDoctrineEnvelope(); $connection = $this->createMock(Connection::class); - $connection->method('get')->willReturn($doctrineEnvelop); + $connection->method('get')->willReturn([$doctrineEnvelop]); $connection->expects($this->once())->method('reject'); $receiver = new DoctrineReceiver($connection, $serializer); diff --git a/src/Symfony/Component/Messenger/Bridge/Doctrine/Tests/Transport/DoctrineTransportTest.php b/src/Symfony/Component/Messenger/Bridge/Doctrine/Tests/Transport/DoctrineTransportTest.php index fea497d14409c..512b355873970 100644 --- a/src/Symfony/Component/Messenger/Bridge/Doctrine/Tests/Transport/DoctrineTransportTest.php +++ b/src/Symfony/Component/Messenger/Bridge/Doctrine/Tests/Transport/DoctrineTransportTest.php @@ -46,7 +46,7 @@ public function testReceivesMessages() ]; $serializer->method('decode')->with(['body' => 'body', 'headers' => ['my' => 'header']])->willReturn(new Envelope($decodedMessage)); - $connection->method('get')->willReturn($doctrineEnvelope); + $connection->method('get')->willReturn([$doctrineEnvelope]); $envelopes = $transport->get(); $this->assertSame($decodedMessage, $envelopes[0]->getMessage()); diff --git a/src/Symfony/Component/Messenger/Bridge/Doctrine/Transport/Connection.php b/src/Symfony/Component/Messenger/Bridge/Doctrine/Transport/Connection.php index 73c38e30aa3df..acf1281a74798 100644 --- a/src/Symfony/Component/Messenger/Bridge/Doctrine/Transport/Connection.php +++ b/src/Symfony/Component/Messenger/Bridge/Doctrine/Transport/Connection.php @@ -12,6 +12,7 @@ namespace Symfony\Component\Messenger\Bridge\Doctrine\Transport; use Doctrine\DBAL\Abstraction\Result as AbstractionResult; +use Doctrine\DBAL\ArrayParameterType; use Doctrine\DBAL\Connection as DBALConnection; use Doctrine\DBAL\Driver\Exception as DriverException; use Doctrine\DBAL\Driver\ResultStatement; @@ -51,6 +52,7 @@ class Connection implements ResetInterface protected const DEFAULT_OPTIONS = [ 'table_name' => 'messenger_messages', 'queue_name' => 'default', + 'batch_size' => 1, 'redeliver_timeout' => 3600, 'auto_setup' => true, ]; @@ -73,6 +75,8 @@ class Connection implements ResetInterface protected $queueEmptiedAt; private ?SchemaSynchronizer $schemaSynchronizer; private bool $autoSetup; + /** @var array */ + private array $lastDeliveredButNotYetHandledEnvelopesIds = []; public function __construct(array $configuration, DBALConnection $driverConnection, SchemaSynchronizer $schemaSynchronizer = null) { @@ -162,7 +166,7 @@ public function send(string $body, array $headers, int $delay = 0): string return $this->driverConnection->lastInsertId(); } - public function get(): ?array + public function get(): array { if ($this->driverConnection->getDatabasePlatform() instanceof MySQLPlatform) { try { @@ -177,7 +181,7 @@ public function get(): ?array try { $query = $this->createAvailableMessagesQueryBuilder() ->orderBy('available_at', 'ASC') - ->setMaxResults(1); + ->setMaxResults($this->configuration['batch_size']); if ($this->driverConnection->getDatabasePlatform() instanceof OraclePlatform) { $query->select('m.id'); @@ -211,36 +215,41 @@ public function get(): ?array $query->getParameters(), $query->getParameterTypes() ); - $doctrineEnvelope = $stmt instanceof Result ? $stmt->fetchAssociative() : $stmt->fetch(); + $doctrineEnvelopeList = $stmt instanceof Result ? $stmt->fetchAllAssociative() : $stmt->fetchAll(); - if (false === $doctrineEnvelope) { + if ([] === $doctrineEnvelopeList) { $this->driverConnection->commit(); $this->queueEmptiedAt = microtime(true) * 1000; - return null; + return []; } // Postgres can "group" notifications having the same channel and payload // We need to be sure to empty the queue before blocking again $this->queueEmptiedAt = null; - $doctrineEnvelope = $this->decodeEnvelopeHeaders($doctrineEnvelope); + $this->lastDeliveredButNotYetHandledEnvelopesIds = $this->extractDoctrineEnvelopeListIds($doctrineEnvelopeList); $queryBuilder = $this->driverConnection->createQueryBuilder() ->update($this->configuration['table_name']) ->set('delivered_at', '?') - ->where('id = ?'); - $now = new \DateTimeImmutable('UTC'); - $this->executeStatement($queryBuilder->getSQL(), [ - $now, - $doctrineEnvelope['id'], - ], [ - Types::DATETIME_IMMUTABLE, - ]); + ->where($this->driverConnection->createQueryBuilder()->expr()->in('id', '?')); + $this->executeStatement( + $queryBuilder->getSQL(), + [ + new \DateTimeImmutable('UTC'), + $this->lastDeliveredButNotYetHandledEnvelopesIds, + ], + [ + Types::DATETIME_IMMUTABLE, + ArrayParameterType::INTEGER, + ] + ); $this->driverConnection->commit(); - return $doctrineEnvelope; + return $this->decodeDoctrineEnvelopeListHeaders($doctrineEnvelopeList); } catch (\Throwable $e) { + $this->lastDeliveredButNotYetHandledEnvelopesIds = []; $this->driverConnection->rollBack(); if ($this->autoSetup && $e instanceof TableNotFoundException) { @@ -254,6 +263,7 @@ public function get(): ?array public function ack(string $id): bool { + $this->removeHandledFromDeliveredList($id); try { if ($this->driverConnection->getDatabasePlatform() instanceof MySQLPlatform) { return $this->driverConnection->update($this->configuration['table_name'], ['delivered_at' => self::FLAGGED_FOR_DELETION], ['id' => $id]) > 0; @@ -348,6 +358,27 @@ public function getExtraSetupSqlForTable(Table $createdTable): array return []; } + public function undeliverNotHandled(): void + { + if (count($this->lastDeliveredButNotYetHandledEnvelopesIds) === 0) { + return; + } + + $queryBuilder = $this->driverConnection->createQueryBuilder() + ->update($this->configuration['table_name']) + ->set('delivered_at', 'NULL') + ->where($this->driverConnection->createQueryBuilder()->expr()->in('id', '?')); + $this->executeStatement( + $queryBuilder->getSQL(), + [ + $this->lastDeliveredButNotYetHandledEnvelopesIds, + ], + [ + ArrayParameterType::INTEGER, + ] + ); + } + private function createAvailableMessagesQueryBuilder(): QueryBuilder { $now = new \DateTimeImmutable('UTC'); @@ -563,4 +594,52 @@ public function addSkipLocked(string $sql): string return $sql; } + + /** + * @return int[] + */ + public function extractDoctrineEnvelopeListIds(array $doctrineEnvelopeList): array + { + return array_map( + function ($doctrineEnvelopeRow) { + return $doctrineEnvelopeRow['id']; + }, + $doctrineEnvelopeList + ); + } + + /** + * @param array $doctrineEnvelopeList + * + * @return array + */ + public function decodeDoctrineEnvelopeListHeaders(array $doctrineEnvelopeList): array + { + $doctrineEnvelopeListWithDecodedHeaders = []; + foreach ($doctrineEnvelopeList as $doctrineEnvelope) { + $doctrineEnvelopeListWithDecodedHeaders[] = $this->decodeEnvelopeHeaders($doctrineEnvelope); + } + return $doctrineEnvelopeListWithDecodedHeaders; + } + + public function removeHandledFromDeliveredList(string $id): void + { + $this->lastDeliveredButNotYetHandledEnvelopesIds = array_diff($this->lastDeliveredButNotYetHandledEnvelopesIds, [$id]); + } } diff --git a/src/Symfony/Component/Messenger/Bridge/Doctrine/Transport/DoctrineReceiver.php b/src/Symfony/Component/Messenger/Bridge/Doctrine/Transport/DoctrineReceiver.php index d7ab930b0a987..73ee4ed995196 100644 --- a/src/Symfony/Component/Messenger/Bridge/Doctrine/Transport/DoctrineReceiver.php +++ b/src/Symfony/Component/Messenger/Bridge/Doctrine/Transport/DoctrineReceiver.php @@ -26,6 +26,7 @@ /** * @author Vincent Touzet * @author Herberto Graca + * @author Alexander Malyk */ class DoctrineReceiver implements ListableReceiverInterface, MessageCountAwareInterface { @@ -43,7 +44,7 @@ public function __construct(Connection $connection, SerializerInterface $seriali public function get(): iterable { try { - $doctrineEnvelope = $this->connection->get(); + $doctrineEnvelopeList = $this->connection->get(); $this->retryingSafetyCounter = 0; // reset counter } catch (RetryableException $exception) { // Do nothing when RetryableException occurs less than "MAX_RETRIES" @@ -59,11 +60,16 @@ public function get(): iterable throw new TransportException($exception->getMessage(), 0, $exception); } - if (null === $doctrineEnvelope) { + if (null === $doctrineEnvelopeList) { return []; } - return [$this->createEnvelopeFromData($doctrineEnvelope)]; + $envelopeList = []; + foreach ($doctrineEnvelopeList as $doctrineEnvelope) { + $envelopeList[] = $this->createEnvelopeFromData($doctrineEnvelope); + } + + return $envelopeList; } public function ack(Envelope $envelope): void @@ -84,6 +90,11 @@ public function reject(Envelope $envelope): void } } + public function undeliverNotHandled(): void + { + $this->connection->undeliverNotHandled(); + } + public function getMessageCount(): int { try { diff --git a/src/Symfony/Component/Messenger/Bridge/Doctrine/Transport/DoctrineTransport.php b/src/Symfony/Component/Messenger/Bridge/Doctrine/Transport/DoctrineTransport.php index dac4dd538b731..3e15f5fa0f3a9 100644 --- a/src/Symfony/Component/Messenger/Bridge/Doctrine/Transport/DoctrineTransport.php +++ b/src/Symfony/Component/Messenger/Bridge/Doctrine/Transport/DoctrineTransport.php @@ -23,6 +23,8 @@ /** * @author Vincent Touzet + * @author Herberto Graca + * @author Alexander Malyk */ class DoctrineTransport implements TransportInterface, SetupableTransportInterface, MessageCountAwareInterface, ListableReceiverInterface { @@ -37,6 +39,18 @@ public function __construct(Connection $connection, SerializerInterface $seriali $this->serializer = $serializer; } + public function __destruct() + { + // The worker using this transport might have pulled 50 msgs out of the mq, marking them as "in process", + // but because of its options (ie --limit, --failure-limit, --memory-limit, --time-limit) it might terminate + // before it actually handles them all, leaving messages in limbo where they will not be handled by the + // consumer that pulled them, and they won't be picked up by any other consumer because they are + // "in process" already. + // Thus, when the consumer stops, and its transport gets destroyed, we need to put the messages not handled, + // back in "waiting to be processed" so that they can be picked up by other consumers. + $this->getReceiver()->undeliverNotHandled(); + } + public function get(): iterable { return $this->getReceiver()->get(); diff --git a/src/Symfony/Component/Messenger/Bridge/Doctrine/Transport/PostgreSqlConnection.php b/src/Symfony/Component/Messenger/Bridge/Doctrine/Transport/PostgreSqlConnection.php index 2adb31be54b7b..201ce8f83f721 100644 --- a/src/Symfony/Component/Messenger/Bridge/Doctrine/Transport/PostgreSqlConnection.php +++ b/src/Symfony/Component/Messenger/Bridge/Doctrine/Transport/PostgreSqlConnection.php @@ -57,7 +57,7 @@ public function reset(): void $this->unlisten(); } - public function get(): ?array + public function get(): array { if (null === $this->queueEmptiedAt) { return parent::get(); @@ -85,7 +85,7 @@ public function get(): ?array ) { usleep(1000); - return null; + return []; } return parent::get(); diff --git a/src/Symfony/Component/Messenger/CHANGELOG.md b/src/Symfony/Component/Messenger/CHANGELOG.md index 8bb52272dd9ad..8bf098e6fc21d 100644 --- a/src/Symfony/Component/Messenger/CHANGELOG.md +++ b/src/Symfony/Component/Messenger/CHANGELOG.md @@ -4,6 +4,7 @@ CHANGELOG 6.4 --- + * Add support for the doctrine transport to pull a batch of messages in one go (`batch_size` option) * Use `SKIP LOCKED` in the doctrine transport for MySQL, PostgreSQL and MSSQL 6.3 From 54360a38159c4b9295abfa531b7aefb1abb7ba77 Mon Sep 17 00:00:00 2001 From: Herberto Graca Date: Fri, 14 Jul 2023 00:06:47 +0200 Subject: [PATCH 08/11] fixup! Replace usage of DoctrineReceivedStamp --- .../Bridge/Doctrine/Tests/Transport/DoctrineReceiverTest.php | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Symfony/Component/Messenger/Bridge/Doctrine/Tests/Transport/DoctrineReceiverTest.php b/src/Symfony/Component/Messenger/Bridge/Doctrine/Tests/Transport/DoctrineReceiverTest.php index 01410708bd370..1391408eb3788 100644 --- a/src/Symfony/Component/Messenger/Bridge/Doctrine/Tests/Transport/DoctrineReceiverTest.php +++ b/src/Symfony/Component/Messenger/Bridge/Doctrine/Tests/Transport/DoctrineReceiverTest.php @@ -18,7 +18,6 @@ use PHPUnit\Framework\TestCase; use Symfony\Component\Messenger\Bridge\Doctrine\Tests\Fixtures\DummyMessage; use Symfony\Component\Messenger\Bridge\Doctrine\Transport\Connection; -use Symfony\Component\Messenger\Bridge\Doctrine\Transport\DoctrineReceivedStamp; use Symfony\Component\Messenger\Bridge\Doctrine\Transport\DoctrineReceiver; use Symfony\Component\Messenger\Envelope; use Symfony\Component\Messenger\Exception\MessageDecodingFailedException; From 0f126f9944d2052f4aac645f6f24dcee35d3da7d Mon Sep 17 00:00:00 2001 From: Herberto Graca Date: Thu, 13 Jul 2023 23:58:55 +0200 Subject: [PATCH 09/11] fixup! Add "SKIP LOCKED" to the query that retrieves messages --- .../Bridge/Doctrine/Tests/Transport/ConnectionTest.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Symfony/Component/Messenger/Bridge/Doctrine/Tests/Transport/ConnectionTest.php b/src/Symfony/Component/Messenger/Bridge/Doctrine/Tests/Transport/ConnectionTest.php index db89c201fda0b..2d34757b5ebbd 100644 --- a/src/Symfony/Component/Messenger/Bridge/Doctrine/Tests/Transport/ConnectionTest.php +++ b/src/Symfony/Component/Messenger/Bridge/Doctrine/Tests/Transport/ConnectionTest.php @@ -466,7 +466,7 @@ public static function providePlatformSql(): iterable yield 'SQL Server|batch_size=1' => [ new SQLServer2012Platform(), 1, - 'SELECT m.* FROM messenger_messages m WITH (UPDLOCK, ROWLOCK, READPAST) WHERE (m.delivered_at is null OR m.delivered_at < ?) AND (m.available_at <= ?) AND (m.queue_name = ?) ORDER BY available_at ASC OFFSET 0 ROWS FETCH NEXT 1 ROWS ONLY ', + 'SELECT m.* FROM messenger_messages m WITH (UPDLOCK, ROWLOCK, READPAST) WHERE (m.delivered_at is null OR m.delivered_at < ?) AND (m.available_at <= ?) AND (m.queue_name = ?) ORDER BY available_at ASC OFFSET 0 ROWS FETCH NEXT 1 ROWS ONLY ', ]; yield 'SQL Server|batch_size=50' => [ From 2013bdcee3719b8dfe4f09f6de800744c12dc294 Mon Sep 17 00:00:00 2001 From: Herberto Graca Date: Thu, 13 Jul 2023 23:59:24 +0200 Subject: [PATCH 10/11] fixup! Add batch delivery --- .../Bridge/Doctrine/Tests/Transport/ConnectionTest.php | 2 +- .../Tests/Transport/DoctrinePostgreSqlIntegrationTest.php | 4 ++-- .../Messenger/Bridge/Doctrine/Transport/Connection.php | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Symfony/Component/Messenger/Bridge/Doctrine/Tests/Transport/ConnectionTest.php b/src/Symfony/Component/Messenger/Bridge/Doctrine/Tests/Transport/ConnectionTest.php index 2d34757b5ebbd..577746eda717f 100644 --- a/src/Symfony/Component/Messenger/Bridge/Doctrine/Tests/Transport/ConnectionTest.php +++ b/src/Symfony/Component/Messenger/Bridge/Doctrine/Tests/Transport/ConnectionTest.php @@ -49,7 +49,7 @@ public function testGetAMessageWillChangeItsStatus() 'id' => 2, 'body' => '{"message":"There"}', 'headers' => json_encode(['type' => DummyMessage::class]), - ] + ], ]; $stmt = $this->getResultMockForGet($resultMock); diff --git a/src/Symfony/Component/Messenger/Bridge/Doctrine/Tests/Transport/DoctrinePostgreSqlIntegrationTest.php b/src/Symfony/Component/Messenger/Bridge/Doctrine/Tests/Transport/DoctrinePostgreSqlIntegrationTest.php index 63fe3ebd4b513..969b58c3adb0f 100644 --- a/src/Symfony/Component/Messenger/Bridge/Doctrine/Tests/Transport/DoctrinePostgreSqlIntegrationTest.php +++ b/src/Symfony/Component/Messenger/Bridge/Doctrine/Tests/Transport/DoctrinePostgreSqlIntegrationTest.php @@ -61,11 +61,11 @@ public function testPostgreSqlConnectionSendAndGet() { $this->connection->send('{"message": "Hi"}', ['type' => DummyMessage::class]); - $encoded = $this->connection->get(); + $encoded = $this->connection->get()[0]; $this->assertEquals('{"message": "Hi"}', $encoded['body']); $this->assertEquals(['type' => DummyMessage::class], $encoded['headers']); - $this->assertNull($this->connection->get()); + $this->assertEmpty($this->connection->get()); } private function createSchemaManager(): AbstractSchemaManager diff --git a/src/Symfony/Component/Messenger/Bridge/Doctrine/Transport/Connection.php b/src/Symfony/Component/Messenger/Bridge/Doctrine/Transport/Connection.php index acf1281a74798..86c7277897fe0 100644 --- a/src/Symfony/Component/Messenger/Bridge/Doctrine/Transport/Connection.php +++ b/src/Symfony/Component/Messenger/Bridge/Doctrine/Transport/Connection.php @@ -360,7 +360,7 @@ public function getExtraSetupSqlForTable(Table $createdTable): array public function undeliverNotHandled(): void { - if (count($this->lastDeliveredButNotYetHandledEnvelopesIds) === 0) { + if (0 === \count($this->lastDeliveredButNotYetHandledEnvelopesIds)) { return; } From 8a4e1a7815084d53b65204f8d232a9c466186469 Mon Sep 17 00:00:00 2001 From: Herberto Graca Date: Fri, 14 Jul 2023 00:07:04 +0200 Subject: [PATCH 11/11] Remove obsolete comment --- .../Messenger/Bridge/Doctrine/Transport/DoctrineTransport.php | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/Symfony/Component/Messenger/Bridge/Doctrine/Transport/DoctrineTransport.php b/src/Symfony/Component/Messenger/Bridge/Doctrine/Transport/DoctrineTransport.php index 3e15f5fa0f3a9..71e015aa06a4d 100644 --- a/src/Symfony/Component/Messenger/Bridge/Doctrine/Transport/DoctrineTransport.php +++ b/src/Symfony/Component/Messenger/Bridge/Doctrine/Transport/DoctrineTransport.php @@ -93,8 +93,6 @@ public function setup(): void /** * Adds the Table to the Schema if this transport uses this connection. - * - * @param \Closure $isSameDatabase */ public function configureSchema(Schema $schema, DbalConnection $forConnection/* , \Closure $isSameDatabase */): void {