diff --git a/src/Symfony/Component/Messenger/Bridge/Doctrine/Tests/Transport/ConnectionTest.php b/src/Symfony/Component/Messenger/Bridge/Doctrine/Tests/Transport/ConnectionTest.php index 3dae29a4cbd66..577746eda717f 100644 --- a/src/Symfony/Component/Messenger/Bridge/Doctrine/Tests/Transport/ConnectionTest.php +++ b/src/Symfony/Component/Messenger/Bridge/Doctrine/Tests/Transport/ConnectionTest.php @@ -17,8 +17,11 @@ use Doctrine\DBAL\Platforms\AbstractPlatform; use Doctrine\DBAL\Platforms\MariaDBPlatform; use Doctrine\DBAL\Platforms\MySQL57Platform; +use Doctrine\DBAL\Platforms\MySQL80Platform; use Doctrine\DBAL\Platforms\OraclePlatform; +use Doctrine\DBAL\Platforms\PostgreSQLPlatform; use Doctrine\DBAL\Platforms\SQLServer2012Platform; +use Doctrine\DBAL\Query\Expression\ExpressionBuilder; use Doctrine\DBAL\Query\QueryBuilder; use Doctrine\DBAL\Result; use Doctrine\DBAL\Schema\AbstractSchemaManager; @@ -36,11 +39,19 @@ public function testGetAMessageWillChangeItsStatus() { $queryBuilder = $this->getQueryBuilderMock(); $driverConnection = $this->getDBALConnectionMock(); - $stmt = $this->getResultMock([ - 'id' => 1, - 'body' => '{"message":"Hi"}', - 'headers' => json_encode(['type' => DummyMessage::class]), - ]); + $resultMock = [ + [ + 'id' => 1, + 'body' => '{"message":"Hi"}', + 'headers' => json_encode(['type' => DummyMessage::class]), + ], + [ + 'id' => 2, + 'body' => '{"message":"There"}', + 'headers' => json_encode(['type' => DummyMessage::class]), + ], + ]; + $stmt = $this->getResultMockForGet($resultMock); $driverConnection ->method('createQueryBuilder') @@ -62,17 +73,19 @@ public function testGetAMessageWillChangeItsStatus() ->willReturn(1); $connection = new Connection([], $driverConnection); - $doctrineEnvelope = $connection->get(); - $this->assertEquals(1, $doctrineEnvelope['id']); - $this->assertEquals('{"message":"Hi"}', $doctrineEnvelope['body']); - $this->assertEquals(['type' => DummyMessage::class], $doctrineEnvelope['headers']); + $doctrineEnvelopeList = $connection->get(); + foreach ($doctrineEnvelopeList as $key => $doctrineEnvelope) { + $this->assertEquals($resultMock[$key]['id'], $doctrineEnvelope['id']); + $this->assertEquals($resultMock[$key]['body'], $doctrineEnvelope['body']); + $this->assertEquals(json_decode($resultMock[$key]['headers'], true), $doctrineEnvelope['headers']); + } } - public function testGetWithNoPendingMessageWillReturnNull() + public function testGetWithNoPendingMessageWillReturnEmptyList() { $queryBuilder = $this->getQueryBuilderMock(); $driverConnection = $this->getDBALConnectionMock(); - $stmt = $this->getResultMock(false); + $stmt = $this->getResultMockForGet([]); $queryBuilder ->method('getParameters') @@ -90,8 +103,9 @@ public function testGetWithNoPendingMessageWillReturnNull() ->willReturn($stmt); $connection = new Connection([], $driverConnection); - $doctrineEnvelope = $connection->get(); - $this->assertNull($doctrineEnvelope); + $doctrineEnvelopeList = $connection->get(); + $this->assertIsArray($doctrineEnvelopeList); + $this->assertEmpty($doctrineEnvelopeList); } public function testItThrowsATransportExceptionIfItCannotAcknowledgeMessage() @@ -151,11 +165,27 @@ private function getQueryBuilderMock() $queryBuilder->method('setMaxResults')->willReturn($queryBuilder); $queryBuilder->method('setParameter')->willReturn($queryBuilder); $queryBuilder->method('setParameters')->willReturn($queryBuilder); + $queryBuilder->method('addOrderBy')->willReturn($queryBuilder); + + $expressionBuilderMock = $this->createMock(ExpressionBuilder::class); + $expressionBuilderMock->method('in')->willReturn(''); + $queryBuilder->method('expr')->willReturn($expressionBuilderMock); return $queryBuilder; } - private function getResultMock($expectedResult) + private function getResultMockForGet($expectedResult) + { + $stmt = $this->createMock(class_exists(Result::class) ? Result::class : ResultStatement::class); + + $stmt->expects($this->once()) + ->method(class_exists(Result::class) ? 'fetchAllAssociative' : 'fetchAll') + ->willReturn($expectedResult); + + return $stmt; + } + + private function getResultMockForFind($expectedResult) { $stmt = $this->createMock(class_exists(Result::class) ? Result::class : ResultStatement::class); @@ -255,14 +285,14 @@ public static function buildConfigurationProvider(): iterable public function testItThrowsAnExceptionIfAnExtraOptionsInDefined() { $this->expectException(InvalidArgumentException::class); - $this->expectExceptionMessage('Unknown option found: [new_option]. Allowed options are [table_name, queue_name, redeliver_timeout, auto_setup]'); + $this->expectExceptionMessage('Unknown option found: [new_option]. Allowed options are [table_name, queue_name, batch_size, redeliver_timeout, auto_setup]'); Connection::buildConfiguration('doctrine://default', ['new_option' => 'woops']); } public function testItThrowsAnExceptionIfAnExtraOptionsInDefinedInDSN() { $this->expectException(InvalidArgumentException::class); - $this->expectExceptionMessage('Unknown option found in DSN: [new_option]. Allowed options are [table_name, queue_name, redeliver_timeout, auto_setup]'); + $this->expectExceptionMessage('Unknown option found in DSN: [new_option]. Allowed options are [table_name, queue_name, batch_size, redeliver_timeout, auto_setup]'); Connection::buildConfiguration('doctrine://default?new_option=woops'); } @@ -271,7 +301,7 @@ public function testFind() $queryBuilder = $this->getQueryBuilderMock(); $driverConnection = $this->getDBALConnectionMock(); $id = 1; - $stmt = $this->getResultMock([ + $stmt = $this->getResultMockForFind([ 'id' => $id, 'body' => '{"message":"Hi"}', 'headers' => json_encode(['type' => DummyMessage::class]), @@ -355,7 +385,7 @@ public function testFindAll() /** * @dataProvider providePlatformSql */ - public function testGeneratedSql(AbstractPlatform $platform, string $expectedSql) + public function testGeneratedSql(AbstractPlatform $platform, int $batchSize, string $expectedSql) { $driverConnection = $this->createMock(DBALConnection::class); $driverConnection->method('getDatabasePlatform')->willReturn($platform); @@ -378,33 +408,84 @@ public function testGeneratedSql(AbstractPlatform $platform, string $expectedSql ; $driverConnection->expects($this->once())->method('commit'); - $connection = new Connection([], $driverConnection); + $connection = new Connection(['batch_size' => $batchSize], $driverConnection); $connection->get(); } public static function providePlatformSql(): iterable { - yield 'MySQL' => [ + yield 'MySQL57|batch_size=1' => [ new MySQL57Platform(), + 1, 'SELECT m.* FROM messenger_messages m WHERE (m.delivered_at is null OR m.delivered_at < ?) AND (m.available_at <= ?) AND (m.queue_name = ?) ORDER BY available_at ASC LIMIT 1 FOR UPDATE', ]; + yield 'MySQL57|batch_size=50' => [ + new MySQL57Platform(), + 50, + 'SELECT m.* FROM messenger_messages m WHERE (m.delivered_at is null OR m.delivered_at < ?) AND (m.available_at <= ?) AND (m.queue_name = ?) ORDER BY available_at ASC LIMIT 50 FOR UPDATE', + ]; + + yield 'MySQL8|batch_size=1' => [ + new MySQL80Platform(), + 1, + 'SELECT m.* FROM messenger_messages m WHERE (m.delivered_at is null OR m.delivered_at < ?) AND (m.available_at <= ?) AND (m.queue_name = ?) ORDER BY available_at ASC LIMIT 1 FOR UPDATE SKIP LOCKED', + ]; + + yield 'MySQL8|batch_size=50' => [ + new MySQL80Platform(), + 50, + 'SELECT m.* FROM messenger_messages m WHERE (m.delivered_at is null OR m.delivered_at < ?) AND (m.available_at <= ?) AND (m.queue_name = ?) ORDER BY available_at ASC LIMIT 50 FOR UPDATE SKIP LOCKED', + ]; + + yield 'Postgres|batch_size=1' => [ + new PostgreSQLPlatform(), + 1, + 'SELECT m.* FROM messenger_messages m WHERE (m.delivered_at is null OR m.delivered_at < ?) AND (m.available_at <= ?) AND (m.queue_name = ?) ORDER BY available_at ASC LIMIT 1 FOR UPDATE SKIP LOCKED', + ]; + + yield 'Postgres|batch_size=50' => [ + new PostgreSQLPlatform(), + 50, + 'SELECT m.* FROM messenger_messages m WHERE (m.delivered_at is null OR m.delivered_at < ?) AND (m.available_at <= ?) AND (m.queue_name = ?) ORDER BY available_at ASC LIMIT 50 FOR UPDATE SKIP LOCKED', + ]; + if (class_exists(MariaDBPlatform::class)) { - yield 'MariaDB' => [ + yield 'MariaDB|batch_size=1' => [ new MariaDBPlatform(), + 1, 'SELECT m.* FROM messenger_messages m WHERE (m.delivered_at is null OR m.delivered_at < ?) AND (m.available_at <= ?) AND (m.queue_name = ?) ORDER BY available_at ASC LIMIT 1 FOR UPDATE', ]; + yield 'MariaDB|batch_size=50' => [ + new MariaDBPlatform(), + 50, + 'SELECT m.* FROM messenger_messages m WHERE (m.delivered_at is null OR m.delivered_at < ?) AND (m.available_at <= ?) AND (m.queue_name = ?) ORDER BY available_at ASC LIMIT 50 FOR UPDATE', + ]; } - yield 'SQL Server' => [ + yield 'SQL Server|batch_size=1' => [ new SQLServer2012Platform(), - 'SELECT m.* FROM messenger_messages m WITH (UPDLOCK, ROWLOCK) WHERE (m.delivered_at is null OR m.delivered_at < ?) AND (m.available_at <= ?) AND (m.queue_name = ?) ORDER BY available_at ASC OFFSET 0 ROWS FETCH NEXT 1 ROWS ONLY ', + 1, + 'SELECT m.* FROM messenger_messages m WITH (UPDLOCK, ROWLOCK, READPAST) WHERE (m.delivered_at is null OR m.delivered_at < ?) AND (m.available_at <= ?) AND (m.queue_name = ?) ORDER BY available_at ASC OFFSET 0 ROWS FETCH NEXT 1 ROWS ONLY ', ]; - yield 'Oracle' => [ + yield 'SQL Server|batch_size=50' => [ + new SQLServer2012Platform(), + 50, + 'SELECT m.* FROM messenger_messages m WITH (UPDLOCK, ROWLOCK, READPAST) WHERE (m.delivered_at is null OR m.delivered_at < ?) AND (m.available_at <= ?) AND (m.queue_name = ?) ORDER BY available_at ASC OFFSET 0 ROWS FETCH NEXT 50 ROWS ONLY ', + ]; + + yield 'Oracle|batch_size=1' => [ new OraclePlatform(), + 1, 'SELECT w.id AS "id", w.body AS "body", w.headers AS "headers", w.queue_name AS "queue_name", w.created_at AS "created_at", w.available_at AS "available_at", w.delivered_at AS "delivered_at" FROM messenger_messages w WHERE w.id IN (SELECT a.id FROM (SELECT m.id FROM messenger_messages m WHERE (m.delivered_at is null OR m.delivered_at < ?) AND (m.available_at <= ?) AND (m.queue_name = ?) ORDER BY available_at ASC) a WHERE ROWNUM <= 1) FOR UPDATE', ]; + + yield 'Oracle|batch_size=50' => [ + new OraclePlatform(), + 50, + 'SELECT w.id AS "id", w.body AS "body", w.headers AS "headers", w.queue_name AS "queue_name", w.created_at AS "created_at", w.available_at AS "available_at", w.delivered_at AS "delivered_at" FROM messenger_messages w WHERE w.id IN (SELECT a.id FROM (SELECT m.id FROM messenger_messages m WHERE (m.delivered_at is null OR m.delivered_at < ?) AND (m.available_at <= ?) AND (m.queue_name = ?) ORDER BY available_at ASC) a WHERE ROWNUM <= 50) FOR UPDATE', + ]; } public function testConfigureSchema() diff --git a/src/Symfony/Component/Messenger/Bridge/Doctrine/Tests/Transport/DoctrineIntegrationTest.php b/src/Symfony/Component/Messenger/Bridge/Doctrine/Tests/Transport/DoctrineIntegrationTest.php index c9ff7c851e845..043e8db3b9ba0 100644 --- a/src/Symfony/Component/Messenger/Bridge/Doctrine/Tests/Transport/DoctrineIntegrationTest.php +++ b/src/Symfony/Component/Messenger/Bridge/Doctrine/Tests/Transport/DoctrineIntegrationTest.php @@ -52,9 +52,9 @@ protected function tearDown(): void public function testConnectionSendAndGet() { $this->connection->send('{"message": "Hi"}', ['type' => DummyMessage::class]); - $encoded = $this->connection->get(); - $this->assertEquals('{"message": "Hi"}', $encoded['body']); - $this->assertEquals(['type' => DummyMessage::class], $encoded['headers']); + $encodedList = $this->connection->get()[0]; + $this->assertEquals('{"message": "Hi"}', $encodedList['body']); + $this->assertEquals(['type' => DummyMessage::class], $encodedList['headers']); } public function testSendWithDelay() @@ -108,7 +108,7 @@ public function testItRetrieveTheFirstAvailableMessage() 'available_at' => $this->formatDateTime(new \DateTimeImmutable('2019-03-15 12:30:00', new \DateTimeZone('UTC'))), ]); - $encoded = $this->connection->get(); + $encoded = $this->connection->get()[0]; $this->assertEquals('{"message": "Hi available"}', $encoded['body']); } @@ -173,7 +173,7 @@ public function testItRetrieveTheMessageThatIsOlderThanRedeliverTimeout() 'available_at' => $this->formatDateTime(new \DateTimeImmutable('2019-03-15 12:30:00', new \DateTimeZone('UTC'))), ]); - $next = $this->connection->get(); + $next = $this->connection->get()[0]; $this->assertEquals('{"message": "Hi requeued"}', $next['body']); $this->connection->reject($next['id']); } @@ -181,10 +181,12 @@ public function testItRetrieveTheMessageThatIsOlderThanRedeliverTimeout() public function testTheTransportIsSetupOnGet() { $this->assertFalse($this->createSchemaManager()->tablesExist(['messenger_messages'])); - $this->assertNull($this->connection->get()); + $result = $this->connection->get(); + $this->assertIsArray($result); + $this->assertEmpty($result); $this->connection->send('the body', ['my' => 'header']); - $envelope = $this->connection->get(); + $envelope = $this->connection->get()[0]; $this->assertEquals('the body', $envelope['body']); } diff --git a/src/Symfony/Component/Messenger/Bridge/Doctrine/Tests/Transport/DoctrinePostgreSqlIntegrationTest.php b/src/Symfony/Component/Messenger/Bridge/Doctrine/Tests/Transport/DoctrinePostgreSqlIntegrationTest.php index 63fe3ebd4b513..969b58c3adb0f 100644 --- a/src/Symfony/Component/Messenger/Bridge/Doctrine/Tests/Transport/DoctrinePostgreSqlIntegrationTest.php +++ b/src/Symfony/Component/Messenger/Bridge/Doctrine/Tests/Transport/DoctrinePostgreSqlIntegrationTest.php @@ -61,11 +61,11 @@ public function testPostgreSqlConnectionSendAndGet() { $this->connection->send('{"message": "Hi"}', ['type' => DummyMessage::class]); - $encoded = $this->connection->get(); + $encoded = $this->connection->get()[0]; $this->assertEquals('{"message": "Hi"}', $encoded['body']); $this->assertEquals(['type' => DummyMessage::class], $encoded['headers']); - $this->assertNull($this->connection->get()); + $this->assertEmpty($this->connection->get()); } private function createSchemaManager(): AbstractSchemaManager diff --git a/src/Symfony/Component/Messenger/Bridge/Doctrine/Tests/Transport/DoctrineReceiverTest.php b/src/Symfony/Component/Messenger/Bridge/Doctrine/Tests/Transport/DoctrineReceiverTest.php index 43a0772371a97..1391408eb3788 100644 --- a/src/Symfony/Component/Messenger/Bridge/Doctrine/Tests/Transport/DoctrineReceiverTest.php +++ b/src/Symfony/Component/Messenger/Bridge/Doctrine/Tests/Transport/DoctrineReceiverTest.php @@ -18,7 +18,6 @@ use PHPUnit\Framework\TestCase; use Symfony\Component\Messenger\Bridge\Doctrine\Tests\Fixtures\DummyMessage; use Symfony\Component\Messenger\Bridge\Doctrine\Transport\Connection; -use Symfony\Component\Messenger\Bridge\Doctrine\Transport\DoctrineReceivedStamp; use Symfony\Component\Messenger\Bridge\Doctrine\Transport\DoctrineReceiver; use Symfony\Component\Messenger\Envelope; use Symfony\Component\Messenger\Exception\MessageDecodingFailedException; @@ -38,7 +37,7 @@ public function testItReturnsTheDecodedMessageToTheHandler() $doctrineEnvelope = $this->createDoctrineEnvelope(); $connection = $this->createMock(Connection::class); - $connection->method('get')->willReturn($doctrineEnvelope); + $connection->method('get')->willReturn([$doctrineEnvelope]); $receiver = new DoctrineReceiver($connection, $serializer); $actualEnvelopes = $receiver->get(); @@ -47,11 +46,6 @@ public function testItReturnsTheDecodedMessageToTheHandler() $actualEnvelope = $actualEnvelopes[0]; $this->assertEquals(new DummyMessage('Hi'), $actualEnvelopes[0]->getMessage()); - /** @var DoctrineReceivedStamp $doctrineReceivedStamp */ - $doctrineReceivedStamp = $actualEnvelope->last(DoctrineReceivedStamp::class); - $this->assertNotNull($doctrineReceivedStamp); - $this->assertSame('1', $doctrineReceivedStamp->getId()); - /** @var TransportMessageIdStamp $transportMessageIdStamp */ $transportMessageIdStamp = $actualEnvelope->last(TransportMessageIdStamp::class); $this->assertNotNull($transportMessageIdStamp); @@ -66,7 +60,7 @@ public function testItRejectTheMessageIfThereIsAMessageDecodingFailedException() $doctrineEnvelop = $this->createDoctrineEnvelope(); $connection = $this->createMock(Connection::class); - $connection->method('get')->willReturn($doctrineEnvelop); + $connection->method('get')->willReturn([$doctrineEnvelop]); $connection->expects($this->once())->method('reject'); $receiver = new DoctrineReceiver($connection, $serializer); diff --git a/src/Symfony/Component/Messenger/Bridge/Doctrine/Tests/Transport/DoctrineTransportTest.php b/src/Symfony/Component/Messenger/Bridge/Doctrine/Tests/Transport/DoctrineTransportTest.php index fea497d14409c..512b355873970 100644 --- a/src/Symfony/Component/Messenger/Bridge/Doctrine/Tests/Transport/DoctrineTransportTest.php +++ b/src/Symfony/Component/Messenger/Bridge/Doctrine/Tests/Transport/DoctrineTransportTest.php @@ -46,7 +46,7 @@ public function testReceivesMessages() ]; $serializer->method('decode')->with(['body' => 'body', 'headers' => ['my' => 'header']])->willReturn(new Envelope($decodedMessage)); - $connection->method('get')->willReturn($doctrineEnvelope); + $connection->method('get')->willReturn([$doctrineEnvelope]); $envelopes = $transport->get(); $this->assertSame($decodedMessage, $envelopes[0]->getMessage()); diff --git a/src/Symfony/Component/Messenger/Bridge/Doctrine/Transport/Connection.php b/src/Symfony/Component/Messenger/Bridge/Doctrine/Transport/Connection.php index 85f95e088ffe5..86c7277897fe0 100644 --- a/src/Symfony/Component/Messenger/Bridge/Doctrine/Transport/Connection.php +++ b/src/Symfony/Component/Messenger/Bridge/Doctrine/Transport/Connection.php @@ -12,14 +12,18 @@ namespace Symfony\Component\Messenger\Bridge\Doctrine\Transport; use Doctrine\DBAL\Abstraction\Result as AbstractionResult; +use Doctrine\DBAL\ArrayParameterType; use Doctrine\DBAL\Connection as DBALConnection; use Doctrine\DBAL\Driver\Exception as DriverException; use Doctrine\DBAL\Driver\ResultStatement; use Doctrine\DBAL\Exception as DBALException; use Doctrine\DBAL\Exception\TableNotFoundException; use Doctrine\DBAL\LockMode; +use Doctrine\DBAL\Platforms\MySQL80Platform; use Doctrine\DBAL\Platforms\MySQLPlatform; use Doctrine\DBAL\Platforms\OraclePlatform; +use Doctrine\DBAL\Platforms\PostgreSQLPlatform; +use Doctrine\DBAL\Platforms\SQLServerPlatform; use Doctrine\DBAL\Query\QueryBuilder; use Doctrine\DBAL\Result; use Doctrine\DBAL\Schema\AbstractSchemaManager; @@ -38,6 +42,8 @@ * * @author Vincent Touzet * @author Kévin Dunglas + * @author Herberto Graca + * @author Alexander Malyk */ class Connection implements ResetInterface { @@ -46,10 +52,13 @@ class Connection implements ResetInterface protected const DEFAULT_OPTIONS = [ 'table_name' => 'messenger_messages', 'queue_name' => 'default', + 'batch_size' => 1, 'redeliver_timeout' => 3600, 'auto_setup' => true, ]; + public const FLAGGED_FOR_DELETION = '9999-12-31 23:59:59'; + /** * Configuration of the connection. * @@ -66,6 +75,8 @@ class Connection implements ResetInterface protected $queueEmptiedAt; private ?SchemaSynchronizer $schemaSynchronizer; private bool $autoSetup; + /** @var array */ + private array $lastDeliveredButNotYetHandledEnvelopesIds = []; public function __construct(array $configuration, DBALConnection $driverConnection, SchemaSynchronizer $schemaSynchronizer = null) { @@ -155,11 +166,11 @@ public function send(string $body, array $headers, int $delay = 0): string return $this->driverConnection->lastInsertId(); } - public function get(): ?array + public function get(): array { if ($this->driverConnection->getDatabasePlatform() instanceof MySQLPlatform) { try { - $this->driverConnection->delete($this->configuration['table_name'], ['delivered_at' => '9999-12-31 23:59:59']); + $this->driverConnection->delete($this->configuration['table_name'], ['delivered_at' => self::FLAGGED_FOR_DELETION]); } catch (DriverException $e) { // Ignore the exception } @@ -170,7 +181,7 @@ public function get(): ?array try { $query = $this->createAvailableMessagesQueryBuilder() ->orderBy('available_at', 'ASC') - ->setMaxResults(1); + ->setMaxResults($this->configuration['batch_size']); if ($this->driverConnection->getDatabasePlatform() instanceof OraclePlatform) { $query->select('m.id'); @@ -198,41 +209,47 @@ public function get(): ?array } // use SELECT ... FOR UPDATE to lock table + $sql .= ' '.$this->driverConnection->getDatabasePlatform()->getWriteLockSQL(); $stmt = $this->executeQuery( - $sql.' '.$this->driverConnection->getDatabasePlatform()->getWriteLockSQL(), + $this->addSkipLocked($sql),// use SELECT ... FOR UPDATE SKIP LOCKED to lock only the relevant rows $query->getParameters(), $query->getParameterTypes() ); - $doctrineEnvelope = $stmt instanceof Result ? $stmt->fetchAssociative() : $stmt->fetch(); + $doctrineEnvelopeList = $stmt instanceof Result ? $stmt->fetchAllAssociative() : $stmt->fetchAll(); - if (false === $doctrineEnvelope) { + if ([] === $doctrineEnvelopeList) { $this->driverConnection->commit(); $this->queueEmptiedAt = microtime(true) * 1000; - return null; + return []; } // Postgres can "group" notifications having the same channel and payload // We need to be sure to empty the queue before blocking again $this->queueEmptiedAt = null; - $doctrineEnvelope = $this->decodeEnvelopeHeaders($doctrineEnvelope); + $this->lastDeliveredButNotYetHandledEnvelopesIds = $this->extractDoctrineEnvelopeListIds($doctrineEnvelopeList); $queryBuilder = $this->driverConnection->createQueryBuilder() ->update($this->configuration['table_name']) ->set('delivered_at', '?') - ->where('id = ?'); - $now = new \DateTimeImmutable('UTC'); - $this->executeStatement($queryBuilder->getSQL(), [ - $now, - $doctrineEnvelope['id'], - ], [ - Types::DATETIME_IMMUTABLE, - ]); + ->where($this->driverConnection->createQueryBuilder()->expr()->in('id', '?')); + $this->executeStatement( + $queryBuilder->getSQL(), + [ + new \DateTimeImmutable('UTC'), + $this->lastDeliveredButNotYetHandledEnvelopesIds, + ], + [ + Types::DATETIME_IMMUTABLE, + ArrayParameterType::INTEGER, + ] + ); $this->driverConnection->commit(); - return $doctrineEnvelope; + return $this->decodeDoctrineEnvelopeListHeaders($doctrineEnvelopeList); } catch (\Throwable $e) { + $this->lastDeliveredButNotYetHandledEnvelopesIds = []; $this->driverConnection->rollBack(); if ($this->autoSetup && $e instanceof TableNotFoundException) { @@ -246,9 +263,10 @@ public function get(): ?array public function ack(string $id): bool { + $this->removeHandledFromDeliveredList($id); try { if ($this->driverConnection->getDatabasePlatform() instanceof MySQLPlatform) { - return $this->driverConnection->update($this->configuration['table_name'], ['delivered_at' => '9999-12-31 23:59:59'], ['id' => $id]) > 0; + return $this->driverConnection->update($this->configuration['table_name'], ['delivered_at' => self::FLAGGED_FOR_DELETION], ['id' => $id]) > 0; } return $this->driverConnection->delete($this->configuration['table_name'], ['id' => $id]) > 0; @@ -261,7 +279,7 @@ public function reject(string $id): bool { try { if ($this->driverConnection->getDatabasePlatform() instanceof MySQLPlatform) { - return $this->driverConnection->update($this->configuration['table_name'], ['delivered_at' => '9999-12-31 23:59:59'], ['id' => $id]) > 0; + return $this->driverConnection->update($this->configuration['table_name'], ['delivered_at' => self::FLAGGED_FOR_DELETION], ['id' => $id]) > 0; } return $this->driverConnection->delete($this->configuration['table_name'], ['id' => $id]) > 0; @@ -340,6 +358,27 @@ public function getExtraSetupSqlForTable(Table $createdTable): array return []; } + public function undeliverNotHandled(): void + { + if (0 === \count($this->lastDeliveredButNotYetHandledEnvelopesIds)) { + return; + } + + $queryBuilder = $this->driverConnection->createQueryBuilder() + ->update($this->configuration['table_name']) + ->set('delivered_at', 'NULL') + ->where($this->driverConnection->createQueryBuilder()->expr()->in('id', '?')); + $this->executeStatement( + $queryBuilder->getSQL(), + [ + $this->lastDeliveredButNotYetHandledEnvelopesIds, + ], + [ + ArrayParameterType::INTEGER, + ] + ); + } + private function createAvailableMessagesQueryBuilder(): QueryBuilder { $now = new \DateTimeImmutable('UTC'); @@ -537,4 +576,70 @@ private function compareSchemas(Comparator $comparator, Schema $from, Schema $to ? $comparator->compareSchemas($from, $to) : $comparator->compare($from, $to); } + + /** + * @throws DBALException + */ + public function addSkipLocked(string $sql): string + { + switch (true) { + case $this->driverConnection->getDatabasePlatform() instanceof MySQL80Platform: + case $this->driverConnection->getDatabasePlatform() instanceof PostgreSQLPlatform: + $sql .= ' SKIP LOCKED'; // use skip locked so workers pulling in parallel don't wait on each other + break; + case $this->driverConnection->getDatabasePlatform() instanceof SQLServerPlatform: + $sql = str_replace('(UPDLOCK, ROWLOCK)', '(UPDLOCK, ROWLOCK, READPAST)', $sql); + break; + } + + return $sql; + } + + /** + * @return int[] + */ + public function extractDoctrineEnvelopeListIds(array $doctrineEnvelopeList): array + { + return array_map( + function ($doctrineEnvelopeRow) { + return $doctrineEnvelopeRow['id']; + }, + $doctrineEnvelopeList + ); + } + + /** + * @param array $doctrineEnvelopeList + * + * @return array + */ + public function decodeDoctrineEnvelopeListHeaders(array $doctrineEnvelopeList): array + { + $doctrineEnvelopeListWithDecodedHeaders = []; + foreach ($doctrineEnvelopeList as $doctrineEnvelope) { + $doctrineEnvelopeListWithDecodedHeaders[] = $this->decodeEnvelopeHeaders($doctrineEnvelope); + } + return $doctrineEnvelopeListWithDecodedHeaders; + } + + public function removeHandledFromDeliveredList(string $id): void + { + $this->lastDeliveredButNotYetHandledEnvelopesIds = array_diff($this->lastDeliveredButNotYetHandledEnvelopesIds, [$id]); + } } diff --git a/src/Symfony/Component/Messenger/Bridge/Doctrine/Transport/DoctrineReceivedStamp.php b/src/Symfony/Component/Messenger/Bridge/Doctrine/Transport/DoctrineReceivedStamp.php deleted file mode 100644 index 513f4b9f01c4c..0000000000000 --- a/src/Symfony/Component/Messenger/Bridge/Doctrine/Transport/DoctrineReceivedStamp.php +++ /dev/null @@ -1,32 +0,0 @@ - - * - * For the full copyright and license information, please view the LICENSE - * file that was distributed with this source code. - */ - -namespace Symfony\Component\Messenger\Bridge\Doctrine\Transport; - -use Symfony\Component\Messenger\Stamp\NonSendableStampInterface; - -/** - * @author Vincent Touzet - */ -class DoctrineReceivedStamp implements NonSendableStampInterface -{ - private string $id; - - public function __construct(string $id) - { - $this->id = $id; - } - - public function getId(): string - { - return $this->id; - } -} diff --git a/src/Symfony/Component/Messenger/Bridge/Doctrine/Transport/DoctrineReceiver.php b/src/Symfony/Component/Messenger/Bridge/Doctrine/Transport/DoctrineReceiver.php index 79310f35a9a9c..73ee4ed995196 100644 --- a/src/Symfony/Component/Messenger/Bridge/Doctrine/Transport/DoctrineReceiver.php +++ b/src/Symfony/Component/Messenger/Bridge/Doctrine/Transport/DoctrineReceiver.php @@ -25,6 +25,8 @@ /** * @author Vincent Touzet + * @author Herberto Graca + * @author Alexander Malyk */ class DoctrineReceiver implements ListableReceiverInterface, MessageCountAwareInterface { @@ -42,7 +44,7 @@ public function __construct(Connection $connection, SerializerInterface $seriali public function get(): iterable { try { - $doctrineEnvelope = $this->connection->get(); + $doctrineEnvelopeList = $this->connection->get(); $this->retryingSafetyCounter = 0; // reset counter } catch (RetryableException $exception) { // Do nothing when RetryableException occurs less than "MAX_RETRIES" @@ -58,17 +60,22 @@ public function get(): iterable throw new TransportException($exception->getMessage(), 0, $exception); } - if (null === $doctrineEnvelope) { + if (null === $doctrineEnvelopeList) { return []; } - return [$this->createEnvelopeFromData($doctrineEnvelope)]; + $envelopeList = []; + foreach ($doctrineEnvelopeList as $doctrineEnvelope) { + $envelopeList[] = $this->createEnvelopeFromData($doctrineEnvelope); + } + + return $envelopeList; } public function ack(Envelope $envelope): void { try { - $this->connection->ack($this->findDoctrineReceivedStamp($envelope)->getId()); + $this->connection->ack($this->findMessageIdStamp($envelope)->getId()); } catch (DBALException $exception) { throw new TransportException($exception->getMessage(), 0, $exception); } @@ -77,12 +84,17 @@ public function ack(Envelope $envelope): void public function reject(Envelope $envelope): void { try { - $this->connection->reject($this->findDoctrineReceivedStamp($envelope)->getId()); + $this->connection->reject($this->findMessageIdStamp($envelope)->getId()); } catch (DBALException $exception) { throw new TransportException($exception->getMessage(), 0, $exception); } } + public function undeliverNotHandled(): void + { + $this->connection->undeliverNotHandled(); + } + public function getMessageCount(): int { try { @@ -120,16 +132,16 @@ public function find(mixed $id): ?Envelope return $this->createEnvelopeFromData($doctrineEnvelope); } - private function findDoctrineReceivedStamp(Envelope $envelope): DoctrineReceivedStamp + private function findMessageIdStamp(Envelope $envelope): TransportMessageIdStamp { - /** @var DoctrineReceivedStamp|null $doctrineReceivedStamp */ - $doctrineReceivedStamp = $envelope->last(DoctrineReceivedStamp::class); + /** @var TransportMessageIdStamp|null $transportMessageIdStamp */ + $transportMessageIdStamp = $envelope->last(TransportMessageIdStamp::class); - if (null === $doctrineReceivedStamp) { - throw new LogicException('No DoctrineReceivedStamp found on the Envelope.'); + if (null === $transportMessageIdStamp) { + throw new LogicException('No TransportMessageIdStamp found on the Envelope.'); } - return $doctrineReceivedStamp; + return $transportMessageIdStamp; } private function createEnvelopeFromData(array $data): Envelope @@ -146,7 +158,6 @@ private function createEnvelopeFromData(array $data): Envelope } return $envelope->with( - new DoctrineReceivedStamp($data['id']), new TransportMessageIdStamp($data['id']) ); } diff --git a/src/Symfony/Component/Messenger/Bridge/Doctrine/Transport/DoctrineTransport.php b/src/Symfony/Component/Messenger/Bridge/Doctrine/Transport/DoctrineTransport.php index dac4dd538b731..71e015aa06a4d 100644 --- a/src/Symfony/Component/Messenger/Bridge/Doctrine/Transport/DoctrineTransport.php +++ b/src/Symfony/Component/Messenger/Bridge/Doctrine/Transport/DoctrineTransport.php @@ -23,6 +23,8 @@ /** * @author Vincent Touzet + * @author Herberto Graca + * @author Alexander Malyk */ class DoctrineTransport implements TransportInterface, SetupableTransportInterface, MessageCountAwareInterface, ListableReceiverInterface { @@ -37,6 +39,18 @@ public function __construct(Connection $connection, SerializerInterface $seriali $this->serializer = $serializer; } + public function __destruct() + { + // The worker using this transport might have pulled 50 msgs out of the mq, marking them as "in process", + // but because of its options (ie --limit, --failure-limit, --memory-limit, --time-limit) it might terminate + // before it actually handles them all, leaving messages in limbo where they will not be handled by the + // consumer that pulled them, and they won't be picked up by any other consumer because they are + // "in process" already. + // Thus, when the consumer stops, and its transport gets destroyed, we need to put the messages not handled, + // back in "waiting to be processed" so that they can be picked up by other consumers. + $this->getReceiver()->undeliverNotHandled(); + } + public function get(): iterable { return $this->getReceiver()->get(); @@ -79,8 +93,6 @@ public function setup(): void /** * Adds the Table to the Schema if this transport uses this connection. - * - * @param \Closure $isSameDatabase */ public function configureSchema(Schema $schema, DbalConnection $forConnection/* , \Closure $isSameDatabase */): void { diff --git a/src/Symfony/Component/Messenger/Bridge/Doctrine/Transport/DoctrineTransportFactory.php b/src/Symfony/Component/Messenger/Bridge/Doctrine/Transport/DoctrineTransportFactory.php index cff92bf11c19f..80b89fa084cb6 100644 --- a/src/Symfony/Component/Messenger/Bridge/Doctrine/Transport/DoctrineTransportFactory.php +++ b/src/Symfony/Component/Messenger/Bridge/Doctrine/Transport/DoctrineTransportFactory.php @@ -11,6 +11,7 @@ namespace Symfony\Component\Messenger\Bridge\Doctrine\Transport; +use Doctrine\DBAL\Connection as DbalConnection; use Doctrine\DBAL\Platforms\PostgreSQLPlatform; use Doctrine\Persistence\ConnectionRegistry; use Symfony\Component\Messenger\Exception\TransportException; @@ -20,9 +21,12 @@ /** * @author Vincent Touzet + * @author Herberto Graca */ class DoctrineTransportFactory implements TransportFactoryInterface { + public const DSN_PREFIX = 'doctrine://'; + private ConnectionRegistry $registry; public function __construct(ConnectionRegistry $registry) @@ -38,6 +42,7 @@ public function createTransport(#[\SensitiveParameter] string $dsn, array $optio $configuration = PostgreSqlConnection::buildConfiguration($dsn, $options); try { + /** @var DbalConnection $driverConnection */ $driverConnection = $this->registry->getConnection($configuration['connection']); } catch (\InvalidArgumentException $e) { throw new TransportException('Could not find Doctrine connection from Messenger DSN.', 0, $e); @@ -54,6 +59,6 @@ public function createTransport(#[\SensitiveParameter] string $dsn, array $optio public function supports(#[\SensitiveParameter] string $dsn, array $options): bool { - return str_starts_with($dsn, 'doctrine://'); + return str_starts_with($dsn, self::DSN_PREFIX); } } diff --git a/src/Symfony/Component/Messenger/Bridge/Doctrine/Transport/PostgreSqlConnection.php b/src/Symfony/Component/Messenger/Bridge/Doctrine/Transport/PostgreSqlConnection.php index 2adb31be54b7b..201ce8f83f721 100644 --- a/src/Symfony/Component/Messenger/Bridge/Doctrine/Transport/PostgreSqlConnection.php +++ b/src/Symfony/Component/Messenger/Bridge/Doctrine/Transport/PostgreSqlConnection.php @@ -57,7 +57,7 @@ public function reset(): void $this->unlisten(); } - public function get(): ?array + public function get(): array { if (null === $this->queueEmptiedAt) { return parent::get(); @@ -85,7 +85,7 @@ public function get(): ?array ) { usleep(1000); - return null; + return []; } return parent::get(); diff --git a/src/Symfony/Component/Messenger/CHANGELOG.md b/src/Symfony/Component/Messenger/CHANGELOG.md index aabf140f6ce73..8bf098e6fc21d 100644 --- a/src/Symfony/Component/Messenger/CHANGELOG.md +++ b/src/Symfony/Component/Messenger/CHANGELOG.md @@ -1,6 +1,12 @@ CHANGELOG ========= +6.4 +--- + + * Add support for the doctrine transport to pull a batch of messages in one go (`batch_size` option) + * Use `SKIP LOCKED` in the doctrine transport for MySQL, PostgreSQL and MSSQL + 6.3 ---