From bc1fbeb5d1ea1493319da4bfa34082c006dd3f7d Mon Sep 17 00:00:00 2001 From: janbeumer Date: Sun, 13 Apr 2025 09:03:51 +0200 Subject: [PATCH 1/5] [Messenger][Doctrine][Transport] introduce a algorithm in `Connection` for mysql platforms to minimize exclusive locking --- .../Tests/Transport/ConnectionTest.php | 110 +++++++++++----- .../Bridge/Doctrine/Transport/Connection.php | 122 +++++++++++++++++- 2 files changed, 191 insertions(+), 41 deletions(-) diff --git a/src/Symfony/Component/Messenger/Bridge/Doctrine/Tests/Transport/ConnectionTest.php b/src/Symfony/Component/Messenger/Bridge/Doctrine/Tests/Transport/ConnectionTest.php index bba17a49eb64c..73b6dc5f41075 100644 --- a/src/Symfony/Component/Messenger/Bridge/Doctrine/Tests/Transport/ConnectionTest.php +++ b/src/Symfony/Component/Messenger/Bridge/Doctrine/Tests/Transport/ConnectionTest.php @@ -668,46 +668,41 @@ public function testGeneratedSql(AbstractPlatform $platform, string $expectedSql $connection->get(); } - public static function providePlatformSql(): iterable + /** + * @dataProvider providePlatformMySQLSql + */ + public function testGeneratedMySQLSqlOnGet(AbstractPlatform $platform, string $expectedSql, string $expectedUpdateSql) { - yield 'MySQL' => [ - class_exists(MySQLPlatform::class) ? new MySQLPlatform() : new MySQL57Platform(), - 'SELECT m.* FROM messenger_messages m WHERE (m.queue_name = ?) AND (m.delivered_at is null OR m.delivered_at < ?) AND (m.available_at <= ?) ORDER BY available_at ASC LIMIT 1 FOR UPDATE', - ]; - - if (class_exists(MySQL80Platform::class) && !method_exists(QueryBuilder::class, 'forUpdate')) { - yield 'MySQL8 & DBAL<3.8' => [ - new MySQL80Platform(), - 'SELECT m.* FROM messenger_messages m WHERE (m.queue_name = ?) AND (m.delivered_at is null OR m.delivered_at < ?) AND (m.available_at <= ?) ORDER BY available_at ASC LIMIT 1 FOR UPDATE', - ]; - } - - if (class_exists(MySQL80Platform::class) && method_exists(QueryBuilder::class, 'forUpdate')) { - yield 'MySQL8 & DBAL>=3.8' => [ - new MySQL80Platform(), - 'SELECT m.* FROM messenger_messages m WHERE (m.queue_name = ?) AND (m.delivered_at is null OR m.delivered_at < ?) AND (m.available_at <= ?) ORDER BY available_at ASC LIMIT 1 FOR UPDATE SKIP LOCKED', - ]; - } + $driverConnection = $this->createMock(DBALConnection::class); + $driverConnection->method('getDatabasePlatform')->willReturn($platform); + $driverConnection->method('createQueryBuilder')->willReturnCallback(fn () => new QueryBuilder($driverConnection)); - yield 'MariaDB' => [ - new MariaDBPlatform(), - 'SELECT m.* FROM messenger_messages m WHERE (m.queue_name = ?) AND (m.delivered_at is null OR m.delivered_at < ?) AND (m.available_at <= ?) ORDER BY available_at ASC LIMIT 1 FOR UPDATE', - ]; + $result = $this->createMock(Result::class); + $result->method('fetchFirstColumn')->willReturn(['id' => 3]); - if (interface_exists(DBALException::class)) { - // DBAL 4+ - $mariaDbPlatformClass = 'Doctrine\DBAL\Platforms\MariaDB1060Platform'; - } else { - $mariaDbPlatformClass = 'Doctrine\DBAL\Platforms\MariaDb1060Platform'; - } + $driverConnection + ->expects($this->once()) + ->method('executeQuery') + ->with($this->callback(function ($sql) use ($expectedSql) { + return trim($expectedSql) === trim($sql); + })) + ->willReturn($result) + ; + $driverConnection + ->expects($this->once()) + ->method('executeStatement') + ->with($this->callback(function ($sql) use ($expectedUpdateSql) { + return trim($expectedUpdateSql) === trim($sql); + })) + ->willReturn(0) + ; - if (class_exists($mariaDbPlatformClass)) { - yield 'MariaDB106' => [ - new $mariaDbPlatformClass(), - 'SELECT m.* FROM messenger_messages m WHERE (m.queue_name = ?) AND (m.delivered_at is null OR m.delivered_at < ?) AND (m.available_at <= ?) ORDER BY available_at ASC LIMIT 1 FOR UPDATE SKIP LOCKED', - ]; - } + $connection = new Connection([], $driverConnection); + $connection->get(); + } + public static function providePlatformSql(): iterable + { if (class_exists(MySQL57Platform::class)) { yield 'Postgres & DBAL<4' => [ new PostgreSQLPlatform(), @@ -773,6 +768,51 @@ class_exists(SQLServerPlatform::class) && !class_exists(SQLServer2012Platform::c } } + public static function providePlatformMySQLSql(): iterable + { + yield 'MySQL' => [ + class_exists(MySQLPlatform::class) ? new MySQLPlatform() : new MySQL57Platform(), + 'SELECT id FROM messenger_messages m WHERE (m.queue_name = ?) AND (m.delivered_at is null OR m.delivered_at < ?) AND (m.available_at <= ?) ORDER BY available_at ASC LIMIT 100', + 'UPDATE messenger_messages SET delivered_at = :now WHERE (id = :id) AND (queue_name = :queue_name) AND (delivered_at is null OR delivered_at < :redeliver_imit)', + ]; + + if (class_exists(MySQL80Platform::class) && !method_exists(QueryBuilder::class, 'forUpdate')) { + yield 'MySQL8 & DBAL<3.8' => [ + new MySQL80Platform(), + 'SELECT id FROM messenger_messages m WHERE (m.queue_name = ?) AND (m.delivered_at is null OR m.delivered_at < ?) AND (m.available_at <= ?) ORDER BY available_at ASC LIMIT 100', + 'UPDATE messenger_messages SET delivered_at = :now WHERE (id = :id) AND (queue_name = :queue_name) AND (delivered_at is null OR delivered_at < :redeliver_imit)', + ]; + } + + if (class_exists(MySQL80Platform::class) && method_exists(QueryBuilder::class, 'forUpdate')) { + yield 'MySQL8 & DBAL>=3.8' => [ + new MySQL80Platform(), + 'SELECT id FROM messenger_messages m WHERE (m.queue_name = ?) AND (m.delivered_at is null OR m.delivered_at < ?) AND (m.available_at <= ?) ORDER BY available_at ASC LIMIT 100', + 'UPDATE messenger_messages SET delivered_at = :now WHERE (id = :id) AND (queue_name = :queue_name) AND (delivered_at is null OR delivered_at < :redeliver_imit)', + ]; + } + yield 'MariaDB' => [ + new MariaDBPlatform(), + 'SELECT id FROM messenger_messages m WHERE (m.queue_name = ?) AND (m.delivered_at is null OR m.delivered_at < ?) AND (m.available_at <= ?) ORDER BY available_at ASC LIMIT 100', + 'UPDATE messenger_messages SET delivered_at = :now WHERE (id = :id) AND (queue_name = :queue_name) AND (delivered_at is null OR delivered_at < :redeliver_imit)', + ]; + + if (interface_exists(DBALException::class)) { + // DBAL 4+ + $mariaDbPlatformClass = 'Doctrine\DBAL\Platforms\MariaDB1060Platform'; + } else { + $mariaDbPlatformClass = 'Doctrine\DBAL\Platforms\MariaDb1060Platform'; + } + + if (class_exists($mariaDbPlatformClass)) { + yield 'MariaDB106' => [ + new $mariaDbPlatformClass(), + 'SELECT id FROM messenger_messages m WHERE (m.queue_name = ?) AND (m.delivered_at is null OR m.delivered_at < ?) AND (m.available_at <= ?) ORDER BY available_at ASC LIMIT 100', + 'UPDATE messenger_messages SET delivered_at = :now WHERE (id = :id) AND (queue_name = :queue_name) AND (delivered_at is null OR delivered_at < :redeliver_imit)', + ]; + } + } + public function testConfigureSchema() { $driverConnection = $this->getDBALConnectionMock(); diff --git a/src/Symfony/Component/Messenger/Bridge/Doctrine/Transport/Connection.php b/src/Symfony/Component/Messenger/Bridge/Doctrine/Transport/Connection.php index 6ed21386b6434..b5af97042cbfa 100644 --- a/src/Symfony/Component/Messenger/Bridge/Doctrine/Transport/Connection.php +++ b/src/Symfony/Component/Messenger/Bridge/Doctrine/Transport/Connection.php @@ -11,6 +11,7 @@ namespace Symfony\Component\Messenger\Bridge\Doctrine\Transport; +use Doctrine\DBAL\ArrayParameterType; use Doctrine\DBAL\Connection as DBALConnection; use Doctrine\DBAL\Driver\Exception as DriverException; use Doctrine\DBAL\Exception as DBALException; @@ -158,15 +159,17 @@ public function send(string $body, array $headers, int $delay = 0): string public function get(): ?array { - if ($this->doMysqlCleanup && $this->driverConnection->getDatabasePlatform() instanceof AbstractMySQLPlatform) { + if ($this->driverConnection->getDatabasePlatform() instanceof AbstractMySQLPlatform) { + if ($this->doMysqlCleanup) { + $this->deleteDeliveredMessageForMySQLPlatform(); + } try { - $this->driverConnection->delete($this->configuration['table_name'], ['delivered_at' => '9999-12-31 23:59:59']); - $this->doMysqlCleanup = false; - } catch (DriverException $e) { - // Ignore the exception - } catch (TableNotFoundException $e) { + return $this->getMessageForMySQLPlatform(); + } catch (TableNotFoundException) { if ($this->autoSetup) { $this->setup(); + + return $this->getMessageForMySQLPlatform(); } } } @@ -627,4 +630,111 @@ private function fallBackToForUpdate(QueryBuilder $query, string $sql): string return $sql; } } + + private function getMessageForMySQLPlatform(): ?array + { + $possibleIdsToClaim = $this->createAvailableMessagesQueryBuilder() + ->select('id') + ->orderBy('available_at', 'ASC') + ->setMaxResults(100)->fetchFirstColumn(); + + if (0 === \count($possibleIdsToClaim)) { + $this->queueEmptiedAt = microtime(true) * 1000; + + return null; + } + $this->queueEmptiedAt = null; + + foreach ($possibleIdsToClaim as $id) { + $claimedId = $this->claimMessage($id); + if (null !== $claimedId) { + break; + } + } + + if (!isset($claimedId)) { + // all open messages already have been claimed by other workers. + return null; + } + + $messageData = $this->createQueryBuilder() + ->andWhere('m.id = :id') + ->setParameter(':id', $claimedId) + ->fetchAssociative(); + + return $this->decodeEnvelopeHeaders($messageData); + } + + /** + * @throws DBALException + */ + private function deleteDeliveredMessageForMySQLPlatform(): void + { + try { + $ids = $this->selectMessageIdsToDelete(); + $this->driverConnection->createQueryBuilder() + ->delete($this->configuration['table_name']) + ->where('id IN (:ids)') + ->setParameter('ids', $ids, ArrayParameterType::INTEGER) + ->executeQuery() + ; + $this->doMysqlCleanup = false; + } catch (DriverException $e) { + // Ignore the exception + } catch (TableNotFoundException $e) { + if ($this->autoSetup) { + $this->setup(); + } + } + } + + /** + * @return array + * + * @throws DBALException + */ + private function selectMessageIdsToDelete(): array + { + return $this->driverConnection->createQueryBuilder() + ->select('m.id') + ->from($this->configuration['table_name'], 'm') + ->where('m.queue_name = ?') + ->andWhere('m.delivered_at = ?') + ->setParameters([ + $this->configuration['queue_name'], + '9999-12-31 23:59:59', + ], [ + Types::STRING, + Types::STRING, + ]) + ->setMaxResults(5_000) + ->fetchFirstColumn(); + } + + private function claimMessage(mixed $id): mixed + { + $now = new \DateTimeImmutable('UTC'); + $redeliverLimit = $now->modify(\sprintf('-%d seconds', $this->configuration['redeliver_timeout'])); + + $claimed = $this->driverConnection->createQueryBuilder() + ->update($this->configuration['table_name']) + ->set('delivered_at', ':now') + ->andWhere('id = :id') + ->andWhere('queue_name = :queue_name') + ->andWhere('delivered_at is null OR delivered_at < :redeliver_imit') + ->setParameters([ + 'now' => $now, + 'id' => $id, + 'queue_name' => $this->configuration['queue_name'], + 'redeliver_imit' => $redeliverLimit, + ], [ + 'now' => Types::DATETIME_IMMUTABLE, + 'id' => Types::INTEGER, + 'queue_name' => Types::STRING, + 'redeliver_imit' => Types::DATETIME_IMMUTABLE, + ]) + ->executeStatement() > 0; + + return $claimed ? $id : null; + } } From ef8458226ee4b102bda4145f0fb7abbdb6040069 Mon Sep 17 00:00:00 2001 From: Jan Beumer <147642917+JanPaulBeumer@users.noreply.github.com> Date: Sun, 20 Apr 2025 08:44:36 +0200 Subject: [PATCH 2/5] Update src/Symfony/Component/Messenger/Bridge/Doctrine/Transport/Connection.php Co-authored-by: Nicolas Grekas --- .../Messenger/Bridge/Doctrine/Transport/Connection.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Symfony/Component/Messenger/Bridge/Doctrine/Transport/Connection.php b/src/Symfony/Component/Messenger/Bridge/Doctrine/Transport/Connection.php index b5af97042cbfa..d4b6a6e0ef6ed 100644 --- a/src/Symfony/Component/Messenger/Bridge/Doctrine/Transport/Connection.php +++ b/src/Symfony/Component/Messenger/Bridge/Doctrine/Transport/Connection.php @@ -638,7 +638,7 @@ private function getMessageForMySQLPlatform(): ?array ->orderBy('available_at', 'ASC') ->setMaxResults(100)->fetchFirstColumn(); - if (0 === \count($possibleIdsToClaim)) { + if (!$possibleIdsToClaim) { $this->queueEmptiedAt = microtime(true) * 1000; return null; From 110629a44d575d47a4c508363d584d8d6fdff7f3 Mon Sep 17 00:00:00 2001 From: Jan Beumer <147642917+JanPaulBeumer@users.noreply.github.com> Date: Sun, 20 Apr 2025 08:47:22 +0200 Subject: [PATCH 3/5] Update src/Symfony/Component/Messenger/Bridge/Doctrine/Transport/Connection.php Co-authored-by: Nicolas Grekas --- .../Messenger/Bridge/Doctrine/Transport/Connection.php | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/Symfony/Component/Messenger/Bridge/Doctrine/Transport/Connection.php b/src/Symfony/Component/Messenger/Bridge/Doctrine/Transport/Connection.php index d4b6a6e0ef6ed..8bfd7fda3036a 100644 --- a/src/Symfony/Component/Messenger/Bridge/Doctrine/Transport/Connection.php +++ b/src/Symfony/Component/Messenger/Bridge/Doctrine/Transport/Connection.php @@ -646,8 +646,7 @@ private function getMessageForMySQLPlatform(): ?array $this->queueEmptiedAt = null; foreach ($possibleIdsToClaim as $id) { - $claimedId = $this->claimMessage($id); - if (null !== $claimedId) { + if (null === $claimedId = $this->claimMessage($id)) { break; } } From 4259f68d9221fe4ccc4a593e5317854eb2485eac Mon Sep 17 00:00:00 2001 From: Jan Beumer <147642917+JanPaulBeumer@users.noreply.github.com> Date: Sun, 20 Apr 2025 08:49:22 +0200 Subject: [PATCH 4/5] Update src/Symfony/Component/Messenger/Bridge/Doctrine/Transport/Connection.php Co-authored-by: Nicolas Grekas --- .../Messenger/Bridge/Doctrine/Transport/Connection.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Symfony/Component/Messenger/Bridge/Doctrine/Transport/Connection.php b/src/Symfony/Component/Messenger/Bridge/Doctrine/Transport/Connection.php index 8bfd7fda3036a..11b0af1da4a02 100644 --- a/src/Symfony/Component/Messenger/Bridge/Doctrine/Transport/Connection.php +++ b/src/Symfony/Component/Messenger/Bridge/Doctrine/Transport/Connection.php @@ -715,7 +715,7 @@ private function claimMessage(mixed $id): mixed $now = new \DateTimeImmutable('UTC'); $redeliverLimit = $now->modify(\sprintf('-%d seconds', $this->configuration['redeliver_timeout'])); - $claimed = $this->driverConnection->createQueryBuilder() + $claimed = 0 < $this->driverConnection->createQueryBuilder() ->update($this->configuration['table_name']) ->set('delivered_at', ':now') ->andWhere('id = :id') From d5db0bff9ddb768db6d3721578dd84abe17ab45e Mon Sep 17 00:00:00 2001 From: janbeumer Date: Sun, 20 Apr 2025 08:50:32 +0200 Subject: [PATCH 5/5] [Messenger][Doctrine][Transport] introduce a variable instead of checking isset. --- .../Messenger/Bridge/Doctrine/Transport/Connection.php | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/Symfony/Component/Messenger/Bridge/Doctrine/Transport/Connection.php b/src/Symfony/Component/Messenger/Bridge/Doctrine/Transport/Connection.php index 11b0af1da4a02..714c9b0d6950c 100644 --- a/src/Symfony/Component/Messenger/Bridge/Doctrine/Transport/Connection.php +++ b/src/Symfony/Component/Messenger/Bridge/Doctrine/Transport/Connection.php @@ -645,13 +645,14 @@ private function getMessageForMySQLPlatform(): ?array } $this->queueEmptiedAt = null; + $claimedId = null; foreach ($possibleIdsToClaim as $id) { if (null === $claimedId = $this->claimMessage($id)) { break; } } - if (!isset($claimedId)) { + if (!null === $claimedId) { // all open messages already have been claimed by other workers. return null; } @@ -732,7 +733,7 @@ private function claimMessage(mixed $id): mixed 'queue_name' => Types::STRING, 'redeliver_imit' => Types::DATETIME_IMMUTABLE, ]) - ->executeStatement() > 0; + ->executeStatement(); return $claimed ? $id : null; }