diff --git a/.gitattributes b/.gitattributes index 84c7add..14c3c35 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,4 +1,3 @@ /Tests export-ignore /phpunit.xml.dist export-ignore -/.gitattributes export-ignore -/.gitignore export-ignore +/.git* export-ignore diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 0000000..4689c4d --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,8 @@ +Please do not submit any Pull Requests here. They will be closed. +--- + +Please submit your PR here instead: +https://github.com/symfony/symfony + +This repository is what we call a "subtree split": a read-only subset of that main repository. +We're looking forward to your PR there! diff --git a/.github/workflows/check-subtree-split.yml b/.github/workflows/check-subtree-split.yml new file mode 100644 index 0000000..16be48b --- /dev/null +++ b/.github/workflows/check-subtree-split.yml @@ -0,0 +1,37 @@ +name: Check subtree split + +on: + pull_request_target: + +jobs: + close-pull-request: + runs-on: ubuntu-latest + + steps: + - name: Close pull request + uses: actions/github-script@v6 + with: + script: | + if (context.repo.owner === "symfony") { + github.rest.issues.createComment({ + owner: "symfony", + repo: context.repo.repo, + issue_number: context.issue.number, + body: ` + Thanks for your Pull Request! We love contributions. + + However, you should instead open your PR on the main repository: + https://github.com/symfony/symfony + + This repository is what we call a "subtree split": a read-only subset of that main repository. + We're looking forward to your PR there! + ` + }); + + github.rest.pulls.update({ + owner: "symfony", + repo: context.repo.repo, + pull_number: context.issue.number, + state: "closed" + }); + } diff --git a/Tests/Transport/ConnectionTest.php b/Tests/Transport/ConnectionTest.php index e2aee3c..25d661a 100644 --- a/Tests/Transport/ConnectionTest.php +++ b/Tests/Transport/ConnectionTest.php @@ -20,6 +20,7 @@ use Doctrine\DBAL\Platforms\OraclePlatform; use Doctrine\DBAL\Platforms\SQLServer2012Platform; use Doctrine\DBAL\Platforms\SQLServerPlatform; +use Doctrine\DBAL\Query\ForUpdate\ConflictResolutionMode; use Doctrine\DBAL\Query\QueryBuilder; use Doctrine\DBAL\Result; use Doctrine\DBAL\Schema\AbstractSchemaManager; @@ -99,6 +100,82 @@ public function testGetWithNoPendingMessageWillReturnNull() $this->assertNull($doctrineEnvelope); } + public function testGetWithSkipLockedWithForUpdateMethod() + { + if (!method_exists(QueryBuilder::class, 'forUpdate')) { + $this->markTestSkipped('This test is for when forUpdate method exists.'); + } + + $queryBuilder = $this->getQueryBuilderMock(); + $driverConnection = $this->getDBALConnectionMock(); + $stmt = $this->getResultMock(false); + + $queryBuilder + ->method('getParameters') + ->willReturn([]); + $queryBuilder + ->method('getParameterTypes') + ->willReturn([]); + $queryBuilder + ->method('forUpdate') + ->with(ConflictResolutionMode::SKIP_LOCKED) + ->willReturn($queryBuilder); + $queryBuilder + ->method('getSQL') + ->willReturn('SELECT FOR UPDATE SKIP LOCKED'); + $driverConnection->expects($this->once()) + ->method('createQueryBuilder') + ->willReturn($queryBuilder); + $driverConnection->expects($this->never()) + ->method('update'); + $driverConnection + ->method('executeQuery') + ->with($this->callback(function ($sql) { + return str_contains($sql, 'SKIP LOCKED'); + })) + ->willReturn($stmt); + + $connection = new Connection(['skip_locked' => true], $driverConnection); + $doctrineEnvelope = $connection->get(); + $this->assertNull($doctrineEnvelope); + } + + public function testGetWithSkipLockedWithoutForUpdateMethod() + { + if (method_exists(QueryBuilder::class, 'forUpdate')) { + $this->markTestSkipped('This test is for when forUpdate method does not exist.'); + } + + $queryBuilder = $this->getQueryBuilderMock(); + $driverConnection = $this->getDBALConnectionMock(); + $stmt = $this->getResultMock(false); + + $queryBuilder + ->method('getParameters') + ->willReturn([]); + $queryBuilder + ->method('getParameterTypes') + ->willReturn([]); + $queryBuilder + ->method('getSQL') + ->willReturn('SELECT'); + $driverConnection->expects($this->once()) + ->method('createQueryBuilder') + ->willReturn($queryBuilder); + $driverConnection->expects($this->never()) + ->method('update'); + $driverConnection + ->method('executeQuery') + ->with($this->callback(function ($sql) { + return str_contains($sql, 'SKIP LOCKED'); + })) + ->willReturn($stmt); + + $connection = new Connection(['skip_locked' => true], $driverConnection); + $doctrineEnvelope = $connection->get(); + $this->assertNull($doctrineEnvelope); + } + public function testItThrowsATransportExceptionIfItCannotAcknowledgeMessage() { $this->expectException(TransportException::class); @@ -223,7 +300,7 @@ private function getDBALConnectionMock() $platform = $this->createMock(AbstractPlatform::class); if (!method_exists(QueryBuilder::class, 'forUpdate')) { - $platform->method('getWriteLockSQL')->willReturn('FOR UPDATE'); + $platform->method('getWriteLockSQL')->willReturn('FOR UPDATE SKIP LOCKED'); } $configuration = $this->createMock(\Doctrine\DBAL\Configuration::class); @@ -496,20 +573,20 @@ class_exists(MySQLPlatform::class) ? new MySQLPlatform() : new MySQL57Platform() yield 'SQL Server' => [ class_exists(SQLServerPlatform::class) && !class_exists(SQLServer2012Platform::class) ? new SQLServerPlatform() : new SQLServer2012Platform(), - 'SELECT m.* FROM messenger_messages m WITH (UPDLOCK, ROWLOCK) WHERE (m.queue_name = ?) AND (m.delivered_at is null OR m.delivered_at < ?) AND (m.available_at <= ?) ORDER BY available_at ASC OFFSET 0 ROWS FETCH NEXT 1 ROWS ONLY ', + sprintf('SELECT m.* FROM messenger_messages m WITH (UPDLOCK, ROWLOCK%s) WHERE (m.queue_name = ?) AND (m.delivered_at is null OR m.delivered_at < ?) AND (m.available_at <= ?) ORDER BY available_at ASC OFFSET 0 ROWS FETCH NEXT 1 ROWS ONLY ', method_exists(QueryBuilder::class, 'forUpdate') ? ', READPAST' : ''), ]; if (!class_exists(MySQL57Platform::class)) { // DBAL >= 4 yield 'Oracle' => [ new OraclePlatform(), - 'SELECT w.id AS "id", w.body AS "body", w.headers AS "headers", w.queue_name AS "queue_name", w.created_at AS "created_at", w.available_at AS "available_at", w.delivered_at AS "delivered_at" FROM messenger_messages w WHERE w.id IN (SELECT m.id FROM messenger_messages m WHERE (m.queue_name = ?) AND (m.delivered_at is null OR m.delivered_at < ?) AND (m.available_at <= ?) ORDER BY available_at ASC FETCH NEXT 1 ROWS ONLY) FOR UPDATE', + 'SELECT w.id AS "id", w.body AS "body", w.headers AS "headers", w.queue_name AS "queue_name", w.created_at AS "created_at", w.available_at AS "available_at", w.delivered_at AS "delivered_at" FROM messenger_messages w WHERE w.id IN (SELECT m.id FROM messenger_messages m WHERE (m.queue_name = ?) AND (m.delivered_at is null OR m.delivered_at < ?) AND (m.available_at <= ?) ORDER BY available_at ASC FETCH NEXT 1 ROWS ONLY) FOR UPDATE SKIP LOCKED', ]; } else { // DBAL < 4 yield 'Oracle' => [ new OraclePlatform(), - 'SELECT w.id AS "id", w.body AS "body", w.headers AS "headers", w.queue_name AS "queue_name", w.created_at AS "created_at", w.available_at AS "available_at", w.delivered_at AS "delivered_at" FROM messenger_messages w WHERE w.id IN (SELECT a.id FROM (SELECT m.id FROM messenger_messages m WHERE (m.queue_name = ?) AND (m.delivered_at is null OR m.delivered_at < ?) AND (m.available_at <= ?) ORDER BY available_at ASC) a WHERE ROWNUM <= 1) FOR UPDATE', + sprintf('SELECT w.id AS "id", w.body AS "body", w.headers AS "headers", w.queue_name AS "queue_name", w.created_at AS "created_at", w.available_at AS "available_at", w.delivered_at AS "delivered_at" FROM messenger_messages w WHERE w.id IN (SELECT a.id FROM (SELECT m.id FROM messenger_messages m WHERE (m.queue_name = ?) AND (m.delivered_at is null OR m.delivered_at < ?) AND (m.available_at <= ?) ORDER BY available_at ASC) a WHERE ROWNUM <= 1) FOR UPDATE%s', method_exists(QueryBuilder::class, 'forUpdate') ? ' SKIP LOCKED' : ''), ]; } } diff --git a/Tests/Transport/DoctrineIntegrationTest.php b/Tests/Transport/DoctrineIntegrationTest.php index 3310aac..fbb50bf 100644 --- a/Tests/Transport/DoctrineIntegrationTest.php +++ b/Tests/Transport/DoctrineIntegrationTest.php @@ -191,6 +191,7 @@ public function testItRetrieveTheMessageThatIsOlderThanRedeliverTimeout() public function testTheTransportIsSetupOnGet() { + $this->driverConnection->executeStatement('CREATE TABLE unrelated (unknown_type_column)'); $this->assertFalse($this->driverConnection->createSchemaManager()->tablesExist(['messenger_messages'])); $this->assertNull($this->connection->get()); diff --git a/Tests/Transport/DoctrinePostgreSqlIntegrationTest.php b/Tests/Transport/DoctrinePostgreSqlIntegrationTest.php index 60f39eb..392f991 100644 --- a/Tests/Transport/DoctrinePostgreSqlIntegrationTest.php +++ b/Tests/Transport/DoctrinePostgreSqlIntegrationTest.php @@ -64,4 +64,17 @@ public function testPostgreSqlConnectionSendAndGet() $this->assertNull($this->connection->get()); } + + public function testSkipLocked() + { + $connection = new PostgreSqlConnection(['table_name' => 'queue_table', 'skip_locked' => true], $this->driverConnection); + + $connection->send('{"message": "Hi"}', ['type' => DummyMessage::class]); + + $encoded = $connection->get(); + $this->assertEquals('{"message": "Hi"}', $encoded['body']); + $this->assertEquals(['type' => DummyMessage::class], $encoded['headers']); + + $this->assertNull($connection->get()); + } } diff --git a/Tests/Transport/DoctrinePostgreSqlRegularIntegrationTest.php b/Tests/Transport/DoctrinePostgreSqlRegularIntegrationTest.php index c8abede..f462d45 100644 --- a/Tests/Transport/DoctrinePostgreSqlRegularIntegrationTest.php +++ b/Tests/Transport/DoctrinePostgreSqlRegularIntegrationTest.php @@ -57,6 +57,20 @@ public function testSendAndGetWithAutoSetupEnabledAndSetupAlready() $this->assertNull($this->connection->get()); } + public function testSendAndGetWithSkipLockedEnabled() + { + $connection = new Connection(['table_name' => 'queue_table', 'skip_locked' => true], $this->driverConnection); + $connection->setup(); + + $connection->send('{"message": "Hi"}', ['type' => DummyMessage::class]); + + $encoded = $connection->get(); + $this->assertSame('{"message": "Hi"}', $encoded['body']); + $this->assertSame(['type' => DummyMessage::class], $encoded['headers']); + + $this->assertNull($this->connection->get()); + } + protected function setUp(): void { if (!$host = getenv('POSTGRES_HOST')) { diff --git a/Tests/Transport/DoctrineReceiverTest.php b/Tests/Transport/DoctrineReceiverTest.php index 3ec11c6..b37b7db 100644 --- a/Tests/Transport/DoctrineReceiverTest.php +++ b/Tests/Transport/DoctrineReceiverTest.php @@ -119,6 +119,188 @@ public function testFind() $this->assertEquals(new DummyMessage('Hi'), $actualEnvelope->getMessage()); } + public function testAck() + { + $serializer = $this->createSerializer(); + $connection = $this->createMock(Connection::class); + + $envelope = new Envelope(new \stdClass(), [new DoctrineReceivedStamp('1')]); + $receiver = new DoctrineReceiver($connection, $serializer); + + $connection + ->expects($this->once()) + ->method('ack') + ->with('1') + ->willReturn(true); + + $receiver->ack($envelope); + } + + public function testAckThrowsRetryableException() + { + $serializer = $this->createSerializer(); + $connection = $this->createMock(Connection::class); + + $envelope = new Envelope(new \stdClass(), [new DoctrineReceivedStamp('1')]); + $receiver = new DoctrineReceiver($connection, $serializer); + + $driverException = class_exists(Exception::class) ? Exception::new(new \PDOException('Deadlock', 40001)) : new PDOException(new \PDOException('Deadlock', 40001)); + if (!class_exists(Version::class)) { + // This is doctrine/dbal 3.x + $deadlockException = new DeadlockException($driverException, null); + } else { + $deadlockException = new DeadlockException('Deadlock', $driverException); + } + + $connection + ->expects($this->exactly(2)) + ->method('ack') + ->with('1') + ->willReturnOnConsecutiveCalls( + $this->throwException($deadlockException), + true, + ); + + $receiver->ack($envelope); + } + + public function testAckThrowsRetryableExceptionAndRetriesFail() + { + $serializer = $this->createSerializer(); + $connection = $this->createMock(Connection::class); + + $envelope = new Envelope(new \stdClass(), [new DoctrineReceivedStamp('1')]); + $receiver = new DoctrineReceiver($connection, $serializer); + + $driverException = class_exists(Exception::class) ? Exception::new(new \PDOException('Deadlock', 40001)) : new PDOException(new \PDOException('Deadlock', 40001)); + if (!class_exists(Version::class)) { + // This is doctrine/dbal 3.x + $deadlockException = new DeadlockException($driverException, null); + } else { + $deadlockException = new DeadlockException('Deadlock', $driverException); + } + + $connection + ->expects($this->exactly(4)) + ->method('ack') + ->with('1') + ->willThrowException($deadlockException); + + self::expectException(TransportException::class); + $receiver->ack($envelope); + } + + public function testAckThrowsException() + { + $serializer = $this->createSerializer(); + $connection = $this->createMock(Connection::class); + + $envelope = new Envelope(new \stdClass(), [new DoctrineReceivedStamp('1')]); + $receiver = new DoctrineReceiver($connection, $serializer); + + $exception = new \RuntimeException(); + + $connection + ->expects($this->once()) + ->method('ack') + ->with('1') + ->willThrowException($exception); + + self::expectException($exception::class); + $receiver->ack($envelope); + } + + public function testReject() + { + $serializer = $this->createSerializer(); + $connection = $this->createMock(Connection::class); + + $envelope = new Envelope(new \stdClass(), [new DoctrineReceivedStamp('1')]); + $receiver = new DoctrineReceiver($connection, $serializer); + + $connection + ->expects($this->once()) + ->method('reject') + ->with('1') + ->willReturn(true); + + $receiver->reject($envelope); + } + + public function testRejectThrowsRetryableException() + { + $serializer = $this->createSerializer(); + $connection = $this->createMock(Connection::class); + + $envelope = new Envelope(new \stdClass(), [new DoctrineReceivedStamp('1')]); + $receiver = new DoctrineReceiver($connection, $serializer); + + $driverException = class_exists(Exception::class) ? Exception::new(new \PDOException('Deadlock', 40001)) : new PDOException(new \PDOException('Deadlock', 40001)); + if (!class_exists(Version::class)) { + // This is doctrine/dbal 3.x + $deadlockException = new DeadlockException($driverException, null); + } else { + $deadlockException = new DeadlockException('Deadlock', $driverException); + } + + $connection + ->expects($this->exactly(2)) + ->method('reject') + ->with('1') + ->willReturnOnConsecutiveCalls( + $this->throwException($deadlockException), + true, + ); + + $receiver->reject($envelope); + } + + public function testRejectThrowsRetryableExceptionAndRetriesFail() + { + $serializer = $this->createSerializer(); + $connection = $this->createMock(Connection::class); + + $envelope = new Envelope(new \stdClass(), [new DoctrineReceivedStamp('1')]); + $receiver = new DoctrineReceiver($connection, $serializer); + + $driverException = class_exists(Exception::class) ? Exception::new(new \PDOException('Deadlock', 40001)) : new PDOException(new \PDOException('Deadlock', 40001)); + if (!class_exists(Version::class)) { + // This is doctrine/dbal 3.x + $deadlockException = new DeadlockException($driverException, null); + } else { + $deadlockException = new DeadlockException('Deadlock', $driverException); + } + + $connection + ->expects($this->exactly(4)) + ->method('reject') + ->with('1') + ->willThrowException($deadlockException); + + self::expectException(TransportException::class); + $receiver->reject($envelope); + } + + public function testRejectThrowsException() + { + $serializer = $this->createSerializer(); + $connection = $this->createMock(Connection::class); + + $envelope = new Envelope(new \stdClass(), [new DoctrineReceivedStamp('1')]); + $receiver = new DoctrineReceiver($connection, $serializer); + + $exception = new \RuntimeException(); + + $connection + ->expects($this->once()) + ->method('reject') + ->with('1') + ->willThrowException($exception); + + self::expectException($exception::class); + $receiver->reject($envelope); + } + private function createDoctrineEnvelope(): array { return [ diff --git a/Transport/Connection.php b/Transport/Connection.php index e0b0c8c..dc4c5d8 100644 --- a/Transport/Connection.php +++ b/Transport/Connection.php @@ -19,6 +19,7 @@ use Doctrine\DBAL\Platforms\AbstractMySQLPlatform; use Doctrine\DBAL\Platforms\OraclePlatform; use Doctrine\DBAL\Platforms\PostgreSQLPlatform; +use Doctrine\DBAL\Query\ForUpdate\ConflictResolutionMode; use Doctrine\DBAL\Query\QueryBuilder; use Doctrine\DBAL\Result; use Doctrine\DBAL\Schema\Schema; @@ -182,15 +183,22 @@ public function get(): ?array ->setParameters($query->getParameters(), $query->getParameterTypes()); if (method_exists(QueryBuilder::class, 'forUpdate')) { - $query->forUpdate(); + $query->forUpdate(ConflictResolutionMode::SKIP_LOCKED); } $sql = $query->getSQL(); } elseif (method_exists(QueryBuilder::class, 'forUpdate')) { - $query->forUpdate(); + $query->forUpdate(ConflictResolutionMode::SKIP_LOCKED); try { $sql = $query->getSQL(); } catch (DBALException $e) { + // If SKIP_LOCKED is not supported, fallback to without SKIP_LOCKED + $query->forUpdate(); + + try { + $sql = $query->getSQL(); + } catch (DBALException $e) { + } } } elseif (preg_match('/FROM (.+) WHERE/', (string) $sql, $matches)) { $fromClause = $matches[1]; @@ -281,7 +289,7 @@ public function setup(): void { $configuration = $this->driverConnection->getConfiguration(); $assetFilter = $configuration->getSchemaAssetsFilter(); - $configuration->setSchemaAssetsFilter(static fn () => true); + $configuration->setSchemaAssetsFilter(fn (string $tableName) => $tableName === $this->configuration['table_name']); $this->updateSchema(); $configuration->setSchemaAssetsFilter($assetFilter); $this->autoSetup = false; diff --git a/Transport/DoctrineReceiver.php b/Transport/DoctrineReceiver.php index 2f6e4a5..20bd611 100644 --- a/Transport/DoctrineReceiver.php +++ b/Transport/DoctrineReceiver.php @@ -67,20 +67,16 @@ public function get(): iterable public function ack(Envelope $envelope): void { - try { + $this->withRetryableExceptionRetry(function() use ($envelope) { $this->connection->ack($this->findDoctrineReceivedStamp($envelope)->getId()); - } catch (DBALException $exception) { - throw new TransportException($exception->getMessage(), 0, $exception); - } + }); } public function reject(Envelope $envelope): void { - try { + $this->withRetryableExceptionRetry(function() use ($envelope) { $this->connection->reject($this->findDoctrineReceivedStamp($envelope)->getId()); - } catch (DBALException $exception) { - throw new TransportException($exception->getMessage(), 0, $exception); - } + }); } public function getMessageCount(): int @@ -150,4 +146,32 @@ private function createEnvelopeFromData(array $data): Envelope new TransportMessageIdStamp($data['id']) ); } + + private function withRetryableExceptionRetry(callable $callable): void + { + $delay = 100; + $multiplier = 2; + $jitter = 0.1; + $retries = 0; + + retry: + try { + $callable(); + } catch (RetryableException $exception) { + if (++$retries <= self::MAX_RETRIES) { + $delay *= $multiplier; + + $randomness = (int) ($delay * $jitter); + $delay += random_int(-$randomness, +$randomness); + + usleep($delay * 1000); + + goto retry; + } + + throw new TransportException($exception->getMessage(), 0, $exception); + } catch (DBALException $exception) { + throw new TransportException($exception->getMessage(), 0, $exception); + } + } }