symfony · andythorne · Jul 21, 2023 · Jul 21, 2023 · Jul 21, 2023 · Jul 21, 2023
diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml
@@ -99,7 +99,7 @@ jobs:
         ports:
           - 9092:9092
         env:
-          KAFKA_AUTO_CREATE_TOPICS_ENABLE: false
+          KAFKA_AUTO_CREATE_TOPICS_ENABLE: true
           KAFKA_CREATE_TOPICS: 'test-topic:1:1:compact'
           KAFKA_ADVERTISED_HOST_NAME: 127.0.0.1
           KAFKA_ZOOKEEPER_CONNECT: 'zookeeper:2181'

diff --git a/.github/workflows/psalm.yml b/.github/workflows/psalm.yml
@@ -22,10 +22,21 @@ jobs:
     env:
       php-version: '8.1'
     steps:
+      - name: Install system dependencies
+        run: |
+          echo "::group::apt-get update"
+          sudo apt-get update
+          echo "::endgroup::"
+
+          echo "::group::install tools & libraries"
+          sudo apt-get install librdkafka-dev
+          echo "::endgroup::"
+
       - name: Setup PHP
         uses: shivammathur/setup-php@v2
         with:
           php-version: ${{ env.php-version }}
+          extensions: "rdkafka"
           ini-values: "memory_limit=-1"
           coverage: none
 

diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml
@@ -21,7 +21,7 @@ jobs:
     name: Unit Tests
 
     env:
-      extensions: amqp,apcu,igbinary,intl,mbstring,memcached,redis,relay
+      extensions: amqp,apcu,igbinary,intl,mbstring,memcached,redis,relay,rdkafka
 
     strategy:
       matrix:
@@ -43,6 +43,16 @@ jobs:
         with:
           fetch-depth: 2
 
+      - name: Install system dependencies
+        run: |
+          echo "::group::apt-get update"
+          sudo apt-get update
+          echo "::endgroup::"
+
+          echo "::group::install tools & libraries"
+          sudo apt-get install librdkafka-dev
+          echo "::endgroup::"
+
       - name: Setup PHP
         uses: shivammathur/setup-php@v2
         with:

diff --git a/src/Symfony/Bundle/FrameworkBundle/DependencyInjection/Compiler/UnusedTagsPass.php b/src/Symfony/Bundle/FrameworkBundle/DependencyInjection/Compiler/UnusedTagsPass.php
@@ -68,6 +68,7 @@ class UnusedTagsPass implements CompilerPassInterface
         'messenger.bus',
         'messenger.message_handler',
         'messenger.receiver',
+        'messenger.transport.kafka.callback_processor',
         'messenger.transport_factory',
         'mime.mime_type_guesser',
         'monolog.logger',

diff --git a/src/Symfony/Bundle/FrameworkBundle/DependencyInjection/FrameworkExtension.php b/src/Symfony/Bundle/FrameworkBundle/DependencyInjection/FrameworkExtension.php
@@ -113,6 +113,9 @@
 use Symfony\Component\Mercure\HubRegistry;
 use Symfony\Component\Messenger\Attribute\AsMessageHandler;
 use Symfony\Component\Messenger\Bridge as MessengerBridge;
+use Symfony\Component\Messenger\Bridge\Kafka\Callback\CallbackManager;
+use Symfony\Component\Messenger\Bridge\Kafka\Callback\CallbackProcessorInterface;
+use Symfony\Component\Messenger\Bridge\Kafka\Transport\KafkaFactory;
 use Symfony\Component\Messenger\Command\StatsCommand;
 use Symfony\Component\Messenger\EventListener\StopWorkerOnSignalsListener;
 use Symfony\Component\Messenger\Handler\BatchHandlerInterface;
@@ -2151,6 +2154,13 @@ private function registerMessengerConfiguration(array $config, ContainerBuilder
             $container->getDefinition('messenger.transport.beanstalkd.factory')->addTag('messenger.transport_factory');
         }
 
+        if (ContainerBuilder::willBeAvailable('symfony/kafka-messenger', MessengerBridge\Kafka\Transport\KafkaTransportFactory::class, ['symfony/framework-bundle', 'symfony/messenger'])) {
+            $container->getDefinition('messenger.transport.kafka.factory')->addTag('messenger.transport_factory');
+
+            $container->registerForAutoconfiguration(CallbackProcessorInterface::class)
+                ->addTag('messenger.transport.kafka.callback_processor');
+        }
+
         if (!class_exists(StopWorkerOnSignalsListener::class)) {
             $container->removeDefinition('messenger.listener.stop_worker_signals_listener');
         } elseif ($config['stop_worker_on_signals']) {
@@ -2213,6 +2223,9 @@ private function registerMessengerConfiguration(array $config, ContainerBuilder
             $container->removeDefinition('messenger.transport.redis.factory');
             $container->removeDefinition('messenger.transport.sqs.factory');
             $container->removeDefinition('messenger.transport.beanstalkd.factory');
+            $container->removeDefinition('messenger.transport.kafka.factory');
+            $container->removeDefinition(CallbackManager::class);
+            $container->removeDefinition(KafkaFactory::class);
             $container->removeAlias(SerializerInterface::class);
         } else {
             $container->getDefinition('messenger.transport.symfony_serializer')

diff --git a/src/Symfony/Bundle/FrameworkBundle/Resources/config/messenger.php b/src/Symfony/Bundle/FrameworkBundle/Resources/config/messenger.php
@@ -15,6 +15,9 @@
 use Symfony\Component\Messenger\Bridge\AmazonSqs\Transport\AmazonSqsTransportFactory;
 use Symfony\Component\Messenger\Bridge\Amqp\Transport\AmqpTransportFactory;
 use Symfony\Component\Messenger\Bridge\Beanstalkd\Transport\BeanstalkdTransportFactory;
+use Symfony\Component\Messenger\Bridge\Kafka\Callback\CallbackManager;
+use Symfony\Component\Messenger\Bridge\Kafka\Transport\KafkaFactory;
+use Symfony\Component\Messenger\Bridge\Kafka\Transport\KafkaTransportFactory;
 use Symfony\Component\Messenger\Bridge\Redis\Transport\RedisTransportFactory;
 use Symfony\Component\Messenger\EventListener\AddErrorDetailsStampListener;
 use Symfony\Component\Messenger\EventListener\DispatchPcntlSignalListener;
@@ -147,6 +150,19 @@
 
         ->set('messenger.transport.beanstalkd.factory', BeanstalkdTransportFactory::class)
 
+        ->set(CallbackManager::class)
+            ->args([
+                tagged_iterator('messenger.transport.kafka.callback_processor'),
+            ])
+        ->set(KafkaFactory::class)
+            ->args([
+                service(CallbackManager::class),
+            ])
+        ->set('messenger.transport.kafka.factory', KafkaTransportFactory::class)
+            ->args([
+                service(KafkaFactory::class),
+            ])
+
         // retry
         ->set('messenger.retry_strategy_locator', ServiceLocator::class)
             ->args([

diff --git a/src/Symfony/Bundle/FrameworkBundle/Tests/DependencyInjection/FrameworkExtensionTestCase.php b/src/Symfony/Bundle/FrameworkBundle/Tests/DependencyInjection/FrameworkExtensionTestCase.php
@@ -59,6 +59,8 @@
 use Symfony\Component\Messenger\Bridge\AmazonSqs\Transport\AmazonSqsTransportFactory;
 use Symfony\Component\Messenger\Bridge\Amqp\Transport\AmqpTransportFactory;
 use Symfony\Component\Messenger\Bridge\Beanstalkd\Transport\BeanstalkdTransportFactory;
+use Symfony\Component\Messenger\Bridge\Kafka\Callback\CallbackManager;
+use Symfony\Component\Messenger\Bridge\Kafka\Transport\KafkaTransportFactory;
 use Symfony\Component\Messenger\Bridge\Redis\Transport\RedisTransportFactory;
 use Symfony\Component\Messenger\Transport\TransportFactory;
 use Symfony\Component\Notifier\ChatterInterface;
@@ -841,6 +843,12 @@ public function testMessenger()
             $expectedFactories[] = 'messenger.transport.beanstalkd.factory';
         }
 
+        if (class_exists(KafkaTransportFactory::class)) {
+            $expectedFactories[] = 'messenger.transport.kafka.factory';
+
+            $this->assertTrue($container->hasDefinition(CallbackManager::class));
+        }
+
         $this->assertTrue($container->hasDefinition('messenger.receiver_locator'));
         $this->assertTrue($container->hasDefinition('console.command.messenger_consume_messages'));
         $this->assertTrue($container->hasAlias('messenger.default_bus'));

diff --git a/src/Symfony/Component/Messenger/Bridge/Kafka/.gitattributes b/src/Symfony/Component/Messenger/Bridge/Kafka/.gitattributes
@@ -0,0 +1,4 @@
+/Tests export-ignore
+/phpunit.xml.dist export-ignore
+/.gitattributes export-ignore
+/.gitignore export-ignore
diff --git a/src/Symfony/Component/Messenger/Bridge/Kafka/.gitignore b/src/Symfony/Component/Messenger/Bridge/Kafka/.gitignore
@@ -0,0 +1,3 @@
+vendor/
+composer.lock
+phpunit.xml
diff --git a/src/Symfony/Component/Messenger/Bridge/Kafka/CHANGELOG.md b/src/Symfony/Component/Messenger/Bridge/Kafka/CHANGELOG.md
@@ -0,0 +1,7 @@
+CHANGELOG
+=========
+
+6.4
-6.4
+7.4
-6.4
+7.4
+---
+
+ * Introduce the Kafka bridge.
diff --git a/src/Symfony/Component/Messenger/Bridge/Kafka/Callback/AbstractCallbackProcessor.php b/src/Symfony/Component/Messenger/Bridge/Kafka/Callback/AbstractCallbackProcessor.php
@@ -0,0 +1,51 @@
+<?php
+
+/*
+ * This file is part of the Symfony package.
+ *
+ * (c) Fabien Potencier <[email protected]>
+ *
+ * For the full copyright and license information, please view the LICENSE
+ * file that was distributed with this source code.
+ */
+
+namespace Symfony\Component\Messenger\Bridge\Kafka\Callback;
+
+use RdKafka\KafkaConsumer;
+use RdKafka\Message;
+use RdKafka\Producer;
+
+abstract class AbstractCallbackProcessor implements CallbackProcessorInterface
+{
+    public function log(object $kafka, int $level, string $facility, string $message): void
+    {
+    }
+
+    public function consumerError(KafkaConsumer $kafka, int $err, string $reason): void
+    {
+    }
+
+    public function producerError(Producer $kafka, int $err, string $reason): void
+    {
+    }
+
+    public function stats(object $kafka, string $json, int $jsonLength): void
+    {
+    }
+
+    public function rebalance(KafkaConsumer $kafka, int $err, array $partitions): void
+    {
+    }
+
+    public function consume(Message $message): void
+    {
+    }
+
+    public function offsetCommit(object $kafka, int $err, array $partitions): void
+    {
+    }
+
+    public function deliveryReport(object $kafka, Message $message): void
+    {
+    }
+}
diff --git a/src/Symfony/Component/Messenger/Bridge/Kafka/Callback/CallbackManager.php b/src/Symfony/Component/Messenger/Bridge/Kafka/Callback/CallbackManager.php
@@ -0,0 +1,93 @@
+<?php
+
+/*
+ * This file is part of the Symfony package.
+ *
+ * (c) Fabien Potencier <[email protected]>
+ *
+ * For the full copyright and license information, please view the LICENSE
+ * file that was distributed with this source code.
+ */
+
+namespace Symfony\Component\Messenger\Bridge\Kafka\Callback;
+
+use RdKafka\KafkaConsumer;
+use RdKafka\Message;
+use RdKafka\Producer;
+use RdKafka\TopicPartition;
+
+/**
+ * @see https://arnaud.le-blanc.net/php-rdkafka-doc/phpdoc/class.rdkafka-conf.html for more information on callback parameters.
+ */
+final class CallbackManager
+{
+    /**
+     * @param list<CallbackProcessorInterface> $callbackProcessors
+     */
+    public function __construct(
+        private readonly iterable $callbackProcessors,
+    ) {
+    }
+
+    public function log(object $kafka, int $level, string $facility, string $message): void
+    {
+        foreach ($this->callbackProcessors as $callbackProcessor) {
+            $callbackProcessor->log($kafka, $level, $facility, $message);
+        }
+    }
+
+    public function consumerError(KafkaConsumer $kafka, int $err, string $reason): void
+    {
+        foreach ($this->callbackProcessors as $callbackProcessor) {
+            $callbackProcessor->consumerError($kafka, $err, $reason);
+        }
+    }
+
+    public function producerError(Producer $kafka, int $err, string $reason): void
+    {
+        foreach ($this->callbackProcessors as $callbackProcessor) {
+            $callbackProcessor->producerError($kafka, $err, $reason);
+        }
+    }
+
+    public function stats(object $kafka, string $json, int $jsonLength): void
+    {
+        foreach ($this->callbackProcessors as $callbackProcessor) {
+            $callbackProcessor->stats($kafka, $json, $jsonLength);
+        }
+    }
+
+    /**
+     * @param list<TopicPartition> $partitions
+     */
+    public function rebalance(KafkaConsumer $kafka, int $err, array $partitions): void
+    {
+        foreach ($this->callbackProcessors as $callbackProcessor) {
+            $callbackProcessor->rebalance($kafka, $err, $partitions);
+        }
+    }
+
+    public function consume(Message $message): void
+    {
+        foreach ($this->callbackProcessors as $callbackProcessor) {
+            $callbackProcessor->consume($message);
+        }
+    }
+
+    /**
+     * @param list<TopicPartition> $partitions
+     */
+    public function offsetCommit(object $kafka, int $err, array $partitions): void
+    {
+        foreach ($this->callbackProcessors as $callbackProcessor) {
+            $callbackProcessor->offsetCommit($kafka, $err, $partitions);
+        }
+    }
+
+    public function deliveryReport(object $kafka, Message $message): void
+    {
+        foreach ($this->callbackProcessors as $callbackProcessor) {
+            $callbackProcessor->deliveryReport($kafka, $message);
+        }
+    }
+}
diff --git a/src/Symfony/Component/Messenger/Bridge/Kafka/Callback/CallbackProcessorInterface.php b/src/Symfony/Component/Messenger/Bridge/Kafka/Callback/CallbackProcessorInterface.php
@@ -0,0 +1,45 @@
+<?php
+
+/*
+ * This file is part of the Symfony package.
+ *
+ * (c) Fabien Potencier <[email protected]>
+ *
+ * For the full copyright and license information, please view the LICENSE
+ * file that was distributed with this source code.
+ */
+
+namespace Symfony\Component\Messenger\Bridge\Kafka\Callback;
+
+use RdKafka\KafkaConsumer;
+use RdKafka\Message;
+use RdKafka\Producer;
+use RdKafka\TopicPartition;
+
+/**
+ * @see https://arnaud.le-blanc.net/php-rdkafka-doc/phpdoc/class.rdkafka-conf.html for more information on callback parameters.
+ */
+interface CallbackProcessorInterface
+{
+    public function log(object $kafka, int $level, string $facility, string $message): void;
+
+    public function consumerError(KafkaConsumer $kafka, int $err, string $reason): void;
+
+    public function producerError(Producer $kafka, int $err, string $reason): void;
+
+    public function stats(object $kafka, string $json, int $jsonLength): void;
+
+    /**
+     * @param list<TopicPartition> $partitions
+     */
+    public function rebalance(KafkaConsumer $kafka, int $err, array $partitions): void;
+
+    public function consume(Message $message): void;
+
+    /**
+     * @param list<TopicPartition> $partitions
+     */
+    public function offsetCommit(object $kafka, int $err, array $partitions): void;
+
+    public function deliveryReport(object $kafka, Message $message): void;
+}