Add handling for delayed message to redis transport

alexander-schranz · alexander-schranz · commit e3a27626aef1 · 2019-07-09T10:42:10.000+02:00
diff --git a/src/Symfony/Component/Messenger/Transport/RedisExt/Connection.php b/src/Symfony/Component/Messenger/Transport/RedisExt/Connection.php
@@ -36,6 +36,7 @@ class Connection
 
     private $connection;
     private $stream;
+    private $queue;
     private $group;
     private $consumer;
     private $autoSetup;
@@ -57,6 +58,7 @@ public function __construct(array $configuration, array $connectionCredentials =
         }
 
         $this->stream = $configuration['stream'] ?? self::DEFAULT_OPTIONS['stream'];
+        $this->queue = $this->stream.'_queue';
         $this->group = $configuration['group'] ?? self::DEFAULT_OPTIONS['group'];
         $this->consumer = $configuration['consumer'] ?? self::DEFAULT_OPTIONS['consumer'];
         $this->autoSetup = $configuration['auto_setup'] ?? self::DEFAULT_OPTIONS['auto_setup'];
@@ -112,6 +114,18 @@ public function get(): ?array
             $this->setup();
         }
 
+        $queuedMessageCount = $this->connection->zcount($this->queue, 0, time());
+
+        if ($queuedMessageCount) {
+            foreach ($this->connection->zpopmin($this->queue, $queuedMessageCount) as $queuedMessage => $time) {
+                $queuedMessage = json_encode($queuedMessage, true);
+                // if a futured placed message is actually popped because of a race condition with
+                // another running message consumer, the message is readded to the queue by add function
+                // else its just added stream and will be available for all stream consumers
+                $this->add($queuedMessage['body'], $queuedMessage['headers'], (time() - $time) * 1000);
+            }
+        }
+
         $messageId = '>'; // will receive new messages
 
         if ($this->couldHavePendingMessages) {
@@ -182,22 +196,27 @@ public function reject(string $id): void
         }
     }
 
-    public function add(string $body, array $headers): void
+    /**
+     * @param int $delay The delay in milliseconds
+     */
+    public function add(string $body, array $headers, int $delay = 0): void
     {
         if ($this->autoSetup) {
             $this->setup();
         }
 
         $e = null;
         try {
-            if ($this->maxEntries) {
-                $added = $this->connection->xadd($this->stream, '*', ['message' => json_encode(
-                    ['body' => $body, 'headers' => $headers]
-                )], $this->maxEntries, true);
+            $message = json_encode(['body' => $body, 'headers' => $headers]);
+
+            if ($delay > 0) { // the delay could be smaller 0 in a queued message
+                $added = $this->connection->zadd($this->queue, ['NX'], time() + ($delay / 1000), $message);
             } else {
-                $added = $this->connection->xadd($this->stream, '*', ['message' => json_encode(
-                    ['body' => $body, 'headers' => $headers]
-                )]);
+                if ($this->maxEntries) {
+                    $added = $this->connection->xadd($this->stream, '*', ['message' => $message], $this->maxEntries, true);
+                } else {
+                    $added = $this->connection->xadd($this->stream, '*', ['message' => $message]);
+                }
             }
         } catch (\RedisException $e) {
         }
diff --git a/src/Symfony/Component/Messenger/Transport/RedisExt/RedisSender.php b/src/Symfony/Component/Messenger/Transport/RedisExt/RedisSender.php
@@ -12,6 +12,7 @@
 namespace Symfony\Component\Messenger\Transport\RedisExt;
 
 use Symfony\Component\Messenger\Envelope;
+use Symfony\Component\Messenger\Stamp\DelayStamp;
 use Symfony\Component\Messenger\Transport\Sender\SenderInterface;
 use Symfony\Component\Messenger\Transport\Serialization\SerializerInterface;
 
@@ -37,7 +38,11 @@ public function send(Envelope $envelope): Envelope
     {
         $encodedMessage = $this->serializer->encode($envelope);
 
-        $this->connection->add($encodedMessage['body'], $encodedMessage['headers'] ?? []);
+        /** @var DelayStamp|null $delayStamp */
+        $delayStamp = $envelope->last(DelayStamp::class);
+        $delay = null !== $delayStamp ? $delayStamp->getDelay() : 0;
+
+        $this->connection->add($encodedMessage['body'], $encodedMessage['headers'] ?? [], $delay);
 
         return $envelope;
     }
