Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit cfece10

Browse files
alexander-schranzRobin Chalas
alexander-schranz
authored and
Robin Chalas
committed
Add handling for delayed message to redis transport
1 parent a0cefaa commit cfece10

File tree

5 files changed

+119
-18
lines changed

5 files changed

+119
-18
lines changed

‎.travis.yml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ env:
2020
- MIN_PHP=7.1.3
2121
- SYMFONY_PROCESS_PHP_TEST_BINARY=~/.phpenv/shims/php
2222
- MESSENGER_AMQP_DSN=amqp://localhost/%2f/messages
23-
- MESSENGER_REDIS_DSN=redis://127.0.0.1:7001/messages
23+
- MESSENGER_REDIS_DSN=redis://127.0.0.1:7006/messages
2424
- SYMFONY_PHPUNIT_DISABLE_RESULT_CACHE=1
2525

2626
matrix:
@@ -59,7 +59,7 @@ before_install:
5959
- |
6060
# Start Redis cluster
6161
docker pull grokzen/redis-cluster:5.0.4
62-
docker run -d -p 7000:7000 -p 7001:7001 -p 7002:7002 -p 7003:7003 -p 7004:7004 -p 7005:7005 --name redis-cluster grokzen/redis-cluster:5.0.4
62+
docker run -d -p 7000:7000 -p 7001:7001 -p 7002:7002 -p 7003:7003 -p 7004:7004 -p 7005:7005 -p 7006:7006 -p 7007:7007 -e "STANDALONE=true" --name redis-cluster grokzen/redis-cluster:5.0.4
6363
export REDIS_CLUSTER_HOSTS='localhost:7000 localhost:7001 localhost:7002 localhost:7003 localhost:7004 localhost:7005'
6464
6565
- |

‎src/Symfony/Component/Messenger/CHANGELOG.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ CHANGELOG
2323
* [BC BREAK] Removed `StopWhenRestartSignalIsReceived` in favor of `StopWorkerOnRestartSignalListener`.
2424
* The component is not marked as `@experimental` anymore.
2525
* Marked the `MessengerDataCollector` class as `@final`.
26+
* Added support for `DelayStamp` to the `redis` transport.
2627

2728
4.3.0
2829
-----

‎src/Symfony/Component/Messenger/Tests/Transport/RedisExt/RedisExtIntegrationTest.php

Lines changed: 44 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717

1818
/**
1919
* @requires extension redis
20+
* @group time-sensitive
2021
*/
2122
class RedisExtIntegrationTest extends TestCase
2223
{
@@ -31,7 +32,7 @@ protected function setUp(): void
3132

3233
$this->redis = new \Redis();
3334
$this->connection = Connection::fromDsn(getenv('MESSENGER_REDIS_DSN'), [], $this->redis);
34-
$this->clearRedis();
35+
$this->connection->cleanup();
3536
$this->connection->setup();
3637
}
3738

@@ -55,11 +56,48 @@ public function testGetTheFirstAvailableMessage()
5556
$this->assertEquals(['type' => DummyMessage::class], $encoded['headers']);
5657
}
5758

58-
private function clearRedis()
59+
public function testConnectionSendWithSameContent()
5960
{
60-
$parsedUrl = parse_url(getenv('MESSENGER_REDIS_DSN'));
61-
$pathParts = explode('/', $parsedUrl['path'] ?? '');
62-
$stream = $pathParts[1] ?? 'symfony';
63-
$this->redis->del($stream);
61+
$body = '{"message": "Hi"}';
62+
$headers = ['type' => DummyMessage::class];
63+
64+
$this->connection->add($body, $headers);
65+
$this->connection->add($body, $headers);
66+
67+
$encoded = $this->connection->get();
68+
$this->assertEquals($body, $encoded['body']);
69+
$this->assertEquals($headers, $encoded['headers']);
70+
71+
$encoded = $this->connection->get();
72+
$this->assertEquals($body, $encoded['body']);
73+
$this->assertEquals($headers, $encoded['headers']);
74+
}
75+
76+
public function testConnectionSendAndGetDelayed()
77+
{
78+
$this->connection->add('{"message": "Hi"}', ['type' => DummyMessage::class], 500);
79+
$encoded = $this->connection->get();
80+
$this->assertNull($encoded);
81+
sleep(2);
82+
$encoded = $this->connection->get();
83+
$this->assertEquals('{"message": "Hi"}', $encoded['body']);
84+
$this->assertEquals(['type' => DummyMessage::class], $encoded['headers']);
85+
}
86+
87+
public function testConnectionSendDelayedMessagesWithSameContent()
88+
{
89+
$body = '{"message": "Hi"}';
90+
$headers = ['type' => DummyMessage::class];
91+
92+
$this->connection->add($body, $headers, 500);
93+
$this->connection->add($body, $headers, 500);
94+
sleep(2);
95+
$encoded = $this->connection->get();
96+
$this->assertEquals($body, $encoded['body']);
97+
$this->assertEquals($headers, $encoded['headers']);
98+
99+
$encoded = $this->connection->get();
100+
$this->assertEquals($body, $encoded['body']);
101+
$this->assertEquals($headers, $encoded['headers']);
64102
}
65103
}

‎src/Symfony/Component/Messenger/Transport/RedisExt/Connection.php

Lines changed: 66 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@ class Connection
3838

3939
private $connection;
4040
private $stream;
41+
private $queue;
4142
private $group;
4243
private $consumer;
4344
private $autoSetup;
@@ -65,6 +66,7 @@ public function __construct(array $configuration, array $connectionCredentials =
6566
$this->stream = $configuration['stream'] ?? self::DEFAULT_OPTIONS['stream'];
6667
$this->group = $configuration['group'] ?? self::DEFAULT_OPTIONS['group'];
6768
$this->consumer = $configuration['consumer'] ?? self::DEFAULT_OPTIONS['consumer'];
69+
$this->queue = $this->stream.'__queue';
6870
$this->autoSetup = $configuration['auto_setup'] ?? self::DEFAULT_OPTIONS['auto_setup'];
6971
$this->maxEntries = $configuration['stream_max_entries'] ?? self::DEFAULT_OPTIONS['stream_max_entries'];
7072
}
@@ -125,6 +127,34 @@ public function get(): ?array
125127
$this->setup();
126128
}
127129

130+
try {
131+
$queuedMessageCount = $this->connection->zcount($this->queue, 0, $this->getCurrentTimeInMilliseconds());
132+
} catch (\RedisException $e) {
133+
throw new TransportException($e->getMessage(), 0, $e);
134+
}
135+
136+
if ($queuedMessageCount) {
137+
for ($i = 0; $i < $queuedMessageCount; ++$i) {
138+
try {
139+
$queuedMessages = $this->connection->zpopmin($this->queue, 1);
140+
} catch (\RedisException $e) {
141+
throw new TransportException($e->getMessage(), 0, $e);
142+
}
143+
144+
foreach ($queuedMessages as $queuedMessage => $time) {
145+
$queuedMessage = json_decode($queuedMessage, true);
146+
// if a futured placed message is actually popped because of a race condition with
147+
// another running message consumer, the message is readded to the queue by add function
148+
// else its just added stream and will be available for all stream consumers
149+
$this->add(
150+
$queuedMessage['body'],
151+
$queuedMessage['headers'],
152+
$time - $this->getCurrentTimeInMilliseconds()
153+
);
154+
}
155+
}
156+
}
157+
128158
$messageId = '>'; // will receive new messages
129159

130160
if ($this->couldHavePendingMessages) {
@@ -203,24 +233,40 @@ public function reject(string $id): void
203233
}
204234
}
205235

206-
public function add(string $body, array $headers): void
236+
public function add(string $body, array $headers, int $delayInMs = 0): void
207237
{
208238
if ($this->autoSetup) {
209239
$this->setup();
210240
}
211241

212242
try {
213-
if ($this->maxEntries) {
214-
$added = $this->connection->xadd($this->stream, '*', ['message' => json_encode(
215-
['body' => $body, 'headers' => $headers]
216-
)], $this->maxEntries, true);
243+
if ($delayInMs > 0) { // the delay could be smaller 0 in a queued message
244+
$message = json_encode([
245+
'body' => $body,
246+
'headers' => $headers,
247+
// Entry need to be unique in the sorted set else it would only be added once to the delayed messages queue
248+
'uniqid' => uniqid('', true),
249+
]);
250+
251+
$score = (int) ($this->getCurrentTimeInMilliseconds() + $delayInMs);
252+
$added = $this->connection->zadd($this->queue, ['NX'], $score, $message);
217253
} else {
218-
$added = $this->connection->xadd($this->stream, '*', ['message' => json_encode(
219-
['body' => $body, 'headers' => $headers]
220-
)]);
254+
$message = json_encode([
255+
'body' => $body,
256+
'headers' => $headers,
257+
]);
258+
259+
if ($this->maxEntries) {
260+
$added = $this->connection->xadd($this->stream, '*', ['message' => $message], $this->maxEntries, true);
261+
} else {
262+
$added = $this->connection->xadd($this->stream, '*', ['message' => $message]);
263+
}
221264
}
222265
} catch (\RedisException $e) {
223-
throw new TransportException($e->getMessage(), 0, $e);
266+
if ($error = $this->connection->getLastError() ?: null) {
267+
$this->connection->clearLastError();
268+
}
269+
throw new TransportException($error ?? $e->getMessage(), 0, $e);
224270
}
225271

226272
if (!$added) {
@@ -246,4 +292,15 @@ public function setup(): void
246292

247293
$this->autoSetup = false;
248294
}
295+
296+
private function getCurrentTimeInMilliseconds(): int
297+
{
298+
return (int) (microtime(true) * 1000);
299+
}
300+
301+
public function cleanup(): void
302+
{
303+
$this->connection->del($this->stream);
304+
$this->connection->del($this->queue);
305+
}
249306
}

‎src/Symfony/Component/Messenger/Transport/RedisExt/RedisSender.php

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
namespace Symfony\Component\Messenger\Transport\RedisExt;
1313

1414
use Symfony\Component\Messenger\Envelope;
15+
use Symfony\Component\Messenger\Stamp\DelayStamp;
1516
use Symfony\Component\Messenger\Transport\Sender\SenderInterface;
1617
use Symfony\Component\Messenger\Transport\Serialization\SerializerInterface;
1718

@@ -37,7 +38,11 @@ public function send(Envelope $envelope): Envelope
3738
{
3839
$encodedMessage = $this->serializer->encode($envelope);
3940

40-
$this->connection->add($encodedMessage['body'], $encodedMessage['headers'] ?? []);
41+
/** @var DelayStamp|null $delayStamp */
42+
$delayStamp = $envelope->last(DelayStamp::class);
43+
$delayInMs = null !== $delayStamp ? $delayStamp->getDelay() : 0;
44+
45+
$this->connection->add($encodedMessage['body'], $encodedMessage['headers'] ?? [], $delayInMs);
4146

4247
return $envelope;
4348
}

0 commit comments

Comments
 (0)