Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit 7bd0a27

Browse files
srozesroze
committed
feature #31825 [Messenger] Added support for auto trimming of redis streams (Toflar)
This PR was squashed before being merged into the 4.4 branch (closes #31825). Discussion ---------- [Messenger] Added support for auto trimming of redis streams | Q | A | ------------- | --- | Branch? | 4.4 | Bug fix? | no | New feature? | yes | BC breaks? | no | Deprecations? | no | Tests pass? | yes | Fixed tickets | | License | MIT | Doc PR | will submit if concept is okay Right now the Redis stream will just grow indefinitely. However, there are means to have it delete old entries from time to time. Note: I could not use the `XADD mystream MAXLEN ~ 1000 *` notation because the PHP redis extension does not support the `MAXLEN` option afaics so I went for the extra `XTRIM` command. I explicitly enabled the approximate flag because it makes absolutely no sense to hardcode the limit for us although we could even have this configurable too (but I don't think we should). The whole idea of this PR is to enable occasional trimming of the stream so it doesn't grow forever, so when you configure something like `20000` it may well happen that trimming only happens at `25000` depending on your settings. Ping @soyuka @alexander-schranz @chalasr :) Commits ------- 7fe06bc [Messenger] Added support for auto trimming of redis streams
2 parents d97f9ab + 7fe06bc commit 7bd0a27

File tree

3 files changed

+40
-6
lines changed

3 files changed

+40
-6
lines changed

‎src/Symfony/Component/Messenger/CHANGELOG.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ CHANGELOG
66

77
* Deprecated passing a `ContainerInterface` instance as first argument of the `ConsumeMessagesCommand` constructor,
88
pass a `RoutableMessageBus` instance instead.
9+
* Added support for auto trimming of Redis streams.
910

1011
4.3.0
1112
-----

‎src/Symfony/Component/Messenger/Tests/Transport/RedisExt/ConnectionTest.php

Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -42,13 +42,13 @@ public function testFromDsn()
4242
public function testFromDsnWithOptions()
4343
{
4444
$this->assertEquals(
45-
new Connection(['stream' => 'queue', 'group' => 'group1', 'consumer' => 'consumer1', 'auto_setup' => false], [
45+
new Connection(['stream' => 'queue', 'group' => 'group1', 'consumer' => 'consumer1', 'auto_setup' => false, 'stream_max_entries' => 20000], [
4646
'host' => 'localhost',
4747
'port' => 6379,
4848
], [
4949
'serializer' => 2,
5050
]),
51-
Connection::fromDsn('redis://localhost/queue/group1/consumer1', ['serializer' => 2, 'auto_setup' => false])
51+
Connection::fromDsn('redis://localhost/queue/group1/consumer1', ['serializer' => 2, 'auto_setup' => false, 'stream_max_entries' => 20000])
5252
);
5353
}
5454

@@ -142,4 +142,16 @@ public function testGetNonBlocking()
142142
$connection->reject($message['id']);
143143
$redis->del('messenger-getnonblocking');
144144
}
145+
146+
public function testMaxEntries()
147+
{
148+
$redis = $this->getMockBuilder(\Redis::class)->disableOriginalConstructor()->getMock();
149+
150+
$redis->expects($this->exactly(1))->method('xadd')
151+
->with('queue', '*', ['message' => '{"body":"1","headers":[]}'], 20000, true)
152+
->willReturn(1);
153+
154+
$connection = Connection::fromDsn('redis://localhost/queue?stream_max_entries=20000', [], $redis); // 1 = always
155+
$connection->add('1', []);
156+
}
145157
}

‎src/Symfony/Component/Messenger/Transport/RedisExt/Connection.php

Lines changed: 25 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -31,13 +31,15 @@ class Connection
3131
'group' => 'symfony',
3232
'consumer' => 'consumer',
3333
'auto_setup' => true,
34+
'stream_max_entries' => 0, // any value higher than 0 defines an approximate maximum number of stream entries
3435
];
3536

3637
private $connection;
3738
private $stream;
3839
private $group;
3940
private $consumer;
4041
private $autoSetup;
42+
private $maxEntries;
4143
private $couldHavePendingMessages = true;
4244

4345
public function __construct(array $configuration, array $connectionCredentials = [], array $redisOptions = [], \Redis $redis = null)
@@ -53,6 +55,7 @@ public function __construct(array $configuration, array $connectionCredentials =
5355
$this->group = $configuration['group'] ?? self::DEFAULT_OPTIONS['group'];
5456
$this->consumer = $configuration['consumer'] ?? self::DEFAULT_OPTIONS['consumer'];
5557
$this->autoSetup = $configuration['auto_setup'] ?? self::DEFAULT_OPTIONS['auto_setup'];
58+
$this->maxEntries = $configuration['stream_max_entries'] ?? self::DEFAULT_OPTIONS['stream_max_entries'];
5659
}
5760

5861
public static function fromDsn(string $dsn, array $redisOptions = [], \Redis $redis = null): self
@@ -82,7 +85,19 @@ public static function fromDsn(string $dsn, array $redisOptions = [], \Redis $re
8285
unset($redisOptions['auto_setup']);
8386
}
8487

85-
return new self(['stream' => $stream, 'group' => $group, 'consumer' => $consumer, 'auto_setup' => $autoSetup], $connectionCredentials, $redisOptions, $redis);
88+
$maxEntries = null;
89+
if (\array_key_exists('stream_max_entries', $redisOptions)) {
90+
$maxEntries = filter_var($redisOptions['stream_max_entries'], FILTER_VALIDATE_INT);
91+
unset($redisOptions['stream_max_entries']);
92+
}
93+
94+
return new self([
95+
'stream' => $stream,
96+
'group' => $group,
97+
'consumer' => $consumer,
98+
'auto_setup' => $autoSetup,
99+
'stream_max_entries' => $maxEntries,
100+
], $connectionCredentials, $redisOptions, $redis);
86101
}
87102

88103
public function get(): ?array
@@ -169,9 +184,15 @@ public function add(string $body, array $headers): void
169184

170185
$e = null;
171186
try {
172-
$added = $this->connection->xadd($this->stream, '*', ['message' => json_encode(
173-
['body' => $body, 'headers' => $headers]
174-
)]);
187+
if ($this->maxEntries) {
188+
$added = $this->connection->xadd($this->stream, '*', ['message' => json_encode(
189+
['body' => $body, 'headers' => $headers]
190+
)], $this->maxEntries, true);
191+
} else {
192+
$added = $this->connection->xadd($this->stream, '*', ['message' => json_encode(
193+
['body' => $body, 'headers' => $headers]
194+
)]);
195+
}
175196
} catch (\RedisException $e) {
176197
}
177198

0 commit comments

Comments
 (0)