Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit 5ece961

Browse files
andreromnicolas-grekas
andrerom
authored and
nicolas-grekas
committed
[Cache] Improve RedisTagAwareAdapter invalidation logic & requirements
1 parent a0bbae7 commit 5ece961

File tree

4 files changed

+86
-95
lines changed

4 files changed

+86
-95
lines changed

‎src/Symfony/Component/Cache/Adapter/RedisTagAwareAdapter.php

Lines changed: 72 additions & 49 deletions
Original file line numberDiff line numberDiff line change
@@ -11,33 +11,32 @@
1111

1212
namespace Symfony\Component\Cache\Adapter;
1313

14-
use Predis;
1514
use Predis\Connection\Aggregate\ClusterInterface;
15+
use Predis\Connection\Aggregate\PredisCluster;
1616
use Predis\Response\Status;
17-
use Symfony\Component\Cache\CacheItem;
18-
use Symfony\Component\Cache\Exception\LogicException;
17+
use Symfony\Component\Cache\Exception\InvalidArgumentException;
1918
use Symfony\Component\Cache\Marshaller\MarshallerInterface;
2019
use Symfony\Component\Cache\Traits\RedisTrait;
2120

2221
/**
23-
* Stores tag id <> cache id relationship as a Redis Set, lookup on invalidation using sPOP.
22+
* Stores tag id <> cache id relationship as a Redis Set, lookup on invalidation using RENAME+SMEMBERS.
2423
*
2524
* Set (tag relation info) is stored without expiry (non-volatile), while cache always gets an expiry (volatile) even
2625
* if not set by caller. Thus if you configure redis with the right eviction policy you can be safe this tag <> cache
2726
* relationship survives eviction (cache cleanup when Redis runs out of memory).
2827
*
2928
* Requirements:
30-
* - Server: Redis 3.2+
31-
* - Client: PHP Redis 3.1.3+ OR Predis
32-
* - Redis Server(s) configured with any `volatile-*` eviction policy, OR `noeviction` if it will NEVER fill up memory
29+
* - Client: PHP Redis or Predis
30+
* Note: Due to lack of RENAME support it is NOT recommended to use Cluster on Predis, instead use phpredis.
31+
* - Server: Redis 2.8+
32+
* Configured with any `volatile-*` eviction policy, OR `noeviction` if it will NEVER fill up memory
3333
*
3434
* Design limitations:
35-
* - Max 2 billion cache keys per cache tag
36-
* E.g. If you use a "all" items tag for expiry instead of clear(), that limits you to 2 billion cache items as well
35+
* - Max 4 billion cache keys per cache tag as limited by Redis Set datatype.
36+
* E.g. If you use a "all" items tag for expiry instead of clear(), that limits you to 4 billion cache items also.
3737
*
3838
* @see https://redis.io/topics/lru-cache#eviction-policies Documentation for Redis eviction policies.
3939
* @see https://redis.io/topics/data-types#sets Documentation for Redis Set datatype.
40-
* @see https://redis.io/commands/spop Documentation for sPOP operation, capable of retriving AND emptying a Set at once.
4140
*
4241
* @author Nicolas Grekas <[email protected]>
4342
* @author André Rømcke <[email protected]>
@@ -46,11 +45,6 @@ class RedisTagAwareAdapter extends AbstractTagAwareAdapter
4645
{
4746
use RedisTrait;
4847

49-
/**
50-
* Redis "Set" can hold more than 4 billion members, here we limit ourselves to PHP's > 2 billion max int (32Bit).
51-
*/
52-
private const POP_MAX_LIMIT = 2147483647 - 1;
53-
5448
/**
5549
* Limits for how many keys are deleted in batch.
5650
*/
@@ -62,26 +56,18 @@ class RedisTagAwareAdapter extends AbstractTagAwareAdapter
6256
*/
6357
private const DEFAULT_CACHE_TTL = 8640000;
6458

65-
/**
66-
* @var bool|null
67-
*/
68-
private $redisServerSupportSPOP = null;
69-
7059
/**
7160
* @param \Redis|\RedisArray|\RedisCluster|\Predis\ClientInterface $redisClient The redis client
7261
* @param string $namespace The default namespace
7362
* @param int $defaultLifetime The default lifetime
74-
*
75-
* @throws \Symfony\Component\Cache\Exception\LogicException If phpredis with version lower than 3.1.3.
7663
*/
7764
public function __construct($redisClient, string $namespace = '', int $defaultLifetime = 0, MarshallerInterface $marshaller = null)
7865
{
79-
$this->init($redisClient, $namespace, $defaultLifetime, $marshaller);
80-
81-
// Make sure php-redis is 3.1.3 or higher configured for Redis classes
82-
if (!$this->redis instanceof \Predis\ClientInterface && version_compare(phpversion('redis'), '3.1.3', '<')) {
83-
throw new LogicException('RedisTagAwareAdapter requires php-redis 3.1.3 or higher, alternatively use predis/predis');
66+
if ($redisClient instanceof \Predis\ClientInterface && $redisClient->getConnection() instanceof ClusterInterface && !$redisClient->getConnection() instanceof PredisCluster) {
67+
throw new InvalidArgumentException(sprintf('Unsupported Predis cluster connection: only "%s" is, "%s" given.', PredisCluster::class, \get_class($redisClient->getConnection())));
8468
}
69+
70+
$this->init($redisClient, $namespace, $defaultLifetime, $marshaller);
8571
}
8672

8773
/**
@@ -138,9 +124,10 @@ protected function doDelete(array $ids, array $tagData = []): bool
138124
return true;
139125
}
140126

141-
$predisCluster = $this->redis instanceof \Predis\ClientInterface && $this->redis->getConnection() instanceof ClusterInterface;
127+
$predisCluster = $this->redis instanceof \Predis\ClientInterface && $this->redis->getConnection() instanceof PredisCluster;
142128
$this->pipeline(static function () use ($ids, $tagData, $predisCluster) {
143129
if ($predisCluster) {
130+
// Unlike phpredis, Predis does not handle bulk calls for us against cluster
144131
foreach ($ids as $id) {
145132
yield 'del' => [$id];
146133
}
@@ -161,46 +148,82 @@ protected function doDelete(array $ids, array $tagData = []): bool
161148
*/
162149
protected function doInvalidate(array $tagIds): bool
163150
{
164-
if (!$this->redisServerSupportSPOP()) {
151+
if (($redis = $this->redis) instanceof \Predis\ClientInterface && $redis->getOptions()->exceptions) {
152+
$options = clone $redis->getOptions();
153+
\Closure::bind(function () { $this->options['exceptions'] = false; }, $options, $options)();
154+
$redis = new $redis($redis->getConnection(), $options);
155+
}
156+
157+
if (!$redis instanceof \Predis\ClientInterface || !$redis->getConnection() instanceof PredisCluster) {
158+
$movedTagSetIds = $this->renameKeys($redis, $tagIds);
159+
} else {
160+
$clusterConnection = $redis->getConnection();
161+
$tagIdsByConnection = new \SplObjectStorage();
162+
$movedTagSetIds = [];
163+
164+
foreach ($tagIds as $id) {
165+
$connection = $clusterConnection->getConnectionByKey($id);
166+
$slot = $tagIdsByConnection[$connection] ?? $tagIdsByConnection[$connection] = new \ArrayObject();
167+
$slot[] = $id;
168+
}
169+
170+
foreach ($tagIdsByConnection as $connection) {
171+
$slot = $tagIdsByConnection[$connection];
172+
$movedTagSetIds = array_merge($movedTagSetIds, $this->renameKeys(new $redis($connection, $options), $slot->getArrayCopy()));
173+
}
174+
}
175+
176+
// No Sets found
177+
if (!$movedTagSetIds) {
165178
return false;
166179
}
167180

168-
// Pop all tag info at once to avoid race conditions
169-
$tagIdSets = $this->pipeline(static function () use ($tagIds) {
170-
foreach ($tagIds as $tagId) {
171-
// Client: Predis or PHP Redis 3.1.3+ (https://github.com/phpredis/phpredis/commit/d2e203a6)
172-
// Server: Redis 3.2 or higher (https://redis.io/commands/spop)
173-
yield 'sPop' => [$tagId, self::POP_MAX_LIMIT];
181+
// Now safely take the time to read the keys in each set and collect ids we need to delete
182+
$tagIdSets = $this->pipeline(static function () use ($movedTagSetIds) {
183+
foreach ($movedTagSetIds as $movedTagId) {
184+
yield 'sMembers' => [$movedTagId];
174185
}
175186
});
176187

177-
// Flatten generator result from pipeline, ignore keys (tag ids)
178-
$ids = array_unique(array_merge(...iterator_to_array($tagIdSets, false)));
188+
// Return combination of the temporary Tag Set ids and their values (cache ids)
189+
$ids = array_merge($movedTagSetIds, ...iterator_to_array($tagIdSets, false));
179190

180191
// Delete cache in chunks to avoid overloading the connection
181-
foreach (array_chunk($ids, self::BULK_DELETE_LIMIT) as $chunkIds) {
192+
foreach (array_chunk(array_unique($ids), self::BULK_DELETE_LIMIT) as $chunkIds) {
182193
$this->doDelete($chunkIds);
183194
}
184195

185196
return true;
186197
}
187198

188-
private function redisServerSupportSPOP(): bool
199+
/**
200+
* Renames several keys in order to be able to operate on them without risk of race conditions.
201+
*
202+
* Filters out keys that do not exist before returning new keys.
203+
*
204+
* @see https://redis.io/commands/rename
205+
* @see https://redis.io/topics/cluster-spec#keys-hash-tags
206+
*
207+
* @return array Filtered list of the valid moved keys (only those that existed)
208+
*/
209+
private function renameKeys($redis, array $ids): array
189210
{
190-
if (null !== $this->redisServerSupportSPOP) {
191-
return $this->redisServerSupportSPOP;
192-
}
211+
$newIds = [];
212+
$uniqueToken = bin2hex(random_bytes(10));
193213

194-
foreach ($this->getHosts() as $host) {
195-
$info = $host->info('Server');
196-
$info = isset($info['Server']) ? $info['Server'] : $info;
197-
if (version_compare($info['redis_version'], '3.2', '<')) {
198-
CacheItem::log($this->logger, 'Redis server needs to be version 3.2 or higher, your Redis server was detected as '.$info['redis_version']);
214+
$results = $this->pipeline(static function () use ($ids, $uniqueToken) {
215+
foreach ($ids as $id) {
216+
yield 'rename' => [$id, '{'.$id.'}'.$uniqueToken];
217+
}
218+
}, $redis);
199219

200-
return $this->redisServerSupportSPOP = false;
220+
foreach ($results as $id => $ok) {
221+
if (true === $ok || ($ok instanceof Status && Status::get('OK') === $ok)) {
222+
// Only take into account if ok (key existed), will be false on phpredis if it did not exist
223+
$newIds[] = '{'.$id.'}'.$uniqueToken;
201224
}
202225
}
203226

204-
return $this->redisServerSupportSPOP = true;
227+
return $newIds;
205228
}
206229
}

‎src/Symfony/Component/Cache/CHANGELOG.md

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,8 @@ CHANGELOG
66

77
* added support for connecting to Redis Sentinel clusters
88
* added argument `$prefix` to `AdapterInterface::clear()`
9+
* improved `RedisTagAwareAdapter` to support Redis server >= 2.8 and up to 4B items per tag
10+
* [BC BREAK] `RedisTagAwareAdapter` is not compatible with `RedisCluster` from `Predis` anymore, use `phpredis` instead
911

1012
4.3.0
1113
-----

‎src/Symfony/Component/Cache/Tests/Adapter/PredisTagAwareRedisClusterAdapterTest.php

Lines changed: 0 additions & 35 deletions
This file was deleted.

‎src/Symfony/Component/Cache/Traits/RedisTrait.php

Lines changed: 12 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -423,31 +423,32 @@ protected function doSave(array $values, $lifetime)
423423
return $failed;
424424
}
425425

426-
private function pipeline(\Closure $generator): \Generator
426+
private function pipeline(\Closure $generator, $redis = null): \Generator
427427
{
428428
$ids = [];
429+
$redis = $redis ?? $this->redis;
429430

430-
if ($this->redis instanceof RedisClusterProxy || $this->redis instanceof \RedisCluster || ($this->redis instanceof \Predis\ClientInterface && $this->redis->getConnection() instanceof RedisCluster)) {
431+
if ($redis instanceof RedisClusterProxy || $redis instanceof \RedisCluster || ($redis instanceof \Predis\ClientInterface && $redis->getConnection() instanceof RedisCluster)) {
431432
// phpredis & predis don't support pipelining with RedisCluster
432433
// see https://github.com/phpredis/phpredis/blob/develop/cluster.markdown#pipelining
433434
// see https://github.com/nrk/predis/issues/267#issuecomment-123781423
434435
$results = [];
435436
foreach ($generator() as $command => $args) {
436-
$results[] = $this->redis->{$command}(...$args);
437+
$results[] = $redis->{$command}(...$args);
437438
$ids[] = $args[0];
438439
}
439-
} elseif ($this->redis instanceof \Predis\ClientInterface) {
440-
$results = $this->redis->pipeline(function ($redis) use ($generator, &$ids) {
440+
} elseif ($redis instanceof \Predis\ClientInterface) {
441+
$results = $redis->pipeline(static function ($redis) use ($generator, &$ids) {
441442
foreach ($generator() as $command => $args) {
442443
$redis->{$command}(...$args);
443444
$ids[] = $args[0];
444445
}
445446
});
446-
} elseif ($this->redis instanceof \RedisArray) {
447+
} elseif ($redis instanceof \RedisArray) {
447448
$connections = $results = $ids = [];
448449
foreach ($generator() as $command => $args) {
449-
if (!isset($connections[$h = $this->redis->_target($args[0])])) {
450-
$connections[$h] = [$this->redis->_instance($h), -1];
450+
if (!isset($connections[$h = $redis->_target($args[0])])) {
451+
$connections[$h] = [$redis->_instance($h), -1];
451452
$connections[$h][0]->multi(\Redis::PIPELINE);
452453
}
453454
$connections[$h][0]->{$command}(...$args);
@@ -461,12 +462,12 @@ private function pipeline(\Closure $generator): \Generator
461462
$results[$k] = $connections[$h][$c];
462463
}
463464
} else {
464-
$this->redis->multi(\Redis::PIPELINE);
465+
$redis->multi(\Redis::PIPELINE);
465466
foreach ($generator() as $command => $args) {
466-
$this->redis->{$command}(...$args);
467+
$redis->{$command}(...$args);
467468
$ids[] = $args[0];
468469
}
469-
$results = $this->redis->exec();
470+
$results = $redis->exec();
470471
}
471472

472473
foreach ($ids as $k => $id) {

0 commit comments

Comments
 (0)