Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit e28a53e

Browse files
andreromnicolas-grekas
andrerom
authored and
nicolas-grekas
committed
[Cache] Improve RedisTagAwareAdapter invalidation logic & requirements
1 parent a0bbae7 commit e28a53e

File tree

2 files changed

+72
-86
lines changed

2 files changed

+72
-86
lines changed

‎src/Symfony/Component/Cache/Adapter/RedisTagAwareAdapter.php

Lines changed: 72 additions & 51 deletions
Original file line numberDiff line numberDiff line change
@@ -11,33 +11,32 @@
1111

1212
namespace Symfony\Component\Cache\Adapter;
1313

14-
use Predis;
1514
use Predis\Connection\Aggregate\ClusterInterface;
15+
use Predis\Connection\Aggregate\PredisCluster;
1616
use Predis\Response\Status;
17-
use Symfony\Component\Cache\CacheItem;
18-
use Symfony\Component\Cache\Exception\LogicException;
17+
use Symfony\Component\Cache\Exception\InvalidArgumentException;
1918
use Symfony\Component\Cache\Marshaller\MarshallerInterface;
2019
use Symfony\Component\Cache\Traits\RedisTrait;
2120

2221
/**
23-
* Stores tag id <> cache id relationship as a Redis Set, lookup on invalidation using sPOP.
22+
* Stores tag id <> cache id relationship as a Redis Set, lookup on invalidation using RENAME+SMEMBERS.
2423
*
2524
* Set (tag relation info) is stored without expiry (non-volatile), while cache always gets an expiry (volatile) even
2625
* if not set by caller. Thus if you configure redis with the right eviction policy you can be safe this tag <> cache
2726
* relationship survives eviction (cache cleanup when Redis runs out of memory).
2827
*
2928
* Requirements:
30-
* - Server: Redis 3.2+
31-
* - Client: PHP Redis 3.1.3+ OR Predis
32-
* - Redis Server(s) configured with any `volatile-*` eviction policy, OR `noeviction` if it will NEVER fill up memory
29+
* - Client: PHP Redis or Predis
30+
* Note: Due to lack of RENAME support it is NOT recommended to use Cluster on Predis, instead use phpredis.
31+
* - Server: Redis 2.8+
32+
* Configured with any `volatile-*` eviction policy, OR `noeviction` if it will NEVER fill up memory
3333
*
3434
* Design limitations:
35-
* - Max 2 billion cache keys per cache tag
36-
* E.g. If you use a "all" items tag for expiry instead of clear(), that limits you to 2 billion cache items as well
35+
* - Max 4 billion cache keys per cache tag as limited by Redis Set datatype.
36+
* E.g. If you use a "all" items tag for expiry instead of clear(), that limits you to 4 billion cache items also.
3737
*
3838
* @see https://redis.io/topics/lru-cache#eviction-policies Documentation for Redis eviction policies.
3939
* @see https://redis.io/topics/data-types#sets Documentation for Redis Set datatype.
40-
* @see https://redis.io/commands/spop Documentation for sPOP operation, capable of retriving AND emptying a Set at once.
4140
*
4241
* @author Nicolas Grekas <[email protected]>
4342
* @author André Rømcke <[email protected]>
@@ -46,11 +45,6 @@ class RedisTagAwareAdapter extends AbstractTagAwareAdapter
4645
{
4746
use RedisTrait;
4847

49-
/**
50-
* Redis "Set" can hold more than 4 billion members, here we limit ourselves to PHP's > 2 billion max int (32Bit).
51-
*/
52-
private const POP_MAX_LIMIT = 2147483647 - 1;
53-
5448
/**
5549
* Limits for how many keys are deleted in batch.
5650
*/
@@ -62,26 +56,18 @@ class RedisTagAwareAdapter extends AbstractTagAwareAdapter
6256
*/
6357
private const DEFAULT_CACHE_TTL = 8640000;
6458

65-
/**
66-
* @var bool|null
67-
*/
68-
private $redisServerSupportSPOP = null;
69-
7059
/**
7160
* @param \Redis|\RedisArray|\RedisCluster|\Predis\ClientInterface $redisClient The redis client
7261
* @param string $namespace The default namespace
7362
* @param int $defaultLifetime The default lifetime
74-
*
75-
* @throws \Symfony\Component\Cache\Exception\LogicException If phpredis with version lower than 3.1.3.
7663
*/
7764
public function __construct($redisClient, string $namespace = '', int $defaultLifetime = 0, MarshallerInterface $marshaller = null)
7865
{
79-
$this->init($redisClient, $namespace, $defaultLifetime, $marshaller);
80-
81-
// Make sure php-redis is 3.1.3 or higher configured for Redis classes
82-
if (!$this->redis instanceof \Predis\ClientInterface && version_compare(phpversion('redis'), '3.1.3', '<')) {
83-
throw new LogicException('RedisTagAwareAdapter requires php-redis 3.1.3 or higher, alternatively use predis/predis');
66+
if ($redisClient instanceof \Predis\ClientInterface && $redisClient->getConnection() instanceof ClusterInterface && !$redisClient->getConnection() instanceof PredisCluster) {
67+
throw new InvalidArgumentException(sprintf('Unsupported Predis cluster connection: only "%s" is, "%s" given.', PredisCluster::class, \get_class($redisClient->getConnection())));
8468
}
69+
70+
$this->init($redisClient, $namespace, $defaultLifetime, $marshaller);
8571
}
8672

8773
/**
@@ -138,9 +124,10 @@ protected function doDelete(array $ids, array $tagData = []): bool
138124
return true;
139125
}
140126

141-
$predisCluster = $this->redis instanceof \Predis\ClientInterface && $this->redis->getConnection() instanceof ClusterInterface;
127+
$predisCluster = $this->redis instanceof \Predis\ClientInterface && $this->redis->getConnection() instanceof PredisCluster;
142128
$this->pipeline(static function () use ($ids, $tagData, $predisCluster) {
143129
if ($predisCluster) {
130+
// Unlike phpredis, Predis does not handle bulk calls for us against cluster
144131
foreach ($ids as $id) {
145132
yield 'del' => [$id];
146133
}
@@ -161,46 +148,80 @@ protected function doDelete(array $ids, array $tagData = []): bool
161148
*/
162149
protected function doInvalidate(array $tagIds): bool
163150
{
164-
if (!$this->redisServerSupportSPOP()) {
151+
if (!$this->redis instanceof \Predis\ClientInterface || !$this->redis->getConnection() instanceof PredisCluster) {
152+
$movedTagSetIds = $this->renameKeys($this->redis, $tagIds);
153+
} else {
154+
$clusterConnection = $this->redis->getConnection();
155+
$tagIdsByConnection = new \SplObjectStorage();
156+
$movedTagSetIds = [];
157+
158+
foreach ($tagIds as $id) {
159+
$connection = $clusterConnection->getConnectionByKey($id);
160+
$slot = $tagIdsByConnection[$connection] ?? $tagIdsByConnection[$connection] = new \ArrayObject();
161+
$slot[] = $id;
162+
}
163+
164+
foreach ($tagIdsByConnection as $connection) {
165+
$slot = $tagIdsByConnection[$connection];
166+
$redis = new \Predis\Client($connection, $this->redis->getOptions());
167+
$movedTagSetIds = array_merge($movedTagSetIds, $this->renameKeys($redis, $slot->getArrayCopy()));
168+
}
169+
}
170+
171+
// No Sets found
172+
if (!$movedTagSetIds) {
165173
return false;
166174
}
167175

168-
// Pop all tag info at once to avoid race conditions
169-
$tagIdSets = $this->pipeline(static function () use ($tagIds) {
170-
foreach ($tagIds as $tagId) {
171-
// Client: Predis or PHP Redis 3.1.3+ (https://github.com/phpredis/phpredis/commit/d2e203a6)
172-
// Server: Redis 3.2 or higher (https://redis.io/commands/spop)
173-
yield 'sPop' => [$tagId, self::POP_MAX_LIMIT];
176+
// Now safely take the time to read the keys in each set and collect ids we need to delete
177+
$tagIdSets = $this->pipeline(static function () use ($movedTagSetIds) {
178+
foreach ($movedTagSetIds as $movedTagId) {
179+
yield 'sMembers' => [$movedTagId];
174180
}
175181
});
176182

177-
// Flatten generator result from pipeline, ignore keys (tag ids)
178-
$ids = array_unique(array_merge(...iterator_to_array($tagIdSets, false)));
183+
// Return combination of the temporary Tag Set ids and their values (cache ids)
184+
$ids = array_merge($movedTagSetIds, ...iterator_to_array($tagIdSets, false));
179185

180186
// Delete cache in chunks to avoid overloading the connection
181-
foreach (array_chunk($ids, self::BULK_DELETE_LIMIT) as $chunkIds) {
187+
foreach (array_chunk(array_unique($ids), self::BULK_DELETE_LIMIT) as $chunkIds) {
182188
$this->doDelete($chunkIds);
183189
}
184190

185191
return true;
186192
}
187193

188-
private function redisServerSupportSPOP(): bool
194+
/**
195+
* Renames several keys in order to be able to operate on them without risk of race conditions.
196+
*
197+
* Filters out keys that do not exist before returning new keys.
198+
*
199+
* @see https://redis.io/commands/rename
200+
*
201+
* @return array Filtered list of the valid moved keys (only those that existed)
202+
*/
203+
private function renameKeys($redis, array $ids): array
189204
{
190-
if (null !== $this->redisServerSupportSPOP) {
191-
return $this->redisServerSupportSPOP;
192-
}
193-
194-
foreach ($this->getHosts() as $host) {
195-
$info = $host->info('Server');
196-
$info = isset($info['Server']) ? $info['Server'] : $info;
197-
if (version_compare($info['redis_version'], '3.2', '<')) {
198-
CacheItem::log($this->logger, 'Redis server needs to be version 3.2 or higher, your Redis server was detected as '.$info['redis_version']);
199-
200-
return $this->redisServerSupportSPOP = false;
205+
// 1. Due to Predis exception we don't do this in pipeline
206+
// 2. https://redis.io/topics/cluster-spec#keys-hash-tags is used to place in same hash slot on cluster
207+
$newIds = [];
208+
$uniqueToken = bin2hex(random_bytes(10));
209+
foreach ($ids as $id) {
210+
$newId = '{'.$id.'}'.$uniqueToken;
211+
try {
212+
$ok = $redis->rename($id, $newId);
213+
if (true === $ok || ($ok instanceof Status && $ok === Status::get('OK'))) {
214+
// Only take into account if ok (key existed), will be false on phpredis if it did not exist
215+
$newIds[] = $newId;
216+
}
217+
} catch (\Predis\Response\ServerException $e) {
218+
// Silence errors when key does not exists on Predis. Otherwise re-throw exception
219+
if ('ERR no such key' !== $e->getMessage()) {
220+
throw $e;
221+
}
201222
}
202223
}
203224

204-
return $this->redisServerSupportSPOP = true;
225+
return $newIds;
205226
}
206227
}

‎src/Symfony/Component/Cache/Tests/Adapter/PredisTagAwareRedisClusterAdapterTest.php

Lines changed: 0 additions & 35 deletions
This file was deleted.

0 commit comments

Comments
 (0)