Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit bb32e6f

Browse files
yatsukhnenkoyatsukhnenko
committed
Implement consistent hashing algorithm for RedisArray
1 parent 3e7e1c8 commit bb32e6f

7 files changed

Lines changed: 117 additions & 20 deletions

File tree

‎common.h‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -644,6 +644,17 @@ typedef enum _PUBSUB_TYPE {
644644
#define REDIS_ENABLE_MODE(redis_sock, m) (redis_sock->mode |= m)
645645
#define REDIS_DISABLE_MODE(redis_sock, m) (redis_sock->mode &= ~m)
646646

647+
/* HOST_NAME_MAX doesn't exist everywhere */
648+
#ifndef HOST_NAME_MAX
649+
#if defined(_POSIX_HOST_NAME_MAX)
650+
#define HOST_NAME_MAX _POSIX_HOST_NAME_MAX
651+
#elif defined(MAXHOSTNAMELEN)
652+
#define HOST_NAME_MAX MAXHOSTNAMELEN
653+
#else
654+
#define HOST_NAME_MAX 255
655+
#endif
656+
#endif
657+
647658
typedef struct fold_item {
648659
zval * (*fun)(INTERNAL_FUNCTION_PARAMETERS, void *, ...);
649660
void *ctx;

‎redis.c‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,7 @@ PHP_INI_BEGIN()
6767
PHP_INI_ENTRY("redis.arrays.previous", "", PHP_INI_ALL, NULL)
6868
PHP_INI_ENTRY("redis.arrays.readtimeout", "0", PHP_INI_ALL, NULL)
6969
PHP_INI_ENTRY("redis.arrays.retryinterval", "0", PHP_INI_ALL, NULL)
70+
PHP_INI_ENTRY("redis.arrays.consistent", "0", PHP_INI_ALL, NULL)
7071

7172
/* redis cluster */
7273
PHP_INI_ENTRY("redis.clusters.persistent", "0", PHP_INI_ALL, NULL)

‎redis_array.c‎

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -138,6 +138,12 @@ redis_array_free(RedisArray *ra)
138138
{
139139
int i;
140140

141+
/* continuum */
142+
if (ra->continuum) {
143+
efree(ra->continuum->points);
144+
efree(ra->continuum);
145+
}
146+
141147
/* Redis objects */
142148
for(i = 0; i< ra->count; i++) {
143149
zval_dtor(&ra->redis[i]);
@@ -264,7 +270,7 @@ PHP_METHOD(RedisArray, __construct)
264270
{
265271
zval *z0, z_fun, z_dist, *zpData, *z_opts = NULL;
266272
RedisArray *ra = NULL;
267-
zend_bool b_index = 0, b_autorehash = 0, b_pconnect = 0;
273+
zend_bool b_index = 0, b_autorehash = 0, b_pconnect = 0, consistent = 0;
268274
HashTable *hPrev = NULL, *hOpts = NULL;
269275
long l_retry_interval = 0;
270276
zend_bool b_lazy_connect = 0;
@@ -349,6 +355,11 @@ PHP_METHOD(RedisArray, __construct)
349355
read_timeout = atof(Z_STRVAL_P(zpData));
350356
}
351357
}
358+
359+
/* consistent */
360+
if ((zpData = zend_hash_str_find(hOpts, "consistent", sizeof("consistent") - 1)) != NULL) {
361+
consistent = zval_is_true(zpData);
362+
}
352363
}
353364

354365
/* extract either name of list of hosts from z0 */
@@ -358,7 +369,7 @@ PHP_METHOD(RedisArray, __construct)
358369
break;
359370

360371
case IS_ARRAY:
361-
ra = ra_make_array(Z_ARRVAL_P(z0), &z_fun, &z_dist, hPrev, b_index, b_pconnect, l_retry_interval, b_lazy_connect, d_connect_timeout, read_timeout TSRMLS_CC);
372+
ra = ra_make_array(Z_ARRVAL_P(z0), &z_fun, &z_dist, hPrev, b_index, b_pconnect, l_retry_interval, b_lazy_connect, d_connect_timeout, read_timeout, consistent TSRMLS_CC);
362373
break;
363374

364375
default:

‎redis_array.h‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,15 @@ PHP_METHOD(RedisArray, exec);
3737
PHP_METHOD(RedisArray, discard);
3838
PHP_METHOD(RedisArray, unwatch);
3939

40+
typedef struct {
41+
uint32_t value;
42+
int index;
43+
} ContinuumPoint;
44+
45+
typedef struct {
46+
size_t nb_points;
47+
ContinuumPoint *points;
48+
} Continuum;
4049

4150
typedef struct RedisArray_ {
4251

@@ -52,7 +61,7 @@ typedef struct RedisArray_ {
5261
HashTable *pure_cmds; /* hash table */
5362
double connect_timeout; /* socket connect timeout */
5463
double read_timeout; /* socket read timeout */
55-
64+
Continuum *continuum;
5665
struct RedisArray_ *prev;
5766
} RedisArray;
5867

‎redis_array_impl.c‎

Lines changed: 81 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424
#include "SAPI.h"
2525
#include "ext/standard/url.h"
2626
#include "ext/standard/crc32.h"
27+
#include "ext/standard/md5.h"
2728

2829
#define PHPREDIS_INDEX_NAME "__phpredis_array_index__"
2930

@@ -173,9 +174,10 @@ RedisArray *ra_load_array(const char *name TSRMLS_DC) {
173174
zval z_params_connect_timeout;
174175
zval z_params_read_timeout;
175176
zval z_params_lazy_connect;
177+
zval z_params_consistent;
176178
RedisArray *ra = NULL;
177179

178-
zend_bool b_index = 0, b_autorehash = 0, b_pconnect = 0;
180+
zend_bool b_index = 0, b_autorehash = 0, b_pconnect = 0, consistent = 0;
179181
long l_retry_interval = 0;
180182
zend_bool b_lazy_connect = 0;
181183
double d_connect_timeout = 0, read_timeout = 0.0;
@@ -312,9 +314,20 @@ RedisArray *ra_load_array(const char *name TSRMLS_DC) {
312314
}
313315
}
314316

317+
/* find consistent option */
318+
array_init(&z_params_consistent);
319+
if ((iptr = INI_STR("redis.arrays.consistent")) != NULL) {
320+
sapi_module.treat_data(PARSE_STRING, estrdup(iptr), &z_params_consistent TSRMLS_CC);
321+
}
322+
if ((z_data = zend_hash_str_find(Z_ARRVAL(z_params_consistent), name, name_len)) != NULL) {
323+
if (Z_TYPE_P(z_data) == IS_STRING && strncmp(Z_STRVAL_P(z_data), "1", 1) == 0) {
324+
consistent = 1;
325+
}
326+
}
327+
315328

316329
/* create RedisArray object */
317-
ra = ra_make_array(hHosts, &z_fun, &z_dist, hPrev, b_index, b_pconnect, l_retry_interval, b_lazy_connect, d_connect_timeout, read_timeout TSRMLS_CC);
330+
ra = ra_make_array(hHosts, &z_fun, &z_dist, hPrev, b_index, b_pconnect, l_retry_interval, b_lazy_connect, d_connect_timeout, read_timeout, consistent TSRMLS_CC);
318331
if (ra) {
319332
ra->auto_rehash = b_autorehash;
320333
if(ra->prev) ra->prev->auto_rehash = b_autorehash;
@@ -332,14 +345,55 @@ RedisArray *ra_load_array(const char *name TSRMLS_DC) {
332345
zval_dtor(&z_params_connect_timeout);
333346
zval_dtor(&z_params_read_timeout);
334347
zval_dtor(&z_params_lazy_connect);
348+
zval_dtor(&z_params_consistent);
335349
zval_dtor(&z_dist);
336350
zval_dtor(&z_fun);
337351

338352
return ra;
339353
}
340354

355+
static int
356+
ra_points_cmp(const void *v1, const void *v2)
357+
{
358+
const ContinuumPoint *p1 = v1, *p2 = v2;
359+
360+
return p1->value < p2->value ? - 1 : p1->value > p2->value;
361+
}
362+
363+
static Continuum *
364+
ra_make_continuum(zend_string **hosts, int nb_hosts)
365+
{
366+
int i, j, k, len, idx = 0;
367+
char host[HOST_NAME_MAX];
368+
unsigned char digest[16];
369+
PHP_MD5_CTX ctx;
370+
Continuum *c;
371+
372+
c = ecalloc(1, sizeof(*c));
373+
c->nb_points = nb_hosts * 160; /* 40 hashes, 4 numbers per hash = 160 points per server */
374+
c->points = ecalloc(c->nb_points, sizeof(*c->points));
375+
376+
for (i = 0; i < nb_hosts; ++i) {
377+
for (j = 0; j < 40; ++j) {
378+
len = snprintf(host, sizeof(host), "%.*s-%u", ZSTR_LEN(hosts[i]), ZSTR_VAL(hosts[i]), j);
379+
PHP_MD5Init(&ctx);
380+
PHP_MD5Update(&ctx, host, len);
381+
PHP_MD5Final(digest, &ctx);
382+
for (k = 0; k < 4; ++k) {
383+
c->points[idx].index = i;
384+
c->points[idx++].value = (digest[3 + k * 4] << 24)
385+
| (digest[2 + k * 4] << 16)
386+
| (digest[1 + k * 4] << 8)
387+
| (digest[k * 4]);
388+
}
389+
}
390+
}
391+
qsort(c->points, c->nb_points, sizeof(*c->points), ra_points_cmp);
392+
return c;
393+
}
394+
341395
RedisArray *
342-
ra_make_array(HashTable *hosts, zval *z_fun, zval *z_dist, HashTable *hosts_prev, zend_bool b_index, zend_bool b_pconnect, long retry_interval, zend_bool b_lazy_connect, double connect_timeout, double read_timeout TSRMLS_DC) {
396+
ra_make_array(HashTable *hosts, zval *z_fun, zval *z_dist, HashTable *hosts_prev, zend_bool b_index, zend_bool b_pconnect, long retry_interval, zend_bool b_lazy_connect, double connect_timeout, double read_timeout, zend_bool consistent TSRMLS_DC) {
343397

344398
int i, count;
345399
RedisArray *ra;
@@ -357,6 +411,7 @@ ra_make_array(HashTable *hosts, zval *z_fun, zval *z_dist, HashTable *hosts_prev
357411
ra->pconnect = b_pconnect;
358412
ra->connect_timeout = connect_timeout;
359413
ra->read_timeout = read_timeout;
414+
ra->continuum = NULL;
360415

361416
if (ra_load_hosts(ra, hosts, retry_interval, b_lazy_connect TSRMLS_CC) == NULL || !ra->count) {
362417
for (i = 0; i < ra->count; ++i) {
@@ -368,7 +423,7 @@ ra_make_array(HashTable *hosts, zval *z_fun, zval *z_dist, HashTable *hosts_prev
368423
efree(ra);
369424
return NULL;
370425
}
371-
ra->prev = hosts_prev ? ra_make_array(hosts_prev, z_fun, z_dist, NULL, b_index, b_pconnect, retry_interval, b_lazy_connect, connect_timeout, read_timeout TSRMLS_CC) : NULL;
426+
ra->prev = hosts_prev ? ra_make_array(hosts_prev, z_fun, z_dist, NULL, b_index, b_pconnect, retry_interval, b_lazy_connect, connect_timeout, read_timeout, consistent TSRMLS_CC) : NULL;
372427

373428
/* init array data structures */
374429
ra_init_function_table(ra);
@@ -377,6 +432,11 @@ ra_make_array(HashTable *hosts, zval *z_fun, zval *z_dist, HashTable *hosts_prev
377432
ZVAL_ZVAL(&ra->z_fun, z_fun, 1, 0);
378433
ZVAL_ZVAL(&ra->z_dist, z_dist, 1, 0);
379434

435+
/* init continuum */
436+
if (consistent) {
437+
ra->continuum = ra_make_continuum(ra->hosts, ra->count);
438+
}
439+
380440
return ra;
381441
}
382442

@@ -480,7 +540,23 @@ ra_find_node(RedisArray *ra, const char *key, int key_len, int *out_pos TSRMLS_D
480540
}
481541

482542
/* get position on ring */
483-
pos = (int)((ret ^ 0xffffffff) * ra->count / 0xffffffff);
543+
if (ra->continuum) {
544+
int left = 0, right = ra->continuum->nb_points;
545+
while (left < right) {
546+
i = (int)((left + right) / 2);
547+
if (ra->continuum->points[i].value < ret) {
548+
left = i + 1;
549+
} else {
550+
right = i;
551+
}
552+
}
553+
if (right == ra->continuum->nb_points) {
554+
right = 0;
555+
}
556+
pos = ra->continuum->points[right].index;
557+
} else {
558+
pos = (int)((ret ^ 0xffffffff) * ra->count / 0xffffffff);
559+
}
484560
} else {
485561
pos = ra_call_distributor(ra, key, key_len TSRMLS_CC);
486562
if (pos < 0 || pos >= ra->count) {

‎redis_array_impl.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@
1111

1212
RedisArray *ra_load_hosts(RedisArray *ra, HashTable *hosts, long retry_interval, zend_bool b_lazy_connect TSRMLS_DC);
1313
RedisArray *ra_load_array(const char *name TSRMLS_DC);
14-
RedisArray *ra_make_array(HashTable *hosts, zval *z_fun, zval *z_dist, HashTable *hosts_prev, zend_bool b_index, zend_bool b_pconnect, long retry_interval, zend_bool b_lazy_connect, double connect_timeout, double read_timeout TSRMLS_DC);
14+
RedisArray *ra_make_array(HashTable *hosts, zval *z_fun, zval *z_dist, HashTable *hosts_prev, zend_bool b_index, zend_bool b_pconnect, long retry_interval, zend_bool b_lazy_connect, double connect_timeout, double read_timeout, zend_bool consistent TSRMLS_DC);
1515
zval *ra_find_node_by_name(RedisArray *ra, const char *host, int host_len TSRMLS_DC);
1616
zval *ra_find_node(RedisArray *ra, const char *key, int key_len, int *out_pos TSRMLS_DC);
1717
void ra_init_function_table(RedisArray *ra);

‎redis_session.c‎

Lines changed: 0 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -41,17 +41,6 @@
4141
#include "SAPI.h"
4242
#include "ext/standard/url.h"
4343

44-
/* HOST_NAME_MAX doesn't exist everywhere */
45-
#ifndef HOST_NAME_MAX
46-
#if defined(_POSIX_HOST_NAME_MAX)
47-
#define HOST_NAME_MAX _POSIX_HOST_NAME_MAX
48-
#elif defined(MAXHOSTNAMELEN)
49-
#define HOST_NAME_MAX MAXHOSTNAMELEN
50-
#else
51-
#define HOST_NAME_MAX 255
52-
#endif
53-
#endif
54-
5544
/* Session lock LUA as well as its SHA1 hash */
5645
#define LOCK_RELEASE_LUA_STR "if redis.call(\"get\",KEYS[1]) == ARGV[1] then return redis.call(\"del\",KEYS[1]) else return 0 end"
5746
#define LOCK_RELEASE_LUA_LEN (sizeof(LOCK_RELEASE_LUA_STR) - 1)

0 commit comments

Comments
 (0)