Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit 05129c3

Browse files
nbraun-amazonmichael-grunder
nbraun-amazon
authored and
michael-grunder
committed
Add support for exponential backoff on retry
1 parent 502d09f commit 05129c3

10 files changed

Lines changed: 270 additions & 48 deletions

File tree

‎backoff.c‎

Lines changed: 90 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,90 @@
1+
#include "common.h"
2+
3+
#include <ext/standard/php_rand.h>
4+
5+
#if PHP_VERSION_ID >= 70100
6+
#include <ext/standard/php_mt_rand.h>
7+
#else
8+
static zend_long php_mt_rand_range(zend_long min, zend_long max) {
9+
zend_long number = php_rand();
10+
RAND_RANGE(number, min, max, PHP_RAND_MAX);
11+
return number;
12+
}
13+
#endif
14+
15+
#include "backoff.h"
16+
17+
static zend_ulong random_range(zend_ulong min, zend_ulong max) {
18+
if (max < min) {
19+
return php_mt_rand_range(max, min);
20+
}
21+
22+
return php_mt_rand_range(min, max);
23+
}
24+
25+
static zend_ulong redis_default_backoff(struct RedisBackoff *self, unsigned int retry_index) {
26+
zend_ulong backoff = retry_index ? self->base : random_range(0, self->base);
27+
return MIN(self->cap, backoff);
28+
}
29+
30+
static zend_ulong redis_constant_backoff(struct RedisBackoff *self, unsigned int retry_index) {
31+
zend_ulong backoff = self->base;
32+
return MIN(self->cap, backoff);
33+
}
34+
35+
static zend_ulong redis_uniform_backoff(struct RedisBackoff *self, unsigned int retry_index) {
36+
zend_ulong backoff = random_range(0, self->base);
37+
return MIN(self->cap, backoff);
38+
}
39+
40+
static zend_ulong redis_exponential_backoff(struct RedisBackoff *self, unsigned int retry_index) {
41+
zend_ulong pow = MIN(retry_index, 10);
42+
zend_ulong backoff = self->base * (1 << pow);
43+
return MIN(self->cap, backoff);
44+
}
45+
46+
static zend_ulong redis_full_jitter_backoff(struct RedisBackoff *self, unsigned int retry_index) {
47+
zend_ulong pow = MIN(retry_index, 10);
48+
zend_ulong backoff = self->base * (1 << pow);
49+
zend_ulong cap = MIN(self->cap, backoff);
50+
return random_range(0, self->cap);
51+
}
52+
53+
static zend_ulong redis_equal_jitter_backoff(struct RedisBackoff *self, unsigned int retry_index) {
54+
zend_ulong pow = MIN(retry_index, 10);
55+
zend_ulong backoff = self->base * (1 << pow);
56+
zend_ulong temp = MIN(self->cap, backoff);
57+
return temp / 2 + random_range(0, temp) / 2;
58+
}
59+
60+
static zend_ulong redis_decorrelated_jitter_backoff(struct RedisBackoff *self, unsigned int retry_index) {
61+
self->previous_backoff = random_range(self->base, self->previous_backoff * 3);
62+
return MIN(self->cap, self->previous_backoff);
63+
}
64+
65+
typedef zend_ulong (*redis_backoff_algorithm)(struct RedisBackoff *self, unsigned int retry_index);
66+
67+
static redis_backoff_algorithm redis_backoff_algorithms[REDIS_BACKOFF_ALGORITHMS] = {
68+
redis_default_backoff,
69+
redis_decorrelated_jitter_backoff,
70+
redis_full_jitter_backoff,
71+
redis_equal_jitter_backoff,
72+
redis_exponential_backoff,
73+
redis_uniform_backoff,
74+
redis_constant_backoff,
75+
};
76+
77+
void redis_initialize_backoff(struct RedisBackoff *self, unsigned long retry_interval) {
78+
self->algorithm = 0; // default backoff
79+
self->base = retry_interval;
80+
self->cap = retry_interval;
81+
self->previous_backoff = 0;
82+
}
83+
84+
void redis_backoff_reset(struct RedisBackoff *self) {
85+
self->previous_backoff = 0;
86+
}
87+
88+
zend_ulong redis_backoff_compute(struct RedisBackoff *self, unsigned int retry_index) {
89+
return redis_backoff_algorithms[self->algorithm](self, retry_index);
90+
}

‎backoff.h‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
#ifndef REDIS_BACKOFF_H
2+
#define REDIS_BACKOFF_H
3+
4+
/* {{{ struct RedisBackoff */
5+
struct RedisBackoff {
6+
unsigned int algorithm; /* index of algorithm function, returns backoff in microseconds*/
7+
zend_ulong base; /* base backoff in microseconds */
8+
zend_ulong cap; /* max backoff in microseconds */
9+
zend_ulong previous_backoff; /* previous backoff in microseconds */
10+
};
11+
/* }}} */
12+
13+
void redis_initialize_backoff(struct RedisBackoff *self, unsigned long retry_interval);
14+
void redis_backoff_reset(struct RedisBackoff *self);
15+
zend_ulong redis_backoff_compute(struct RedisBackoff *self, unsigned int retry_index);
16+
17+
#endif

‎common.h‎

Lines changed: 53 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,8 @@
2020
#define NULL ((void *) 0)
2121
#endif
2222

23+
#include "backoff.h"
24+
2325
typedef enum {
2426
REDIS_SOCK_STATUS_FAILED = -1,
2527
REDIS_SOCK_STATUS_DISCONNECTED,
@@ -82,6 +84,10 @@ typedef enum _PUBSUB_TYPE {
8284
#define REDIS_OPT_REPLY_LITERAL 8
8385
#define REDIS_OPT_COMPRESSION_LEVEL 9
8486
#define REDIS_OPT_NULL_MBULK_AS_NULL 10
87+
#define REDIS_OPT_MAX_RETRIES 11
88+
#define REDIS_OPT_BACKOFF_ALGORITHM 12
89+
#define REDIS_OPT_BACKOFF_BASE 13
90+
#define REDIS_OPT_BACKOFF_CAP 14
8591

8692
/* cluster options */
8793
#define REDIS_FAILOVER_NONE 0
@@ -108,6 +114,16 @@ typedef enum {
108114
#define REDIS_SCAN_PREFIX 2
109115
#define REDIS_SCAN_NOPREFIX 3
110116

117+
/* BACKOFF_ALGORITHM options */
118+
#define REDIS_BACKOFF_ALGORITHMS 7
119+
#define REDIS_BACKOFF_ALGORITHM_DEFAULT 0
120+
#define REDIS_BACKOFF_ALGORITHM_DECORRELATED_JITTER 1
121+
#define REDIS_BACKOFF_ALGORITHM_FULL_JITTER 2
122+
#define REDIS_BACKOFF_ALGORITHM_EQUAL_JITTER 3
123+
#define REDIS_BACKOFF_ALGORITHM_EXPONENTIAL 4
124+
#define REDIS_BACKOFF_ALGORITHM_UNIFORM 5
125+
#define REDIS_BACKOFF_ALGORITHM_CONSTANT 6
126+
111127
/* GETBIT/SETBIT offset range limits */
112128
#define BITOP_MIN_OFFSET 0
113129
#define BITOP_MAX_OFFSET 4294967295U
@@ -257,41 +273,43 @@ typedef enum {
257273

258274
/* {{{ struct RedisSock */
259275
typedef struct {
260-
php_stream *stream;
261-
php_stream_context *stream_ctx;
262-
zend_string *host;
263-
int port;
264-
zend_string *user;
265-
zend_string *pass;
266-
double timeout;
267-
double read_timeout;
268-
long retry_interval;
269-
redis_sock_status status;
270-
int persistent;
271-
int watching;
272-
zend_string *persistent_id;
273-
274-
redis_serializer serializer;
275-
int compression;
276-
int compression_level;
277-
long dbNumber;
278-
279-
zend_string *prefix;
280-
281-
short mode;
282-
struct fold_item *head;
283-
struct fold_item *current;
284-
285-
zend_string *pipeline_cmd;
286-
287-
zend_string *err;
288-
289-
int scan;
290-
291-
int readonly;
292-
int reply_literal;
293-
int null_mbulk_as_null;
294-
int tcp_keepalive;
276+
php_stream *stream;
277+
php_stream_context *stream_ctx;
278+
zend_string *host;
279+
int port;
280+
zend_string *user;
281+
zend_string *pass;
282+
double timeout;
283+
double read_timeout;
284+
long retry_interval;
285+
int max_retries;
286+
struct RedisBackoff backoff;
287+
redis_sock_status status;
288+
int persistent;
289+
int watching;
290+
zend_string *persistent_id;
291+
292+
redis_serializer serializer;
293+
int compression;
294+
int compression_level;
295+
long dbNumber;
296+
297+
zend_string *prefix;
298+
299+
short mode;
300+
struct fold_item *head;
301+
struct fold_item *current;
302+
303+
zend_string *pipeline_cmd;
304+
305+
zend_string *err;
306+
307+
int scan;
308+
309+
int readonly;
310+
int reply_literal;
311+
int null_mbulk_as_null;
312+
int tcp_keepalive;
295313
} RedisSock;
296314
/* }}} */
297315

‎config.m4‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -323,5 +323,5 @@ if test "$PHP_REDIS" != "no"; then
323323
fi
324324

325325
PHP_SUBST(REDIS_SHARED_LIBADD)
326-
PHP_NEW_EXTENSION(redis, redis.c redis_commands.c library.c redis_session.c redis_array.c redis_array_impl.c redis_cluster.c cluster_library.c redis_sentinel.c sentinel_library.c $lzf_sources, $ext_shared)
326+
PHP_NEW_EXTENSION(redis, redis.c redis_commands.c library.c redis_session.c redis_array.c redis_array_impl.c redis_cluster.c cluster_library.c redis_sentinel.c sentinel_library.c backoff.c $lzf_sources, $ext_shared)
327327
fi

‎config.w32‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ ARG_ENABLE("redis-session", "whether to enable sessions", "yes");
55
ARG_ENABLE("redis-igbinary", "whether to enable igbinary serializer support", "no");
66

77
if (PHP_REDIS != "no") {
8-
var sources = "redis.c redis_commands.c library.c redis_session.c redis_array.c redis_array_impl.c redis_cluster.c cluster_library.c redis_sentinel.c sentinel_library.c";
8+
var sources = "redis.c redis_commands.c library.c redis_session.c redis_array.c redis_array_impl.c redis_cluster.c cluster_library.c redis_sentinel.c sentinel_library.c backoff.c";
99
if (PHP_REDIS_SESSION != "no") {
1010
ADD_EXTENSION_DEP("redis", "session");
1111
ADD_FLAG("CFLAGS_REDIS", ' /D PHP_SESSION=1 ');

‎library.c‎

Lines changed: 10 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -301,7 +301,7 @@ redis_error_throw(RedisSock *redis_sock)
301301
PHP_REDIS_API int
302302
redis_check_eof(RedisSock *redis_sock, int no_throw)
303303
{
304-
int count;
304+
unsigned int retry_index;
305305
char *errmsg;
306306

307307
if (!redis_sock || !redis_sock->stream || redis_sock->status == REDIS_SOCK_STATUS_FAILED) {
@@ -333,18 +333,17 @@ redis_check_eof(RedisSock *redis_sock, int no_throw)
333333
errmsg = "Connection lost and socket is in MULTI/watching mode";
334334
} else {
335335
errmsg = "Connection lost";
336-
/* TODO: configurable max retry count */
337-
for (count = 0; count < 10; ++count) {
336+
redis_backoff_reset(&redis_sock->backoff);
337+
for (retry_index = 0; retry_index < redis_sock->max_retries; ++retry_index) {
338338
/* close existing stream before reconnecting */
339339
if (redis_sock->stream) {
340340
redis_sock_disconnect(redis_sock, 1);
341341
}
342-
// Wait for a while before trying to reconnect
343-
if (redis_sock->retry_interval) {
344-
// Random factor to avoid having several (or many) concurrent connections trying to reconnect at the same time
345-
long retry_interval = (count ? redis_sock->retry_interval : (php_rand() % redis_sock->retry_interval));
346-
usleep(retry_interval);
347-
}
342+
/* Sleep based on our backoff algorithm */
343+
zend_ulong delay = redis_backoff_compute(&redis_sock->backoff, retry_index);
344+
if (delay != 0)
345+
usleep(delay);
346+
348347
/* reconnect */
349348
if (redis_sock_connect(redis_sock) == 0) {
350349
/* check for EOF again. */
@@ -2127,6 +2126,8 @@ redis_sock_create(char *host, int host_len, int port,
21272126
redis_sock->host = zend_string_init(host, host_len, 0);
21282127
redis_sock->status = REDIS_SOCK_STATUS_DISCONNECTED;
21292128
redis_sock->retry_interval = retry_interval * 1000;
2129+
redis_sock->max_retries = 10;
2130+
redis_initialize_backoff(&redis_sock->backoff, retry_interval);
21302131
redis_sock->persistent = persistent;
21312132

21322133
if (persistent && persistent_id != NULL) {

‎package.xml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,8 @@ http://pear.php.net/dtd/package-2.0.xsd">
6868
<file role='doc' name='arrays.markdown'/>
6969
<file role='doc' name='cluster.markdown'/>
7070
<file role='doc' name='sentinel.markdown'/>
71+
<file role='src' name='backoff.c'/>
72+
<file role='src' name='backoff.h'/>
7173
<file role='src' name='cluster_library.c'/>
7274
<file role='src' name='cluster_library.h'/>
7375
<file role='src' name='common.h'/>

‎redis.c‎

Lines changed: 15 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -768,8 +768,21 @@ static void add_class_constants(zend_class_entry *ce, int is_cluster) {
768768
zend_declare_class_constant_long(ce, ZEND_STRL("FAILOVER_DISTRIBUTE_SLAVES"), REDIS_FAILOVER_DISTRIBUTE_SLAVES);
769769
}
770770

771-
zend_declare_class_constant_stringl(ce, "AFTER", 5, "after", 5);
772-
zend_declare_class_constant_stringl(ce, "BEFORE", 6, "before", 6);
771+
/* retry/backoff options*/
772+
zend_declare_class_constant_long(ce, ZEND_STRL("OPT_MAX_RETRIES"), REDIS_OPT_MAX_RETRIES);
773+
774+
zend_declare_class_constant_long(ce, ZEND_STRL("OPT_BACKOFF_ALGORITHM"), REDIS_OPT_BACKOFF_ALGORITHM);
775+
zend_declare_class_constant_long(ce, ZEND_STRL("BACKOFF_ALGORITHM_DEFAULT"), REDIS_BACKOFF_ALGORITHM_DEFAULT);
776+
zend_declare_class_constant_long(ce, ZEND_STRL("BACKOFF_ALGORITHM_CONSTANT"), REDIS_BACKOFF_ALGORITHM_CONSTANT);
777+
zend_declare_class_constant_long(ce, ZEND_STRL("BACKOFF_ALGORITHM_UNIFORM"), REDIS_BACKOFF_ALGORITHM_UNIFORM);
778+
zend_declare_class_constant_long(ce, ZEND_STRL("BACKOFF_ALGORITHM_EXPONENTIAL"), REDIS_BACKOFF_ALGORITHM_EXPONENTIAL);
779+
zend_declare_class_constant_long(ce, ZEND_STRL("BACKOFF_ALGORITHM_FULL_JITTER"), REDIS_BACKOFF_ALGORITHM_FULL_JITTER);
780+
zend_declare_class_constant_long(ce, ZEND_STRL("BACKOFF_ALGORITHM_EQUAL_JITTER"), REDIS_BACKOFF_ALGORITHM_EQUAL_JITTER);
781+
zend_declare_class_constant_long(ce, ZEND_STRL("BACKOFF_ALGORITHM_DECORRELATED_JITTER"), REDIS_BACKOFF_ALGORITHM_DECORRELATED_JITTER);
782+
783+
zend_declare_class_constant_long(ce, ZEND_STRL("OPT_BACKOFF_BASE"), REDIS_OPT_BACKOFF_BASE);
784+
785+
zend_declare_class_constant_long(ce, ZEND_STRL("OPT_BACKOFF_CAP"), REDIS_OPT_BACKOFF_CAP);
773786
}
774787

775788
static ZEND_RSRC_DTOR_FUNC(redis_connections_pool_dtor)

‎redis_commands.c‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4008,6 +4008,14 @@ void redis_getoption_handler(INTERNAL_FUNCTION_PARAMETERS,
40084008
RETURN_LONG(redis_sock->null_mbulk_as_null);
40094009
case REDIS_OPT_FAILOVER:
40104010
RETURN_LONG(c->failover);
4011+
case REDIS_OPT_MAX_RETRIES:
4012+
RETURN_LONG(redis_sock->max_retries);
4013+
case REDIS_OPT_BACKOFF_ALGORITHM:
4014+
RETURN_LONG(redis_sock->backoff.algorithm);
4015+
case REDIS_OPT_BACKOFF_BASE:
4016+
RETURN_LONG(redis_sock->backoff.base / 1000);
4017+
case REDIS_OPT_BACKOFF_CAP:
4018+
RETURN_LONG(redis_sock->backoff.cap / 1000);
40114019
default:
40124020
RETURN_FALSE;
40134021
}
@@ -4141,6 +4149,35 @@ void redis_setoption_handler(INTERNAL_FUNCTION_PARAMETERS,
41414149
RETURN_TRUE;
41424150
}
41434151
break;
4152+
case REDIS_OPT_MAX_RETRIES:
4153+
val_long = zval_get_long(val);
4154+
if(val_long >= 0) {
4155+
redis_sock->max_retries = val_long;
4156+
RETURN_TRUE;
4157+
}
4158+
break;
4159+
case REDIS_OPT_BACKOFF_ALGORITHM:
4160+
val_long = zval_get_long(val);
4161+
if(val_long >= 0 &&
4162+
val_long < REDIS_BACKOFF_ALGORITHMS) {
4163+
redis_sock->backoff.algorithm = val_long;
4164+
RETURN_TRUE;
4165+
}
4166+
break;
4167+
case REDIS_OPT_BACKOFF_BASE:
4168+
val_long = zval_get_long(val);
4169+
if(val_long >= 0) {
4170+
redis_sock->backoff.base = val_long * 1000;
4171+
RETURN_TRUE;
4172+
}
4173+
break;
4174+
case REDIS_OPT_BACKOFF_CAP:
4175+
val_long = zval_get_long(val);
4176+
if(val_long >= 0) {
4177+
redis_sock->backoff.cap = val_long * 1000;
4178+
RETURN_TRUE;
4179+
}
4180+
break;
41444181
EMPTY_SWITCH_DEFAULT_CASE()
41454182
}
41464183
RETURN_FALSE;

0 commit comments

Comments
 (0)