Add a new failover option prioritising nodes

aolley · aolley · commit 84f44c617130 · 2021-03-04T23:47:34.000+10:30
This makes it possible to define a list of nodes you'd prefer to communicate
with in cluster mode.

This behaves similarly to FAILOVER_DISTRIBUTE, where it randomises the list of
nodes before trying them in sequence - however it takes that random list and
sorts any preferred nodes to the top first. All candidate nodes are still in
the list, but the preferred ones get tried first.

This is extremely helpful for setups where you know which nodes are closer
(i.e. not crossing an AZ boundary).
diff --git a/cluster_library.c b/cluster_library.c
@@ -843,6 +843,10 @@ PHP_REDIS_API redisCluster *cluster_create(double timeout, double read_timeout,
     ALLOC_HASHTABLE(c->nodes);
     zend_hash_init(c->nodes, 0, NULL, ht_free_node, 0);
 
+    /* Allocate our preferred nodes HashTable */
+    ALLOC_HASHTABLE(c->preferred_nodes);
+    zend_hash_init(c->preferred_nodes, 0, NULL, NULL, 0);
+
     return c;
 }
 
@@ -861,10 +865,12 @@ cluster_free(redisCluster *c, int free_ctx)
     /* Call hash table destructors */
     zend_hash_destroy(c->seeds);
     zend_hash_destroy(c->nodes);
+    zend_hash_destroy(c->preferred_nodes);
 
     /* Free hash tables themselves */
     efree(c->seeds);
     efree(c->nodes);
+    efree(c->preferred_nodes);
 
     /* Free any error we've got */
     if (c->err) zend_string_release(c->err);
@@ -1235,10 +1241,74 @@ PHP_REDIS_API void cluster_disconnect(redisCluster *c, int force) {
     } ZEND_HASH_FOREACH_END();
 }
 
+int preferred_compare(const void *const first, const void *const second)
+{
+    const preferredNode* a = (const preferredNode*)first;
+    const preferredNode* b = (const preferredNode*)second;
+    if (a->preferred == 1 && b->preferred == 0)
+        return -1;
+    else if (a->preferred == 0 && b->preferred == 1)
+        return 1;
+    else if (a->original_order < b->original_order)
+        return -1;
+    else if (a->original_order > b->original_order)
+        return 1;
+    else
+        return 0;
+}
+
+/* This method takes the randomised list of nodes and sorts preferred nodes to
+ * the top. */
+static void preferredsort(int *array, size_t len, redisCluster *c,
+							unsigned short slot)
+{
+    int i, temp, key_len, *prefnodes;
+    size_t r;
+    RedisSock *redis_sock;
+    char key[1024];
+    zval *node;
+
+    struct preferredNode ab[len];
+
+    // array: key => order; value => node-idx (0=master)
+    for (i = 0; i < len; i++) {
+        // Get node host+port string.
+        redis_sock = cluster_slot_sock(c, c->cmd_slot, array[i]);
+
+        ab[i].idx = array[i];
+        ab[i].original_order = i;
+        ab[i].preferred = 0;
+        if (!redis_sock) {
+            continue;
+        }
+
+        // Is it in the preferred_nodes map?
+        key_len = snprintf(key, sizeof(key), "%s:%d", ZSTR_VAL(redis_sock->host), redis_sock->port);
+        // Perhaps the preferred_nodes table should be keyed on the host:port string to make this
+        // easier and faster.
+        ZEND_HASH_FOREACH_VAL(c->preferred_nodes, node) {
+            ZVAL_DEREF(node);
+
+            if (strcmp(Z_STRVAL_P(node), key) == 0) {
+                ab[i].preferred = 1;
+                break;
+            }
+
+        } ZEND_HASH_FOREACH_END();
+    }
+
+    // Sort preferred nodes to the top of the list.
+    qsort(ab, len, sizeof(*ab), preferred_compare);
+    for (i = 0; i < len; i++) {
+        array[i] = ab[i].idx;
+    }
+}
+
+
 /* This method attempts to write our command at random to the master and any
  * attached slaves, until we either successufly do so, or fail. */
 static int cluster_dist_write(redisCluster *c, const char *cmd, size_t sz,
-                              int nomaster)
+                              int nomaster, int preferred)
 {
     int i, count = 1, *nodes;
     RedisSock *redis_sock;
@@ -1256,6 +1326,11 @@ static int cluster_dist_write(redisCluster *c, const char *cmd, size_t sz,
     for (i = 0; i < count; i++) nodes[i] = i;
     fyshuffle(nodes, count);
 
+    /* Shift preferred nodes to the top of the list if we're in preferred
+     * mode */
+    if (preferred && zend_hash_num_elements(c->preferred_nodes) > 0)
+        preferredsort(nodes, count, c, c->cmd_slot);
+
     /* Iterate through our nodes until we find one we can write to or fail */
     for (i = 0; i < count; i++) {
         /* Skip if this is the master node and we don't want to query that */
@@ -1306,6 +1381,8 @@ static int cluster_dist_write(redisCluster *c, const char *cmd, size_t sz,
  * REDIS_FAILOVER_DISTRIBUTE_SLAVES:
  *   We pick at random from slave nodes of a given master.  This option is
  *   used to load balance read queries against N slaves.
+ * REDIS_FAILOVER_PREFERRED:
+ *   Similar to DISTRIBUTE, but with a list of nodes we prefer over others.
  *
  * Once we are able to find a node we can write to, we check for MOVED or
  * ASKING redirection, such that the keyspace can be updated.
@@ -1315,7 +1392,7 @@ static int cluster_sock_write(redisCluster *c, const char *cmd, size_t sz,
 {
     redisClusterNode *seed_node;
     RedisSock *redis_sock;
-    int failover, nomaster;
+    int failover, nomaster, preferred;
 
     /* First try the socket requested */
     redis_sock = c->cmd_sock;
@@ -1343,12 +1420,13 @@ static int cluster_sock_write(redisCluster *c, const char *cmd, size_t sz,
     } else if (failover == REDIS_FAILOVER_ERROR) {
         /* Try the master, then fall back to any slaves we may have */
         if (CLUSTER_SEND_PAYLOAD(redis_sock, cmd, sz) ||
-           !cluster_dist_write(c, cmd, sz, 1)) return 0;
+           !cluster_dist_write(c, cmd, sz, 1, 0)) return 0;
     } else {
         /* Include or exclude master node depending on failover option and
          * attempt to make our write */
         nomaster = failover == REDIS_FAILOVER_DISTRIBUTE_SLAVES;
-        if (!cluster_dist_write(c, cmd, sz, nomaster)) {
+        preferred = failover == REDIS_FAILOVER_PREFERRED;
+        if (!cluster_dist_write(c, cmd, sz, nomaster, preferred)) {
             /* We were able to write to a master or slave at random */
             return 0;
         }
diff --git a/cluster_library.h b/cluster_library.h
@@ -208,6 +208,9 @@ typedef struct redisCluster {
     /* All RedisCluster objects we've created/are connected to */
     HashTable *nodes;
 
+    /* A list of nodes we prefer to talk to when in FAILOVER_PREFERRED mode */
+    HashTable *preferred_nodes;
+
     /* Transaction handling linked list, and where we are as we EXEC */
     clusterFoldItem *multi_head;
     clusterFoldItem *multi_curr;
@@ -259,6 +262,12 @@ typedef struct redisCluster {
 /* RedisCluster response processing callback */
 typedef void (*cluster_cb)(INTERNAL_FUNCTION_PARAMETERS, redisCluster*, void*);
 
+typedef struct preferredNode {
+    int idx;
+    int preferred;
+    int original_order;
+} preferredNode;
+
 /* Context for processing transactions */
 struct clusterFoldItem {
     /* Response processing callback */
diff --git a/common.h b/common.h
@@ -82,12 +82,14 @@ typedef enum _PUBSUB_TYPE {
 #define REDIS_OPT_REPLY_LITERAL      8
 #define REDIS_OPT_COMPRESSION_LEVEL  9
 #define REDIS_OPT_NULL_MBULK_AS_NULL 10
+#define REDIS_OPT_PREFERRED_NODES    11
 
 /* cluster options */
 #define REDIS_FAILOVER_NONE              0
 #define REDIS_FAILOVER_ERROR             1
 #define REDIS_FAILOVER_DISTRIBUTE        2
 #define REDIS_FAILOVER_DISTRIBUTE_SLAVES 3
+#define REDIS_FAILOVER_PREFERRED         4
 /* serializers */
 typedef enum {
     REDIS_SERIALIZER_NONE,
diff --git a/redis.c b/redis.c
@@ -688,6 +688,7 @@ static void add_class_constants(zend_class_entry *ce, int is_cluster) {
     zend_declare_class_constant_long(ce, ZEND_STRL("OPT_REPLY_LITERAL"), REDIS_OPT_REPLY_LITERAL);
     zend_declare_class_constant_long(ce, ZEND_STRL("OPT_COMPRESSION_LEVEL"), REDIS_OPT_COMPRESSION_LEVEL);
     zend_declare_class_constant_long(ce, ZEND_STRL("OPT_NULL_MULTIBULK_AS_NULL"), REDIS_OPT_NULL_MBULK_AS_NULL);
+    zend_declare_class_constant_long(ce, ZEND_STRL("OPT_PREFERRED_NODES"), REDIS_OPT_PREFERRED_NODES);
 
     /* serializer */
     zend_declare_class_constant_long(ce, ZEND_STRL("SERIALIZER_NONE"), REDIS_SERIALIZER_NONE);
@@ -737,6 +738,7 @@ static void add_class_constants(zend_class_entry *ce, int is_cluster) {
         zend_declare_class_constant_long(ce, ZEND_STRL("FAILOVER_ERROR"), REDIS_FAILOVER_ERROR);
         zend_declare_class_constant_long(ce, ZEND_STRL("FAILOVER_DISTRIBUTE"), REDIS_FAILOVER_DISTRIBUTE);
         zend_declare_class_constant_long(ce, ZEND_STRL("FAILOVER_DISTRIBUTE_SLAVES"), REDIS_FAILOVER_DISTRIBUTE_SLAVES);
+        zend_declare_class_constant_long(ce, ZEND_STRL("FAILOVER_PREFERRED"), REDIS_FAILOVER_PREFERRED);
     } else {
         /* Cluster doesn't support pipelining at this time */
         zend_declare_class_constant_long(ce, ZEND_STRL("PIPELINE"), PIPELINE);
diff --git a/redis_cluster.c b/redis_cluster.c
@@ -326,6 +326,10 @@ zend_object * create_cluster_context(zend_class_entry *class_type) {
     ALLOC_HASHTABLE(cluster->nodes);
     zend_hash_init(cluster->nodes, 0, NULL, ht_free_node, 0);
 
+    /* Allocate our preferred nodes HashTable */
+    ALLOC_HASHTABLE(cluster->preferred_nodes);
+    zend_hash_init(cluster->preferred_nodes, 0, NULL, NULL, 0);
+
     // Initialize it
     zend_object_std_init(&cluster->std, class_type);
 
diff --git a/redis_commands.c b/redis_commands.c
@@ -4041,6 +4041,8 @@ void redis_getoption_handler(INTERNAL_FUNCTION_PARAMETERS,
             RETURN_LONG(redis_sock->null_mbulk_as_null);
         case REDIS_OPT_FAILOVER:
             RETURN_LONG(c->failover);
+        case REDIS_OPT_PREFERRED_NODES:
+            RETURN_ARR(zend_array_dup(c->preferred_nodes));
         default:
             RETURN_FALSE;
     }
@@ -4168,12 +4170,22 @@ void redis_setoption_handler(INTERNAL_FUNCTION_PARAMETERS,
             if (val_long == REDIS_FAILOVER_NONE ||
                 val_long == REDIS_FAILOVER_ERROR ||
                 val_long == REDIS_FAILOVER_DISTRIBUTE ||
-                val_long == REDIS_FAILOVER_DISTRIBUTE_SLAVES)
+                val_long == REDIS_FAILOVER_DISTRIBUTE_SLAVES ||
+                val_long == REDIS_FAILOVER_PREFERRED)
             {
                 c->failover = val_long;
                 RETURN_TRUE;
             }
             break;
+        case REDIS_OPT_PREFERRED_NODES:
+            if (Z_TYPE_P(val) != IS_ARRAY)
+                RETURN_FALSE;
+            if (c->preferred_nodes) {
+                zend_hash_destroy(c->preferred_nodes);
+                FREE_HASHTABLE(c->preferred_nodes);
+            }
+            c->preferred_nodes = zend_array_dup(Z_ARRVAL_P(val));
+            RETURN_TRUE;
         EMPTY_SWITCH_DEFAULT_CASE()
     }
     RETURN_FALSE;
diff --git a/tests/RedisClusterTest.php b/tests/RedisClusterTest.php
@@ -654,6 +654,32 @@ public function testFailOver() {
         }
     }
 
+    public function testFailOverPreferred() {
+        return $this->markTestSkipped(); // this test is racy.
+        $arr_value_ref = [];
+        $arr_type_ref  = [];
+
+        /* Set a bunch of keys of various redis types*/
+        for ($i = 0; $i < 1; $i++) {
+            foreach ($this->_arr_redis_types as $i_type) {
+                $str_key = $this->setKeyVals($i, $i_type, $arr_value_ref);
+                $arr_type_ref[$str_key] = $i_type;
+            }
+        }
+
+        // Give slaves a chance to catch up with the master.
+        // This is racy... which is probably why the above test skips the
+        // failover modes other than 'NONE'.
+        usleep(1000);
+
+        $this->redis->setOption(RedisCluster::OPT_SLAVE_FAILOVER, RedisCluster::FAILOVER_PREFERRED);
+        $this->redis->setOption(RedisCluster::OPT_PREFERRED_NODES, ['127.0.0.1:7000']);
+
+        foreach ($arr_value_ref as $str_key => $value) {
+            $this->checkKeyValue($str_key, $arr_type_ref[$str_key], $value);
+        }
+    }
+
     /* Test a 'raw' command */
     public function testRawCommand() {
         $this->redis->rawCommand('mykey', 'set', 'mykey', 'my-value');
