item locks now lock hash table buckets

dormando · dormando · commit 1c94e12c3d76 · 2012-09-03T00:35:54.000-07:00
expansion requires switching to a global lock temporarily, so all buckets have
a covered read lock.

slab rebalancer is paused during hash table expansion.

internal item "trylocks" are always issued, and tracked as the hash power
variable can change out from under it.
diff --git a/assoc.c b/assoc.c
@@ -32,7 +32,7 @@ typedef  unsigned long  int  ub4;   /* unsigned 4-byte quantities */
 typedef  unsigned       char ub1;   /* unsigned 1-byte quantities */
 
 /* how many powers of 2's worth of buckets we use */
-static unsigned int hashpower = HASHPOWER_DEFAULT;
+unsigned int hashpower = HASHPOWER_DEFAULT;
 
 #define hashsize(n) ((ub4)1<<(n))
 #define hashmask(n) (hashsize(n)-1)
@@ -51,6 +51,7 @@ static unsigned int hash_items = 0;
 
 /* Flag: Are we in the middle of expanding now? */
 static bool expanding = false;
+static bool started_expanding = false;
 
 /*
  * During expansion we migrate values with bucket granularity; this is how
@@ -136,13 +137,19 @@ static void assoc_expand(void) {
         stats.hash_bytes += hashsize(hashpower) * sizeof(void *);
         stats.hash_is_expanding = 1;
         STATS_UNLOCK();
-        pthread_cond_signal(&maintenance_cond);
     } else {
         primary_hashtable = old_hashtable;
         /* Bad news, but we can keep running. */
     }
 }
 
+static void assoc_start_expand(void) {
+    if (started_expanding)
+        return;
+    started_expanding = true;
+    pthread_cond_signal(&maintenance_cond);
+}
+
 /* Note: this isn't an assoc_update.  The key must not already exist to call this */
 int assoc_insert(item *it, const uint32_t hv) {
     unsigned int oldbucket;
@@ -161,7 +168,7 @@ int assoc_insert(item *it, const uint32_t hv) {
 
     hash_items++;
     if (! expanding && hash_items > (hashsize(hashpower) * 3) / 2) {
-        assoc_expand();
+        assoc_start_expand();
     }
 
     MEMCACHED_ASSOC_INSERT(ITEM_key(it), it->nkey, hash_items);
@@ -201,6 +208,7 @@ static void *assoc_maintenance_thread(void *arg) {
 
         /* Lock the cache, and bulk move multiple buckets to the new
          * hash table. */
+        item_lock_global();
         mutex_lock(&cache_lock);
 
         for (ii = 0; ii < hash_bulk_move && expanding; ++ii) {
@@ -230,12 +238,24 @@ static void *assoc_maintenance_thread(void *arg) {
             }
         }
 
+        mutex_unlock(&cache_lock);
+        item_unlock_global();
+
         if (!expanding) {
+            /* finished expanding. tell all threads to use fine-grained locks */
+            switch_item_lock_type(ITEM_LOCK_GRANULAR);
+            started_expanding = false;
+            slabs_rebalancer_resume();
             /* We are done expanding.. just wait for next invocation */
             pthread_cond_wait(&maintenance_cond, &cache_lock);
+            /* Before doing anything, tell threads to use a global lock */
+            mutex_unlock(&cache_lock);
+            slabs_rebalancer_pause();
+            switch_item_lock_type(ITEM_LOCK_GLOBAL);
+            mutex_lock(&cache_lock);
+            assoc_expand();
+            mutex_unlock(&cache_lock);
         }
-
-        mutex_unlock(&cache_lock);
     }
     return NULL;
 }
diff --git a/assoc.h b/assoc.h
@@ -6,4 +6,4 @@ void assoc_delete(const char *key, const size_t nkey, const uint32_t hv);
 void do_assoc_move_next_bucket(void);
 int start_assoc_maintenance_thread(void);
 void stop_assoc_maintenance_thread(void);
-
+extern unsigned int hashpower;
diff --git a/items.c b/items.c
@@ -105,6 +105,7 @@ item *do_item_alloc(char *key, const size_t nkey, const int flags,
     int tries = 5;
     int tried_alloc = 0;
     item *search;
+    void *hold_lock = NULL;
     rel_time_t oldest_live = settings.oldest_live;
 
     search = tails[id];
@@ -115,7 +116,8 @@ item *do_item_alloc(char *key, const size_t nkey, const int flags,
         /* Attempt to hash item lock the "search" item. If locked, no
          * other callers can incr the refcount
          */
-        if (hv != cur_hv && item_trylock(hv) != 0)
+        /* FIXME: I think we need to mask the hv here for comparison? */
+        if (hv != cur_hv && (hold_lock = item_trylock(hv)) == NULL)
             continue;
         /* Now see if the item is refcount locked */
         if (refcount_incr(&search->refcount) != 2) {
@@ -128,7 +130,8 @@ item *do_item_alloc(char *key, const size_t nkey, const int flags,
                 search->refcount = 1;
                 do_item_unlink_nolock(search, hv);
             }
-            item_unlock(hv);
+            if (hold_lock)
+                item_trylock_unlock(hold_lock);
             continue;
         }
 
@@ -187,7 +190,9 @@ item *do_item_alloc(char *key, const size_t nkey, const int flags,
         }
 
         refcount_decr(&search->refcount);
-        item_unlock(hv);
+        /* If hash values were equal, we don't grab a second lock */
+        if (hold_lock)
+            item_trylock_unlock(hold_lock);
         break;
     }
 
@@ -506,7 +511,7 @@ void do_item_stats_sizes(ADD_STAT add_stats, void *c) {
 
 /** wrapper around assoc_find which does the lazy expiration logic */
 item *do_item_get(const char *key, const size_t nkey, const uint32_t hv) {
-    mutex_lock(&cache_lock);
+    //mutex_lock(&cache_lock);
     item *it = assoc_find(key, nkey, hv);
     if (it != NULL) {
         refcount_incr(&it->refcount);
@@ -520,7 +525,7 @@ item *do_item_get(const char *key, const size_t nkey, const uint32_t hv) {
             it = NULL;
         }
     }
-    mutex_unlock(&cache_lock);
+    //mutex_unlock(&cache_lock);
     int was_found = 0;
 
     if (settings.verbose > 2) {
diff --git a/memcached.h b/memcached.h
@@ -180,6 +180,11 @@ enum network_transport {
     udp_transport
 };
 
+enum item_lock_types {
+    ITEM_LOCK_GRANULAR = 0,
+    ITEM_LOCK_GLOBAL
+};
+
 #define IS_UDP(x) (x == udp_transport)
 
 #define NREAD_ADD 1
@@ -352,6 +357,7 @@ typedef struct {
     struct thread_stats stats;  /* Stats generated by this thread */
     struct conn_queue *new_conn_queue; /* queue of new connections to handle */
     cache_t *suffix_cache;      /* suffix cache */
+    uint8_t item_lock_type;     /* use fine-grained or global item lock */
 } LIBEVENT_THREAD;
 
 typedef struct {
@@ -531,9 +537,13 @@ void  item_stats_sizes(ADD_STAT add_stats, void *c);
 void  item_unlink(item *it);
 void  item_update(item *it);
 
+void item_lock_global(void);
+void item_unlock_global(void);
 void item_lock(uint32_t hv);
-int item_trylock(uint32_t hv);
+void *item_trylock(uint32_t hv);
+void item_trylock_unlock(void *arg);
 void item_unlock(uint32_t hv);
+void switch_item_lock_type(enum item_lock_types type);
 unsigned short refcount_incr(unsigned short *refcount);
 unsigned short refcount_decr(unsigned short *refcount);
 void STATS_LOCK(void);
diff --git a/slabs.c b/slabs.c
@@ -522,8 +522,9 @@ static int slab_rebalance_move(void) {
         item *it = slab_rebal.slab_pos;
         status = MOVE_PASS;
         if (it->slabs_clsid != 255) {
+            void *hold_lock = NULL;
             uint32_t hv = hash(ITEM_key(it), it->nkey, 0);
-            if (item_trylock(hv) != 0) {
+            if ((hold_lock = item_trylock(hv)) == NULL) {
                 status = MOVE_LOCKED;
             } else {
                 refcount = refcount_incr(&it->refcount);
@@ -557,7 +558,7 @@ static int slab_rebalance_move(void) {
                     }
                     status = MOVE_BUSY;
                 }
-                item_unlock(hv);
+                item_trylock_unlock(hold_lock);
             }
         }
 
@@ -738,6 +739,8 @@ static void *slab_maintenance_thread(void *arg) {
  */
 static void *slab_rebalance_thread(void *arg) {
     int was_busy = 0;
+    /* So we first pass into cond_wait with the mutex held */
+    mutex_lock(&slabs_rebalance_lock);
 
     while (do_run_slab_rebalance_thread) {
         if (slab_rebalance_signal == 1) {
@@ -826,6 +829,15 @@ enum reassign_result_type slabs_reassign(int src, int dst) {
     return ret;
 }
 
+/* If we hold this lock, rebalancer can't wake up or move */
+void slabs_rebalancer_pause(void) {
+    pthread_mutex_lock(&slabs_rebalance_lock);
+}
+
+void slabs_rebalancer_resume(void) {
+    pthread_mutex_unlock(&slabs_rebalance_lock);
+}
+
 static pthread_t maintenance_tid;
 static pthread_t rebalance_tid;
 
diff --git a/slabs.h b/slabs.h
@@ -43,4 +43,7 @@ enum reassign_result_type {
 
 enum reassign_result_type slabs_reassign(int src, int dst);
 
+void slabs_rebalancer_pause(void);
+void slabs_rebalancer_resume(void);
+
 #endif
diff --git a/thread.c b/thread.c
