Added new metric: reap_too_many_dead

johnousterhout · johnousterhout · commit e8cd12f3793e · 2020-09-18T17:43:26.000-07:00
diff --git a/homa_impl.h b/homa_impl.h
@@ -1889,6 +1889,12 @@ struct homa_metrics {
 	 */
 	__u64 reaper_dead_skbs;
 	
+	/**
+	 * @reap_too_many_dead: total number of times that homa_wait_for_message
+	 * invoked the reaper because dead_skbs was too high.
+	 */
+	__u64 reap_too_many_dead;
+	
 	/**
 	 * @throttle_list_adds: Total number of calls to homa_add_to_throttled.
 	 */
diff --git a/homa_incoming.c b/homa_incoming.c
@@ -1123,13 +1123,16 @@ struct homa_rpc *homa_wait_for_message(struct homa_sock *hsk, int flags,
 	int error;
 	
 	homa_interest_init(&interest);
-	while (hsk->dead_skbs > hsk->homa->max_dead_buffs) {
-		/* Way too many dead RPCs; must cleanup immediately. */
-		if (!homa_rpc_reap(hsk))
-			break;
+	if (hsk->dead_skbs > hsk->homa->max_dead_buffs) {
+		/* Too many dead RPCs; must cleanup immediately. */
+		INC_METRIC(reap_too_many_dead, 1);
+		do {
+			if (!homa_rpc_reap(hsk))
+				break;
 
-		/* Give NAPI and SoftIRQ tasks a chance to run. */
-		schedule();
+			/* Give NAPI and SoftIRQ tasks a chance to run. */
+			schedule();
+		} while (hsk->dead_skbs > hsk->homa->max_dead_buffs);
 	}
 	
 	/* Normally this loop only gets executed once, but we may have
diff --git a/homa_utils.c b/homa_utils.c
@@ -1311,6 +1311,10 @@ char *homa_print_metrics(struct homa *homa)
 				"Sum of hsk->dead_skbs across all reaper "
 				"calls\n",
 				m->reaper_dead_skbs);
+		homa_append_metric(homa,
+				"reap_too_many_dead        %15llu  "
+				"Reaps forced by dead RPC buildup\n",
+				m->reap_too_many_dead);
 		homa_append_metric(homa,
 				"throttle_list_adds        %15llu  "
 				"Calls to homa_add_to_throttled\n",
diff --git a/notes.txt b/notes.txt
@@ -9,7 +9,6 @@ Notes for Homa implementation in Linux:
 * Performance-related tasks:
   * See if turning off c-states allows shorter polling intervals?
   * Are Meltdown mitigations really disabled?
-  * Add metric for forced reaps because of too many messages.
   * Consider a permanent reduction in rtt_bytes.
   * Consider reducing throttle_min_bytes to see if it helps region 1
     in the CDF?
diff --git a/test/unit_homa_incoming.c b/test/unit_homa_incoming.c
@@ -1862,6 +1862,7 @@ TEST_F(homa_incoming, homa_wait_for_message__dead_buffs_exceeded)
 	rpc = homa_wait_for_message(&self->hsk, HOMA_RECV_RESPONSE, 44);
 	EXPECT_EQ(EINVAL, -PTR_ERR(rpc));
 	EXPECT_EQ(10, self->hsk.dead_skbs);
+	EXPECT_EQ(1, homa_cores[cpu_number]->metrics.reap_too_many_dead);
 }
 TEST_F(homa_incoming, homa_wait_for_message__rpc_from_register_interests)
 {
diff --git a/util/metrics.py b/util/metrics.py
@@ -392,7 +392,8 @@ def scale_number(number):
             "server_cant_create_rpcs", "server_cant_create_rpcs",
             "short_packets", "redundant_packets",
             "client_peer_timeouts", "server_rpc_discards",
-            "server_rpcs_unknown", "stale_generations", "generation_overflows"]:
+            "server_rpcs_unknown", "stale_generations", "generation_overflows",
+            "reap_too_many_dead"]:
         if deltas[symbol] == 0:
             continue
         rate = float(deltas[symbol])/elapsed_secs

Original file line number	Diff line number	Diff line change
`@@ -1862,6 +1862,7 @@ TEST_F(homa_incoming, homa_wait_for_message__dead_buffs_exceeded)`
`1862`	`1862`	`rpc = homa_wait_for_message(&self->hsk, HOMA_RECV_RESPONSE, 44);`
`1863`	`1863`	`EXPECT_EQ(EINVAL, -PTR_ERR(rpc));`
`1864`	`1864`	`EXPECT_EQ(10, self->hsk.dead_skbs);`
	`1865`	`+ EXPECT_EQ(1, homa_cores[cpu_number]->metrics.reap_too_many_dead);`
`1865`	`1866`	`}`
`1866`	`1867`	`TEST_F(homa_incoming, homa_wait_for_message__rpc_from_register_interests)`
`1867`	`1868`	`{`