Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit cd7c22a

Browse files
johnousterhoutjohnousterhout
committed
Rework metrics for homa_qdisc
* Added some new metrics, plus new "Pacer" section in homa_metrics.py * Removed pacer_lost_cycles: didn't seem to be meaningful. * Renamed throttled_cycles to nic_backlog_cycles. * Also removed all STRIP ifdefs from homa_pacer.c
1 parent b75d4f6 commit cd7c22a

File tree

10 files changed

+248
-296
lines changed

10 files changed

+248
-296
lines changed

‎homa_metrics.c‎

Lines changed: 12 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -289,22 +289,24 @@ char *homa_metrics_print(void)
289289
"Time in homa_data_pkt spent reaping RPCs\n");
290290
M("idle_time_conflicts", m->idle_time_conflicts,
291291
"Cache conflicts when updating link_idle_time\n");
292+
M("nic_backlog_cycles", m->nic_backlog_cycles,
293+
"Time when NIC queue was backlogged\n");
292294
M("pacer_cycles", m->pacer_cycles,
293-
"Time spent in homa_pacer_main\n");
295+
"Execution time in pacer thread\n");
296+
M("pacer_xmit_cycles", m->pacer_xmit_cycles,
297+
"Time pacer spent xmitting packets (vs. polling NIC queue)\n");
298+
M("pacer_packets", m->pacer_packets,
299+
"Packets transmitted by the pacer\n");
300+
M("pacer_bytes", m->pacer_bytes,
301+
"Bytes transmitted by the pacer (including headers)\n");
302+
M("pacer_help_bytes", m->pacer_help_bytes,
303+
"Bytes transmitted via homa_qdisc_pacer_check");
294304
M("homa_cycles",
295305
m->softirq_cycles + m->napi_cycles +
296306
m->send_cycles + m->recv_cycles +
297307
m->reply_cycles - m->blocked_cycles +
298-
m->timer_cycles + m->pacer_cycles,
308+
m->timer_cycles + m->nic_backlog_cycles,
299309
"Total time in all Homa-related functions\n");
300-
M("pacer_lost_cycles", m->pacer_lost_cycles,
301-
"Lost transmission time because pacer was slow\n");
302-
M("pacer_bytes", m->pacer_bytes,
303-
"Bytes transmitted when the pacer was active\n");
304-
M("pacer_needed_help", m->pacer_needed_help,
305-
"homa_pacer_xmit invocations from homa_check_pacer\n");
306-
M("throttled_cycles", m->throttled_cycles,
307-
"Time when output was throttled because NIC was backlogged\n");
308310
M("resent_packets", m->resent_packets,
309311
"DATA packets sent in response to RESENDs\n");
310312
M("peer_allocs", m->peer_allocs,

‎homa_metrics.h‎

Lines changed: 25 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -375,36 +375,45 @@ struct homa_metrics {
375375
__u64 idle_time_conflicts;
376376

377377
/**
378-
* @pacer_cycles: total time spent executing in homa_pacer_main
379-
* (not including blocked time).
378+
* @nic_backlog_cycles: total amount of time when there were packets
379+
* waiting to be transmitted in homa_qdisc because the NIC queue was
380+
* too long.
381+
*/
382+
u64 nic_backlog_cycles;
383+
384+
/**
385+
* @pacer_cycles: total execution time in the pacer thread (excluding
386+
* blocked time).
380387
*/
381388
u64 pacer_cycles;
382389

383390
/**
384-
* @pacer_lost_cycles: unnecessary delays in transmitting packets
385-
* (i.e. wasted output bandwidth) because the pacer was slow or got
386-
* descheduled.
391+
* @pacer_xmit_cycles: total time spent by the pacer actually
392+
* transmitting packets (as opposed to polling waiting for the
393+
* NIC queue to subside).
387394
*/
388-
u64 pacer_lost_cycles;
395+
u64 pacer_xmit_cycles;
389396

390397
/**
391-
* @pacer_bytes: total number of bytes transmitted when
392-
* @homa->throttled_rpcs is nonempty.
398+
* @pacer_packets: total number of Homa packets that were transmitted
399+
* by homa_qdisc_pacer (they were deferred because of NIC queue
400+
* overload).
393401
*/
394-
u64 pacer_bytes;
402+
u64 pacer_packets;
395403

396404
/**
397-
* @pacer_needed_help: total number of times that homa_check_pacer
398-
* found that the pacer was running behind, so it actually invoked
399-
* homa_pacer_xmit.
405+
* @pacer_bytes: total number of bytes in packets that were
406+
* transmitted by homa_qdisc_pacer (they were deferred because of
407+
* NIC queue overload).
400408
*/
401-
u64 pacer_needed_help;
409+
u64 pacer_bytes;
402410

403411
/**
404-
* @throttled_cycles: total amount of time that @homa->throttled_rpcs
405-
* is nonempty.
412+
* @pacer_help_bytes: bytes in @pacer_bytes that were transmitted via
413+
* calls to homa_qdisc_pacer_check (presumably because the pacer thread
414+
* wasn't keeping up). Includes header bytes.
406415
*/
407-
u64 throttled_cycles;
416+
u64 pacer_help_bytes;
408417

409418
/**
410419
* @resent_packets: total number of data packets issued in response to

‎homa_pacer.c‎

Lines changed: 9 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,6 @@
99
#include "homa_pacer.h"
1010
#include "homa_rpc.h"
1111

12-
#ifndef __STRIP__ /* See strip.py */
1312
/* Used to enable sysctl access to pacer-specific configuration parameters. The
1413
* @data fields are actually offsets within a struct homa_pacer; these are
1514
* converted to pointers into a net-specific struct homa later.
@@ -38,7 +37,6 @@ static struct ctl_table pacer_ctl_table[] = {
3837
.proc_handler = homa_pacer_dointvec
3938
},
4039
};
41-
#endif /* See strip.py */
4240

4341
/**
4442
* homa_pacer_alloc() - Allocate and initialize a new pacer object, which
@@ -71,15 +69,13 @@ struct homa_pacer *homa_pacer_alloc(struct homa *homa)
7169
}
7270
atomic64_set(&pacer->link_idle_time, homa_clock());
7371

74-
#ifndef __STRIP__ /* See strip.py */
7572
pacer->sysctl_header = register_net_sysctl(&init_net, "net/homa",
7673
pacer_ctl_table);
7774
if (!pacer->sysctl_header) {
7875
err = -ENOMEM;
7976
pr_err("couldn't register sysctl parameters for Homa pacer\n");
8077
goto error;
8178
}
82-
#endif /* See strip.py */
8379
homa_pacer_update_sysctl_deps(pacer);
8480
return pacer;
8581

@@ -96,12 +92,10 @@ struct homa_pacer *homa_pacer_alloc(struct homa *homa)
9692
*/
9793
void homa_pacer_free(struct homa_pacer *pacer)
9894
{
99-
#ifndef __STRIP__ /* See strip.py */
10095
if (pacer->sysctl_header) {
10196
unregister_net_sysctl_table(pacer->sysctl_header);
10297
pacer->sysctl_header = NULL;
10398
}
104-
#endif /* See strip.py */
10599
if (pacer->kthread) {
106100
kthread_stop(pacer->kthread);
107101
pacer->kthread = NULL;
@@ -141,27 +135,12 @@ int homa_pacer_check_nic_q(struct homa_pacer *pacer, struct sk_buff *skb,
141135
if ((clock + pacer->max_nic_queue_cycles) < idle && !force &&
142136
!(pacer->homa->flags & HOMA_FLAG_DONT_THROTTLE))
143137
return 0;
144-
#ifndef __STRIP__ /* See strip.py */
145138
if (!list_empty(&pacer->throttled_rpcs))
146139
INC_METRIC(pacer_bytes, bytes);
147-
if (idle < clock) {
148-
if (pacer->wake_time) {
149-
u64 lost = (pacer->wake_time > idle)
150-
? clock - pacer->wake_time
151-
: clock - idle;
152-
INC_METRIC(pacer_lost_cycles, lost);
153-
tt_record1("pacer lost %d cycles", lost);
154-
}
155-
new_idle = clock + cycles_for_packet;
156-
} else {
157-
new_idle = idle + cycles_for_packet;
158-
}
159-
#else /* See strip.py */
160140
if (idle < clock)
161141
new_idle = clock + cycles_for_packet;
162142
else
163143
new_idle = idle + cycles_for_packet;
164-
#endif /* See strip.py */
165144

166145
/* This method must be thread-safe. */
167146
if (atomic64_cmpxchg_relaxed(&pacer->link_idle_time, idle,
@@ -180,15 +159,15 @@ int homa_pacer_check_nic_q(struct homa_pacer *pacer, struct sk_buff *skb,
180159
int homa_pacer_main(void *arg)
181160
{
182161
struct homa_pacer *pacer = arg;
162+
u64 wake_time;
183163
int status;
184164

185165
while (1) {
186166
if (kthread_should_stop())
187167
break;
188-
pacer->wake_time = homa_clock();
168+
wake_time = homa_clock();
189169
homa_pacer_xmit(pacer);
190-
INC_METRIC(pacer_cycles, homa_clock() - pacer->wake_time);
191-
pacer->wake_time = 0;
170+
INC_METRIC(pacer_cycles, homa_clock() - wake_time);
192171
if (!list_empty(&pacer->throttled_rpcs)) {
193172
/* NIC queue is full; before calling pacer again,
194173
* give other threads a chance to run (otherwise
@@ -285,12 +264,8 @@ void homa_pacer_xmit(struct homa_pacer *pacer)
285264
/* Note: rpc->state could be RPC_DEAD here, but the code
286265
* below should work anyway.
287266
*/
288-
#ifndef __STRIP__ /* See strip.py */
289267
if (!*rpc->msgout.next_xmit || rpc->msgout.next_xmit_offset >=
290268
rpc->msgout.granted) {
291-
#else /* See strip.py */
292-
if (!*rpc->msgout.next_xmit) {
293-
#endif /* See strip.py */
294269
/* No more data can be transmitted from this message
295270
* (right now), so remove it from the throttled list.
296271
*/
@@ -317,25 +292,22 @@ void homa_pacer_manage_rpc(struct homa_rpc *rpc)
317292
struct homa_pacer *pacer = rpc->hsk->homa->pacer;
318293
struct homa_rpc *candidate;
319294
int bytes_left;
320-
321-
IF_NO_STRIP(int checks = 0);
322-
IF_NO_STRIP(u64 now);
295+
int checks = 0;
296+
u64 now;
323297

324298
if (!list_empty(&rpc->throttled_links))
325299
return;
326-
IF_NO_STRIP(now = homa_clock());
327-
#ifndef __STRIP__ /* See strip.py */
300+
now = homa_clock();
328301
if (!list_empty(&pacer->throttled_rpcs))
329-
INC_METRIC(throttled_cycles, now - pacer->throttle_add);
302+
INC_METRIC(nic_backlog_cycles, now - pacer->throttle_add);
330303
pacer->throttle_add = now;
331-
#endif /* See strip.py */
332304
bytes_left = rpc->msgout.length - rpc->msgout.next_xmit_offset;
333305
homa_pacer_throttle_lock(pacer);
334306
list_for_each_entry(candidate, &pacer->throttled_rpcs,
335307
throttled_links) {
336308
int bytes_left_cand;
337309

338-
IF_NO_STRIP(checks++);
310+
checks++;
339311

340312
/* Watch out: the pacer might have just transmitted the last
341313
* packet from candidate.
@@ -371,11 +343,9 @@ void homa_pacer_unmanage_rpc(struct homa_rpc *rpc)
371343
UNIT_LOG("; ", "removing id %llu from throttled list", rpc->id);
372344
homa_pacer_throttle_lock(pacer);
373345
list_del_init(&rpc->throttled_links);
374-
#ifndef __STRIP__ /* See strip.py */
375346
if (list_empty(&pacer->throttled_rpcs))
376-
INC_METRIC(throttled_cycles, homa_clock()
347+
INC_METRIC(nic_backlog_cycles, homa_clock()
377348
- pacer->throttle_add);
378-
#endif /* See strip.py */
379349
homa_pacer_throttle_unlock(pacer);
380350
}
381351
}
@@ -399,7 +369,6 @@ void homa_pacer_update_sysctl_deps(struct homa_pacer *pacer)
399369
pacer->cycles_per_mbyte = tmp;
400370
}
401371

402-
#ifndef __STRIP__ /* See strip.py */
403372
/**
404373
* homa_pacer_dointvec() - This function is a wrapper around proc_dointvec. It
405374
* is invoked to read and write pacer-related sysctl values.
@@ -481,4 +450,3 @@ void homa_pacer_throttle_lock_slow(struct homa_pacer *pacer)
481450
INC_METRIC(throttle_lock_misses, 1);
482451
INC_METRIC(throttle_lock_miss_cycles, homa_clock() - start);
483452
}
484-
#endif /* See strip.py */

‎homa_pacer.h‎

Lines changed: 0 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -35,12 +35,6 @@ struct homa_pacer {
3535
*/
3636
int fifo_count;
3737

38-
/**
39-
* @wake_time: homa_clock() time when the pacer woke up (if the pacer
40-
* is running) or 0 if the pacer is sleeping.
41-
*/
42-
u64 wake_time;
43-
4438
/**
4539
* @throttle_lock: Used to synchronize access to @throttled_rpcs. Must
4640
* hold when inserting or removing an RPC from throttled_rpcs.
@@ -54,13 +48,11 @@ struct homa_pacer {
5448
*/
5549
struct list_head throttled_rpcs;
5650

57-
#ifndef __STRIP__ /* See strip.py */
5851
/**
5952
* @throttle_add: The most recent homa_clock() time when an RPC was
6053
* added to @throttled_rpcs.
6154
*/
6255
u64 throttle_add;
63-
#endif /* See strip.py */
6456

6557
/**
6658
* @fifo_fraction: Out of every 1000 packets transmitted by the
@@ -168,7 +160,6 @@ static inline void homa_pacer_check(struct homa_pacer *pacer)
168160
return;
169161
tt_record("homa_check_pacer calling homa_pacer_xmit");
170162
homa_pacer_xmit(pacer);
171-
INC_METRIC(pacer_needed_help, 1);
172163
}
173164

174165
#ifndef __STRIP__ /* See strip.py */

0 commit comments

Comments
 (0)