Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit 335dc89

Browse files
Dym03Kobzol
Dym03
authored and
Kobzol
committed
Add worker managed cpu utilization
1 parent 9ab0d36 commit 335dc89

3 files changed

Lines changed: 182 additions & 24 deletions

File tree

‎crates/hyperqueue/src/dashboard/ui/screens/cluster/worker/cpu_util_table.rs‎

Lines changed: 87 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,16 @@
11
use crate::common::format::human_size;
2+
use crate::dashboard::ui::screens::cluster::worker::UtilizationRenderMode;
23
use ratatui::layout::{Constraint, Rect};
34
use ratatui::style::Style;
45
use ratatui::widgets::{Cell, Row, Table};
56
use std::cmp;
67
use tako::hwstats::MemoryStats;
8+
use tako::resources::ResourceIndex;
79

810
use crate::dashboard::ui::styles;
911
use crate::dashboard::ui::terminal::DashboardFrame;
1012
use crate::dashboard::ui::widgets::progressbar::{
11-
ProgressPrintStyle, get_progress_bar_color, render_progress_bar_at,
13+
ProgressPrintStyle, get_progress_bar_color, get_progress_bar_cpu_color, render_progress_bar_at,
1214
};
1315
use crate::dashboard::utils::calculate_average;
1416

@@ -19,6 +21,8 @@ const CPU_METER_WIDTH: u8 = CPU_METER_PROGRESSBAR_WIDTH + 4;
1921
pub fn render_cpu_util_table(
2022
cpu_util_list: &[f64],
2123
mem_util: &MemoryStats,
24+
used_cpus: &[ResourceIndex],
25+
util_render_mode: &UtilizationRenderMode,
2226
rect: Rect,
2327
frame: &mut DashboardFrame,
2428
table_style: Style,
@@ -31,38 +35,50 @@ pub fn render_cpu_util_table(
3135
let width = constraints.len();
3236
let height = (cpu_util_list.len() as f64 / width as f64).ceil() as usize;
3337

34-
let mut rows: Vec<Vec<(f64, usize)>> = vec![vec![]; height];
35-
for (position, &cpu_util) in cpu_util_list.iter().enumerate() {
36-
let row = position % height;
37-
rows[row].push((cpu_util, position));
38+
let mut rows: Vec<Vec<(f64, usize, bool)>> = vec![vec![]; height];
39+
if *util_render_mode == UtilizationRenderMode::Worker {
40+
rows = get_utilization_sorted_by_usage(cpu_util_list, used_cpus, height)
41+
} else {
42+
for (position, &cpu_util) in cpu_util_list.iter().enumerate() {
43+
let row = position % height;
44+
let used = used_cpus.contains(&ResourceIndex::new(position as u32));
45+
rows[row].push((cpu_util, position, used));
46+
}
3847
}
3948

4049
let rows: Vec<Row> = rows
4150
.into_iter()
4251
.map(|targets| {
4352
let columns: Vec<Cell> = targets
4453
.into_iter()
45-
.map(|(cpu_util, position)| {
54+
.map(|(cpu_util, position, used)| {
4655
let progress = cpu_util / 100.00;
56+
let style = match util_render_mode {
57+
UtilizationRenderMode::Global => get_progress_bar_color(progress),
58+
UtilizationRenderMode::Worker => get_progress_bar_cpu_color(progress, used),
59+
};
60+
4761
Cell::from(render_progress_bar_at(
4862
Some(format!("{position:>3} ")),
4963
progress,
5064
CPU_METER_PROGRESSBAR_WIDTH,
5165
ProgressPrintStyle::default(),
5266
))
53-
.style(get_progress_bar_color(progress))
67+
.style(style)
5468
})
5569
.collect();
5670
Row::new(columns)
5771
})
5872
.collect();
5973

60-
let avg_cpu = calculate_average(cpu_util_list);
61-
6274
let mem_used = mem_util.total - mem_util.free;
75+
let (which_util, num_cpus, avg_cpu) =
76+
create_title_info(cpu_util_list, used_cpus, util_render_mode);
77+
6378
let title = styles::table_title(format!(
64-
"Worker Utilization ({} CPUs), Avg CPU = {:.0}%, Mem = {:.0}% ({}/{})",
65-
cpu_util_list.len(),
79+
"{} Utilization ({} CPUs), Avg CPU = {:.0}%, Mem = {:.0}% ({}/{})",
80+
which_util,
81+
num_cpus,
6682
avg_cpu,
6783
(mem_used as f64 / mem_util.total as f64) * 100.0,
6884
human_size(mem_used),
@@ -89,3 +105,63 @@ fn get_column_constraints(rect: Rect, num_cpus: usize) -> Vec<Constraint> {
89105
)
90106
.collect()
91107
}
108+
109+
fn get_utilization_sorted_by_usage(
110+
cpu_util_list: &[f64],
111+
used_cpus: &[ResourceIndex],
112+
height: usize,
113+
) -> Vec<Vec<(f64, usize, bool)>> {
114+
let mut all_cpus: Vec<(f64, usize, bool)> = cpu_util_list
115+
.iter()
116+
.enumerate()
117+
.map(|(position, &cpu_util)| {
118+
let used = used_cpus.contains(&ResourceIndex::new(position as u32));
119+
(cpu_util, position, used)
120+
})
121+
.collect();
122+
123+
all_cpus.sort_by_key(|&(_, _, used)| std::cmp::Reverse(used));
124+
125+
let mut rows: Vec<Vec<(f64, usize, bool)>> = vec![vec![]; height];
126+
for (index, cpu_data) in all_cpus.into_iter().enumerate() {
127+
let row = index % height;
128+
rows[row].push(cpu_data);
129+
}
130+
rows
131+
}
132+
133+
fn create_title_info(
134+
cpu_util_list: &[f64],
135+
used_cpus: &[ResourceIndex],
136+
util_render_mode: &UtilizationRenderMode,
137+
) -> (String, usize, f64) {
138+
let which_util = match util_render_mode {
139+
UtilizationRenderMode::Global => "Node".to_string(),
140+
UtilizationRenderMode::Worker => "Worker".to_string(),
141+
};
142+
143+
let num_cpus = match util_render_mode {
144+
UtilizationRenderMode::Global => cpu_util_list.len(),
145+
UtilizationRenderMode::Worker => used_cpus.len(),
146+
};
147+
148+
let avg_usage = match util_render_mode {
149+
UtilizationRenderMode::Global => calculate_average(cpu_util_list),
150+
UtilizationRenderMode::Worker => {
151+
let used_cpu_util_list: Vec<f64> = cpu_util_list
152+
.iter()
153+
.enumerate()
154+
.filter_map(|(idx, utilization)| {
155+
if used_cpus.contains(&ResourceIndex::new(idx as u32)) {
156+
Some(*utilization)
157+
} else {
158+
None
159+
}
160+
})
161+
.collect();
162+
calculate_average(&used_cpu_util_list)
163+
}
164+
};
165+
166+
(which_util, num_cpus, avg_usage)
167+
}

‎crates/hyperqueue/src/dashboard/ui/screens/cluster/worker/mod.rs‎

Lines changed: 76 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ use crate::dashboard::ui::widgets::text::draw_text;
1212
use crossterm::event::{KeyCode, KeyEvent};
1313
use ratatui::layout::{Constraint, Direction, Layout, Rect};
1414
use tako::hwstats::MemoryStats;
15+
use tako::resources::{CPU_RESOURCE_NAME, ResourceIndex};
1516
use tako::{JobTaskId, WorkerId};
1617

1718
mod cpu_util_table;
@@ -26,6 +27,7 @@ pub struct WorkerDetail {
2627
worker_tasks_table: TasksTable,
2728

2829
utilization: Option<Utilization>,
30+
utilization_render_mode: UtilizationRenderMode,
2931
}
3032

3133
impl Default for WorkerDetail {
@@ -36,13 +38,37 @@ impl Default for WorkerDetail {
3638
worker_config_table: Default::default(),
3739
worker_tasks_table: TasksTable::non_interactive(),
3840
utilization: None,
41+
utilization_render_mode: UtilizationRenderMode::Worker,
42+
}
43+
}
44+
}
45+
46+
#[derive(PartialEq)]
47+
enum UtilizationRenderMode {
48+
Global,
49+
Worker,
50+
}
51+
52+
impl UtilizationRenderMode {
53+
fn next(&mut self) {
54+
*self = match self {
55+
UtilizationRenderMode::Global => UtilizationRenderMode::Worker,
56+
UtilizationRenderMode::Worker => UtilizationRenderMode::Global,
57+
}
58+
}
59+
60+
fn next_text(&self) -> &str {
61+
match self {
62+
UtilizationRenderMode::Global => "Show worker CPU utilization",
63+
UtilizationRenderMode::Worker => "Show global CPU utilization",
3964
}
4065
}
4166
}
4267

4368
struct Utilization {
4469
cpu: Vec<f64>,
4570
memory: MemoryStats,
71+
used_cpus: Vec<ResourceIndex>,
4672
}
4773

4874
impl WorkerDetail {
@@ -66,12 +92,24 @@ impl WorkerDetail {
6692
frame,
6793
style_header_text(),
6894
);
69-
draw_text("<backspace>: Back", layout.footer, frame, style_footer());
95+
96+
draw_text(
97+
format!(
98+
"<backspace>: Back, <c>: {}",
99+
self.utilization_render_mode.next_text()
100+
)
101+
.as_str(),
102+
layout.footer,
103+
frame,
104+
style_footer(),
105+
);
70106

71107
if let Some(util) = &self.utilization {
72108
render_cpu_util_table(
73109
&util.cpu,
74110
&util.memory,
111+
&util.used_cpus,
112+
&self.utilization_render_mode,
75113
layout.current_utilization,
76114
frame,
77115
table_style_deselected(),
@@ -95,21 +133,45 @@ impl WorkerDetail {
95133
if let Some(worker_id) = self.worker_id {
96134
self.utilization_history.update(data, worker_id);
97135

98-
if let Some((cpu_util, mem_util)) = data
136+
if let Some(overview) = data
99137
.workers()
100138
.query_worker_overview_at(worker_id, data.current_time())
101-
.and_then(|overview| overview.item.hw_state.as_ref())
102-
.map(|hw_state| {
103-
(
104-
&hw_state.state.cpu_usage.cpu_per_core_percent_usage,
105-
&hw_state.state.memory_usage,
106-
)
107-
})
108139
{
109-
self.utilization = Some(Utilization {
110-
cpu: cpu_util.iter().map(|&v| v as f64).collect(),
111-
memory: mem_util.clone(),
112-
});
140+
let worker_used_cpus: Vec<ResourceIndex> = match self.utilization_render_mode {
141+
UtilizationRenderMode::Worker => overview
142+
.item
143+
.running_tasks
144+
.iter()
145+
.flat_map(|(_id, task_resource_alloc)| {
146+
task_resource_alloc
147+
.resources
148+
.iter()
149+
.filter_map(|resource_alloc| {
150+
if resource_alloc.resource == CPU_RESOURCE_NAME {
151+
Some(resource_alloc.indices.iter().map(|(index, _)| *index))
152+
} else {
153+
None
154+
}
155+
})
156+
})
157+
.flatten()
158+
.collect(),
159+
UtilizationRenderMode::Global => vec![],
160+
};
161+
162+
if let Some(hw_state) = overview.item.hw_state.as_ref() {
163+
self.utilization = Some(Utilization {
164+
cpu: hw_state
165+
.state
166+
.cpu_usage
167+
.cpu_per_core_percent_usage
168+
.iter()
169+
.map(|&v| v as f64)
170+
.collect(),
171+
memory: hw_state.state.memory_usage.clone(),
172+
used_cpus: worker_used_cpus,
173+
})
174+
}
113175
}
114176

115177
let tasks_info: Vec<(JobTaskId, &TaskInfo)> =
@@ -127,6 +189,7 @@ impl WorkerDetail {
127189
pub fn handle_key(&mut self, key: KeyEvent) {
128190
match key.code {
129191
KeyCode::Backspace => self.worker_tasks_table.clear_selection(),
192+
KeyCode::Char('c') => self.utilization_render_mode.next(),
130193
_ => self.worker_tasks_table.handle_key(key),
131194
}
132195
}

‎crates/hyperqueue/src/dashboard/ui/widgets/progressbar.rs‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,25 @@ pub fn get_progress_bar_color(progress: f64) -> Style {
3030
}
3131
}
3232

33+
pub fn get_progress_bar_cpu_color(progress: f64, used: bool) -> Style {
34+
let color = if !used {
35+
Color::Gray
36+
} else if progress <= GREEN_THRESHOLD {
37+
Color::Green
38+
} else if progress <= YELLOW_THRESHOLD {
39+
Color::Yellow
40+
} else {
41+
Color::Red
42+
};
43+
44+
Style {
45+
fg: Some(color),
46+
bg: None,
47+
add_modifier: Modifier::empty(),
48+
sub_modifier: Modifier::empty(),
49+
}
50+
}
51+
3352
/**
3453
* Creates a string progress bar for 0 < progress < 1
3554
*/

0 commit comments

Comments
 (0)