forked from It4innovations/hyperqueue
-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathmain.py
More file actions
120 lines (100 loc) · 3.98 KB
/
main.py
File metadata and controls
120 lines (100 loc) · 3.98 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
import itertools
import logging
import multiprocessing
from pathlib import Path
from typing import List, Optional
import dataclasses
import typer
from src.benchmark.identifier import BenchmarkDescriptor
from src.benchmark_defs import create_basic_hq_benchmarks, get_hq_binary, single_node_hq_cluster
from src.build.hq import BuildConfig, iterate_binaries
from src.build.repository import TAG_WORKSPACE
from src.clusterutils import ClusterInfo
from src.clusterutils.node_list import Local
from src.environment import EnvironmentDescriptor
from src.environment.dask import DaskClusterInfo, DaskWorkerConfig
from src.environment.hq import HqClusterInfo, HqWorkerConfig
from src.environment.snake import SnakeEnvironmentDescriptor
from src.utils.benchmark import run_benchmarks_with_postprocessing
from src.workloads import Workload
from src.workloads.sleep import SleepHQ, SleepSnake, SleepDask
app = typer.Typer()
@app.command()
def compare_hq_version(baseline: str, modified: Optional[str] = None, zero_worker: Optional[bool] = False):
"""
Compares the performance of two HQ versions.
If `modified` is not set, the current git workspace version will be used.
"""
modified = modified if modified else TAG_WORKSPACE
configs = [
BuildConfig(git_ref=modified),
BuildConfig(git_ref=baseline),
]
if zero_worker:
configs += [dataclasses.replace(config, zero_worker=True) for config in configs]
artifacts = list(iterate_binaries(configs))
identifiers = create_basic_hq_benchmarks(artifacts)
workdir = Path(f"benchmark/cmp-{baseline}-{modified}")
run_benchmarks_with_postprocessing(workdir, identifiers)
@app.command()
def compare_zw():
"""
Compares the performance of HQ vs zero-worker HQ.
"""
artifacts = list(iterate_binaries([BuildConfig(), BuildConfig(zero_worker=True)]))
identifiers = create_basic_hq_benchmarks(artifacts)
workdir = Path("benchmark/zw")
run_benchmarks_with_postprocessing(workdir, identifiers)
@app.command()
def sleep():
"""
Compare the sleep benchmarks between various tools.
"""
hq_path = list(iterate_binaries([BuildConfig()]))[0].binary_path
workdir = Path("benchmark/sleep")
task_counts = (10, 100, 1000)
descriptions = []
def add_product(workloads: List[Workload], environments: List[EnvironmentDescriptor]):
for env, workload in itertools.product(environments, workloads):
descriptions.append(BenchmarkDescriptor(env_descriptor=env, workload=workload))
add_product([SleepSnake(tc) for tc in task_counts], [SnakeEnvironmentDescriptor()])
add_product(
[SleepDask(tc) for tc in task_counts],
[
DaskClusterInfo(
cluster_info=ClusterInfo(monitor_nodes=True, node_list=Local()),
workers=[DaskWorkerConfig(cores=multiprocessing.cpu_count())],
)
],
)
add_product(
[SleepHQ(tc) for tc in task_counts],
[
HqClusterInfo(
cluster=ClusterInfo(monitor_nodes=True, node_list=Local()),
environment_params=dict(worker_count=1),
workers=[HqWorkerConfig()],
binary=hq_path,
)
],
)
run_benchmarks_with_postprocessing(workdir, descriptions)
def profile():
from src.clusterutils.profiler import SamplyProfiler
hq_path = get_hq_binary(debug_symbols=True)
hq_env = dataclasses.replace(
single_node_hq_cluster(hq_path, worker_threads=16), worker_profilers=[SamplyProfiler(100)]
)
sleep_time = 250
task_count = 50000
workload = SleepHQ(task_count=task_count, sleep_duration=sleep_time / task_count)
run_benchmarks_with_postprocessing(
Path("benchmarks"), [BenchmarkDescriptor(env_descriptor=hq_env, workload=workload)]
)
if __name__ == "__main__":
logging.basicConfig(
level=logging.INFO,
format="%(levelname)s:%(asctime)s.%(msecs)03d:%(funcName)s: %(message)s",
datefmt="%Y-%m-%d %H:%M:%S",
)
app()