forked from It4innovations/hyperqueue
-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathmain.py
More file actions
105 lines (89 loc) · 3.24 KB
/
main.py
File metadata and controls
105 lines (89 loc) · 3.24 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
import dataclasses
import itertools
import logging
from pathlib import Path
from typing import List, Optional
import typer
from src.benchmark.identifier import BenchmarkDescriptor
from src.benchmark_defs import create_basic_hq_benchmarks
from src.build.hq import BuildConfig, iterate_binaries
from src.build.repository import TAG_WORKSPACE
from src.clusterutils import ClusterInfo
from src.clusterutils.node_list import Local
# from src.clusterutils.profiler import (
# PerfEventsProfiler,
# FlamegraphProfiler,
# CachegrindProfiler,
# )
from src.environment import EnvironmentDescriptor
from src.environment.hq import HqClusterInfo, HqWorkerConfig
from src.environment.snake import SnakeEnvironmentDescriptor
from src.utils.benchmark import run_benchmarks_with_postprocessing
from src.workloads import Workload
from src.workloads.sleep import SleepHQ, SleepSnake
app = typer.Typer()
@app.command()
def compare_hq_version(
baseline: str, modified: Optional[str] = None, zero_worker: Optional[bool] = False
):
"""
Compares the performance of two HQ versions.
If `modified` is not set, the current git workspace version will be used.
"""
modified = modified if modified else TAG_WORKSPACE
configs = [
BuildConfig(git_ref=modified),
BuildConfig(git_ref=baseline),
]
if zero_worker:
configs += [dataclasses.replace(config, zero_worker=True) for config in configs]
artifacts = list(iterate_binaries(configs))
identifiers = create_basic_hq_benchmarks(artifacts)
workdir = Path(f"benchmark/cmp-{baseline}-{modified}")
run_benchmarks_with_postprocessing(workdir, identifiers)
@app.command()
def compare_zw():
"""
Compares the performance of HQ vs zero-worker HQ.
"""
artifacts = list(iterate_binaries([BuildConfig(), BuildConfig(zero_worker=True)]))
identifiers = create_basic_hq_benchmarks(artifacts)
workdir = Path("benchmark/zw")
run_benchmarks_with_postprocessing(workdir, identifiers)
@app.command()
def sleep():
"""
Compare the sleep benchmarks between various tools.
"""
hq_path = list(iterate_binaries([BuildConfig()]))[0].binary_path
workdir = Path("benchmark/sleep")
task_counts = (10, 100, 1000)
descriptions = []
def add_product(
workloads: List[Workload], environments: List[EnvironmentDescriptor]
):
for env, workload in itertools.product(environments, workloads):
descriptions.append(
BenchmarkDescriptor(env_descriptor=env, workload=workload)
)
add_product([SleepSnake(tc) for tc in task_counts], [SnakeEnvironmentDescriptor()])
add_product(
[SleepHQ(tc) for tc in task_counts],
[
HqClusterInfo(
cluster=ClusterInfo(monitor_nodes=True, node_list=Local()),
environment_params=dict(worker_count=1),
workers=[HqWorkerConfig()],
binary=hq_path,
worker_profilers=[],
)
],
)
run_benchmarks_with_postprocessing(workdir, descriptions)
if __name__ == "__main__":
logging.basicConfig(
level=logging.INFO,
format="%(levelname)s:%(asctime)s.%(msecs)03d:%(funcName)s: %(message)s",
datefmt="%Y-%m-%d %H:%M:%S",
)
app()