From 19c6cd76430cde4eb57cdfc1b6376a8c02faf815 Mon Sep 17 00:00:00 2001 From: lilydjwg Date: Sat, 27 Sep 2025 10:55:00 +0800 Subject: [PATCH 01/46] make python -m lilac2.tools to show resource usage --- lilac2/tools.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lilac2/tools.py b/lilac2/tools.py index f2c9b479..51b5335f 100644 --- a/lilac2/tools.py +++ b/lilac2/tools.py @@ -49,3 +49,8 @@ def get_avail_memory() -> int: if l.startswith('MemAvailable:'): return int(l.split()[1]) * 1024 return 10 * 1024 ** 3 + +if __name__ == '__main__': + cpu = get_running_task_cpu_ratio() + mem = get_avail_memory() + print(cpu, mem) From 0c8a1805a5a70f59def2df1639e8c03dc6247b6f Mon Sep 17 00:00:00 2001 From: lilydjwg Date: Sat, 27 Sep 2025 11:22:07 +0800 Subject: [PATCH 02/46] starting introducing remoteworker --- config.toml.sample | 7 +++++++ lilac | 35 +++++++++++++++++++++++++------- lilac2/building.py | 9 +++++---- lilac2/workerman.py | 49 +++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 89 insertions(+), 11 deletions(-) create mode 100644 lilac2/workerman.py diff --git a/config.toml.sample b/config.toml.sample index c0c28de8..95ff021b 100644 --- a/config.toml.sample +++ b/config.toml.sample @@ -41,6 +41,13 @@ logurl = "https://example.com/${pkgbase}/${datetime}.html" # schema = "lilac" max_concurrency = 1 +# build packages over ssh +[[remoteworker]] +host = "builder.example.org" +# the same repodir but on remote host +repodir = "/path/to/gitrepo" +max_concurrency = 2 + [nvchecker] # set proxy for nvchecker # proxy = "http://localhost:8000" diff --git a/lilac b/lilac index 85bc27bc..a1f71214 100755 --- a/lilac +++ b/lilac @@ -49,6 +49,7 @@ from lilac2 import pkgbuild from lilac2.building import build_package, MissingDependencies from lilac2 import slogconf from lilac2 import intl +from lilac2 import workerman from lilac2.typing import PkgToBuild try: from lilac2 import db @@ -57,6 +58,7 @@ except ImportError: USE = False config = tools.read_config() +TLS = threading.local() # Setting up environment variables os.environ.update(config.get('envvars', ())) @@ -246,24 +248,28 @@ def start_build( logdir: Path, failed: dict[str, tuple[str, ...]], built: set[str], - max_concurrency: int, + workermans: list[workerman.WorkerManager], ) -> None: # built is used to collect built package names sorter, depmap = packages_with_depends(repo) + max_workers = sum(wm.max_concurrency for wm in workermans) try: buildsorter = BuildSorter(sorter, depmap) futures: dict[Future, str] = {} with ThreadPoolExecutor( - max_workers = max_concurrency, + max_workers = max_workers, initializer = setup_thread, ) as executor: while True: + # TODO: use other workermans + workers_before_wm = 0 + wm = workermans[0] pkgs = try_pick_some( repo, buildsorter, failed, running = frozenset(futures.values()), - limit = max_concurrency - len(futures), + limit = wm.max_concurrency - len(futures), starving = not bool(futures), ) for pkg in pkgs: @@ -274,7 +280,9 @@ def start_build( buildsorter.done(pkg.pkgbase) continue fu = executor.submit( - build_it, pkg, repo, buildsorter, built, failed) + build_it, pkg, repo, buildsorter, built, failed, + wm, workers_before_wm + ) futures[fu] = pkg.pkgbase if not pkgs and not futures: @@ -464,10 +472,13 @@ def check_buildability( def build_it( to_build: PkgToBuild, repo: Repo, buildsorter: BuildSorter, built: set[str], failed: dict[str, tuple[str, ...]], + wm: workerman.WorkerManager, + workers_before_wm: int, ) -> None: pkg = to_build.pkgbase logger.info('building %s', pkg) logfile = logdir / f'{pkg}.log' + worker_no = TLS.worker_no - workers_before_wm if db.USE: with db.get_session() as s: @@ -485,6 +496,7 @@ def build_it( else: commit_msg_template.append('unknown reasons?!') + # TODO: use wm r, version = build_package( to_build, repo.lilacinfos[pkg], update_info = nvdata[pkg], @@ -496,6 +508,7 @@ def build_it( myname = MYNAME, destdir = DESTDIR, logfile = logfile, + worker_no = worker_no, ) elapsed = r.elapsed @@ -613,7 +626,6 @@ WORKER_NO_LOCK = threading.Lock() def setup_thread() -> None: global WORKER_NO - from lilac2.building import TLS with WORKER_NO_LOCK: TLS.worker_no = WORKER_NO WORKER_NO += 1 @@ -628,6 +640,15 @@ def build_nvchecker_reason( changes.append((ver_change.oldver, ver_change.newver)) return BuildReason.NvChecker(items, changes) +def get_workermans(): + max_concurrency = config['lilac'].get('max_concurrency', 1) + local = workerman.LocalWorkerManager(max_concurrency) + remotes = [ + workerman.RemoteWorkerManager(remote) + for remote in config.get('remoteworker', []) + ] + return [local] + remotes + def main_may_raise( D: dict[str, Any], pkgs_from_args: List[str], logdir: Path, ) -> None: @@ -763,6 +784,7 @@ def main_may_raise( build_reasons[p].append(BuildReason.OnBuild(update_on_build)) update_succeeded: set[str] = set() + workermans = get_workermans() try: build_logger.info('build start') @@ -772,8 +794,7 @@ def main_may_raise( s.execute('insert into batch (event, logdir) values (%s, %s)', ('start', logdir_name)) db.build_updated(s) - start_build(REPO, logdir, failed, update_succeeded, - config['lilac'].get('max_concurrency', 1)) + start_build(REPO, logdir, failed, update_succeeded, workermans) finally: D['last_commit'] = git_last_commit() # handle what has been processed even on exception diff --git a/lilac2/building.py b/lilac2/building.py index 02c07b5d..d99cc0c1 100644 --- a/lilac2/building.py +++ b/lilac2/building.py @@ -11,7 +11,6 @@ from pathlib import Path import time import json -import threading import signal from contextlib import suppress @@ -29,7 +28,6 @@ del Repo logger = logging.getLogger(__name__) -TLS = threading.local() class MissingDependencies(Exception): def __init__(self, pkgs: Set[str]) -> None: @@ -55,6 +53,7 @@ def build_package( myname: str, destdir: Path, logfile: Path, + worker_no: int, ) -> tuple[BuildResult, Optional[str]]: '''return BuildResult and version string if successful''' start_time = time.time() @@ -82,6 +81,7 @@ def build_package( logfile = logfile, deadline = start_time + time_limit_hours * 3600, packager = packager, + worker_no = worker_no, ) if error: raise error @@ -187,6 +187,7 @@ def call_worker( tmpfs: list[str], deadline: float, packager: str, + worker_no: int, ) -> tuple[Optional[str], RUsage, Optional[Exception]]: ''' return: package version, resource usage, error information @@ -199,7 +200,7 @@ def call_worker( 'bindmounts': bindmounts, 'tmpfs': tmpfs, 'logfile': str(logfile), # for sending error reports - 'worker_no': TLS.worker_no, + 'worker_no': worker_no, } fd, resultpath = tempfile.mkstemp(prefix=f'{pkgbase}-', suffix='.lilac') os.close(fd) @@ -217,7 +218,7 @@ def call_worker( _call_cmd = _call_cmd_systemd else: _call_cmd = _call_cmd_subprocess - name = f'lilac-worker-{TLS.worker_no}' + name = f'lilac-worker-{worker_no}' rusage, timedout = _call_cmd( name, cmd, logfile, pkgdir, deadline, input_bytes, packager, diff --git a/lilac2/workerman.py b/lilac2/workerman.py new file mode 100644 index 00000000..f46087bb --- /dev/null +++ b/lilac2/workerman.py @@ -0,0 +1,49 @@ +from pathlib import Path +from typing import override + +class WorkerManager: + max_concurrency: int + + def prepare(self) -> None: + raise NotImplementedError + + def sync_depended_packages(self, depends): + raise NotImplementedError + + def build_package(self): + raise NotImplementedError + +class LocalWorkerManager(WorkerManager): + max_concurrency: int + def __init__(self, max_concurrency) -> None: + self.max_concurrency = max_concurrency + + @override + def prepare(self): + # git pull + ... + + @override + def sync_depended_packages(self, depends): + ... + + @override + def build_package(self): + ... + +class RemoteWorkerManager(WorkerManager): + max_concurrency: int + repodir: Path + + @override + def prepare(self): + # git pull + ... + + @override + def sync_depended_packages(self, depends): + ... + + @override + def build_package(self): + ... From c5ffd212401b1c2f8c4a2127b7e0302910181aef Mon Sep 17 00:00:00 2001 From: lilydjwg Date: Sat, 27 Sep 2025 11:31:49 +0800 Subject: [PATCH 03/46] record builder in pkglog --- README.md | 8 ++++++++ config.toml.sample | 1 + lilac | 6 +++--- lilac2/workerman.py | 4 ++++ scripts/dbsetup.sql | 3 ++- scripts/tailf-build-log | 3 ++- scripts/useful.sql | 2 +- 7 files changed, 21 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 7551ddf4..27e34571 100644 --- a/README.md +++ b/README.md @@ -25,6 +25,14 @@ Docs Update ---- +### 2025-09-27 +if database is in use, run the following SQL to update: + +```sql + +alter table lilac.pkglog add column builder text not null default 'local'; +``` + ### 2024-06-28 if database is in use, run the following SQL to update: diff --git a/config.toml.sample b/config.toml.sample index 95ff021b..0a5a2e2c 100644 --- a/config.toml.sample +++ b/config.toml.sample @@ -43,6 +43,7 @@ max_concurrency = 1 # build packages over ssh [[remoteworker]] +name = "remotebuilder" host = "builder.example.org" # the same repodir but on remote host repodir = "/path/to/gitrepo" diff --git a/lilac b/lilac index a1f71214..a111135e 100755 --- a/lilac +++ b/lilac @@ -607,10 +607,10 @@ def build_it( s.execute( '''insert into pkglog (pkgbase, nv_version, pkg_version, elapsed, result, cputime, memory, - msg, build_reasons, maintainers) values - (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s)''', + msg, build_reasons, maintainers, builder) values + (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)''', (pkg, newver, version, elapsed, r.__class__.__name__, cputime, memory, - msg, reason_s, maintainers)) + msg, reason_s, maintainers, wm.name)) db.mark_pkg_as(s, pkg, 'done') db.build_updated(s) diff --git a/lilac2/workerman.py b/lilac2/workerman.py index f46087bb..ef1f3068 100644 --- a/lilac2/workerman.py +++ b/lilac2/workerman.py @@ -2,6 +2,7 @@ from typing import override class WorkerManager: + name: str max_concurrency: int def prepare(self) -> None: @@ -14,7 +15,9 @@ def build_package(self): raise NotImplementedError class LocalWorkerManager(WorkerManager): + name: str = 'local' max_concurrency: int + def __init__(self, max_concurrency) -> None: self.max_concurrency = max_concurrency @@ -32,6 +35,7 @@ def build_package(self): ... class RemoteWorkerManager(WorkerManager): + name: str max_concurrency: int repodir: Path diff --git a/scripts/dbsetup.sql b/scripts/dbsetup.sql index ad8a7da2..83871092 100644 --- a/scripts/dbsetup.sql +++ b/scripts/dbsetup.sql @@ -15,7 +15,8 @@ create table pkglog ( memory bigint, msg text, build_reasons jsonb, - maintainers jsonb + maintainers jsonb, + builder text not null ); create index pkglog_ts_idx on pkglog (ts); diff --git a/scripts/tailf-build-log b/scripts/tailf-build-log index 039a518e..6ca0e4e1 100755 --- a/scripts/tailf-build-log +++ b/scripts/tailf-build-log @@ -18,7 +18,7 @@ FMT = { 'staged': f'[{c(12)}%(ts)s{c(7)}] {c(15)}%(pkgbase)s{c(7)} %(nv_version)s %(action)s{c(7)} as {c(15)}%(pkg_version)s{c(7)} in {c(6)}%(elapsed)s', 'failed': f'[{c(12)}%(ts)s{c(7)}] {c(15)}%(pkgbase)s{c(7)} %(nv_version)s %(action)s{c(7)} to build as {c(15)}%(pkg_version)s{c(7)} in {c(6)}%(elapsed)s', 'skipped': f'[{c(12)}%(ts)s{c(7)}] {c(15)}%(pkgbase)s{c(7)} %(nv_version)s %(action)s{c(7)} because {c(15)}%(msg)s', - '_rusage': f'{c(7)}; CPU time: {c(6)}%(cputime)s{c(7)} (%(cpupercent)s%%{c(7)}), Memory: {c(5)}%(memory)s\n', + '_rusage': f'{c(7)}; CPU time: {c(6)}%(cputime)s{c(7)} (%(cpupercent)s%%{c(7)}), Memory: {c(5)}%(memory)s on %(builder)s\n', '_batch': f'[{c(12)}%(ts)s{c(7)}] {c(14)}build %(event)s\n', } @@ -80,6 +80,7 @@ def pretty_print(log): 'cputime': humantime(cputime), 'cpupercent': cpupercent, 'memory': filesize(memory), + 'builder': log['builder'], } fmt = FMT[result] diff --git a/scripts/useful.sql b/scripts/useful.sql index 83f1268c..6be7f869 100644 --- a/scripts/useful.sql +++ b/scripts/useful.sql @@ -1,7 +1,7 @@ -- some useful SQL commands (for PostgreSQL) -- show build log -select id, ts, pkgbase, nv_version, pkg_version, elapsed, result, cputime, case when elapsed = 0 then 0 else cputime * 100 / elapsed end as "cpu%", round(memory / 1073741824.0, 3) as "memory (GiB)", substring(msg for 20) as msg, build_reasons, (select array_agg(github) from jsonb_to_recordset(maintainers) as m(github text)) as maintainers from pkglog order by id desc limit 10; +select id, ts, pkgbase, nv_version, pkg_version, elapsed, result, cputime, case when elapsed = 0 then 0 else cputime * 100 / elapsed end as "cpu%", round(memory / 1073741824.0, 3) as "memory (GiB)", substring(msg for 20) as msg, build_reasons, (select array_agg(github) from jsonb_to_recordset(maintainers) as m(github text)) as maintainers, builder from pkglog order by id desc limit 10; -- show current build status and expected time select index, c.pkgbase, updated_at, status, elapsed as last_time, c.build_reasons from pkgcurrent as c left join lateral ( From e2a672ab2a914d91813ad6cf0385a68c95821e3a Mon Sep 17 00:00:00 2001 From: lilydjwg Date: Sat, 27 Sep 2025 16:34:33 +0800 Subject: [PATCH 04/46] distribute build tasks to WorkerManagers --- lilac | 135 +++++++++++++++++++------------------------- lilac2/building.py | 1 + lilac2/db.py | 17 +++--- lilac2/typing.py | 29 +++++++++- lilac2/workerman.py | 90 ++++++++++++++++++++++++++++- 5 files changed, 184 insertions(+), 88 deletions(-) diff --git a/lilac b/lilac index a111135e..67e5f995 100755 --- a/lilac +++ b/lilac @@ -8,7 +8,7 @@ import traceback import logging import time from collections import defaultdict -from typing import List, Any, DefaultDict, Tuple, Optional +from typing import List, Any, DefaultDict, Tuple, Optional, cast from collections.abc import Set from pathlib import Path import graphlib @@ -49,8 +49,8 @@ from lilac2 import pkgbuild from lilac2.building import build_package, MissingDependencies from lilac2 import slogconf from lilac2 import intl -from lilac2 import workerman -from lilac2.typing import PkgToBuild +from lilac2.workerman import WorkerManager +from lilac2.typing import PkgToBuild, Rusages try: from lilac2 import db except ImportError: @@ -248,7 +248,7 @@ def start_build( logdir: Path, failed: dict[str, tuple[str, ...]], built: set[str], - workermans: list[workerman.WorkerManager], + workermans: list[WorkerManager], ) -> None: # built is used to collect built package names sorter, depmap = packages_with_depends(repo) @@ -256,21 +256,18 @@ def start_build( max_workers = sum(wm.max_concurrency for wm in workermans) try: buildsorter = BuildSorter(sorter, depmap) - futures: dict[Future, str] = {} + futures: dict[Future, PkgToBuild] = {} with ThreadPoolExecutor( max_workers = max_workers, initializer = setup_thread, ) as executor: while True: - # TODO: use other workermans - workers_before_wm = 0 - wm = workermans[0] pkgs = try_pick_some( repo, buildsorter, failed, - running = frozenset(futures.values()), - limit = wm.max_concurrency - len(futures), + running = frozenset(x.pkgbase for x in futures.values()), starving = not bool(futures), + workermans = workermans, ) for pkg in pkgs: if pkg.pkgbase not in nvdata: @@ -278,12 +275,12 @@ def start_build( # a package is pulled in by OnBuild logger.warning('%s not in nvdata, skipping', pkg.pkgbase) buildsorter.done(pkg.pkgbase) + wm = cast(WorkerManager, pkg.workerman) + wm.current_task_count -= 1 continue fu = executor.submit( - build_it, pkg, repo, buildsorter, built, failed, - wm, workers_before_wm - ) - futures[fu] = pkg.pkgbase + build_it, pkg, repo, buildsorter, built, failed) + futures[fu] = pkg if not pkgs and not futures: # no more packages and no task is running: we're done @@ -291,7 +288,9 @@ def start_build( done, pending = futures_wait(futures, return_when=FIRST_COMPLETED) for fu in done: - del futures[fu] + pkg = futures.pop(fu) + wm = cast(WorkerManager, pkg.workerman) + wm.current_task_count -= 1 fu.result() # at least one task is done, try pick new tasks @@ -304,8 +303,8 @@ def try_pick_some( buildsorter: BuildSorter, failed: dict[str, tuple[str, ...]], running: Set[str], - limit: int, starving: bool, + workermans: list[WorkerManager], ) -> list[PkgToBuild]: if not buildsorter.is_active(): return [] @@ -314,9 +313,6 @@ def try_pick_some( if not ready: return [] - cpu_ratio = tools.get_running_task_cpu_ratio() - memory_avail = tools.get_avail_memory() - ready_to_build = [pkg for pkg in ready if pkg not in running] if not ready_to_build: return [] @@ -324,65 +320,41 @@ def try_pick_some( if db.USE: rusages = db.get_pkgs_last_rusage(ready_to_build) else: - rusages = {} - - def sort_key(pkg): - p = buildsorter.priority_func(pkg) - cpu = (r := rusages.get(pkg)) and (r.cputime / r.elapsed) or 1.0 - return (p, cpu) - ready_to_build.sort(key=sort_key) - logger.debug('sorted ready_to_build: %r', ready_to_build) - if cpu_ratio < 1.0: - # low cpu usage, build a big package - p = buildsorter.priority_func(ready_to_build[0]) - for idx, pkg in enumerate(ready_to_build): - if buildsorter.priority_func(pkg) != p: - if idx > 2: - ready_to_build.insert(0, ready_to_build.pop(idx-1)) - break - else: - logger.info('high cpu usage (%.2f), preferring low-cpu-usage builds', cpu_ratio) + rusages = Rusages({}) ret: list[PkgToBuild] = [] - limited_by_memory = False - for pkg in ready_to_build: - if (r := rusages.get(pkg)) and r.memory > memory_avail: - logger.debug('package %s used %d memory last time, but now only %d is available', pkg, r.memory, memory_avail) - limited_by_memory = True - continue - - to_build = check_buildability(pkg, repo, buildsorter, failed) - if to_build is None: - continue - - ret.append(to_build) - if len(ret) == limit: + for wm in workermans: + to_builds = wm.try_accept_package( + ready_to_build, + rusages, + buildsorter.priority_func, + lambda pkg: check_buildability(pkg,repo, buildsorter, failed), + ) + ret.extend(to_builds) + + if not ret and starving: + wm = workermans[0] + def sort_key(pkg): + p = buildsorter.priority_func(pkg) + r = rusages.for_package(pkg, [wm.name]) + if r is not None: + m = r.memory + else: + m = 10 * 1024**3 + return (p, m) + ready_to_build.sort(key=sort_key) + logger.debug('sorted ready_to_build: %r', ready_to_build) + memory_avail = wm.get_resource_usage()[1] + logger.info('insufficient memory, starting only one build on %s (available: %d)', wm.name, memory_avail) + for pkg in ready_to_build: + to_build = check_buildability(pkg, repo, buildsorter, failed) + if to_build is None: + continue + to_build.workerman = wm + ret.append(to_build) break - if r := rusages.get(pkg): - memory_avail -= r.memory - else: - memory_avail -= 10 * 1024 ** 3 - - if not ret and limited_by_memory: - if starving: - def sort_key(pkg): - p = buildsorter.priority_func(pkg) - r = (r := rusages.get(pkg)) and r.memory or 10 * 1024**3 - return (p, r) - ready_to_build.sort(key=sort_key) - logger.debug('sorted ready_to_build: %r', ready_to_build) - logger.info('insufficient memory, starting only one build (available: %d)', memory_avail) - for pkg in ready_to_build: - to_build = check_buildability(pkg, repo, buildsorter, failed) - if to_build is None: - continue - ret.append(to_build) - break - else: - logger.info('insufficient memory, not starting another concurrent build (available: %d)', memory_avail) - return ret def check_buildability( @@ -391,6 +363,7 @@ def check_buildability( buildsorter: BuildSorter, failed: dict[str, tuple[str, ...]], ) -> Optional[PkgToBuild]: + '''NOTE: caller needs to set workerman on returned value''' to_build = PkgToBuild(pkg) if pkg in failed: @@ -472,13 +445,12 @@ def check_buildability( def build_it( to_build: PkgToBuild, repo: Repo, buildsorter: BuildSorter, built: set[str], failed: dict[str, tuple[str, ...]], - wm: workerman.WorkerManager, - workers_before_wm: int, ) -> None: pkg = to_build.pkgbase logger.info('building %s', pkg) logfile = logdir / f'{pkg}.log' - worker_no = TLS.worker_no - workers_before_wm + wm = cast(WorkerManager, to_build.workerman) + worker_no = TLS.worker_no - wm.workers_before_me if db.USE: with db.get_session() as s: @@ -496,7 +468,6 @@ def build_it( else: commit_msg_template.append('unknown reasons?!') - # TODO: use wm r, version = build_package( to_build, repo.lilacinfos[pkg], update_info = nvdata[pkg], @@ -641,13 +612,21 @@ def build_nvchecker_reason( return BuildReason.NvChecker(items, changes) def get_workermans(): + from lilac2 import workerman + max_concurrency = config['lilac'].get('max_concurrency', 1) local = workerman.LocalWorkerManager(max_concurrency) remotes = [ workerman.RemoteWorkerManager(remote) for remote in config.get('remoteworker', []) ] - return [local] + remotes + ret = [local] + remotes + + workers_before = max_concurrency + for remote in remotes: + remote.workers_before_me = workers_before + workers_before += remote.max_concurrency + return ret def main_may_raise( D: dict[str, Any], pkgs_from_args: List[str], logdir: Path, diff --git a/lilac2/building.py b/lilac2/building.py index d99cc0c1..7fd3353f 100644 --- a/lilac2/building.py +++ b/lilac2/building.py @@ -69,6 +69,7 @@ def build_package( depend_packages = resolve_depends(repo, depends) pkgdir = repo.repodir / pkgbase try: + # TODO: use to_build.wm pkg_version, rusage, error = call_worker( pkgbase = pkgbase, pkgdir = pkgdir, diff --git a/lilac2/db.py b/lilac2/db.py index 5f3b7827..b87e0442 100644 --- a/lilac2/db.py +++ b/lilac2/db.py @@ -3,11 +3,12 @@ import re import logging from functools import partial +from itertools import groupby import psycopg2 import psycopg2.pool -from .typing import UsedResource, OnBuildEntry, OnBuildVers +from .typing import UsedResource, OnBuildEntry, OnBuildVers, Rusages logger = logging.getLogger(__name__) @@ -67,21 +68,23 @@ def get_pkgs_last_success_times(pkgs: list[str]) -> list[tuple[str, datetime.dat r = s.fetchall() return r -def get_pkgs_last_rusage(pkgs: list[str]) -> dict[str, UsedResource]: +def get_pkgs_last_rusage(pkgs: list[str]) -> Rusages: if not pkgs: - return {} + return Rusages({}) with get_session() as s: s.execute(''' - select pkgbase, cputime, memory, elapsed from ( - select pkgbase, cputime, memory, elapsed, row_number() over (partition by pkgbase order by ts desc) as k + select pkgbase, builder, cputime, memory, elapsed from ( + select pkgbase, builder, cputime, memory, elapsed, row_number() over (partition by pkgbase, builder order by ts desc) as k from pkglog where pkgbase = any(%s) and result in ('successful', 'staged') ) as w where k = 1''', (pkgs,)) rs = s.fetchall() - ret = {r[0]: UsedResource(r[1], r[2], r[3]) for r in rs} + ret = {} + for pkgbase, rr in groupby(rs, lambda r: r[0]): + ret[pkgbase] = {r[1]: UsedResource(r[2], r[3], r[4]) for r in rr} - return ret + return Rusages(ret) def _get_last_two_versions(s, pkg: str) -> tuple[str, str]: s.execute( diff --git a/lilac2/typing.py b/lilac2/typing.py index d6b425a8..4873f866 100644 --- a/lilac2/typing.py +++ b/lilac2/typing.py @@ -3,12 +3,15 @@ import types from typing import ( Union, Dict, Tuple, Type, NamedTuple, Optional, - Sequence, + Sequence, TYPE_CHECKING, ) from pathlib import Path import dataclasses import datetime +if TYPE_CHECKING: + from .workerman import WorkerManager + class LilacMod(types.ModuleType): time_limit_hours: float pkgbase: str @@ -90,7 +93,29 @@ class UsedResource(NamedTuple): memory: int elapsed: int +class Rusages: + def __init__(self, data: dict[str, dict[str, UsedResource]]) -> None: + '''data: pkgbase -> builder -> UsedResource''' + self.data = data + + def for_package( + self, + pkgbase: str, + builder_hints: list[str], + ) -> Optional[UsedResource]: + if a := self.data.get(pkgbase): + for builder in builder_hints: + if b := a.get(builder): + return b + if a: + return next(iter(a.values())) + + return None + OnBuildVers = list[tuple[str, str]] -class PkgToBuild(NamedTuple): + +@dataclasses.dataclass +class PkgToBuild: pkgbase: str on_build_vers: OnBuildVers = [] + workerman: Optional[WorkerManager] = None diff --git a/lilac2/workerman.py b/lilac2/workerman.py index ef1f3068..02b8d19b 100644 --- a/lilac2/workerman.py +++ b/lilac2/workerman.py @@ -1,9 +1,88 @@ from pathlib import Path -from typing import override +from typing import override, Callable, Optional +import logging + +from . import tools +from .typing import PkgToBuild, Rusages + +logger = logging.getLogger(__name__) class WorkerManager: name: str max_concurrency: int + workers_before_me: int = 0 + current_task_count: int = 0 + + def get_resource_usage(self) -> tuple[float, int]: + raise NotImplementedError + + def try_accept_package( + self, + ready_to_build: list[str], + rusages: Rusages, + priority_func: Callable[[str], int], + check_buildability: Callable[[str], Optional[PkgToBuild]], + ) -> list[PkgToBuild]: + if self.current_task_count >= self.max_concurrency: + return [] + + cpu_ratio, memory_avail = self.get_resource_usage() + + if cpu_ratio > 1.0 and self.current_task_count > 0: + return [] + + def sort_key(pkg): + p = priority_func(pkg) + r = rusages.for_package(pkg, [self.name]) + if r is not None: + cpu = r.cputime / r.elapsed + else: + cpu = 1.0 + return (p, cpu) + ready_to_build.sort(key=sort_key) + logger.debug('[%s] sorted ready_to_build: %r', + self.name, ready_to_build) + + if cpu_ratio < 0.9: + # low cpu usage, build a big package + p = priority_func(ready_to_build[0]) + for idx, pkg in enumerate(ready_to_build): + if priority_func(pkg) != p: + if idx > 2: + ready_to_build.insert(0, ready_to_build.pop(idx-1)) + break + else: + logger.info('high cpu usage (%.2f), preferring low-cpu-usage builds', cpu_ratio) + + ret: list[PkgToBuild] = [] + + limited_by_memory = False + for pkg in ready_to_build: + r = rusages.for_package(pkg, [self.name]) + if r and r.memory > memory_avail: + logger.debug('package %s used %d memory last time, but now only %d is available', pkg, r.memory, memory_avail) + limited_by_memory = True + continue + + to_build = check_buildability(pkg) + if to_build is None: + continue + + to_build.workerman = self + ret.append(to_build) + if len(ret) + self.current_task_count >= self.max_concurrency: + break + + if r: + memory_avail -= r.memory + else: + memory_avail -= 10 * 1024 ** 3 + + if not ret and limited_by_memory: + logger.info('insufficient memory, not starting another concurrent build (available: %d)', memory_avail) + + self.current_task_count += len(ret) + return ret def prepare(self) -> None: raise NotImplementedError @@ -21,6 +100,12 @@ class LocalWorkerManager(WorkerManager): def __init__(self, max_concurrency) -> None: self.max_concurrency = max_concurrency + @override + def get_resource_usage(self) -> tuple[float, int]: + cpu_ratio = tools.get_running_task_cpu_ratio() + memory_avail = tools.get_avail_memory() + return cpu_ratio, memory_avail + @override def prepare(self): # git pull @@ -39,6 +124,9 @@ class RemoteWorkerManager(WorkerManager): max_concurrency: int repodir: Path + def get_resource_usage(self) -> tuple[float, int]: + raise NotImplementedError + @override def prepare(self): # git pull From 47df80c3956fe4b648902f106deaa454d6107785 Mon Sep 17 00:00:00 2001 From: lilydjwg Date: Thu, 9 Oct 2025 16:56:12 +0800 Subject: [PATCH 05/46] remoteworker done --- config.toml.sample | 4 +- lilac | 1 + lilac2/building.py | 43 +++++--- lilac2/remote/__init__.py | 0 lilac2/remote/runner.py | 51 ++++++++++ lilac2/remote/worker.py | 53 ++++++++++ lilac2/repo.py | 29 +++--- lilac2/systemd.py | 3 + lilac2/worker.py | 93 +++++++++++------- lilac2/workerman.py | 202 ++++++++++++++++++++++++++++++++------ processes.md | 9 ++ 11 files changed, 386 insertions(+), 102 deletions(-) create mode 100644 lilac2/remote/__init__.py create mode 100644 lilac2/remote/runner.py create mode 100644 lilac2/remote/worker.py create mode 100644 processes.md diff --git a/config.toml.sample b/config.toml.sample index 0a5a2e2c..96ff3500 100644 --- a/config.toml.sample +++ b/config.toml.sample @@ -44,10 +44,12 @@ max_concurrency = 1 # build packages over ssh [[remoteworker]] name = "remotebuilder" +# ssh host string; ControlMaster is advised to be used host = "builder.example.org" -# the same repodir but on remote host +# the directory on remote host that stores files repodir = "/path/to/gitrepo" max_concurrency = 2 +enabled = true [nvchecker] # set proxy for nvchecker diff --git a/lilac b/lilac index 67e5f995..2fd72bee 100755 --- a/lilac +++ b/lilac @@ -619,6 +619,7 @@ def get_workermans(): remotes = [ workerman.RemoteWorkerManager(remote) for remote in config.get('remoteworker', []) + if remote.get('enabled', False) ] ret = [local] + remotes diff --git a/lilac2/building.py b/lilac2/building.py index 7fd3353f..8e0a73bf 100644 --- a/lilac2/building.py +++ b/lilac2/building.py @@ -1,11 +1,10 @@ from __future__ import annotations import os -import sys import logging import subprocess from typing import ( - Optional, Iterable, List, Set, TYPE_CHECKING, + Optional, Iterable, Set, TYPE_CHECKING, ) import tempfile from pathlib import Path @@ -21,6 +20,7 @@ from .nomypy import BuildResult # type: ignore from . import systemd from . import intl +from .workerman import WorkerManager if TYPE_CHECKING: from .repo import Repo @@ -66,14 +66,17 @@ def build_package( packager = '%s (on behalf of %s) <%s>' % ( myname, maintainer.name, maintainer.email) + assert to_build.workerman is not None depend_packages = resolve_depends(repo, depends) + to_build.workerman.sync_depended_packages(depend_packages) pkgdir = repo.repodir / pkgbase try: - # TODO: use to_build.wm pkg_version, rusage, error = call_worker( + repo = repo, + lilacinfo = lilacinfo, pkgbase = pkgbase, pkgdir = pkgdir, - depend_packages = [str(x) for x in depend_packages], + depend_packages = depend_packages, update_info = update_info, on_build_vers = to_build.on_build_vers, bindmounts = bindmounts, @@ -83,6 +86,7 @@ def build_package( deadline = start_time + time_limit_hours * 3600, packager = packager, worker_no = worker_no, + workerman = to_build.workerman, ) if error: raise error @@ -129,9 +133,10 @@ def build_package( ) return result, pkg_version -def resolve_depends(repo: Optional[Repo], depends: Iterable[Dependency]) -> List[str]: +def resolve_depends(repo: Optional[Repo], depends: Iterable[Dependency]) -> list[str]: need_build_first = set() depend_packages = [] + cwd = os.getcwd() for x in depends: p = x.resolve() @@ -141,7 +146,7 @@ def resolve_depends(repo: Optional[Repo], depends: Iterable[Dependency]) -> List continue need_build_first.add(x.pkgname) else: - depend_packages.append(str(p)) + depend_packages.append(f'../{p.relative_to(cwd)}') if need_build_first: raise MissingDependencies(need_build_first) @@ -177,10 +182,12 @@ def notify_maintainers( repo.sendmail(addresses, subject, body) def call_worker( + repo: Repo, + lilacinfo: LilacInfo, pkgbase: str, pkgdir: Path, logfile: Path, - depend_packages: List[str], + depend_packages: list[str], update_info: NvResults, on_build_vers: OnBuildVers, commit_msg_template: str, @@ -189,6 +196,7 @@ def call_worker( deadline: float, packager: str, worker_no: int, + workerman: WorkerManager, ) -> tuple[Optional[str], RUsage, Optional[Exception]]: ''' return: package version, resource usage, error information @@ -200,8 +208,9 @@ def call_worker( 'commit_msg_template': commit_msg_template, 'bindmounts': bindmounts, 'tmpfs': tmpfs, - 'logfile': str(logfile), # for sending error reports 'worker_no': worker_no, + 'workerman': workerman.name, + 'deadline': deadline, } fd, resultpath = tempfile.mkstemp(prefix=f'{pkgbase}-', suffix='.lilac') os.close(fd) @@ -209,17 +218,12 @@ def call_worker( input_bytes = json.dumps(input).encode() logger.debug('worker input: %r', input_bytes) - cmd = [ - sys.executable, - '-Xno_debug_ranges', # save space - '-P', # don't prepend cwd to sys.path where unexpected directories may exist - '-m', 'lilac2.worker', pkgbase, - ] + cmd = workerman.get_worker_cmd(pkgbase) if systemd.available(): _call_cmd = _call_cmd_systemd else: _call_cmd = _call_cmd_subprocess - name = f'lilac-worker-{worker_no}' + name = f'lilac-worker-{workerman.name}-{worker_no}' rusage, timedout = _call_cmd( name, cmd, logfile, pkgdir, deadline, input_bytes, packager, @@ -249,11 +253,20 @@ def call_worker( elif st == 'skipped': error = SkipBuild(r['msg']) elif st == 'failed': + if report := r.get('report'): + repo.send_error_report( + lilacinfo, + subject = report['subject'], + msg = report['msg'], + logfile = logfile, + ) error = BuildFailed(r['msg']) else: error = RuntimeError('unknown status from worker', st) version = r['version'] + if ru2 := r.get('rusage'): + rusage = RUsage(*ru2) return version, rusage, error def _call_cmd_subprocess( diff --git a/lilac2/remote/__init__.py b/lilac2/remote/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/lilac2/remote/runner.py b/lilac2/remote/runner.py new file mode 100644 index 00000000..c29ca396 --- /dev/null +++ b/lilac2/remote/runner.py @@ -0,0 +1,51 @@ +import sys +import json +import subprocess +import logging +import tempfile +import os + +from .. import systemd +from ..vendor.nicelogger import enable_pretty_logging + +logger = logging.getLogger(__name__) + +def main() -> None: + enable_pretty_logging('DEBUG') + + input = json.load(sys.stdin) + logger.debug('[remote.runner] got input: %r', input) + + name = input.pop('name') + deadline = input.pop('deadline') + myresultpath = input.pop('result') + + cmd = ['python', '-m', 'lilac2.worker'] + sys.argv[1:] + + input2: dict[str, str] = {} + fd, resultpath = tempfile.mkstemp(prefix='remoterunner-', suffix='.lilac') + os.close(fd) + input['result'] = resultpath + + p = systemd.start_cmd( + name, + cmd, + stdin = subprocess.PIPE, + cwd = input.pop('pkgdir'), + setenv = input.pop('setenv'), + ) + p.stdin.write(json.dumps(input2).encode()) # type: ignore + p.stdin.close() # type: ignore + + rusage, _ = systemd.poll_rusage(name, deadline) + p.wait() + + with open(resultpath, 'rb') as f: + r = json.load(f) + r['rusage'] = rusage + + with open(myresultpath, 'w') as f2: + json.dump(r, f2) + +if __name__ == '__main__': + main() diff --git a/lilac2/remote/worker.py b/lilac2/remote/worker.py new file mode 100644 index 00000000..9e4b1956 --- /dev/null +++ b/lilac2/remote/worker.py @@ -0,0 +1,53 @@ +from __future__ import annotations + +import logging +import json +import sys +import os + +from ..vendor.nicelogger import enable_pretty_logging + +from ..tools import kill_child_processes, read_config +from ..workerman import WorkerManager + +logger = logging.getLogger(__name__) + +def main() -> None: + enable_pretty_logging('DEBUG') + + config = read_config() + + input = json.load(sys.stdin) + logger.debug('[remote.worker] got input: %r', input) + workerman = WorkerManager.from_name(config, input.pop('workerman')) + worker_no = input['worker_no'] + deadline = input['deadline'] + myresultpath = input.pop('result') + + try: + pkgname = os.path.basename(os.getcwd()) + workerman.prepare_files(pkgname) + workerman.run_remote(pkgname, deadline, worker_no, input) + workerman.fetch_files(pkgname) + r = {'status': 'done'} + except Exception as e: + r = { + 'status': 'failed', + 'msg': repr(e), + } + sys.stdout.flush() + except KeyboardInterrupt: + logger.info('KeyboardInterrupt received') + r = { + 'status': 'failed', + 'msg': 'KeyboardInterrupt', + } + finally: + # say goodbye to all our children + kill_child_processes() + + with open(myresultpath, 'w') as f: + json.dump(r, f) + +if __name__ == '__main__': + main() diff --git a/lilac2/repo.py b/lilac2/repo.py index dcd9939d..181f10f7 100644 --- a/lilac2/repo.py +++ b/lilac2/repo.py @@ -20,7 +20,7 @@ from .mail import MailService from .packages import get_built_package_files from .tools import ansi_escape_re -from . import api, lilacyaml, intl +from . import lilacyaml, intl from .typing import LilacMod, Maintainer, LilacInfos, LilacInfo from .nomypy import BuildResult # type: ignore if TYPE_CHECKING: @@ -44,7 +44,7 @@ def __init__(self, config: dict[str, Any]) -> None: self.commit_msg_prefix = config['lilac'].get('commit_msg_prefix', '') self.repodir = Path(config['repository']['repodir']).expanduser() - self.bindmounts = self._get_bindmounts(config.get('bindmounts')) + self.bindmounts = config.get('bindmounts', []) self.tmpfs = config.get('misc', {}).get('tmpfs', []) self.ms = MailService(config) @@ -286,12 +286,6 @@ def send_error_report( msgs.append(msg1 + '\n\n' + exc.output) msg1 = l10n.format_value('packaging-error-traceback') msgs.append(msg1 + '\n\n' + tb) - elif isinstance(exc, api.AurDownloadError): - subject_real = subject or l10n.format_value('packaging-error-aur-subject') - msg1 = l10n.format_value('packaging-error-aur') - msgs.append(msg1 + '\n\n') - msg1 = l10n.format_value('packaging-error-traceback') - msgs.append(msg1 + '\n\n' + tb) elif isinstance(exc, TimeoutError): subject_real = subject or l10n.format_value('packaging-error-timeout-subject') else: @@ -312,6 +306,15 @@ def send_error_report( # strictly encoded, disallowing surrogate pairs with logfile.open(errors='replace') as f: build_output = f.read() + + if len(build_output) > 200 * 1024: + too_long = l10n.format_value('log-too-long') + build_output = ( + build_output[:100 * 1024] + + '\n\n' + too_long + '\n\n' + + build_output[-100 * 1024:] + ) + if build_output: log_header = l10n.format_value('packaging-log') with suppress(ValueError, KeyError): # invalid template or wrong key @@ -380,13 +383,3 @@ def on_built(self, pkg: str, result: BuildResult, version: Optional[str]) -> Non except Exception: logger.exception('postbuild cmd error for %r', cmd) - def _get_bindmounts( - self, bindmounts: Optional[dict[str, str]], - ) -> list[str]: - if bindmounts is None: - return [] - - items = [(os.path.expanduser(src), dst) - for src, dst in bindmounts.items()] - items.sort(reverse=True) - return [f'{src}:{dst}' for src, dst in items] diff --git a/lilac2/systemd.py b/lilac2/systemd.py index 91257d65..b98f0ef7 100644 --- a/lilac2/systemd.py +++ b/lilac2/systemd.py @@ -144,6 +144,9 @@ def _poll_cmd(pid: int) -> Generator[None, None, None]: logger.debug('worker exited') return yield + except KeyboardInterrupt: + # give up the service and continue + pass finally: os.close(pidfd) diff --git a/lilac2/worker.py b/lilac2/worker.py index e1170f83..cf1e663c 100644 --- a/lilac2/worker.py +++ b/lilac2/worker.py @@ -3,13 +3,14 @@ import os import logging import subprocess -from typing import Optional, List, Generator, Union +from typing import Optional, Generator, Any from types import SimpleNamespace import contextlib import json import sys from pathlib import Path import platform +import traceback import pyalpm @@ -17,7 +18,7 @@ from .vendor.myutils import file_lock from . import pkgbuild -from .typing import LilacMod, LilacInfo, Cmd, OnBuildVers +from .typing import LilacMod, Cmd, OnBuildVers from .cmd import run_cmd, UNTRUSTED_PREFIX from .api import ( vcs_update, get_pkgver_and_pkgrel, update_pkgrel, @@ -26,10 +27,8 @@ from .nvchecker import NvResults from .tools import kill_child_processes from .lilacpy import load_lilac -from .lilacyaml import load_lilacinfo from .const import _G, PACMAN_DB_DIR, mydir -from .repo import Repo -from . import intl +from . import intl, api logger = logging.getLogger(__name__) @@ -57,6 +56,12 @@ def may_update_pkgrel() -> Generator[None, None, None]: # pkgrel is not a number, resetting to 1 update_pkgrel(1) +def get_bindmounts(bindmounts: dict[str, str]) -> list[str]: + items = [(os.path.expanduser(src), dst) + for src, dst in bindmounts.items()] + items.sort(reverse=True) + return [f'{src}:{dst}' for src, dst in items] + def lilac_build( worker_no: int, mod: LilacMod, @@ -109,7 +114,7 @@ def lilac_build( if not isinstance(build_prefix, str): raise TypeError('build_prefix', build_prefix) - build_args: List[str] = [] + build_args: list[str] = [] if hasattr(mod, 'build_args'): build_args = mod.build_args @@ -141,12 +146,12 @@ def lilac_build( post_build_always(success=success) def call_build_cmd( - build_prefix: str, depends: List[str], + build_prefix: str, depends: list[str], bindmounts: list[str] = [], tmpfs: list[str] = [], build_args: list[str] = [], - makechrootpkg_args: List[str] = [], - makepkg_args: List[str] = [], + makechrootpkg_args: list[str] = [], + makepkg_args: list[str] = [], ) -> None: cmd: Cmd if build_prefix == 'makepkg': @@ -207,9 +212,6 @@ def run_build_cmd(cmd: Cmd) -> None: def main() -> None: enable_pretty_logging('DEBUG') - from .tools import read_config - config = read_config() - repo = _G.repo = Repo(config) pkgbuild.load_data(PACMAN_DB_DIR) input = json.load(sys.stdin) @@ -217,6 +219,7 @@ def main() -> None: _G.commit_msg_template = input['commit_msg_template'] + r: dict[str, Any] try: with load_lilac(Path('.')) as mod: _G.mod = mod @@ -226,7 +229,7 @@ def main() -> None: depend_packages = input['depend_packages'], update_info = NvResults.from_list(input['update_info']), on_build_vers = input.get('on_build_vers', []), - bindmounts = input['bindmounts'], + bindmounts = get_bindmounts(input['bindmounts']), tmpfs = input['tmpfs'], ) r = {'status': 'done'} @@ -241,12 +244,7 @@ def main() -> None: 'msg': repr(e), } sys.stdout.flush() - try: - handle_failure(e, repo, mod, Path(input['logfile'])) - except UnboundLocalError: - # mod failed to load - info = load_lilacinfo(Path('.')) - handle_failure(e, repo, info, Path(input['logfile'])) + r['report'] = gen_failure_report(e) except KeyboardInterrupt: logger.info('KeyboardInterrupt received') r = { @@ -257,17 +255,17 @@ def main() -> None: # say goodbye to all our children kill_child_processes() - r['version'] = getattr(_G, 'built_version', None) # type: ignore + r['version'] = getattr(_G, 'built_version', None) with open(input['result'], 'w') as f: json.dump(r, f) -def handle_failure( - e: Exception, repo: Repo, mod: Union[LilacMod, LilacInfo], logfile: Path, -) -> None: +def gen_failure_report(e: Exception) -> dict[str, str]: logger.error('build failed', exc_info=e) l10n = intl.get_l10n('mail') + report = {} + if isinstance(e, pkgbuild.ConflictWithOfficialError): reason = '' if e.groups: @@ -275,23 +273,46 @@ def handle_failure( if e.packages: reason += l10n.format_value('package-replacing-official-package', {'packages': repr(e.packages)}) + '\n' subj = l10n.format_value('package-conflicts-with-official-repos') - repo.send_error_report( - mod, subject = subj, msg = reason, - ) + report['subject'] = subj + report['msg'] = reason, elif isinstance(e, pkgbuild.DowngradingError): - repo.send_error_report( - mod, - subject = l10n.format_value('package-older-subject'), - msg = l10n.format_value('package-older-body', { - 'pkg': e.pkgname, - 'built_version': e.built_version, - 'repo_version': e.repo_version, - }) + '\n', - ) + report['subject'] = l10n.format_value('package-older-subject') + report['msg'] = l10n.format_value('package-older-body', { + 'pkg': e.pkgname, + 'built_version': e.built_version, + 'repo_version': e.repo_version, + }) + '\n' else: - repo.send_error_report(mod, exc=e, logfile=logfile) + msgs = [] + tb = ''.join(traceback.format_exception(type(e), e, e.__traceback__)) + if isinstance(e, subprocess.CalledProcessError): + subject = l10n.format_value('packaging-error-subprocess-subject') + msg1 = l10n.format_value('packaging-error-subprocess', { + 'cmd': repr(e.cmd), + 'returncode': e.returncode, + }) + msgs.append(msg1) + if e.output: + msg1 = l10n.format_value('packaging-error-subprocess-output') + msgs.append(msg1 + '\n\n' + e.output) + msg1 = l10n.format_value('packaging-error-traceback') + msgs.append(msg1 + '\n\n' + tb) + elif isinstance(e, api.AurDownloadError): + subject = l10n.format_value('packaging-error-aur-subject') + msg1 = l10n.format_value('packaging-error-aur') + msgs.append(msg1 + '\n\n') + msg1 = l10n.format_value('packaging-error-traceback') + msgs.append(msg1 + '\n\n' + tb) + else: + subject = l10n.format_value('packaging-error-unknown-subject') + msg1 = l10n.format_value('packaging-error-unknown') + msgs.append(msg1 + '\n\n' + tb) + report['subject'] = subject + report['msg'] = '\n'.join(msgs) + + return report if __name__ == '__main__': main() diff --git a/lilac2/workerman.py b/lilac2/workerman.py index 02b8d19b..e2a18735 100644 --- a/lilac2/workerman.py +++ b/lilac2/workerman.py @@ -1,9 +1,15 @@ -from pathlib import Path -from typing import override, Callable, Optional +from typing import override, Callable, Optional, Any import logging +import subprocess +import os +import json +import signal +import sys +import tempfile from . import tools -from .typing import PkgToBuild, Rusages +from .typing import PkgToBuild, Rusages, RUsage +from .cmd import run_cmd logger = logging.getLogger(__name__) @@ -12,10 +18,17 @@ class WorkerManager: max_concurrency: int workers_before_me: int = 0 current_task_count: int = 0 + rusage: Optional[RUsage] = None + + def get_worker_cmd(self, pkgbase: str) -> list[str]: + raise NotImplementedError def get_resource_usage(self) -> tuple[float, int]: raise NotImplementedError + def sync_depended_packages(self, depends: list[str]) -> None: + raise NotImplementedError + def try_accept_package( self, ready_to_build: list[str], @@ -84,14 +97,17 @@ def sort_key(pkg): self.current_task_count += len(ret) return ret - def prepare(self) -> None: - raise NotImplementedError - - def sync_depended_packages(self, depends): - raise NotImplementedError - - def build_package(self): - raise NotImplementedError + @staticmethod + def from_name(config: dict[str, Any], name: str): + if name == 'local': + max_concurrency = config['lilac'].get('max_concurrency', 1) + return LocalWorkerManager(max_concurrency) + else: + remote = [ + x for x in config['remoteworker'] + if x.get('enabled', False) and x['name'] == name + ][0] + return RemoteWorkerManager(remote) class LocalWorkerManager(WorkerManager): name: str = 'local' @@ -100,6 +116,15 @@ class LocalWorkerManager(WorkerManager): def __init__(self, max_concurrency) -> None: self.max_concurrency = max_concurrency + @override + def get_worker_cmd(self, pkgbase: str) -> list[str]: + return [ + sys.executable, + '-Xno_debug_ranges', # save space + '-P', # don't prepend cwd to sys.path where unexpected directories may exist + '-m', 'lilac2.worker', pkgbase, + ] + @override def get_resource_usage(self) -> tuple[float, int]: cpu_ratio = tools.get_running_task_cpu_ratio() @@ -107,35 +132,148 @@ def get_resource_usage(self) -> tuple[float, int]: return cpu_ratio, memory_avail @override - def prepare(self): - # git pull - ... - - @override - def sync_depended_packages(self, depends): - ... - - @override - def build_package(self): - ... + def sync_depended_packages(self, depends: list[str]) -> None: + pass class RemoteWorkerManager(WorkerManager): name: str max_concurrency: int - repodir: Path + repodir: str + host: str - def get_resource_usage(self) -> tuple[float, int]: - raise NotImplementedError + def __init__(self, remote) -> None: + self.name = remote['name'] + self.repodir = remote['repodir'] + self.host = remote['host'] + self.max_concurrency = remote.get('max_concurrency', 1) @override - def prepare(self): - # git pull - ... + def get_worker_cmd(self, pkgbase: str) -> list[str]: + return [ + sys.executable, + '-Xno_debug_ranges', # save space + '-P', # don't prepend cwd to sys.path where unexpected directories may exist + '-m', 'lilac2.remote.worker', pkgbase, self.name, + ] @override - def sync_depended_packages(self, depends): - ... + def get_resource_usage(self) -> tuple[float, int]: + sshcmd = self.get_sshcmd_prefix() + ['python', '-m', 'lilac2.tools'] + out = subprocess.check_output(sshcmd, text=True) + cpu, mem = out.split() + return float(cpu), int(mem) @override - def build_package(self): - ... + def sync_depended_packages(self, depends: list[str]) -> None: + includes = ''.join(f'/{p.rsplit('/', 2)[1]}\n' for p in depends) + rsync_cmd = [ + 'rsync', '-avi', + '--include-from=-', + '--exclude=/.*', '--exclude=*/', '--include=*.pkg.tar.zst', '--exclude=*/*', + '--delete', + './', f'{self.host}:{self.repodir.removesuffix('/')}', + ] + subprocess.run(rsync_cmd, text=True, input=includes, check=True) + + def prepare_files(self, pkgname: str) -> None: + # run in remote.worker + run_cmd(['recv_gpg_keys']) + out = subprocess.check_output(['git', 'ls-files', '.'], text=True) + rsync_cmd = [ + 'rsync', '-avi', + '--include-from=-', + './', f'{self.host}:{self.repodir.removesuffix('/')}/{pkgname}', + ] + subprocess.run(rsync_cmd, input=out, text=True, check=True) + + def fetch_files(self, pkgname: str) -> None: + # run in remote.worker + rsync_cmd = [ + 'rsync', '-avi', + '--include=*.pkg.tar.zst', '--exclude=*' + f'{self.host}:{self.repodir.removesuffix('/')}/{pkgname}/', + '.', + ] + subprocess.run(rsync_cmd, check=True) + + def run_remote( + self, + pkgname: str, + deadline: float, + worker_no: int, + input: dict[str, Any], + ) -> None: + # run in remote.worker + + setenv = { + 'MAKEFLAGS': os.environ.get('MAKEFLAGS', ''), + 'PACKAGER': os.environ.get('PACKAGER', ''), + } + name = f'lilac-worker-{worker_no}' + + fd, resultpath = tempfile.mkstemp(prefix=f'{name}-', suffix='.lilac') + os.close(fd) + + input = { + 'name': name, + 'deadline': deadline, + 'result': resultpath, + 'pkgdir': os.path.join(self.repodir, pkgname), + 'setenv': setenv, + **input, + } + + input_bytes = json.dumps(input).encode() + sshcmd: list[str] = self.get_sshcmd_prefix(pty=True) + [ + 'python', '-m', 'lilac2.remote.runner', pkgname, str(worker_no), + ] + p = subprocess.Popen( + sshcmd, + stdin = subprocess.PIPE, + ) + p.stdin.write(input_bytes) # type: ignore + p.stdin.close() # type: ignore + + e: Optional[BaseException] = None + stop_countdown = None + while True: + try: + # timeout tor waiting subprocess to terminate + if stop_countdown is not None: + stop_countdown -= 1 + if stop_countdown == 0: + break + + try: + code = p.wait(10) + except subprocess.TimeoutExpired: + st = os.stat(1) + if st.st_size > 1024 ** 3: # larger than 1G + logger.error('\n\nToo much output, killed.') + else: + if code != 0 and e is None: + e = subprocess.CalledProcessError(code, 'lilac2.remote.runner') + break + except KeyboardInterrupt as e2: + logger.info('SIGINT received, relaying to remoteworker') + p.send_signal(signal.SIGINT) + stop_countdown = 6 + e = e2 + p.wait() + + try: + sshcmd = self.get_sshcmd_prefix() + ['cat', resultpath] + out = subprocess.check_output(sshcmd, text=True) + r = json.loads(out) + self.rusage = RUsage(*r['rusage']) + if r['status'] == 'failed': + raise Exception(r['msg']) + finally: + if e: + raise e + + def get_sshcmd_prefix(self, pty: bool = False) -> list[str]: + if pty: + return ['ssh', '-t', self.host] + else: + return ['ssh', '-T', self.host] diff --git a/processes.md b/processes.md new file mode 100644 index 00000000..cb30afe3 --- /dev/null +++ b/processes.md @@ -0,0 +1,9 @@ +* lilac + * local workerman thread (collect rusage) + * systemd-run lilac2.worker (handle SIGINT) + * build cmd + * remote workerman thread (collect rusage) + * systemd-run lilac2.remote.worker (handle SIGINT) + * ssh host lilac2.remote.runner (collect rusage) + * systemd-run lilac2.worker (handle SIGINT) + * build cmd From 760bb994880d1299f528fa91cee289df0eb048dc Mon Sep 17 00:00:00 2001 From: lilydjwg Date: Thu, 9 Oct 2025 17:00:48 +0800 Subject: [PATCH 06/46] no need to run recv_gpg_keys in remote.worker --- lilac2/workerman.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/lilac2/workerman.py b/lilac2/workerman.py index e2a18735..b7764f94 100644 --- a/lilac2/workerman.py +++ b/lilac2/workerman.py @@ -9,7 +9,6 @@ from . import tools from .typing import PkgToBuild, Rusages, RUsage -from .cmd import run_cmd logger = logging.getLogger(__name__) @@ -177,7 +176,6 @@ def sync_depended_packages(self, depends: list[str]) -> None: def prepare_files(self, pkgname: str) -> None: # run in remote.worker - run_cmd(['recv_gpg_keys']) out = subprocess.check_output(['git', 'ls-files', '.'], text=True) rsync_cmd = [ 'rsync', '-avi', From 313690b36422283ae1909f179db534e875e7b585 Mon Sep 17 00:00:00 2001 From: lilydjwg Date: Thu, 9 Oct 2025 17:46:29 +0800 Subject: [PATCH 07/46] fix dataclass definition ValueError: mutable default for field on_build_vers is not allowed: use default_factory --- lilac2/typing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lilac2/typing.py b/lilac2/typing.py index 4873f866..0e49e382 100644 --- a/lilac2/typing.py +++ b/lilac2/typing.py @@ -117,5 +117,5 @@ def for_package( @dataclasses.dataclass class PkgToBuild: pkgbase: str - on_build_vers: OnBuildVers = [] + on_build_vers: OnBuildVers = dataclasses.field(default_factory=list) workerman: Optional[WorkerManager] = None From 2952bfd819c0bd89027948c1039701495c59f442 Mon Sep 17 00:00:00 2001 From: lilydjwg Date: Thu, 9 Oct 2025 20:08:59 +0800 Subject: [PATCH 08/46] fix _G.repo.name no longer available in worker and remoteworker didn't update its pacman databases. --- lilac | 6 +++--- lilac2/building.py | 1 + lilac2/lilacyaml.py | 4 ++-- lilac2/pkgbuild.py | 6 ++++++ lilac2/worker.py | 1 + lilac2/workerman.py | 18 +++++++++++++++++- 6 files changed, 30 insertions(+), 6 deletions(-) diff --git a/lilac b/lilac index 2fd72bee..824fff5b 100755 --- a/lilac +++ b/lilac @@ -45,7 +45,6 @@ from lilac2.repo import Repo from lilac2.const import mydir, _G, PACMAN_DB_DIR from lilac2.nvchecker import packages_need_update, nvtake, NvResults from lilac2.nomypy import BuildResult, BuildReason # type: ignore -from lilac2 import pkgbuild from lilac2.building import build_package, MissingDependencies from lilac2 import slogconf from lilac2 import intl @@ -641,7 +640,9 @@ def main_may_raise( logger.warning('/etc/resolv.conf is a symlink; this might not work!') pacman_conf = config['misc'].get('pacman_conf') - pkgbuild.update_data(PACMAN_DB_DIR, pacman_conf) + workermans = get_workermans() + for wm in workermans: + wm.update_pacmandb(PACMAN_DB_DIR, pacman_conf) if dburl := config['lilac'].get('dburl'): schema = config['lilac'].get('schema') @@ -764,7 +765,6 @@ def main_may_raise( build_reasons[p].append(BuildReason.OnBuild(update_on_build)) update_succeeded: set[str] = set() - workermans = get_workermans() try: build_logger.info('build start') diff --git a/lilac2/building.py b/lilac2/building.py index 8e0a73bf..43a19fef 100644 --- a/lilac2/building.py +++ b/lilac2/building.py @@ -211,6 +211,7 @@ def call_worker( 'worker_no': worker_no, 'workerman': workerman.name, 'deadline': deadline, + 'reponame': repo.name, } fd, resultpath = tempfile.mkstemp(prefix=f'{pkgbase}-', suffix='.lilac') os.close(fd) diff --git a/lilac2/lilacyaml.py b/lilac2/lilacyaml.py index a49c3ab6..ba94fdc9 100644 --- a/lilac2/lilacyaml.py +++ b/lilac2/lilacyaml.py @@ -108,7 +108,7 @@ def load_lilacinfo(dir: Path) -> LilacInfo: def expand_alias_arg(value: str) -> str: return value.format( pacman_db_dir = PACMAN_DB_DIR, - repo_name = _G.repo.name, + repo_name = _G.reponame, ) def parse_update_on( @@ -134,7 +134,7 @@ def parse_update_on( if alias == 'alpm-lilac': entry['source'] = 'alpm' entry.setdefault('dbpath', str(PACMAN_DB_DIR)) - entry.setdefault('repo', _G.repo.name) + entry.setdefault('repo', _G.reponame) elif alias is not None: for k, v in ALIASES[alias].items(): diff --git a/lilac2/pkgbuild.py b/lilac2/pkgbuild.py index 09f0ec20..2d99d772 100644 --- a/lilac2/pkgbuild.py +++ b/lilac2/pkgbuild.py @@ -167,3 +167,9 @@ def _get_package_version(srcinfo: List[str]) -> PkgVers: assert pkgver is not None assert pkgrel is not None return PkgVers(epoch, pkgver, pkgrel) + +if __name__ == '__main__': + import sys + dbdir = Path(sys.argv[1]) + conf = sys.argv[2] or None + update_data(dbdir, conf) diff --git a/lilac2/worker.py b/lilac2/worker.py index cf1e663c..e1d22a98 100644 --- a/lilac2/worker.py +++ b/lilac2/worker.py @@ -218,6 +218,7 @@ def main() -> None: logger.debug('got input: %r', input) _G.commit_msg_template = input['commit_msg_template'] + _G.reponame = input['reponame'] r: dict[str, Any] try: diff --git a/lilac2/workerman.py b/lilac2/workerman.py index b7764f94..a5ee6250 100644 --- a/lilac2/workerman.py +++ b/lilac2/workerman.py @@ -6,8 +6,8 @@ import signal import sys import tempfile +from pathlib import Path -from . import tools from .typing import PkgToBuild, Rusages, RUsage logger = logging.getLogger(__name__) @@ -28,6 +28,9 @@ def get_resource_usage(self) -> tuple[float, int]: def sync_depended_packages(self, depends: list[str]) -> None: raise NotImplementedError + def update_pacmandb(self, dbdir: Path, conf: Optional[str]) -> None: + raise NotImplementedError + def try_accept_package( self, ready_to_build: list[str], @@ -126,6 +129,7 @@ def get_worker_cmd(self, pkgbase: str) -> list[str]: @override def get_resource_usage(self) -> tuple[float, int]: + from . import tools cpu_ratio = tools.get_running_task_cpu_ratio() memory_avail = tools.get_avail_memory() return cpu_ratio, memory_avail @@ -134,6 +138,11 @@ def get_resource_usage(self) -> tuple[float, int]: def sync_depended_packages(self, depends: list[str]) -> None: pass + @override + def update_pacmandb(self, dbdir: Path, conf: Optional[str]) -> None: + from . import pkgbuild + pkgbuild.update_data(dbdir, conf) + class RemoteWorkerManager(WorkerManager): name: str max_concurrency: int @@ -174,6 +183,13 @@ def sync_depended_packages(self, depends: list[str]) -> None: ] subprocess.run(rsync_cmd, text=True, input=includes, check=True) + @override + def update_pacmandb(self, dbdir: Path, conf: Optional[str]) -> None: + sshcmd = self.get_sshcmd_prefix() + [ + 'python', '-m', 'lilac2.pkgbuild', str(dbdir), conf or '', + ] + subprocess.check_call(sshcmd) + def prepare_files(self, pkgname: str) -> None: # run in remote.worker out = subprocess.check_output(['git', 'ls-files', '.'], text=True) From 31e7b4b644bcb5d5651f27d08d174d026443f99b Mon Sep 17 00:00:00 2001 From: lilydjwg Date: Thu, 9 Oct 2025 20:10:41 +0800 Subject: [PATCH 09/46] minor update in comments --- lilac2/const.py | 1 - 1 file changed, 1 deletion(-) diff --git a/lilac2/const.py b/lilac2/const.py index 8462217b..67d3a7f5 100644 --- a/lilac2/const.py +++ b/lilac2/const.py @@ -18,6 +18,5 @@ # repo: Repo # mod: LilacMod # worker: -# repo: Repo (for sending reports; not loading all lilacinfos) # mod: LilacMod # built_version: Optional[str] From 73531ce53bf755a554f219c94caa37207b78c777 Mon Sep 17 00:00:00 2001 From: lilydjwg Date: Thu, 9 Oct 2025 20:12:09 +0800 Subject: [PATCH 10/46] fix _G.reponame not available for master process --- lilac | 1 + 1 file changed, 1 insertion(+) diff --git a/lilac b/lilac index 824fff5b..67f40526 100755 --- a/lilac +++ b/lilac @@ -75,6 +75,7 @@ logger = logging.getLogger(__name__) build_logger_old = logging.getLogger('build') build_logger = structlog.get_logger(logger_name='build') REPO = _G.repo = Repo(config) +_G.reponame = REPO.name EMPTY_COMMIT = '4b825dc642cb6eb9a060e54bf8d69288fbee4904' From a66eb2fa75a031ab682015cc870904a049363c07 Mon Sep 17 00:00:00 2001 From: lilydjwg Date: Thu, 9 Oct 2025 20:30:36 +0800 Subject: [PATCH 11/46] fix wrong "not in lilacinfos" warning --- lilac | 4 ++-- lilac2/repo.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lilac b/lilac index 67f40526..c28408a7 100755 --- a/lilac +++ b/lilac @@ -437,8 +437,8 @@ def check_buildability( (new, new) for _, new in db.get_update_on_build_vers(update_on_build) ] to_build = PkgToBuild(pkg, vers) - else: - logger.warning('%s not in lilacinfos.', pkg) + else: + logger.warning('%s not in lilacinfos.', pkg) return to_build diff --git a/lilac2/repo.py b/lilac2/repo.py index 181f10f7..65aa4351 100644 --- a/lilac2/repo.py +++ b/lilac2/repo.py @@ -56,7 +56,7 @@ def __init__(self, config: dict[str, Any]) -> None: self.on_built_cmds = config.get('misc', {}).get('postbuild', []) - self.lilacinfos: LilacInfos = {} # to be filled by self.load_all_lilac_and_report() + self.lilacinfos: LilacInfos = {} # to be filled by self.load_managed_lilac_and_report() self.yamls: dict[str, Any] = {} self._maint_cache: dict[str, list[Maintainer]] = {} From 99b7469789f4f4ebb97a651645614f09d394f743 Mon Sep 17 00:00:00 2001 From: lilydjwg Date: Thu, 9 Oct 2025 20:35:48 +0800 Subject: [PATCH 12/46] option to disable local worker --- config.toml.sample | 2 ++ lilac | 14 ++++++++++---- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/config.toml.sample b/config.toml.sample index 96ff3500..f4860249 100644 --- a/config.toml.sample +++ b/config.toml.sample @@ -40,6 +40,8 @@ logurl = "https://example.com/${pkgbase}/${datetime}.html" # the schema to use; by default lilac uses the schema "lilac" # schema = "lilac" max_concurrency = 1 +# whether to disable local worker (and use remote only) +# disable_local_worker = false # build packages over ssh [[remoteworker]] diff --git a/lilac b/lilac index c28408a7..67c2a334 100755 --- a/lilac +++ b/lilac @@ -614,16 +614,22 @@ def build_nvchecker_reason( def get_workermans(): from lilac2 import workerman - max_concurrency = config['lilac'].get('max_concurrency', 1) - local = workerman.LocalWorkerManager(max_concurrency) + ret = [] + workers_before = 0 + + if config['lilac'].get('disable_local_worker', False): + max_concurrency = config['lilac'].get('max_concurrency', 1) + local = workerman.LocalWorkerManager(max_concurrency) + ret.append(local) + workers_before = max_concurrency + remotes = [ workerman.RemoteWorkerManager(remote) for remote in config.get('remoteworker', []) if remote.get('enabled', False) ] - ret = [local] + remotes + ret.extend(remotes) - workers_before = max_concurrency for remote in remotes: remote.workers_before_me = workers_before workers_before += remote.max_concurrency From fbf224fc778b4365abaf4d041d341d3a39e27a37 Mon Sep 17 00:00:00 2001 From: lilydjwg Date: Thu, 9 Oct 2025 20:46:57 +0800 Subject: [PATCH 13/46] fix -m lilac2.pkgbuild --- lilac | 4 ++-- lilac2/pkgbuild.py | 9 +++++---- lilac2/workerman.py | 11 +++++------ 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/lilac b/lilac index 67c2a334..05917810 100755 --- a/lilac +++ b/lilac @@ -42,7 +42,7 @@ from lilac2.cmd import ( ) from lilac2 import tools from lilac2.repo import Repo -from lilac2.const import mydir, _G, PACMAN_DB_DIR +from lilac2.const import mydir, _G from lilac2.nvchecker import packages_need_update, nvtake, NvResults from lilac2.nomypy import BuildResult, BuildReason # type: ignore from lilac2.building import build_package, MissingDependencies @@ -649,7 +649,7 @@ def main_may_raise( pacman_conf = config['misc'].get('pacman_conf') workermans = get_workermans() for wm in workermans: - wm.update_pacmandb(PACMAN_DB_DIR, pacman_conf) + wm.update_pacmandb(pacman_conf) if dburl := config['lilac'].get('dburl'): schema = config['lilac'].get('schema') diff --git a/lilac2/pkgbuild.py b/lilac2/pkgbuild.py index 2d99d772..747a5e56 100644 --- a/lilac2/pkgbuild.py +++ b/lilac2/pkgbuild.py @@ -71,8 +71,10 @@ def update_pacmandb(dbpath: Path, pacman_conf: Optional[str] = None, else: p.check_returncode() -def update_data(dbpath: Path, pacman_conf: Optional[str], +def update_data(pacman_conf: Optional[str], *, quiet: bool = False) -> None: + from .const import PACMAN_DB_DIR + dbpath = PACMAN_DB_DIR update_pacmandb(dbpath, pacman_conf, quiet=quiet) now = int(time.time()) @@ -170,6 +172,5 @@ def _get_package_version(srcinfo: List[str]) -> PkgVers: if __name__ == '__main__': import sys - dbdir = Path(sys.argv[1]) - conf = sys.argv[2] or None - update_data(dbdir, conf) + conf = sys.argv[1] if len(sys.argv) == 2 else None + update_data(conf) diff --git a/lilac2/workerman.py b/lilac2/workerman.py index a5ee6250..27328ffa 100644 --- a/lilac2/workerman.py +++ b/lilac2/workerman.py @@ -6,7 +6,6 @@ import signal import sys import tempfile -from pathlib import Path from .typing import PkgToBuild, Rusages, RUsage @@ -28,7 +27,7 @@ def get_resource_usage(self) -> tuple[float, int]: def sync_depended_packages(self, depends: list[str]) -> None: raise NotImplementedError - def update_pacmandb(self, dbdir: Path, conf: Optional[str]) -> None: + def update_pacmandb(self, conf: Optional[str]) -> None: raise NotImplementedError def try_accept_package( @@ -139,9 +138,9 @@ def sync_depended_packages(self, depends: list[str]) -> None: pass @override - def update_pacmandb(self, dbdir: Path, conf: Optional[str]) -> None: + def update_pacmandb(self, conf: Optional[str]) -> None: from . import pkgbuild - pkgbuild.update_data(dbdir, conf) + pkgbuild.update_data(conf) class RemoteWorkerManager(WorkerManager): name: str @@ -184,9 +183,9 @@ def sync_depended_packages(self, depends: list[str]) -> None: subprocess.run(rsync_cmd, text=True, input=includes, check=True) @override - def update_pacmandb(self, dbdir: Path, conf: Optional[str]) -> None: + def update_pacmandb(self, conf: Optional[str]) -> None: sshcmd = self.get_sshcmd_prefix() + [ - 'python', '-m', 'lilac2.pkgbuild', str(dbdir), conf or '', + 'python', '-m', 'lilac2.pkgbuild', conf or '', ] subprocess.check_call(sshcmd) From 0bd51ac3cecb3714cc8ba5b08353b340e8a5e920 Mon Sep 17 00:00:00 2001 From: lilydjwg Date: Thu, 9 Oct 2025 20:57:25 +0800 Subject: [PATCH 14/46] fix reversed logic for disable_local_worker --- lilac | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lilac b/lilac index 05917810..dc59d926 100755 --- a/lilac +++ b/lilac @@ -617,7 +617,7 @@ def get_workermans(): ret = [] workers_before = 0 - if config['lilac'].get('disable_local_worker', False): + if not config['lilac'].get('disable_local_worker', False): max_concurrency = config['lilac'].get('max_concurrency', 1) local = workerman.LocalWorkerManager(max_concurrency) ret.append(local) From dafbe5b77d299dcb81d1a5ed74193a7a555e1ae3 Mon Sep 17 00:00:00 2001 From: lilydjwg Date: Thu, 9 Oct 2025 21:01:30 +0800 Subject: [PATCH 15/46] remove picked packages from ready_to_build --- lilac | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lilac b/lilac index dc59d926..67d41ae5 100755 --- a/lilac +++ b/lilac @@ -332,6 +332,10 @@ def try_pick_some( lambda pkg: check_buildability(pkg,repo, buildsorter, failed), ) ret.extend(to_builds) + # remove picked packages from ready_to_build + picked = {x.pkgbase for x in to_builds} + ready_to_build = [x for x in ready_to_build + if x not in picked] if not ret and starving: wm = workermans[0] From e4821d63a11604028cc2f4b147e91e2ec87afdda Mon Sep 17 00:00:00 2001 From: lilydjwg Date: Thu, 9 Oct 2025 21:08:35 +0800 Subject: [PATCH 16/46] fix version not passed back from remote worker --- lilac2/remote/worker.py | 5 +++-- lilac2/workerman.py | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/lilac2/remote/worker.py b/lilac2/remote/worker.py index 9e4b1956..070b7a98 100644 --- a/lilac2/remote/worker.py +++ b/lilac2/remote/worker.py @@ -27,7 +27,7 @@ def main() -> None: try: pkgname = os.path.basename(os.getcwd()) workerman.prepare_files(pkgname) - workerman.run_remote(pkgname, deadline, worker_no, input) + remote_r = workerman.run_remote(pkgname, deadline, worker_no, input) workerman.fetch_files(pkgname) r = {'status': 'done'} except Exception as e: @@ -47,7 +47,8 @@ def main() -> None: kill_child_processes() with open(myresultpath, 'w') as f: - json.dump(r, f) + remote_r.update(r) + json.dump(remote_r, f) if __name__ == '__main__': main() diff --git a/lilac2/workerman.py b/lilac2/workerman.py index 27328ffa..1e211952 100644 --- a/lilac2/workerman.py +++ b/lilac2/workerman.py @@ -215,7 +215,7 @@ def run_remote( deadline: float, worker_no: int, input: dict[str, Any], - ) -> None: + ) -> dict[str, Any]: # run in remote.worker setenv = { @@ -278,9 +278,10 @@ def run_remote( sshcmd = self.get_sshcmd_prefix() + ['cat', resultpath] out = subprocess.check_output(sshcmd, text=True) r = json.loads(out) - self.rusage = RUsage(*r['rusage']) + self.rusage = RUsage(*r.pop('rusage')) if r['status'] == 'failed': raise Exception(r['msg']) + return r finally: if e: raise e From 556da5c8ecf543115e5754d74174338fe4dc9b86 Mon Sep 17 00:00:00 2001 From: lilydjwg Date: Thu, 9 Oct 2025 21:15:04 +0800 Subject: [PATCH 17/46] fix input not passed from remote.runner to worker --- lilac2/remote/runner.py | 3 +-- lilac2/worker.py | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/lilac2/remote/runner.py b/lilac2/remote/runner.py index c29ca396..b2eb8848 100644 --- a/lilac2/remote/runner.py +++ b/lilac2/remote/runner.py @@ -22,7 +22,6 @@ def main() -> None: cmd = ['python', '-m', 'lilac2.worker'] + sys.argv[1:] - input2: dict[str, str] = {} fd, resultpath = tempfile.mkstemp(prefix='remoterunner-', suffix='.lilac') os.close(fd) input['result'] = resultpath @@ -34,7 +33,7 @@ def main() -> None: cwd = input.pop('pkgdir'), setenv = input.pop('setenv'), ) - p.stdin.write(json.dumps(input2).encode()) # type: ignore + p.stdin.write(json.dumps(input).encode()) # type: ignore p.stdin.close() # type: ignore rusage, _ = systemd.poll_rusage(name, deadline) diff --git a/lilac2/worker.py b/lilac2/worker.py index e1d22a98..cd6bebfb 100644 --- a/lilac2/worker.py +++ b/lilac2/worker.py @@ -215,7 +215,7 @@ def main() -> None: pkgbuild.load_data(PACMAN_DB_DIR) input = json.load(sys.stdin) - logger.debug('got input: %r', input) + logger.debug('[worker] got input: %r', input) _G.commit_msg_template = input['commit_msg_template'] _G.reponame = input['reponame'] From da0bfbe04705af25fa40761e9fb97d6399028fac Mon Sep 17 00:00:00 2001 From: lilydjwg Date: Thu, 9 Oct 2025 21:17:57 +0800 Subject: [PATCH 18/46] pass LANG and TZ to remote worker --- lilac2/workerman.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lilac2/workerman.py b/lilac2/workerman.py index 1e211952..0d976333 100644 --- a/lilac2/workerman.py +++ b/lilac2/workerman.py @@ -221,7 +221,11 @@ def run_remote( setenv = { 'MAKEFLAGS': os.environ.get('MAKEFLAGS', ''), 'PACKAGER': os.environ.get('PACKAGER', ''), + 'LANG': os.environ.get('LANG', 'C.UTF-8'), } + if tz := os.environ.get('TZ'): + setenv['TZ'] = tz + name = f'lilac-worker-{worker_no}' fd, resultpath = tempfile.mkstemp(prefix=f'{name}-', suffix='.lilac') From 24d486f4cd7213a119b1d726b55b9d39e7be201f Mon Sep 17 00:00:00 2001 From: lilydjwg Date: Fri, 10 Oct 2025 11:21:48 +0800 Subject: [PATCH 19/46] remote's repodir needs to be the git repository run git pull at startup --- config.toml.sample | 9 +++++++ lilac | 5 +++- lilac2/remote/worker.py | 1 - lilac2/workerman.py | 57 +++++++++++++++++++++++++++++++---------- 4 files changed, 56 insertions(+), 16 deletions(-) diff --git a/config.toml.sample b/config.toml.sample index f4860249..4f0c4b61 100644 --- a/config.toml.sample +++ b/config.toml.sample @@ -52,6 +52,15 @@ host = "builder.example.org" repodir = "/path/to/gitrepo" max_concurrency = 2 enabled = true +# run some commands before each run +# prerun = [ +# ["sudo", "update_something"], +# ] + +# run some commands after each run +# postrun = [ +# ["upload-packages"], +# ] [nvchecker] # set proxy for nvchecker diff --git a/lilac b/lilac index 67d41ae5..891c6426 100755 --- a/lilac +++ b/lilac @@ -653,7 +653,7 @@ def main_may_raise( pacman_conf = config['misc'].get('pacman_conf') workermans = get_workermans() for wm in workermans: - wm.update_pacmandb(pacman_conf) + wm.prepare_batch(pacman_conf) if dburl := config['lilac'].get('dburl'): schema = config['lilac'].get('schema') @@ -830,6 +830,9 @@ def main_may_raise( if config['lilac']['git_push']: git_push() + for wm in workermans: + wm.finish_batch() + if cmds := config.get('misc', {}).get('postrun'): for cmd in cmds: subprocess.check_call(cmd) diff --git a/lilac2/remote/worker.py b/lilac2/remote/worker.py index 070b7a98..796f0086 100644 --- a/lilac2/remote/worker.py +++ b/lilac2/remote/worker.py @@ -26,7 +26,6 @@ def main() -> None: try: pkgname = os.path.basename(os.getcwd()) - workerman.prepare_files(pkgname) remote_r = workerman.run_remote(pkgname, deadline, worker_no, input) workerman.fetch_files(pkgname) r = {'status': 'done'} diff --git a/lilac2/workerman.py b/lilac2/workerman.py index 0d976333..945d1701 100644 --- a/lilac2/workerman.py +++ b/lilac2/workerman.py @@ -27,7 +27,13 @@ def get_resource_usage(self) -> tuple[float, int]: def sync_depended_packages(self, depends: list[str]) -> None: raise NotImplementedError - def update_pacmandb(self, conf: Optional[str]) -> None: + def prepare_batch( + self, + pacman_conf: Optional[str], + ) -> None: + raise NotImplementedError + + def finish_batch(self) -> None: raise NotImplementedError def try_accept_package( @@ -138,21 +144,30 @@ def sync_depended_packages(self, depends: list[str]) -> None: pass @override - def update_pacmandb(self, conf: Optional[str]) -> None: + def prepare_batch( + self, + pacman_conf: Optional[str], + ) -> None: from . import pkgbuild - pkgbuild.update_data(conf) + pkgbuild.update_data(pacman_conf) + + @override + def finish_batch(self) -> None: + pass class RemoteWorkerManager(WorkerManager): name: str max_concurrency: int repodir: str host: str + config: dict[str, Any] - def __init__(self, remote) -> None: + def __init__(self, remote: dict[str, Any]) -> None: self.name = remote['name'] self.repodir = remote['repodir'] self.host = remote['host'] self.max_concurrency = remote.get('max_concurrency', 1) + self.config = remote @override def get_worker_cmd(self, pkgbase: str) -> list[str]: @@ -172,6 +187,9 @@ def get_resource_usage(self) -> tuple[float, int]: @override def sync_depended_packages(self, depends: list[str]) -> None: + if not depends: + return + includes = ''.join(f'/{p.rsplit('/', 2)[1]}\n' for p in depends) rsync_cmd = [ 'rsync', '-avi', @@ -183,21 +201,28 @@ def sync_depended_packages(self, depends: list[str]) -> None: subprocess.run(rsync_cmd, text=True, input=includes, check=True) @override - def update_pacmandb(self, conf: Optional[str]) -> None: + def prepare_batch( + self, + pacman_conf: Optional[str], + ) -> None: + # update pacman databases sshcmd = self.get_sshcmd_prefix() + [ - 'python', '-m', 'lilac2.pkgbuild', conf or '', + 'python', '-m', 'lilac2.pkgbuild', pacman_conf or '', ] subprocess.check_call(sshcmd) - def prepare_files(self, pkgname: str) -> None: - # run in remote.worker - out = subprocess.check_output(['git', 'ls-files', '.'], text=True) - rsync_cmd = [ - 'rsync', '-avi', - '--include-from=-', - './', f'{self.host}:{self.repodir.removesuffix('/')}/{pkgname}', + sshcmd = self.get_sshcmd_prefix() + [ + 'python', '-m', 'lilac2.remote.git_pull', f'"{self.repodir}"', ] - subprocess.run(rsync_cmd, input=out, text=True, check=True) + subprocess.run(sshcmd, check=True) + + if prerun := self.config.get('prerun'): + run_cmds(prerun) + + @override + def finish_batch(self) -> None: + if postrun := self.config.get('postrun'): + run_cmds(postrun) def fetch_files(self, pkgname: str) -> None: # run in remote.worker @@ -295,3 +320,7 @@ def get_sshcmd_prefix(self, pty: bool = False) -> list[str]: return ['ssh', '-t', self.host] else: return ['ssh', '-T', self.host] + +def run_cmds(cmds: list[list[str]]) -> None: + for cmd in cmds: + subprocess.check_call(cmd) From 3cce5d8e63b7a41d40ee45af1b73c02eade150f8 Mon Sep 17 00:00:00 2001 From: lilydjwg Date: Fri, 10 Oct 2025 11:38:23 +0800 Subject: [PATCH 20/46] remote needs to have the git repo too --- config.toml.sample | 10 ++++++---- lilac | 7 ++++--- lilac2/remote/git_pull.py | 9 +++++++++ lilac2/workerman.py | 21 ++++++++++++++++----- 4 files changed, 35 insertions(+), 12 deletions(-) create mode 100644 lilac2/remote/git_pull.py diff --git a/config.toml.sample b/config.toml.sample index 4f0c4b61..e331f1c7 100644 --- a/config.toml.sample +++ b/config.toml.sample @@ -45,21 +45,23 @@ max_concurrency = 1 # build packages over ssh [[remoteworker]] +# this is also used to name the git remote name = "remotebuilder" -# ssh host string; ControlMaster is advised to be used +# ssh host string; ControlMaster is advised to be enabled in ~/.ssh/config host = "builder.example.org" -# the directory on remote host that stores files +# the same as local "repodir" but in remote host repodir = "/path/to/gitrepo" max_concurrency = 2 enabled = true + # run some commands before each run # prerun = [ -# ["sudo", "update_something"], +# "sudo update_something", # ] # run some commands after each run # postrun = [ -# ["upload-packages"], +# "do_something", # ] [nvchecker] diff --git a/lilac b/lilac index 891c6426..d5655a9d 100755 --- a/lilac +++ b/lilac @@ -787,6 +787,10 @@ def main_may_raise( db.build_updated(s) start_build(REPO, logdir, failed, update_succeeded, workermans) finally: + # fetch remote commits + for wm in workermans: + wm.finish_batch() + D['last_commit'] = git_last_commit() # handle what has been processed even on exception for k, v in failed.items(): @@ -830,9 +834,6 @@ def main_may_raise( if config['lilac']['git_push']: git_push() - for wm in workermans: - wm.finish_batch() - if cmds := config.get('misc', {}).get('postrun'): for cmd in cmds: subprocess.check_call(cmd) diff --git a/lilac2/remote/git_pull.py b/lilac2/remote/git_pull.py new file mode 100644 index 00000000..f8989edd --- /dev/null +++ b/lilac2/remote/git_pull.py @@ -0,0 +1,9 @@ +import os +import sys + +from ..cmd import git_pull_override + +if __name__ == '__main__': + wd = sys.argv[1] + os.chdir(wd) + git_pull_override() diff --git a/lilac2/workerman.py b/lilac2/workerman.py index 945d1701..13941243 100644 --- a/lilac2/workerman.py +++ b/lilac2/workerman.py @@ -217,12 +217,23 @@ def prepare_batch( subprocess.run(sshcmd, check=True) if prerun := self.config.get('prerun'): - run_cmds(prerun) + self.run_cmds(prerun) @override def finish_batch(self) -> None: + out = subprocess.check_output(['git', 'remote']) + remotes = out.splitlines() + if self.name not in remotes: + subprocess.check_call([ + 'git', 'remote', 'add', self.name, + f'{self.host}:{self.repodir}', + ]) + subprocess.check_call([ + 'git', 'pull', '--no-edit', self.name, 'master', + ]) + if postrun := self.config.get('postrun'): - run_cmds(postrun) + self.run_cmds(postrun) def fetch_files(self, pkgname: str) -> None: # run in remote.worker @@ -321,6 +332,6 @@ def get_sshcmd_prefix(self, pty: bool = False) -> list[str]: else: return ['ssh', '-T', self.host] -def run_cmds(cmds: list[list[str]]) -> None: - for cmd in cmds: - subprocess.check_call(cmd) + def run_cmds(self, cmds: list[str]) -> None: + for cmd in cmds: + subprocess.check_call(self.get_sshcmd_prefix() + [cmd]) From eb98ebca0956c6a27373a19c61a01290643e5a64 Mon Sep 17 00:00:00 2001 From: lilydjwg Date: Fri, 10 Oct 2025 11:39:44 +0800 Subject: [PATCH 21/46] initialize remote_r so an exception doesn't cause the following code to fail --- lilac2/remote/worker.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lilac2/remote/worker.py b/lilac2/remote/worker.py index 796f0086..5723c928 100644 --- a/lilac2/remote/worker.py +++ b/lilac2/remote/worker.py @@ -24,6 +24,7 @@ def main() -> None: deadline = input['deadline'] myresultpath = input.pop('result') + remote_r = {} try: pkgname = os.path.basename(os.getcwd()) remote_r = workerman.run_remote(pkgname, deadline, worker_no, input) From 9f054a4ca09bdb474f971b2166de475e9c36a603 Mon Sep 17 00:00:00 2001 From: lilydjwg Date: Fri, 10 Oct 2025 11:51:09 +0800 Subject: [PATCH 22/46] RemoteWorkerManager.finish_batch: fix remote repo path --- lilac2/workerman.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/lilac2/workerman.py b/lilac2/workerman.py index 13941243..a0a43c17 100644 --- a/lilac2/workerman.py +++ b/lilac2/workerman.py @@ -224,9 +224,14 @@ def finish_batch(self) -> None: out = subprocess.check_output(['git', 'remote']) remotes = out.splitlines() if self.name not in remotes: + sshcmd = self.get_sshcmd_prefix() + ['git rev-parse --show-prefix'] + out = subprocess.check_output(sshcmd).strip('\n/') + if out: + reporoot = self.repodir.removesuffix(out).rstrip('/') + else: + reporoot = self.repodir subprocess.check_call([ - 'git', 'remote', 'add', self.name, - f'{self.host}:{self.repodir}', + 'git', 'remote', 'add', self.name, f'{self.host}:{reporoot}', ]) subprocess.check_call([ 'git', 'pull', '--no-edit', self.name, 'master', From a15f441b721604e2a87a6636673005c31a9ffd61 Mon Sep 17 00:00:00 2001 From: lilydjwg Date: Fri, 10 Oct 2025 11:58:03 +0800 Subject: [PATCH 23/46] make sure r['version'] is available --- lilac2/remote/worker.py | 2 +- lilac2/workerman.py | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/lilac2/remote/worker.py b/lilac2/remote/worker.py index 5723c928..2e40c50c 100644 --- a/lilac2/remote/worker.py +++ b/lilac2/remote/worker.py @@ -29,7 +29,7 @@ def main() -> None: pkgname = os.path.basename(os.getcwd()) remote_r = workerman.run_remote(pkgname, deadline, worker_no, input) workerman.fetch_files(pkgname) - r = {'status': 'done'} + r = {'status': 'done', 'version': None} except Exception as e: r = { 'status': 'failed', diff --git a/lilac2/workerman.py b/lilac2/workerman.py index a0a43c17..7ec1a988 100644 --- a/lilac2/workerman.py +++ b/lilac2/workerman.py @@ -324,8 +324,6 @@ def run_remote( out = subprocess.check_output(sshcmd, text=True) r = json.loads(out) self.rusage = RUsage(*r.pop('rusage')) - if r['status'] == 'failed': - raise Exception(r['msg']) return r finally: if e: From 2c4832ef919808d0b6e86fb9d74652c93131bb13 Mon Sep 17 00:00:00 2001 From: lilydjwg Date: Fri, 10 Oct 2025 12:00:09 +0800 Subject: [PATCH 24/46] compact traceback for remote.runner --- lilac2/workerman.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lilac2/workerman.py b/lilac2/workerman.py index 7ec1a988..02d5a50d 100644 --- a/lilac2/workerman.py +++ b/lilac2/workerman.py @@ -283,7 +283,10 @@ def run_remote( input_bytes = json.dumps(input).encode() sshcmd: list[str] = self.get_sshcmd_prefix(pty=True) + [ - 'python', '-m', 'lilac2.remote.runner', pkgname, str(worker_no), + 'python', + '-Xno_debug_ranges', # save space + '-P', # don't prepend cwd to sys.path where unexpected directories may exist + '-m', 'lilac2.remote.runner', pkgname, str(worker_no), ] p = subprocess.Popen( sshcmd, From 910fbc9a110c278d5a759a5e913cd9fcaa037dfd Mon Sep 17 00:00:00 2001 From: lilydjwg Date: Fri, 10 Oct 2025 12:15:30 +0800 Subject: [PATCH 25/46] only run build-cleaner after local builds --- lilac2/building.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lilac2/building.py b/lilac2/building.py index 43a19fef..cecd823d 100644 --- a/lilac2/building.py +++ b/lilac2/building.py @@ -91,7 +91,8 @@ def build_package( if error: raise error finally: - may_need_cleanup() + if to_build.workerman.name == 'local': + may_need_cleanup() reap_zombies() staging = lilacinfo.staging From 940062f16c3e5a5b74bb143560de67515e97779b Mon Sep 17 00:00:00 2001 From: lilydjwg Date: Fri, 10 Oct 2025 12:26:18 +0800 Subject: [PATCH 26/46] workerman: fix git rev-parse path --- lilac2/workerman.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lilac2/workerman.py b/lilac2/workerman.py index 02d5a50d..a8f02134 100644 --- a/lilac2/workerman.py +++ b/lilac2/workerman.py @@ -224,7 +224,9 @@ def finish_batch(self) -> None: out = subprocess.check_output(['git', 'remote']) remotes = out.splitlines() if self.name not in remotes: - sshcmd = self.get_sshcmd_prefix() + ['git rev-parse --show-prefix'] + sshcmd = self.get_sshcmd_prefix() + [ + f'cd "{self.repodir}" && git rev-parse --show-prefix' + ] out = subprocess.check_output(sshcmd).strip('\n/') if out: reporoot = self.repodir.removesuffix(out).rstrip('/') From 7fc7361e194704dd2f69de53065110312b445c22 Mon Sep 17 00:00:00 2001 From: lilydjwg Date: Fri, 10 Oct 2025 12:33:34 +0800 Subject: [PATCH 27/46] remote.worker: don't override remote_r with local r also some more logging --- lilac2/remote/worker.py | 4 ++-- lilac2/workerman.py | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/lilac2/remote/worker.py b/lilac2/remote/worker.py index 2e40c50c..dd785721 100644 --- a/lilac2/remote/worker.py +++ b/lilac2/remote/worker.py @@ -24,12 +24,12 @@ def main() -> None: deadline = input['deadline'] myresultpath = input.pop('result') - remote_r = {} + remote_r = {'status': 'done', 'version': None} + r = {} try: pkgname = os.path.basename(os.getcwd()) remote_r = workerman.run_remote(pkgname, deadline, worker_no, input) workerman.fetch_files(pkgname) - r = {'status': 'done', 'version': None} except Exception as e: r = { 'status': 'failed', diff --git a/lilac2/workerman.py b/lilac2/workerman.py index a8f02134..67a4282e 100644 --- a/lilac2/workerman.py +++ b/lilac2/workerman.py @@ -198,6 +198,7 @@ def sync_depended_packages(self, depends: list[str]) -> None: '--delete', './', f'{self.host}:{self.repodir.removesuffix('/')}', ] + logger.info('[%s] sync_depended_packages: %s', self.name, rsync_cmd) subprocess.run(rsync_cmd, text=True, input=includes, check=True) @override @@ -250,6 +251,7 @@ def fetch_files(self, pkgname: str) -> None: f'{self.host}:{self.repodir.removesuffix('/')}/{pkgname}/', '.', ] + logger.info('[%s] fetch_files: %s', self.name, rsync_cmd) subprocess.run(rsync_cmd, check=True) def run_remote( From c4987727b52414419644c5ffddfa4568b649c666 Mon Sep 17 00:00:00 2001 From: lilydjwg Date: Fri, 10 Oct 2025 12:37:20 +0800 Subject: [PATCH 28/46] missing comma in rsync args --- lilac2/workerman.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lilac2/workerman.py b/lilac2/workerman.py index 67a4282e..cbb4c500 100644 --- a/lilac2/workerman.py +++ b/lilac2/workerman.py @@ -247,7 +247,7 @@ def fetch_files(self, pkgname: str) -> None: # run in remote.worker rsync_cmd = [ 'rsync', '-avi', - '--include=*.pkg.tar.zst', '--exclude=*' + '--include=*.pkg.tar.zst', '--exclude=*', f'{self.host}:{self.repodir.removesuffix('/')}/{pkgname}/', '.', ] From dc0bf07d5a83d238748226a7403d297e68065645 Mon Sep 17 00:00:00 2001 From: lilydjwg Date: Fri, 10 Oct 2025 12:38:59 +0800 Subject: [PATCH 29/46] missing text=True for a subprocess --- lilac2/workerman.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lilac2/workerman.py b/lilac2/workerman.py index cbb4c500..780f9fe8 100644 --- a/lilac2/workerman.py +++ b/lilac2/workerman.py @@ -222,13 +222,13 @@ def prepare_batch( @override def finish_batch(self) -> None: - out = subprocess.check_output(['git', 'remote']) + out = subprocess.check_output(['git', 'remote'], text=True) remotes = out.splitlines() if self.name not in remotes: sshcmd = self.get_sshcmd_prefix() + [ f'cd "{self.repodir}" && git rev-parse --show-prefix' ] - out = subprocess.check_output(sshcmd).strip('\n/') + out = subprocess.check_output(sshcmd, text=True).strip('\n/') if out: reporoot = self.repodir.removesuffix(out).rstrip('/') else: From f0c05f8cbe5b74f57a0f1979c34673bf61e84410 Mon Sep 17 00:00:00 2001 From: lilydjwg Date: Fri, 10 Oct 2025 12:43:52 +0800 Subject: [PATCH 30/46] sign_and_copy: overwrite any existing signature --- lilac2/building.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lilac2/building.py b/lilac2/building.py index cecd823d..836087c6 100644 --- a/lilac2/building.py +++ b/lilac2/building.py @@ -168,7 +168,7 @@ def sign_and_copy(pkgdir: Path, dest: Path) -> None: for pkg in pkgs: subprocess.run([ 'gpg', '--pinentry-mode', 'loopback', - '--passphrase', '', '--detach-sign', '--', pkg, + '--passphrase', '', '--detach-sign', '--yes', '--', pkg, ]) for f in pkgs + [x.with_name(x.name + '.sig') for x in pkgs]: with suppress(FileExistsError): From 6a1f735f5acee8910c5d81c618732918e1d7cd69 Mon Sep 17 00:00:00 2001 From: lilydjwg Date: Fri, 10 Oct 2025 12:51:14 +0800 Subject: [PATCH 31/46] docs: don't forget to configure git user --- docs/setup.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/setup.rst b/docs/setup.rst index 19fcf194..f4968919 100644 --- a/docs/setup.rst +++ b/docs/setup.rst @@ -45,6 +45,8 @@ Make sure in ``/etc/makepkg.conf`` or similar files there aren't any changes to The ``PKGBUILD`` files needs to be in a git repo. A subdirectory inside it is recommended. +Configure the desired git committer name and email in ``~/.gitconfig`` or using a ``git config`` command. + Setup a passphrase-less GPG key for the build user to sign packages: .. code-block:: sh From 55df42bcef3a728e038058092536516a7a21f5b1 Mon Sep 17 00:00:00 2001 From: lilydjwg Date: Fri, 10 Oct 2025 12:58:59 +0800 Subject: [PATCH 32/46] docs: more git config --- docs/setup.rst | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/docs/setup.rst b/docs/setup.rst index f4968919..a50fda7a 100644 --- a/docs/setup.rst +++ b/docs/setup.rst @@ -47,6 +47,22 @@ The ``PKGBUILD`` files needs to be in a git repo. A subdirectory inside it is re Configure the desired git committer name and email in ``~/.gitconfig`` or using a ``git config`` command. +To avoid an issue with recent git version, configure the follow git option too: + +.. code-block:: sh + + git config --global maintenance.autoDetach false + +Or you may get the following errors from time to time:: + + fatal: update_ref failed for ref 'HEAD': cannot lock ref 'HEAD': Unable to create '...repo/.git/HEAD.lock': File exists. + + Another git process seems to be running in this repository, e.g. + an editor opened by 'git commit'. Please make sure all processes + are terminated then try again. If it still fails, a git process + may have crashed in this repository earlier: + remove the file manually to continue. + Setup a passphrase-less GPG key for the build user to sign packages: .. code-block:: sh From 8d3817300d6d22c1fd83be8f23641861672e5f5e Mon Sep 17 00:00:00 2001 From: lilydjwg Date: Fri, 10 Oct 2025 13:18:11 +0800 Subject: [PATCH 33/46] remote.runner: call worker with -Xno_debug_ranges and -P --- lilac2/remote/runner.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/lilac2/remote/runner.py b/lilac2/remote/runner.py index b2eb8848..45983463 100644 --- a/lilac2/remote/runner.py +++ b/lilac2/remote/runner.py @@ -20,7 +20,12 @@ def main() -> None: deadline = input.pop('deadline') myresultpath = input.pop('result') - cmd = ['python', '-m', 'lilac2.worker'] + sys.argv[1:] + cmd = [ + sys.executable, + '-Xno_debug_ranges', # save space + '-P', # don't prepend cwd to sys.path where unexpected directories may exist + '-m', 'lilac2.worker', + ] + sys.argv[1:] fd, resultpath = tempfile.mkstemp(prefix='remoterunner-', suffix='.lilac') os.close(fd) From 8b149d01a52748ba9b3ec72424de61e3bbfd60a1 Mon Sep 17 00:00:00 2001 From: lilydjwg Date: Fri, 10 Oct 2025 13:41:14 +0800 Subject: [PATCH 34/46] pass remote rusage via inputs, not workerman's attribute --- lilac2/workerman.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/lilac2/workerman.py b/lilac2/workerman.py index 780f9fe8..d2614b36 100644 --- a/lilac2/workerman.py +++ b/lilac2/workerman.py @@ -7,7 +7,7 @@ import sys import tempfile -from .typing import PkgToBuild, Rusages, RUsage +from .typing import PkgToBuild, Rusages logger = logging.getLogger(__name__) @@ -16,7 +16,6 @@ class WorkerManager: max_concurrency: int workers_before_me: int = 0 current_task_count: int = 0 - rusage: Optional[RUsage] = None def get_worker_cmd(self, pkgbase: str) -> list[str]: raise NotImplementedError @@ -330,7 +329,6 @@ def run_remote( sshcmd = self.get_sshcmd_prefix() + ['cat', resultpath] out = subprocess.check_output(sshcmd, text=True) r = json.loads(out) - self.rusage = RUsage(*r.pop('rusage')) return r finally: if e: From 804063c9f44d0d9a82ef2b0fcfec166e519c47a9 Mon Sep 17 00:00:00 2001 From: lilydjwg Date: Fri, 10 Oct 2025 20:26:31 +0800 Subject: [PATCH 35/46] MAKEFLAGS shouldn't be passed to remote remote determines its value itself. --- lilac | 2 +- lilac2/remote/runner.py | 10 +++++++++- lilac2/workerman.py | 1 - setup.py | 3 +-- 4 files changed, 11 insertions(+), 5 deletions(-) diff --git a/lilac b/lilac index d5655a9d..702277a2 100755 --- a/lilac +++ b/lilac @@ -864,7 +864,7 @@ def setup() -> Path: enable_pretty_logging('DEBUG') if 'MAKEFLAGS' not in os.environ: - cores = os.cpu_count() + cores = os.process_cpu_count() if cores is not None: os.environ['MAKEFLAGS'] = '-j{0}'.format(cores) diff --git a/lilac2/remote/runner.py b/lilac2/remote/runner.py index 45983463..1b26d0ea 100644 --- a/lilac2/remote/runner.py +++ b/lilac2/remote/runner.py @@ -31,12 +31,20 @@ def main() -> None: os.close(fd) input['result'] = resultpath + setenv = input.pop('setenv') + if v := os.environ.get('MAKEFLAGS'): + setenv['MAKEFLAGS'] = v + else: + cores = os.process_cpu_count() + if cores is not None: + setenv['MAKEFLAGS'] = '-j{0}'.format(cores) + p = systemd.start_cmd( name, cmd, stdin = subprocess.PIPE, cwd = input.pop('pkgdir'), - setenv = input.pop('setenv'), + setenv = setenv, ) p.stdin.write(json.dumps(input).encode()) # type: ignore p.stdin.close() # type: ignore diff --git a/lilac2/workerman.py b/lilac2/workerman.py index d2614b36..f88fa949 100644 --- a/lilac2/workerman.py +++ b/lilac2/workerman.py @@ -263,7 +263,6 @@ def run_remote( # run in remote.worker setenv = { - 'MAKEFLAGS': os.environ.get('MAKEFLAGS', ''), 'PACKAGER': os.environ.get('PACKAGER', ''), 'LANG': os.environ.get('LANG', 'C.UTF-8'), } diff --git a/setup.py b/setup.py index 4fb829ca..cb778fdc 100755 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ description = 'The build bot for archlinuxcn', author = 'lilydjwg', author_email = 'lilydjwg@gmail.com', - python_requires = '>=3.12.0', + python_requires = '>=3.13.0', url = 'https://github.com/archlinuxcn/lilac', zip_safe = False, packages = find_packages(exclude=('tests',)) + ['nvchecker_source'], @@ -26,7 +26,6 @@ classifiers = [ 'Programming Language :: Python', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.12', 'Programming Language :: Python :: 3.13', ], ) From 88b28cc0481e1c0437cca956c598969c6fd8fd0c Mon Sep 17 00:00:00 2001 From: lilydjwg Date: Sat, 11 Oct 2025 10:10:35 +0800 Subject: [PATCH 36/46] remote.git_pull: reset to remote before pull --- lilac2/remote/git_pull.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/lilac2/remote/git_pull.py b/lilac2/remote/git_pull.py index f8989edd..0ea1b084 100644 --- a/lilac2/remote/git_pull.py +++ b/lilac2/remote/git_pull.py @@ -1,9 +1,13 @@ import os import sys -from ..cmd import git_pull_override +from ..cmd import git_pull_override, run_cmd + +def main(): + cmd = ['git', 'reset', '--hard', 'origin/master'] + run_cmd(cmd) + git_pull_override() if __name__ == '__main__': wd = sys.argv[1] os.chdir(wd) - git_pull_override() From 0b2732f27e42335b99c5702f444450de6357f25f Mon Sep 17 00:00:00 2001 From: lilydjwg Date: Sat, 11 Oct 2025 10:12:05 +0800 Subject: [PATCH 37/46] more -Xno_debug_ranges -P --- lilac2/workerman.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lilac2/workerman.py b/lilac2/workerman.py index f88fa949..35a7ff16 100644 --- a/lilac2/workerman.py +++ b/lilac2/workerman.py @@ -207,12 +207,14 @@ def prepare_batch( ) -> None: # update pacman databases sshcmd = self.get_sshcmd_prefix() + [ - 'python', '-m', 'lilac2.pkgbuild', pacman_conf or '', + 'python', '-Xno_debug_ranges', '-P', + '-m', 'lilac2.pkgbuild', pacman_conf or '', ] subprocess.check_call(sshcmd) sshcmd = self.get_sshcmd_prefix() + [ - 'python', '-m', 'lilac2.remote.git_pull', f'"{self.repodir}"', + 'python', '-Xno_debug_ranges', '-P', + '-m', 'lilac2.remote.git_pull', f'"{self.repodir}"', ] subprocess.run(sshcmd, check=True) From 92d3bc9094d6abda13b7a700be2a92cbd1c88292 Mon Sep 17 00:00:00 2001 From: lilydjwg Date: Sat, 11 Oct 2025 10:37:19 +0800 Subject: [PATCH 38/46] remote.runner: make check unit name unique for each worker_no --- lilac2/remote/runner.py | 3 ++- lilac2/systemd.py | 27 +++++++++++++++++++-------- 2 files changed, 21 insertions(+), 9 deletions(-) diff --git a/lilac2/remote/runner.py b/lilac2/remote/runner.py index 1b26d0ea..5a2366f5 100644 --- a/lilac2/remote/runner.py +++ b/lilac2/remote/runner.py @@ -19,6 +19,7 @@ def main() -> None: name = input.pop('name') deadline = input.pop('deadline') myresultpath = input.pop('result') + worker_no = input['worker_no'] cmd = [ sys.executable, @@ -49,7 +50,7 @@ def main() -> None: p.stdin.write(json.dumps(input).encode()) # type: ignore p.stdin.close() # type: ignore - rusage, _ = systemd.poll_rusage(name, deadline) + rusage, _ = systemd.poll_rusage(name, deadline, worker_no=worker_no) p.wait() with open(resultpath, 'rb') as f: diff --git a/lilac2/systemd.py b/lilac2/systemd.py index b98f0ef7..140394ab 100644 --- a/lilac2/systemd.py +++ b/lilac2/systemd.py @@ -13,12 +13,12 @@ _available = None _check_lock = threading.Lock() -def available() -> bool | dict[str, bool]: +def available(worker_no: Optional[int] = None) -> bool | dict[str, bool]: global _available with _check_lock: if _available is None: - _available = _check_availability() + _available = _check_availability(worker_no) logger.debug('systemd availability: %s', _available) return _available @@ -35,15 +35,21 @@ def _cgroup_cpu_usage(cgroup: str) -> int: return int(l.split()[1]) * 1000 return 0 -def _check_availability() -> bool | dict[str, bool]: +def _check_availability(worker_no: Optional[int]) -> bool | dict[str, bool]: if 'DBUS_SESSION_BUS_ADDRESS' not in os.environ: dbus = f'/run/user/{os.getuid()}/bus' if not os.path.exists(dbus): return False os.environ['DBUS_SESSION_BUS_ADDRESS'] = f'unix:path={dbus}' + + if worker_no is None: + unit_name = 'lilac-check' + else: + unit_name = f'lilac-check-{worker_no}' + p = subprocess.run([ 'systemd-run', '--quiet', '--user', - '--remain-after-exit', '-u', 'lilac-check', 'true', + '--remain-after-exit', '-u', unit_name, 'true', ]) if p.returncode != 0: return False @@ -55,7 +61,7 @@ def _check_availability() -> bool | dict[str, bool]: 'MemoryPeak': None, 'MainPID': None, } - _read_service_int_properties('lilac-check', ps) + _read_service_int_properties(unit_name, ps) if ps['MainPID'] != 0: time.sleep(0.01) continue @@ -66,7 +72,7 @@ def _check_availability() -> bool | dict[str, bool]: return ret finally: - subprocess.run(['systemctl', '--user', 'stop', '--quiet', 'lilac-check']) + subprocess.run(['systemctl', '--user', 'stop', '--quiet', unit_name]) def _read_service_int_properties(name: str, properties: dict[str, Optional[int]]) -> None: cmd = [ @@ -150,7 +156,12 @@ def _poll_cmd(pid: int) -> Generator[None, None, None]: finally: os.close(pidfd) -def poll_rusage(name: str, deadline: float) -> tuple[RUsage, bool]: +def poll_rusage( + name: str, + deadline: float, + worker_no: Optional[int] = None, +) -> tuple[RUsage, bool]: + '''worker_no: for remote.runner and used to make check unit name unique''' timedout = False done_state = ['exited', 'failed'] @@ -174,7 +185,7 @@ def poll_rusage(name: str, deadline: float) -> tuple[RUsage, bool]: nsec = 0 mem_max = 0 - availability = available() + availability = available(worker_no) assert isinstance(availability, dict) for _ in _poll_cmd(pid): if not availability['CPUUsageNSec']: From 57f657356226cbe4f01497aa2e56dcbe56515257 Mon Sep 17 00:00:00 2001 From: lilydjwg Date: Sat, 11 Oct 2025 13:33:54 +0800 Subject: [PATCH 39/46] workerman: more logging --- lilac2/workerman.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lilac2/workerman.py b/lilac2/workerman.py index 35a7ff16..429ef33c 100644 --- a/lilac2/workerman.py +++ b/lilac2/workerman.py @@ -148,6 +148,7 @@ def prepare_batch( pacman_conf: Optional[str], ) -> None: from . import pkgbuild + logger.info('[%s] updating pacman databases', self.name) pkgbuild.update_data(pacman_conf) @override @@ -210,12 +211,14 @@ def prepare_batch( 'python', '-Xno_debug_ranges', '-P', '-m', 'lilac2.pkgbuild', pacman_conf or '', ] + logger.info('[%s] running %s', self.name, sshcmd) subprocess.check_call(sshcmd) sshcmd = self.get_sshcmd_prefix() + [ 'python', '-Xno_debug_ranges', '-P', '-m', 'lilac2.remote.git_pull', f'"{self.repodir}"', ] + logger.info('[%s] running %s', self.name, sshcmd) subprocess.run(sshcmd, check=True) if prerun := self.config.get('prerun'): From b6e865e4ab7c74fa2a915dbff689e99425d0ca1f Mon Sep 17 00:00:00 2001 From: lilydjwg Date: Sat, 11 Oct 2025 13:37:53 +0800 Subject: [PATCH 40/46] remote.git_pull: fix main not called --- lilac2/remote/git_pull.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lilac2/remote/git_pull.py b/lilac2/remote/git_pull.py index 0ea1b084..03161f23 100644 --- a/lilac2/remote/git_pull.py +++ b/lilac2/remote/git_pull.py @@ -11,3 +11,4 @@ def main(): if __name__ == '__main__': wd = sys.argv[1] os.chdir(wd) + main() From 17f4496b2d22863f5925482a62621832fb99e2f9 Mon Sep 17 00:00:00 2001 From: lilydjwg Date: Sat, 11 Oct 2025 14:06:19 +0800 Subject: [PATCH 41/46] tailf-build-log: fix color --- scripts/tailf-build-log | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/tailf-build-log b/scripts/tailf-build-log index 6ca0e4e1..8da827bd 100755 --- a/scripts/tailf-build-log +++ b/scripts/tailf-build-log @@ -18,7 +18,7 @@ FMT = { 'staged': f'[{c(12)}%(ts)s{c(7)}] {c(15)}%(pkgbase)s{c(7)} %(nv_version)s %(action)s{c(7)} as {c(15)}%(pkg_version)s{c(7)} in {c(6)}%(elapsed)s', 'failed': f'[{c(12)}%(ts)s{c(7)}] {c(15)}%(pkgbase)s{c(7)} %(nv_version)s %(action)s{c(7)} to build as {c(15)}%(pkg_version)s{c(7)} in {c(6)}%(elapsed)s', 'skipped': f'[{c(12)}%(ts)s{c(7)}] {c(15)}%(pkgbase)s{c(7)} %(nv_version)s %(action)s{c(7)} because {c(15)}%(msg)s', - '_rusage': f'{c(7)}; CPU time: {c(6)}%(cputime)s{c(7)} (%(cpupercent)s%%{c(7)}), Memory: {c(5)}%(memory)s on %(builder)s\n', + '_rusage': f'{c(7)}; CPU time: {c(6)}%(cputime)s{c(7)} (%(cpupercent)s%%{c(7)}), Memory: {c(5)}%(memory)s{c(7)} on %(builder)s\n', '_batch': f'[{c(12)}%(ts)s{c(7)}] {c(14)}build %(event)s\n', } From 256ca96ced4d517026568f056fd79ef74404ad38 Mon Sep 17 00:00:00 2001 From: lilydjwg Date: Sat, 11 Oct 2025 14:08:27 +0800 Subject: [PATCH 42/46] tailf-build-log: better coloring --- scripts/tailf-build-log | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/scripts/tailf-build-log b/scripts/tailf-build-log index 8da827bd..450e32d5 100755 --- a/scripts/tailf-build-log +++ b/scripts/tailf-build-log @@ -18,7 +18,7 @@ FMT = { 'staged': f'[{c(12)}%(ts)s{c(7)}] {c(15)}%(pkgbase)s{c(7)} %(nv_version)s %(action)s{c(7)} as {c(15)}%(pkg_version)s{c(7)} in {c(6)}%(elapsed)s', 'failed': f'[{c(12)}%(ts)s{c(7)}] {c(15)}%(pkgbase)s{c(7)} %(nv_version)s %(action)s{c(7)} to build as {c(15)}%(pkg_version)s{c(7)} in {c(6)}%(elapsed)s', 'skipped': f'[{c(12)}%(ts)s{c(7)}] {c(15)}%(pkgbase)s{c(7)} %(nv_version)s %(action)s{c(7)} because {c(15)}%(msg)s', - '_rusage': f'{c(7)}; CPU time: {c(6)}%(cputime)s{c(7)} (%(cpupercent)s%%{c(7)}), Memory: {c(5)}%(memory)s{c(7)} on %(builder)s\n', + '_rusage': f'{c(7)}; CPU time: {c(6)}%(cputime)s{c(7)} (%(cpupercent)s%%{c(7)}), Memory: {c(5)}%(memory)s{c(7)} on {c(15)}%(builder)s\n', '_batch': f'[{c(12)}%(ts)s{c(7)}] {c(14)}build %(event)s\n', } @@ -29,7 +29,9 @@ ACTION = { 'skipped': f'{c(3)}skipped', } -N_CORES = os.cpu_count() +N_CORES = { + 'local': os.cpu_count(), +} def color_gradient(v): r = 255 - v * 255 @@ -67,7 +69,8 @@ def pretty_print(log): cpupercent = round(100 * cputime / log['elapsed']) else: cpupercent = 0 - cpupercent = color_gradient(1 - cpupercent / 100 / N_CORES) + str(cpupercent) + n_cores = N_CORES.get(log['builder'], 'local') + cpupercent = color_gradient(1 - cpupercent / 100 / n_cores) + str(cpupercent) args = { 'ts': log['ts'].strftime('%Y-%m-%d %H:%M:%S'), From f4722c425adb8fe8f9e7be4ef3531a1be52438a2 Mon Sep 17 00:00:00 2001 From: lilydjwg Date: Sat, 11 Oct 2025 14:09:37 +0800 Subject: [PATCH 43/46] tailf-build-log: change builder color --- scripts/tailf-build-log | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/tailf-build-log b/scripts/tailf-build-log index 450e32d5..4c9998f9 100755 --- a/scripts/tailf-build-log +++ b/scripts/tailf-build-log @@ -18,7 +18,7 @@ FMT = { 'staged': f'[{c(12)}%(ts)s{c(7)}] {c(15)}%(pkgbase)s{c(7)} %(nv_version)s %(action)s{c(7)} as {c(15)}%(pkg_version)s{c(7)} in {c(6)}%(elapsed)s', 'failed': f'[{c(12)}%(ts)s{c(7)}] {c(15)}%(pkgbase)s{c(7)} %(nv_version)s %(action)s{c(7)} to build as {c(15)}%(pkg_version)s{c(7)} in {c(6)}%(elapsed)s', 'skipped': f'[{c(12)}%(ts)s{c(7)}] {c(15)}%(pkgbase)s{c(7)} %(nv_version)s %(action)s{c(7)} because {c(15)}%(msg)s', - '_rusage': f'{c(7)}; CPU time: {c(6)}%(cputime)s{c(7)} (%(cpupercent)s%%{c(7)}), Memory: {c(5)}%(memory)s{c(7)} on {c(15)}%(builder)s\n', + '_rusage': f'{c(7)}; CPU time: {c(6)}%(cputime)s{c(7)} (%(cpupercent)s%%{c(7)}), Memory: {c(5)}%(memory)s{c(7)} on {c(6)}%(builder)s\n', '_batch': f'[{c(12)}%(ts)s{c(7)}] {c(14)}build %(event)s\n', } From 72e74e25c35d8fb322dd793d129caafc4bc90a35 Mon Sep 17 00:00:00 2001 From: lilydjwg Date: Sat, 11 Oct 2025 19:17:38 +0800 Subject: [PATCH 44/46] tailf-build-log: don't show "on local" builder --- scripts/tailf-build-log | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/scripts/tailf-build-log b/scripts/tailf-build-log index 4c9998f9..9135cd37 100755 --- a/scripts/tailf-build-log +++ b/scripts/tailf-build-log @@ -18,7 +18,7 @@ FMT = { 'staged': f'[{c(12)}%(ts)s{c(7)}] {c(15)}%(pkgbase)s{c(7)} %(nv_version)s %(action)s{c(7)} as {c(15)}%(pkg_version)s{c(7)} in {c(6)}%(elapsed)s', 'failed': f'[{c(12)}%(ts)s{c(7)}] {c(15)}%(pkgbase)s{c(7)} %(nv_version)s %(action)s{c(7)} to build as {c(15)}%(pkg_version)s{c(7)} in {c(6)}%(elapsed)s', 'skipped': f'[{c(12)}%(ts)s{c(7)}] {c(15)}%(pkgbase)s{c(7)} %(nv_version)s %(action)s{c(7)} because {c(15)}%(msg)s', - '_rusage': f'{c(7)}; CPU time: {c(6)}%(cputime)s{c(7)} (%(cpupercent)s%%{c(7)}), Memory: {c(5)}%(memory)s{c(7)} on {c(6)}%(builder)s\n', + '_rusage': f'{c(7)}; CPU time: {c(6)}%(cputime)s{c(7)} (%(cpupercent)s%%{c(7)}), Memory: {c(5)}%(memory)s{c(7)}', '_batch': f'[{c(12)}%(ts)s{c(7)}] {c(14)}build %(event)s\n', } @@ -65,11 +65,12 @@ def pretty_print(log): cputime = 0 memory = 0 + builder = log['builder'] + n_cores = N_CORES.get(builder, N_CORES['local']) if log['elapsed']: cpupercent = round(100 * cputime / log['elapsed']) else: cpupercent = 0 - n_cores = N_CORES.get(log['builder'], 'local') cpupercent = color_gradient(1 - cpupercent / 100 / n_cores) + str(cpupercent) args = { @@ -83,11 +84,14 @@ def pretty_print(log): 'cputime': humantime(cputime), 'cpupercent': cpupercent, 'memory': filesize(memory), - 'builder': log['builder'], + 'builder': builder, } fmt = FMT[result] out = c(7) + fmt % args + FMT['_rusage'] % args + if builder != 'local': + out += f' on {c(6)}{builder}' + out += '\n' if result == 'failed': out += f'{c(8)}{log["msg"][:1000]}\n' sys.stdout.write(out) From dbea91d97aa1f76f4db1da3c851ab7bb8952aeaf Mon Sep 17 00:00:00 2001 From: lilydjwg Date: Sat, 11 Oct 2025 19:25:03 +0800 Subject: [PATCH 45/46] disable hpack debug logging --- lilac2/api.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lilac2/api.py b/lilac2/api.py index 83c67374..e6b81daf 100644 --- a/lilac2/api.py +++ b/lilac2/api.py @@ -39,6 +39,7 @@ UserAgent = 'lilac/0.2b (package auto-build bot, by lilydjwg)' logging.getLogger('httpcore').setLevel(logging.ERROR) +logging.getLogger('hpack').setLevel(logging.ERROR) s = httpx.Client(http2=True) s.headers['User-Agent'] = UserAgent From 8817e83bcb6a4c117d0e35e79535237be7a2bcfe Mon Sep 17 00:00:00 2001 From: lilydjwg Date: Sat, 11 Oct 2025 20:15:29 +0800 Subject: [PATCH 46/46] make remote process's deadline 60s earlier --- lilac2/remote/worker.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lilac2/remote/worker.py b/lilac2/remote/worker.py index dd785721..8cdfb35d 100644 --- a/lilac2/remote/worker.py +++ b/lilac2/remote/worker.py @@ -21,7 +21,8 @@ def main() -> None: logger.debug('[remote.worker] got input: %r', input) workerman = WorkerManager.from_name(config, input.pop('workerman')) worker_no = input['worker_no'] - deadline = input['deadline'] + # make remote process to exit 60s earlier so that we could do some cleanup + deadline = input.pop('deadline') - 60 myresultpath = input.pop('result') remote_r = {'status': 'done', 'version': None}