From 20c0e48594fa65fddf418ae7a2ee55ae14b30f9c Mon Sep 17 00:00:00 2001 From: lilydjwg Date: Fri, 12 Jun 2020 19:10:34 +0800 Subject: [PATCH] download from github actions artifacts and fallback to repo https://github.com/petronny/action-tools/blob/2f2a7a1d547907e564daffb07c67ab3d235f20cc/download-package-from-artifact.sh https://github.com/petronny/action-tools/blob/39fbeed3222d26480f64dcc0b69758d8deb3a7fb/action_tools.py --- lilac2/remote/__init__.py | 0 lilac2/remote/gh_actions.py | 80 +++++++++++++++++++++++++++++++++++++ lilac2/remote/utils.py | 69 ++++++++++++++++++++++++++++++++ lilac2/typing.py | 4 ++ vendor/github.py | 10 ++++- 5 files changed, 162 insertions(+), 1 deletion(-) create mode 100644 lilac2/remote/__init__.py create mode 100644 lilac2/remote/gh_actions.py create mode 100644 lilac2/remote/utils.py diff --git a/lilac2/remote/__init__.py b/lilac2/remote/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/lilac2/remote/gh_actions.py b/lilac2/remote/gh_actions.py new file mode 100644 index 00000000..ac47cb03 --- /dev/null +++ b/lilac2/remote/gh_actions.py @@ -0,0 +1,80 @@ +from __future__ import annotations + +from typing import List, Iterable, Dict, Any, Tuple +from collections import defaultdict +import logging +import os +import zipfile + +from github import GitHub +from archpkg import pkgfile_pat + +from ..typing import DepDesc +from .utils import RepoDownloader, PKG_CACHE_DIR + +logger = logging.getLogger(__name__) + +def _download_to(r: Any, filename: str) -> None: + partfile = filename + '.part' + with open(partfile, 'wb') as fd: + for chunk in r.iter_content(chunk_size=4096): + fd.write(chunk) + os.rename(partfile, filename) + +def _extract_zip(filename: str, pkgname: str) -> str: + logger.info('Extracting pkgname %s from file %s', + pkgname, filename) + dir = os.path.dirname(filename) + with zipfile.ZipFile(filename) as zip: + for name in zip.namelist(): + if pkgfile_pat.match(name): + zip.extract(name, path=dir) + logger.info('Found %s', name) + return os.path.join(dir, name) + raise FileNotFoundError(pkgname) + +def download_packages( + repo: str, + gh: GitHub, + pkgs: Iterable[DepDesc], + repo_name: str, + servers: List[str], +) -> List[str]: + artifacts = [x for x in gh.get_actions_artifacts(repo)] + want = defaultdict(list) + for d in pkgs: + want[d.pkgbase].append(d.pkgname) + + download: Dict[Tuple[str, str], List[str]] = {} + for artifact in artifacts: + if artifact['name'] in want: + download[ + (artifact['id'], artifact['archive_download_url']) + ] = want.pop(artifact['name']) + + files: List[str] = [] + + logger.info('Downloading artifacts: %r', download) + s = gh.session + for (aid, url), pkgnames in download.items(): + filename = os.path.join(PKG_CACHE_DIR, f'{aid}.zip') + if not os.path.exists(filename): + logger.info('Downloading %s', url) + r = s.get(url, stream=True) + _download_to(r, filename) + else: + logger.info('Using cached file for %s', url) + + for pkgname in pkgnames: + files.append(_extract_zip(filename, pkgname)) + + logger.info('Downloading from repo: %r', want) + + pkgnames2: List[str] = [] + for v in want.values(): + pkgnames2.extend(v) + + r = RepoDownloader(repo_name, servers) + files.extend(r.download(pkgnames2)) + + return files diff --git a/lilac2/remote/utils.py b/lilac2/remote/utils.py new file mode 100644 index 00000000..39e2f3c9 --- /dev/null +++ b/lilac2/remote/utils.py @@ -0,0 +1,69 @@ +from __future__ import annotations + +import os +from typing import List +import logging + +import pyalpm + +logger = logging.getLogger(__name__) + +PKG_CACHE_DIR = '/var/cache/pacman/pkg' +# PKG_CACHE_DIR = '/tmp' + +class RepoDownloader: + _last_dl_name = None + _last_dl_percent = 0 + + def __init__(self, name: str, servers: List[str]) -> None: + self.name = name + + dir = os.path.expanduser('~/pacmandb') + os.makedirs(dir, exist_ok=True) + + self._handle = pyalpm.Handle('/', dir) + self._db = db = self._handle.register_syncdb(name, 0) + db.servers = servers + db.update(False) + + def download(self, pkg_names: List[str]) -> List[str]: + logger.info('Downloading packages repo %s: %r', + self.name, pkg_names) + db = self._db + pkgs = [db.get_pkg(p) for p in pkg_names] + + h = self._handle + h.cachedirs = [PKG_CACHE_DIR] + h.dlcb = self.cb_dl + + tx = h.init_transaction(nodeps=True, downloadonly=True) + try: + for pkg in pkgs: + tx.add_pkg(pkg) + tx.prepare() + tx.commit() + finally: + self._last_dl_name = None + self._last_dl_percent = 0 + tx.release() + + return [ + os.path.join(PKG_CACHE_DIR, pkg.filename) + for pkg in pkgs + ] + + def cb_dl(self, filename: str, tx: int, total: int) -> None: + if self._last_dl_name == filename: + if not total: + return + p = tx * 100 // total + print('.' * (p-self._last_dl_percent), + end='', flush=True) + self._last_dl_percent = p + if p == 100: + print('done') + else: + self._last_dl_name = filename + self._last_dl_percent = 0 + print(f'Downloading {filename}', end='', flush=True) + diff --git a/lilac2/typing.py b/lilac2/typing.py index 4e558caf..88cb8802 100644 --- a/lilac2/typing.py +++ b/lilac2/typing.py @@ -45,3 +45,7 @@ def from_email_address( return cls(name, email, github) PkgRel = Union[int, str] + +class DepDesc(NamedTuple): + pkgbase: str + pkgname: str diff --git a/vendor/github.py b/vendor/github.py index 460a4917..cbb82e9e 100644 --- a/vendor/github.py +++ b/vendor/github.py @@ -2,6 +2,7 @@ import datetime import weakref +from typing import Any, Iterator import requestsutils @@ -51,10 +52,17 @@ def get_repo_issues(self, repo, *, state='open', labels=''): r = self.api_request(r.links['next']) yield from (Issue(x, self) for x in r.json()) - def get_user_info(self, username): + def get_user_info(self, username: str) -> Any: r = self.api_request(f'/users/{username}') return r.json() + def get_actions_artifacts(self, repo: str) -> Iterator[Any]: + r = self.api_request(f'/repos/{repo}/actions/artifacts') + yield from r.json()['artifacts'] + while 'next' in r.links: + r = self.api_request(r.links['next']) + yield from r.json()['artifacts'] + class Issue: def __init__(self, data, gh): self.gh = weakref.proxy(gh)