Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit 9e4754f

Browse files
authored
Merge pull request #1616 from pre-commit/zipapp
add zipapp support
2 parents 01f1a00 + fbd5292 commit 9e4754f

5 files changed

Lines changed: 240 additions & 0 deletions

File tree

requirements-dev.txt

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
covdefaults
22
coverage
3+
distlib
34
pytest
45
pytest-env
56
re-assert

testing/zipapp/Dockerfile

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
FROM ubuntu:bionic
2+
RUN : \
3+
&& apt-get update \
4+
&& DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
5+
python3 \
6+
python3-distutils \
7+
python3-venv \
8+
&& apt-get clean \
9+
&& rm -rf /var/lib/apt/lists/*
10+
11+
ENV LANG=C.UTF-8 PATH=/venv/bin:$PATH
12+
RUN : \
13+
&& python3.6 -mvenv /venv \
14+
&& pip install --no-cache-dir pip setuptools wheel no-manylinux --upgrade

testing/zipapp/entry

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
#!/usr/bin/env python3
2+
import os.path
3+
import shutil
4+
import stat
5+
import sys
6+
import tempfile
7+
import zipfile
8+
9+
from pre_commit.file_lock import lock
10+
11+
CACHE_DIR = os.path.expanduser('~/.cache/pre-commit-zipapp')
12+
13+
14+
def _make_executable(filename: str) -> None:
15+
os.chmod(filename, os.stat(filename).st_mode | stat.S_IXUSR)
16+
17+
18+
def _ensure_cache(zipf: zipfile.ZipFile, cache_key: str) -> str:
19+
os.makedirs(CACHE_DIR, exist_ok=True)
20+
21+
cache_dest = os.path.join(CACHE_DIR, cache_key)
22+
lock_filename = os.path.join(CACHE_DIR, f'{cache_key}.lock')
23+
24+
if os.path.exists(cache_dest):
25+
return cache_dest
26+
27+
with lock(lock_filename, blocked_cb=lambda: None):
28+
# another process may have completed this work
29+
if os.path.exists(cache_dest):
30+
return cache_dest
31+
32+
tmpdir = tempfile.mkdtemp(prefix=os.path.join(CACHE_DIR, ''))
33+
try:
34+
zipf.extractall(tmpdir)
35+
# zip doesn't maintain permissions
36+
_make_executable(os.path.join(tmpdir, 'python'))
37+
_make_executable(os.path.join(tmpdir, 'python.exe'))
38+
os.rename(tmpdir, cache_dest)
39+
except BaseException:
40+
shutil.rmtree(tmpdir)
41+
raise
42+
43+
return cache_dest
44+
45+
46+
def main() -> int:
47+
with zipfile.ZipFile(os.path.dirname(__file__)) as zipf:
48+
with zipf.open('CACHE_KEY') as f:
49+
cache_key = f.read().decode().strip()
50+
51+
cache_dest = _ensure_cache(zipf, cache_key)
52+
53+
if sys.platform != 'win32':
54+
exe = os.path.join(cache_dest, 'python')
55+
else:
56+
exe = os.path.join(cache_dest, 'python.exe')
57+
58+
cmd = (exe, '-mpre_commit', *sys.argv[1:])
59+
if sys.platform == 'win32': # https://bugs.python.org/issue19124
60+
import subprocess
61+
62+
if sys.version_info < (3, 7): # https://bugs.python.org/issue25942
63+
return subprocess.Popen(cmd).wait()
64+
else:
65+
return subprocess.call(cmd)
66+
else:
67+
os.execvp(cmd[0], cmd)
68+
69+
70+
if __name__ == '__main__':
71+
exit(main())

testing/zipapp/make

Lines changed: 106 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,106 @@
1+
#!/usr/bin/env python3
2+
import argparse
3+
import base64
4+
import hashlib
5+
import importlib.resources
6+
import io
7+
import os.path
8+
import shutil
9+
import subprocess
10+
import tempfile
11+
import zipapp
12+
import zipfile
13+
14+
HERE = os.path.dirname(os.path.realpath(__file__))
15+
IMG = 'make-pre-commit-zipapp'
16+
17+
18+
def _msg(s: str) -> None:
19+
print(f'\033[7m{s}\033[m')
20+
21+
22+
def _exit_if_retv(*cmd: str) -> None:
23+
if subprocess.call(cmd):
24+
raise SystemExit(1)
25+
26+
27+
def _check_no_shared_objects(wheeldir: str) -> None:
28+
for zip_filename in os.listdir(wheeldir):
29+
with zipfile.ZipFile(os.path.join(wheeldir, zip_filename)) as zipf:
30+
for filename in zipf.namelist():
31+
if filename.endswith('.so') or '.so.' in filename:
32+
raise AssertionError(zip_filename, filename)
33+
34+
35+
def _add_shim(dest: str) -> None:
36+
shim = os.path.join(HERE, 'python')
37+
shutil.copy(shim, dest)
38+
39+
bio = io.BytesIO()
40+
with zipfile.ZipFile(bio, 'w') as zipf:
41+
zipf.write(shim, arcname='__main__.py')
42+
43+
with open(os.path.join(dest, 'python.exe'), 'wb') as f:
44+
f.write(importlib.resources.read_binary('distlib', 't32.exe'))
45+
f.write(b'#!py.exe -3\n')
46+
f.write(bio.getvalue())
47+
48+
49+
def _write_cache_key(version: str, wheeldir: str, dest: str) -> None:
50+
cache_hash = hashlib.sha256(f'{version}\n'.encode())
51+
for filename in sorted(os.listdir(wheeldir)):
52+
cache_hash.update(f'{filename}\n'.encode())
53+
with open(os.path.join(HERE, 'python'), 'rb') as f:
54+
cache_hash.update(f.read())
55+
with open(os.path.join(dest, 'CACHE_KEY'), 'wb') as f:
56+
f.write(base64.urlsafe_b64encode(cache_hash.digest()).rstrip(b'='))
57+
58+
59+
def main() -> int:
60+
parser = argparse.ArgumentParser()
61+
parser.add_argument('version')
62+
args = parser.parse_args()
63+
64+
with tempfile.TemporaryDirectory() as tmpdir:
65+
wheeldir = os.path.join(tmpdir, 'wheels')
66+
os.mkdir(wheeldir)
67+
68+
_msg('building podman image...')
69+
_exit_if_retv('podman', 'build', '-q', '-t', IMG, HERE)
70+
71+
_msg('populating wheels...')
72+
_exit_if_retv(
73+
'podman', 'run', '--rm', '--volume', f'{wheeldir}:/wheels:rw', IMG,
74+
'pip', 'wheel', f'pre_commit=={args.version}',
75+
'--wheel-dir', '/wheels',
76+
)
77+
78+
_msg('validating wheels...')
79+
_check_no_shared_objects(wheeldir)
80+
81+
_msg('adding __main__.py...')
82+
mainfile = os.path.join(tmpdir, '__main__.py')
83+
shutil.copy(os.path.join(HERE, 'entry'), mainfile)
84+
85+
_msg('adding shim...')
86+
_add_shim(tmpdir)
87+
88+
_msg('copying file_lock.py...')
89+
file_lock_py = os.path.join(HERE, '../../pre_commit/file_lock.py')
90+
file_lock_py_dest = os.path.join(tmpdir, 'pre_commit/file_lock.py')
91+
os.makedirs(os.path.dirname(file_lock_py_dest))
92+
shutil.copy(file_lock_py, file_lock_py_dest)
93+
94+
_msg('writing CACHE_KEY...')
95+
_write_cache_key(args.version, wheeldir, tmpdir)
96+
97+
filename = f'pre-commit-{args.version}.pyz'
98+
_msg(f'writing {filename}...')
99+
shebang = '/usr/bin/env python3'
100+
zipapp.create_archive(tmpdir, filename, interpreter=shebang)
101+
102+
return 0
103+
104+
105+
if __name__ == '__main__':
106+
exit(main())

testing/zipapp/python

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
#!/usr/bin/env python3
2+
"""A shim executable to put dependencies on sys.path"""
3+
import argparse
4+
import os.path
5+
import runpy
6+
import sys
7+
8+
# an exe-zipapp will have a __file__ of shim.exe/__main__.py
9+
EXE = __file__ if os.path.isfile(__file__) else os.path.dirname(__file__)
10+
EXE = os.path.realpath(EXE)
11+
HERE = os.path.dirname(EXE)
12+
WHEELDIR = os.path.join(HERE, 'wheels')
13+
SITE_DIRS = frozenset(('dist-packages', 'site-packages'))
14+
15+
16+
def main() -> int:
17+
parser = argparse.ArgumentParser(add_help=False)
18+
parser.add_argument('-m')
19+
args, rest = parser.parse_known_args()
20+
21+
if args.m:
22+
# try and remove site-packages from sys.path so our packages win
23+
sys.path[:] = [
24+
p for p in sys.path
25+
if os.path.split(p)[1] not in SITE_DIRS
26+
]
27+
for wheel in sorted(os.listdir(WHEELDIR)):
28+
sys.path.append(os.path.join(WHEELDIR, wheel))
29+
if args.m == 'pre_commit' or args.m.startswith('pre_commit.'):
30+
sys.executable = EXE
31+
sys.argv[1:] = rest
32+
runpy.run_module(args.m, run_name='__main__', alter_sys=True)
33+
return 0
34+
else:
35+
cmd = (sys.executable, *sys.argv[1:])
36+
if sys.platform == 'win32': # https://bugs.python.org/issue19124
37+
import subprocess
38+
39+
if sys.version_info < (3, 7): # https://bugs.python.org/issue25942
40+
return subprocess.Popen(cmd).wait()
41+
else:
42+
return subprocess.call(cmd)
43+
else:
44+
os.execvp(cmd[0], cmd)
45+
46+
47+
if __name__ == '__main__':
48+
exit(main())

0 commit comments

Comments
 (0)