From a4e6f9b61eb4c2a855f26ec0756857b43bac919c Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Wed, 27 Sep 2023 08:57:26 -0700 Subject: [PATCH 01/40] Set up minimal arg parsing --- .gitignore | 1 + Tools/wasm/wasi.py | 102 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 103 insertions(+) create mode 100644 Tools/wasm/wasi.py diff --git a/.gitignore b/.gitignore index dddf28da016192..1179f7d7635a3d 100644 --- a/.gitignore +++ b/.gitignore @@ -124,6 +124,7 @@ Tools/unicode/data/ /config.status.lineno # hendrikmuhs/ccache-action@v1 /.ccache +/cross-build /platform /profile-clean-stamp /profile-run-stamp diff --git a/Tools/wasm/wasi.py b/Tools/wasm/wasi.py new file mode 100644 index 00000000000000..dac51ba3df8129 --- /dev/null +++ b/Tools/wasm/wasi.py @@ -0,0 +1,102 @@ +import argparse +import contextlib +import os +import pathlib +import subprocess +import sysconfig + +CHECKOUT = pathlib.Path(".").absolute() +CROSS_BUILD_DIR = CHECKOUT / "cross-build" +HOST_TRIPLE = "wasm32-unknown-wasi" + +# - Make sure `Modules/Setup.local` exists +# - Make sure the necessary build tools are installed: +# - `make` +# - `pkg-config` (on Linux) +# - Create the build Python +# - `mkdir -p builddir/build` +# - `pushd builddir/build` +# - Get the build platform +# - Python: `sysconfig.get_config_var("BUILD_GNU_TYPE")` +# - Shell: `../../config.guess` +# - `../../configure -C` +# - `make all` +# - ```PYTHON_VERSION=`./python -c 'import sys; print(f"{sys.version_info.major}.{sys.version_info.minor}")'` ``` +# - `popd` +# - Create the host/WASI Python +# - `mkdir builddir/wasi` +# - `pushd builddir/wasi` +# - `../../Tools/wasm/wasi-env ../../configure -C --host=wasm32-unknown-wasi --build=$(../../config.guess) --with-build-python=../build/python` +# - `CONFIG_SITE=../../Tools/wasm/config.site-wasm32-wasi` +# - `HOSTRUNNER="wasmtime run --mapdir /::$(dirname $(dirname $(pwd))) --env PYTHONPATH=/builddir/wasi/build/lib.wasi-wasm32-$PYTHON_VERSION $(pwd)/python.wasm --"` +# - Maps the source checkout to `/` in the WASI runtime +# - Stdlib gets loaded from `/Lib` +# - Gets `_sysconfigdata__wasi_wasm32-wasi.py` on to `sys.path` via `PYTHONPATH` +# - Set by `wasi-env` +# - `WASI_SDK_PATH` +# - `WASI_SYSROOT`; enough for `--sysroot`? +# - `CC` +# - `CPP` +# - `CXX` +# - `LDSHARED` +# - `AR` +# - `RANLIB` +# - `CFLAGS` +# - `LDFLAGS` +# - `PKG_CONFIG_PATH` +# - `PKG_CONFIG_LIBDIR` +# - `PKG_CONFIG_SYSROOT_DIR` +# - `PATH` +# - `make all` +# - Create `run_wasi.sh` + +def build_platform(): + """The name of the build/host platform.""" + # Can also be found via `config.guess`.` + return sysconfig.get_config_var("BUILD_GNU_TYPE") + + +def compile_host_python(): + """Compile the build/host Python. + + Returns the path to the new interpreter and it's major.minor version. + """ + build_dir = CROSS_BUILD_DIR / "build" + build_dir.mkdir(parents=True, exist_ok=True) + + + with contextlib.chdir(build_dir): + subprocess.check_call([CHECKOUT / "configure", "-C"]) + subprocess.check_call(["make", "all"]) + # XXX + # - ```PYTHON_VERSION=`./python -c 'import sys; print(f"{sys.version_info.major}.{sys.version_info.minor}")'` ``` + # XXX Check if the binary is named `python` on macOS + return build_dir / "python", XXX + + +def find_wasi_sdk(): + """Find the path to wasi-sdk.""" + if wasi_sdk_path := os.environ.get("WASI_SDK_PATH"): + return pathlib.Path(wasi_sdk_path) + elif (default_path := pathlib.Path("/opt/wasi-sdk")).exists(): + return default_path + + +def main(): + parser = argparse.ArgumentParser() + subcommands = parser.add_subparsers(dest="subcommand") + build = subcommands.add_parser("build") + build.add_argument("--wasi-sdk", type=pathlib.Path, dest="wasi_sdk_path", + default=find_wasi_sdk(), + help="Path to wasi-sdk; defaults to " + "$WASI_SDK_PATH or /opt/wasi-sdk") + + args = parser.parse_args() + if not args.wasi_sdk_path or not args.wasi_sdk_path.exists(): + raise ValueError("wasi-sdk not found; see https://github.com/WebAssembly/wasi-sdk") + + compile_host_python() + + +if __name__ == "__main__": + main() From f1be30ac2b675d74e7ccecf9ab6371a153cd4028 Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Wed, 27 Sep 2023 18:23:03 -0700 Subject: [PATCH 02/40] Get the WASI build working --- Tools/wasm/wasi.py | 131 ++++++++++++++++++++++++++++----------------- 1 file changed, 82 insertions(+), 49 deletions(-) diff --git a/Tools/wasm/wasi.py b/Tools/wasm/wasi.py index dac51ba3df8129..ddd75e0b4e0542 100644 --- a/Tools/wasm/wasi.py +++ b/Tools/wasm/wasi.py @@ -2,53 +2,17 @@ import contextlib import os import pathlib +import shutil import subprocess import sysconfig -CHECKOUT = pathlib.Path(".").absolute() +CHECKOUT = pathlib.Path(__file__).parent.parent.parent CROSS_BUILD_DIR = CHECKOUT / "cross-build" -HOST_TRIPLE = "wasm32-unknown-wasi" - -# - Make sure `Modules/Setup.local` exists -# - Make sure the necessary build tools are installed: -# - `make` -# - `pkg-config` (on Linux) -# - Create the build Python -# - `mkdir -p builddir/build` -# - `pushd builddir/build` -# - Get the build platform -# - Python: `sysconfig.get_config_var("BUILD_GNU_TYPE")` -# - Shell: `../../config.guess` -# - `../../configure -C` -# - `make all` -# - ```PYTHON_VERSION=`./python -c 'import sys; print(f"{sys.version_info.major}.{sys.version_info.minor}")'` ``` -# - `popd` -# - Create the host/WASI Python -# - `mkdir builddir/wasi` -# - `pushd builddir/wasi` -# - `../../Tools/wasm/wasi-env ../../configure -C --host=wasm32-unknown-wasi --build=$(../../config.guess) --with-build-python=../build/python` -# - `CONFIG_SITE=../../Tools/wasm/config.site-wasm32-wasi` -# - `HOSTRUNNER="wasmtime run --mapdir /::$(dirname $(dirname $(pwd))) --env PYTHONPATH=/builddir/wasi/build/lib.wasi-wasm32-$PYTHON_VERSION $(pwd)/python.wasm --"` -# - Maps the source checkout to `/` in the WASI runtime -# - Stdlib gets loaded from `/Lib` -# - Gets `_sysconfigdata__wasi_wasm32-wasi.py` on to `sys.path` via `PYTHONPATH` -# - Set by `wasi-env` -# - `WASI_SDK_PATH` -# - `WASI_SYSROOT`; enough for `--sysroot`? -# - `CC` -# - `CPP` -# - `CXX` -# - `LDSHARED` -# - `AR` -# - `RANLIB` -# - `CFLAGS` -# - `LDFLAGS` -# - `PKG_CONFIG_PATH` -# - `PKG_CONFIG_LIBDIR` -# - `PKG_CONFIG_SYSROOT_DIR` -# - `PATH` -# - `make all` -# - Create `run_wasi.sh` +HOST_TRIPLE = "wasm32-wasi" + + +def section(title): + print("#" * 5, title, "#" * 5) def build_platform(): """The name of the build/host platform.""" @@ -64,14 +28,20 @@ def compile_host_python(): build_dir = CROSS_BUILD_DIR / "build" build_dir.mkdir(parents=True, exist_ok=True) + section(build_dir) with contextlib.chdir(build_dir): subprocess.check_call([CHECKOUT / "configure", "-C"]) - subprocess.check_call(["make", "all"]) - # XXX - # - ```PYTHON_VERSION=`./python -c 'import sys; print(f"{sys.version_info.major}.{sys.version_info.minor}")'` ``` + subprocess.check_call(["make", "--jobs", str(os.cpu_count()), "all"]) + + binary = build_dir / "python" + cmd = [binary, "-c", + "import sys; " + "print(f'{sys.version_info.major}.{sys.version_info.minor}')"] + version = subprocess.check_output(cmd, encoding="utf-8").strip() + # XXX Check if the binary is named `python` on macOS - return build_dir / "python", XXX + return binary, version def find_wasi_sdk(): @@ -82,6 +52,67 @@ def find_wasi_sdk(): return default_path +def wasi_sdk_env(context): + """Calculate environment variables for building with wasi-sdk.""" + wasi_sdk_path = context.wasi_sdk_path + sysroot = wasi_sdk_path / "share" / "wasi-sysroot" + env = {"CC": "clang", "CPP": "clang-cpp", "CXX": "clang++", + "LDSHARED": "wasm-ld", "AR": "llvm-ar", "RANLIB": "ranlib"} + + for env_var, binary_name in list(env.items()): + env[env_var] = os.fsdecode(wasi_sdk_path / "bin" / binary_name) + + if wasi_sdk_path != pathlib.Path("/opt/wasi-sdk"): + for compiler in ["CC", "CPP", "CXX"]: + env[compiler] += f" --sysroot={sysroot}" + + env["PKG_CONFIG_PATH"] = "" + env["PKG_CONFIG_LIBDIR"] = os.pathsep.join( + map(os.fsdecode, + [sysroot / "lib" / "pkgconfig", + sysroot / "share" / "pkgconfig"])) + env["PKG_CONFIG_SYSROOT_DIR"] = os.fsdecode(sysroot) + + env["WASI_SDK_PATH"] = os.fsdecode(wasi_sdk_path) + env["WASI_SYSROOT"] = os.fsdecode(sysroot) + + env["PATH"] = os.pathsep.join([os.fsdecode(wasi_sdk_path / "bin"), + os.environ["PATH"]]) + + return env + + +def compile_wasi_python(context, build_python, version): + """Compile the wasm32-wasi Python.""" + build_dir = CROSS_BUILD_DIR / HOST_TRIPLE + build_dir.mkdir(exist_ok=True) + + section(build_dir) + + config_site = os.fsdecode(CHECKOUT / "Tools" / "wasm" / "config.site-wasm32-wasi") + # Map the checkout to / to load the stdlib from /Lib. Also means paths for + # PYTHONPATH to include sysconfig data must be anchored to the WASI + # runtime's / directory. + host_runner = (f"{shutil.which('wasmtime')} run " + f"--mapdir /::{CHECKOUT} " + f"--env PYTHONPATH=/{CROSS_BUILD_DIR.name}/wasi/build/lib.wasi-wasm32-{version} " + f"{build_dir / 'python.wasm'} --") + env_additions = {"CONFIG_SITE": config_site, "HOSTRUNNER": host_runner} + + with contextlib.chdir(build_dir): + # The path to `configure` MUST be relative, else `python.wasm` is unable + # to find the stdlib for unknown reasons. + configure = [os.path.relpath(CHECKOUT / 'configure', build_dir), + "-C", + f"--host={HOST_TRIPLE}", + f"--build={build_platform()}", + f"--with-build-python={build_python}"] + configure_env = os.environ | env_additions | wasi_sdk_env(context) + subprocess.check_call(configure, env=configure_env) + subprocess.check_call(["make", "--jobs", str(os.cpu_count()), "all"], + env=os.environ | env_additions) + + def main(): parser = argparse.ArgumentParser() subcommands = parser.add_subparsers(dest="subcommand") @@ -93,9 +124,11 @@ def main(): args = parser.parse_args() if not args.wasi_sdk_path or not args.wasi_sdk_path.exists(): - raise ValueError("wasi-sdk not found; see https://github.com/WebAssembly/wasi-sdk") + raise ValueError("wasi-sdk not found or specified; " + "see https://github.com/WebAssembly/wasi-sdk") - compile_host_python() + build_python, version = compile_host_python() + compile_wasi_python(args, build_python, version) if __name__ == "__main__": From 7031bcef8b6d002612832a414bd754648ef5ab9b Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Tue, 3 Oct 2023 11:23:40 -0700 Subject: [PATCH 03/40] Create a `python.sh` command --- Tools/wasm/wasi.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/Tools/wasm/wasi.py b/Tools/wasm/wasi.py index ddd75e0b4e0542..9830b59b1e3dff 100644 --- a/Tools/wasm/wasi.py +++ b/Tools/wasm/wasi.py @@ -101,7 +101,8 @@ def compile_wasi_python(context, build_python, version): with contextlib.chdir(build_dir): # The path to `configure` MUST be relative, else `python.wasm` is unable - # to find the stdlib for unknown reasons. + # to find the stdlib due to Python not recognizing that it's being + # executed from within a checkout. configure = [os.path.relpath(CHECKOUT / 'configure', build_dir), "-C", f"--host={HOST_TRIPLE}", @@ -112,6 +113,11 @@ def compile_wasi_python(context, build_python, version): subprocess.check_call(["make", "--jobs", str(os.cpu_count()), "all"], env=os.environ | env_additions) + exec_script = build_dir / "python.sh" + with exec_script.open("w", encoding="utf-8") as file: + file.write(f'#!/bin/sh\nexec {host_runner} "$@"\n') + exec_script.chmod(0o755) + def main(): parser = argparse.ArgumentParser() From 3db23886630b98e3abfbd7832ab1aea280ed5b94 Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Tue, 3 Oct 2023 13:42:09 -0700 Subject: [PATCH 04/40] Switch to `os.process_cpu_count()` --- Tools/wasm/wasi.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Tools/wasm/wasi.py b/Tools/wasm/wasi.py index 9830b59b1e3dff..e325952d42b9c1 100644 --- a/Tools/wasm/wasi.py +++ b/Tools/wasm/wasi.py @@ -32,7 +32,7 @@ def compile_host_python(): with contextlib.chdir(build_dir): subprocess.check_call([CHECKOUT / "configure", "-C"]) - subprocess.check_call(["make", "--jobs", str(os.cpu_count()), "all"]) + subprocess.check_call(["make", "--jobs", str(os.process_cpu_count()), "all"]) binary = build_dir / "python" cmd = [binary, "-c", @@ -110,7 +110,7 @@ def compile_wasi_python(context, build_python, version): f"--with-build-python={build_python}"] configure_env = os.environ | env_additions | wasi_sdk_env(context) subprocess.check_call(configure, env=configure_env) - subprocess.check_call(["make", "--jobs", str(os.cpu_count()), "all"], + subprocess.check_call(["make", "--jobs", str(os.process_cpu_count()), "all"], env=os.environ | env_additions) exec_script = build_dir / "python.sh" From 238eed11879f238d52e799b821d4a56a1f38b2cf Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Tue, 3 Oct 2023 17:42:26 -0700 Subject: [PATCH 05/40] Tweak some formatting --- Tools/wasm/wasi.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Tools/wasm/wasi.py b/Tools/wasm/wasi.py index e325952d42b9c1..0bb6b9a75911b2 100644 --- a/Tools/wasm/wasi.py +++ b/Tools/wasm/wasi.py @@ -12,7 +12,8 @@ def section(title): - print("#" * 5, title, "#" * 5) + print(title, "#" * 20) + def build_platform(): """The name of the build/host platform.""" From 01319de5cc531cbaa20f7df76f5fc3198425f8bf Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Wed, 4 Oct 2023 16:08:32 -0700 Subject: [PATCH 06/40] Add a `--skip-build-python` flag --- Tools/wasm/wasi.py | 30 ++++++++++++++++++++---------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/Tools/wasm/wasi.py b/Tools/wasm/wasi.py index 0bb6b9a75911b2..e2cd78320a579d 100644 --- a/Tools/wasm/wasi.py +++ b/Tools/wasm/wasi.py @@ -1,6 +1,10 @@ import argparse import contextlib import os +try: + from os import process_cpu_count as cpu_count +except ImportError: + from os import cpu_count import pathlib import shutil import subprocess @@ -12,7 +16,8 @@ def section(title): - print(title, "#" * 20) + title = str(title) + print(title, "#" * (79 - len(title))) def build_platform(): @@ -21,7 +26,7 @@ def build_platform(): return sysconfig.get_config_var("BUILD_GNU_TYPE") -def compile_host_python(): +def compile_host_python(context): """Compile the build/host Python. Returns the path to the new interpreter and it's major.minor version. @@ -31,9 +36,11 @@ def compile_host_python(): section(build_dir) - with contextlib.chdir(build_dir): - subprocess.check_call([CHECKOUT / "configure", "-C"]) - subprocess.check_call(["make", "--jobs", str(os.process_cpu_count()), "all"]) + if context.build_python: + with contextlib.chdir(build_dir): + subprocess.check_call([CHECKOUT / "configure", "-C"]) + subprocess.check_call(["make", "--jobs", + str(cpu_count()), "all"]) binary = build_dir / "python" cmd = [binary, "-c", @@ -111,7 +118,7 @@ def compile_wasi_python(context, build_python, version): f"--with-build-python={build_python}"] configure_env = os.environ | env_additions | wasi_sdk_env(context) subprocess.check_call(configure, env=configure_env) - subprocess.check_call(["make", "--jobs", str(os.process_cpu_count()), "all"], + subprocess.check_call(["make", "--jobs", str(cpu_count()), "all"], env=os.environ | env_additions) exec_script = build_dir / "python.sh" @@ -128,14 +135,17 @@ def main(): default=find_wasi_sdk(), help="Path to wasi-sdk; defaults to " "$WASI_SDK_PATH or /opt/wasi-sdk") + build.add_argument("--skip-build-python", action="store_false", + dest="build_python", default=True, + help="Skip building the build/host Python") - args = parser.parse_args() - if not args.wasi_sdk_path or not args.wasi_sdk_path.exists(): + context = parser.parse_args() + if not context.wasi_sdk_path or not context.wasi_sdk_path.exists(): raise ValueError("wasi-sdk not found or specified; " "see https://github.com/WebAssembly/wasi-sdk") - build_python, version = compile_host_python() - compile_wasi_python(args, build_python, version) + build_python, version = compile_host_python(context) + compile_wasi_python(context, build_python, version) if __name__ == "__main__": From 0120446c54d4ae63403372470f65afcbd5b86d59 Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Wed, 4 Oct 2023 16:36:02 -0700 Subject: [PATCH 07/40] Add a `--quiet` flag --- Tools/wasm/wasi.py | 48 ++++++++++++++++++++++++++++++++++++---------- 1 file changed, 38 insertions(+), 10 deletions(-) diff --git a/Tools/wasm/wasi.py b/Tools/wasm/wasi.py index e2cd78320a579d..9c8c0c5d4a5af5 100644 --- a/Tools/wasm/wasi.py +++ b/Tools/wasm/wasi.py @@ -9,15 +9,37 @@ import shutil import subprocess import sysconfig +import tempfile CHECKOUT = pathlib.Path(__file__).parent.parent.parent CROSS_BUILD_DIR = CHECKOUT / "cross-build" HOST_TRIPLE = "wasm32-wasi" -def section(title): - title = str(title) - print(title, "#" * (79 - len(title))) +def section(working_dir): + """Print out a visible section header based on a working directory.""" + print("#" * 80) + print("📁", working_dir) + + +def call(command, *, quiet, **kwargs): + """Execute a command. + + If 'quiet' is true, then redirect stdout and stderr to a temporary file. + """ + print("❯", " ".join(command)) + if not quiet: + stdout = None + stderr = None + else: + stdout = tempfile.NamedTemporaryFile("w", encoding="utf-8", + delete=False, + prefix="cpython-wasi-", + suffix=".log") + stderr = subprocess.STDOUT + print(f"Logging output to {stdout.name} ...") + + subprocess.check_call(command, **kwargs, stdout=stdout, stderr=stderr) def build_platform(): @@ -36,11 +58,12 @@ def compile_host_python(context): section(build_dir) - if context.build_python: + if not context.build_python: + print("Skipped via --skip-build-python ...") + else: with contextlib.chdir(build_dir): - subprocess.check_call([CHECKOUT / "configure", "-C"]) - subprocess.check_call(["make", "--jobs", - str(cpu_count()), "all"]) + call([CHECKOUT / "configure", "-C"], quiet=context.quiet) + call(["make", "--jobs", str(cpu_count()), "all"], quiet=context.quiet) binary = build_dir / "python" cmd = [binary, "-c", @@ -117,9 +140,10 @@ def compile_wasi_python(context, build_python, version): f"--build={build_platform()}", f"--with-build-python={build_python}"] configure_env = os.environ | env_additions | wasi_sdk_env(context) - subprocess.check_call(configure, env=configure_env) - subprocess.check_call(["make", "--jobs", str(cpu_count()), "all"], - env=os.environ | env_additions) + call(configure, env=configure_env, quiet=context.quiet) + call(["make", "--jobs", str(cpu_count()), "all"], + env=os.environ | env_additions, + quiet=context.quiet) exec_script = build_dir / "python.sh" with exec_script.open("w", encoding="utf-8") as file: @@ -138,6 +162,9 @@ def main(): build.add_argument("--skip-build-python", action="store_false", dest="build_python", default=True, help="Skip building the build/host Python") + build.add_argument("--quiet", action="store_true", default=False, + dest="quiet", + help="Redirect output from subprocesses to a log file") context = parser.parse_args() if not context.wasi_sdk_path or not context.wasi_sdk_path.exists(): @@ -145,6 +172,7 @@ def main(): "see https://github.com/WebAssembly/wasi-sdk") build_python, version = compile_host_python(context) + print() compile_wasi_python(context, build_python, version) From 4642afedf5cd52566467d98d895d4a6095c75e09 Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Thu, 5 Oct 2023 13:22:29 -0700 Subject: [PATCH 08/40] Add `SOURCE_DATE_EPOCH` Set to the initial commit of Python. --- Tools/wasm/wasi.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/Tools/wasm/wasi.py b/Tools/wasm/wasi.py index 9c8c0c5d4a5af5..7810708a8c23ad 100644 --- a/Tools/wasm/wasi.py +++ b/Tools/wasm/wasi.py @@ -128,7 +128,11 @@ def compile_wasi_python(context, build_python, version): f"--mapdir /::{CHECKOUT} " f"--env PYTHONPATH=/{CROSS_BUILD_DIR.name}/wasi/build/lib.wasi-wasm32-{version} " f"{build_dir / 'python.wasm'} --") - env_additions = {"CONFIG_SITE": config_site, "HOSTRUNNER": host_runner} + env_additions = {"CONFIG_SITE": config_site, "HOSTRUNNER": host_runner, + # Python's first commit: + # Thu, 09 Aug 1990 14:25:15 +0000 (1990-08-09) + # https://hg.python.org/cpython/rev/3cd033e6b530 + "SOURCE_DATE_EPOCH": "650211915"} with contextlib.chdir(build_dir): # The path to `configure` MUST be relative, else `python.wasm` is unable From 705f7468fa15285716a9e1779478307fcb2f86eb Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Thu, 5 Oct 2023 15:37:36 -0700 Subject: [PATCH 09/40] Add `--debug` --- Tools/wasm/wasi.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/Tools/wasm/wasi.py b/Tools/wasm/wasi.py index 7810708a8c23ad..d51a4e1545375e 100644 --- a/Tools/wasm/wasi.py +++ b/Tools/wasm/wasi.py @@ -58,11 +58,15 @@ def compile_host_python(context): section(build_dir) + configure = [CHECKOUT / "configure", "-C"] + if context.debug: + configure.append("--with-pydebug") + if not context.build_python: print("Skipped via --skip-build-python ...") else: with contextlib.chdir(build_dir): - call([CHECKOUT / "configure", "-C"], quiet=context.quiet) + call(configure, quiet=context.quiet) call(["make", "--jobs", str(cpu_count()), "all"], quiet=context.quiet) binary = build_dir / "python" @@ -143,6 +147,8 @@ def compile_wasi_python(context, build_python, version): f"--host={HOST_TRIPLE}", f"--build={build_platform()}", f"--with-build-python={build_python}"] + if context.debug: + configure.append("--with-pydebug") configure_env = os.environ | env_additions | wasi_sdk_env(context) call(configure, env=configure_env, quiet=context.quiet) call(["make", "--jobs", str(cpu_count()), "all"], @@ -169,6 +175,9 @@ def main(): build.add_argument("--quiet", action="store_true", default=False, dest="quiet", help="Redirect output from subprocesses to a log file") + build.add_argument("--debug", action="store_true", default=False, + dest="debug", + help="Debug build (i.e., pydebug)") context = parser.parse_args() if not context.wasi_sdk_path or not context.wasi_sdk_path.exists(): From 134e6b42768888c7c043ae47ed8ff50c2760ca1f Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Fri, 6 Oct 2023 11:39:53 -0700 Subject: [PATCH 10/40] Fix a bug related to passing a `pathlib.Path` to `section()` --- Tools/wasm/wasi.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tools/wasm/wasi.py b/Tools/wasm/wasi.py index d51a4e1545375e..56923fa384f18e 100644 --- a/Tools/wasm/wasi.py +++ b/Tools/wasm/wasi.py @@ -27,7 +27,7 @@ def call(command, *, quiet, **kwargs): If 'quiet' is true, then redirect stdout and stderr to a temporary file. """ - print("❯", " ".join(command)) + print("❯", " ".join(map(str, command))) if not quiet: stdout = None stderr = None From 95029ee6dc3ddb9b9dbccfe1c2b5c5dfcc9d156e Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Fri, 6 Oct 2023 12:48:08 -0700 Subject: [PATCH 11/40] Rename `--debug` to `--with-pydebug` Want to minimize confusion for core devs. --- Tools/wasm/wasi.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tools/wasm/wasi.py b/Tools/wasm/wasi.py index 56923fa384f18e..a0e7aa291b3cab 100644 --- a/Tools/wasm/wasi.py +++ b/Tools/wasm/wasi.py @@ -175,7 +175,7 @@ def main(): build.add_argument("--quiet", action="store_true", default=False, dest="quiet", help="Redirect output from subprocesses to a log file") - build.add_argument("--debug", action="store_true", default=False, + build.add_argument("--with-pydebug", action="store_true", default=False, dest="debug", help="Debug build (i.e., pydebug)") From 8717974ad5b32f6f3213171822c05adc3298abff Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Fri, 6 Oct 2023 14:32:25 -0700 Subject: [PATCH 12/40] Get everything working with a pydebug build --- Tools/wasm/wasi.py | 39 ++++++++++++++++++++++++++++++++------- 1 file changed, 32 insertions(+), 7 deletions(-) diff --git a/Tools/wasm/wasi.py b/Tools/wasm/wasi.py index a0e7aa291b3cab..b01761a23fb99a 100644 --- a/Tools/wasm/wasi.py +++ b/Tools/wasm/wasi.py @@ -48,6 +48,20 @@ def build_platform(): return sysconfig.get_config_var("BUILD_GNU_TYPE") +def prep_checkout(): + """Prepare the source checkout for cross-compiling.""" + # Without `Setup.local`, in-place execution fails to realize it's in a + # build tree/checkout (the dreaded "No module named 'encodings'" error). + section(CHECKOUT) + + local_setup = CHECKOUT / "Modules" / "Setup.local" + if local_setup.exists(): + print("Modules/Setup.local already exists ...") + else: + print("Touching Modules/Setup.local ...") + local_setup.touch() + + def compile_host_python(context): """Compile the build/host Python. @@ -58,7 +72,7 @@ def compile_host_python(context): section(build_dir) - configure = [CHECKOUT / "configure", "-C"] + configure = [os.path.relpath(CHECKOUT / 'configure', build_dir), "-C"] if context.debug: configure.append("--with-pydebug") @@ -67,15 +81,16 @@ def compile_host_python(context): else: with contextlib.chdir(build_dir): call(configure, quiet=context.quiet) - call(["make", "--jobs", str(cpu_count()), "all"], quiet=context.quiet) + call(["make", "--jobs", str(cpu_count()), "all"], + quiet=context.quiet) + # XXX `python.exe` on Mac binary = build_dir / "python" cmd = [binary, "-c", "import sys; " "print(f'{sys.version_info.major}.{sys.version_info.minor}')"] version = subprocess.check_output(cmd, encoding="utf-8").strip() - # XXX Check if the binary is named `python` on macOS return binary, version @@ -125,12 +140,20 @@ def compile_wasi_python(context, build_python, version): section(build_dir) config_site = os.fsdecode(CHECKOUT / "Tools" / "wasm" / "config.site-wasm32-wasi") - # Map the checkout to / to load the stdlib from /Lib. Also means paths for - # PYTHONPATH to include sysconfig data must be anchored to the WASI - # runtime's / directory. + # Use PYTHONPATH to include sysconfig data (which must be anchored to the + # WASI guest's / directory. + guest_build_dir = build_dir.relative_to(CHECKOUT) + sysconfig_data = f"{guest_build_dir}/build/lib.wasi-wasm32-{version}" + if context.debug: + sysconfig_data += "-pydebug" host_runner = (f"{shutil.which('wasmtime')} run " + # Make sure the stack size will work in a pydebug build. + # The value comes from `ulimit -s` under Linux which is + # 8291 KiB. + "--max-wasm-stack 8388608 " + # Map the checkout to / to load the stdlib from /Lib. f"--mapdir /::{CHECKOUT} " - f"--env PYTHONPATH=/{CROSS_BUILD_DIR.name}/wasi/build/lib.wasi-wasm32-{version} " + f"--env PYTHONPATH=/{sysconfig_data} " f"{build_dir / 'python.wasm'} --") env_additions = {"CONFIG_SITE": config_site, "HOSTRUNNER": host_runner, # Python's first commit: @@ -184,6 +207,8 @@ def main(): raise ValueError("wasi-sdk not found or specified; " "see https://github.com/WebAssembly/wasi-sdk") + prep_checkout() + print() build_python, version = compile_host_python(context) print() compile_wasi_python(context, build_python, version) From d66fd8b1b8a4edd45d79483c795a538552ad7167 Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Tue, 17 Oct 2023 13:40:01 -0700 Subject: [PATCH 13/40] Support macOS --- Tools/wasm/wasi.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/Tools/wasm/wasi.py b/Tools/wasm/wasi.py index b01761a23fb99a..a44b8c42e002bc 100644 --- a/Tools/wasm/wasi.py +++ b/Tools/wasm/wasi.py @@ -84,13 +84,18 @@ def compile_host_python(context): call(["make", "--jobs", str(cpu_count()), "all"], quiet=context.quiet) - # XXX `python.exe` on Mac binary = build_dir / "python" + if not binary.is_file(): + binary = binary.with_suffix(".exe") + if not binary.is_file(): + raise FileNotFoundError(f"Unable to find `python(.exe)` in {build_dir}") cmd = [binary, "-c", "import sys; " "print(f'{sys.version_info.major}.{sys.version_info.minor}')"] version = subprocess.check_output(cmd, encoding="utf-8").strip() + print(f"Python {version} @ {binary}") + return binary, version From b21d59a6d858d7c6caa8a8ea3e3d37667e6b07a5 Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Wed, 18 Oct 2023 11:09:04 -0700 Subject: [PATCH 14/40] Add a verification check for the `python.sh` file Implicity verifies `python.wasm` also works. --- Tools/wasm/wasi.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Tools/wasm/wasi.py b/Tools/wasm/wasi.py index a44b8c42e002bc..e52eaa3c2a60f8 100644 --- a/Tools/wasm/wasi.py +++ b/Tools/wasm/wasi.py @@ -8,6 +8,7 @@ import pathlib import shutil import subprocess +import sys import sysconfig import tempfile @@ -187,6 +188,9 @@ def compile_wasi_python(context, build_python, version): with exec_script.open("w", encoding="utf-8") as file: file.write(f'#!/bin/sh\nexec {host_runner} "$@"\n') exec_script.chmod(0o755) + print(f"Created {exec_script} ... ", end="") + sys.stdout.flush() + subprocess.check_call([exec_script, "--version"]) def main(): From a14348e510f0d9560b6c6735c6bf65407c3c0391 Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Fri, 27 Oct 2023 15:43:52 -0700 Subject: [PATCH 15/40] Allow for `HOST_RUNNER` to be specified as a CLI option --- Tools/wasm/wasi.py | 37 ++++++++++++++++++++++++++++--------- 1 file changed, 28 insertions(+), 9 deletions(-) diff --git a/Tools/wasm/wasi.py b/Tools/wasm/wasi.py index e52eaa3c2a60f8..69d66d88a3cce4 100644 --- a/Tools/wasm/wasi.py +++ b/Tools/wasm/wasi.py @@ -152,15 +152,20 @@ def compile_wasi_python(context, build_python, version): sysconfig_data = f"{guest_build_dir}/build/lib.wasi-wasm32-{version}" if context.debug: sysconfig_data += "-pydebug" - host_runner = (f"{shutil.which('wasmtime')} run " - # Make sure the stack size will work in a pydebug build. - # The value comes from `ulimit -s` under Linux which is - # 8291 KiB. - "--max-wasm-stack 8388608 " - # Map the checkout to / to load the stdlib from /Lib. - f"--mapdir /::{CHECKOUT} " - f"--env PYTHONPATH=/{sysconfig_data} " - f"{build_dir / 'python.wasm'} --") + # host_runner = (f"{shutil.which('wasmtime')} run " + # # Make sure the stack size will work in a pydebug build. + # # The value comes from `ulimit -s` under Linux which is + # # 8291 KiB. + # "--max-wasm-stack 8388608 " + # # Map the checkout to / to load the stdlib from /Lib. + # f"--mapdir /::{CHECKOUT} " + # f"--env PYTHONPATH=/{sysconfig_data} " + # f"{build_dir / 'python.wasm'} --") + host_runner = context.host_runner.format(GUEST_DIR="/", + HOST_DIR=CHECKOUT, + ENV_VAR_NAME="PYTHONPATH", + ENV_VAR_VALUE=f"/{sysconfig_data}", + PYTHON_WASM=build_dir / "python.wasm") env_additions = {"CONFIG_SITE": config_site, "HOSTRUNNER": host_runner, # Python's first commit: # Thu, 09 Aug 1990 14:25:15 +0000 (1990-08-09) @@ -210,6 +215,20 @@ def main(): build.add_argument("--with-pydebug", action="store_true", default=False, dest="debug", help="Debug build (i.e., pydebug)") + default_host_runner = (f"{shutil.which('wasmtime')} run " + # Make sure the stack size will work in a pydebug build. + # The value comes from `ulimit -s` under Linux which is + # 8291 KiB. + "--wasm max-wasm-stack=8388608 " + # Map the checkout to / to load the stdlib from /Lib. + "--dir {HOST_DIR}::{GUEST_DIR} " + "--env {ENV_VAR_NAME}={ENV_VAR_VALUE} " + "{PYTHON_WASM}") + build.add_argument("--host-runner", action="store", + default=default_host_runner, dest="host_runner", + help="Command template for running the WebAssembly code " + "(default for wasmtime 14 or newer: " + f"`{default_host_runner}`)") context = parser.parse_args() if not context.wasi_sdk_path or not context.wasi_sdk_path.exists(): From 57f56fa3a21de09a93fd4ae4b260a574e4bb2262 Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Fri, 3 Nov 2023 12:53:20 -0700 Subject: [PATCH 16/40] Turn on thread support for the default host runner --- Tools/wasm/wasi.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/Tools/wasm/wasi.py b/Tools/wasm/wasi.py index 69d66d88a3cce4..c529603e18f97e 100644 --- a/Tools/wasm/wasi.py +++ b/Tools/wasm/wasi.py @@ -216,14 +216,18 @@ def main(): dest="debug", help="Debug build (i.e., pydebug)") default_host_runner = (f"{shutil.which('wasmtime')} run " - # Make sure the stack size will work in a pydebug build. - # The value comes from `ulimit -s` under Linux which is - # 8291 KiB. - "--wasm max-wasm-stack=8388608 " + # Make sure the stack size will work for a pydebug build. + # The 8388608 value comes from `ulimit -s` under Linux which + # is 8291 KiB. + "--wasm max-wasm-stack=8388608 " + # Enable thread support. + "--wasm threads=y --wasi threads=y " # Map the checkout to / to load the stdlib from /Lib. - "--dir {HOST_DIR}::{GUEST_DIR} " - "--env {ENV_VAR_NAME}={ENV_VAR_VALUE} " - "{PYTHON_WASM}") + "--dir {HOST_DIR}::{GUEST_DIR} " + # Set PYTHONPATH to the sysconfig data. + "--env {ENV_VAR_NAME}={ENV_VAR_VALUE} " + # Path to the WASM binary. + "{PYTHON_WASM}") build.add_argument("--host-runner", action="store", default=default_host_runner, dest="host_runner", help="Command template for running the WebAssembly code " From 94851f69289adb6c5250e4873e6113461c8e1e9a Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Fri, 3 Nov 2023 13:34:31 -0700 Subject: [PATCH 17/40] Add threaded build support --- Tools/wasm/wasi.py | 48 ++++++++++++++++++++++++++++------------------ 1 file changed, 29 insertions(+), 19 deletions(-) diff --git a/Tools/wasm/wasi.py b/Tools/wasm/wasi.py index c529603e18f97e..f3fe34942cdccd 100644 --- a/Tools/wasm/wasi.py +++ b/Tools/wasm/wasi.py @@ -38,7 +38,7 @@ def call(command, *, quiet, **kwargs): prefix="cpython-wasi-", suffix=".log") stderr = subprocess.STDOUT - print(f"Logging output to {stdout.name} ...") + print(f"Logging output to {stdout.name} (--quiet)...") subprocess.check_call(command, **kwargs, stdout=stdout, stderr=stderr) @@ -78,8 +78,12 @@ def compile_host_python(context): configure.append("--with-pydebug") if not context.build_python: - print("Skipped via --skip-build-python ...") + print("Skipping build (--skip-build-python)...") else: + if context.clean: + print(f"Deleting {build_dir} (--clean)...") + shutil.rmtree(build_dir, ignore_errors=True) + with contextlib.chdir(build_dir): call(configure, quiet=context.quiet) call(["make", "--jobs", str(cpu_count()), "all"], @@ -141,10 +145,15 @@ def wasi_sdk_env(context): def compile_wasi_python(context, build_python, version): """Compile the wasm32-wasi Python.""" build_dir = CROSS_BUILD_DIR / HOST_TRIPLE - build_dir.mkdir(exist_ok=True) section(build_dir) + if context.clean: + print(f"Deleting {build_dir} (--clean)...") + shutil.rmtree(build_dir, ignore_errors=True) + + build_dir.mkdir(exist_ok=True) + config_site = os.fsdecode(CHECKOUT / "Tools" / "wasm" / "config.site-wasm32-wasi") # Use PYTHONPATH to include sysconfig data (which must be anchored to the # WASI guest's / directory. @@ -152,15 +161,7 @@ def compile_wasi_python(context, build_python, version): sysconfig_data = f"{guest_build_dir}/build/lib.wasi-wasm32-{version}" if context.debug: sysconfig_data += "-pydebug" - # host_runner = (f"{shutil.which('wasmtime')} run " - # # Make sure the stack size will work in a pydebug build. - # # The value comes from `ulimit -s` under Linux which is - # # 8291 KiB. - # "--max-wasm-stack 8388608 " - # # Map the checkout to / to load the stdlib from /Lib. - # f"--mapdir /::{CHECKOUT} " - # f"--env PYTHONPATH=/{sysconfig_data} " - # f"{build_dir / 'python.wasm'} --") + host_runner = context.host_runner.format(GUEST_DIR="/", HOST_DIR=CHECKOUT, ENV_VAR_NAME="PYTHONPATH", @@ -183,6 +184,8 @@ def compile_wasi_python(context, build_python, version): f"--with-build-python={build_python}"] if context.debug: configure.append("--with-pydebug") + if context.threads: + configure.append("--with-wasm-pthreads") configure_env = os.environ | env_additions | wasi_sdk_env(context) call(configure, env=configure_env, quiet=context.quiet) call(["make", "--jobs", str(cpu_count()), "all"], @@ -202,19 +205,22 @@ def main(): parser = argparse.ArgumentParser() subcommands = parser.add_subparsers(dest="subcommand") build = subcommands.add_parser("build") - build.add_argument("--wasi-sdk", type=pathlib.Path, dest="wasi_sdk_path", - default=find_wasi_sdk(), - help="Path to wasi-sdk; defaults to " - "$WASI_SDK_PATH or /opt/wasi-sdk") - build.add_argument("--skip-build-python", action="store_false", - dest="build_python", default=True, - help="Skip building the build/host Python") build.add_argument("--quiet", action="store_true", default=False, dest="quiet", help="Redirect output from subprocesses to a log file") + build.add_argument("--clean", action="store_true", default=False, + dest="clean", + help="Delete any build directories before building") build.add_argument("--with-pydebug", action="store_true", default=False, dest="debug", help="Debug build (i.e., pydebug)") + build.add_argument("--skip-build-python", action="store_false", + dest="build_python", default=True, + help="Skip building the build/host Python") + build.add_argument("--wasi-sdk", type=pathlib.Path, dest="wasi_sdk_path", + default=find_wasi_sdk(), + help="Path to wasi-sdk; defaults to " + "$WASI_SDK_PATH or /opt/wasi-sdk") default_host_runner = (f"{shutil.which('wasmtime')} run " # Make sure the stack size will work for a pydebug build. # The 8388608 value comes from `ulimit -s` under Linux which @@ -233,6 +239,10 @@ def main(): help="Command template for running the WebAssembly code " "(default for wasmtime 14 or newer: " f"`{default_host_runner}`)") + build.add_argument("--threads", action="store_true", default=False, + dest="threads", + help="Compile with threads support (off by default as " + "thread support is experimental in WASI)") context = parser.parse_args() if not context.wasi_sdk_path or not context.wasi_sdk_path.exists(): From 7060516605df671e86745a9a26953c5f6ce4023e Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Tue, 7 Nov 2023 13:20:03 -0800 Subject: [PATCH 18/40] Check if the expected sysconfig directory was created If it isn't then it's usually a sign that one went to/from a pydebug build from/to a release one --- Tools/wasm/wasi.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Tools/wasm/wasi.py b/Tools/wasm/wasi.py index f3fe34942cdccd..f985d1b2fe0da7 100644 --- a/Tools/wasm/wasi.py +++ b/Tools/wasm/wasi.py @@ -192,6 +192,10 @@ def compile_wasi_python(context, build_python, version): env=os.environ | env_additions, quiet=context.quiet) + if not pathlib.Path(sysconfig_data).exists(): + raise FileNotFoundError(f"Unable to find {sysconfig_data}; " + "check if build Python is a different build type") + exec_script = build_dir / "python.sh" with exec_script.open("w", encoding="utf-8") as file: file.write(f'#!/bin/sh\nexec {host_runner} "$@"\n') From 347d5c0f340c3e3f50fe47f3da701730d1755c84 Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Tue, 7 Nov 2023 13:20:27 -0800 Subject: [PATCH 19/40] Do a better job of deleting build directories when run with `--clean` --- Tools/wasm/wasi.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/Tools/wasm/wasi.py b/Tools/wasm/wasi.py index f985d1b2fe0da7..3f0c7a9034be96 100644 --- a/Tools/wasm/wasi.py +++ b/Tools/wasm/wasi.py @@ -69,7 +69,6 @@ def compile_host_python(context): Returns the path to the new interpreter and it's major.minor version. """ build_dir = CROSS_BUILD_DIR / "build" - build_dir.mkdir(parents=True, exist_ok=True) section(build_dir) @@ -80,9 +79,11 @@ def compile_host_python(context): if not context.build_python: print("Skipping build (--skip-build-python)...") else: - if context.clean: + if context.clean and build_dir.exists(): print(f"Deleting {build_dir} (--clean)...") - shutil.rmtree(build_dir, ignore_errors=True) + shutil.rmtree(build_dir) + + build_dir.mkdir(parents=True, exist_ok=True) with contextlib.chdir(build_dir): call(configure, quiet=context.quiet) @@ -148,9 +149,9 @@ def compile_wasi_python(context, build_python, version): section(build_dir) - if context.clean: + if context.clean and build_dir.exists(): print(f"Deleting {build_dir} (--clean)...") - shutil.rmtree(build_dir, ignore_errors=True) + shutil.rmtree(build_dir) build_dir.mkdir(exist_ok=True) From 0aa5599f63e1df3c93d72939af5de696051ece9b Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Wed, 8 Nov 2023 12:02:51 -0800 Subject: [PATCH 20/40] Make the check for the sysconfig build directory use an absolute path --- Tools/wasm/wasi.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Tools/wasm/wasi.py b/Tools/wasm/wasi.py index 3f0c7a9034be96..34f9cb937c1269 100644 --- a/Tools/wasm/wasi.py +++ b/Tools/wasm/wasi.py @@ -193,7 +193,8 @@ def compile_wasi_python(context, build_python, version): env=os.environ | env_additions, quiet=context.quiet) - if not pathlib.Path(sysconfig_data).exists(): + + if not (CHECKOUT / sysconfig_data).exists(): raise FileNotFoundError(f"Unable to find {sysconfig_data}; " "check if build Python is a different build type") From b1268cee782f54d26ab07b01499b3da13e0abbd5 Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Thu, 16 Nov 2023 12:04:32 -0800 Subject: [PATCH 21/40] Switch to `--platform` from `--skip-build-python` --- Tools/wasm/wasi.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/Tools/wasm/wasi.py b/Tools/wasm/wasi.py index 34f9cb937c1269..80d930bb75a9b7 100644 --- a/Tools/wasm/wasi.py +++ b/Tools/wasm/wasi.py @@ -76,8 +76,8 @@ def compile_host_python(context): if context.debug: configure.append("--with-pydebug") - if not context.build_python: - print("Skipping build (--skip-build-python)...") + if context.platform not in {"all", "build"}: + print("Skipping build (--platform=host)...") else: if context.clean and build_dir.exists(): print(f"Deleting {build_dir} (--clean)...") @@ -220,9 +220,10 @@ def main(): build.add_argument("--with-pydebug", action="store_true", default=False, dest="debug", help="Debug build (i.e., pydebug)") - build.add_argument("--skip-build-python", action="store_false", - dest="build_python", default=True, - help="Skip building the build/host Python") + build.add_argument("--platform", choices=["all", "build", "host"], + default="all", + help="specify which platform(s) to build for " + "(default is 'all')") build.add_argument("--wasi-sdk", type=pathlib.Path, dest="wasi_sdk_path", default=find_wasi_sdk(), help="Path to wasi-sdk; defaults to " @@ -258,8 +259,9 @@ def main(): prep_checkout() print() build_python, version = compile_host_python(context) - print() - compile_wasi_python(context, build_python, version) + if context.platform in {"all", "host"}: + print() + compile_wasi_python(context, build_python, version) if __name__ == "__main__": From c3539c1a598ebe04ae6f2a5679d026020fdc59e5 Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Wed, 22 Nov 2023 12:10:37 -0800 Subject: [PATCH 22/40] Add the `configure-build-python` and `make-build-python` subcommands --- Tools/wasm/wasi.py | 100 +++++++++++++++++++++++++++++++++++++-------- 1 file changed, 83 insertions(+), 17 deletions(-) diff --git a/Tools/wasm/wasi.py b/Tools/wasm/wasi.py index 80d930bb75a9b7..5bf299a3a9015d 100644 --- a/Tools/wasm/wasi.py +++ b/Tools/wasm/wasi.py @@ -19,7 +19,12 @@ def section(working_dir): """Print out a visible section header based on a working directory.""" - print("#" * 80) + try: + tput_output = subprocess.check_output(["tput", "cols"], encoding="utf-8") + terminal_width = int(tput_output.strip()) + except subprocess.CalledProcessError: + terminal_width = 80 + print("#" * terminal_width) print("📁", working_dir) @@ -63,6 +68,50 @@ def prep_checkout(): local_setup.touch() +def configure_build_python(context): + """Configure the build/host Python.""" + + build_dir = CROSS_BUILD_DIR / "build" + + section(build_dir) + + configure = [os.path.relpath(CHECKOUT / 'configure', build_dir)] + if context.args: + configure.extend(context.args) + + if context.clean and build_dir.exists(): + print(f"Deleting {build_dir} (--clean)...") + shutil.rmtree(build_dir) + + build_dir.mkdir(parents=True, exist_ok=True) + + with contextlib.chdir(build_dir): + call(configure, quiet=context.quiet) + + +def make_build_python(context): + """Make/build the build/host Python.""" + build_dir = CROSS_BUILD_DIR / "build" + + section(build_dir) + + with contextlib.chdir(build_dir): + call(["make", "--jobs", str(cpu_count()), "all"], + quiet=context.quiet) + + binary = build_dir / "python" + if not binary.is_file(): + binary = binary.with_suffix(".exe") + if not binary.is_file(): + raise FileNotFoundError(f"Unable to find `python(.exe)` in {build_dir}") + cmd = [binary, "-c", + "import sys; " + "print(f'{sys.version_info.major}.{sys.version_info.minor}')"] + version = subprocess.check_output(cmd, encoding="utf-8").strip() + + print(f"{binary} {version}") + + def compile_host_python(context): """Compile the build/host Python. @@ -210,13 +259,23 @@ def compile_wasi_python(context, build_python, version): def main(): parser = argparse.ArgumentParser() subcommands = parser.add_subparsers(dest="subcommand") - build = subcommands.add_parser("build") - build.add_argument("--quiet", action="store_true", default=False, - dest="quiet", - help="Redirect output from subprocesses to a log file") - build.add_argument("--clean", action="store_true", default=False, - dest="clean", - help="Delete any build directories before building") + build = subcommands.add_parser("build", help="Build everything") + configure_build = subcommands.add_parser("configure-build-python", + help="Run `configure` for the build Python") + make_build = subcommands.add_parser("make-build-python", + help="Run `make` for the build Python") + for subcommand in build, configure_build, make_build: + subcommand.add_argument("--quiet", action="store_true", default=False, + dest="quiet", + help="Redirect output from subprocesses to a log file") + for subcommand in build, configure_build: + subcommand.add_argument("--clean", action="store_true", default=False, + dest="clean", + help="Delete any relevant directories before building") + # configure-build-python + configure_build.add_argument("args", nargs="*", + help="Extra arguments to pass to `configure`") + # build build.add_argument("--with-pydebug", action="store_true", default=False, dest="debug", help="Debug build (i.e., pydebug)") @@ -252,16 +311,23 @@ def main(): "thread support is experimental in WASI)") context = parser.parse_args() - if not context.wasi_sdk_path or not context.wasi_sdk_path.exists(): - raise ValueError("wasi-sdk not found or specified; " - "see https://github.com/WebAssembly/wasi-sdk") - - prep_checkout() - print() - build_python, version = compile_host_python(context) - if context.platform in {"all", "host"}: + if context.subcommand == "configure-build-python": + prep_checkout() + print() + configure_build_python(context) + elif context.subcommand == "make-build-python": + make_build_python(context) + else: + if not context.wasi_sdk_path or not context.wasi_sdk_path.exists(): + raise ValueError("wasi-sdk not found or specified; " + "see https://github.com/WebAssembly/wasi-sdk") + + prep_checkout() print() - compile_wasi_python(context, build_python, version) + build_python, version = compile_host_python(context) + if context.platform in {"all", "host"}: + print() + compile_wasi_python(context, build_python, version) if __name__ == "__main__": From 2a6c541894447528c4f2e83cb5f89419edc2d959 Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Wed, 22 Nov 2023 12:29:42 -0800 Subject: [PATCH 23/40] Create a decorator to handle some boilerplate --- Tools/wasm/wasi.py | 87 ++++++++++++++++++++++++---------------------- 1 file changed, 45 insertions(+), 42 deletions(-) diff --git a/Tools/wasm/wasi.py b/Tools/wasm/wasi.py index 5bf299a3a9015d..deeb1582f3601b 100644 --- a/Tools/wasm/wasi.py +++ b/Tools/wasm/wasi.py @@ -1,5 +1,6 @@ import argparse import contextlib +import functools import os try: from os import process_cpu_count as cpu_count @@ -14,6 +15,7 @@ CHECKOUT = pathlib.Path(__file__).parent.parent.parent CROSS_BUILD_DIR = CHECKOUT / "cross-build" +BUILD_DIR = CROSS_BUILD_DIR / "build" HOST_TRIPLE = "wasm32-wasi" @@ -28,6 +30,25 @@ def section(working_dir): print("📁", working_dir) +def subdir(working_dir): + """Decorator to change to a working directory.""" + def decorator(func): + @functools.wraps(func) + def wrapper(context): + if getattr(context, "clean", None) and working_dir.exists(): + print(f"Deleting {working_dir} (--clean)...") + shutil.rmtree(working_dir) + + working_dir.mkdir(parents=True, exist_ok=True) + section(working_dir) + + with contextlib.chdir(working_dir): + return func(context, working_dir) + + return wrapper + + return decorator + def call(command, *, quiet, **kwargs): """Execute a command. @@ -54,13 +75,12 @@ def build_platform(): return sysconfig.get_config_var("BUILD_GNU_TYPE") -def prep_checkout(): +@subdir(CHECKOUT) +def prep_checkout(context, working_dir): """Prepare the source checkout for cross-compiling.""" # Without `Setup.local`, in-place execution fails to realize it's in a # build tree/checkout (the dreaded "No module named 'encodings'" error). - section(CHECKOUT) - - local_setup = CHECKOUT / "Modules" / "Setup.local" + local_setup = working_dir / "Modules" / "Setup.local" if local_setup.exists(): print("Modules/Setup.local already exists ...") else: @@ -68,42 +88,27 @@ def prep_checkout(): local_setup.touch() -def configure_build_python(context): +@subdir(BUILD_DIR) +def configure_build_python(context, working_dir): """Configure the build/host Python.""" - - build_dir = CROSS_BUILD_DIR / "build" - - section(build_dir) - - configure = [os.path.relpath(CHECKOUT / 'configure', build_dir)] + configure = [os.path.relpath(CHECKOUT / 'configure', working_dir)] if context.args: configure.extend(context.args) - if context.clean and build_dir.exists(): - print(f"Deleting {build_dir} (--clean)...") - shutil.rmtree(build_dir) - - build_dir.mkdir(parents=True, exist_ok=True) - - with contextlib.chdir(build_dir): - call(configure, quiet=context.quiet) + call(configure, quiet=context.quiet) -def make_build_python(context): +@subdir(BUILD_DIR) +def make_build_python(context, working_dir): """Make/build the build/host Python.""" - build_dir = CROSS_BUILD_DIR / "build" - - section(build_dir) + call(["make", "--jobs", str(cpu_count()), "all"], + quiet=context.quiet) - with contextlib.chdir(build_dir): - call(["make", "--jobs", str(cpu_count()), "all"], - quiet=context.quiet) - - binary = build_dir / "python" + binary = working_dir / "python" if not binary.is_file(): binary = binary.with_suffix(".exe") if not binary.is_file(): - raise FileNotFoundError(f"Unable to find `python(.exe)` in {build_dir}") + raise FileNotFoundError(f"Unable to find `python(.exe)` in {working_dir}") cmd = [binary, "-c", "import sys; " "print(f'{sys.version_info.major}.{sys.version_info.minor}')"] @@ -117,33 +122,31 @@ def compile_host_python(context): Returns the path to the new interpreter and it's major.minor version. """ - build_dir = CROSS_BUILD_DIR / "build" - - section(build_dir) + section(BUILD_DIR) - configure = [os.path.relpath(CHECKOUT / 'configure', build_dir), "-C"] + configure = [os.path.relpath(CHECKOUT / 'configure', BUILD_DIR), "-C"] if context.debug: configure.append("--with-pydebug") if context.platform not in {"all", "build"}: print("Skipping build (--platform=host)...") else: - if context.clean and build_dir.exists(): - print(f"Deleting {build_dir} (--clean)...") - shutil.rmtree(build_dir) + if context.clean and BUILD_DIR.exists(): + print(f"Deleting {BUILD_DIR} (--clean)...") + shutil.rmtree(BUILD_DIR) - build_dir.mkdir(parents=True, exist_ok=True) + BUILD_DIR.mkdir(parents=True, exist_ok=True) - with contextlib.chdir(build_dir): + with contextlib.chdir(BUILD_DIR): call(configure, quiet=context.quiet) call(["make", "--jobs", str(cpu_count()), "all"], quiet=context.quiet) - binary = build_dir / "python" + binary = BUILD_DIR / "python" if not binary.is_file(): binary = binary.with_suffix(".exe") if not binary.is_file(): - raise FileNotFoundError(f"Unable to find `python(.exe)` in {build_dir}") + raise FileNotFoundError(f"Unable to find `python(.exe)` in {BUILD_DIR}") cmd = [binary, "-c", "import sys; " "print(f'{sys.version_info.major}.{sys.version_info.minor}')"] @@ -312,7 +315,7 @@ def main(): context = parser.parse_args() if context.subcommand == "configure-build-python": - prep_checkout() + prep_checkout(context) print() configure_build_python(context) elif context.subcommand == "make-build-python": @@ -322,7 +325,7 @@ def main(): raise ValueError("wasi-sdk not found or specified; " "see https://github.com/WebAssembly/wasi-sdk") - prep_checkout() + prep_checkout(context) print() build_python, version = compile_host_python(context) if context.platform in {"all", "host"}: From c7dcc95ae467ad8362736ee5d2c04b593d60a27a Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Wed, 22 Nov 2023 12:38:15 -0800 Subject: [PATCH 24/40] Don't accidentally delete the checkout --- Tools/wasm/wasi.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Tools/wasm/wasi.py b/Tools/wasm/wasi.py index deeb1582f3601b..84d3f067ff8b89 100644 --- a/Tools/wasm/wasi.py +++ b/Tools/wasm/wasi.py @@ -75,12 +75,12 @@ def build_platform(): return sysconfig.get_config_var("BUILD_GNU_TYPE") -@subdir(CHECKOUT) -def prep_checkout(context, working_dir): +# Don't use subdir(); it will delete the checkout! +def prep_checkout(context): """Prepare the source checkout for cross-compiling.""" # Without `Setup.local`, in-place execution fails to realize it's in a # build tree/checkout (the dreaded "No module named 'encodings'" error). - local_setup = working_dir / "Modules" / "Setup.local" + local_setup = CHECKOUT / "Modules" / "Setup.local" if local_setup.exists(): print("Modules/Setup.local already exists ...") else: From 19cbc6ed62e2a5ea40bb39b038ead574f98da45d Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Wed, 22 Nov 2023 12:39:52 -0800 Subject: [PATCH 25/40] Add back in the `section()` call to `prep_checkout()` --- Tools/wasm/wasi.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Tools/wasm/wasi.py b/Tools/wasm/wasi.py index 84d3f067ff8b89..9a192f021b5038 100644 --- a/Tools/wasm/wasi.py +++ b/Tools/wasm/wasi.py @@ -80,6 +80,7 @@ def prep_checkout(context): """Prepare the source checkout for cross-compiling.""" # Without `Setup.local`, in-place execution fails to realize it's in a # build tree/checkout (the dreaded "No module named 'encodings'" error). + section(CHECKOUT) local_setup = CHECKOUT / "Modules" / "Setup.local" if local_setup.exists(): print("Modules/Setup.local already exists ...") From 12a788681225121ab9315d0344f53f21f4f36247 Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Wed, 22 Nov 2023 12:43:18 -0800 Subject: [PATCH 26/40] Make automatic cleaning an opt-in experience --- Tools/wasm/wasi.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/Tools/wasm/wasi.py b/Tools/wasm/wasi.py index 9a192f021b5038..cacf16a31f7e4e 100644 --- a/Tools/wasm/wasi.py +++ b/Tools/wasm/wasi.py @@ -30,12 +30,12 @@ def section(working_dir): print("📁", working_dir) -def subdir(working_dir): +def subdir(working_dir, *, clean_ok=False): """Decorator to change to a working directory.""" def decorator(func): @functools.wraps(func) def wrapper(context): - if getattr(context, "clean", None) and working_dir.exists(): + if clean_ok and context.clean and working_dir.exists(): print(f"Deleting {working_dir} (--clean)...") shutil.rmtree(working_dir) @@ -75,13 +75,12 @@ def build_platform(): return sysconfig.get_config_var("BUILD_GNU_TYPE") -# Don't use subdir(); it will delete the checkout! -def prep_checkout(context): +@subdir(CHECKOUT) +def prep_checkout(context, working_dir): """Prepare the source checkout for cross-compiling.""" # Without `Setup.local`, in-place execution fails to realize it's in a # build tree/checkout (the dreaded "No module named 'encodings'" error). - section(CHECKOUT) - local_setup = CHECKOUT / "Modules" / "Setup.local" + local_setup = working_dir / "Modules" / "Setup.local" if local_setup.exists(): print("Modules/Setup.local already exists ...") else: @@ -89,7 +88,7 @@ def prep_checkout(context): local_setup.touch() -@subdir(BUILD_DIR) +@subdir(BUILD_DIR, clean_ok=True) def configure_build_python(context, working_dir): """Configure the build/host Python.""" configure = [os.path.relpath(CHECKOUT / 'configure', working_dir)] From 62b2da87067aec10d393bea10f222b227c9c3531 Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Wed, 22 Nov 2023 13:37:53 -0800 Subject: [PATCH 27/40] Clobber `cross-build` --- .gitignore | 2 +- Makefile.pre.in | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 87cc93fe675c01..c424a894c2a6e0 100644 --- a/.gitignore +++ b/.gitignore @@ -125,7 +125,7 @@ Tools/unicode/data/ /config.status.lineno # hendrikmuhs/ccache-action@v1 /.ccache -/cross-build +/cross-build/ /platform /profile-clean-stamp /profile-run-stamp diff --git a/Makefile.pre.in b/Makefile.pre.in index 3d766425abba34..e7f8abce43d648 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -2732,6 +2732,7 @@ clobber: clean -rm -rf build platform -rm -rf $(PYTHONFRAMEWORKDIR) -rm -f python-config.py python-config + -rm -rf cross-build # Make things extra clean, before making a distribution: # remove all generated files, even Makefile[.pre] From 789338532f84589ec88448826a01eb683e4c7ca8 Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Thu, 23 Nov 2023 14:13:31 -0800 Subject: [PATCH 28/40] Add `configure-host` --- Tools/wasm/wasi.py | 165 +++++++++++++++++++++++++++++++++++---------- 1 file changed, 129 insertions(+), 36 deletions(-) diff --git a/Tools/wasm/wasi.py b/Tools/wasm/wasi.py index cacf16a31f7e4e..e8aa6dac807de1 100644 --- a/Tools/wasm/wasi.py +++ b/Tools/wasm/wasi.py @@ -13,10 +13,12 @@ import sysconfig import tempfile + CHECKOUT = pathlib.Path(__file__).parent.parent.parent CROSS_BUILD_DIR = CHECKOUT / "cross-build" BUILD_DIR = CROSS_BUILD_DIR / "build" HOST_TRIPLE = "wasm32-wasi" +HOST_DIR = CROSS_BUILD_DIR / HOST_TRIPLE def section(working_dir): @@ -30,6 +32,29 @@ def section(working_dir): print("📁", working_dir) +def updated_env(updates): + """Create a new dict representing the environment to use. + + The changes made to the execution environment are printed out. + """ + # Python's first commit: + # Thu, 09 Aug 1990 14:25:15 +0000 (1990-08-09) + # https://hg.python.org/cpython/rev/3cd033e6b530 + env_defaults = {"SOURCE_DATE_EPOCH": "650211915"} + environment = env_defaults | os.environ | updates + + env_diff = {} + for key, value in environment.items(): + if os.environ.get(key) != value: + env_diff[key] = value + + print("Environment changes:") + for key in sorted(env_diff.keys()): + print(f" {key}={env_diff[key]}") + + return environment + + def subdir(working_dir, *, clean_ok=False): """Decorator to change to a working directory.""" def decorator(func): @@ -49,6 +74,7 @@ def wrapper(context): return decorator + def call(command, *, quiet, **kwargs): """Execute a command. @@ -75,6 +101,17 @@ def build_platform(): return sysconfig.get_config_var("BUILD_GNU_TYPE") +def build_python_path(): + """The path to the build Python binary.""" + binary = BUILD_DIR / "python" + if not binary.is_file(): + binary = binary.with_suffix(".exe") + if not binary.is_file(): + raise FileNotFoundError(f"Unable to find `python(.exe)` in {BUILD_DIR}") + + return binary + + @subdir(CHECKOUT) def prep_checkout(context, working_dir): """Prepare the source checkout for cross-compiling.""" @@ -100,15 +137,11 @@ def configure_build_python(context, working_dir): @subdir(BUILD_DIR) def make_build_python(context, working_dir): - """Make/build the build/host Python.""" + """Make/build the build Python.""" call(["make", "--jobs", str(cpu_count()), "all"], quiet=context.quiet) - binary = working_dir / "python" - if not binary.is_file(): - binary = binary.with_suffix(".exe") - if not binary.is_file(): - raise FileNotFoundError(f"Unable to find `python(.exe)` in {working_dir}") + binary = build_python_path() cmd = [binary, "-c", "import sys; " "print(f'{sys.version_info.major}.{sys.version_info.minor}')"] @@ -118,7 +151,7 @@ def make_build_python(context, working_dir): def compile_host_python(context): - """Compile the build/host Python. + """Compile the build Python. Returns the path to the new interpreter and it's major.minor version. """ @@ -195,6 +228,59 @@ def wasi_sdk_env(context): return env +@subdir(HOST_DIR, clean_ok=True) +def configure_wasi_python(context, working_dir): + """Configure the WASI/host build.""" + if not context.wasi_sdk_path or not context.wasi_sdk_path.exists(): + raise ValueError("wasi-sdk not found or specified; " + "download from https://github.com/WebAssembly/wasi-sdk") + + config_site = os.fsdecode(CHECKOUT / "Tools" / "wasm" / "config.site-wasm32-wasi") + + wasi_build_dir = working_dir.relative_to(CHECKOUT) + + python_build_dir = BUILD_DIR / "build" + lib_dirs = list(python_build_dir.glob("lib.*")) + assert len(lib_dirs) == 1, f"Expected one lib.* directory in {python_build_dir}" + lib_dir = os.fsdecode(lib_dirs[0]) + pydebug = lib_dir.endswith("-pydebug") + python_version = lib_dir.removesuffix("-pydebug").rpartition("-")[-1] + sysconfig_data = f"{wasi_build_dir}/build/lib.wasi-wasm32-{python_version}" + if pydebug: + sysconfig_data += "-pydebug" + + # Use PYTHONPATH to include sysconfig data which must be anchored to the + # WASI guest's `/` directory. + host_runner = context.host_runner.format(GUEST_DIR="/", + HOST_DIR=CHECKOUT, + ENV_VAR_NAME="PYTHONPATH", + ENV_VAR_VALUE=f"/{sysconfig_data}", + PYTHON_WASM=working_dir / "python.wasm") + env_additions = {"CONFIG_SITE": config_site, "HOSTRUNNER": host_runner} + build_python = os.fsdecode(build_python_path()) + # The path to `configure` MUST be relative, else `python.wasm` is unable + # to find the stdlib due to Python not recognizing that it's being + # executed from within a checkout. + configure = [os.path.relpath(CHECKOUT / 'configure', working_dir), + f"--host={HOST_TRIPLE}", + f"--build={build_platform()}", + f"--with-build-python={build_python}"] + if pydebug: + configure.append("--with-pydebug") + if context.args: + configure.extend(context.args) + call(configure, + env=updated_env(env_additions | wasi_sdk_env(context)), + quiet=context.quiet) + + exec_script = working_dir / "python.sh" + with exec_script.open("w", encoding="utf-8") as file: + file.write(f'#!/bin/sh\nexec {host_runner} "$@"\n') + exec_script.chmod(0o755) + print(f"Created {exec_script} for easier execution ... ") + sys.stdout.flush() + + def compile_wasi_python(context, build_python, version): """Compile the wasm32-wasi Python.""" build_dir = CROSS_BUILD_DIR / HOST_TRIPLE @@ -224,7 +310,8 @@ def compile_wasi_python(context, build_python, version): # Python's first commit: # Thu, 09 Aug 1990 14:25:15 +0000 (1990-08-09) # https://hg.python.org/cpython/rev/3cd033e6b530 - "SOURCE_DATE_EPOCH": "650211915"} + "SOURCE_DATE_EPOCH": + os.environ.get("SOURCE_DATE_EPOCH", "650211915")} with contextlib.chdir(build_dir): # The path to `configure` MUST be relative, else `python.wasm` is unable @@ -260,6 +347,20 @@ def compile_wasi_python(context, build_python, version): def main(): + default_host_runner = (f"{shutil.which('wasmtime')} run " + # Make sure the stack size will work for a pydebug build. + # The 8388608 value comes from `ulimit -s` under Linux which + # is 8291 KiB. + "--wasm max-wasm-stack=8388608 " + # Enable thread support. + "--wasm threads=y --wasi threads=y " + # Map the checkout to / to load the stdlib from /Lib. + "--dir {HOST_DIR}::{GUEST_DIR} " + # Set PYTHONPATH to the sysconfig data. + "--env {ENV_VAR_NAME}={ENV_VAR_VALUE} " + # Path to the WASM binary. + "{PYTHON_WASM}") + parser = argparse.ArgumentParser() subcommands = parser.add_subparsers(dest="subcommand") build = subcommands.add_parser("build", help="Build everything") @@ -267,17 +368,29 @@ def main(): help="Run `configure` for the build Python") make_build = subcommands.add_parser("make-build-python", help="Run `make` for the build Python") - for subcommand in build, configure_build, make_build: + configure_host = subcommands.add_parser("configure-host", + help="Run `configure` for the host/WASI") + for subcommand in build, configure_build, make_build, configure_host: subcommand.add_argument("--quiet", action="store_true", default=False, dest="quiet", help="Redirect output from subprocesses to a log file") - for subcommand in build, configure_build: + for subcommand in build, configure_build, configure_host: subcommand.add_argument("--clean", action="store_true", default=False, dest="clean", help="Delete any relevant directories before building") - # configure-build-python - configure_build.add_argument("args", nargs="*", - help="Extra arguments to pass to `configure`") + for subcommand in configure_build, configure_host: + subcommand.add_argument("args", nargs="*", + help="Extra arguments to pass to `configure`") + for subcommand in build, configure_host: + subcommand.add_argument("--wasi-sdk", type=pathlib.Path, dest="wasi_sdk_path", + default=find_wasi_sdk(), + help="Path to wasi-sdk; defaults to " + "$WASI_SDK_PATH or /opt/wasi-sdk") + subcommand.add_argument("--host-runner", action="store", + default=default_host_runner, dest="host_runner", + help="Command template for running the WebAssembly code " + "(default meant for wasmtime 14 or newer: " + f"`{default_host_runner}`)") # build build.add_argument("--with-pydebug", action="store_true", default=False, dest="debug", @@ -286,28 +399,6 @@ def main(): default="all", help="specify which platform(s) to build for " "(default is 'all')") - build.add_argument("--wasi-sdk", type=pathlib.Path, dest="wasi_sdk_path", - default=find_wasi_sdk(), - help="Path to wasi-sdk; defaults to " - "$WASI_SDK_PATH or /opt/wasi-sdk") - default_host_runner = (f"{shutil.which('wasmtime')} run " - # Make sure the stack size will work for a pydebug build. - # The 8388608 value comes from `ulimit -s` under Linux which - # is 8291 KiB. - "--wasm max-wasm-stack=8388608 " - # Enable thread support. - "--wasm threads=y --wasi threads=y " - # Map the checkout to / to load the stdlib from /Lib. - "--dir {HOST_DIR}::{GUEST_DIR} " - # Set PYTHONPATH to the sysconfig data. - "--env {ENV_VAR_NAME}={ENV_VAR_VALUE} " - # Path to the WASM binary. - "{PYTHON_WASM}") - build.add_argument("--host-runner", action="store", - default=default_host_runner, dest="host_runner", - help="Command template for running the WebAssembly code " - "(default for wasmtime 14 or newer: " - f"`{default_host_runner}`)") build.add_argument("--threads", action="store_true", default=False, dest="threads", help="Compile with threads support (off by default as " @@ -315,11 +406,13 @@ def main(): context = parser.parse_args() if context.subcommand == "configure-build-python": - prep_checkout(context) + prep_checkout(context) # TODO: merge w/ configure_build_python() once `build` is removed print() configure_build_python(context) elif context.subcommand == "make-build-python": make_build_python(context) + elif context.subcommand == "configure-host": + configure_wasi_python(context) else: if not context.wasi_sdk_path or not context.wasi_sdk_path.exists(): raise ValueError("wasi-sdk not found or specified; " From 9994841cbf8e4e33ea80d587b3528cab5fdcf15b Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Thu, 23 Nov 2023 15:22:11 -0800 Subject: [PATCH 29/40] Re-implement `build` in terms of all the other subcommands --- Tools/wasm/wasi.py | 210 ++++++++++----------------------------------- 1 file changed, 45 insertions(+), 165 deletions(-) diff --git a/Tools/wasm/wasi.py b/Tools/wasm/wasi.py index e8aa6dac807de1..342ffaf222ba00 100644 --- a/Tools/wasm/wasi.py +++ b/Tools/wasm/wasi.py @@ -21,18 +21,7 @@ HOST_DIR = CROSS_BUILD_DIR / HOST_TRIPLE -def section(working_dir): - """Print out a visible section header based on a working directory.""" - try: - tput_output = subprocess.check_output(["tput", "cols"], encoding="utf-8") - terminal_width = int(tput_output.strip()) - except subprocess.CalledProcessError: - terminal_width = 80 - print("#" * terminal_width) - print("📁", working_dir) - - -def updated_env(updates): +def updated_env(updates={}): """Create a new dict representing the environment to use. The changes made to the execution environment are printed out. @@ -48,7 +37,7 @@ def updated_env(updates): if os.environ.get(key) != value: env_diff[key] = value - print("Environment changes:") + print("🌎 Environment changes:") for key in sorted(env_diff.keys()): print(f" {key}={env_diff[key]}") @@ -60,12 +49,18 @@ def subdir(working_dir, *, clean_ok=False): def decorator(func): @functools.wraps(func) def wrapper(context): + try: + tput_output = subprocess.check_output(["tput", "cols"], encoding="utf-8") + terminal_width = int(tput_output.strip()) + except subprocess.CalledProcessError: + terminal_width = 80 + print("⎯" * terminal_width) + print("📁", working_dir) if clean_ok and context.clean and working_dir.exists(): - print(f"Deleting {working_dir} (--clean)...") + print(f"🚮 Deleting directory (--clean)...") shutil.rmtree(working_dir) working_dir.mkdir(parents=True, exist_ok=True) - section(working_dir) with contextlib.chdir(working_dir): return func(context, working_dir) @@ -90,7 +85,7 @@ def call(command, *, quiet, **kwargs): prefix="cpython-wasi-", suffix=".log") stderr = subprocess.STDOUT - print(f"Logging output to {stdout.name} (--quiet)...") + print(f"📝 Logging output to {stdout.name} (--quiet)...") subprocess.check_call(command, **kwargs, stdout=stdout, stderr=stderr) @@ -112,22 +107,16 @@ def build_python_path(): return binary -@subdir(CHECKOUT) -def prep_checkout(context, working_dir): - """Prepare the source checkout for cross-compiling.""" - # Without `Setup.local`, in-place execution fails to realize it's in a - # build tree/checkout (the dreaded "No module named 'encodings'" error). - local_setup = working_dir / "Modules" / "Setup.local" +@subdir(BUILD_DIR, clean_ok=True) +def configure_build_python(context, working_dir): + """Configure the build/host Python.""" + local_setup = CHECKOUT / "Modules" / "Setup.local" if local_setup.exists(): - print("Modules/Setup.local already exists ...") + print(f"👍 {local_setup} exists ...") else: - print("Touching Modules/Setup.local ...") + print(f"📝 Touching {local_setup} ...") local_setup.touch() - -@subdir(BUILD_DIR, clean_ok=True) -def configure_build_python(context, working_dir): - """Configure the build/host Python.""" configure = [os.path.relpath(CHECKOUT / 'configure', working_dir)] if context.args: configure.extend(context.args) @@ -147,47 +136,7 @@ def make_build_python(context, working_dir): "print(f'{sys.version_info.major}.{sys.version_info.minor}')"] version = subprocess.check_output(cmd, encoding="utf-8").strip() - print(f"{binary} {version}") - - -def compile_host_python(context): - """Compile the build Python. - - Returns the path to the new interpreter and it's major.minor version. - """ - section(BUILD_DIR) - - configure = [os.path.relpath(CHECKOUT / 'configure', BUILD_DIR), "-C"] - if context.debug: - configure.append("--with-pydebug") - - if context.platform not in {"all", "build"}: - print("Skipping build (--platform=host)...") - else: - if context.clean and BUILD_DIR.exists(): - print(f"Deleting {BUILD_DIR} (--clean)...") - shutil.rmtree(BUILD_DIR) - - BUILD_DIR.mkdir(parents=True, exist_ok=True) - - with contextlib.chdir(BUILD_DIR): - call(configure, quiet=context.quiet) - call(["make", "--jobs", str(cpu_count()), "all"], - quiet=context.quiet) - - binary = BUILD_DIR / "python" - if not binary.is_file(): - binary = binary.with_suffix(".exe") - if not binary.is_file(): - raise FileNotFoundError(f"Unable to find `python(.exe)` in {BUILD_DIR}") - cmd = [binary, "-c", - "import sys; " - "print(f'{sys.version_info.major}.{sys.version_info.minor}')"] - version = subprocess.check_output(cmd, encoding="utf-8").strip() - - print(f"Python {version} @ {binary}") - - return binary, version + print(f"🎉 {binary} {version}") def find_wasi_sdk(): @@ -232,8 +181,9 @@ def wasi_sdk_env(context): def configure_wasi_python(context, working_dir): """Configure the WASI/host build.""" if not context.wasi_sdk_path or not context.wasi_sdk_path.exists(): - raise ValueError("wasi-sdk not found or specified; " - "download from https://github.com/WebAssembly/wasi-sdk") + raise ValueError("WASI-SDK not found; " + "download from https://github.com/WebAssembly/wasi-sdk " + "and/or specify via $WASI_SDK_PATH or --wasi-sdk") config_site = os.fsdecode(CHECKOUT / "Tools" / "wasm" / "config.site-wasm32-wasi") @@ -241,7 +191,7 @@ def configure_wasi_python(context, working_dir): python_build_dir = BUILD_DIR / "build" lib_dirs = list(python_build_dir.glob("lib.*")) - assert len(lib_dirs) == 1, f"Expected one lib.* directory in {python_build_dir}" + assert len(lib_dirs) == 1, f"Expected a single lib.* directory in {python_build_dir}" lib_dir = os.fsdecode(lib_dirs[0]) pydebug = lib_dir.endswith("-pydebug") python_version = lib_dir.removesuffix("-pydebug").rpartition("-")[-1] @@ -277,73 +227,25 @@ def configure_wasi_python(context, working_dir): with exec_script.open("w", encoding="utf-8") as file: file.write(f'#!/bin/sh\nexec {host_runner} "$@"\n') exec_script.chmod(0o755) - print(f"Created {exec_script} for easier execution ... ") + print(f"🏃‍♀️ Created {exec_script} ... ") sys.stdout.flush() -def compile_wasi_python(context, build_python, version): - """Compile the wasm32-wasi Python.""" - build_dir = CROSS_BUILD_DIR / HOST_TRIPLE - - section(build_dir) - - if context.clean and build_dir.exists(): - print(f"Deleting {build_dir} (--clean)...") - shutil.rmtree(build_dir) - - build_dir.mkdir(exist_ok=True) - - config_site = os.fsdecode(CHECKOUT / "Tools" / "wasm" / "config.site-wasm32-wasi") - # Use PYTHONPATH to include sysconfig data (which must be anchored to the - # WASI guest's / directory. - guest_build_dir = build_dir.relative_to(CHECKOUT) - sysconfig_data = f"{guest_build_dir}/build/lib.wasi-wasm32-{version}" - if context.debug: - sysconfig_data += "-pydebug" - - host_runner = context.host_runner.format(GUEST_DIR="/", - HOST_DIR=CHECKOUT, - ENV_VAR_NAME="PYTHONPATH", - ENV_VAR_VALUE=f"/{sysconfig_data}", - PYTHON_WASM=build_dir / "python.wasm") - env_additions = {"CONFIG_SITE": config_site, "HOSTRUNNER": host_runner, - # Python's first commit: - # Thu, 09 Aug 1990 14:25:15 +0000 (1990-08-09) - # https://hg.python.org/cpython/rev/3cd033e6b530 - "SOURCE_DATE_EPOCH": - os.environ.get("SOURCE_DATE_EPOCH", "650211915")} - - with contextlib.chdir(build_dir): - # The path to `configure` MUST be relative, else `python.wasm` is unable - # to find the stdlib due to Python not recognizing that it's being - # executed from within a checkout. - configure = [os.path.relpath(CHECKOUT / 'configure', build_dir), - "-C", - f"--host={HOST_TRIPLE}", - f"--build={build_platform()}", - f"--with-build-python={build_python}"] - if context.debug: - configure.append("--with-pydebug") - if context.threads: - configure.append("--with-wasm-pthreads") - configure_env = os.environ | env_additions | wasi_sdk_env(context) - call(configure, env=configure_env, quiet=context.quiet) - call(["make", "--jobs", str(cpu_count()), "all"], - env=os.environ | env_additions, +@subdir(HOST_DIR) +def make_wasi_python(context, working_dir): + """Run `make` for the WASI/host build.""" + call(["make", "--jobs", str(cpu_count()), "all"], + env=updated_env(), quiet=context.quiet) + exec_script = working_dir / "python.sh" + subprocess.check_call([exec_script, "--version"]) - if not (CHECKOUT / sysconfig_data).exists(): - raise FileNotFoundError(f"Unable to find {sysconfig_data}; " - "check if build Python is a different build type") - exec_script = build_dir / "python.sh" - with exec_script.open("w", encoding="utf-8") as file: - file.write(f'#!/bin/sh\nexec {host_runner} "$@"\n') - exec_script.chmod(0o755) - print(f"Created {exec_script} ... ", end="") - sys.stdout.flush() - subprocess.check_call([exec_script, "--version"]) +def build_all(context): + """Build everything.""" + for step in configure_build_python, make_build_python, configure_wasi_python, make_wasi_python: + step(context) def main(): @@ -370,7 +272,9 @@ def main(): help="Run `make` for the build Python") configure_host = subcommands.add_parser("configure-host", help="Run `configure` for the host/WASI") - for subcommand in build, configure_build, make_build, configure_host: + make_host = subcommands.add_parser("make-host", + help="Run `make` for the host/WASI") + for subcommand in build, configure_build, make_build, configure_host, make_host: subcommand.add_argument("--quiet", action="store_true", default=False, dest="quiet", help="Redirect output from subprocesses to a log file") @@ -378,7 +282,7 @@ def main(): subcommand.add_argument("--clean", action="store_true", default=False, dest="clean", help="Delete any relevant directories before building") - for subcommand in configure_build, configure_host: + for subcommand in build, configure_build, configure_host: subcommand.add_argument("args", nargs="*", help="Extra arguments to pass to `configure`") for subcommand in build, configure_host: @@ -391,39 +295,15 @@ def main(): help="Command template for running the WebAssembly code " "(default meant for wasmtime 14 or newer: " f"`{default_host_runner}`)") - # build - build.add_argument("--with-pydebug", action="store_true", default=False, - dest="debug", - help="Debug build (i.e., pydebug)") - build.add_argument("--platform", choices=["all", "build", "host"], - default="all", - help="specify which platform(s) to build for " - "(default is 'all')") - build.add_argument("--threads", action="store_true", default=False, - dest="threads", - help="Compile with threads support (off by default as " - "thread support is experimental in WASI)") context = parser.parse_args() - if context.subcommand == "configure-build-python": - prep_checkout(context) # TODO: merge w/ configure_build_python() once `build` is removed - print() - configure_build_python(context) - elif context.subcommand == "make-build-python": - make_build_python(context) - elif context.subcommand == "configure-host": - configure_wasi_python(context) - else: - if not context.wasi_sdk_path or not context.wasi_sdk_path.exists(): - raise ValueError("wasi-sdk not found or specified; " - "see https://github.com/WebAssembly/wasi-sdk") - - prep_checkout(context) - print() - build_python, version = compile_host_python(context) - if context.platform in {"all", "host"}: - print() - compile_wasi_python(context, build_python, version) + + dispatch = {"configure-build-python": configure_build_python, + "make-build-python": make_build_python, + "configure-host": configure_wasi_python, + "make-host": make_wasi_python, + "build": build_all} + dispatch[context.subcommand](context) if __name__ == "__main__": From d975c4776f84716e4dc6a4dfa35afab46cab0c32 Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Thu, 23 Nov 2023 16:08:38 -0800 Subject: [PATCH 30/40] Touch up formatting --- Tools/wasm/wasi.py | 45 +++++++++++++++++++++++++++++---------------- 1 file changed, 29 insertions(+), 16 deletions(-) diff --git a/Tools/wasm/wasi.py b/Tools/wasm/wasi.py index 342ffaf222ba00..fa1b292b344a8a 100644 --- a/Tools/wasm/wasi.py +++ b/Tools/wasm/wasi.py @@ -1,3 +1,5 @@ +#!/usr/bin/env python3 + import argparse import contextlib import functools @@ -50,7 +52,8 @@ def decorator(func): @functools.wraps(func) def wrapper(context): try: - tput_output = subprocess.check_output(["tput", "cols"], encoding="utf-8") + tput_output = subprocess.check_output(["tput", "cols"], + encoding="utf-8") terminal_width = int(tput_output.strip()) except subprocess.CalledProcessError: terminal_width = 80 @@ -102,7 +105,8 @@ def build_python_path(): if not binary.is_file(): binary = binary.with_suffix(".exe") if not binary.is_file(): - raise FileNotFoundError(f"Unable to find `python(.exe)` in {BUILD_DIR}") + raise FileNotFoundError("Unable to find `python(.exe)` in " + f"{BUILD_DIR}") return binary @@ -182,8 +186,9 @@ def configure_wasi_python(context, working_dir): """Configure the WASI/host build.""" if not context.wasi_sdk_path or not context.wasi_sdk_path.exists(): raise ValueError("WASI-SDK not found; " - "download from https://github.com/WebAssembly/wasi-sdk " - "and/or specify via $WASI_SDK_PATH or --wasi-sdk") + "download from " + "https://github.com/WebAssembly/wasi-sdk and/or " + "specify via $WASI_SDK_PATH or --wasi-sdk") config_site = os.fsdecode(CHECKOUT / "Tools" / "wasm" / "config.site-wasm32-wasi") @@ -244,15 +249,18 @@ def make_wasi_python(context, working_dir): def build_all(context): """Build everything.""" - for step in configure_build_python, make_build_python, configure_wasi_python, make_wasi_python: + steps = [configure_build_python, make_build_python, configure_wasi_python, + make_wasi_python] + for step in steps: step(context) def main(): default_host_runner = (f"{shutil.which('wasmtime')} run " - # Make sure the stack size will work for a pydebug build. - # The 8388608 value comes from `ulimit -s` under Linux which - # is 8291 KiB. + # Make sure the stack size will work for a pydebug + # build. + # The 8388608 value comes from `ulimit -s` under Linux + # which equates to 8291 KiB. "--wasm max-wasm-stack=8388608 " # Enable thread support. "--wasm threads=y --wasi threads=y " @@ -267,11 +275,15 @@ def main(): subcommands = parser.add_subparsers(dest="subcommand") build = subcommands.add_parser("build", help="Build everything") configure_build = subcommands.add_parser("configure-build-python", - help="Run `configure` for the build Python") + help="Run `configure` for the " + "build Python") make_build = subcommands.add_parser("make-build-python", help="Run `make` for the build Python") configure_host = subcommands.add_parser("configure-host", - help="Run `configure` for the host/WASI") + help="Run `configure` for the " + "host/WASI (pydebug builds " + "are inferred from the build " + "Python)") make_host = subcommands.add_parser("make-host", help="Run `make` for the host/WASI") for subcommand in build, configure_build, make_build, configure_host, make_host: @@ -286,14 +298,15 @@ def main(): subcommand.add_argument("args", nargs="*", help="Extra arguments to pass to `configure`") for subcommand in build, configure_host: - subcommand.add_argument("--wasi-sdk", type=pathlib.Path, dest="wasi_sdk_path", - default=find_wasi_sdk(), - help="Path to wasi-sdk; defaults to " - "$WASI_SDK_PATH or /opt/wasi-sdk") + subcommand.add_argument("--wasi-sdk", type=pathlib.Path, + dest="wasi_sdk_path", + default=find_wasi_sdk(), + help="Path to wasi-sdk; defaults to " + "$WASI_SDK_PATH or /opt/wasi-sdk") subcommand.add_argument("--host-runner", action="store", default=default_host_runner, dest="host_runner", - help="Command template for running the WebAssembly code " - "(default meant for wasmtime 14 or newer: " + help="Command template for running the WebAssembly " + "code (default meant for wasmtime 14 or newer: " f"`{default_host_runner}`)") context = parser.parse_args() From 27702ca6d12572179921b359414cc5ce0af18d23 Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Thu, 23 Nov 2023 16:20:02 -0800 Subject: [PATCH 31/40] Update README --- Tools/wasm/README.md | 102 ++++++++++--------------------------------- 1 file changed, 24 insertions(+), 78 deletions(-) diff --git a/Tools/wasm/README.md b/Tools/wasm/README.md index 8ef63c6dcd9ddc..9f7c653e014c44 100644 --- a/Tools/wasm/README.md +++ b/Tools/wasm/README.md @@ -298,100 +298,44 @@ AddType application/wasm wasm ## WASI (wasm32-wasi) -WASI builds require the [WASI SDK](https://github.com/WebAssembly/wasi-sdk) 16.0+. -See `.devcontainer/Dockerfile` for an example of how to download and -install the WASI SDK. +**NOTE**: The instructions below assume a Unix-based OS due to cross-compilation for CPython being set up for `./configure`. -### Build - -The script ``wasi-env`` sets necessary compiler and linker flags as well as -``pkg-config`` overrides. The script assumes that WASI-SDK is installed in -``/opt/wasi-sdk`` or ``$WASI_SDK_PATH``. +### Prerequisites -There are two scripts you can use to do a WASI build from a source checkout. You can either use: +Developing WASI requires two things to be installed: -```shell -./Tools/wasm/wasm_build.py wasi build -``` - -or: -```shell -./Tools/wasm/build_wasi.sh -``` +1. [WASI SDK](https://github.com/WebAssembly/wasi-sdk) 16.0+ + (see `.devcontainer/Dockerfile` for an example of how to download and install the WASI SDK) +2. A WASI host/runtime ([wasmtime](https://wasmtime.dev) is recommended) -The commands are equivalent to the following steps: - -- Make sure `Modules/Setup.local` exists -- Make sure the necessary build tools are installed: - - [WASI SDK](https://github.com/WebAssembly/wasi-sdk) (which ships with `clang`) - - `make` - - `pkg-config` (on Linux) -- Create the build Python - - `mkdir -p builddir/build` - - `pushd builddir/build` - - Get the build platform - - Python: `sysconfig.get_config_var("BUILD_GNU_TYPE")` - - Shell: `../../config.guess` - - `../../configure -C` - - `make all` - - ```PYTHON_VERSION=`./python -c 'import sys; print(f"{sys.version_info.major}.{sys.version_info.minor}")'` ``` - - `popd` -- Create the host/WASI Python - - `mkdir builddir/wasi` - - `pushd builddir/wasi` - - `../../Tools/wasm/wasi-env ../../configure -C --host=wasm32-unknown-wasi --build=$(../../config.guess) --with-build-python=../build/python` - - `CONFIG_SITE=../../Tools/wasm/config.site-wasm32-wasi` - - `HOSTRUNNER="wasmtime run --mapdir /::$(dirname $(dirname $(pwd))) --env PYTHONPATH=/builddir/wasi/build/lib.wasi-wasm32-$PYTHON_VERSION $(pwd)/python.wasm --"` - - Maps the source checkout to `/` in the WASI runtime - - Stdlib gets loaded from `/Lib` - - Gets `_sysconfigdata__wasi_wasm32-wasi.py` on to `sys.path` via `PYTHONPATH` - - Set by `wasi-env` - - `WASI_SDK_PATH` - - `WASI_SYSROOT` - - `CC` - - `CPP` - - `CXX` - - `LDSHARED` - - `AR` - - `RANLIB` - - `CFLAGS` - - `LDFLAGS` - - `PKG_CONFIG_PATH` - - `PKG_CONFIG_LIBDIR` - - `PKG_CONFIG_SYSROOT_DIR` - - `PATH` - - `make all` +### Building -### Running +Building for WASI requires doing a cross-build where you have a "build" Python and then your WASI build (technically it's a "host x host" cross-build because the build Python is also the target Python while the host build is the WASI build). What this leads to is you building two separate copies of Python. In the end you should have a build Python in `cross-build/build` and `cross-build/wasm32-wasi`. -If you followed the instructions above, you can run the interpreter via e.g., `wasmtime` from within the `Tools/wasi` directory (make sure to set/change `$PYTHON_VERSION` and do note the paths are relative to running in`builddir/wasi` for simplicity only): +The easiest way to do a build is to use the `wasi.py` script. You can either have it perform the entire build process from start to finish in one command, or you can do it in discrete steps (which later on are beneficial when you only need to do a specific step). +To do it in a single command, run: ```shell -wasmtime run --mapdir /::../.. --env PYTHONPATH=/builddir/wasi/build/lib.wasi-wasm32-$PYTHON_VERSION python.wasm -- +python Tools/wasm/wasi.py build ``` -There are also helpers provided by `Tools/wasm/wasm_build.py` as listed below. Also, if you used `Tools/wasm/build_wasi.sh`, a `run_wasi.sh` file will be created in `builddir/wasi` which will run the above command for you (it also uses absolute paths, so it can be executed from anywhere). +That will: -#### REPL +1. Run `configure` for the build Python (same as `wasi.py configure-build-python`) +2. Run `make` for the build Python (`wasi.py make-build-python`) +3. Run `configure` for the WASI build (`wasi.py configure-host`) +4. Run `make` for the WASI build (`wasi.py make-host`) -```shell -./Tools/wasm/wasm_build.py wasi repl -``` +See the `--help` for the various options available for any of the subcommands. Extra options passed in after `build --` will be given to **both** `configure` commands (e.g., `build -- --with-pydebug` for a pydebug build). This also applies to `configure-build-python` and `configure-host`. -#### Tests +The output from the script should clearly document what it is going on so that you can replicate any step manually if desired. But there are some included niceties like inferring a pydebug build for the host/WASI build based on the build Python which make using `wasi.py` a bit easier. -```shell -./Tools/wasm/wasm_build.py wasi test -``` -### Debugging +### Running + +If you used `wasi.py` then there will be a `cross-build/wasm32-wasi/python.sh` file which you can use to run the WASI build. While there is a `python.wasm`, various details to run from a checkout need to be set up -- e.g. `PYTHONPATH` for `sysconfig` data -- which are annoying to do manually. -* ``wasmtime run -g`` generates debugging symbols for gdb and lldb. The - feature is currently broken, see - https://github.com/bytecodealliance/wasmtime/issues/4669 . -* The environment variable ``RUST_LOG=wasi_common`` enables debug and - trace logging. ## Detect WebAssembly builds @@ -402,15 +346,17 @@ import os, sys if sys.platform == "emscripten": # Python on Emscripten + ... if sys.platform == "wasi": # Python on WASI + ... if os.name == "posix": # WASM platforms identify as POSIX-like. # Windows does not provide os.uname(). machine = os.uname().machine if machine.startswith("wasm"): - # WebAssembly (wasm32, wasm64 in the future) + # WebAssembly (wasm32, wasm64 potentially in the future) ``` ```python From c8cbd7a0356163ec7cc21208427fa3cff47a1de3 Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Mon, 27 Nov 2023 13:34:07 -0800 Subject: [PATCH 32/40] Set `SOURCE_DATE_EPOCH` based on `git log -1 --pretty=%ct` Command from https://reproducible-builds.org/docs/source-date-epoch/ . --- Tools/wasm/wasi.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/Tools/wasm/wasi.py b/Tools/wasm/wasi.py index fa1b292b344a8a..4aeddda81aacb2 100644 --- a/Tools/wasm/wasi.py +++ b/Tools/wasm/wasi.py @@ -28,10 +28,14 @@ def updated_env(updates={}): The changes made to the execution environment are printed out. """ - # Python's first commit: - # Thu, 09 Aug 1990 14:25:15 +0000 (1990-08-09) - # https://hg.python.org/cpython/rev/3cd033e6b530 - env_defaults = {"SOURCE_DATE_EPOCH": "650211915"} + env_defaults = {} + # https://reproducible-builds.org/docs/source-date-epoch/ + git_epoch_cmd = ["git", "log", "-1", "--pretty=%ct"] + try: + epoch = subprocess.check_output(git_epoch_cmd, encoding="utf-8").strip() + env_defaults["SOURCE_DATE_EPOCH"] = epoch + except subprocess.CalledProcessError: + pass # Might be building from a tarball. environment = env_defaults | os.environ | updates env_diff = {} From e6d947a4ac2143e99de013b42e5007b6d81ae8fb Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Mon, 27 Nov 2023 13:44:20 -0800 Subject: [PATCH 33/40] Flesh out the README --- Tools/wasm/README.md | 25 +++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/Tools/wasm/README.md b/Tools/wasm/README.md index 9f7c653e014c44..9808f2ce95528d 100644 --- a/Tools/wasm/README.md +++ b/Tools/wasm/README.md @@ -313,7 +313,15 @@ Developing WASI requires two things to be installed: Building for WASI requires doing a cross-build where you have a "build" Python and then your WASI build (technically it's a "host x host" cross-build because the build Python is also the target Python while the host build is the WASI build). What this leads to is you building two separate copies of Python. In the end you should have a build Python in `cross-build/build` and `cross-build/wasm32-wasi`. -The easiest way to do a build is to use the `wasi.py` script. You can either have it perform the entire build process from start to finish in one command, or you can do it in discrete steps (which later on are beneficial when you only need to do a specific step). +The easiest way to do a build is to use the `wasi.py` script. You can either have it perform the entire build process from start to finish in one command, or you can do it in discrete steps (which are beneficial when you only need to do a specific step after getting a complete build). + +The discrete steps are: +```shell +python Tools/wasm/wasi.py configure-build-python +python Tools/wasm/wasi.py make-build-python +python Tools/wasm/wasi.py configure-host +python Tools/wasm/wasi.py make-host +``` To do it in a single command, run: ```shell @@ -327,14 +335,23 @@ That will: 3. Run `configure` for the WASI build (`wasi.py configure-host`) 4. Run `make` for the WASI build (`wasi.py make-host`) -See the `--help` for the various options available for any of the subcommands. Extra options passed in after `build --` will be given to **both** `configure` commands (e.g., `build -- --with-pydebug` for a pydebug build). This also applies to `configure-build-python` and `configure-host`. +See the `--help` for the various options available for any of the subcommands which can control things like the location of the WASI SDK, the command to use with the WASI host/runtime, etc. Also note that you can use `--` as a separtor for any of the `configure`-related commands -- including `build` -- to pass arguments to `configure` itself. As an example, if you want a pydebug command that also caches the results from `configure`, you can do: +```shell +python Tools/wasm/wasi.py build -- -C --with-pydebug +``` -The output from the script should clearly document what it is going on so that you can replicate any step manually if desired. But there are some included niceties like inferring a pydebug build for the host/WASI build based on the build Python which make using `wasi.py` a bit easier. +The `wasi.py` script is able to infer details from the build Python, and so you only technically need to specify `--with-pydebug` once for `configure-build-python` and `configure-host` will detect its use: +```shell +python Tools/wasm/wasi.py configure-build-python -- -C --with-pydebug +python Tools/wasm/wasi.py make-build-python +python Tools/wasm/wasi.py configure-host -- -C +python Tools/wasm/wasi.py make-host +``` ### Running -If you used `wasi.py` then there will be a `cross-build/wasm32-wasi/python.sh` file which you can use to run the WASI build. While there is a `python.wasm`, various details to run from a checkout need to be set up -- e.g. `PYTHONPATH` for `sysconfig` data -- which are annoying to do manually. +If you used `wasi.py` then there will be a `cross-build/wasm32-wasi/python.sh` file which you can use to run the WASI build (see the output from the `configure-host` subcommand). While there is a `python.wasm` file, you can't run it naively as there are various things need to be set (e.g. `PYTHONPATH` for `sysconfig` data). As such, the `python.sh` file handles these details for you. ## Detect WebAssembly builds From e160bafd6bab8f020419bffae45a2f85a0498a5e Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Mon, 27 Nov 2023 13:53:05 -0800 Subject: [PATCH 34/40] Some more README flourishes --- Tools/wasm/README.md | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/Tools/wasm/README.md b/Tools/wasm/README.md index 9808f2ce95528d..acbece214ed9bd 100644 --- a/Tools/wasm/README.md +++ b/Tools/wasm/README.md @@ -302,18 +302,18 @@ AddType application/wasm wasm ### Prerequisites -Developing WASI requires two things to be installed: +Developing for WASI requires two things to be installed: -1. [WASI SDK](https://github.com/WebAssembly/wasi-sdk) 16.0+ +1. The [WASI SDK](https://github.com/WebAssembly/wasi-sdk) 16.0+ (see `.devcontainer/Dockerfile` for an example of how to download and install the WASI SDK) -2. A WASI host/runtime ([wasmtime](https://wasmtime.dev) is recommended) +2. A WASI host/runtime ([wasmtime](https://wasmtime.dev) 14+ is recommended and what the instructions below assume) ### Building -Building for WASI requires doing a cross-build where you have a "build" Python and then your WASI build (technically it's a "host x host" cross-build because the build Python is also the target Python while the host build is the WASI build). What this leads to is you building two separate copies of Python. In the end you should have a build Python in `cross-build/build` and `cross-build/wasm32-wasi`. +Building for WASI requires doing a cross-build where you have a "build" Python to help produce a WASI build of CPython (technically it's a "host x host" cross-build because the build Python is also the target Python while the host build is the WASI build; yes, it's confusing terminology). In the end you should have a build Python in `cross-build/build` and a WASI build in `cross-build/wasm32-wasi`. -The easiest way to do a build is to use the `wasi.py` script. You can either have it perform the entire build process from start to finish in one command, or you can do it in discrete steps (which are beneficial when you only need to do a specific step after getting a complete build). +The easiest way to do a build is to use the `wasi.py` script. You can either have it perform the entire build process from start to finish in one step, or you can do it in discrete steps that mirror running `configure` and `make` for each of the two builds of Python you end up producing (which are beneficial when you only need to do a specific step after getting a complete build, e.g. editing some code and you just need to run `make` for the WASI build). The discrete steps are: ```shell @@ -335,12 +335,12 @@ That will: 3. Run `configure` for the WASI build (`wasi.py configure-host`) 4. Run `make` for the WASI build (`wasi.py make-host`) -See the `--help` for the various options available for any of the subcommands which can control things like the location of the WASI SDK, the command to use with the WASI host/runtime, etc. Also note that you can use `--` as a separtor for any of the `configure`-related commands -- including `build` -- to pass arguments to `configure` itself. As an example, if you want a pydebug command that also caches the results from `configure`, you can do: +See the `--help` for the various options available for each of the subcommands which controls things like the location of the WASI SDK, the command to use with the WASI host/runtime, etc. Also note that you can use `--` as a separtor for any of the `configure`-related commands -- including `build` -- to pass arguments to `configure` itself. For example, if you want a pydebug build that also caches the results from `configure`, you can do: ```shell python Tools/wasm/wasi.py build -- -C --with-pydebug ``` -The `wasi.py` script is able to infer details from the build Python, and so you only technically need to specify `--with-pydebug` once for `configure-build-python` and `configure-host` will detect its use: +The `wasi.py` script is able to infer details from the build Python, and so you only technically need to specify `--with-pydebug` once for `configure-build-python` and `configure-host` will detect its use if you use the discrete steps: ```shell python Tools/wasm/wasi.py configure-build-python -- -C --with-pydebug python Tools/wasm/wasi.py make-build-python @@ -351,7 +351,12 @@ python Tools/wasm/wasi.py make-host ### Running -If you used `wasi.py` then there will be a `cross-build/wasm32-wasi/python.sh` file which you can use to run the WASI build (see the output from the `configure-host` subcommand). While there is a `python.wasm` file, you can't run it naively as there are various things need to be set (e.g. `PYTHONPATH` for `sysconfig` data). As such, the `python.sh` file handles these details for you. +If you used `wasi.py` to do your build then there will be a `cross-build/wasm32-wasi/python.sh` file which you can use to run the `python.wasm` file (see the output from the `configure-host` subcommand): +```shell +cross-build/wasm32-wasi/python.sh --version +``` + +While you _can_ run `python.wasm` directly, Python will fail to start up without certain things being set (e.g. `PYTHONPATH` for `sysconfig` data). As such, the `python.sh` file records these details for you. ## Detect WebAssembly builds From 37619db380d621d09b628cde22b7c033ec7bb932 Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Mon, 27 Nov 2023 13:55:57 -0800 Subject: [PATCH 35/40] Add a news entry --- .../next/Build/2023-11-27-13-55-47.gh-issue-103065.o72OiA.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 Misc/NEWS.d/next/Build/2023-11-27-13-55-47.gh-issue-103065.o72OiA.rst diff --git a/Misc/NEWS.d/next/Build/2023-11-27-13-55-47.gh-issue-103065.o72OiA.rst b/Misc/NEWS.d/next/Build/2023-11-27-13-55-47.gh-issue-103065.o72OiA.rst new file mode 100644 index 00000000000000..e2240b7c656a2f --- /dev/null +++ b/Misc/NEWS.d/next/Build/2023-11-27-13-55-47.gh-issue-103065.o72OiA.rst @@ -0,0 +1 @@ +Introduce ``Tools/wasm/wasi.py`` to simplify doing a WASI build. From 5b01cbe5f13b5c9dbaf79159fa6c424d81fede10 Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Mon, 27 Nov 2023 14:03:28 -0800 Subject: [PATCH 36/40] Skip mypy checks --- Tools/wasm/mypy.ini | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/Tools/wasm/mypy.ini b/Tools/wasm/mypy.ini index c62598f89eba69..4de0a30c260f5f 100644 --- a/Tools/wasm/mypy.ini +++ b/Tools/wasm/mypy.ini @@ -1,5 +1,5 @@ [mypy] -files = Tools/wasm +files = Tools/wasm/wasm_*.py pretty = True show_traceback = True @@ -9,6 +9,3 @@ python_version = 3.8 # Be strict... strict = True enable_error_code = truthy-bool,ignore-without-code - -# except for incomplete defs, which are useful for module authors: -disallow_incomplete_defs = False From 71946e7846a3d5995863b7cf45d41311ae321446 Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Mon, 27 Nov 2023 14:16:37 -0800 Subject: [PATCH 37/40] Fix an over-indentation --- Tools/wasm/wasi.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Tools/wasm/wasi.py b/Tools/wasm/wasi.py index 4aeddda81aacb2..2cd1e4f9d78ba8 100644 --- a/Tools/wasm/wasi.py +++ b/Tools/wasm/wasi.py @@ -189,10 +189,10 @@ def wasi_sdk_env(context): def configure_wasi_python(context, working_dir): """Configure the WASI/host build.""" if not context.wasi_sdk_path or not context.wasi_sdk_path.exists(): - raise ValueError("WASI-SDK not found; " - "download from " - "https://github.com/WebAssembly/wasi-sdk and/or " - "specify via $WASI_SDK_PATH or --wasi-sdk") + raise ValueError("WASI-SDK not found; " + "download from " + "https://github.com/WebAssembly/wasi-sdk and/or " + "specify via $WASI_SDK_PATH or --wasi-sdk") config_site = os.fsdecode(CHECKOUT / "Tools" / "wasm" / "config.site-wasm32-wasi") From bd488b212857e7f71f3acb8c4ba9ba00a215cb2d Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Mon, 27 Nov 2023 14:17:33 -0800 Subject: [PATCH 38/40] Only set `SOURCE_DATE_EPOCH` if it has not already been set --- Tools/wasm/wasi.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/Tools/wasm/wasi.py b/Tools/wasm/wasi.py index 2cd1e4f9d78ba8..e1c192d7e68a43 100644 --- a/Tools/wasm/wasi.py +++ b/Tools/wasm/wasi.py @@ -29,13 +29,14 @@ def updated_env(updates={}): The changes made to the execution environment are printed out. """ env_defaults = {} - # https://reproducible-builds.org/docs/source-date-epoch/ - git_epoch_cmd = ["git", "log", "-1", "--pretty=%ct"] - try: - epoch = subprocess.check_output(git_epoch_cmd, encoding="utf-8").strip() - env_defaults["SOURCE_DATE_EPOCH"] = epoch - except subprocess.CalledProcessError: - pass # Might be building from a tarball. + if "SOURCE_DATE_EPOCH" not in os.environ: + # https://reproducible-builds.org/docs/source-date-epoch/ + git_epoch_cmd = ["git", "log", "-1", "--pretty=%ct"] + try: + epoch = subprocess.check_output(git_epoch_cmd, encoding="utf-8").strip() + env_defaults["SOURCE_DATE_EPOCH"] = epoch + except subprocess.CalledProcessError: + pass # Might be building from a tarball. environment = env_defaults | os.environ | updates env_diff = {} From 547ba0f9628f7ba6374ea9dfa7cfc6c42834c849 Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Tue, 28 Nov 2023 11:13:15 -0800 Subject: [PATCH 39/40] Simplify some code --- Tools/wasm/wasi.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/Tools/wasm/wasi.py b/Tools/wasm/wasi.py index e1c192d7e68a43..34c0e9375e24c8 100644 --- a/Tools/wasm/wasi.py +++ b/Tools/wasm/wasi.py @@ -29,14 +29,14 @@ def updated_env(updates={}): The changes made to the execution environment are printed out. """ env_defaults = {} - if "SOURCE_DATE_EPOCH" not in os.environ: - # https://reproducible-builds.org/docs/source-date-epoch/ - git_epoch_cmd = ["git", "log", "-1", "--pretty=%ct"] - try: - epoch = subprocess.check_output(git_epoch_cmd, encoding="utf-8").strip() - env_defaults["SOURCE_DATE_EPOCH"] = epoch - except subprocess.CalledProcessError: - pass # Might be building from a tarball. + # https://reproducible-builds.org/docs/source-date-epoch/ + git_epoch_cmd = ["git", "log", "-1", "--pretty=%ct"] + try: + epoch = subprocess.check_output(git_epoch_cmd, encoding="utf-8").strip() + env_defaults["SOURCE_DATE_EPOCH"] = epoch + except subprocess.CalledProcessError: + pass # Might be building from a tarball. + # This layering lets SOURCE_DATE_EPOCH from os.environ takes precedence. environment = env_defaults | os.environ | updates env_diff = {} From dea525ec8dff5b3e49928bee93e1ed6c743691fc Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Tue, 28 Nov 2023 15:11:50 -0800 Subject: [PATCH 40/40] Bump the version of wasmtime installed into the dev container --- .devcontainer/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 590d7834b2b8be..9f808af38e69df 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -6,7 +6,7 @@ ENV WASI_SDK_VERSION=20 ENV WASI_SDK_PATH=/opt/wasi-sdk ENV WASMTIME_HOME=/opt/wasmtime -ENV WASMTIME_VERSION=9.0.1 +ENV WASMTIME_VERSION=14.0.4 ENV WASMTIME_CPU_ARCH=x86_64 RUN dnf -y --nodocs --setopt=install_weak_deps=False install /usr/bin/{blurb,clang,curl,git,ln,tar,xz} 'dnf-command(builddep)' && \