From 85dd6ec9a2460283942281ce931d0dfcd662da5b Mon Sep 17 00:00:00 2001 From: Christian Heimes Date: Wed, 1 Dec 2021 17:27:33 +0100 Subject: [PATCH 1/3] bpo-45949: Pure Python freeze module for cross builds Use `_bootstrap_python` interpreter and pure Python implementation of `freeze_module` to generate frozen byte code files. Only importlib bootstrap files are generated with `Programs/_freeze_module`. This simplifies cross building, as the build system no longer needs a `_freeze_module` binary. A standard Python installation with same version is sufficient. Signed-off-by: Christian Heimes --- Doc/using/configure.rst | 14 +- Makefile.pre.in | 133 +++++++++++------- .../2021-12-02-23-21-18.bpo-45949.OTSo9X.rst | 3 + Tools/scripts/deepfreeze.py | 5 +- Tools/scripts/freeze_module.py | 69 +++++++++ Tools/scripts/freeze_modules.py | 19 ++- configure | 45 ++---- configure.ac | 32 +---- 8 files changed, 189 insertions(+), 131 deletions(-) create mode 100644 Misc/NEWS.d/next/Build/2021-12-02-23-21-18.bpo-45949.OTSo9X.rst create mode 100644 Tools/scripts/freeze_module.py diff --git a/Doc/using/configure.rst b/Doc/using/configure.rst index 7802a7c810b29e..b2a2936db738bc 100644 --- a/Doc/using/configure.rst +++ b/Doc/using/configure.rst @@ -518,9 +518,8 @@ Cross Compiling Options Cross compiling, also known as cross building, can be used to build Python for another CPU architecture or platform. Cross compiling requires a Python -interpreter and the :program:`_freeze_module` binary from another build. The -version of the build Python and :program:`_freeze_module` command must be -the same as the cross compiled host Python. +interpreter for the build platform. The version of the build Python must match +the version of the cross compiled host Python. .. cmdoption:: --build=BUILD @@ -530,13 +529,7 @@ the same as the cross compiled host Python. cross-compile to build programs to run on HOST (target platform) -.. cmdoption:: --with-freeze-module=Programs/_freeze_module - - path to ``_freeze_module`` binary for cross compiling. - - .. versionadded:: 3.11 - -.. cmdoption:: --with-build-python=python3.xx +.. cmdoption:: --with-build-python=path/to/python path to build ``python`` binary for cross compiling @@ -559,7 +552,6 @@ Cross compiling example:: CONFIG_SITE=config.site-aarch64 ../configure \ --build=x86_64-pc-linux-gnu \ --host=aarch64-unknown-linux-gnu \ - --with-freeze-module=../x86_64/Programs/_freeze_module \ --with-build-python=../x86_64/python diff --git a/Makefile.pre.in b/Makefile.pre.in index a182786e5b5873..db11892b0f3f80 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -286,11 +286,14 @@ PYTHON_FOR_BUILD=@PYTHON_FOR_BUILD@ # Standard builds use _bootstrap_python for freezing, cross compiling # uses build Python, which must have the same version and bytecode, PYTHON_FOR_FREEZE?=@PYTHON_FOR_FREEZE@ +FREEZE_MODULE_C=Programs/_freeze_module +FREEZE_MODULE_PY=$(PYTHON_FOR_FREEZE) $(srcdir)/Tools/scripts/freeze_module.py +# bootstrap freezing uses FREEZE_MODULE_C for normal builds and _PY for cross builds +FREEZE_MODULE_BOOTSTRAP=@FREEZE_MODULE_BOOTSTRAP@ + _PYTHON_HOST_PLATFORM=@_PYTHON_HOST_PLATFORM@ BUILD_GNU_TYPE= @build@ HOST_GNU_TYPE= @host@ -# Allow developers to override freeze_module command for cross building (bpo-45886) -FREEZE_MODULE?=@FREEZE_MODULE@ # Tcl and Tk config info from --with-tcltk-includes and -libs options TCLTK_INCLUDES= @TCLTK_INCLUDES@ @@ -1044,6 +1047,32 @@ Python/deepfreeze/frozen_only.c: Python/frozen_modules/frozen_only.h $(DEEPFREEZ ############################################################################ # frozen modules (including importlib) +# +# Freezing is a multi step process. It works differently for standard builds +# and cross builds. Standard builds use Programs/_freeze_module and +# _bootstrap_python for freezing and deepfreezing, so users can build Python +# without an existing Python installation. Cross builds cannot execute +# compiled binaries and therefore rely on an external build Python +# interpreter. The build interpreter must have same version and same bytecode +# as the host (target) binary. +# +# Standard build process: +# 1) compile minimal core objects for Py_Compile*() and PyMarshal_Write*(). +# 2) build Programs/_freeze_module binary. +# 3) create frozen module headers for importlib and getpath. +# 4) build _bootstrap_python binary. +# 5) create remaining frozen module headers with _bootstrap_python and +# Python freeze_module.py (so we test this code path). +# 6) deepfreeze modules with _bootstrap_python +# +# Cross compile proces: +# 1) create all frozen module headers with external build Python and +# freeze_module.py script. +# 2) deepfreeze modules with external build Python. +# +# Cross compiling also creates Programs/_freeze_module and _bootstrap_python +# artifacts, but does not use or ship them. +# # FROZEN_FILES_* are auto-generated by Tools/scripts/freeze_modules.py. FROZEN_FILES_IN = \ @@ -1104,83 +1133,83 @@ Modules/getpath_noop.o: $(srcdir)/Modules/getpath_noop.c Makefile Programs/_freeze_module: Programs/_freeze_module.o Modules/getpath_noop.o $(LIBRARY_OBJS_OMIT_FROZEN) $(LINKCC) $(PY_CORE_LDFLAGS) -o $@ Programs/_freeze_module.o Modules/getpath_noop.o $(LIBRARY_OBJS_OMIT_FROZEN) $(LIBS) $(MODLIBS) $(SYSLIBS) +# We manually freeze getpath.py rather than through freeze_modules +Python/frozen_modules/getpath.h: $(FREEZE_MODULE_C) Modules/getpath.py + $(FREEZE_MODULE_BOOTSTRAP) getpath $(srcdir)/Modules/getpath.py Python/frozen_modules/getpath.h + # BEGIN: freezing modules -Python/frozen_modules/importlib._bootstrap.h: $(FREEZE_MODULE) Lib/importlib/_bootstrap.py - $(FREEZE_MODULE) importlib._bootstrap $(srcdir)/Lib/importlib/_bootstrap.py Python/frozen_modules/importlib._bootstrap.h +Python/frozen_modules/importlib._bootstrap.h: Lib/importlib/_bootstrap.py $(FREEZE_MODULE_C) $(srcdir)/Tools/scripts/freeze_module.py + $(FREEZE_MODULE_BOOTSTRAP) importlib._bootstrap $(srcdir)/Lib/importlib/_bootstrap.py Python/frozen_modules/importlib._bootstrap.h -Python/frozen_modules/importlib._bootstrap_external.h: $(FREEZE_MODULE) Lib/importlib/_bootstrap_external.py - $(FREEZE_MODULE) importlib._bootstrap_external $(srcdir)/Lib/importlib/_bootstrap_external.py Python/frozen_modules/importlib._bootstrap_external.h +Python/frozen_modules/importlib._bootstrap_external.h: Lib/importlib/_bootstrap_external.py $(FREEZE_MODULE_C) $(srcdir)/Tools/scripts/freeze_module.py + $(FREEZE_MODULE_BOOTSTRAP) importlib._bootstrap_external $(srcdir)/Lib/importlib/_bootstrap_external.py Python/frozen_modules/importlib._bootstrap_external.h -Python/frozen_modules/zipimport.h: $(FREEZE_MODULE) Lib/zipimport.py - $(FREEZE_MODULE) zipimport $(srcdir)/Lib/zipimport.py Python/frozen_modules/zipimport.h +Python/frozen_modules/zipimport.h: Lib/zipimport.py $(FREEZE_MODULE_C) $(srcdir)/Tools/scripts/freeze_module.py + $(FREEZE_MODULE_BOOTSTRAP) zipimport $(srcdir)/Lib/zipimport.py Python/frozen_modules/zipimport.h -Python/frozen_modules/abc.h: $(FREEZE_MODULE) Lib/abc.py - $(FREEZE_MODULE) abc $(srcdir)/Lib/abc.py Python/frozen_modules/abc.h +Python/frozen_modules/abc.h: Lib/abc.py _bootstrap_python $(srcdir)/Tools/scripts/freeze_module.py + $(FREEZE_MODULE_PY) abc $(srcdir)/Lib/abc.py Python/frozen_modules/abc.h -Python/frozen_modules/codecs.h: $(FREEZE_MODULE) Lib/codecs.py - $(FREEZE_MODULE) codecs $(srcdir)/Lib/codecs.py Python/frozen_modules/codecs.h +Python/frozen_modules/codecs.h: Lib/codecs.py _bootstrap_python $(srcdir)/Tools/scripts/freeze_module.py + $(FREEZE_MODULE_PY) codecs $(srcdir)/Lib/codecs.py Python/frozen_modules/codecs.h -Python/frozen_modules/io.h: $(FREEZE_MODULE) Lib/io.py - $(FREEZE_MODULE) io $(srcdir)/Lib/io.py Python/frozen_modules/io.h +Python/frozen_modules/io.h: Lib/io.py _bootstrap_python $(srcdir)/Tools/scripts/freeze_module.py + $(FREEZE_MODULE_PY) io $(srcdir)/Lib/io.py Python/frozen_modules/io.h -Python/frozen_modules/_collections_abc.h: $(FREEZE_MODULE) Lib/_collections_abc.py - $(FREEZE_MODULE) _collections_abc $(srcdir)/Lib/_collections_abc.py Python/frozen_modules/_collections_abc.h +Python/frozen_modules/_collections_abc.h: Lib/_collections_abc.py _bootstrap_python $(srcdir)/Tools/scripts/freeze_module.py + $(FREEZE_MODULE_PY) _collections_abc $(srcdir)/Lib/_collections_abc.py Python/frozen_modules/_collections_abc.h -Python/frozen_modules/_sitebuiltins.h: $(FREEZE_MODULE) Lib/_sitebuiltins.py - $(FREEZE_MODULE) _sitebuiltins $(srcdir)/Lib/_sitebuiltins.py Python/frozen_modules/_sitebuiltins.h +Python/frozen_modules/_sitebuiltins.h: Lib/_sitebuiltins.py _bootstrap_python $(srcdir)/Tools/scripts/freeze_module.py + $(FREEZE_MODULE_PY) _sitebuiltins $(srcdir)/Lib/_sitebuiltins.py Python/frozen_modules/_sitebuiltins.h -Python/frozen_modules/genericpath.h: $(FREEZE_MODULE) Lib/genericpath.py - $(FREEZE_MODULE) genericpath $(srcdir)/Lib/genericpath.py Python/frozen_modules/genericpath.h +Python/frozen_modules/genericpath.h: Lib/genericpath.py _bootstrap_python $(srcdir)/Tools/scripts/freeze_module.py + $(FREEZE_MODULE_PY) genericpath $(srcdir)/Lib/genericpath.py Python/frozen_modules/genericpath.h -Python/frozen_modules/ntpath.h: $(FREEZE_MODULE) Lib/ntpath.py - $(FREEZE_MODULE) ntpath $(srcdir)/Lib/ntpath.py Python/frozen_modules/ntpath.h +Python/frozen_modules/ntpath.h: Lib/ntpath.py _bootstrap_python $(srcdir)/Tools/scripts/freeze_module.py + $(FREEZE_MODULE_PY) ntpath $(srcdir)/Lib/ntpath.py Python/frozen_modules/ntpath.h -Python/frozen_modules/posixpath.h: $(FREEZE_MODULE) Lib/posixpath.py - $(FREEZE_MODULE) posixpath $(srcdir)/Lib/posixpath.py Python/frozen_modules/posixpath.h +Python/frozen_modules/posixpath.h: Lib/posixpath.py _bootstrap_python $(srcdir)/Tools/scripts/freeze_module.py + $(FREEZE_MODULE_PY) posixpath $(srcdir)/Lib/posixpath.py Python/frozen_modules/posixpath.h -Python/frozen_modules/os.h: $(FREEZE_MODULE) Lib/os.py - $(FREEZE_MODULE) os $(srcdir)/Lib/os.py Python/frozen_modules/os.h +Python/frozen_modules/os.h: Lib/os.py _bootstrap_python $(srcdir)/Tools/scripts/freeze_module.py + $(FREEZE_MODULE_PY) os $(srcdir)/Lib/os.py Python/frozen_modules/os.h -Python/frozen_modules/site.h: $(FREEZE_MODULE) Lib/site.py - $(FREEZE_MODULE) site $(srcdir)/Lib/site.py Python/frozen_modules/site.h +Python/frozen_modules/site.h: Lib/site.py _bootstrap_python $(srcdir)/Tools/scripts/freeze_module.py + $(FREEZE_MODULE_PY) site $(srcdir)/Lib/site.py Python/frozen_modules/site.h -Python/frozen_modules/stat.h: $(FREEZE_MODULE) Lib/stat.py - $(FREEZE_MODULE) stat $(srcdir)/Lib/stat.py Python/frozen_modules/stat.h +Python/frozen_modules/stat.h: Lib/stat.py _bootstrap_python $(srcdir)/Tools/scripts/freeze_module.py + $(FREEZE_MODULE_PY) stat $(srcdir)/Lib/stat.py Python/frozen_modules/stat.h -Python/frozen_modules/importlib.util.h: $(FREEZE_MODULE) Lib/importlib/util.py - $(FREEZE_MODULE) importlib.util $(srcdir)/Lib/importlib/util.py Python/frozen_modules/importlib.util.h +Python/frozen_modules/importlib.util.h: Lib/importlib/util.py _bootstrap_python $(srcdir)/Tools/scripts/freeze_module.py + $(FREEZE_MODULE_PY) importlib.util $(srcdir)/Lib/importlib/util.py Python/frozen_modules/importlib.util.h -Python/frozen_modules/importlib.machinery.h: $(FREEZE_MODULE) Lib/importlib/machinery.py - $(FREEZE_MODULE) importlib.machinery $(srcdir)/Lib/importlib/machinery.py Python/frozen_modules/importlib.machinery.h +Python/frozen_modules/importlib.machinery.h: Lib/importlib/machinery.py _bootstrap_python $(srcdir)/Tools/scripts/freeze_module.py + $(FREEZE_MODULE_PY) importlib.machinery $(srcdir)/Lib/importlib/machinery.py Python/frozen_modules/importlib.machinery.h -Python/frozen_modules/runpy.h: $(FREEZE_MODULE) Lib/runpy.py - $(FREEZE_MODULE) runpy $(srcdir)/Lib/runpy.py Python/frozen_modules/runpy.h +Python/frozen_modules/runpy.h: Lib/runpy.py _bootstrap_python $(srcdir)/Tools/scripts/freeze_module.py + $(FREEZE_MODULE_PY) runpy $(srcdir)/Lib/runpy.py Python/frozen_modules/runpy.h -Python/frozen_modules/__hello__.h: $(FREEZE_MODULE) Lib/__hello__.py - $(FREEZE_MODULE) __hello__ $(srcdir)/Lib/__hello__.py Python/frozen_modules/__hello__.h +Python/frozen_modules/__hello__.h: Lib/__hello__.py _bootstrap_python $(srcdir)/Tools/scripts/freeze_module.py + $(FREEZE_MODULE_PY) __hello__ $(srcdir)/Lib/__hello__.py Python/frozen_modules/__hello__.h -Python/frozen_modules/__phello__.h: $(FREEZE_MODULE) Lib/__phello__/__init__.py - $(FREEZE_MODULE) __phello__ $(srcdir)/Lib/__phello__/__init__.py Python/frozen_modules/__phello__.h +Python/frozen_modules/__phello__.h: Lib/__phello__/__init__.py _bootstrap_python $(srcdir)/Tools/scripts/freeze_module.py + $(FREEZE_MODULE_PY) __phello__ $(srcdir)/Lib/__phello__/__init__.py Python/frozen_modules/__phello__.h -Python/frozen_modules/__phello__.ham.h: $(FREEZE_MODULE) Lib/__phello__/ham/__init__.py - $(FREEZE_MODULE) __phello__.ham $(srcdir)/Lib/__phello__/ham/__init__.py Python/frozen_modules/__phello__.ham.h +Python/frozen_modules/__phello__.ham.h: Lib/__phello__/ham/__init__.py _bootstrap_python $(srcdir)/Tools/scripts/freeze_module.py + $(FREEZE_MODULE_PY) __phello__.ham $(srcdir)/Lib/__phello__/ham/__init__.py Python/frozen_modules/__phello__.ham.h -Python/frozen_modules/__phello__.ham.eggs.h: $(FREEZE_MODULE) Lib/__phello__/ham/eggs.py - $(FREEZE_MODULE) __phello__.ham.eggs $(srcdir)/Lib/__phello__/ham/eggs.py Python/frozen_modules/__phello__.ham.eggs.h +Python/frozen_modules/__phello__.ham.eggs.h: Lib/__phello__/ham/eggs.py _bootstrap_python $(srcdir)/Tools/scripts/freeze_module.py + $(FREEZE_MODULE_PY) __phello__.ham.eggs $(srcdir)/Lib/__phello__/ham/eggs.py Python/frozen_modules/__phello__.ham.eggs.h -Python/frozen_modules/__phello__.spam.h: $(FREEZE_MODULE) Lib/__phello__/spam.py - $(FREEZE_MODULE) __phello__.spam $(srcdir)/Lib/__phello__/spam.py Python/frozen_modules/__phello__.spam.h +Python/frozen_modules/__phello__.spam.h: Lib/__phello__/spam.py _bootstrap_python $(srcdir)/Tools/scripts/freeze_module.py + $(FREEZE_MODULE_PY) __phello__.spam $(srcdir)/Lib/__phello__/spam.py Python/frozen_modules/__phello__.spam.h -Python/frozen_modules/frozen_only.h: $(FREEZE_MODULE) Tools/freeze/flag.py - $(FREEZE_MODULE) frozen_only $(srcdir)/Tools/freeze/flag.py Python/frozen_modules/frozen_only.h +Python/frozen_modules/frozen_only.h: Tools/freeze/flag.py _bootstrap_python $(srcdir)/Tools/scripts/freeze_module.py + $(FREEZE_MODULE_PY) frozen_only $(srcdir)/Tools/freeze/flag.py Python/frozen_modules/frozen_only.h # END: freezing modules -# We manually freeze getpath.py rather than through freeze_modules -Python/frozen_modules/getpath.h: $(FREEZE_MODULE) Modules/getpath.py - $(FREEZE_MODULE) getpath $(srcdir)/Modules/getpath.py Python/frozen_modules/getpath.h - Tools/scripts/freeze_modules.py: $(FREEZE_MODULE) .PHONY: regen-frozen diff --git a/Misc/NEWS.d/next/Build/2021-12-02-23-21-18.bpo-45949.OTSo9X.rst b/Misc/NEWS.d/next/Build/2021-12-02-23-21-18.bpo-45949.OTSo9X.rst new file mode 100644 index 00000000000000..c746d71b1dcae9 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2021-12-02-23-21-18.bpo-45949.OTSo9X.rst @@ -0,0 +1,3 @@ +Use pure Python ``freeze_module`` for all but importlib bootstrap files. +``--with-freeze-module`` :program:`configure` option is no longer needed for +cross builds. diff --git a/Tools/scripts/deepfreeze.py b/Tools/scripts/deepfreeze.py index 30ca7bb0b6090e..7cd7eb674d4511 100644 --- a/Tools/scripts/deepfreeze.py +++ b/Tools/scripts/deepfreeze.py @@ -393,13 +393,14 @@ def generate(self, name: str, obj: object) -> str: } """ -FROZEN_COMMENT = "/* Auto-generated by Programs/_freeze_module.c */" +FROZEN_COMMENT_C = "/* Auto-generated by Programs/_freeze_module.c */" +FROZEN_COMMENT_PY = "/* Auto-generated by Tools/scripts/freeze_module.py */" FROZEN_DATA_LINE = r"\s*(\d+,\s*)+\s*" def is_frozen_header(source: str) -> bool: - return source.startswith(FROZEN_COMMENT) + return source.startswith((FROZEN_COMMENT_C, FROZEN_COMMENT_PY)) def decode_frozen_data(source: str) -> types.CodeType: diff --git a/Tools/scripts/freeze_module.py b/Tools/scripts/freeze_module.py new file mode 100644 index 00000000000000..6b5ad35708f788 --- /dev/null +++ b/Tools/scripts/freeze_module.py @@ -0,0 +1,69 @@ +"""Python implementation of Programs/_freeze_module.c + +The pure Python implementation uses same functions and arguments as the C +implementation. + +The generated byte code is slightly different because +compile() sets the PyCF_SOURCE_IS_UTF8 flag and objects have a +reference count > 1. Marshal adds the `FLAG_REF` flag and creates a +reference `hashtable`. +""" + +import marshal +import sys + +header = "/* Auto-generated by Tools/scripts/freeze_module.py */" + + +def read_text(inpath: str) -> bytes: + with open(inpath, "rb") as f: + return f.read() + + +def compile_and_marshal(name: str, text: bytes) -> bytes: + filename = f"" + # exec == Py_file_input + code = compile(text, filename, "exec", optimize=0, dont_inherit=True) + # defaults to Py_MARSHAL_VERSION) + return marshal.dumps(code) + + +def get_varname(name: str, prefix: str) -> str: + return f"{prefix}{name.replace('.', '_')}" + + +def write_code(outfile, marshalled: bytes, varname: str) -> None: + data_size = len(marshalled) + + outfile.write(f"const unsigned char {varname}[] = {{\n") + + for n in range(0, data_size, 16): + outfile.write(" ") + outfile.write(",".join(str(i) for i in marshalled[n : n + 16])) + outfile.write(",\n") + outfile.write("};\n") + + +def write_frozen(outpath: str, inpath: str, name: str, marshalled: bytes) -> None: + with open(outpath, "w") as outfile: + outfile.write(header) + outfile.write("\n") + arrayname = get_varname(name, "_Py_M__") + write_code(outfile, marshalled, arrayname) + + +def main(): + if len(sys.argv) != 4: + sys.exit("need to specify the name, input and output paths\n") + + name = sys.argv[1] + inpath = sys.argv[2] + outpath = sys.argv[3] + + text = read_text(inpath) + marshalled = compile_and_marshal(name, text) + write_frozen(outpath, inpath, name, marshalled) + + +if __name__ == "__main__": + main() diff --git a/Tools/scripts/freeze_modules.py b/Tools/scripts/freeze_modules.py index 58d13985608359..05d59cc6bcd452 100644 --- a/Tools/scripts/freeze_modules.py +++ b/Tools/scripts/freeze_modules.py @@ -264,6 +264,10 @@ def ispkg(self): else: return os.path.basename(self.pyfile) == '__init__.py' + @property + def isbootstrap(self): + return self.id in BOOTSTRAP + def resolve_frozen_file(frozenid, destdir): """Return the filename corresponding to the given frozen ID. @@ -476,7 +480,7 @@ def regen_frozen(modules): indent = ' ' lastsection = None for mod in modules: - if mod.frozenid in BOOTSTRAP: + if mod.isbootstrap: lines = bootstraplines elif mod.section == TESTS_SECTION: lines = testlines @@ -585,10 +589,17 @@ def regen_makefile(modules): pyfile = relpath_for_posix_display(src.pyfile, ROOT_DIR) pyfiles.append(f'\t\t{pyfile} \\') - freeze = (f'$(FREEZE_MODULE) {src.frozenid} ' - f'$(srcdir)/{pyfile} {frozen_header}') + if src.isbootstrap: + freezecmd = '$(FREEZE_MODULE_BOOTSTRAP)' + freezedep = '$(FREEZE_MODULE_C)' + else: + freezecmd = '$(FREEZE_MODULE_PY)' + freezedep = '_bootstrap_python' + + freeze = (f'{freezecmd} {src.frozenid} ' + f'$(srcdir)/{pyfile} {frozen_header}') rules.extend([ - f'{frozen_header}: $(FREEZE_MODULE) {pyfile}', + f'{frozen_header}: {pyfile} {freezedep} $(srcdir)/Tools/scripts/freeze_module.py', f'\t{freeze}', '', ]) diff --git a/configure b/configure index 3c2a9cf0a249dd..e3ad4a8875c5dd 100755 --- a/configure +++ b/configure @@ -933,9 +933,9 @@ CONFIG_ARGS SOVERSION VERSION PYTHON_FOR_REGEN +FREEZE_MODULE_BOOTSTRAP PYTHON_FOR_FREEZE PYTHON_FOR_BUILD -FREEZE_MODULE host_os host_vendor host_cpu @@ -991,7 +991,6 @@ SHELL' ac_subst_files='' ac_user_opts=' enable_option_checking -with_freeze_module with_build_python with_pkg_config enable_universalsdk @@ -1726,8 +1725,6 @@ Optional Features: Optional Packages: --with-PACKAGE[=ARG] use PACKAGE [ARG=yes] --without-PACKAGE do not use PACKAGE (same as --with-PACKAGE=no) - --with-freeze-module=Programs/_freeze_module - path to _freeze_module binary for cross compiling --with-build-python=python3.11 path to build python binary for cross compiling (default: _bootstrap_python or python3.11) @@ -3196,38 +3193,6 @@ fi rm -f pybuilddir.txt -# Check whether --with-freeze-module was given. -if test "${with_freeze_module+set}" = set; then : - withval=$with_freeze_module; - { $as_echo "$as_me:${as_lineno-$LINENO}: checking for --with-freeze-module" >&5 -$as_echo_n "checking for --with-freeze-module... " >&6; } - if test "x$cross_compiling" = xno; then : - as_fn_error $? "--with-freeze-module only applies to cross compiling" "$LINENO" 5 -fi - if test "$with_freeze_module" = yes -o "$with_freeze_module" = no; then - as_fn_error $? "invalid --with-freeze-module option: expected path, not \"$with_freeze_module\"" "$LINENO" 5 - fi - if ! $(command -v "$with_freeze_module" >/dev/null 2>&1); then - as_fn_error $? "invalid or missing freeze module binary \"$with_freeze_module\"" "$LINENO" 5 - fi - FREEZE_MODULE="$with_freeze_module" - { $as_echo "$as_me:${as_lineno-$LINENO}: result: $FREEZE_MODULE" >&5 -$as_echo "$FREEZE_MODULE" >&6; } - -else - - if test "x$cross_compiling" = xyes; then : - as_fn_error $? "Cross compiling requires --with-freeze-module" "$LINENO" 5 - -fi - FREEZE_MODULE=Programs/_freeze_module - - -fi - - - - # Check whether --with-build-python was given. if test "${with_build_python+set}" = set; then : withval=$with_build_python; @@ -3274,6 +3239,14 @@ $as_echo_n "checking for Python interpreter freezing... " >&6; } $as_echo "$PYTHON_FOR_FREEZE" >&6; } +if test "x$cross_compiling" = xyes; then : + FREEZE_MODULE_BOOTSTRAP='$(FREEZE_MODULE_PY)' +else + FREEZE_MODULE_BOOTSTRAP='$(FREEZE_MODULE_C)' + +fi + + for ac_prog in python$PACKAGE_VERSION python3.10 python3.9 python3.8 python3.7 python3.6 python3 python do # Extract the first word of "$ac_prog", so it can be a program name with args. diff --git a/configure.ac b/configure.ac index fa5e63cf5c3f98..f5e6ea46b455ee 100644 --- a/configure.ac +++ b/configure.ac @@ -100,32 +100,6 @@ AS_VAR_IF([cross_compiling], [maybe], # pybuilddir.txt will be created by --generate-posix-vars in the Makefile rm -f pybuilddir.txt -dnl cross-compiling needs a freeze_module binary for build platform -AC_ARG_WITH( - [freeze-module], - [AS_HELP_STRING([--with-freeze-module=Programs/_freeze_module], - [path to _freeze_module binary for cross compiling])], - [ - AC_MSG_CHECKING([for --with-freeze-module]) - AS_VAR_IF([cross_compiling], [no], AC_MSG_ERROR([--with-freeze-module only applies to cross compiling])) - if test "$with_freeze_module" = yes -o "$with_freeze_module" = no; then - AC_MSG_ERROR([invalid --with-freeze-module option: expected path, not "$with_freeze_module"]) - fi - if ! $(command -v "$with_freeze_module" >/dev/null 2>&1); then - AC_MSG_ERROR([invalid or missing freeze module binary "$with_freeze_module"]) - fi - FREEZE_MODULE="$with_freeze_module" - AC_MSG_RESULT([$FREEZE_MODULE]) - ], [ - AS_VAR_IF([cross_compiling], [yes], - [AC_MSG_ERROR([Cross compiling requires --with-freeze-module])] - ) - FREEZE_MODULE=Programs/_freeze_module - ] -) -AC_SUBST([FREEZE_MODULE]) - -dnl build-python is used for cross compiling and macOS framework builds. AC_ARG_WITH( [build-python], [AS_HELP_STRING([--with-build-python=python]PYTHON_VERSION, @@ -162,6 +136,12 @@ AC_MSG_CHECKING([for Python interpreter freezing]) AC_MSG_RESULT([$PYTHON_FOR_FREEZE]) AC_SUBST([PYTHON_FOR_FREEZE]) +AS_VAR_IF([cross_compiling], [yes], + [FREEZE_MODULE_BOOTSTRAP='$(FREEZE_MODULE_PY)'], + [FREEZE_MODULE_BOOTSTRAP='$(FREEZE_MODULE_C)'] +) +AC_SUBST([FREEZE_MODULE_BOOTSTRAP]) + AC_CHECK_PROGS([PYTHON_FOR_REGEN], [python$PACKAGE_VERSION python3.10 python3.9 python3.8 python3.7 python3.6 python3 python], [python3]) From df657a9b980cce1c4d0deeb8b87be96d657d30c1 Mon Sep 17 00:00:00 2001 From: Christian Heimes Date: Thu, 9 Dec 2021 15:25:35 +0100 Subject: [PATCH 2/3] Move script next to _freeze_module.c --- Makefile.pre.in | 59 ++++++++++--------- .../_freeze_module.py | 2 +- Tools/scripts/deepfreeze.py | 2 +- Tools/scripts/freeze_modules.py | 4 +- 4 files changed, 34 insertions(+), 33 deletions(-) rename Tools/scripts/freeze_module.py => Programs/_freeze_module.py (96%) diff --git a/Makefile.pre.in b/Makefile.pre.in index db11892b0f3f80..4eff3ab415f193 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -287,7 +287,7 @@ PYTHON_FOR_BUILD=@PYTHON_FOR_BUILD@ # uses build Python, which must have the same version and bytecode, PYTHON_FOR_FREEZE?=@PYTHON_FOR_FREEZE@ FREEZE_MODULE_C=Programs/_freeze_module -FREEZE_MODULE_PY=$(PYTHON_FOR_FREEZE) $(srcdir)/Tools/scripts/freeze_module.py +FREEZE_MODULE_PY=$(PYTHON_FOR_FREEZE) $(srcdir)/Programs/_freeze_module.py # bootstrap freezing uses FREEZE_MODULE_C for normal builds and _PY for cross builds FREEZE_MODULE_BOOTSTRAP=@FREEZE_MODULE_BOOTSTRAP@ @@ -1061,13 +1061,14 @@ Python/deepfreeze/frozen_only.c: Python/frozen_modules/frozen_only.h $(DEEPFREEZ # 2) build Programs/_freeze_module binary. # 3) create frozen module headers for importlib and getpath. # 4) build _bootstrap_python binary. -# 5) create remaining frozen module headers with _bootstrap_python and -# Python freeze_module.py (so we test this code path). +# 5) create remaining frozen module headers with +# ``./_bootstrap_python Programs/_freeze_module.py``. The pure Python +# script is used to test the cross compile code path. # 6) deepfreeze modules with _bootstrap_python # # Cross compile proces: # 1) create all frozen module headers with external build Python and -# freeze_module.py script. +# Programs/_freeze_module.py script. # 2) deepfreeze modules with external build Python. # # Cross compiling also creates Programs/_freeze_module and _bootstrap_python @@ -1130,82 +1131,82 @@ Programs/_freeze_module.o: Programs/_freeze_module.c Makefile Modules/getpath_noop.o: $(srcdir)/Modules/getpath_noop.c Makefile -Programs/_freeze_module: Programs/_freeze_module.o Modules/getpath_noop.o $(LIBRARY_OBJS_OMIT_FROZEN) +$(FREEZE_MODULE_C): Programs/_freeze_module.o Modules/getpath_noop.o $(LIBRARY_OBJS_OMIT_FROZEN) $(LINKCC) $(PY_CORE_LDFLAGS) -o $@ Programs/_freeze_module.o Modules/getpath_noop.o $(LIBRARY_OBJS_OMIT_FROZEN) $(LIBS) $(MODLIBS) $(SYSLIBS) # We manually freeze getpath.py rather than through freeze_modules -Python/frozen_modules/getpath.h: $(FREEZE_MODULE_C) Modules/getpath.py +Python/frozen_modules/getpath.h: Modules/getpath.py $(FREEZE_MODULE_C) $(FREEZE_MODULE_BOOTSTRAP) getpath $(srcdir)/Modules/getpath.py Python/frozen_modules/getpath.h # BEGIN: freezing modules -Python/frozen_modules/importlib._bootstrap.h: Lib/importlib/_bootstrap.py $(FREEZE_MODULE_C) $(srcdir)/Tools/scripts/freeze_module.py +Python/frozen_modules/importlib._bootstrap.h: Lib/importlib/_bootstrap.py $(FREEZE_MODULE_C) $(FREEZE_MODULE_BOOTSTRAP) importlib._bootstrap $(srcdir)/Lib/importlib/_bootstrap.py Python/frozen_modules/importlib._bootstrap.h -Python/frozen_modules/importlib._bootstrap_external.h: Lib/importlib/_bootstrap_external.py $(FREEZE_MODULE_C) $(srcdir)/Tools/scripts/freeze_module.py +Python/frozen_modules/importlib._bootstrap_external.h: Lib/importlib/_bootstrap_external.py $(FREEZE_MODULE_C) $(FREEZE_MODULE_BOOTSTRAP) importlib._bootstrap_external $(srcdir)/Lib/importlib/_bootstrap_external.py Python/frozen_modules/importlib._bootstrap_external.h -Python/frozen_modules/zipimport.h: Lib/zipimport.py $(FREEZE_MODULE_C) $(srcdir)/Tools/scripts/freeze_module.py +Python/frozen_modules/zipimport.h: Lib/zipimport.py $(FREEZE_MODULE_C) $(FREEZE_MODULE_BOOTSTRAP) zipimport $(srcdir)/Lib/zipimport.py Python/frozen_modules/zipimport.h -Python/frozen_modules/abc.h: Lib/abc.py _bootstrap_python $(srcdir)/Tools/scripts/freeze_module.py +Python/frozen_modules/abc.h: Lib/abc.py _bootstrap_python $(srcdir)/Programs/_freeze_module.py $(FREEZE_MODULE_PY) abc $(srcdir)/Lib/abc.py Python/frozen_modules/abc.h -Python/frozen_modules/codecs.h: Lib/codecs.py _bootstrap_python $(srcdir)/Tools/scripts/freeze_module.py +Python/frozen_modules/codecs.h: Lib/codecs.py _bootstrap_python $(srcdir)/Programs/_freeze_module.py $(FREEZE_MODULE_PY) codecs $(srcdir)/Lib/codecs.py Python/frozen_modules/codecs.h -Python/frozen_modules/io.h: Lib/io.py _bootstrap_python $(srcdir)/Tools/scripts/freeze_module.py +Python/frozen_modules/io.h: Lib/io.py _bootstrap_python $(srcdir)/Programs/_freeze_module.py $(FREEZE_MODULE_PY) io $(srcdir)/Lib/io.py Python/frozen_modules/io.h -Python/frozen_modules/_collections_abc.h: Lib/_collections_abc.py _bootstrap_python $(srcdir)/Tools/scripts/freeze_module.py +Python/frozen_modules/_collections_abc.h: Lib/_collections_abc.py _bootstrap_python $(srcdir)/Programs/_freeze_module.py $(FREEZE_MODULE_PY) _collections_abc $(srcdir)/Lib/_collections_abc.py Python/frozen_modules/_collections_abc.h -Python/frozen_modules/_sitebuiltins.h: Lib/_sitebuiltins.py _bootstrap_python $(srcdir)/Tools/scripts/freeze_module.py +Python/frozen_modules/_sitebuiltins.h: Lib/_sitebuiltins.py _bootstrap_python $(srcdir)/Programs/_freeze_module.py $(FREEZE_MODULE_PY) _sitebuiltins $(srcdir)/Lib/_sitebuiltins.py Python/frozen_modules/_sitebuiltins.h -Python/frozen_modules/genericpath.h: Lib/genericpath.py _bootstrap_python $(srcdir)/Tools/scripts/freeze_module.py +Python/frozen_modules/genericpath.h: Lib/genericpath.py _bootstrap_python $(srcdir)/Programs/_freeze_module.py $(FREEZE_MODULE_PY) genericpath $(srcdir)/Lib/genericpath.py Python/frozen_modules/genericpath.h -Python/frozen_modules/ntpath.h: Lib/ntpath.py _bootstrap_python $(srcdir)/Tools/scripts/freeze_module.py +Python/frozen_modules/ntpath.h: Lib/ntpath.py _bootstrap_python $(srcdir)/Programs/_freeze_module.py $(FREEZE_MODULE_PY) ntpath $(srcdir)/Lib/ntpath.py Python/frozen_modules/ntpath.h -Python/frozen_modules/posixpath.h: Lib/posixpath.py _bootstrap_python $(srcdir)/Tools/scripts/freeze_module.py +Python/frozen_modules/posixpath.h: Lib/posixpath.py _bootstrap_python $(srcdir)/Programs/_freeze_module.py $(FREEZE_MODULE_PY) posixpath $(srcdir)/Lib/posixpath.py Python/frozen_modules/posixpath.h -Python/frozen_modules/os.h: Lib/os.py _bootstrap_python $(srcdir)/Tools/scripts/freeze_module.py +Python/frozen_modules/os.h: Lib/os.py _bootstrap_python $(srcdir)/Programs/_freeze_module.py $(FREEZE_MODULE_PY) os $(srcdir)/Lib/os.py Python/frozen_modules/os.h -Python/frozen_modules/site.h: Lib/site.py _bootstrap_python $(srcdir)/Tools/scripts/freeze_module.py +Python/frozen_modules/site.h: Lib/site.py _bootstrap_python $(srcdir)/Programs/_freeze_module.py $(FREEZE_MODULE_PY) site $(srcdir)/Lib/site.py Python/frozen_modules/site.h -Python/frozen_modules/stat.h: Lib/stat.py _bootstrap_python $(srcdir)/Tools/scripts/freeze_module.py +Python/frozen_modules/stat.h: Lib/stat.py _bootstrap_python $(srcdir)/Programs/_freeze_module.py $(FREEZE_MODULE_PY) stat $(srcdir)/Lib/stat.py Python/frozen_modules/stat.h -Python/frozen_modules/importlib.util.h: Lib/importlib/util.py _bootstrap_python $(srcdir)/Tools/scripts/freeze_module.py +Python/frozen_modules/importlib.util.h: Lib/importlib/util.py _bootstrap_python $(srcdir)/Programs/_freeze_module.py $(FREEZE_MODULE_PY) importlib.util $(srcdir)/Lib/importlib/util.py Python/frozen_modules/importlib.util.h -Python/frozen_modules/importlib.machinery.h: Lib/importlib/machinery.py _bootstrap_python $(srcdir)/Tools/scripts/freeze_module.py +Python/frozen_modules/importlib.machinery.h: Lib/importlib/machinery.py _bootstrap_python $(srcdir)/Programs/_freeze_module.py $(FREEZE_MODULE_PY) importlib.machinery $(srcdir)/Lib/importlib/machinery.py Python/frozen_modules/importlib.machinery.h -Python/frozen_modules/runpy.h: Lib/runpy.py _bootstrap_python $(srcdir)/Tools/scripts/freeze_module.py +Python/frozen_modules/runpy.h: Lib/runpy.py _bootstrap_python $(srcdir)/Programs/_freeze_module.py $(FREEZE_MODULE_PY) runpy $(srcdir)/Lib/runpy.py Python/frozen_modules/runpy.h -Python/frozen_modules/__hello__.h: Lib/__hello__.py _bootstrap_python $(srcdir)/Tools/scripts/freeze_module.py +Python/frozen_modules/__hello__.h: Lib/__hello__.py _bootstrap_python $(srcdir)/Programs/_freeze_module.py $(FREEZE_MODULE_PY) __hello__ $(srcdir)/Lib/__hello__.py Python/frozen_modules/__hello__.h -Python/frozen_modules/__phello__.h: Lib/__phello__/__init__.py _bootstrap_python $(srcdir)/Tools/scripts/freeze_module.py +Python/frozen_modules/__phello__.h: Lib/__phello__/__init__.py _bootstrap_python $(srcdir)/Programs/_freeze_module.py $(FREEZE_MODULE_PY) __phello__ $(srcdir)/Lib/__phello__/__init__.py Python/frozen_modules/__phello__.h -Python/frozen_modules/__phello__.ham.h: Lib/__phello__/ham/__init__.py _bootstrap_python $(srcdir)/Tools/scripts/freeze_module.py +Python/frozen_modules/__phello__.ham.h: Lib/__phello__/ham/__init__.py _bootstrap_python $(srcdir)/Programs/_freeze_module.py $(FREEZE_MODULE_PY) __phello__.ham $(srcdir)/Lib/__phello__/ham/__init__.py Python/frozen_modules/__phello__.ham.h -Python/frozen_modules/__phello__.ham.eggs.h: Lib/__phello__/ham/eggs.py _bootstrap_python $(srcdir)/Tools/scripts/freeze_module.py +Python/frozen_modules/__phello__.ham.eggs.h: Lib/__phello__/ham/eggs.py _bootstrap_python $(srcdir)/Programs/_freeze_module.py $(FREEZE_MODULE_PY) __phello__.ham.eggs $(srcdir)/Lib/__phello__/ham/eggs.py Python/frozen_modules/__phello__.ham.eggs.h -Python/frozen_modules/__phello__.spam.h: Lib/__phello__/spam.py _bootstrap_python $(srcdir)/Tools/scripts/freeze_module.py +Python/frozen_modules/__phello__.spam.h: Lib/__phello__/spam.py _bootstrap_python $(srcdir)/Programs/_freeze_module.py $(FREEZE_MODULE_PY) __phello__.spam $(srcdir)/Lib/__phello__/spam.py Python/frozen_modules/__phello__.spam.h -Python/frozen_modules/frozen_only.h: Tools/freeze/flag.py _bootstrap_python $(srcdir)/Tools/scripts/freeze_module.py +Python/frozen_modules/frozen_only.h: Tools/freeze/flag.py _bootstrap_python $(srcdir)/Programs/_freeze_module.py $(FREEZE_MODULE_PY) frozen_only $(srcdir)/Tools/freeze/flag.py Python/frozen_modules/frozen_only.h # END: freezing modules diff --git a/Tools/scripts/freeze_module.py b/Programs/_freeze_module.py similarity index 96% rename from Tools/scripts/freeze_module.py rename to Programs/_freeze_module.py index 6b5ad35708f788..cc97e4195bfa2e 100644 --- a/Tools/scripts/freeze_module.py +++ b/Programs/_freeze_module.py @@ -12,7 +12,7 @@ import marshal import sys -header = "/* Auto-generated by Tools/scripts/freeze_module.py */" +header = "/* Auto-generated by Programs/_freeze_module.py */" def read_text(inpath: str) -> bytes: diff --git a/Tools/scripts/deepfreeze.py b/Tools/scripts/deepfreeze.py index 7cd7eb674d4511..002d680e10c2f6 100644 --- a/Tools/scripts/deepfreeze.py +++ b/Tools/scripts/deepfreeze.py @@ -394,7 +394,7 @@ def generate(self, name: str, obj: object) -> str: """ FROZEN_COMMENT_C = "/* Auto-generated by Programs/_freeze_module.c */" -FROZEN_COMMENT_PY = "/* Auto-generated by Tools/scripts/freeze_module.py */" +FROZEN_COMMENT_PY = "/* Auto-generated by Programs/_freeze_module.py */" FROZEN_DATA_LINE = r"\s*(\d+,\s*)+\s*" diff --git a/Tools/scripts/freeze_modules.py b/Tools/scripts/freeze_modules.py index 05d59cc6bcd452..f68a0932142bd9 100644 --- a/Tools/scripts/freeze_modules.py +++ b/Tools/scripts/freeze_modules.py @@ -594,12 +594,12 @@ def regen_makefile(modules): freezedep = '$(FREEZE_MODULE_C)' else: freezecmd = '$(FREEZE_MODULE_PY)' - freezedep = '_bootstrap_python' + freezedep = '_bootstrap_python $(srcdir)/Programs/_freeze_module.py' freeze = (f'{freezecmd} {src.frozenid} ' f'$(srcdir)/{pyfile} {frozen_header}') rules.extend([ - f'{frozen_header}: {pyfile} {freezedep} $(srcdir)/Tools/scripts/freeze_module.py', + f'{frozen_header}: {pyfile} {freezedep}', f'\t{freeze}', '', ]) From 0d328a9377a7eba677e5ae38f812bd51ad18a835 Mon Sep 17 00:00:00 2001 From: Christian Heimes Date: Sat, 11 Dec 2021 15:14:17 +0100 Subject: [PATCH 3/3] Address review comments - Fix typo - Remove confusing comment - Improve dependency handling for freezing --- Makefile.pre.in | 111 ++++++++++++++++---------------- Programs/_freeze_module.c | 2 + Programs/_freeze_module.py | 1 - Tools/scripts/freeze_modules.py | 6 +- configure | 20 +++++- configure.ac | 19 +++++- 6 files changed, 95 insertions(+), 64 deletions(-) diff --git a/Makefile.pre.in b/Makefile.pre.in index 4eff3ab415f193..fb114979a5dff9 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -283,13 +283,15 @@ BUILDPYTHON= python$(BUILDEXE) PYTHON_FOR_REGEN?=@PYTHON_FOR_REGEN@ UPDATE_FILE=$(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/update_file.py PYTHON_FOR_BUILD=@PYTHON_FOR_BUILD@ -# Standard builds use _bootstrap_python for freezing, cross compiling -# uses build Python, which must have the same version and bytecode, -PYTHON_FOR_FREEZE?=@PYTHON_FOR_FREEZE@ -FREEZE_MODULE_C=Programs/_freeze_module -FREEZE_MODULE_PY=$(PYTHON_FOR_FREEZE) $(srcdir)/Programs/_freeze_module.py -# bootstrap freezing uses FREEZE_MODULE_C for normal builds and _PY for cross builds + +# Normal builds use Programs/_freeze_module.c for bootstrapping and +# ./_bootstrap_python Programs/_freeze_module.py for remaining modules +# Cross builds use an external "build Python" for all modules. +PYTHON_FOR_FREEZE=@PYTHON_FOR_FREEZE@ FREEZE_MODULE_BOOTSTRAP=@FREEZE_MODULE_BOOTSTRAP@ +FREEZE_MODULE_BOOTSTRAP_DEPS=@FREEZE_MODULE_BOOTSTRAP_DEPS@ +FREEZE_MODULE=@FREEZE_MODULE@ +FREEZE_MODULE_DEPS=@FREEZE_MODULE_DEPS@ _PYTHON_HOST_PLATFORM=@_PYTHON_HOST_PLATFORM@ BUILD_GNU_TYPE= @build@ @@ -970,7 +972,7 @@ _bootstrap_python: $(LIBRARY_OBJS_OMIT_FROZEN) Programs/_bootstrap_python.o Modu .PHONY: regen-deepfreeze regen-deepfreeze: $(DEEPFREEZE_OBJS) -DEEPFREEZE_DEPS=$(srcdir)/Tools/scripts/deepfreeze.py _bootstrap_python +DEEPFREEZE_DEPS=$(srcdir)/Tools/scripts/deepfreeze.py $(FREEZE_MODULE_DEPS) # BEGIN: deepfreeze modules @@ -1066,14 +1068,11 @@ Python/deepfreeze/frozen_only.c: Python/frozen_modules/frozen_only.h $(DEEPFREEZ # script is used to test the cross compile code path. # 6) deepfreeze modules with _bootstrap_python # -# Cross compile proces: +# Cross compile process: # 1) create all frozen module headers with external build Python and # Programs/_freeze_module.py script. # 2) deepfreeze modules with external build Python. # -# Cross compiling also creates Programs/_freeze_module and _bootstrap_python -# artifacts, but does not use or ship them. -# # FROZEN_FILES_* are auto-generated by Tools/scripts/freeze_modules.py. FROZEN_FILES_IN = \ @@ -1131,83 +1130,83 @@ Programs/_freeze_module.o: Programs/_freeze_module.c Makefile Modules/getpath_noop.o: $(srcdir)/Modules/getpath_noop.c Makefile -$(FREEZE_MODULE_C): Programs/_freeze_module.o Modules/getpath_noop.o $(LIBRARY_OBJS_OMIT_FROZEN) +Programs/_freeze_module: Programs/_freeze_module.o Modules/getpath_noop.o $(LIBRARY_OBJS_OMIT_FROZEN) $(LINKCC) $(PY_CORE_LDFLAGS) -o $@ Programs/_freeze_module.o Modules/getpath_noop.o $(LIBRARY_OBJS_OMIT_FROZEN) $(LIBS) $(MODLIBS) $(SYSLIBS) # We manually freeze getpath.py rather than through freeze_modules -Python/frozen_modules/getpath.h: Modules/getpath.py $(FREEZE_MODULE_C) +Python/frozen_modules/getpath.h: Modules/getpath.py $(FREEZE_MODULE_BOOTSTRAP_DEPS) $(FREEZE_MODULE_BOOTSTRAP) getpath $(srcdir)/Modules/getpath.py Python/frozen_modules/getpath.h # BEGIN: freezing modules -Python/frozen_modules/importlib._bootstrap.h: Lib/importlib/_bootstrap.py $(FREEZE_MODULE_C) +Python/frozen_modules/importlib._bootstrap.h: Lib/importlib/_bootstrap.py $(FREEZE_MODULE_BOOTSTRAP_DEPS) $(FREEZE_MODULE_BOOTSTRAP) importlib._bootstrap $(srcdir)/Lib/importlib/_bootstrap.py Python/frozen_modules/importlib._bootstrap.h -Python/frozen_modules/importlib._bootstrap_external.h: Lib/importlib/_bootstrap_external.py $(FREEZE_MODULE_C) +Python/frozen_modules/importlib._bootstrap_external.h: Lib/importlib/_bootstrap_external.py $(FREEZE_MODULE_BOOTSTRAP_DEPS) $(FREEZE_MODULE_BOOTSTRAP) importlib._bootstrap_external $(srcdir)/Lib/importlib/_bootstrap_external.py Python/frozen_modules/importlib._bootstrap_external.h -Python/frozen_modules/zipimport.h: Lib/zipimport.py $(FREEZE_MODULE_C) +Python/frozen_modules/zipimport.h: Lib/zipimport.py $(FREEZE_MODULE_BOOTSTRAP_DEPS) $(FREEZE_MODULE_BOOTSTRAP) zipimport $(srcdir)/Lib/zipimport.py Python/frozen_modules/zipimport.h -Python/frozen_modules/abc.h: Lib/abc.py _bootstrap_python $(srcdir)/Programs/_freeze_module.py - $(FREEZE_MODULE_PY) abc $(srcdir)/Lib/abc.py Python/frozen_modules/abc.h +Python/frozen_modules/abc.h: Lib/abc.py $(FREEZE_MODULE_DEPS) + $(FREEZE_MODULE) abc $(srcdir)/Lib/abc.py Python/frozen_modules/abc.h -Python/frozen_modules/codecs.h: Lib/codecs.py _bootstrap_python $(srcdir)/Programs/_freeze_module.py - $(FREEZE_MODULE_PY) codecs $(srcdir)/Lib/codecs.py Python/frozen_modules/codecs.h +Python/frozen_modules/codecs.h: Lib/codecs.py $(FREEZE_MODULE_DEPS) + $(FREEZE_MODULE) codecs $(srcdir)/Lib/codecs.py Python/frozen_modules/codecs.h -Python/frozen_modules/io.h: Lib/io.py _bootstrap_python $(srcdir)/Programs/_freeze_module.py - $(FREEZE_MODULE_PY) io $(srcdir)/Lib/io.py Python/frozen_modules/io.h +Python/frozen_modules/io.h: Lib/io.py $(FREEZE_MODULE_DEPS) + $(FREEZE_MODULE) io $(srcdir)/Lib/io.py Python/frozen_modules/io.h -Python/frozen_modules/_collections_abc.h: Lib/_collections_abc.py _bootstrap_python $(srcdir)/Programs/_freeze_module.py - $(FREEZE_MODULE_PY) _collections_abc $(srcdir)/Lib/_collections_abc.py Python/frozen_modules/_collections_abc.h +Python/frozen_modules/_collections_abc.h: Lib/_collections_abc.py $(FREEZE_MODULE_DEPS) + $(FREEZE_MODULE) _collections_abc $(srcdir)/Lib/_collections_abc.py Python/frozen_modules/_collections_abc.h -Python/frozen_modules/_sitebuiltins.h: Lib/_sitebuiltins.py _bootstrap_python $(srcdir)/Programs/_freeze_module.py - $(FREEZE_MODULE_PY) _sitebuiltins $(srcdir)/Lib/_sitebuiltins.py Python/frozen_modules/_sitebuiltins.h +Python/frozen_modules/_sitebuiltins.h: Lib/_sitebuiltins.py $(FREEZE_MODULE_DEPS) + $(FREEZE_MODULE) _sitebuiltins $(srcdir)/Lib/_sitebuiltins.py Python/frozen_modules/_sitebuiltins.h -Python/frozen_modules/genericpath.h: Lib/genericpath.py _bootstrap_python $(srcdir)/Programs/_freeze_module.py - $(FREEZE_MODULE_PY) genericpath $(srcdir)/Lib/genericpath.py Python/frozen_modules/genericpath.h +Python/frozen_modules/genericpath.h: Lib/genericpath.py $(FREEZE_MODULE_DEPS) + $(FREEZE_MODULE) genericpath $(srcdir)/Lib/genericpath.py Python/frozen_modules/genericpath.h -Python/frozen_modules/ntpath.h: Lib/ntpath.py _bootstrap_python $(srcdir)/Programs/_freeze_module.py - $(FREEZE_MODULE_PY) ntpath $(srcdir)/Lib/ntpath.py Python/frozen_modules/ntpath.h +Python/frozen_modules/ntpath.h: Lib/ntpath.py $(FREEZE_MODULE_DEPS) + $(FREEZE_MODULE) ntpath $(srcdir)/Lib/ntpath.py Python/frozen_modules/ntpath.h -Python/frozen_modules/posixpath.h: Lib/posixpath.py _bootstrap_python $(srcdir)/Programs/_freeze_module.py - $(FREEZE_MODULE_PY) posixpath $(srcdir)/Lib/posixpath.py Python/frozen_modules/posixpath.h +Python/frozen_modules/posixpath.h: Lib/posixpath.py $(FREEZE_MODULE_DEPS) + $(FREEZE_MODULE) posixpath $(srcdir)/Lib/posixpath.py Python/frozen_modules/posixpath.h -Python/frozen_modules/os.h: Lib/os.py _bootstrap_python $(srcdir)/Programs/_freeze_module.py - $(FREEZE_MODULE_PY) os $(srcdir)/Lib/os.py Python/frozen_modules/os.h +Python/frozen_modules/os.h: Lib/os.py $(FREEZE_MODULE_DEPS) + $(FREEZE_MODULE) os $(srcdir)/Lib/os.py Python/frozen_modules/os.h -Python/frozen_modules/site.h: Lib/site.py _bootstrap_python $(srcdir)/Programs/_freeze_module.py - $(FREEZE_MODULE_PY) site $(srcdir)/Lib/site.py Python/frozen_modules/site.h +Python/frozen_modules/site.h: Lib/site.py $(FREEZE_MODULE_DEPS) + $(FREEZE_MODULE) site $(srcdir)/Lib/site.py Python/frozen_modules/site.h -Python/frozen_modules/stat.h: Lib/stat.py _bootstrap_python $(srcdir)/Programs/_freeze_module.py - $(FREEZE_MODULE_PY) stat $(srcdir)/Lib/stat.py Python/frozen_modules/stat.h +Python/frozen_modules/stat.h: Lib/stat.py $(FREEZE_MODULE_DEPS) + $(FREEZE_MODULE) stat $(srcdir)/Lib/stat.py Python/frozen_modules/stat.h -Python/frozen_modules/importlib.util.h: Lib/importlib/util.py _bootstrap_python $(srcdir)/Programs/_freeze_module.py - $(FREEZE_MODULE_PY) importlib.util $(srcdir)/Lib/importlib/util.py Python/frozen_modules/importlib.util.h +Python/frozen_modules/importlib.util.h: Lib/importlib/util.py $(FREEZE_MODULE_DEPS) + $(FREEZE_MODULE) importlib.util $(srcdir)/Lib/importlib/util.py Python/frozen_modules/importlib.util.h -Python/frozen_modules/importlib.machinery.h: Lib/importlib/machinery.py _bootstrap_python $(srcdir)/Programs/_freeze_module.py - $(FREEZE_MODULE_PY) importlib.machinery $(srcdir)/Lib/importlib/machinery.py Python/frozen_modules/importlib.machinery.h +Python/frozen_modules/importlib.machinery.h: Lib/importlib/machinery.py $(FREEZE_MODULE_DEPS) + $(FREEZE_MODULE) importlib.machinery $(srcdir)/Lib/importlib/machinery.py Python/frozen_modules/importlib.machinery.h -Python/frozen_modules/runpy.h: Lib/runpy.py _bootstrap_python $(srcdir)/Programs/_freeze_module.py - $(FREEZE_MODULE_PY) runpy $(srcdir)/Lib/runpy.py Python/frozen_modules/runpy.h +Python/frozen_modules/runpy.h: Lib/runpy.py $(FREEZE_MODULE_DEPS) + $(FREEZE_MODULE) runpy $(srcdir)/Lib/runpy.py Python/frozen_modules/runpy.h -Python/frozen_modules/__hello__.h: Lib/__hello__.py _bootstrap_python $(srcdir)/Programs/_freeze_module.py - $(FREEZE_MODULE_PY) __hello__ $(srcdir)/Lib/__hello__.py Python/frozen_modules/__hello__.h +Python/frozen_modules/__hello__.h: Lib/__hello__.py $(FREEZE_MODULE_DEPS) + $(FREEZE_MODULE) __hello__ $(srcdir)/Lib/__hello__.py Python/frozen_modules/__hello__.h -Python/frozen_modules/__phello__.h: Lib/__phello__/__init__.py _bootstrap_python $(srcdir)/Programs/_freeze_module.py - $(FREEZE_MODULE_PY) __phello__ $(srcdir)/Lib/__phello__/__init__.py Python/frozen_modules/__phello__.h +Python/frozen_modules/__phello__.h: Lib/__phello__/__init__.py $(FREEZE_MODULE_DEPS) + $(FREEZE_MODULE) __phello__ $(srcdir)/Lib/__phello__/__init__.py Python/frozen_modules/__phello__.h -Python/frozen_modules/__phello__.ham.h: Lib/__phello__/ham/__init__.py _bootstrap_python $(srcdir)/Programs/_freeze_module.py - $(FREEZE_MODULE_PY) __phello__.ham $(srcdir)/Lib/__phello__/ham/__init__.py Python/frozen_modules/__phello__.ham.h +Python/frozen_modules/__phello__.ham.h: Lib/__phello__/ham/__init__.py $(FREEZE_MODULE_DEPS) + $(FREEZE_MODULE) __phello__.ham $(srcdir)/Lib/__phello__/ham/__init__.py Python/frozen_modules/__phello__.ham.h -Python/frozen_modules/__phello__.ham.eggs.h: Lib/__phello__/ham/eggs.py _bootstrap_python $(srcdir)/Programs/_freeze_module.py - $(FREEZE_MODULE_PY) __phello__.ham.eggs $(srcdir)/Lib/__phello__/ham/eggs.py Python/frozen_modules/__phello__.ham.eggs.h +Python/frozen_modules/__phello__.ham.eggs.h: Lib/__phello__/ham/eggs.py $(FREEZE_MODULE_DEPS) + $(FREEZE_MODULE) __phello__.ham.eggs $(srcdir)/Lib/__phello__/ham/eggs.py Python/frozen_modules/__phello__.ham.eggs.h -Python/frozen_modules/__phello__.spam.h: Lib/__phello__/spam.py _bootstrap_python $(srcdir)/Programs/_freeze_module.py - $(FREEZE_MODULE_PY) __phello__.spam $(srcdir)/Lib/__phello__/spam.py Python/frozen_modules/__phello__.spam.h +Python/frozen_modules/__phello__.spam.h: Lib/__phello__/spam.py $(FREEZE_MODULE_DEPS) + $(FREEZE_MODULE) __phello__.spam $(srcdir)/Lib/__phello__/spam.py Python/frozen_modules/__phello__.spam.h -Python/frozen_modules/frozen_only.h: Tools/freeze/flag.py _bootstrap_python $(srcdir)/Programs/_freeze_module.py - $(FREEZE_MODULE_PY) frozen_only $(srcdir)/Tools/freeze/flag.py Python/frozen_modules/frozen_only.h +Python/frozen_modules/frozen_only.h: Tools/freeze/flag.py $(FREEZE_MODULE_DEPS) + $(FREEZE_MODULE) frozen_only $(srcdir)/Tools/freeze/flag.py Python/frozen_modules/frozen_only.h # END: freezing modules diff --git a/Programs/_freeze_module.c b/Programs/_freeze_module.c index e3f6c11c8b0731..d50787666f81cc 100644 --- a/Programs/_freeze_module.c +++ b/Programs/_freeze_module.c @@ -5,6 +5,8 @@ This is used directly by Tools/scripts/freeze_modules.py, and indirectly by "make regen-frozen". See Python/frozen.c for more info. + + Keep this file in sync with Programs/_freeze_module.py. */ #include diff --git a/Programs/_freeze_module.py b/Programs/_freeze_module.py index cc97e4195bfa2e..ba638eef6c4cd6 100644 --- a/Programs/_freeze_module.py +++ b/Programs/_freeze_module.py @@ -24,7 +24,6 @@ def compile_and_marshal(name: str, text: bytes) -> bytes: filename = f"" # exec == Py_file_input code = compile(text, filename, "exec", optimize=0, dont_inherit=True) - # defaults to Py_MARSHAL_VERSION) return marshal.dumps(code) diff --git a/Tools/scripts/freeze_modules.py b/Tools/scripts/freeze_modules.py index f68a0932142bd9..cbe8bf1ce60cdc 100644 --- a/Tools/scripts/freeze_modules.py +++ b/Tools/scripts/freeze_modules.py @@ -591,10 +591,10 @@ def regen_makefile(modules): if src.isbootstrap: freezecmd = '$(FREEZE_MODULE_BOOTSTRAP)' - freezedep = '$(FREEZE_MODULE_C)' + freezedep = '$(FREEZE_MODULE_BOOTSTRAP_DEPS)' else: - freezecmd = '$(FREEZE_MODULE_PY)' - freezedep = '_bootstrap_python $(srcdir)/Programs/_freeze_module.py' + freezecmd = '$(FREEZE_MODULE)' + freezedep = '$(FREEZE_MODULE_DEPS)' freeze = (f'{freezecmd} {src.frozenid} ' f'$(srcdir)/{pyfile} {frozen_header}') diff --git a/configure b/configure index e3ad4a8875c5dd..583e7d1fe3b32c 100755 --- a/configure +++ b/configure @@ -933,6 +933,9 @@ CONFIG_ARGS SOVERSION VERSION PYTHON_FOR_REGEN +FREEZE_MODULE_DEPS +FREEZE_MODULE +FREEZE_MODULE_BOOTSTRAP_DEPS FREEZE_MODULE_BOOTSTRAP PYTHON_FOR_FREEZE PYTHON_FOR_BUILD @@ -3240,13 +3243,26 @@ $as_echo "$PYTHON_FOR_FREEZE" >&6; } if test "x$cross_compiling" = xyes; then : - FREEZE_MODULE_BOOTSTRAP='$(FREEZE_MODULE_PY)' + + FREEZE_MODULE_BOOTSTRAP='$(PYTHON_FOR_FREEZE) $(srcdir)/Programs/_freeze_module.py' + FREEZE_MODULE_BOOTSTRAP_DEPS='$(srcdir)/Programs/_freeze_module.py' + FREEZE_MODULE='$(FREEZE_MODULE_BOOTSTRAP)' + FREEZE_MODULE_DEPS='$(FREEZE_MODULE_BOOTSTRAP_DEPS)' + else - FREEZE_MODULE_BOOTSTRAP='$(FREEZE_MODULE_C)' + + FREEZE_MODULE_BOOTSTRAP='./Programs/_freeze_module' + FREEZE_MODULE_BOOTSTRAP_DEPS="Programs/_freeze_module" + FREEZE_MODULE='$(PYTHON_FOR_FREEZE) $(srcdir)/Programs/_freeze_module.py' + FREEZE_MODULE_DEPS="_bootstrap_python \$(srcdir)/Programs/_freeze_module.py" + fi + + + for ac_prog in python$PACKAGE_VERSION python3.10 python3.9 python3.8 python3.7 python3.6 python3 python do # Extract the first word of "$ac_prog", so it can be a program name with args. diff --git a/configure.ac b/configure.ac index f5e6ea46b455ee..5256a61289e223 100644 --- a/configure.ac +++ b/configure.ac @@ -137,10 +137,25 @@ AC_MSG_RESULT([$PYTHON_FOR_FREEZE]) AC_SUBST([PYTHON_FOR_FREEZE]) AS_VAR_IF([cross_compiling], [yes], - [FREEZE_MODULE_BOOTSTRAP='$(FREEZE_MODULE_PY)'], - [FREEZE_MODULE_BOOTSTRAP='$(FREEZE_MODULE_C)'] + [ + dnl external build Python, freezing depends on Programs/_freeze_module.py + FREEZE_MODULE_BOOTSTRAP='$(PYTHON_FOR_FREEZE) $(srcdir)/Programs/_freeze_module.py' + FREEZE_MODULE_BOOTSTRAP_DEPS='$(srcdir)/Programs/_freeze_module.py' + FREEZE_MODULE='$(FREEZE_MODULE_BOOTSTRAP)' + FREEZE_MODULE_DEPS='$(FREEZE_MODULE_BOOTSTRAP_DEPS)' + ], + [ + dnl internal build tools also depend on Programs/_freeze_module and _bootstrap_python. + FREEZE_MODULE_BOOTSTRAP='./Programs/_freeze_module' + FREEZE_MODULE_BOOTSTRAP_DEPS="Programs/_freeze_module" + FREEZE_MODULE='$(PYTHON_FOR_FREEZE) $(srcdir)/Programs/_freeze_module.py' + FREEZE_MODULE_DEPS="_bootstrap_python \$(srcdir)/Programs/_freeze_module.py" + ] ) AC_SUBST([FREEZE_MODULE_BOOTSTRAP]) +AC_SUBST([FREEZE_MODULE_BOOTSTRAP_DEPS]) +AC_SUBST([FREEZE_MODULE]) +AC_SUBST([FREEZE_MODULE_DEPS]) AC_CHECK_PROGS([PYTHON_FOR_REGEN], [python$PACKAGE_VERSION python3.10 python3.9 python3.8 python3.7 python3.6 python3 python],