From 919efccc188c0175056053164a70cff9d6fd3ef6 Mon Sep 17 00:00:00 2001 From: Ronald Oussoren Date: Mon, 20 Jul 2020 13:42:17 +0200 Subject: [PATCH 01/11] Add support for ARM64 to the pythonw tool --- Mac/Tools/pythonw.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/Mac/Tools/pythonw.c b/Mac/Tools/pythonw.c index c8bd3ba8d68c15..8d65da79517420 100644 --- a/Mac/Tools/pythonw.c +++ b/Mac/Tools/pythonw.c @@ -95,9 +95,6 @@ setup_spawnattr(posix_spawnattr_t* spawnattr) size_t count; cpu_type_t cpu_types[1]; short flags = 0; -#ifdef __LP64__ - int ch; -#endif if ((errno = posix_spawnattr_init(spawnattr)) != 0) { err(2, "posix_spawnattr_int"); @@ -119,10 +116,16 @@ setup_spawnattr(posix_spawnattr_t* spawnattr) #elif defined(__ppc__) cpu_types[0] = CPU_TYPE_POWERPC; + #elif defined(__i386__) cpu_types[0] = CPU_TYPE_X86; + +#elif defined(__arm64__) + cpu_types[0] = CPU_TYPE_ARM64; + #else # error "Unknown CPU" + #endif if (posix_spawnattr_setbinpref_np(spawnattr, count, From 69c39f36f6efe9b13b2dc06892133ac31e6ca7bc Mon Sep 17 00:00:00 2001 From: Ronald Oussoren Date: Mon, 20 Jul 2020 13:42:43 +0200 Subject: [PATCH 02/11] Add support for "universal2" as a fat binary target on macOS --- Lib/_osx_support.py | 2 ++ configure | 20 ++++++++++++++------ configure.ac | 18 +++++++++++++----- 3 files changed, 29 insertions(+), 11 deletions(-) diff --git a/Lib/_osx_support.py b/Lib/_osx_support.py index e9efce7d7ed5bd..8ed1eeac8525da 100644 --- a/Lib/_osx_support.py +++ b/Lib/_osx_support.py @@ -481,6 +481,8 @@ def get_platform_osx(_config_vars, osname, release, machine): if len(archs) == 1: machine = archs[0] + elif archs == ('arm64', 'x86_64'): + machine = 'universal2' elif archs == ('i386', 'ppc'): machine = 'fat' elif archs == ('i386', 'x86_64'): diff --git a/configure b/configure index 5024860ca4395a..f811d2c23e4a7a 100755 --- a/configure +++ b/configure @@ -1509,8 +1509,8 @@ Optional Packages: specify the kind of universal binary that should be created. this option is only valid when --enable-universalsdk is set; options are: - ("32-bit", "64-bit", "3-way", "intel", "intel-32", - "intel-64", or "all") see Mac/README.rst + ("universal2", "32-bit", "64-bit", "3-way", "intel", + "intel-32", "intel-64", or "all") see Mac/README.rst --with-framework-name=FRAMEWORK specify the name for the python framework on macOS only valid when --enable-framework is set. see @@ -6945,7 +6945,7 @@ fi -# The -arch flags for universal builds on OSX +# The -arch flags for universal builds on macOS UNIVERSAL_ARCH_FLAGS= @@ -7472,6 +7472,11 @@ $as_echo "$CC" >&6; } LIPO_32BIT_FLAGS="-extract ppc7400 -extract i386" ARCH_RUN_32BIT="/usr/bin/arch -i386 -ppc" ;; + universal2) + UNIVERSAL_ARCH_FLAGS="-arch arm64 -arch x86_64" + LIPO_32BIT_FLAGS="" + ARCH_RUN_32BIT="true" + ;; intel) UNIVERSAL_ARCH_FLAGS="-arch i386 -arch x86_64" LIPO_32BIT_FLAGS="-extract i386" @@ -7493,7 +7498,7 @@ $as_echo "$CC" >&6; } ARCH_RUN_32BIT="/usr/bin/arch -i386 -ppc" ;; *) - as_fn_error $? "proper usage is --with-universal-arch=32-bit|64-bit|all|intel|3-way" "$LINENO" 5 + as_fn_error $? "proper usage is --with-universal-arch=universal2|32-bit|64-bit|all|intel|3-way" "$LINENO" 5 ;; esac @@ -9322,7 +9327,7 @@ fi MACOSX_DEFAULT_ARCH="ppc" ;; *) - as_fn_error $? "Unexpected output of 'arch' on OSX" "$LINENO" 5 + as_fn_error $? "Unexpected output of 'arch' on macOS" "$LINENO" 5 ;; esac else @@ -9332,9 +9337,12 @@ fi ;; ppc) MACOSX_DEFAULT_ARCH="ppc64" + ;; + arm64) + MACOSX_DEFAULT_ARCH="arm64" ;; *) - as_fn_error $? "Unexpected output of 'arch' on OSX" "$LINENO" 5 + as_fn_error $? "Unexpected output of 'arch' on macOS" "$LINENO" 5 ;; esac diff --git a/configure.ac b/configure.ac index 5a3e340aa3e72b..d2044d0520e679 100644 --- a/configure.ac +++ b/configure.ac @@ -218,7 +218,7 @@ AC_ARG_WITH(universal-archs, AS_HELP_STRING([--with-universal-archs=ARCH], [specify the kind of universal binary that should be created. this option is only valid when --enable-universalsdk is set; options are: - ("32-bit", "64-bit", "3-way", "intel", "intel-32", "intel-64", or "all") + ("universal2", "32-bit", "64-bit", "3-way", "intel", "intel-32", "intel-64", or "all") see Mac/README.rst]), [ UNIVERSAL_ARCHS="$withval" @@ -1578,7 +1578,7 @@ AC_SUBST(BASECFLAGS) AC_SUBST(CFLAGS_NODIST) AC_SUBST(LDFLAGS_NODIST) -# The -arch flags for universal builds on OSX +# The -arch flags for universal builds on macOS UNIVERSAL_ARCH_FLAGS= AC_SUBST(UNIVERSAL_ARCH_FLAGS) @@ -1879,6 +1879,11 @@ yes) LIPO_32BIT_FLAGS="-extract ppc7400 -extract i386" ARCH_RUN_32BIT="/usr/bin/arch -i386 -ppc" ;; + universal2) + UNIVERSAL_ARCH_FLAGS="-arch arm64 -arch x86_64" + LIPO_32BIT_FLAGS="" + ARCH_RUN_32BIT="true" + ;; intel) UNIVERSAL_ARCH_FLAGS="-arch i386 -arch x86_64" LIPO_32BIT_FLAGS="-extract i386" @@ -1900,7 +1905,7 @@ yes) ARCH_RUN_32BIT="/usr/bin/arch -i386 -ppc" ;; *) - AC_MSG_ERROR([proper usage is --with-universal-arch=32-bit|64-bit|all|intel|3-way]) + AC_MSG_ERROR([proper usage is --with-universal-arch=universal2|32-bit|64-bit|all|intel|3-way]) ;; esac @@ -2467,7 +2472,7 @@ case $ac_sys_system/$ac_sys_release in MACOSX_DEFAULT_ARCH="ppc" ;; *) - AC_MSG_ERROR([Unexpected output of 'arch' on OSX]) + AC_MSG_ERROR([Unexpected output of 'arch' on macOS]) ;; esac else @@ -2477,9 +2482,12 @@ case $ac_sys_system/$ac_sys_release in ;; ppc) MACOSX_DEFAULT_ARCH="ppc64" + ;; + arm64) + MACOSX_DEFAULT_ARCH="arm64" ;; *) - AC_MSG_ERROR([Unexpected output of 'arch' on OSX]) + AC_MSG_ERROR([Unexpected output of 'arch' on macOS]) ;; esac From 3940c867b6394d8950bfff725133e8a8241081b0 Mon Sep 17 00:00:00 2001 From: Ronald Oussoren Date: Mon, 20 Jul 2020 15:47:44 +0200 Subject: [PATCH 03/11] Tweak ctypes.macholib.dyld to work with the shared library cache. --- Lib/ctypes/macholib/dyld.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/Lib/ctypes/macholib/dyld.py b/Lib/ctypes/macholib/dyld.py index 9d86b058765a3e..9ab9e79ab816c4 100644 --- a/Lib/ctypes/macholib/dyld.py +++ b/Lib/ctypes/macholib/dyld.py @@ -122,7 +122,19 @@ def dyld_find(name, executable_path=None, env=None): dyld_executable_path_search(name, executable_path), dyld_default_search(name, env), ), env): - if os.path.isfile(path): + + # on macOS 11 system libraries are in a shared library + # cache, not in the regular place in the filesystem. + # There still are symlinks (libz.dylib -> libz.1.dylib), + # but the target of the symlink no longer is there. + # + # There shouldn't be 3th-party libraries in these + # system locations + if path.startswith("/System/") and os.path.islink(path): + return path + elif path.startswith("/usr/lib/") and os.path.islink(path): + return path + elif os.path.isfile(path): return path raise ValueError("dylib %s could not be found" % (name,)) From deda5f0c830517969c619821f5d932495d7d27ef Mon Sep 17 00:00:00 2001 From: Ronald Oussoren Date: Mon, 20 Jul 2020 16:11:40 +0200 Subject: [PATCH 04/11] Silence compile time warning We should consider just dropping support for macOS 10.4 here to simplify the code. --- Mac/Tools/pythonw.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Mac/Tools/pythonw.c b/Mac/Tools/pythonw.c index 8d65da79517420..78813e818e7dac 100644 --- a/Mac/Tools/pythonw.c +++ b/Mac/Tools/pythonw.c @@ -223,7 +223,8 @@ main(int argc, char **argv) { /* We're weak-linking to posix-spawnv to ensure that * an executable build on 10.5 can work on 10.4. */ - if (posix_spawn != NULL) { + + if (&posix_spawn != NULL) { posix_spawnattr_t spawnattr = NULL; setup_spawnattr(&spawnattr); From ea3c200890e1dc606745f6a5308725018595d001 Mon Sep 17 00:00:00 2001 From: Ronald Oussoren Date: Mon, 20 Jul 2020 16:12:37 +0200 Subject: [PATCH 05/11] macOS/arm64 support, based on GH-21249 This is support for ctypes on macOS/arm64 based on PR 21249 by Lawrence D'Anna (Apple). Changes: - changed __builtin_available tests from 11.0 to 10.15 - added test to setup.py for ffi_closure_alloc and use that in malloc_closure.c - Minor change in the code path for ffi_prep_closure_var (coding style change) --- Lib/test/test_bytes.py | 1 + Lib/test/test_unicode.py | 2 + Modules/_ctypes/callbacks.c | 39 +++++++++--- Modules/_ctypes/callproc.c | 72 ++++++++++++++++++---- Modules/_ctypes/ctypes.h | 8 +++ Modules/_ctypes/malloc_closure.c | 15 ++++- setup.py | 102 ++++++++++++++++--------------- 7 files changed, 168 insertions(+), 71 deletions(-) diff --git a/Lib/test/test_bytes.py b/Lib/test/test_bytes.py index 61b4b9162ccc54..e26ea4704bf5c7 100644 --- a/Lib/test/test_bytes.py +++ b/Lib/test/test_bytes.py @@ -1035,6 +1035,7 @@ def test_from_format(self): c_char_p) PyBytes_FromFormat = pythonapi.PyBytes_FromFormat + PyBytes_FromFormat.argtypes = (c_char_p,) PyBytes_FromFormat.restype = py_object # basic tests diff --git a/Lib/test/test_unicode.py b/Lib/test/test_unicode.py index d485bc7ede2b92..4cf60591d2dd4d 100644 --- a/Lib/test/test_unicode.py +++ b/Lib/test/test_unicode.py @@ -2513,11 +2513,13 @@ class CAPITest(unittest.TestCase): def test_from_format(self): import_helper.import_module('ctypes') from ctypes import ( + c_char_p, pythonapi, py_object, sizeof, c_int, c_long, c_longlong, c_ssize_t, c_uint, c_ulong, c_ulonglong, c_size_t, c_void_p) name = "PyUnicode_FromFormat" _PyUnicode_FromFormat = getattr(pythonapi, name) + _PyUnicode_FromFormat.argtypes = (c_char_p,) _PyUnicode_FromFormat.restype = py_object def PyUnicode_FromFormat(format, *args): diff --git a/Modules/_ctypes/callbacks.c b/Modules/_ctypes/callbacks.c index 2abfa67cdc06bf..77d4855277ecec 100644 --- a/Modules/_ctypes/callbacks.c +++ b/Modules/_ctypes/callbacks.c @@ -1,6 +1,8 @@ #include "Python.h" #include "frameobject.h" +#include + #include #ifdef MS_WIN32 #include @@ -18,7 +20,7 @@ CThunkObject_dealloc(PyObject *myself) Py_XDECREF(self->callable); Py_XDECREF(self->restype); if (self->pcl_write) - ffi_closure_free(self->pcl_write); + Py_ffi_closure_free(self->pcl_write); PyObject_GC_Del(self); } @@ -361,8 +363,7 @@ CThunkObject *_ctypes_alloc_callback(PyObject *callable, assert(CThunk_CheckExact((PyObject *)p)); - p->pcl_write = ffi_closure_alloc(sizeof(ffi_closure), - &p->pcl_exec); + p->pcl_write = Py_ffi_closure_alloc(sizeof(ffi_closure), &p->pcl_exec); if (p->pcl_write == NULL) { PyErr_NoMemory(); goto error; @@ -408,13 +409,35 @@ CThunkObject *_ctypes_alloc_callback(PyObject *callable, "ffi_prep_cif failed with %d", result); goto error; } -#if defined(X86_DARWIN) || defined(POWERPC_DARWIN) - result = ffi_prep_closure(p->pcl_write, &p->cif, closure_fcn, p); +#if HAVE_FFI_PREP_CLOSURE_LOC +# if USING_APPLE_OS_LIBFFI +# define HAVE_FFI_PREP_CLOSURE_LOC_RUNTIME __builtin_available(macos 10.15, ios 13, watchos 6, tvos 13, *) +# else +# define HAVE_FFI_PREP_CLOSURE_LOC_RUNTIME true +# endif + if (HAVE_FFI_PREP_CLOSURE_LOC_RUNTIME) { + result = ffi_prep_closure_loc(p->pcl_write, &p->cif, closure_fcn, + p, + p->pcl_exec); + } else +#endif + { +#if USING_APPLE_OS_LIBFFI && defined(__arm64__) + PyErr_Format(PyExc_NotImplementedError, "ffi_prep_closure_loc() is missing"); + goto error; #else - result = ffi_prep_closure_loc(p->pcl_write, &p->cif, closure_fcn, - p, - p->pcl_exec); +#ifdef MACOSX + #pragma clang diagnostic push + #pragma clang diagnostic ignored "-Wdeprecated-declarations" #endif + result = ffi_prep_closure(p->pcl_write, &p->cif, closure_fcn, p); + +#ifdef MACOSX + #pragma clang diagnostic pop +#endif + +#endif + } if (result != FFI_OK) { PyErr_Format(PyExc_RuntimeError, "ffi_prep_closure failed with %d", result); diff --git a/Modules/_ctypes/callproc.c b/Modules/_ctypes/callproc.c index 261ae5ceb9e48a..ac407c93c240e4 100644 --- a/Modules/_ctypes/callproc.c +++ b/Modules/_ctypes/callproc.c @@ -57,6 +57,8 @@ #include "Python.h" #include "structmember.h" // PyMemberDef +#include + #ifdef MS_WIN32 #include #include @@ -812,7 +814,8 @@ static int _call_function_pointer(int flags, ffi_type **atypes, ffi_type *restype, void *resmem, - int argcount) + int argcount, + int argtypecount) { PyThreadState *_save = NULL; /* For Py_BLOCK_THREADS and Py_UNBLOCK_THREADS */ PyObject *error_object = NULL; @@ -835,14 +838,60 @@ static int _call_function_pointer(int flags, if ((flags & FUNCFLAG_CDECL) == 0) cc = FFI_STDCALL; #endif - if (FFI_OK != ffi_prep_cif(&cif, - cc, - argcount, - restype, - atypes)) { - PyErr_SetString(PyExc_RuntimeError, - "ffi_prep_cif failed"); - return -1; + +# if USING_APPLE_OS_LIBFFI +# define HAVE_FFI_PREP_CIF_VAR_RUNTIME __builtin_available(macos 10.15, ios 13, watchos 6, tvos 13, *) +# elif HAVE_FFI_PREP_CIF_VAR +# define HAVE_FFI_PREP_CIF_VAR_RUNTIME true +# else +# define HAVE_FFI_PREP_CIF_VAR_RUNTIME false +# endif + + /* Even on Apple-arm64 the calling convention for variadic functions conincides + * with the standard calling convention in the case that the function called + * only with its fixed arguments. Thus, we do not need a special flag to be + * set on variadic functions. We treat a function as variadic if it is called + * with a nonzero number of variadic arguments */ + bool is_variadic = (argtypecount != 0 && argcount > argtypecount); + (void) is_variadic; + +#if defined(__APPLE__) && defined(__arm64__) + if (is_variadic) { + if (HAVE_FFI_PREP_CIF_VAR_RUNTIME) { + } else { + PyErr_SetString(PyExc_NotImplementedError, "ffi_prep_cif_var() is missing"); + return -1; + } + } +#endif + +#if HAVE_FFI_PREP_CIF_VAR + if (is_variadic) { + if (HAVE_FFI_PREP_CIF_VAR_RUNTIME) { + if (FFI_OK != ffi_prep_cif_var(&cif, + cc, + argtypecount, + argcount, + restype, + atypes)) { + PyErr_SetString(PyExc_RuntimeError, + "ffi_prep_cif_var failed"); + return -1; + } + } + } else +#endif + + { + if (FFI_OK != ffi_prep_cif(&cif, + cc, + argcount, + restype, + atypes)) { + PyErr_SetString(PyExc_RuntimeError, + "ffi_prep_cif failed"); + return -1; + } } if (flags & (FUNCFLAG_USE_ERRNO | FUNCFLAG_USE_LASTERROR)) { @@ -1212,9 +1261,8 @@ PyObject *_ctypes_callproc(PPROC pProc, if (-1 == _call_function_pointer(flags, pProc, avalues, atypes, rtype, resbuf, - Py_SAFE_DOWNCAST(argcount, - Py_ssize_t, - int))) + Py_SAFE_DOWNCAST(argcount, Py_ssize_t, int), + Py_SAFE_DOWNCAST(argtype_count, Py_ssize_t, int))) goto cleanup; #ifdef WORDS_BIGENDIAN diff --git a/Modules/_ctypes/ctypes.h b/Modules/_ctypes/ctypes.h index 1effccf9cc5ff9..3f20031d671a8a 100644 --- a/Modules/_ctypes/ctypes.h +++ b/Modules/_ctypes/ctypes.h @@ -366,6 +366,14 @@ PyObject *_ctypes_get_errobj(int **pspace); extern PyObject *ComError; #endif +#if USING_MALLOC_CLOSURE_DOT_C +void Py_ffi_closure_free(void *p); +void *Py_ffi_closure_alloc(size_t size, void** codeloc); +#else +#define Py_ffi_closure_free ffi_closure_free +#define Py_ffi_closure_alloc ffi_closure_alloc +#endif + /* Local Variables: compile-command: "python setup.py -q build install --home ~" diff --git a/Modules/_ctypes/malloc_closure.c b/Modules/_ctypes/malloc_closure.c index f9cdb336958c6f..4f220e42ff3fcc 100644 --- a/Modules/_ctypes/malloc_closure.c +++ b/Modules/_ctypes/malloc_closure.c @@ -89,16 +89,27 @@ static void more_core(void) /******************************************************************/ /* put the item back into the free list */ -void ffi_closure_free(void *p) +void Py_ffi_closure_free(void *p) { +#if USING_APPLE_OS_LIBFFI && HAVE_FFI_CLOSURE_ALLOC + if (__builtin_available(macos 10.15, ios 13, watchos 6, tvos 13, *)) { + ffi_closure_free(p); + return; + } +#endif ITEM *item = (ITEM *)p; item->next = free_list; free_list = item; } /* return one item from the free list, allocating more if needed */ -void *ffi_closure_alloc(size_t ignored, void** codeloc) +void *Py_ffi_closure_alloc(size_t size, void** codeloc) { +#if USING_APPLE_OS_LIBFFI && HAVE_FFI_CLOSURE_ALLOC + if (__builtin_available(macos 10.15, ios 13, watchos 6, tvos 13, *)) { + return ffi_closure_alloc(size, codeloc); + } +#endif ITEM *item; if (!free_list) more_core(); diff --git a/setup.py b/setup.py index 21a5a58981fc15..65d479fcb184c6 100644 --- a/setup.py +++ b/setup.py @@ -229,6 +229,13 @@ def macosx_sdk_specified(): macosx_sdk_root() return MACOS_SDK_SPECIFIED +def is_macosx_at_least(vers): + if MACOS: + dep_target = sysconfig.get_config_var('MACOSX_DEPLOYMENT_TARGET') + if dep_target: + return tuple(map(int, dep_target.split('.'))) >= vers + return False + def is_macosx_sdk_path(path): """ @@ -239,6 +246,13 @@ def is_macosx_sdk_path(path): or path.startswith('/Library/') ) +def grep_headers_for(function, headers): + for header in headers: + with open(header, 'r') as f: + if function in f.read(): + return True + return False + def find_file(filename, std_dirs, paths): """Searches for the directory where a given file is located, and returns a possibly-empty list of additional directories, or None @@ -2100,43 +2114,18 @@ def detect_tkinter(self): library_dirs=added_lib_dirs)) return True - def configure_ctypes_darwin(self, ext): - # Darwin (OS X) uses preconfigured files, in - # the Modules/_ctypes/libffi_osx directory. - ffi_srcdir = os.path.abspath(os.path.join(self.srcdir, 'Modules', - '_ctypes', 'libffi_osx')) - sources = [os.path.join(ffi_srcdir, p) - for p in ['ffi.c', - 'x86/darwin64.S', - 'x86/x86-darwin.S', - 'x86/x86-ffi_darwin.c', - 'x86/x86-ffi64.c', - 'powerpc/ppc-darwin.S', - 'powerpc/ppc-darwin_closure.S', - 'powerpc/ppc-ffi_darwin.c', - 'powerpc/ppc64-darwin_closure.S', - ]] - - # Add .S (preprocessed assembly) to C compiler source extensions. - self.compiler.src_extensions.append('.S') - - include_dirs = [os.path.join(ffi_srcdir, 'include'), - os.path.join(ffi_srcdir, 'powerpc')] - ext.include_dirs.extend(include_dirs) - ext.sources.extend(sources) - return True - def configure_ctypes(self, ext): - if not self.use_system_libffi: - if MACOS: - return self.configure_ctypes_darwin(ext) - print('INFO: Could not locate ffi libs and/or headers') - return False return True def detect_ctypes(self): # Thomas Heller's _ctypes module - self.use_system_libffi = False + + if (not sysconfig.get_config_var("LIBFFI_INCLUDEDIR") and MACOS and + (is_macosx_at_least((10,15)) or '-arch arm64' in sysconfig.get_config_var("CFLAGS"))): + self.use_system_libffi = True + else: + self.use_system_libffi = '--with-system-ffi' in sysconfig.get_config_var("CONFIG_ARGS") + include_dirs = [] extra_compile_args = ['-DPy_BUILD_CORE_MODULE'] extra_link_args = [] @@ -2149,11 +2138,10 @@ def detect_ctypes(self): if MACOS: sources.append('_ctypes/malloc_closure.c') - sources.append('_ctypes/darwin/dlfcn_simple.c') + extra_compile_args.append('-DUSING_MALLOC_CLOSURE_DOT_C=1') + #sources.append('_ctypes/darwin/dlfcn_simple.c') extra_compile_args.append('-DMACOSX') include_dirs.append('_ctypes/darwin') - # XXX Is this still needed? - # extra_link_args.extend(['-read_only_relocs', 'warning']) elif HOST_PLATFORM == 'sunos5': # XXX This shouldn't be necessary; it appears that some @@ -2183,31 +2171,47 @@ def detect_ctypes(self): sources=['_ctypes/_ctypes_test.c'], libraries=['m'])) + ffi_inc = sysconfig.get_config_var("LIBFFI_INCLUDEDIR") + ffi_lib = None + ffi_inc_dirs = self.inc_dirs.copy() if MACOS: - if '--with-system-ffi' not in sysconfig.get_config_var("CONFIG_ARGS"): - return - # OS X 10.5 comes with libffi.dylib; the include files are - # in /usr/include/ffi - ffi_inc_dirs.append('/usr/include/ffi') - - ffi_inc = [sysconfig.get_config_var("LIBFFI_INCLUDEDIR")] - if not ffi_inc or ffi_inc[0] == '': - ffi_inc = find_file('ffi.h', [], ffi_inc_dirs) - if ffi_inc is not None: - ffi_h = ffi_inc[0] + '/ffi.h' + # XXX: The define should only be added when actually using the system + # version (and not a locally compiled one) + ext.extra_compile_args.append("-DUSING_APPLE_OS_LIBFFI=1") + ffi_in_sdk = os.path.join(macosx_sdk_root(), "usr/include/ffi") + if os.path.exists(ffi_in_sdk): + ffi_inc = ffi_in_sdk + ffi_lib = 'ffi' + else: + # OS X 10.5 comes with libffi.dylib; the include files are + # in /usr/include/ffi + ffi_inc_dirs.append('/usr/include/ffi') + + if not ffi_inc: + found = find_file('ffi.h', [], ffi_inc_dirs) + if found: + ffi_inc = found[0] + if ffi_inc: + ffi_h = ffi_inc + '/ffi.h' if not os.path.exists(ffi_h): ffi_inc = None print('Header file {} does not exist'.format(ffi_h)) - ffi_lib = None - if ffi_inc is not None: + if ffi_lib is None and ffi_inc: for lib_name in ('ffi', 'ffi_pic'): if (self.compiler.find_library_file(self.lib_dirs, lib_name)): ffi_lib = lib_name break if ffi_inc and ffi_lib: - ext.include_dirs.extend(ffi_inc) + ffi_headers = glob(os.path.join(ffi_inc, '*.h')) + if grep_headers_for('ffi_prep_cif_var', ffi_headers): + ext.extra_compile_args.append("-DHAVE_FFI_PREP_CIF_VAR=1") + if grep_headers_for('ffi_prep_closure_loc', ffi_headers): + ext.extra_compile_args.append("-DHAVE_FFI_PREP_CLOSURE_LOC=1") + if grep_headers_for('ffi_closure_alloc', ffi_headers): + ext.extra_compile_args.append("-DHAVE_FFI_CLOSURE_ALLOC=1") + ext.include_dirs.append(ffi_inc) ext.libraries.append(ffi_lib) self.use_system_libffi = True From 552bca86a2e7049d0483b779a7776245608d172c Mon Sep 17 00:00:00 2001 From: Ronald Oussoren Date: Mon, 20 Jul 2020 16:23:50 +0200 Subject: [PATCH 06/11] Fix test for macOS before 10.10 --- Lib/distutils/tests/test_build_ext.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/distutils/tests/test_build_ext.py b/Lib/distutils/tests/test_build_ext.py index f9e0d766d870e0..6bb009a86f41eb 100644 --- a/Lib/distutils/tests/test_build_ext.py +++ b/Lib/distutils/tests/test_build_ext.py @@ -493,7 +493,7 @@ def _try_compile_deployment_target(self, operator, target): # format the target value as defined in the Apple # Availability Macros. We can't use the macro names since # at least one value we test with will not exist yet. - if target[1] < 10: + if target[:2] < (10, 10): # for 10.1 through 10.9.x -> "10n0" target = '%02d%01d0' % target else: From e0c23a1d14446079e5cbee0b1d1624f242eebbc5 Mon Sep 17 00:00:00 2001 From: Ronald Oussoren Date: Mon, 20 Jul 2020 16:25:35 +0200 Subject: [PATCH 07/11] ARM64 is a valid platform for macOS --- Lib/test/test_platform.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_platform.py b/Lib/test/test_platform.py index 5ad306e0ed5795..45ca08819fb36c 100644 --- a/Lib/test/test_platform.py +++ b/Lib/test/test_platform.py @@ -246,7 +246,7 @@ def test_mac_ver(self): self.assertEqual(res[1], ('', '', '')) if sys.byteorder == 'little': - self.assertIn(res[2], ('i386', 'x86_64')) + self.assertIn(res[2], ('i386', 'x86_64', 'arm64')) else: self.assertEqual(res[2], 'PowerPC') From e637a77f3017d3149b90c25932805112b52d43c4 Mon Sep 17 00:00:00 2001 From: Ronald Oussoren Date: Mon, 20 Jul 2020 17:21:37 +0200 Subject: [PATCH 08/11] Unconditionally use uint32_t here. The preprocessor guard in the old version doesn't work, and isn't really needed (10.4 only supported 32-bit code where unsigned long is the same as uint32_t). --- Modules/getpath.c | 4 ---- 1 file changed, 4 deletions(-) diff --git a/Modules/getpath.c b/Modules/getpath.c index f7a6dd40443054..44453f29df703a 100644 --- a/Modules/getpath.c +++ b/Modules/getpath.c @@ -923,11 +923,7 @@ static PyStatus calculate_program_macos(wchar_t **abs_path_p) { char execpath[MAXPATHLEN + 1]; -#if MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_4 uint32_t nsexeclength = Py_ARRAY_LENGTH(execpath) - 1; -#else - unsigned long nsexeclength = Py_ARRAY_LENGTH(execpath) - 1; -#endif /* On Mac OS X, if a script uses an interpreter of the form "#!/opt/python2.3/bin/python", the kernel only passes "python" From ba2f5a324fba1f16c0c4f4e98fd906e5a470bb69 Mon Sep 17 00:00:00 2001 From: Ronald Oussoren Date: Sun, 26 Jul 2020 14:52:51 +0200 Subject: [PATCH 09/11] Update build-installer.py for universal2 builds This also adds an option to stop building after compiling the 3th-party dependencies, as well as a script for archiving those dependencies. This makes it easier to work on the build. There are three changes to build-installer.py related to universal2 support: 1. Add 'universal2' information to build-installer.py; 2. Building OpenSSL for arm64 requires a patch at this time; 3. For some reason I had to patch the Tcl build to avoid a build error. --- Mac/BuildScript/archive-deps.py | 39 ++++++++++++++++++++ Mac/BuildScript/build-installer.py | 47 +++++++++++++++++++++++-- Mac/BuildScript/openssl-mac-arm64.patch | 41 +++++++++++++++++++++ 3 files changed, 124 insertions(+), 3 deletions(-) create mode 100755 Mac/BuildScript/archive-deps.py create mode 100644 Mac/BuildScript/openssl-mac-arm64.patch diff --git a/Mac/BuildScript/archive-deps.py b/Mac/BuildScript/archive-deps.py new file mode 100755 index 00000000000000..06d73a0f07617c --- /dev/null +++ b/Mac/BuildScript/archive-deps.py @@ -0,0 +1,39 @@ +#!/usr/bin/python + +import shutil +import getopt +import os +import sys + +WORKDIR = "/tmp/_py" + +def main(): + workdir = WORKDIR + + try: + options, args = getopt.getopt(sys.argv[1:], "?hbo", + [ "build-dir=", "output=", "help" ]) + except getopt.GetoptError: + print(sys.exc_info()[1]) + sys.exit(1) + + if args: + print("Additional arguments") + sys.exit(1) + + for k, v in options: + if k in ("-h", "-?", "--help"): + print(USAGE) + sys.exit(0) + + elif k in ("b", "--build-dir"): + workdir=v + + else: + raise SystemExit("Unknown option") + + root = os.path.join(workdir, "libraries", "Library", "Frameworks") + shutil.make_archive("third-party", "zip", root_dir=root) + +if __name__ == "__main__": + main() diff --git a/Mac/BuildScript/build-installer.py b/Mac/BuildScript/build-installer.py index a58b922ce30b83..56fc20390c3f97 100755 --- a/Mac/BuildScript/build-installer.py +++ b/Mac/BuildScript/build-installer.py @@ -116,7 +116,8 @@ def getFullVersion(): DEPSRC = os.path.join(WORKDIR, 'third-party') DEPSRC = os.path.expanduser('~/Universal/other-sources') -universal_opts_map = { '32-bit': ('i386', 'ppc',), +universal_opts_map = { 'universal2': ('arm64', 'x86_64'), + '32-bit': ('i386', 'ppc',), '64-bit': ('x86_64', 'ppc64',), 'intel': ('i386', 'x86_64'), 'intel-32': ('i386',), @@ -124,6 +125,7 @@ def getFullVersion(): '3-way': ('ppc', 'i386', 'x86_64'), 'all': ('i386', 'ppc', 'x86_64', 'ppc64',) } default_target_map = { + 'universal2': '10.9', '64-bit': '10.5', '3-way': '10.5', 'intel': '10.5', @@ -148,6 +150,10 @@ def getFullVersion(): # $MACOSX_DEPLOYMENT_TARGET -> minimum OS X level DEPTARGET = '10.5' +# If true only builds the 3th-party dependencies +# in $WORKDIR +DEPS_ONLY=False + def getDeptargetTuple(): return tuple([int(n) for n in DEPTARGET.split('.')[0:2]]) @@ -190,6 +196,27 @@ def getTargetCompilers(): def internalTk(): return getDeptargetTuple() >= (10, 6) + +def tweak_tcl_build(basedir, archList): + with open("Makefile", "r") as fp: + contents = fp.readlines() + + # For reasons I don't understand the tcl configure script + # decides that some stdlib symbols aren't present, before + # deciding that strtod is broken. + new_contents = [] + for line in contents: + if line.startswith("COMPAT_OBJS"): + # note: the space before strtod.o is intentional, + # the detection of a broken strtod results in + # "fixstrod.o" on this line. + for nm in ("strstr.o", "strtoul.o", " strtod.o"): + line = line.replace(nm, "") + new_contents.append(line) + + with open("Makefile", "w") as fp: + fp.writelines(new_contents) + # List of names of third party software built with this installer. # The names will be inserted into the rtf version of the License. THIRD_PARTY_LIBS = [] @@ -215,6 +242,9 @@ def library_recipes(): buildrecipe=build_universal_openssl, configure=None, install=None, + patches=[ + "openssl-mac-arm64.patch", + ], ), ]) @@ -231,6 +261,7 @@ def library_recipes(): '--libdir=/Library/Frameworks/Python.framework/Versions/%s/lib'%(getVersion(),), ], useLDFlags=False, + buildrecipe=tweak_tcl_build, install='make TCL_LIBRARY=%(TCL_LIBRARY)s && make install TCL_LIBRARY=%(TCL_LIBRARY)s DESTDIR=%(DESTDIR)s'%{ "DESTDIR": shellQuote(os.path.join(WORKDIR, 'libraries')), "TCL_LIBRARY": shellQuote('/Library/Frameworks/Python.framework/Versions/%s/lib/tcl8.6'%(getVersion())), @@ -596,7 +627,7 @@ def checkEnvironment(): ev, os.environ[ev])) del os.environ[ev] - base_path = '/bin:/sbin:/usr/bin:/usr/sbin' + base_path = '/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin' if 'SDK_TOOLS_BIN' in os.environ: base_path = os.environ['SDK_TOOLS_BIN'] + ':' + base_path # Xcode 2.5 on OS X 10.4 does not include SetFile in its usr/bin; @@ -618,6 +649,7 @@ def parseOptions(args=None): global UNIVERSALOPTS, UNIVERSALARCHS, ARCHLIST, CC, CXX global FW_VERSION_PREFIX global FW_SSL_DIRECTORY + global DEPS_ONLY if args is None: args = sys.argv[1:] @@ -625,7 +657,7 @@ def parseOptions(args=None): try: options, args = getopt.getopt(args, '?hb', [ 'build-dir=', 'third-party=', 'sdk-path=' , 'src-dir=', - 'dep-target=', 'universal-archs=', 'help' ]) + 'dep-target=', 'universal-archs=', 'deps-only', 'help' ]) except getopt.GetoptError: print(sys.exc_info()[1]) sys.exit(1) @@ -646,6 +678,9 @@ def parseOptions(args=None): elif k in ('--third-party',): DEPSRC=v + elif k in ('--deps-only',): + DEPS_ONLY=True + elif k in ('--sdk-path',): print(" WARNING: --sdk-path is no longer supported") @@ -691,6 +726,8 @@ def parseOptions(args=None): print(" -- Building a Python %s framework at patch level %s" % (getVersion(), getFullVersion())) print("") + if DEPS_ONLY: + print("Stopping after building third-party libraries") def extractArchive(builddir, archiveName): """ @@ -801,6 +838,7 @@ def build_openssl_arch(archbase, arch): arch_opts = { "i386": ["darwin-i386-cc"], "x86_64": ["darwin64-x86_64-cc", "enable-ec_nistp_64_gcc_128"], + "arm64": ["darwin64-arm64-cc" ], "ppc": ["darwin-ppc-cc"], "ppc64": ["darwin64-ppc-cc"], } @@ -1656,6 +1694,9 @@ def main(): # Then build third-party libraries such as sleepycat DB4. buildLibraries() + if DEPS_ONLY: + sys.exit(1) + # Now build python itself buildPython() diff --git a/Mac/BuildScript/openssl-mac-arm64.patch b/Mac/BuildScript/openssl-mac-arm64.patch new file mode 100644 index 00000000000000..11267fb118744a --- /dev/null +++ b/Mac/BuildScript/openssl-mac-arm64.patch @@ -0,0 +1,41 @@ +diff -ur openssl-1.1.1g-orig/Configurations/10-main.conf openssl-1.1.1g/Configurations/10-main.conf +--- openssl-1.1.1g-orig/Configurations/10-main.conf 2020-04-21 14:22:39.000000000 +0200 ++++ openssl-1.1.1g/Configurations/10-main.conf 2020-07-26 12:21:32.000000000 +0200 +@@ -1557,6 +1557,14 @@ + bn_ops => "SIXTY_FOUR_BIT_LONG", + perlasm_scheme => "macosx", + }, ++ "darwin64-arm64-cc" => { ++ inherit_from => [ "darwin-common", asm("aarch64_asm") ], ++ CFLAGS => add("-Wall"), ++ cflags => add("-arch arm64"), ++ lib_cppflags => add("-DL_ENDIAN"), ++ bn_ops => "SIXTY_FOUR_BIT_LONG", ++ perlasm_scheme => "ios64", ++ }, + + ##### GNU Hurd + "hurd-x86" => { +diff -ur openssl-1.1.1g-orig/config openssl-1.1.1g/config +--- openssl-1.1.1g-orig/config 2020-04-21 14:22:39.000000000 +0200 ++++ openssl-1.1.1g/config 2020-07-26 12:21:59.000000000 +0200 +@@ -255,6 +255,9 @@ + ;; + x86_64) + echo "x86_64-apple-darwin${VERSION}" ++ ;; ++ arm64) ++ echo "arm64-apple-darwin${VERSION}" + ;; + *) + echo "i686-apple-darwin${VERSION}" +@@ -497,6 +500,9 @@ + else + OUT="darwin64-x86_64-cc" + fi ;; ++ x86_64-apple-darwin*) ++ OUT="darwin64-arm64-cc" ++ ;; + armv6+7-*-iphoneos) + __CNF_CFLAGS="$__CNF_CFLAGS -arch armv6 -arch armv7" + __CNF_CXXFLAGS="$__CNF_CXXFLAGS -arch armv6 -arch armv7" From 8e3b4548f3843518996a9b754f213740479f290d Mon Sep 17 00:00:00 2001 From: Ronald Oussoren Date: Sun, 26 Jul 2020 16:06:43 +0200 Subject: [PATCH 10/11] Merge ctypes changes from Apple (PR 21241) Needed because my previous workaround doesn't work anymore. This uses a private API, that should be made public later... --- Lib/ctypes/macholib/dyld.py | 24 ++++++++-------- Lib/ctypes/test/test_macholib.py | 15 ++++++---- Modules/_ctypes/callproc.c | 48 ++++++++++++++++++++++++++++++++ setup.py | 1 + 4 files changed, 70 insertions(+), 18 deletions(-) diff --git a/Lib/ctypes/macholib/dyld.py b/Lib/ctypes/macholib/dyld.py index 9ab9e79ab816c4..22825d0d15b290 100644 --- a/Lib/ctypes/macholib/dyld.py +++ b/Lib/ctypes/macholib/dyld.py @@ -6,6 +6,11 @@ from ctypes.macholib.framework import framework_info from ctypes.macholib.dylib import dylib_info from itertools import * +try: + from _ctypes import _dyld_shared_cache_contains_path +except ImportError: + def _dyld_shared_cache_contains_path(*args): + raise NotImplementedError __all__ = [ 'dyld_find', 'framework_find', @@ -123,19 +128,14 @@ def dyld_find(name, executable_path=None, env=None): dyld_default_search(name, env), ), env): - # on macOS 11 system libraries are in a shared library - # cache, not in the regular place in the filesystem. - # There still are symlinks (libz.dylib -> libz.1.dylib), - # but the target of the symlink no longer is there. - # - # There shouldn't be 3th-party libraries in these - # system locations - if path.startswith("/System/") and os.path.islink(path): - return path - elif path.startswith("/usr/lib/") and os.path.islink(path): - return path - elif os.path.isfile(path): + if os.path.isfile(path): return path + try: + if _dyld_shared_cache_contains_path(path): + return path + except NotImplementedError: + pass + raise ValueError("dylib %s could not be found" % (name,)) def framework_find(fn, executable_path=None, env=None): diff --git a/Lib/ctypes/test/test_macholib.py b/Lib/ctypes/test/test_macholib.py index 6b3526951acfab..a1bac26a7df058 100644 --- a/Lib/ctypes/test/test_macholib.py +++ b/Lib/ctypes/test/test_macholib.py @@ -45,19 +45,22 @@ def find_lib(name): class MachOTest(unittest.TestCase): @unittest.skipUnless(sys.platform == "darwin", 'OSX-specific test') def test_find(self): - - self.assertEqual(find_lib('pthread'), - '/usr/lib/libSystem.B.dylib') + # On Mac OS 11, system dylibs are only present in the shared cache, + # so symlinks like libpthread.dylib -> libSystem.B.dylib will not + # be resolved by dyld_find + self.assertIn(find_lib('pthread'), + ('/usr/lib/libSystem.B.dylib', '/usr/lib/libpthread.dylib')) result = find_lib('z') # Issue #21093: dyld default search path includes $HOME/lib and # /usr/local/lib before /usr/lib, which caused test failures if # a local copy of libz exists in one of them. Now ignore the head # of the path. - self.assertRegex(result, r".*/lib/libz\..*.*\.dylib") + self.assertRegex(result, r".*/lib/libz.*\.dylib") - self.assertEqual(find_lib('IOKit'), - '/System/Library/Frameworks/IOKit.framework/Versions/A/IOKit') + self.assertIn(find_lib('IOKit'), + ('/System/Library/Frameworks/IOKit.framework/Versions/A/IOKit', + '/System/Library/Frameworks/IOKit.framework/IOKit')) if __name__ == "__main__": unittest.main() diff --git a/Modules/_ctypes/callproc.c b/Modules/_ctypes/callproc.c index ac407c93c240e4..824b00af0e60c2 100644 --- a/Modules/_ctypes/callproc.c +++ b/Modules/_ctypes/callproc.c @@ -66,6 +66,18 @@ #include "ctypes_dlfcn.h" #endif +#ifdef __APPLE__ +/* + * The API to query if a shared library is in the shared cache is + * private for now, this should change in beta 4. + * + * TODO: + * - Switch to that API + * - Add feature macro and runtime guards (as with the ffi.*loc API's) + */ +extern bool _dyld_shared_cache_contains_path(const char* path) __attribute__((weak_import)); +#endif + #ifdef MS_WIN32 #include #endif @@ -1446,6 +1458,37 @@ copy_com_pointer(PyObject *self, PyObject *args) } #else +#ifdef __APPLE__ + static PyObject *py_dyld_shared_cache_contains_path(PyObject *self, PyObject *args) + { + PyObject *name, *name2; + char *name_str; + + if (_dyld_shared_cache_contains_path == NULL) { + PyErr_SetString(PyExc_NotImplementedError, "_dyld_shared_cache_contains_path symbol is missing"); + return NULL; + } + + if (!PyArg_ParseTuple(args, "O", &name)) + return NULL; + + if (name == Py_None) + Py_RETURN_FALSE; + + if (PyUnicode_FSConverter(name, &name2) == 0) + return NULL; + if (PyBytes_Check(name2)) + name_str = PyBytes_AS_STRING(name2); + else + name_str = PyByteArray_AS_STRING(name2); + + if(_dyld_shared_cache_contains_path(name_str)) + Py_RETURN_TRUE; + else + Py_RETURN_FALSE; + } + #endif + static PyObject *py_dl_open(PyObject *self, PyObject *args) { PyObject *name, *name2; @@ -1935,6 +1978,8 @@ buffer_info(PyObject *self, PyObject *arg) return Py_BuildValue("siN", dict->format, dict->ndim, shape); } + + PyMethodDef _ctypes_module_methods[] = { {"get_errno", get_errno, METH_NOARGS}, {"set_errno", set_errno, METH_VARARGS}, @@ -1956,6 +2001,9 @@ PyMethodDef _ctypes_module_methods[] = { "dlopen(name, flag={RTLD_GLOBAL|RTLD_LOCAL}) open a shared library"}, {"dlclose", py_dl_close, METH_VARARGS, "dlclose a library"}, {"dlsym", py_dl_sym, METH_VARARGS, "find symbol in shared library"}, +#endif +#ifdef __APPLE__ + {"_dyld_shared_cache_contains_path", py_dyld_shared_cache_contains_path, METH_VARARGS, "check if path is in the shared cache"}, #endif {"alignment", align_func, METH_O, alignment_doc}, {"sizeof", sizeof_func, METH_O, sizeof_doc}, diff --git a/setup.py b/setup.py index 65d479fcb184c6..3bbf17a1c2283e 100644 --- a/setup.py +++ b/setup.py @@ -2211,6 +2211,7 @@ def detect_ctypes(self): ext.extra_compile_args.append("-DHAVE_FFI_PREP_CLOSURE_LOC=1") if grep_headers_for('ffi_closure_alloc', ffi_headers): ext.extra_compile_args.append("-DHAVE_FFI_CLOSURE_ALLOC=1") + ext.include_dirs.append(ffi_inc) ext.libraries.append(ffi_lib) self.use_system_libffi = True From 87c942becc00ac0c4bcb06f049e69aa2df8622e0 Mon Sep 17 00:00:00 2001 From: Ronald Oussoren Date: Sun, 26 Jul 2020 16:36:23 +0200 Subject: [PATCH 11/11] The archive-deps.py script isn't really useful --- Mac/BuildScript/archive-deps.py | 39 --------------------------------- 1 file changed, 39 deletions(-) delete mode 100755 Mac/BuildScript/archive-deps.py diff --git a/Mac/BuildScript/archive-deps.py b/Mac/BuildScript/archive-deps.py deleted file mode 100755 index 06d73a0f07617c..00000000000000 --- a/Mac/BuildScript/archive-deps.py +++ /dev/null @@ -1,39 +0,0 @@ -#!/usr/bin/python - -import shutil -import getopt -import os -import sys - -WORKDIR = "/tmp/_py" - -def main(): - workdir = WORKDIR - - try: - options, args = getopt.getopt(sys.argv[1:], "?hbo", - [ "build-dir=", "output=", "help" ]) - except getopt.GetoptError: - print(sys.exc_info()[1]) - sys.exit(1) - - if args: - print("Additional arguments") - sys.exit(1) - - for k, v in options: - if k in ("-h", "-?", "--help"): - print(USAGE) - sys.exit(0) - - elif k in ("b", "--build-dir"): - workdir=v - - else: - raise SystemExit("Unknown option") - - root = os.path.join(workdir, "libraries", "Library", "Frameworks") - shutil.make_archive("third-party", "zip", root_dir=root) - -if __name__ == "__main__": - main()