diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 00000000000000..ff4bbbc21e7757 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "Stackless/platf/stackman"] + path = Stackless/stackman + url = https://github.com/stackless-dev/stackman.git diff --git a/Makefile.pre.in b/Makefile.pre.in index 176e10fb106143..b6c13ede4fdda5 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -119,6 +119,8 @@ CFLAGS_ALIASING=@CFLAGS_ALIASING@ # Special C flags for slp_transfer.c SLPFLAGS= @SLPFLAGS@ +SLP_STACKMAN_LIB= @SLP_STACKMAN_LIB@ +SLP_STACKMAN_OBJS= @SLP_STACKMAN_OBJS@ # Machine-dependent subdirectories MACHDEP= @MACHDEP@ @@ -348,6 +350,7 @@ PYTHON_OBJS= \ Stackless/module/taskletobject.o \ Stackless/pickling/prickelpit.o \ Stackless/pickling/safe_pickle.o \ + $(SLP_STACKMAN_OBJS) \ Python/codecs.o \ Python/compile.o \ Python/coreconfig.o \ @@ -724,6 +727,17 @@ $(srcdir)/Include/cpython/slp_exttype.h: $(srcdir)/Objects/typeobject.c $(PYTHON_FOR_REGEN) $(srcdir)/Stackless/Tools/extract_slp_info.py $(UPDATE_FILE) $(srcdir)/Include/cpython/slp_exttype.h $(srcdir)/Include/cpython/slp_exttype.h.new +$(SLP_STACKMAN_OBJS): $(SLP_STACKMAN_LIB) + cp $(SLP_STACKMAN_LIB) Stackless/platf/libstackman.a + cd Stackless/platf && $(AR) x libstackman.a && rm libstackman.a + +Stackless/platf/slp_transfer.o: $(srcdir)/Stackless/platf/slp_transfer.c + $(CC) -c $(PY_CORE_CFLAGS) $(SLPFLAGS) -o $@ $(srcdir)/Stackless/platf/slp_transfer.c + +.PHONY: teststackless +teststackless: @DEF_MAKE_RULE@ platform + $(TESTPYTHON) -E $(srcdir)/Stackless/unittests/runAll.py + ############################################################################ # Importlib @@ -990,9 +1004,6 @@ regen-typeslots: $(srcdir)/Objects/typeslots.inc.new $(UPDATE_FILE) $(srcdir)/Objects/typeslots.inc $(srcdir)/Objects/typeslots.inc.new -Stackless/platf/slp_transfer.o: $(srcdir)/Stackless/platf/slp_transfer.c - $(CC) -c $(PY_CORE_CFLAGS) $(SLPFLAGS) -o $@ $(srcdir)/Stackless/platf/slp_transfer.c - ############################################################################ # Header files @@ -1150,7 +1161,7 @@ TESTPYTHON= $(RUNSHARED) ./$(BUILDPYTHON) $(TESTPYTHONOPTS) TESTRUNNER= $(TESTPYTHON) $(srcdir)/Tools/scripts/run_tests.py TESTTIMEOUT= 1200 -.PHONY: test testall testuniversal buildbottest pythoninfo teststackless +.PHONY: test testall testuniversal buildbottest pythoninfo # Run a basic set of regression tests. # This excludes some tests that are particularly resource-intensive. @@ -1171,9 +1182,6 @@ testall: @DEF_MAKE_RULE@ platform -$(TESTRUNNER) -u all $(TESTOPTS) $(TESTRUNNER) -u all $(TESTOPTS) -teststackless: @DEF_MAKE_RULE@ platform - $(TESTPYTHON) -E $(srcdir)/Stackless/unittests/runAll.py - # Run the test suite for both architectures in a Universal build on OSX. # Must be run on an Intel box. testuniversal: @DEF_MAKE_RULE@ platform @@ -1791,7 +1799,7 @@ docclean: -rm -rf Doc/tools/sphinx Doc/tools/pygments Doc/tools/docutils clean: pycremoval - find . -name '*.[oa]' -exec rm -f {} ';' + find . -path '*Stackless/stackman/*.a' -prune -o -name '*.[oa]' -exec rm -f {} ';' find . -name '*.s[ol]' -exec rm -f {} ';' find . -name '*.so.[0-9]*.[0-9]*' -exec rm -f {} ';' find build -name 'fficonfig.h' -exec rm -f {} ';' || true diff --git a/PCbuild/python.props b/PCbuild/python.props index a9dc9db4863f0b..e48151431f8924 100644 --- a/PCbuild/python.props +++ b/PCbuild/python.props @@ -71,6 +71,11 @@ $(BuildPath)python$(PyDebugExt).exe + + + $(PySourcePath)Stackless\stackman\ + $(stackmanDir)\ + stackman.lib; diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index 791a8ea00baf6e..052388c4bfb241 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -88,9 +88,15 @@ $(zlibDir);%(AdditionalIncludeDirectories) _USRDLL;Py_BUILD_CORE;Py_ENABLE_SHARED;MS_DLL_ID="$(SysWinVer)";%(PreprocessorDefinitions) _Py_HAVE_ZLIB;%(PreprocessorDefinitions) + SLP_NO_STACKMAN;%(PreprocessorDefinitions) - version.lib;shlwapi.lib;ws2_32.lib;%(AdditionalDependencies) + version.lib;shlwapi.lib;ws2_32.lib;$(stackmanLib)%(AdditionalDependencies) + $(stackmanDir)lib\win_x64;%(AdditionalLibraryDirectories) + $(stackmanDir)lib\win_x86;%(AdditionalLibraryDirectories) + $(stackmanDir)lib\win_arm;%(AdditionalLibraryDirectories) + $(stackmanDir)lib\win_arm64;%(AdditionalLibraryDirectories) + false @@ -454,7 +460,9 @@ - + + $(stackmanDir);%(AdditionalIncludeDirectories) + @@ -486,23 +494,12 @@ - true - true - true - true + true + true Document - ml64 /nologo /c /Zi /Fo "$(IntDir)%(Filename) .obj" "%(FullPath)" - Running ml64 - $(IntDir)%(Filename) .obj;%(Outputs) - ml64 /nologo /c /Zi /Fo "$(IntDir)%(Filename) .obj" "%(FullPath)" - Running ml64 - $(IntDir)%(Filename) .obj;%(Outputs) - ml64 /nologo /c /Zi /Fo "$(IntDir)%(Filename) .obj" "%(FullPath)" - Running ml64 - $(IntDir)%(Filename) .obj;%(Outputs) - ml64 /nologo /c /Zi /Fo "$(IntDir)%(Filename) .obj" "%(FullPath)" - Running ml64 - $(IntDir)%(Filename) .obj;%(Outputs) + ml64 /nologo /c /Zi /Fo "$(IntDir)%(Filename) .obj" "%(FullPath)" + Running ml64 + $(IntDir)%(Filename) .obj;%(Outputs) diff --git a/Stackless/Tools/create_source_archive.sh b/Stackless/Tools/create_source_archive.sh index e19b475ee4cc6f..25ba184d4c2bf6 100644 --- a/Stackless/Tools/create_source_archive.sh +++ b/Stackless/Tools/create_source_archive.sh @@ -14,7 +14,7 @@ if [ $# != 3 ] ; then exit 1 fi -: ${excludes:=.gitignore .git* .hg* .bzrignore .mention-bot .travis.yml .readthedocs.yml .azure-pipelines} +: ${excludes:=.gitignore .git* .mention-bot .travis.yml .readthedocs.yml .azure-pipelines Stackless/stackman/.git* } git_python_dir="$1" ; shift if [ "x$1" = "x-v" ] ; then @@ -42,6 +42,7 @@ mkdir "$tmpdir" cd "$git_python_dir" git archive --format=tar --prefix="${srcdir}/" "$tag" | \ ( cd "${tmpdir}" && tar xf - ) +git submodule foreach --recursive "git archive --format=tar --prefix=${srcdir}/\$path/ HEAD | ( cd \"${tmpdir}\" && tar xf - )" cd "${tmpdir}" ( cd "${srcdir}" && set +f && rm -rf $excludes ) diff --git a/Stackless/changelog.txt b/Stackless/changelog.txt index a3068734e34fb0..bb41e0f7fc0061 100644 --- a/Stackless/changelog.txt +++ b/Stackless/changelog.txt @@ -9,6 +9,11 @@ What's New in Stackless 3.X.X? *Release date: 20XX-XX-XX* +- https://github.com/stackless-dev/stackless/issues/278 + Stackless now uses Stackman https://github.com/stackless-dev/stackman for + stack switching / hard tasklet switching. The legacy stack switching code + is still available. See 'readme.txt' for details. + - https://github.com/stackless-dev/stackless/issues/254 The Stackless version is now "3.8". diff --git a/Stackless/platf/slp_transfer.c b/Stackless/platf/slp_transfer.c index 522d602d4f4d34..771def9be2444a 100644 --- a/Stackless/platf/slp_transfer.c +++ b/Stackless/platf/slp_transfer.c @@ -39,6 +39,16 @@ */ #define SLP_EVAL /* enable code generation in the included header */ +#ifndef SLP_NO_STACKMAN /* defined by configure --without-stackman */ +/* First, see if stackman an implementation without external + * assembler, use that if possible + */ +#define STACKMAN_OPTIONAL +#include "stackman/stackman.h" +#endif /* #ifndef SLP_NO_STACKMAN */ +#if defined(STACKMAN_PLATFORM) && !defined(STACKMAN_EXTERNAL_ASM) +#include "switch_stackman.h" +#else /* use traditional stackless switching */ #if defined(MS_WIN32) && !defined(MS_WIN64) && defined(_M_IX86) #include "switch_x86_msvc.h" /* MS Visual Studio on X86 */ #elif defined(MS_WIN64) && defined(_M_X64) @@ -74,6 +84,7 @@ Please provide an implementation of the switch_XXX.h or disable the STACKLESS flag. ********** #endif +#endif /* use traditional stackless switching */ /* default definitions if not defined in above files */ diff --git a/Stackless/platf/switch_stackman.h b/Stackless/platf/switch_stackman.h new file mode 100644 index 00000000000000..a358f150c90058 --- /dev/null +++ b/Stackless/platf/switch_stackman.h @@ -0,0 +1,67 @@ +/* + * this is the internal transfer function, using + * the stackman platform library. + * We create a wrapper callback that employs the existing + * stack macros. + * At some later point in time, the callback could be + * written directly. + * + */ + +#define SLP_STACK_REFPLUS 1 + +#ifdef SLP_EVAL +#define SLP_STACK_MAGIC 0 + + +/* need a special function arount SLP_SAVE_STATE() because + * the macro has a built-in return of 0 or -1. Must catch + * that. + */ +static int slp_stackman_cb_save(void *sp, intptr_t *pdiff) +{ + intptr_t diff; + /* first argument must be a pointer to a "stack word", not a void* */ + SLP_SAVE_STATE(((intptr_t *)sp), diff); + *pdiff = diff; + return 1; +} + +static void *slp_stackman_cb(void *_ctxt, int opcode, void *sp) +{ + int *error = (int*)_ctxt; + intptr_t stsizediff; + if (opcode == STACKMAN_OP_SAVE) + { + int ret = slp_stackman_cb_save(sp, &stsizediff); + if (ret == 1) { + /* regular switch */ + return (void*)((char*)sp + stsizediff); + } + if (ret == -1) + { + *error = -1; + } + /* error or save only, no change in sp */ + return sp; + } + else + { + if (*error != -1) + SLP_RESTORE_STATE(); + return sp; + } +} + +static int +slp_switch(void) +{ + /* this can be on the stack, because if there isn't a switch + * then it is intact. (error or save only) */ + int error = 0; + stackman_switch(&slp_stackman_cb, &error); + return error; +} + +#include "stackman/stackman_impl.h" +#endif diff --git a/Stackless/readme.txt b/Stackless/readme.txt index 35d08cdde701e2..10b1ab22f95b84 100644 --- a/Stackless/readme.txt +++ b/Stackless/readme.txt @@ -27,13 +27,26 @@ It is the ultimate truth. Read it at https://github.com/stackless-dev/stackless Building Stackless ------------------ -Just follow the build instructions for regular Python. If you define -the C-preprocessor symbol STACKLESS_OFF, you get a Python interpreter -without Stackless. It should behave exactly like the corresponding -version of regular Python. Any difference constitutes a bug. -(Windows 64bit only: if you define STACKLESS_OFF, you also need -to clear the content of the file Stackless\platf\switch_x64_masm.asm: - c:\> echo END > Stackless\platf\switch_x64_masm.asm ) +Just follow the build instructions for regular C-Python. + +To more easily support new platforms and CPU architectures, Stackless Python +now commonly uses Stackman (https://github.com/stackless-dev/stackman) +for stack switching / hard switching between tasklets. Stackman is a +lightweight library for stack manipulation. The Stackless source archive +already contains a copy of Stackman, which is normally used automatically. +However, the legacy Stackless switching code can still be used without +restrictions. To do this, Stackless must be configured with +`configure --without-stackman` or built under Windows with +`build.bat "/p:stackmanDir=no"`. + +It is also possible to use your own version of Stackman instead of the copy +bundled with Stackless. To do this configure Stackless with +`configure --with-stackman=/path/to/your/stackman` or built under Windows +with `build.bat "/p:stackmanDir=X:\path\to\your\stackman"`. + +If you define the C-preprocessor symbol STACKLESS_OFF in Include/stackless.h, +you get a Python interpreter without Stackless. It should behave exactly like +the corresponding version of regular Python. Any difference constitutes a bug. Contributing ------------ diff --git a/Stackless/stackman b/Stackless/stackman new file mode 160000 index 00000000000000..c572d4d07cc850 --- /dev/null +++ b/Stackless/stackman @@ -0,0 +1 @@ +Subproject commit c572d4d07cc8507bc5a519dbab6e461f718547d7 diff --git a/configure b/configure index caa9de63bcc3de..9488cb523babe3 100755 --- a/configure +++ b/configure @@ -652,6 +652,9 @@ DTRACE TCLTK_LIBS TCLTK_INCLUDES LIBFFI_INCLUDEDIR +SLP_STACKMAN_OBJS +SLP_STACKMAN_LIB +SLPFLAGS PKG_CONFIG_LIBDIR PKG_CONFIG_PATH PKG_CONFIG @@ -729,7 +732,6 @@ CFLAGS CC EXPORT_MACOSX_DEPLOYMENT_TARGET CONFIGURE_MACOSX_DEPLOYMENT_TARGET -SLPFLAGS _PYTHON_HOST_PLATFORM MACHDEP FRAMEWORKINSTALLAPPSPREFIX @@ -826,6 +828,7 @@ with_address_sanitizer with_memory_sanitizer with_undefined_behavior_sanitizer with_libs +with_stackman with_system_expat with_system_ffi with_system_libmpdec @@ -1528,6 +1531,8 @@ Optional Packages: --with-undefined-behavior-sanitizer enable UndefinedBehaviorSanitizer (ubsan) --with-libs='lib1 ...' link against additional libs + --with-stackman build python with an external stackman library or + without using stackman at all --with-system-expat build pyexpat module using an installed expat library --with-system-ffi build _ctypes module using an installed ffi library @@ -3335,17 +3340,6 @@ if test "$cross_compiling" = yes; then _PYTHON_HOST_PLATFORM="$MACHDEP${_host_cpu:+-$_host_cpu}" fi -# Stackless flags for compiling the hard switching code -case $MACHDEP in - darwin) - SLPFLAGS="-fomit-frame-pointer -fno-inline-functions" - ;; - *) - SLPFLAGS="-fno-omit-frame-pointer -fno-inline-functions" - ;; -esac - - # Some systems cannot stand _XOPEN_SOURCE being defined at all; they # disable features if it is defined, without any means to access these # features as extensions. For these systems, we skip the definition of @@ -10336,6 +10330,66 @@ $as_echo "no" >&6; } fi fi +# Stackless flags for compiling the hard switching code +case $MACHDEP in + darwin) + SLPFLAGS="-fomit-frame-pointer -fno-inline-functions" + ;; + *) + SLPFLAGS="-fno-omit-frame-pointer -fno-inline-functions" + ;; +esac + +# Check for use of an external or no stackman library +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for --with-stackman" >&5 +$as_echo_n "checking for --with-stackman... " >&6; } + +# Check whether --with-stackman was given. +if test "${with_stackman+set}" = set; then : + withval=$with_stackman; +fi + + +case "$with_stackman" in + "") + with_stackman="yes" + ;; + "yes"|"no") + ;; + "*") + if test -f "$with_stackman"/Makefile ; then + : + else + as_fn_error $? "--with-stackman argument must be a directory containing the stackman project" "$LINENO" 5 + fi + ;; +esac +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_stackman" >&5 +$as_echo "$with_stackman" >&6; } + +if test "$with_stackman" = "yes" ; then + with_stackman="Stackless/stackman" +fi +SLP_STACKMAN_LIB= +SLP_STACKMAN_OBJS= +if test "$with_stackman" = "no" ; then + SLPFLAGS="-DSLP_NO_STACKMAN $SLPFLAGS" +else + SLPFLAGS="-I$with_stackman $SLPFLAGS" + slp_stackman_abiname=`cd "$with_stackman" && make -s CC="$CC" CXX="$CXX" AR="$AR" abiname 2>/dev/null` + if test "X$slp_stackman_abiname" != "X" ; then + if test -f "$with_stackman/lib/$slp_stackman_abiname/libstackman.a" ; then + SLP_STACKMAN_LIB="$with_stackman/lib/$slp_stackman_abiname/libstackman.a" + SLP_STACKMAN_OBJS=`$AR t "$SLP_STACKMAN_LIB" | sed -e 's|^|Stackless/platf/|' | tr '\n' ' '` + else + as_fn_error $? "Stackman library not found: $with_stackman/lib/$slp_stackman_abiname/libstackman.a" "$LINENO" 5 + fi + fi +fi + + + + # Check for use of the system expat library { $as_echo "$as_me:${as_lineno-$LINENO}: checking for --with-system-expat" >&5 $as_echo_n "checking for --with-system-expat... " >&6; } diff --git a/configure.ac b/configure.ac index 58c1ad2f5c950f..4b9e2baa7c09f6 100644 --- a/configure.ac +++ b/configure.ac @@ -437,17 +437,6 @@ if test "$cross_compiling" = yes; then _PYTHON_HOST_PLATFORM="$MACHDEP${_host_cpu:+-$_host_cpu}" fi -# Stackless flags for compiling the hard switching code -case $MACHDEP in - darwin) - SLPFLAGS="-fomit-frame-pointer -fno-inline-functions" - ;; - *) - SLPFLAGS="-fno-omit-frame-pointer -fno-inline-functions" - ;; -esac -AC_SUBST(SLPFLAGS) - # Some systems cannot stand _XOPEN_SOURCE being defined at all; they # disable features if it is defined, without any means to access these # features as extensions. For these systems, we skip the definition of @@ -2922,6 +2911,60 @@ LIBS="$withval $LIBS" PKG_PROG_PKG_CONFIG +# Stackless flags for compiling the hard switching code +case $MACHDEP in + darwin) + SLPFLAGS="-fomit-frame-pointer -fno-inline-functions" + ;; + *) + SLPFLAGS="-fno-omit-frame-pointer -fno-inline-functions" + ;; +esac + +# Check for use of an external or no stackman library +AC_MSG_CHECKING(for --with-stackman) +AC_ARG_WITH(stackman, + AS_HELP_STRING([--with-stackman], [build python with an external stackman library or without using stackman at all]),,,) + +case "$with_stackman" in + "") + with_stackman="yes" + ;; + "yes"|"no") + ;; + "*") + if test -f "$with_stackman"/Makefile ; then + : + else + AC_MSG_ERROR([--with-stackman argument must be a directory containing the stackman project]) + fi + ;; +esac +AC_MSG_RESULT($with_stackman) + +if test "$with_stackman" = "yes" ; then + with_stackman="Stackless/stackman" +fi +SLP_STACKMAN_LIB= +SLP_STACKMAN_OBJS= +if test "$with_stackman" = "no" ; then + SLPFLAGS="-DSLP_NO_STACKMAN $SLPFLAGS" +else + SLPFLAGS="-I$with_stackman $SLPFLAGS" + slp_stackman_abiname=`cd "$with_stackman" && make -s CC="$CC" CXX="$CXX" AR="$AR" abiname 2>/dev/null` + if test "X$slp_stackman_abiname" != "X" ; then + if test -f "$with_stackman/lib/$slp_stackman_abiname/libstackman.a" ; then + SLP_STACKMAN_LIB="$with_stackman/lib/$slp_stackman_abiname/libstackman.a" + SLP_STACKMAN_OBJS=`$AR t "$SLP_STACKMAN_LIB" | sed -e 's|^|Stackless/platf/|' | tr '\n' ' '` + else + AC_MSG_ERROR([Stackman library not found: $with_stackman/lib/$slp_stackman_abiname/libstackman.a]) + fi + fi +fi +AC_SUBST(SLPFLAGS) +AC_SUBST(SLP_STACKMAN_LIB) +AC_SUBST(SLP_STACKMAN_OBJS) + # Check for use of the system expat library AC_MSG_CHECKING(for --with-system-expat) AC_ARG_WITH(system_expat,