Add option to enable mimalloc secure mode

tiran · tiran · commit d2e159043600 · 2022-08-30T23:38:15.000+02:00
diff --git a/Doc/using/configure.rst b/Doc/using/configure.rst
@@ -268,6 +268,23 @@ also be used to improve performance.
 
    See also :envvar:`PYTHONMALLOC` environment variable.
 
+   .. versionadded:: 3.11
+
+.. cmdoption:: --enable-mimalloc-secure[=yes|no|1|2|3|4]
+
+   Enable mimalloc's secure mode and various mitigations against exploits.
+   Secure mode comes with small performance penalty and uses additional
+   memory for guard pages. Each level includes the previous levels. *yes*
+   enables the highest security level.
+
+   * *1* enables guard pages around metadata
+   * *2* enables guard pages around mimalloc page
+   * *3* enables encoded free lists and detects corrupted free lists as
+     well as invalid pointer frees.
+   * *4* enables expensive checks for double free.
+
+   .. versionadded:: 3.11
+
 .. cmdoption:: --without-doc-strings
 
    Disable static documentation strings to reduce the memory footprint (enabled
diff --git a/Include/internal/pycore_mimalloc.h b/Include/internal/pycore_mimalloc.h
@@ -20,6 +20,8 @@
 #  define MI_DEBUG 2
 // check for double free, buffer overflows and invalid pointer free
 #  define MI_SECURE 4
+#elif defined(PY_MIMALLOC_SECURE)
+#  define MI_SECURE PY_MIMALLOC_SECURE
 #endif
 
 /* Prefix all non-static symbols with "_Py_"
diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py
@@ -933,11 +933,12 @@ def test_debugmallocstats(self):
     @test.support.cpython_only
     def test_malloc_info(self):
         info = sys._malloc_info
-        self.assertEqual(len(info), 4)
+        self.assertEqual(len(info), 5)
         self.assertIsInstance(info.allocator, str)
-        self.assertIsInstance(info.with_freelists, bool)
         self.assertIsInstance(info.with_pymalloc, bool)
         self.assertIsInstance(info.with_mimalloc, bool)
+        self.assertIsInstance(info.mimalloc_secure, int)
+        self.assertIsInstance(info.mimalloc_debug, int)
 
     @unittest.skipUnless(hasattr(sys, "getallocatedblocks"),
                          "sys.getallocatedblocks unavailable on this build")
diff --git a/Makefile.pre.in b/Makefile.pre.in
@@ -1308,8 +1308,9 @@ Python/dynload_hpux.o: $(srcdir)/Python/dynload_hpux.c Makefile
 		-DSHLIB_EXT='"$(EXT_SUFFIX)"' \
 		-o $@ $(srcdir)/Python/dynload_hpux.c
 
-Python/sysmodule.o: $(srcdir)/Python/sysmodule.c Makefile $(srcdir)/Include/pydtrace.h
+Python/sysmodule.o: $(srcdir)/Python/sysmodule.c Makefile $(srcdir)/Include/pydtrace.h @MIMALLOC_INCLUDES@
 	$(CC) -c $(PY_CORE_CFLAGS) \
+		-I$(srcdir)/Include/mimalloc \
 		-DABIFLAGS='"$(ABIFLAGS)"' \
 		$(MULTIARCH_CPPFLAGS) \
 		-o $@ $(srcdir)/Python/sysmodule.c
diff --git a/Python/sysmodule.c b/Python/sysmodule.c
@@ -20,6 +20,7 @@ Data members:
 #include "pycore_code.h"          // _Py_QuickenedCount
 #include "pycore_frame.h"         // _PyInterpreterFrame
 #include "pycore_initconfig.h"    // _PyStatus_EXCEPTION()
+#include "pycore_mimalloc.h"      // MI_SECURE, MI_DEBUG
 #include "pycore_namespace.h"     // _PyNamespace_New()
 #include "pycore_object.h"        // _PyObject_IS_GC()
 #include "pycore_pathconfig.h"    // _PyPathConfig_ComputeSysPath0()
@@ -1945,17 +1946,18 @@ static PyTypeObject MallocInfoType;
 
 static PyStructSequence_Field malloc_info_fields[] = {
     {"allocator", "current memory allocator"},
-    {"with_freelists", "uses freelists"},
     {"with_pymalloc", "supports pymalloc (aka obmalloc)"},
     {"with_mimalloc", "supports mimalloc"},
+    {"mimalloc_secure", "mimalloc security level"},
+    {"mimalloc_debug", "mimalloc debug level"},
     {0}
 };
 
 static PyStructSequence_Desc malloc_info_desc = {
     "sys._malloc_info",     /* name */
     malloc_info__doc__ ,    /* doc */
     malloc_info_fields,     /* fields */
-    4
+    5
 };
 
 static PyObject *
@@ -1971,6 +1973,9 @@ make_malloc_info(void)
         return NULL;
     }
 
+#define SetIntItem(flag) \
+    PyStructSequence_SET_ITEM(malloc_info, pos++, PyLong_FromLong(flag))
+
     name = _PyMem_GetCurrentAllocatorName();
     if (name == NULL) {
         name = "unknown";
@@ -1983,13 +1988,6 @@ make_malloc_info(void)
 
     PyStructSequence_SET_ITEM(malloc_info, pos++, v);
 
-#ifdef WITH_FREELISTS
-    v = Py_True;
-#else
-    v = Py_False;
-#endif
-    PyStructSequence_SET_ITEM(malloc_info, pos++, _Py_NewRef(v));
-
 #ifdef WITH_PYMALLOC
     v = Py_True;
 #else
@@ -2004,6 +2002,11 @@ make_malloc_info(void)
 #endif
     PyStructSequence_SET_ITEM(malloc_info, pos++, _Py_NewRef(v));
 
+    SetIntItem(MI_SECURE);
+    SetIntItem(MI_DEBUG);
+
+#undef SetIntItem
+
     if (PyErr_Occurred()) {
         Py_CLEAR(malloc_info);
         return NULL;
diff --git a/configure b/configure
diff --git a/configure.ac b/configure.ac
@@ -4598,27 +4598,47 @@ fi
 AC_MSG_RESULT($with_doc_strings)
 
 # --with-mimalloc
-AC_MSG_CHECKING(for --with-mimalloc)
-AC_ARG_WITH(mimalloc,
-  AS_HELP_STRING([--with-mimalloc],
-                 [build with mimalloc memory allocator (default is yes)]),
+AC_MSG_CHECKING([for --with-mimalloc])
+AC_ARG_WITH([mimalloc],
+  [AS_HELP_STRING([--with-mimalloc],
+                 [build with mimalloc memory allocator (default is yes)])],
   [],
   [with_mimalloc="yes"]
 )
 
 if test "$with_mimalloc" != no; then
-  AC_MSG_RESULT(yes)
-  # disable pymalloc with mimalloc
-  # with_pymalloc="no"
-  # with_freelists="no"
-  AC_DEFINE(WITH_MIMALLOC, 1, Define Python uses mimalloc memory allocator.)
-  AC_SUBST(WITH_MIMALLOC, 1)
-  AC_SUBST(MIMALLOC_HEADERS, '$(MIMALLOC_HEADERS)')
-  AC_SUBST(MIMALLOC_INCLUDES, '$(MIMALLOC_INCLUDES)')
-else
-  AC_MSG_RESULT(no)
+  with_mimalloc=yes
+  AC_DEFINE([WITH_MIMALLOC], [1], [Define if you want to compile in mimalloc memory allocator.])
+  AC_SUBST([MIMALLOC_HEADERS], ['$(MIMALLOC_HEADERS)'])
+  AC_SUBST([MIMALLOC_INCLUDES], ['$(MIMALLOC_INCLUDES)'])
 fi
+AC_MSG_RESULT([$with_mimalloc])
 
+AC_MSG_CHECKING([for --enable-mimalloc-secure])
+AC_ARG_ENABLE([mimalloc-secure],
+  [AS_HELP_STRING([--enable-mimalloc-secure@<:@=yes|no|1|2|3|4@:>@],
+                  [enable mimalloc security mitigations (default is no), 1: guard metadata, 2: guard pages, 3: encode freelists, 4/yes: detect double free])],
+  [
+    AS_CASE([$enable_mimalloc_secure],
+      [yes], [enable_mimalloc_secure=4],
+      [1|2|3|4], [],
+      [no], [],
+      [enable_mimalloc_secure=invalid]
+    )
+    AS_VAR_IF([enable_mimalloc_secure], [invalid], [AC_MSG_ERROR([bad value $enable_mimalloc_secure for --enable-mimalloc-secure])])
+  ],
+  [enable_mimalloc_secure="no"]
+)
+
+if test "$enable_mimalloc_secure" != "no"; then
+  AS_VAR_IF([with_mimalloc], [no], [AC_MSG_ERROR([--with-mimalloc-secure requires --with-mimalloc])])
+  AC_DEFINE_UNQUOTED(
+    [PY_MIMALLOC_SECURE],
+    [$enable_mimalloc_secure],
+    [Define to 1, 2, 3, or 4 to enable mimalloc's security mitigations (Py_DEBUG forces 4)]
+    )
+fi
+AC_MSG_RESULT([$enable_mimalloc_secure])
 
 # Check for Python-specific malloc support
 AC_MSG_CHECKING(for --with-pymalloc)
@@ -6754,7 +6774,7 @@ AS_VAR_IF([ac_cv_header_stdatomic_h], [yes], [
 
 if test "$ac_cv_header_stdatomic_h" != yes -a "$with_mimalloc" != no; then
     # mimalloc-atomic.h wants C11 stdatomic.h on POSIX
-    AC_MSG_ERROR([--with-mimalloc requires stdatomic.h on your platform])
+    AC_MSG_ERROR([mimalloc requires stdatomic.h, use --without-mimalloc to disable mimalloc. A future version of Python will require stdatomic.h.])
 fi
 
 # Check for GCC >= 4.7 and clang __atomic builtin functions
diff --git a/pyconfig.h.in b/pyconfig.h.in
@@ -1571,6 +1571,10 @@
 /* Define to 1 if you have the perf trampoline. */
 #undef PY_HAVE_PERF_TRAMPOLINE
 
+/* Define to 1, 2, 3, or 4 to enable mimalloc's security mitigations (Py_DEBUG
+   forces 4) */
+#undef PY_MIMALLOC_SECURE
+
 /* Define to 1 to build the sqlite module with loadable extensions support. */
 #undef PY_SQLITE_ENABLE_LOAD_EXTENSION
 
@@ -1737,7 +1741,7 @@
 /* Define to 1 if libintl is needed for locale functions. */
 #undef WITH_LIBINTL
 
-/* Define Python uses mimalloc memory allocator. */
+/* Define if you want to compile in mimalloc memory allocator. */
 #undef WITH_MIMALLOC
 
 /* Define if you want to produce an OpenStep/Rhapsody framework (shared
