Drop m_init and m_copy from PyModuleDef_Base.

ericsnowcurrently · ericsnowcurrently · commit b6098ef7c3f3 · 2024-04-30T19:59:13.000-06:00
diff --git a/Include/moduleobject.h b/Include/moduleobject.h
@@ -38,30 +38,21 @@ PyAPI_DATA(PyTypeObject) PyModuleDef_Type;
 
 typedef struct PyModuleDef_Base {
   PyObject_HEAD
-  /* The function used to re-initialize the module.
-     This is only set for legacy (single-phase init) extension modules
-     and only used for those that support multiple initializations
-     (m_size >= 0).
-     It is set by _PyImport_LoadDynamicModuleWithSpec()
-     and _imp.create_builtin(). */
-  PyObject* (*m_init)(void);
+  /* We keep the following fields, at the given size, for API compatibility. */
+  Py_uintptr_t m_cache_has_m_init;
+  Py_uintptr_t m_cache_has_m_dict;
   /* The module's index into its interpreter's modules_by_index cache.
      This is set for all extension modules but only used for legacy ones.
      (See PyInterpreterState.modules_by_index for more info.)
      It is set by PyModuleDef_Init(). */
   Py_ssize_t m_index;
-  /* A copy of the module's __dict__ after the first time it was loaded.
-     This is only set/used for legacy modules that do not support
-     multiple initializations.
-     It is set by fix_up_extension() in import.c. */
-  PyObject* m_copy;
 } PyModuleDef_Base;
 
 #define PyModuleDef_HEAD_INIT {  \
     PyObject_HEAD_INIT(_Py_NULL) \
-    _Py_NULL, /* m_init */       \
-    0,        /* m_index */      \
-    _Py_NULL, /* m_copy */       \
+    0,  /* m_cache_has_m_init */ \
+    0,  /* m_cache_has_m_dict */ \
+    0,  /* m_index */            \
   }
 
 #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x03050000
diff --git a/Python/import.c b/Python/import.c
@@ -580,19 +580,8 @@ _PyImport_ClearModulesByIndex(PyInterpreterState *interp)
         return;
     }
 
-    Py_ssize_t i;
-    for (i = 0; i < PyList_GET_SIZE(MODULES_BY_INDEX(interp)); i++) {
-        PyObject *m = PyList_GET_ITEM(MODULES_BY_INDEX(interp), i);
-        if (PyModule_Check(m)) {
-            /* cleanup the saved copy of module dicts */
-            PyModuleDef *md = PyModule_GetDef(m);
-            if (md) {
-                // XXX Do this more carefully.  The dict might be owned
-                // by another interpreter.
-                Py_CLEAR(md->m_base.m_copy);
-            }
-        }
-    }
+    /* We clear out the cached module defs and copied module dicts
+     * in _PyImport_Fini(). */
 
     /* Setting modules_by_index to NULL could be dangerous, so we
        clear the list instead. */
@@ -652,7 +641,7 @@ _PyImport_ClearModulesByIndex(PyInterpreterState *interp)
        O.   import_run_extension():  set __file__
        P.   import_run_extension() -> update_global_state_for_extension()
        Q.     update_global_state_for_extension():
-                      copy __dict__ into def->m_base.m_copy
+                      copy __dict__ into the cached m_dict
        R.     update_global_state_for_extension():
                       add it to _PyRuntime.imports.extensions
        S.   import_run_extension() -> finish_singlephase_extension()
@@ -672,7 +661,7 @@ _PyImport_ClearModulesByIndex(PyInterpreterState *interp)
        F.       else:
                   1. import_add_module() -> PyModule_NewObject()
                   2. import_add_module():  set it on sys.modules
-       G.     reload_singlephase_extension():  copy the "m_copy" dict into __dict__
+       G.     reload_singlephase_extension():  copy the "m_dict" dict into __dict__
        H.     reload_singlephase_extension():  add to modules_by_index
 
     (10). (every time):
@@ -854,19 +843,19 @@ We support a number of kinds of single-phase init builtin/extension modules:
 
 * "basic"
     * no module state (PyModuleDef.m_size == -1)
-    * does not support repeated init (we use PyModuleDef.m_base.m_copy)
+    * does not support repeated init (we use the cached m_dict)
     * may have process-global state
     * the module's def is cached in _PyRuntime.imports.extensions,
       by (name, filename)
 * "reinit"
     * no module state (PyModuleDef.m_size == 0)
-    * supports repeated init (m_copy is never used)
+    * supports repeated init (m_dict is never used)
     * should not have any process-global state
     * its def is never cached in _PyRuntime.imports.extensions
       (except, currently, under the main interpreter, for some reason)
 * "with state"  (almost the same as reinit)
     * has module state (PyModuleDef.m_size > 0)
-    * supports repeated init (m_copy is never used)
+    * supports repeated init (m_dict is never used)
     * should not have any process-global state
     * its def is never cached in _PyRuntime.imports.extensions
       (except, currently, under the main interpreter, for some reason)
@@ -892,7 +881,7 @@ notable weirdness happens:
     * (non-basic-only) its init func is used when re-loading any of them
       (via the def's m_init)
     * (basic-only) the copy of its __dict__ is used when re-loading any of them
-      (via the def's m_copy)
+      (via the def's cached m_dict)
 
 However, the following happens as expected:
 
@@ -905,7 +894,7 @@ However, the following happens as expected:
 For "basic" modules there are other quirks:
 
 * (whether sharing a def or not) when loaded the first time,
-  m_copy is set before _init_module_attrs() is called
+  m_dict is set before _init_module_attrs() is called
   in importlib._bootstrap.module_from_spec(),
   so when the module is re-loaded, the previous value
   for __wpec__ (and others) is reset, possibly unexpectedly.
@@ -948,7 +937,7 @@ typedef PyDictObject *cached_m_dict_t;
 struct extensions_cache_value {
     PyModuleDef *def;
 
-    /* m_init and m_copy were formerly part of PyModuleDef_Base. */
+    /* m_init and m_dict were formerly part of PyModuleDef_Base. */
 
     /* The function used to re-initialize the module.
        This is only set for legacy (single-phase init) extension modules
@@ -1008,7 +997,7 @@ set_cached_m_dict(struct extensions_cache_value *value, PyObject *m_dict)
      * isolated interpreter, but there are ways around that.
      * Hence, heere be dragons!  Ideally we would instead do
      * something like make a read-only, immortal copy of the
-     * dict using PyMem_RawMalloc() and store *that* in m_copy.
+     * dict using PyMem_RawMalloc() and store *that* in m_dict.
      * Then we'd need to make sure to clear that when the
      * runtime is finalized, rather than in
      * PyImport_ClearModulesByIndex(). */
@@ -1051,7 +1040,7 @@ update_extensions_cache_value(struct extensions_cache_value *value,
     /* We expect the same symbol to be used and the shared object file
      * to have remained loaded, so it must be the same pointer. */
     assert(value->m_init == NULL || value->m_init == m_init);
-    /* For now we don't worry about comparing value->m_copy. */
+    /* For now we don't worry about comparing value->m_dict. */
     assert(value->m_dict == NULL || m_dict != NULL);
 
     /* We assume that all module defs are statically allocated
@@ -1067,6 +1056,13 @@ update_extensions_cache_value(struct extensions_cache_value *value,
         return -1;
     }
 
+    if (m_init != NULL) {
+        def->m_base.m_cache_has_m_init = 1;
+    }
+    if (temp.m_dict != NULL) {
+        def->m_base.m_cache_has_m_dict = 1;
+    }
+
     *value = temp;
     return 0;
 }
@@ -1396,7 +1392,7 @@ _get_extension_kind(PyModuleDef *def, bool check_size)
     enum _Py_ext_module_kind kind;
     if (def == NULL) {
         /* It must be a module created by reload_singlephase_extension()
-         * from m_copy.  Ideally we'd do away with this case. */
+         * from m_dict.  Ideally we'd do away with this case. */
         kind = _Py_ext_module_kind_SINGLEPHASE;
     }
     else if (def->m_slots != NULL) {
@@ -1405,7 +1401,10 @@ _get_extension_kind(PyModuleDef *def, bool check_size)
     else if (check_size && def->m_size == -1) {
         kind = _Py_ext_module_kind_SINGLEPHASE;
     }
-    else if (def->m_base.m_init != NULL) {
+    else if (def->m_base.m_cache_has_m_init) {
+        kind = _Py_ext_module_kind_SINGLEPHASE;
+    }
+    else if (def->m_base.m_cache_has_m_dict) {
         kind = _Py_ext_module_kind_SINGLEPHASE;
     }
     else {
@@ -1451,8 +1450,9 @@ update_global_state_for_extension(PyThreadState *tstate,
     PyModInitFunction m_init = NULL;
     PyObject *m_dict = NULL;
 
-    assert(def->m_base.m_init == NULL);
-    assert(def->m_base.m_copy == NULL);
+    /* It shouldn't have been set yet. */
+    assert(!def->m_base.m_cache_has_m_init);
+    assert(!def->m_base.m_cache_has_m_dict);
 
     /* Set up for _extensions_cache_set(). */
     if (singlephase != NULL) {
@@ -1473,7 +1473,7 @@ update_global_state_for_extension(PyThreadState *tstate,
         else {
             assert(singlephase->m_dict != NULL
                     && PyDict_Check(singlephase->m_dict));
-            // gh-88216: Extensions and def->m_base.m_copy can be updated
+            // gh-88216: Extensions and the cached m_dict can be updated
             // when the extension module doesn't support sub-interpreters.
             assert(def->m_size == -1);
             assert(!is_core_module(tstate->interp, name, path));
@@ -1496,6 +1496,8 @@ update_global_state_for_extension(PyThreadState *tstate,
             // mark the module as not loadable.
             return -1;
         }
+        assert(def->m_base.m_cache_has_m_init
+                || def->m_base.m_cache_has_m_dict);
     }
 
     return 0;
@@ -1547,14 +1549,15 @@ reload_singlephase_extension(PyThreadState *tstate,
     if (def->m_size == -1) {
         /* Module does not support repeated initialization */
         assert(cached->m_init == NULL);
-        assert(def->m_base.m_init == NULL);
+        assert(!def->m_base.m_cache_has_m_init);
+        assert(def->m_base.m_cache_has_m_dict);
         // XXX Copying the cached dict may break interpreter isolation.
         // We could solve this by temporarily acquiring the original
         // interpreter's GIL.
         PyObject *m_copy = get_cached_m_dict(cached);
         if (m_copy == NULL) {
             /* It might be a core module (e.g. sys & builtins),
-               for which we don't set m_copy. */
+               for which we don't cache m_dict. */
             m_copy = get_core_module_dict(
                     tstate->interp, info->name, info->path);
             if (m_copy == NULL) {
@@ -1589,7 +1592,8 @@ reload_singlephase_extension(PyThreadState *tstate,
     }
     else {
         assert(cached->m_dict == NULL);
-        assert(def->m_base.m_copy == NULL);
+        assert(def->m_base.m_cache_has_m_init);
+        assert(!def->m_base.m_cache_has_m_dict);
         PyModInitFunction p0 = cached->m_init;
         if (p0 == NULL) {
             assert(!PyErr_Occurred());
@@ -1714,18 +1718,16 @@ import_run_extension(PyThreadState *tstate, PyModInitFunction p0,
 
         /* Update global import state. */
         struct singlephase_global_update singlephase = {0};
-        // gh-88216: Extensions and def->m_base.m_copy can be updated
+        // gh-88216: Extensions and def->m_base.m_dict can be updated
         // when the extension module doesn't support sub-interpreters.
         if (def->m_size == -1) {
-            /* We will reload from m_copy. */
-            assert(def->m_base.m_init == NULL);
+            /* We will reload from m_dict. */
             singlephase.m_dict = PyModule_GetDict(mod);
             assert(singlephase.m_dict != NULL);
         }
         else {
             /* We will reload via the init function. */
             assert(def->m_size >= 0);
-            assert(def->m_base.m_copy == NULL);
             singlephase.m_init = p0;
         }
         if (update_global_state_for_extension(
@@ -1765,8 +1767,8 @@ clear_singlephase_extension(PyInterpreterState *interp,
     PyModuleDef *def = value->def;
 
     /* Clear data set when the module was initially loaded. */
-    def->m_base.m_init = NULL;
-    Py_CLEAR(def->m_base.m_copy);
+    def->m_base.m_cache_has_m_init = 0;
+    def->m_base.m_cache_has_m_dict = 0;
     // We leave m_index alone since there's no reason to reset it.
 
     /* Clear the PyState_*Module() cache entry. */
@@ -1808,22 +1810,23 @@ _PyImport_FixupBuiltin(PyThreadState *tstate, PyObject *mod, const char *name,
 
     /* We only use _PyImport_FixupBuiltin() for the core builtin modules
      * (sys and builtins).  These modules are single-phase init with no
-     * module state, but we also don't populate def->m_base.m_copy
+     * module state, but we also don't populate the cached m_dict
      * for them. */
     assert(is_core_module(tstate->interp, nameobj, nameobj));
     assert(check_singlephase(def));
     assert(def->m_size == -1);
-    assert(def->m_base.m_copy == NULL);
 
     struct singlephase_global_update singlephase = {
-        /* We don't want def->m_base.m_copy populated. */
+        /* We don't want the cached m_dict populated. */
         .m_dict=NULL,
     };
     if (update_global_state_for_extension(
             tstate, nameobj, nameobj, def, &singlephase) < 0)
     {
         goto finally;
     }
+    assert(!def->m_base.m_cache_has_m_init);
+    assert(def->m_base.m_cache_has_m_dict);
 
     if (finish_singlephase_extension(tstate, mod, def, nameobj, modules) < 0) {
         goto finally;
