bpo-31404: Revert "remove modules from Py_InterpreterState (#1638)" (#3565)

ericsnowcurrently · web-flow · commit 93c92f7d1dbb · 2017-09-13T23:46:04.000-07:00
PR #1638, for bpo-28411, causes problems in some (very) edge cases. Until that gets sorted out, we're reverting the merge. PR #3506, a fix on top of #1638, is also getting reverted.
diff --git a/Doc/c-api/import.rst b/Doc/c-api/import.rst
@@ -204,13 +204,6 @@ Importing Modules
    Return the dictionary used for the module administration (a.k.a.
    ``sys.modules``).  Note that this is a per-interpreter variable.
 
-.. c:function:: PyObject* PyImport_GetModule(PyObject *name)
-
-   Return the already imported module with the given name.  If the
-   module has not been imported yet then returns NULL but does not set
-   an error.  Returns NULL and sets an error if the lookup failed.
-
-   .. versionadded:: 3.7
 
 .. c:function:: PyObject* PyImport_GetImporter(PyObject *path)
 
diff --git a/Doc/whatsnew/3.7.rst b/Doc/whatsnew/3.7.rst
@@ -486,9 +486,6 @@ Changes in the Python API
   and module are affected by this change. (Contributed by INADA Naoki and
   Eugene Toder in :issue:`29463`.)
 
-* ``PyInterpreterState`` no longer has a ``modules`` field.  Instead use
-  ``sys.modules``.
-
 * The *mode* argument of :func:`os.makedirs` no longer affects the file
   permission bits of newly-created intermediate-level directories.
   To set their file permission bits you can set the umask before invoking
diff --git a/Include/import.h b/Include/import.h
@@ -38,25 +38,11 @@ PyAPI_FUNC(PyObject *) PyImport_ExecCodeModuleObject(
     );
 #endif
 PyAPI_FUNC(PyObject *) PyImport_GetModuleDict(void);
-#ifndef Py_LIMITED_API
-PyAPI_FUNC(int) _PyImport_IsInitialized(PyInterpreterState *);
-#endif
-PyAPI_FUNC(PyObject *) PyImport_GetModule(PyObject *name);
-#ifndef Py_LIMITED_API
-PyAPI_FUNC(PyObject *) _PyImport_GetModule(PyObject *name);
-PyAPI_FUNC(PyObject *) _PyImport_GetModuleWithError(PyObject *name);
-PyAPI_FUNC(PyObject *) _PyImport_GetModuleId(struct _Py_Identifier *name);
-PyAPI_FUNC(int) _PyImport_SetModule(PyObject *name, PyObject *module);
-PyAPI_FUNC(int) _PyImport_SetModuleString(const char *name, PyObject* module);
-#endif
 #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x03030000
 PyAPI_FUNC(PyObject *) PyImport_AddModuleObject(
     PyObject *name
     );
 #endif
-#ifndef Py_LIMITED_API
-PyAPI_FUNC(PyObject *) _PyImport_AddModuleObject(PyObject *, PyObject *);
-#endif
 PyAPI_FUNC(PyObject *) PyImport_AddModule(
     const char *name            /* UTF-8 encoded string */
     );
@@ -106,19 +92,14 @@ PyAPI_FUNC(int) _PyImport_ReleaseLock(void);
 PyAPI_FUNC(void) _PyImport_ReInitLock(void);
 
 PyAPI_FUNC(PyObject *) _PyImport_FindBuiltin(
-    const char *name,            /* UTF-8 encoded string */
-    PyObject *modules
+    const char *name            /* UTF-8 encoded string */
     );
 PyAPI_FUNC(PyObject *) _PyImport_FindExtensionObject(PyObject *, PyObject *);
-PyAPI_FUNC(PyObject *) _PyImport_FindExtensionObjectEx(PyObject *, PyObject *,
-                                                       PyObject *);
 PyAPI_FUNC(int) _PyImport_FixupBuiltin(
     PyObject *mod,
-    const char *name,            /* UTF-8 encoded string */
-    PyObject *modules
+    const char *name            /* UTF-8 encoded string */
     );
-PyAPI_FUNC(int) _PyImport_FixupExtensionObject(PyObject*, PyObject *,
-                                               PyObject *, PyObject *);
+PyAPI_FUNC(int) _PyImport_FixupExtensionObject(PyObject*, PyObject *, PyObject *);
 
 struct _inittab {
     const char *name;           /* ASCII encoded string */
diff --git a/Include/modsupport.h b/Include/modsupport.h
@@ -191,10 +191,6 @@ PyAPI_FUNC(int) PyModule_ExecDef(PyObject *module, PyModuleDef *def);
 
 PyAPI_FUNC(PyObject *) PyModule_Create2(struct PyModuleDef*,
                                      int apiver);
-#ifndef Py_LIMITED_API
-PyAPI_FUNC(PyObject *) _PyModule_CreateInitialized(struct PyModuleDef*,
-                                                   int apiver);
-#endif
 
 #ifdef Py_LIMITED_API
 #define PyModule_Create(module) \
diff --git a/Include/object.h b/Include/object.h
@@ -727,13 +727,14 @@ PyAPI_FUNC(Py_ssize_t) _Py_GetRefTotal(void);
 /* Py_REF_DEBUG also controls the display of refcounts and memory block
  * allocations at the interactive prompt and at interpreter shutdown
  */
-PyAPI_FUNC(PyObject *) _PyDebug_XOptionShowRefCount(void);
 PyAPI_FUNC(void) _PyDebug_PrintTotalRefs(void);
+#define _PY_DEBUG_PRINT_TOTAL_REFS() _PyDebug_PrintTotalRefs()
 #else
 #define _Py_INC_REFTOTAL
 #define _Py_DEC_REFTOTAL
 #define _Py_REF_DEBUG_COMMA
 #define _Py_CHECK_REFCNT(OP)    /* a semicolon */;
+#define _PY_DEBUG_PRINT_TOTAL_REFS()
 #endif /* Py_REF_DEBUG */
 
 #ifdef COUNT_ALLOCS
diff --git a/Include/pystate.h b/Include/pystate.h
@@ -53,13 +53,16 @@ typedef struct _is {
 
     int64_t id;
 
+    PyObject *modules;
     PyObject *modules_by_index;
     PyObject *sysdict;
     PyObject *builtins;
     PyObject *importlib;
 
     /* Used in Python/sysmodule.c. */
     int check_interval;
+    PyObject *warnoptions;
+    PyObject *xoptions;
 
     /* Used in Modules/_threadmodule.c. */
     long num_threads;
diff --git a/Misc/NEWS.d/next/Core and Builtins/2017-09-04-10-46-09.bpo-28411.IU9rQL.rst b/Misc/NEWS.d/next/Core and Builtins/2017-09-04-10-46-09.bpo-28411.IU9rQL.rst
diff --git a/Modules/_pickle.c b/Modules/_pickle.c
@@ -6425,7 +6425,9 @@ _pickle_Unpickler_find_class_impl(UnpicklerObject *self,
 /*[clinic end generated code: output=becc08d7f9ed41e3 input=e2e6a865de093ef4]*/
 {
     PyObject *global;
+    PyObject *modules_dict;
     PyObject *module;
+    _Py_IDENTIFIER(modules);
 
     /* Try to map the old names used in Python 2.x to the new ones used in
        Python 3.x.  We do this only with old pickle protocols and when the
@@ -6482,19 +6484,25 @@ _pickle_Unpickler_find_class_impl(UnpicklerObject *self,
         }
     }
 
-    module = PyImport_GetModule(module_name);
+    modules_dict = _PySys_GetObjectId(&PyId_modules);
+    if (modules_dict == NULL) {
+        PyErr_SetString(PyExc_RuntimeError, "unable to get sys.modules");
+        return NULL;
+    }
+
+    module = PyDict_GetItemWithError(modules_dict, module_name);
     if (module == NULL) {
         if (PyErr_Occurred())
             return NULL;
         module = PyImport_Import(module_name);
         if (module == NULL)
             return NULL;
         global = getattribute(module, global_name, self->proto >= 4);
+        Py_DECREF(module);
     }
     else {
         global = getattribute(module, global_name, self->proto >= 4);
     }
-    Py_DECREF(module);
     return global;
 }
 
diff --git a/Modules/pyexpat.c b/Modules/pyexpat.c
@@ -1643,6 +1643,7 @@ MODULE_INITFUNC(void)
     PyObject *errors_module;
     PyObject *modelmod_name;
     PyObject *model_module;
+    PyObject *sys_modules;
     PyObject *tmpnum, *tmpstr;
     PyObject *codes_dict;
     PyObject *rev_codes_dict;
@@ -1692,6 +1693,11 @@ MODULE_INITFUNC(void)
     */
     PyModule_AddStringConstant(m, "native_encoding", "UTF-8");
 
+    sys_modules = PySys_GetObject("modules");
+    if (sys_modules == NULL) {
+        Py_DECREF(m);
+        return NULL;
+    }
     d = PyModule_GetDict(m);
     if (d == NULL) {
         Py_DECREF(m);
@@ -1701,7 +1707,7 @@ MODULE_INITFUNC(void)
     if (errors_module == NULL) {
         errors_module = PyModule_New(MODULE_NAME ".errors");
         if (errors_module != NULL) {
-            _PyImport_SetModule(errmod_name, errors_module);
+            PyDict_SetItem(sys_modules, errmod_name, errors_module);
             /* gives away the reference to errors_module */
             PyModule_AddObject(m, "errors", errors_module);
         }
@@ -1711,7 +1717,7 @@ MODULE_INITFUNC(void)
     if (model_module == NULL) {
         model_module = PyModule_New(MODULE_NAME ".model");
         if (model_module != NULL) {
-            _PyImport_SetModule(modelmod_name, model_module);
+            PyDict_SetItem(sys_modules, modelmod_name, model_module);
             /* gives away the reference to model_module */
             PyModule_AddObject(m, "model", model_module);
         }
diff --git a/Objects/moduleobject.c b/Objects/moduleobject.c
@@ -161,18 +161,12 @@ _add_methods_to_object(PyObject *module, PyObject *name, PyMethodDef *functions)
 
 PyObject *
 PyModule_Create2(struct PyModuleDef* module, int module_api_version)
-{
-    if (!_PyImport_IsInitialized(PyThreadState_GET()->interp))
-        Py_FatalError("Python import machinery not initialized");
-    return _PyModule_CreateInitialized(module, module_api_version);
-}
-
-PyObject *
-_PyModule_CreateInitialized(struct PyModuleDef* module, int module_api_version)
 {
     const char* name;
     PyModuleObject *m;
-
+    PyInterpreterState *interp = PyThreadState_Get()->interp;
+    if (interp->modules == NULL)
+        Py_FatalError("Python import machinery not initialized");
     if (!PyModuleDef_Init(module))
         return NULL;
     name = module->m_name;
diff --git a/Objects/object.c b/Objects/object.c
@@ -29,23 +29,20 @@ _Py_GetRefTotal(void)
     return total;
 }
 
-PyObject *
-_PyDebug_XOptionShowRefCount(void)
-{
-    PyObject *xoptions = PySys_GetXOptions();
-    if (xoptions == NULL)
-        return NULL;
-
-    _Py_IDENTIFIER(showrefcount);
-    return _PyDict_GetItemId(xoptions, &PyId_showrefcount);
-}
-
 void
 _PyDebug_PrintTotalRefs(void) {
-    fprintf(stderr,
-            "[%" PY_FORMAT_SIZE_T "d refs, "
-            "%" PY_FORMAT_SIZE_T "d blocks]\n",
-            _Py_GetRefTotal(), _Py_GetAllocatedBlocks());
+    PyObject *xoptions, *value;
+    _Py_IDENTIFIER(showrefcount);
+
+    xoptions = PySys_GetXOptions();
+    if (xoptions == NULL)
+        return;
+    value = _PyDict_GetItemId(xoptions, &PyId_showrefcount);
+    if (value == Py_True)
+        fprintf(stderr,
+                "[%" PY_FORMAT_SIZE_T "d refs, "
+                "%" PY_FORMAT_SIZE_T "d blocks]\n",
+                _Py_GetRefTotal(), _Py_GetAllocatedBlocks());
 }
 #endif /* Py_REF_DEBUG */
 
diff --git a/Objects/typeobject.c b/Objects/typeobject.c
@@ -3902,6 +3902,7 @@ import_copyreg(void)
 {
     PyObject *copyreg_str;
     PyObject *copyreg_module;
+    PyInterpreterState *interp = PyThreadState_GET()->interp;
     _Py_IDENTIFIER(copyreg);
 
     copyreg_str = _PyUnicode_FromId(&PyId_copyreg);
@@ -3913,7 +3914,7 @@ import_copyreg(void)
        by storing a reference to the cached module in a static variable, but
        this broke when multiple embedded interpreters were in use (see issue
        #17408 and #19088). */
-    copyreg_module = _PyImport_GetModuleWithError(copyreg_str);
+    copyreg_module = PyDict_GetItemWithError(interp->modules, copyreg_str);
     if (copyreg_module != NULL) {
         Py_INCREF(copyreg_module);
         return copyreg_module;
diff --git a/Python/_warnings.c b/Python/_warnings.c
@@ -38,6 +38,7 @@ static PyObject *
 get_warnings_attr(const char *attr, int try_import)
 {
     static PyObject *warnings_str = NULL;
+    PyObject *all_modules;
     PyObject *warnings_module, *obj;
 
     if (warnings_str == NULL) {
@@ -57,7 +58,9 @@ get_warnings_attr(const char *attr, int try_import)
         }
     }
     else {
-        warnings_module = _PyImport_GetModule(warnings_str);
+        all_modules = PyImport_GetModuleDict();
+
+        warnings_module = PyDict_GetItem(all_modules, warnings_str);
         if (warnings_module == NULL)
             return NULL;
 
diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c
@@ -2685,7 +2685,7 @@ _PyBuiltin_Init(void)
         PyType_Ready(&PyZip_Type) < 0)
         return NULL;
 
-    mod = _PyModule_CreateInitialized(&builtinsmodule, PYTHON_API_VERSION);
+    mod = PyModule_Create(&builtinsmodule);
     if (mod == NULL)
         return NULL;
     dict = PyModule_GetDict(mod);
diff --git a/Python/ceval.c b/Python/ceval.c
@@ -4935,7 +4935,7 @@ import_from(PyObject *v, PyObject *name)
         Py_DECREF(pkgname);
         return NULL;
     }
-    x = _PyImport_GetModule(fullmodname);
+    x = PyDict_GetItem(PyImport_GetModuleDict(), fullmodname);
     Py_DECREF(fullmodname);
     if (x == NULL) {
         goto error;
diff --git a/Python/import.c b/Python/import.c
diff --git a/Python/importdl.c b/Python/importdl.c
diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c
diff --git a/Python/pystate.c b/Python/pystate.c
diff --git a/Python/pythonrun.c b/Python/pythonrun.c
diff --git a/Python/sysmodule.c b/Python/sysmodule.c

Original file line number	Diff line number	Diff line change
`@@ -6425,7 +6425,9 @@ _pickle_Unpickler_find_class_impl(UnpicklerObject *self,`
`6425`	`6425`	`/[clinic end generated code: output=becc08d7f9ed41e3 input=e2e6a865de093ef4]/`
`6426`	`6426`	`{`
`6427`	`6427`	`PyObject *global;`
	`6428`	`+ PyObject *modules_dict;`
`6428`	`6429`	`PyObject *module;`
	`6430`	`+ _Py_IDENTIFIER(modules);`
`6429`	`6431`
`6430`	`6432`	`/* Try to map the old names used in Python 2.x to the new ones used in`
`6431`	`6433`	`Python 3.x. We do this only with old pickle protocols and when the`
`@@ -6482,19 +6484,25 @@ _pickle_Unpickler_find_class_impl(UnpicklerObject *self,`
`6482`	`6484`	`}`
`6483`	`6485`	`}`
`6484`	`6486`
`6485`		`- module = PyImport_GetModule(module_name);`
	`6487`	`+ modules_dict = _PySys_GetObjectId(&PyId_modules);`
	`6488`	`+ if (modules_dict == NULL) {`
	`6489`	`+ PyErr_SetString(PyExc_RuntimeError, "unable to get sys.modules");`
	`6490`	`+ return NULL;`
	`6491`	`+ }`
	`6492`	`+`
	`6493`	`+ module = PyDict_GetItemWithError(modules_dict, module_name);`
`6486`	`6494`	`if (module == NULL) {`
`6487`	`6495`	`if (PyErr_Occurred())`
`6488`	`6496`	`return NULL;`
`6489`	`6497`	`module = PyImport_Import(module_name);`
`6490`	`6498`	`if (module == NULL)`
`6491`	`6499`	`return NULL;`
`6492`	`6500`	`global = getattribute(module, global_name, self->proto >= 4);`
	`6501`	`+ Py_DECREF(module);`
`6493`	`6502`	`}`
`6494`	`6503`	`else {`
`6495`	`6504`	`global = getattribute(module, global_name, self->proto >= 4);`
`6496`	`6505`	`}`
`6497`		`- Py_DECREF(module);`
`6498`	`6506`	`return global;`
`6499`	`6507`	`}`
`6500`	`6508`