From 70f6e1968947811154560f60cb0d6d5401d827f6 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Thu, 6 Oct 2016 23:27:39 -0600 Subject: [PATCH 01/35] Add _PyImport_GetModuleDict(). --- Include/import.h | 3 +++ Python/import.c | 19 +++++++++++++++---- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/Include/import.h b/Include/import.h index bb6beba67b738f..2db3b361da9194 100644 --- a/Include/import.h +++ b/Include/import.h @@ -38,6 +38,9 @@ PyAPI_FUNC(PyObject *) PyImport_ExecCodeModuleObject( ); #endif PyAPI_FUNC(PyObject *) PyImport_GetModuleDict(void); +#ifndef Py_LIMITED_API +PyAPI_FUNC(PyObject *) _PyImport_GetModuleDict(PyThreadState *tstate); +#endif #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x03030000 PyAPI_FUNC(PyObject *) PyImport_AddModuleObject( PyObject *name diff --git a/Python/import.c b/Python/import.c index f27b7cb010ce0b..ba09c0ab89c10f 100644 --- a/Python/import.c +++ b/Python/import.c @@ -300,13 +300,24 @@ _PyImport_Fini(void) /* Helper for sys */ +PyObject * +_PyImport_GetModuleDict(PyThreadState *tstate) +{ + if (tstate->interp->sysdict == NULL) + Py_FatalError("_PyImport_GetModuleDict: no sys module!"); + + _Py_IDENTIFIER(modules); + PyObject *modules = _PyDict_GetItemId(tstate->interp->sysdict, + &PyId_modules); + if (modules == NULL) + Py_FatalError("lost sys.modules"); + return modules; +} + PyObject * PyImport_GetModuleDict(void) { - PyInterpreterState *interp = PyThreadState_GET()->interp; - if (interp->modules == NULL) - Py_FatalError("PyImport_GetModuleDict: no module dictionary!"); - return interp->modules; + return _PyImport_GetModuleDict(PyThreadState_GET()); } From 10f7443c03568d3f8bd3b3e7e476ad1aa98e4845 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 10 Oct 2016 12:11:52 -0600 Subject: [PATCH 02/35] Import the sys module before any other imports. --- Python/pylifecycle.c | 51 ++++++++++++++++++++++++-------------------- 1 file changed, 28 insertions(+), 23 deletions(-) diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index eaa7b7f459953a..17af49a06d13f5 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -647,9 +647,20 @@ void _Py_InitializeCore(const _PyCoreConfig *config) if (!_PyFloat_Init()) Py_FatalError("Py_InitializeCore: can't init float"); - interp->modules = PyDict_New(); - if (interp->modules == NULL) + PyObject *modules = PyDict_New(); + if (modules == NULL) Py_FatalError("Py_InitializeCore: can't make modules dictionary"); + interp->modules = modules; + + sysmod = _PySys_BeginInit(); + if (sysmod == NULL) + Py_FatalError("Py_InitializeCore: can't initialize sys"); + interp->sysdict = PyModule_GetDict(sysmod); + if (interp->sysdict == NULL) + Py_FatalError("Py_InitializeCore: can't initialize sys dict"); + Py_INCREF(interp->sysdict); + PyDict_SetItemString(interp->sysdict, "modules", modules); + _PyImport_FixupBuiltin(sysmod, "sys"); /* Init Unicode implementation; relies on the codec registry */ if (_PyUnicode_Init() < 0) @@ -670,17 +681,6 @@ void _Py_InitializeCore(const _PyCoreConfig *config) /* initialize builtin exceptions */ _PyExc_Init(bimod); - sysmod = _PySys_BeginInit(); - if (sysmod == NULL) - Py_FatalError("Py_InitializeCore: can't initialize sys"); - interp->sysdict = PyModule_GetDict(sysmod); - if (interp->sysdict == NULL) - Py_FatalError("Py_InitializeCore: can't initialize sys dict"); - Py_INCREF(interp->sysdict); - _PyImport_FixupBuiltin(sysmod, "sys"); - PyDict_SetItemString(interp->sysdict, "modules", - interp->modules); - /* Set up a preliminary stderr printer until we have enough infrastructure for the io module in place. */ pstderr = PyFile_NewStdPrinter(fileno(stderr)); @@ -1178,7 +1178,21 @@ Py_NewInterpreter(void) /* XXX The following is lax in error checking */ - interp->modules = PyDict_New(); + PyObject *modules = PyDict_New(); + if (modules == NULL) + Py_FatalError("Py_NewInterpreter: can't make modules dictionary"); + interp->modules = modules; + + sysmod = _PyImport_FindBuiltin("sys"); + if (sysmod != NULL) { + interp->sysdict = PyModule_GetDict(sysmod); + if (interp->sysdict == NULL) + goto handle_error; + Py_INCREF(interp->sysdict); + PyDict_SetItemString(interp->sysdict, "modules", modules); + PySys_SetPath(Py_GetPath()); + _PySys_EndInit(interp->sysdict); + } bimod = _PyImport_FindBuiltin("builtins"); if (bimod != NULL) { @@ -1191,18 +1205,9 @@ Py_NewInterpreter(void) /* initialize builtin exceptions */ _PyExc_Init(bimod); - sysmod = _PyImport_FindBuiltin("sys"); if (bimod != NULL && sysmod != NULL) { PyObject *pstderr; - interp->sysdict = PyModule_GetDict(sysmod); - if (interp->sysdict == NULL) - goto handle_error; - Py_INCREF(interp->sysdict); - _PySys_EndInit(interp->sysdict); - PySys_SetPath(Py_GetPath()); - PyDict_SetItemString(interp->sysdict, "modules", - interp->modules); /* Set up a preliminary stderr printer until we have enough infrastructure for the io module in place. */ pstderr = PyFile_NewStdPrinter(fileno(stderr)); From 5ea0e3b4c1a9ebbd2247a63502611a71b7050946 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 10 Oct 2016 13:01:59 -0600 Subject: [PATCH 03/35] Pass the modules dict to _PyImport_FindBuiltin(). --- Include/import.h | 8 +++++++- Python/import.c | 24 +++++++++++++++++++----- Python/pylifecycle.c | 4 ++-- 3 files changed, 28 insertions(+), 8 deletions(-) diff --git a/Include/import.h b/Include/import.h index 2db3b361da9194..a273d6a39d2cff 100644 --- a/Include/import.h +++ b/Include/import.h @@ -46,6 +46,9 @@ PyAPI_FUNC(PyObject *) PyImport_AddModuleObject( PyObject *name ); #endif +#ifndef Py_LIMITED_API +PyAPI_FUNC(PyObject *) _PyImport_AddModuleObject(PyObject *, PyObject *); +#endif PyAPI_FUNC(PyObject *) PyImport_AddModule( const char *name /* UTF-8 encoded string */ ); @@ -100,9 +103,12 @@ PyAPI_FUNC(int) _PyImport_ReleaseLock(void); PyAPI_FUNC(void) _PyImport_ReInitLock(void); PyAPI_FUNC(PyObject *) _PyImport_FindBuiltin( - const char *name /* UTF-8 encoded string */ + const char *name, /* UTF-8 encoded string */ + PyObject *modules ); PyAPI_FUNC(PyObject *) _PyImport_FindExtensionObject(PyObject *, PyObject *); +PyAPI_FUNC(PyObject *) _PyImport_FindExtensionObjectEx(PyObject *, PyObject *, + PyObject *); PyAPI_FUNC(int) _PyImport_FixupBuiltin( PyObject *mod, const char *name /* UTF-8 encoded string */ diff --git a/Python/import.c b/Python/import.c index ba09c0ab89c10f..9b4d4616ff60c6 100644 --- a/Python/import.c +++ b/Python/import.c @@ -600,6 +600,14 @@ _PyImport_FixupBuiltin(PyObject *mod, const char *name) PyObject * _PyImport_FindExtensionObject(PyObject *name, PyObject *filename) +{ + PyObject *modules = PyImport_GetModuleDict(); + return _PyImport_FindExtensionObjectEx(name, filename, modules); +} + +PyObject * +_PyImport_FindExtensionObjectEx(PyObject *name, PyObject *filename, + PyObject *modules) { PyObject *mod, *mdict, *key; PyModuleDef* def; @@ -616,7 +624,7 @@ _PyImport_FindExtensionObject(PyObject *name, PyObject *filename) /* Module does not support repeated initialization */ if (def->m_base.m_copy == NULL) return NULL; - mod = PyImport_AddModuleObject(name); + mod = _PyImport_AddModuleObject(name, modules); if (mod == NULL) return NULL; mdict = PyModule_GetDict(mod); @@ -631,14 +639,14 @@ _PyImport_FindExtensionObject(PyObject *name, PyObject *filename) mod = def->m_base.m_init(); if (mod == NULL) return NULL; - if (PyDict_SetItem(PyImport_GetModuleDict(), name, mod) == -1) { + if (PyDict_SetItem(modules, name, mod) == -1) { Py_DECREF(mod); return NULL; } Py_DECREF(mod); } if (_PyState_AddModule(mod, def) < 0) { - PyDict_DelItem(PyImport_GetModuleDict(), name); + PyDict_DelItem(modules, name); Py_DECREF(mod); return NULL; } @@ -650,13 +658,13 @@ _PyImport_FindExtensionObject(PyObject *name, PyObject *filename) } PyObject * -_PyImport_FindBuiltin(const char *name) +_PyImport_FindBuiltin(const char *name, PyObject *modules) { PyObject *res, *nameobj; nameobj = PyUnicode_InternFromString(name); if (nameobj == NULL) return NULL; - res = _PyImport_FindExtensionObject(nameobj, nameobj); + res = _PyImport_FindExtensionObjectEx(nameobj, nameobj, modules); Py_DECREF(nameobj); return res; } @@ -671,6 +679,12 @@ PyObject * PyImport_AddModuleObject(PyObject *name) { PyObject *modules = PyImport_GetModuleDict(); + return _PyImport_AddModuleObject(name, modules); +} + +PyObject * +_PyImport_AddModuleObject(PyObject *name, PyObject *modules) +{ PyObject *m; if ((m = PyDict_GetItemWithError(modules, name)) != NULL && diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 17af49a06d13f5..228022f3290b0b 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -1183,7 +1183,7 @@ Py_NewInterpreter(void) Py_FatalError("Py_NewInterpreter: can't make modules dictionary"); interp->modules = modules; - sysmod = _PyImport_FindBuiltin("sys"); + sysmod = _PyImport_FindBuiltin("sys", modules); if (sysmod != NULL) { interp->sysdict = PyModule_GetDict(sysmod); if (interp->sysdict == NULL) @@ -1194,7 +1194,7 @@ Py_NewInterpreter(void) _PySys_EndInit(interp->sysdict); } - bimod = _PyImport_FindBuiltin("builtins"); + bimod = _PyImport_FindBuiltin("builtins", modules); if (bimod != NULL) { interp->builtins = PyModule_GetDict(bimod); if (interp->builtins == NULL) From 29780ec937a55ff1776bdfa8199f7f2bc3fda0af Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Fri, 7 Oct 2016 00:02:53 -0600 Subject: [PATCH 04/35] Replace usage of interp->modules with PyImport_GetModuleDict(). --- Objects/typeobject.c | 4 ++-- Python/import.c | 8 +++++--- Python/pylifecycle.c | 5 ++--- Python/sysmodule.c | 5 +++-- 4 files changed, 12 insertions(+), 10 deletions(-) diff --git a/Objects/typeobject.c b/Objects/typeobject.c index d1a12a7efac2c8..cdde8a87c00397 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -3901,7 +3901,6 @@ import_copyreg(void) { PyObject *copyreg_str; PyObject *copyreg_module; - PyInterpreterState *interp = PyThreadState_GET()->interp; _Py_IDENTIFIER(copyreg); copyreg_str = _PyUnicode_FromId(&PyId_copyreg); @@ -3913,7 +3912,8 @@ import_copyreg(void) by storing a reference to the cached module in a static variable, but this broke when multiple embedded interpreters were in use (see issue #17408 and #19088). */ - copyreg_module = PyDict_GetItemWithError(interp->modules, copyreg_str); + PyObject *modules = PyImport_GetModuleDict(); + copyreg_module = PyDict_GetItemWithError(modules, copyreg_str); if (copyreg_module != NULL) { Py_INCREF(copyreg_module); return copyreg_module; diff --git a/Python/import.c b/Python/import.c index 9b4d4616ff60c6..5179c2cbe02b2d 100644 --- a/Python/import.c +++ b/Python/import.c @@ -347,7 +347,7 @@ PyImport_Cleanup(void) Py_ssize_t pos; PyObject *key, *value, *dict; PyInterpreterState *interp = PyThreadState_GET()->interp; - PyObject *modules = interp->modules; + PyObject *modules = PyImport_GetModuleDict(); PyObject *weaklist = NULL; const char * const *p; @@ -1549,7 +1549,8 @@ PyImport_ImportModuleLevelObject(PyObject *name, PyObject *globals, Py_INCREF(abs_name); } - mod = PyDict_GetItem(interp->modules, abs_name); + PyObject *modules = PyImport_GetModuleDict(); + mod = PyDict_GetItem(modules, abs_name); if (mod != NULL && mod != Py_None) { _Py_IDENTIFIER(__spec__); _Py_IDENTIFIER(_initializing); @@ -1636,7 +1637,8 @@ PyImport_ImportModuleLevelObject(PyObject *name, PyObject *globals, goto error; } - final_mod = PyDict_GetItem(interp->modules, to_return); + PyObject *modules = PyImport_GetModuleDict(); + final_mod = PyDict_GetItem(modules, to_return); Py_DECREF(to_return); if (final_mod == NULL) { PyErr_Format(PyExc_KeyError, diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 228022f3290b0b..243002b8233145 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -1887,9 +1887,8 @@ wait_for_thread_shutdown(void) #ifdef WITH_THREAD _Py_IDENTIFIER(_shutdown); PyObject *result; - PyThreadState *tstate = PyThreadState_GET(); - PyObject *threading = PyMapping_GetItemString(tstate->interp->modules, - "threading"); + PyObject *modules = PyImport_GetModuleDict(); + PyObject *threading = PyMapping_GetItemString(modules, "threading"); if (threading == NULL) { /* threading not imported */ PyErr_Clear(); diff --git a/Python/sysmodule.c b/Python/sysmodule.c index ab435c83104ddc..fff6876d80bdf8 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -159,8 +159,9 @@ static PyObject * sys_displayhook(PyObject *self, PyObject *o) { PyObject *outf; - PyInterpreterState *interp = PyThreadState_GET()->interp; - PyObject *modules = interp->modules; + PyObject *modules = PyImport_GetModuleDict(); + if (modules == NULL) + return NULL; PyObject *builtins; static PyObject *newline = NULL; int err; From e6de21575a5785315e9cacd5ec4c9a7c56cf2027 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 10 Oct 2016 11:23:44 -0600 Subject: [PATCH 05/35] Add _PyImport_EnsureInitialized(). --- Include/import.h | 1 + Python/import.c | 15 +++++++++++++++ 2 files changed, 16 insertions(+) diff --git a/Include/import.h b/Include/import.h index a273d6a39d2cff..af95743a3cf6c5 100644 --- a/Include/import.h +++ b/Include/import.h @@ -40,6 +40,7 @@ PyAPI_FUNC(PyObject *) PyImport_ExecCodeModuleObject( PyAPI_FUNC(PyObject *) PyImport_GetModuleDict(void); #ifndef Py_LIMITED_API PyAPI_FUNC(PyObject *) _PyImport_GetModuleDict(PyThreadState *tstate); +PyAPI_FUNC(void) _PyImport_EnsureInitialized(PyInterpreterState *interp); #endif #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x03030000 PyAPI_FUNC(PyObject *) PyImport_AddModuleObject( diff --git a/Python/import.c b/Python/import.c index 5179c2cbe02b2d..3adaa4451e71c8 100644 --- a/Python/import.c +++ b/Python/import.c @@ -320,6 +320,21 @@ PyImport_GetModuleDict(void) return _PyImport_GetModuleDict(PyThreadState_GET()); } +/* In some corner cases it is important to be sure that the import + machinery has been initialized (or not cleaned up yet). For + example, see issue #4236 and PyModule_Create2(). */ + +void +_PyImport_EnsureInitialized(PyInterpreterState *interp) +{ + if (interp->modules == NULL) + goto notinitialized; + return; + +notinitialized: + Py_FatalError("Python import machinery not initialized"); +} + /* List of names to clear in sys */ static const char * const sys_deletes[] = { From 29f6b7c4ef8ea52d8500650aa51e7eb99452ca32 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 10 Oct 2016 11:30:02 -0600 Subject: [PATCH 06/35] Add _PyModule_Create2(). --- Include/modsupport.h | 4 ++++ Objects/moduleobject.c | 11 ++++++++--- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/Include/modsupport.h b/Include/modsupport.h index 8c7cf39d9a3ea9..3ba15b7853d248 100644 --- a/Include/modsupport.h +++ b/Include/modsupport.h @@ -191,6 +191,10 @@ PyAPI_FUNC(int) PyModule_ExecDef(PyObject *module, PyModuleDef *def); PyAPI_FUNC(PyObject *) PyModule_Create2(struct PyModuleDef*, int apiver); +#ifndef Py_LIMITED_API +PyAPI_FUNC(PyObject *) _PyModule_Create2(struct PyModuleDef*, + int apiver); +#endif #ifdef Py_LIMITED_API #define PyModule_Create(module) \ diff --git a/Objects/moduleobject.c b/Objects/moduleobject.c index 02a8cf0cb6e02f..66ab9a7f89e85e 100644 --- a/Objects/moduleobject.c +++ b/Objects/moduleobject.c @@ -160,12 +160,17 @@ _add_methods_to_object(PyObject *module, PyObject *name, PyMethodDef *functions) PyObject * PyModule_Create2(struct PyModuleDef* module, int module_api_version) +{ + _PyImport_EnsureInitialized(PyThreadState_GET()->interp); + return _PyModule_Create2(module, module_api_version); +} + +PyObject * +_PyModule_Create2(struct PyModuleDef* module, int module_api_version) { const char* name; PyModuleObject *m; - PyInterpreterState *interp = PyThreadState_Get()->interp; - if (interp->modules == NULL) - Py_FatalError("Python import machinery not initialized"); + if (!PyModuleDef_Init(module)) return NULL; name = module->m_name; From b77d73f288b75ab28907cb718c61a3cfc506fde9 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 10 Oct 2016 11:31:51 -0600 Subject: [PATCH 07/35] Use _PyModule_Create2() for key builtin modules. --- Python/bltinmodule.c | 2 +- Python/sysmodule.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index 5e1f1d3854fc67..df84cf1c0d3c16 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -2685,7 +2685,7 @@ _PyBuiltin_Init(void) PyType_Ready(&PyZip_Type) < 0) return NULL; - mod = PyModule_Create(&builtinsmodule); + mod = _PyModule_Create2(&builtinsmodule, PYTHON_API_VERSION); if (mod == NULL) return NULL; dict = PyModule_GetDict(mod); diff --git a/Python/sysmodule.c b/Python/sysmodule.c index fff6876d80bdf8..31e6a864fd40a6 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -1930,7 +1930,7 @@ _PySys_BeginInit(void) PyObject *m, *sysdict, *version_info; int res; - m = PyModule_Create(&sysmodule); + m = _PyModule_Create2(&sysmodule, PYTHON_API_VERSION); if (m == NULL) return NULL; sysdict = PyModule_GetDict(m); From 8275633fbe957281b82da8f13e4ed9f667f788fd Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 10 Oct 2016 11:23:44 -0600 Subject: [PATCH 08/35] Check sys.modules in _PyImport_EnsureInitialized(). --- Python/import.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/Python/import.c b/Python/import.c index 3adaa4451e71c8..5eeeb985096d8d 100644 --- a/Python/import.c +++ b/Python/import.c @@ -327,7 +327,11 @@ PyImport_GetModuleDict(void) void _PyImport_EnsureInitialized(PyInterpreterState *interp) { - if (interp->modules == NULL) + if (interp->sysdict == NULL) + goto notinitialized; + _Py_IDENTIFIER(modules); + PyObject *modules = _PyDict_GetItemId(interp->sysdict, &PyId_modules); + if (modules == NULL) goto notinitialized; return; From 7c62114b15fe66027cc827d8c44dabb78fd7a3e0 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Fri, 7 Oct 2016 00:14:42 -0600 Subject: [PATCH 09/35] Drop PyInterpreterState.modules. --- Include/pystate.h | 1 - Python/import.c | 1 - Python/pylifecycle.c | 2 -- Python/pystate.c | 2 -- 4 files changed, 6 deletions(-) diff --git a/Include/pystate.h b/Include/pystate.h index edfb08b15bc476..8a92f3ec3ed827 100644 --- a/Include/pystate.h +++ b/Include/pystate.h @@ -52,7 +52,6 @@ typedef struct _is { int64_t id; - PyObject *modules; PyObject *modules_by_index; PyObject *sysdict; PyObject *builtins; diff --git a/Python/import.c b/Python/import.c index 5eeeb985096d8d..9774a95f90b6bc 100644 --- a/Python/import.c +++ b/Python/import.c @@ -495,7 +495,6 @@ PyImport_Cleanup(void) /* Clear and delete the modules directory. Actual modules will still be there only if imported during the execution of some destructor. */ - interp->modules = NULL; Py_DECREF(modules); /* Once more */ diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 243002b8233145..28920f530f7588 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -650,7 +650,6 @@ void _Py_InitializeCore(const _PyCoreConfig *config) PyObject *modules = PyDict_New(); if (modules == NULL) Py_FatalError("Py_InitializeCore: can't make modules dictionary"); - interp->modules = modules; sysmod = _PySys_BeginInit(); if (sysmod == NULL) @@ -1181,7 +1180,6 @@ Py_NewInterpreter(void) PyObject *modules = PyDict_New(); if (modules == NULL) Py_FatalError("Py_NewInterpreter: can't make modules dictionary"); - interp->modules = modules; sysmod = _PyImport_FindBuiltin("sys", modules); if (sysmod != NULL) { diff --git a/Python/pystate.c b/Python/pystate.c index 24a08ebf4fe5eb..30a372212ed4db 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -97,7 +97,6 @@ PyInterpreterState_New(void) if (head_mutex == NULL) Py_FatalError("Can't initialize threads for interpreter"); #endif - interp->modules = NULL; interp->modules_by_index = NULL; interp->sysdict = NULL; interp->builtins = NULL; @@ -158,7 +157,6 @@ PyInterpreterState_Clear(PyInterpreterState *interp) Py_CLEAR(interp->codec_search_path); Py_CLEAR(interp->codec_search_cache); Py_CLEAR(interp->codec_error_registry); - Py_CLEAR(interp->modules); Py_CLEAR(interp->modules_by_index); Py_CLEAR(interp->sysdict); Py_CLEAR(interp->builtins); From 428d22322ea3021ae971e1f231cebd53243dec56 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 10 Oct 2016 13:55:55 -0600 Subject: [PATCH 10/35] Pass the modules dict to _PyImport_FixupBuiltin(). --- Include/import.h | 5 ++++- Python/import.c | 15 +++++++++++---- Python/pylifecycle.c | 4 ++-- 3 files changed, 17 insertions(+), 7 deletions(-) diff --git a/Include/import.h b/Include/import.h index af95743a3cf6c5..94dd8d6f29794d 100644 --- a/Include/import.h +++ b/Include/import.h @@ -112,9 +112,12 @@ PyAPI_FUNC(PyObject *) _PyImport_FindExtensionObjectEx(PyObject *, PyObject *, PyObject *); PyAPI_FUNC(int) _PyImport_FixupBuiltin( PyObject *mod, - const char *name /* UTF-8 encoded string */ + const char *name, /* UTF-8 encoded string */ + PyObject *modules ); PyAPI_FUNC(int) _PyImport_FixupExtensionObject(PyObject*, PyObject *, PyObject *); +PyAPI_FUNC(int) _PyImport_FixupExtensionObjectEx(PyObject*, PyObject *, + PyObject *, PyObject *); struct _inittab { const char *name; /* ASCII encoded string */ diff --git a/Python/import.c b/Python/import.c index 9774a95f90b6bc..f3dbe45ddd9ba6 100644 --- a/Python/import.c +++ b/Python/import.c @@ -555,7 +555,15 @@ int _PyImport_FixupExtensionObject(PyObject *mod, PyObject *name, PyObject *filename) { - PyObject *modules, *dict, *key; + PyObject *modules = PyImport_GetModuleDict(); + return _PyImport_FixupExtensionObjectEx(mod, name, filename, modules); +} + +int +_PyImport_FixupExtensionObjectEx(PyObject *mod, PyObject *name, + PyObject *filename, PyObject *modules) +{ + PyObject *dict, *key; struct PyModuleDef *def; int res; if (extensions == NULL) { @@ -572,7 +580,6 @@ _PyImport_FixupExtensionObject(PyObject *mod, PyObject *name, PyErr_BadInternalCall(); return -1; } - modules = PyImport_GetModuleDict(); if (PyDict_SetItem(modules, name, mod) < 0) return -1; if (_PyState_AddModule(mod, def) < 0) { @@ -604,14 +611,14 @@ _PyImport_FixupExtensionObject(PyObject *mod, PyObject *name, } int -_PyImport_FixupBuiltin(PyObject *mod, const char *name) +_PyImport_FixupBuiltin(PyObject *mod, const char *name, PyObject *modules) { int res; PyObject *nameobj; nameobj = PyUnicode_InternFromString(name); if (nameobj == NULL) return -1; - res = _PyImport_FixupExtensionObject(mod, nameobj, nameobj); + res = _PyImport_FixupExtensionObjectEx(mod, nameobj, nameobj, modules); Py_DECREF(nameobj); return res; } diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 28920f530f7588..1ebf5fb680b7be 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -659,7 +659,7 @@ void _Py_InitializeCore(const _PyCoreConfig *config) Py_FatalError("Py_InitializeCore: can't initialize sys dict"); Py_INCREF(interp->sysdict); PyDict_SetItemString(interp->sysdict, "modules", modules); - _PyImport_FixupBuiltin(sysmod, "sys"); + _PyImport_FixupBuiltin(sysmod, "sys", modules); /* Init Unicode implementation; relies on the codec registry */ if (_PyUnicode_Init() < 0) @@ -671,7 +671,7 @@ void _Py_InitializeCore(const _PyCoreConfig *config) bimod = _PyBuiltin_Init(); if (bimod == NULL) Py_FatalError("Py_InitializeCore: can't initialize builtins modules"); - _PyImport_FixupBuiltin(bimod, "builtins"); + _PyImport_FixupBuiltin(bimod, "builtins", modules); interp->builtins = PyModule_GetDict(bimod); if (interp->builtins == NULL) Py_FatalError("Py_InitializeCore: can't initialize builtins dict"); From 833b00ba2ea5344962f1b787df7819b1d0a87934 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 11 Oct 2016 17:00:32 -0600 Subject: [PATCH 11/35] Pass PyInterpreterState to _PyImport_GetModuleDict(). --- Include/import.h | 4 ++-- Python/import.c | 11 +++++------ 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/Include/import.h b/Include/import.h index 94dd8d6f29794d..7bb1fd1cf53597 100644 --- a/Include/import.h +++ b/Include/import.h @@ -39,8 +39,8 @@ PyAPI_FUNC(PyObject *) PyImport_ExecCodeModuleObject( #endif PyAPI_FUNC(PyObject *) PyImport_GetModuleDict(void); #ifndef Py_LIMITED_API -PyAPI_FUNC(PyObject *) _PyImport_GetModuleDict(PyThreadState *tstate); -PyAPI_FUNC(void) _PyImport_EnsureInitialized(PyInterpreterState *interp); +PyAPI_FUNC(PyObject *) _PyImport_GetModuleDict(PyInterpreterState *); +PyAPI_FUNC(void) _PyImport_EnsureInitialized(PyInterpreterState *); #endif #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x03030000 PyAPI_FUNC(PyObject *) PyImport_AddModuleObject( diff --git a/Python/import.c b/Python/import.c index f3dbe45ddd9ba6..429358e8b3c358 100644 --- a/Python/import.c +++ b/Python/import.c @@ -301,14 +301,13 @@ _PyImport_Fini(void) /* Helper for sys */ PyObject * -_PyImport_GetModuleDict(PyThreadState *tstate) +_PyImport_GetModuleDict(PyInterpreterState *interp) { - if (tstate->interp->sysdict == NULL) - Py_FatalError("_PyImport_GetModuleDict: no sys module!"); + if (interp->sysdict == NULL) + Py_FatalError("PyImport_GetModuleDict: no sys module!"); _Py_IDENTIFIER(modules); - PyObject *modules = _PyDict_GetItemId(tstate->interp->sysdict, - &PyId_modules); + PyObject *modules = _PyDict_GetItemId(interp->sysdict, &PyId_modules); if (modules == NULL) Py_FatalError("lost sys.modules"); return modules; @@ -317,7 +316,7 @@ _PyImport_GetModuleDict(PyThreadState *tstate) PyObject * PyImport_GetModuleDict(void) { - return _PyImport_GetModuleDict(PyThreadState_GET()); + return _PyImport_GetModuleDict(PyThreadState_GET()->interp); } /* In some corner cases it is important to be sure that the import From e0e51d2073b711894b970054da21d8a49b113bd9 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 11 Oct 2016 17:04:30 -0600 Subject: [PATCH 12/35] Drop _PyImport_GetModuleDict(). --- Include/import.h | 1 - Python/import.c | 13 ++++--------- 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/Include/import.h b/Include/import.h index 7bb1fd1cf53597..8623fcd640154b 100644 --- a/Include/import.h +++ b/Include/import.h @@ -39,7 +39,6 @@ PyAPI_FUNC(PyObject *) PyImport_ExecCodeModuleObject( #endif PyAPI_FUNC(PyObject *) PyImport_GetModuleDict(void); #ifndef Py_LIMITED_API -PyAPI_FUNC(PyObject *) _PyImport_GetModuleDict(PyInterpreterState *); PyAPI_FUNC(void) _PyImport_EnsureInitialized(PyInterpreterState *); #endif #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x03030000 diff --git a/Python/import.c b/Python/import.c index 429358e8b3c358..e7ba748a021e95 100644 --- a/Python/import.c +++ b/Python/import.c @@ -301,24 +301,19 @@ _PyImport_Fini(void) /* Helper for sys */ PyObject * -_PyImport_GetModuleDict(PyInterpreterState *interp) +PyImport_GetModuleDict(void) { - if (interp->sysdict == NULL) + PyObject *sysdict = PyThreadState_GET()->interp->sysdict; + if (sysdict == NULL) Py_FatalError("PyImport_GetModuleDict: no sys module!"); _Py_IDENTIFIER(modules); - PyObject *modules = _PyDict_GetItemId(interp->sysdict, &PyId_modules); + PyObject *modules = _PyDict_GetItemId(sysdict, &PyId_modules); if (modules == NULL) Py_FatalError("lost sys.modules"); return modules; } -PyObject * -PyImport_GetModuleDict(void) -{ - return _PyImport_GetModuleDict(PyThreadState_GET()->interp); -} - /* In some corner cases it is important to be sure that the import machinery has been initialized (or not cleaned up yet). For example, see issue #4236 and PyModule_Create2(). */ From 88aadeb2e11b513a33419ff14852c7a93ff616cc Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Wed, 17 May 2017 16:55:14 -0700 Subject: [PATCH 13/35] _PyModule_Create2 -> _PyModule_CreateInitialized --- Include/modsupport.h | 4 ++-- Objects/moduleobject.c | 4 ++-- Python/bltinmodule.c | 2 +- Python/sysmodule.c | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Include/modsupport.h b/Include/modsupport.h index 3ba15b7853d248..73d86a94b957f6 100644 --- a/Include/modsupport.h +++ b/Include/modsupport.h @@ -192,8 +192,8 @@ PyAPI_FUNC(int) PyModule_ExecDef(PyObject *module, PyModuleDef *def); PyAPI_FUNC(PyObject *) PyModule_Create2(struct PyModuleDef*, int apiver); #ifndef Py_LIMITED_API -PyAPI_FUNC(PyObject *) _PyModule_Create2(struct PyModuleDef*, - int apiver); +PyAPI_FUNC(PyObject *) _PyModule_CreateInitialized(struct PyModuleDef*, + int apiver); #endif #ifdef Py_LIMITED_API diff --git a/Objects/moduleobject.c b/Objects/moduleobject.c index 66ab9a7f89e85e..7d829bb16b39d4 100644 --- a/Objects/moduleobject.c +++ b/Objects/moduleobject.c @@ -162,11 +162,11 @@ PyObject * PyModule_Create2(struct PyModuleDef* module, int module_api_version) { _PyImport_EnsureInitialized(PyThreadState_GET()->interp); - return _PyModule_Create2(module, module_api_version); + return _PyModule_CreateInitialized(module, module_api_version); } PyObject * -_PyModule_Create2(struct PyModuleDef* module, int module_api_version) +_PyModule_CreateInitialized(struct PyModuleDef* module, int module_api_version) { const char* name; PyModuleObject *m; diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index df84cf1c0d3c16..c363cfe8cea26b 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -2685,7 +2685,7 @@ _PyBuiltin_Init(void) PyType_Ready(&PyZip_Type) < 0) return NULL; - mod = _PyModule_Create2(&builtinsmodule, PYTHON_API_VERSION); + mod = _PyModule_CreateInitialized(&builtinsmodule, PYTHON_API_VERSION); if (mod == NULL) return NULL; dict = PyModule_GetDict(mod); diff --git a/Python/sysmodule.c b/Python/sysmodule.c index 31e6a864fd40a6..3d9202a4809abd 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -1930,7 +1930,7 @@ _PySys_BeginInit(void) PyObject *m, *sysdict, *version_info; int res; - m = _PyModule_Create2(&sysmodule, PYTHON_API_VERSION); + m = _PyModule_CreateInitialized(&sysmodule, PYTHON_API_VERSION); if (m == NULL) return NULL; sysdict = PyModule_GetDict(m); From 79320bd5c5705ef775fb990348987f3f6ed0ba76 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Wed, 17 May 2017 22:41:44 -0700 Subject: [PATCH 14/35] _PyImport_EnsureInitialized() -> _PyImport_IsInitialized() --- Include/import.h | 2 +- Objects/moduleobject.c | 3 ++- Python/import.c | 13 +++++-------- 3 files changed, 8 insertions(+), 10 deletions(-) diff --git a/Include/import.h b/Include/import.h index 8623fcd640154b..090df078ecd525 100644 --- a/Include/import.h +++ b/Include/import.h @@ -39,7 +39,7 @@ PyAPI_FUNC(PyObject *) PyImport_ExecCodeModuleObject( #endif PyAPI_FUNC(PyObject *) PyImport_GetModuleDict(void); #ifndef Py_LIMITED_API -PyAPI_FUNC(void) _PyImport_EnsureInitialized(PyInterpreterState *); +PyAPI_FUNC(int) _PyImport_IsInitialized(PyInterpreterState *); #endif #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x03030000 PyAPI_FUNC(PyObject *) PyImport_AddModuleObject( diff --git a/Objects/moduleobject.c b/Objects/moduleobject.c index 7d829bb16b39d4..89afe29032301d 100644 --- a/Objects/moduleobject.c +++ b/Objects/moduleobject.c @@ -161,7 +161,8 @@ _add_methods_to_object(PyObject *module, PyObject *name, PyMethodDef *functions) PyObject * PyModule_Create2(struct PyModuleDef* module, int module_api_version) { - _PyImport_EnsureInitialized(PyThreadState_GET()->interp); + if (!_PyImport_IsInitialized(PyThreadState_GET()->interp)) + Py_FatalError("Python import machinery not initialized"); return _PyModule_CreateInitialized(module, module_api_version); } diff --git a/Python/import.c b/Python/import.c index e7ba748a021e95..9a19bc0e0863ff 100644 --- a/Python/import.c +++ b/Python/import.c @@ -318,19 +318,16 @@ PyImport_GetModuleDict(void) machinery has been initialized (or not cleaned up yet). For example, see issue #4236 and PyModule_Create2(). */ -void -_PyImport_EnsureInitialized(PyInterpreterState *interp) +int +_PyImport_IsInitialized(PyInterpreterState *interp) { if (interp->sysdict == NULL) - goto notinitialized; + return 0; _Py_IDENTIFIER(modules); PyObject *modules = _PyDict_GetItemId(interp->sysdict, &PyId_modules); if (modules == NULL) - goto notinitialized; - return; - -notinitialized: - Py_FatalError("Python import machinery not initialized"); + return 0; + return 1; } From 6a0215c9c8b583d169f8f67319ae9ba7d7c679b1 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Thu, 18 May 2017 11:44:25 -0700 Subject: [PATCH 15/35] Add _PyImport_GetModule*. --- Include/import.h | 3 +++ Python/_warnings.c | 5 +---- Python/ceval.c | 2 +- Python/import.c | 40 ++++++++++++++++++++++++++++------------ Python/pylifecycle.c | 4 ++-- Python/sysmodule.c | 5 +---- 6 files changed, 36 insertions(+), 23 deletions(-) diff --git a/Include/import.h b/Include/import.h index 090df078ecd525..ecfc29de6b5dc8 100644 --- a/Include/import.h +++ b/Include/import.h @@ -40,6 +40,9 @@ PyAPI_FUNC(PyObject *) PyImport_ExecCodeModuleObject( PyAPI_FUNC(PyObject *) PyImport_GetModuleDict(void); #ifndef Py_LIMITED_API PyAPI_FUNC(int) _PyImport_IsInitialized(PyInterpreterState *); +PyAPI_FUNC(PyObject *) _PyImport_GetModule(PyObject *name); +PyAPI_FUNC(PyObject *) _PyImport_GetModuleString(const char *name); +PyAPI_FUNC(PyObject *) _PyImport_GetModuleId(struct _Py_Identifier *name); #endif #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x03030000 PyAPI_FUNC(PyObject *) PyImport_AddModuleObject( diff --git a/Python/_warnings.c b/Python/_warnings.c index add72e4ebb38af..8616195c4e322a 100644 --- a/Python/_warnings.c +++ b/Python/_warnings.c @@ -44,7 +44,6 @@ static PyObject * get_warnings_attr(const char *attr, int try_import) { static PyObject *warnings_str = NULL; - PyObject *all_modules; PyObject *warnings_module, *obj; if (warnings_str == NULL) { @@ -64,9 +63,7 @@ get_warnings_attr(const char *attr, int try_import) } } else { - all_modules = PyImport_GetModuleDict(); - - warnings_module = PyDict_GetItem(all_modules, warnings_str); + warnings_module = _PyImport_GetModule(warnings_str); if (warnings_module == NULL) return NULL; diff --git a/Python/ceval.c b/Python/ceval.c index 92b13311355ba8..436e5cad25f8c9 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -5054,7 +5054,7 @@ import_from(PyObject *v, PyObject *name) Py_DECREF(pkgname); return NULL; } - x = PyDict_GetItem(PyImport_GetModuleDict(), fullmodname); + x = _PyImport_GetModule(fullmodname); Py_DECREF(fullmodname); if (x == NULL) { goto error; diff --git a/Python/import.c b/Python/import.c index 9a19bc0e0863ff..a080942185cc98 100644 --- a/Python/import.c +++ b/Python/import.c @@ -330,6 +330,27 @@ _PyImport_IsInitialized(PyInterpreterState *interp) return 1; } +PyObject * +_PyImport_GetModule(PyObject *name) +{ + PyObject *modules = PyImport_GetModuleDict(); + return PyDict_GetItem(modules, name); +} + +PyObject * +_PyImport_GetModuleString(const char *name) +{ + PyObject *modules = PyImport_GetModuleDict(); + return PyDict_GetItemString(modules, name); +} + +PyObject * +_PyImport_GetModuleId(struct _Py_Identifier *name) +{ + PyObject *modules = PyImport_GetModuleDict(); + return _PyDict_GetItemId(modules, name); +} + /* List of names to clear in sys */ static const char * const sys_deletes[] = { @@ -853,7 +874,6 @@ module_dict_for_exec(PyObject *name) static PyObject * exec_code_in_module(PyObject *name, PyObject *module_dict, PyObject *code_object) { - PyObject *modules = PyImport_GetModuleDict(); PyObject *v, *m; v = PyEval_EvalCode(code_object, module_dict, module_dict); @@ -863,7 +883,8 @@ exec_code_in_module(PyObject *name, PyObject *module_dict, PyObject *code_object } Py_DECREF(v); - if ((m = PyDict_GetItem(modules, name)) == NULL) { + m = _PyImport_GetModule(name); + if (m == NULL) { PyErr_Format(PyExc_ImportError, "Loaded module %R not found in sys.modules", name); @@ -1565,8 +1586,7 @@ PyImport_ImportModuleLevelObject(PyObject *name, PyObject *globals, Py_INCREF(abs_name); } - PyObject *modules = PyImport_GetModuleDict(); - mod = PyDict_GetItem(modules, abs_name); + mod = _PyImport_GetModule(abs_name); if (mod != NULL && mod != Py_None) { _Py_IDENTIFIER(__spec__); _Py_IDENTIFIER(_initializing); @@ -1653,8 +1673,7 @@ PyImport_ImportModuleLevelObject(PyObject *name, PyObject *globals, goto error; } - PyObject *modules = PyImport_GetModuleDict(); - final_mod = PyDict_GetItem(modules, to_return); + final_mod = _PyImport_GetModule(to_return); Py_DECREF(to_return); if (final_mod == NULL) { PyErr_Format(PyExc_KeyError, @@ -1709,8 +1728,7 @@ PyImport_ReloadModule(PyObject *m) { _Py_IDENTIFIER(reload); PyObject *reloaded_module = NULL; - PyObject *modules = PyImport_GetModuleDict(); - PyObject *imp = PyDict_GetItemString(modules, "imp"); + PyObject *imp = _PyImport_GetModuleString("imp"); if (imp == NULL) { imp = PyImport_ImportModule("imp"); if (imp == NULL) { @@ -1745,7 +1763,6 @@ PyImport_Import(PyObject *module_name) PyObject *globals = NULL; PyObject *import = NULL; PyObject *builtins = NULL; - PyObject *modules = NULL; PyObject *r = NULL; /* Initialize constant string objects */ @@ -1800,9 +1817,8 @@ PyImport_Import(PyObject *module_name) goto err; Py_DECREF(r); - modules = PyImport_GetModuleDict(); - r = PyDict_GetItemWithError(modules, module_name); - if (r != NULL) { + r = _PyImport_GetModule(module_name); + if (r != NULL) Py_INCREF(r); } else if (!PyErr_Occurred()) { diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 1ebf5fb680b7be..273f62ab0e3d42 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -1885,13 +1885,13 @@ wait_for_thread_shutdown(void) #ifdef WITH_THREAD _Py_IDENTIFIER(_shutdown); PyObject *result; - PyObject *modules = PyImport_GetModuleDict(); - PyObject *threading = PyMapping_GetItemString(modules, "threading"); + PyObject *threading = _PyImport_GetModuleString("threading"); if (threading == NULL) { /* threading not imported */ PyErr_Clear(); return; } + Py_INCREF(threading); result = _PyObject_CallMethodId(threading, &PyId__shutdown, NULL); if (result == NULL) { PyErr_WriteUnraisable(threading); diff --git a/Python/sysmodule.c b/Python/sysmodule.c index 3d9202a4809abd..852babbed78964 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -159,14 +159,11 @@ static PyObject * sys_displayhook(PyObject *self, PyObject *o) { PyObject *outf; - PyObject *modules = PyImport_GetModuleDict(); - if (modules == NULL) - return NULL; PyObject *builtins; static PyObject *newline = NULL; int err; - builtins = _PyDict_GetItemId(modules, &PyId_builtins); + builtins = _PyImport_GetModuleId(&PyId_builtins); if (builtins == NULL) { PyErr_SetString(PyExc_RuntimeError, "lost builtins module"); return NULL; From dabd1879a6797bd2dd39e579bb50900c77e6171c Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Thu, 18 May 2017 12:50:48 -0700 Subject: [PATCH 16/35] Allow sys.modules to be any mapping. --- Modules/_pickle.c | 11 ++++++++- Modules/pyexpat.c | 4 ++-- Objects/typeobject.c | 10 +++++++- Python/import.c | 56 ++++++++++++++++++++++++++++++++------------ Python/pylifecycle.c | 2 +- 5 files changed, 63 insertions(+), 20 deletions(-) diff --git a/Modules/_pickle.c b/Modules/_pickle.c index a6f3abeba06f85..76f720c9eec515 100644 --- a/Modules/_pickle.c +++ b/Modules/_pickle.c @@ -6483,7 +6483,16 @@ _pickle_Unpickler_find_class_impl(UnpicklerObject *self, return NULL; } - module = PyDict_GetItemWithError(modules_dict, module_name); + module = PyObject_GetItem(modules_dict, module_name); + if (PyDict_Check(modules_dict)) { + module = PyDict_GetItemWithError(modules_dict, module_name); + } else { + module = PyObject_GetItem(modules_dict, module_name); + if (PyErr_Occurred() && PyErr_ExceptionMatches(PyExc_KeyError)) + // For backward-comaptibility we copy the behavior + // of PyDict_GetItemWithError(). + PyErr_Clear(); + } if (module == NULL) { if (PyErr_Occurred()) return NULL; diff --git a/Modules/pyexpat.c b/Modules/pyexpat.c index d9cfa3e20859d5..4e80c31d9522a5 100644 --- a/Modules/pyexpat.c +++ b/Modules/pyexpat.c @@ -1707,7 +1707,7 @@ MODULE_INITFUNC(void) if (errors_module == NULL) { errors_module = PyModule_New(MODULE_NAME ".errors"); if (errors_module != NULL) { - PyDict_SetItem(sys_modules, errmod_name, errors_module); + PyObject_SetItem(sys_modules, errmod_name, errors_module); /* gives away the reference to errors_module */ PyModule_AddObject(m, "errors", errors_module); } @@ -1717,7 +1717,7 @@ MODULE_INITFUNC(void) if (model_module == NULL) { model_module = PyModule_New(MODULE_NAME ".model"); if (model_module != NULL) { - PyDict_SetItem(sys_modules, modelmod_name, model_module); + PyObject_SetItem(sys_modules, modelmod_name, model_module); /* gives away the reference to model_module */ PyModule_AddObject(m, "model", model_module); } diff --git a/Objects/typeobject.c b/Objects/typeobject.c index cdde8a87c00397..22a0be677a1f17 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -3913,7 +3913,15 @@ import_copyreg(void) this broke when multiple embedded interpreters were in use (see issue #17408 and #19088). */ PyObject *modules = PyImport_GetModuleDict(); - copyreg_module = PyDict_GetItemWithError(modules, copyreg_str); + if (PyDict_Check(modules)) { + copyreg_module = PyDict_GetItemWithError(modules, copyreg_str); + } else { + copyreg_module = PyObject_GetItem(modules, copyreg_str); + if (PyErr_Occurred() && PyErr_ExceptionMatches(PyExc_KeyError)) + // For backward-comaptibility we copy the behavior + // of PyDict_GetItemWithError(). + PyErr_Clear(); + } if (copyreg_module != NULL) { Py_INCREF(copyreg_module); return copyreg_module; diff --git a/Python/import.c b/Python/import.c index a080942185cc98..a46174c667c3a3 100644 --- a/Python/import.c +++ b/Python/import.c @@ -334,21 +334,39 @@ PyObject * _PyImport_GetModule(PyObject *name) { PyObject *modules = PyImport_GetModuleDict(); - return PyDict_GetItem(modules, name); + if (PyDict_Check(modules)) + return PyDict_GetItem(modules, name); + + PyObject *mod = PyObject_GetItem(modules, name); + if (mod == NULL && PyErr_Occurred()) + // For backward-comaptibility we copy the behavior + // of PyDict_GetItem(). + PyErr_Clear(); + return mod; } PyObject * _PyImport_GetModuleString(const char *name) { PyObject *modules = PyImport_GetModuleDict(); - return PyDict_GetItemString(modules, name); + if (PyDict_Check(modules)) + return PyDict_GetItemString(modules, name); + + PyObject *mod = PyMapping_GetItemString(modules, name); + if (mod == NULL && PyErr_Occurred()) + // For backward-comaptibility we copy the behavior + // of PyDict_GetItemString(). + PyErr_Clear(); + return mod; } PyObject * -_PyImport_GetModuleId(struct _Py_Identifier *name) +_PyImport_GetModuleId(struct _Py_Identifier *nameid) { - PyObject *modules = PyImport_GetModuleDict(); - return _PyDict_GetItemId(modules, name); + PyObject *name = _PyUnicode_FromId(nameid); /* borrowed */ + if (name == NULL) + return NULL; + return _PyImport_GetModule(name); } @@ -440,7 +458,7 @@ PyImport_Cleanup(void) if (Py_VerboseFlag && PyUnicode_Check(key)) PySys_FormatStderr("# cleanup[2] removing %U\n", key); STORE_MODULE_WEAKREF(key, value); - PyDict_SetItem(modules, key, Py_None); + PyObject_SetItem(modules, key, Py_None); } } @@ -592,10 +610,10 @@ _PyImport_FixupExtensionObjectEx(PyObject *mod, PyObject *name, PyErr_BadInternalCall(); return -1; } - if (PyDict_SetItem(modules, name, mod) < 0) + if (PyObject_SetItem(modules, name, mod) < 0) return -1; if (_PyState_AddModule(mod, def) < 0) { - PyDict_DelItem(modules, name); + PyMapping_DelItem(modules, name); return -1; } if (def->m_size == -1) { @@ -676,14 +694,14 @@ _PyImport_FindExtensionObjectEx(PyObject *name, PyObject *filename, mod = def->m_base.m_init(); if (mod == NULL) return NULL; - if (PyDict_SetItem(modules, name, mod) == -1) { + if (PyObject_SetItem(modules, name, mod) == -1) { Py_DECREF(mod); return NULL; } Py_DECREF(mod); } if (_PyState_AddModule(mod, def) < 0) { - PyDict_DelItem(modules, name); + PyMapping_DelItem(modules, name); Py_DECREF(mod); return NULL; } @@ -724,8 +742,16 @@ _PyImport_AddModuleObject(PyObject *name, PyObject *modules) { PyObject *m; - if ((m = PyDict_GetItemWithError(modules, name)) != NULL && - PyModule_Check(m)) { + if (PyDict_Check(modules)) + m = PyDict_GetItemWithError(modules, name); + else { + m = PyObject_GetItem(modules, name); + if (PyErr_Occurred() && PyErr_ExceptionMatches(PyExc_KeyError)) + // For backward-comaptibility we copy the behavior + // of PyDict_GetItemWithError(). + PyErr_Clear(); + } + if (m != NULL && PyModule_Check(m)) { return m; } if (PyErr_Occurred()) { @@ -734,7 +760,7 @@ _PyImport_AddModuleObject(PyObject *name, PyObject *modules) m = PyModule_NewObject(name); if (m == NULL) return NULL; - if (PyDict_SetItem(modules, name, m) != 0) { + if (PyObject_SetItem(modules, name, m) != 0) { Py_DECREF(m); return NULL; } @@ -761,9 +787,9 @@ static void remove_module(PyObject *name) { PyObject *modules = PyImport_GetModuleDict(); - if (PyDict_GetItem(modules, name) == NULL) + if (!PyMapping_HasKey(modules, name)) return; - if (PyDict_DelItem(modules, name) < 0) + if (PyMapping_DelItem(modules, name) < 0) Py_FatalError("import: deleting existing key in" "sys.modules failed"); } diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 273f62ab0e3d42..092615b2c46275 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -297,7 +297,7 @@ initimport(PyInterpreterState *interp, PyObject *sysmod) if (Py_VerboseFlag) { PySys_FormatStderr("import sys # builtin\n"); } - if (PyDict_SetItemString(sys_modules, "_imp", impmod) < 0) { + if (PyMapping_SetItemString(sys_modules, "_imp", impmod) < 0) { Py_FatalError("Py_Initialize: can't save _imp to sys.modules"); } From 58d9be77a0ae6ab7d80ac740158a17db86f70477 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Thu, 18 May 2017 13:50:48 -0700 Subject: [PATCH 17/35] Add _PyImport_SetModule*. --- Include/import.h | 4 ++++ Modules/pyexpat.c | 10 ++-------- Python/import.c | 12 ++++++++++++ Python/pylifecycle.c | 7 +------ 4 files changed, 19 insertions(+), 14 deletions(-) diff --git a/Include/import.h b/Include/import.h index ecfc29de6b5dc8..bd6acc15cd4147 100644 --- a/Include/import.h +++ b/Include/import.h @@ -40,9 +40,13 @@ PyAPI_FUNC(PyObject *) PyImport_ExecCodeModuleObject( PyAPI_FUNC(PyObject *) PyImport_GetModuleDict(void); #ifndef Py_LIMITED_API PyAPI_FUNC(int) _PyImport_IsInitialized(PyInterpreterState *); +#endif +#ifndef Py_LIMITED_API PyAPI_FUNC(PyObject *) _PyImport_GetModule(PyObject *name); PyAPI_FUNC(PyObject *) _PyImport_GetModuleString(const char *name); PyAPI_FUNC(PyObject *) _PyImport_GetModuleId(struct _Py_Identifier *name); +PyAPI_FUNC(int) _PyImport_SetModule(PyObject *name, PyObject *module); +PyAPI_FUNC(int) _PyImport_SetModuleString(const char *name, PyObject* module); #endif #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x03030000 PyAPI_FUNC(PyObject *) PyImport_AddModuleObject( diff --git a/Modules/pyexpat.c b/Modules/pyexpat.c index 4e80c31d9522a5..c8a01d4e088ece 100644 --- a/Modules/pyexpat.c +++ b/Modules/pyexpat.c @@ -1643,7 +1643,6 @@ MODULE_INITFUNC(void) PyObject *errors_module; PyObject *modelmod_name; PyObject *model_module; - PyObject *sys_modules; PyObject *tmpnum, *tmpstr; PyObject *codes_dict; PyObject *rev_codes_dict; @@ -1693,11 +1692,6 @@ MODULE_INITFUNC(void) */ PyModule_AddStringConstant(m, "native_encoding", "UTF-8"); - sys_modules = PySys_GetObject("modules"); - if (sys_modules == NULL) { - Py_DECREF(m); - return NULL; - } d = PyModule_GetDict(m); if (d == NULL) { Py_DECREF(m); @@ -1707,7 +1701,7 @@ MODULE_INITFUNC(void) if (errors_module == NULL) { errors_module = PyModule_New(MODULE_NAME ".errors"); if (errors_module != NULL) { - PyObject_SetItem(sys_modules, errmod_name, errors_module); + _PyImport_SetModule(errmod_name, errors_module); /* gives away the reference to errors_module */ PyModule_AddObject(m, "errors", errors_module); } @@ -1717,7 +1711,7 @@ MODULE_INITFUNC(void) if (model_module == NULL) { model_module = PyModule_New(MODULE_NAME ".model"); if (model_module != NULL) { - PyObject_SetItem(sys_modules, modelmod_name, model_module); + _PyImport_SetModule(modelmod_name, model_module); /* gives away the reference to model_module */ PyModule_AddObject(m, "model", model_module); } diff --git a/Python/import.c b/Python/import.c index a46174c667c3a3..01080d635423f8 100644 --- a/Python/import.c +++ b/Python/import.c @@ -369,6 +369,18 @@ _PyImport_GetModuleId(struct _Py_Identifier *nameid) return _PyImport_GetModule(name); } +int +_PyImport_SetModule(PyObject *name, PyObject *m) { + PyObject *modules = PyImport_GetModuleDict(); + return PyObject_SetItem(modules, name, m); +} + +int +_PyImport_SetModuleString(const char *name, PyObject *m) { + PyObject *modules = PyImport_GetModuleDict(); + return PyMapping_SetItemString(modules, name, m); +} + /* List of names to clear in sys */ static const char * const sys_deletes[] = { diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 092615b2c46275..05ac297109fc55 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -262,7 +262,6 @@ initimport(PyInterpreterState *interp, PyObject *sysmod) { PyObject *importlib; PyObject *impmod; - PyObject *sys_modules; PyObject *value; /* Import _importlib through its frozen version, _frozen_importlib. */ @@ -293,11 +292,7 @@ initimport(PyInterpreterState *interp, PyObject *sysmod) else if (Py_VerboseFlag) { PySys_FormatStderr("import _imp # builtin\n"); } - sys_modules = PyImport_GetModuleDict(); - if (Py_VerboseFlag) { - PySys_FormatStderr("import sys # builtin\n"); - } - if (PyMapping_SetItemString(sys_modules, "_imp", impmod) < 0) { + if (_PyImport_SetModuleString("_imp", impmod) < 0) { Py_FatalError("Py_Initialize: can't save _imp to sys.modules"); } From e3b0f0c3741e4a7ae410f94d48935bf8d20ec562 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Thu, 18 May 2017 14:22:21 -0700 Subject: [PATCH 18/35] Add PyImport_GetModule(). --- Doc/c-api/import.rst | 5 +++++ Include/import.h | 2 ++ Modules/_pickle.c | 19 +------------------ Objects/typeobject.c | 11 +---------- Python/import.c | 45 +++++++++++++++++++++++++++++++++----------- 5 files changed, 43 insertions(+), 39 deletions(-) diff --git a/Doc/c-api/import.rst b/Doc/c-api/import.rst index 7c16ece0586610..a79b4e6dc2dc64 100644 --- a/Doc/c-api/import.rst +++ b/Doc/c-api/import.rst @@ -204,6 +204,11 @@ Importing Modules Return the dictionary used for the module administration (a.k.a. ``sys.modules``). Note that this is a per-interpreter variable. +.. c:function:: PyObject* PyImport_GetModule(PyObject *name) + + Return the already imported module with the given name. + + .. versionadded:: 3.7 .. c:function:: PyObject* PyImport_GetImporter(PyObject *path) diff --git a/Include/import.h b/Include/import.h index bd6acc15cd4147..d1ce030d29ebeb 100644 --- a/Include/import.h +++ b/Include/import.h @@ -41,8 +41,10 @@ PyAPI_FUNC(PyObject *) PyImport_GetModuleDict(void); #ifndef Py_LIMITED_API PyAPI_FUNC(int) _PyImport_IsInitialized(PyInterpreterState *); #endif +PyAPI_FUNC(PyObject *) PyImport_GetModule(PyObject *name); #ifndef Py_LIMITED_API PyAPI_FUNC(PyObject *) _PyImport_GetModule(PyObject *name); +PyAPI_FUNC(PyObject *) _PyImport_GetModuleWithError(PyObject *name); PyAPI_FUNC(PyObject *) _PyImport_GetModuleString(const char *name); PyAPI_FUNC(PyObject *) _PyImport_GetModuleId(struct _Py_Identifier *name); PyAPI_FUNC(int) _PyImport_SetModule(PyObject *name, PyObject *module); diff --git a/Modules/_pickle.c b/Modules/_pickle.c index 76f720c9eec515..d25dadf97b8465 100644 --- a/Modules/_pickle.c +++ b/Modules/_pickle.c @@ -6418,9 +6418,7 @@ _pickle_Unpickler_find_class_impl(UnpicklerObject *self, /*[clinic end generated code: output=becc08d7f9ed41e3 input=e2e6a865de093ef4]*/ { PyObject *global; - PyObject *modules_dict; PyObject *module; - _Py_IDENTIFIER(modules); /* Try to map the old names used in Python 2.x to the new ones used in Python 3.x. We do this only with old pickle protocols and when the @@ -6477,22 +6475,7 @@ _pickle_Unpickler_find_class_impl(UnpicklerObject *self, } } - modules_dict = _PySys_GetObjectId(&PyId_modules); - if (modules_dict == NULL) { - PyErr_SetString(PyExc_RuntimeError, "unable to get sys.modules"); - return NULL; - } - - module = PyObject_GetItem(modules_dict, module_name); - if (PyDict_Check(modules_dict)) { - module = PyDict_GetItemWithError(modules_dict, module_name); - } else { - module = PyObject_GetItem(modules_dict, module_name); - if (PyErr_Occurred() && PyErr_ExceptionMatches(PyExc_KeyError)) - // For backward-comaptibility we copy the behavior - // of PyDict_GetItemWithError(). - PyErr_Clear(); - } + module = PyImport_GetModule(module_name); if (module == NULL) { if (PyErr_Occurred()) return NULL; diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 22a0be677a1f17..1d963aae3f8dac 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -3912,16 +3912,7 @@ import_copyreg(void) by storing a reference to the cached module in a static variable, but this broke when multiple embedded interpreters were in use (see issue #17408 and #19088). */ - PyObject *modules = PyImport_GetModuleDict(); - if (PyDict_Check(modules)) { - copyreg_module = PyDict_GetItemWithError(modules, copyreg_str); - } else { - copyreg_module = PyObject_GetItem(modules, copyreg_str); - if (PyErr_Occurred() && PyErr_ExceptionMatches(PyExc_KeyError)) - // For backward-comaptibility we copy the behavior - // of PyDict_GetItemWithError(). - PyErr_Clear(); - } + copyreg_module = _PyImport_GetModuleWithError(copyreg_str); if (copyreg_module != NULL) { Py_INCREF(copyreg_module); return copyreg_module; diff --git a/Python/import.c b/Python/import.c index 01080d635423f8..b778395d321673 100644 --- a/Python/import.c +++ b/Python/import.c @@ -345,6 +345,21 @@ _PyImport_GetModule(PyObject *name) return mod; } +PyObject * +_PyImport_GetModuleWithError(PyObject *name) +{ + PyObject *modules = PyImport_GetModuleDict(); + if (PyDict_Check(modules)) + return PyDict_GetItemWithError(modules, name); + + PyObject *mod = PyObject_GetItem(modules, name); + if (mod == NULL && !PyMapping_HasKey(modules, name)) + // For backward-comaptibility we copy the behavior + // of PyDict_GetItemWithError(). + PyErr_Clear(); + return mod; +} + PyObject * _PyImport_GetModuleString(const char *name) { @@ -381,6 +396,24 @@ _PyImport_SetModuleString(const char *name, PyObject *m) { return PyMapping_SetItemString(modules, name, m); } +PyObject * +PyImport_GetModule(PyObject *name) +{ + _Py_IDENTIFIER(modules); + PyObject *modules = _PySys_GetObjectId(&PyId_modules); + if (modules == NULL) { + PyErr_SetString(PyExc_RuntimeError, "unable to get sys.modules"); + return NULL; + } + if (PyDict_Check(modules)) + return PyDict_GetItemWithError(modules, name); + + PyObject *m = PyObject_GetItem(modules, name); + if (m == NULL && !PyMapping_HasKey(modules, name)) + PyErr_Clear(); + return m; +} + /* List of names to clear in sys */ static const char * const sys_deletes[] = { @@ -752,17 +785,7 @@ PyImport_AddModuleObject(PyObject *name) PyObject * _PyImport_AddModuleObject(PyObject *name, PyObject *modules) { - PyObject *m; - - if (PyDict_Check(modules)) - m = PyDict_GetItemWithError(modules, name); - else { - m = PyObject_GetItem(modules, name); - if (PyErr_Occurred() && PyErr_ExceptionMatches(PyExc_KeyError)) - // For backward-comaptibility we copy the behavior - // of PyDict_GetItemWithError(). - PyErr_Clear(); - } + PyObject *m = _PyImport_GetModuleWithError(name); if (m != NULL && PyModule_Check(m)) { return m; } From d9c6b79364d96237731f57e911204654072a9eaf Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Thu, 18 May 2017 15:12:56 -0700 Subject: [PATCH 19/35] Fix ref counts. --- Python/import.c | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/Python/import.c b/Python/import.c index b778395d321673..5ae1d16b06d33c 100644 --- a/Python/import.c +++ b/Python/import.c @@ -338,10 +338,13 @@ _PyImport_GetModule(PyObject *name) return PyDict_GetItem(modules, name); PyObject *mod = PyObject_GetItem(modules, name); - if (mod == NULL && PyErr_Occurred()) - // For backward-comaptibility we copy the behavior - // of PyDict_GetItem(). - PyErr_Clear(); + // For backward-comaptibility we copy the behavior of PyDict_GetItem(). + if (mod == NULL) { + if (PyErr_Occurred()) + PyErr_Clear(); + } else { + Py_DECREF(mod); + } return mod; } @@ -353,9 +356,9 @@ _PyImport_GetModuleWithError(PyObject *name) return PyDict_GetItemWithError(modules, name); PyObject *mod = PyObject_GetItem(modules, name); + // For backward-comaptibility we copy the behavior + // of PyDict_GetItemWithError(). if (mod == NULL && !PyMapping_HasKey(modules, name)) - // For backward-comaptibility we copy the behavior - // of PyDict_GetItemWithError(). PyErr_Clear(); return mod; } @@ -368,10 +371,14 @@ _PyImport_GetModuleString(const char *name) return PyDict_GetItemString(modules, name); PyObject *mod = PyMapping_GetItemString(modules, name); - if (mod == NULL && PyErr_Occurred()) - // For backward-comaptibility we copy the behavior - // of PyDict_GetItemString(). - PyErr_Clear(); + // For backward-comaptibility we copy the behavior + // of PyDict_GetItemString(). + if (mod == NULL) { + if (PyErr_Occurred()) + PyErr_Clear(); + } else { + Py_DECREF(mod); + } return mod; } From e3565f90a967b7749f2e658339e5847afcc01c65 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Thu, 18 May 2017 16:44:41 -0700 Subject: [PATCH 20/35] Look up "new" modules in the given modules dict. --- Python/import.c | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/Python/import.c b/Python/import.c index 5ae1d16b06d33c..63db3a44eeb101 100644 --- a/Python/import.c +++ b/Python/import.c @@ -792,7 +792,16 @@ PyImport_AddModuleObject(PyObject *name) PyObject * _PyImport_AddModuleObject(PyObject *name, PyObject *modules) { - PyObject *m = _PyImport_GetModuleWithError(name); + PyObject *m; + if (PyDict_Check(modules)) { + m = PyDict_GetItemWithError(modules, name); + } else { + m = PyObject_GetItem(modules, name); + // For backward-comaptibility we copy the behavior + // of PyDict_GetItemWithError(). + if (m == NULL && !PyMapping_HasKey(modules, name)) + PyErr_Clear(); + } if (m != NULL && PyModule_Check(m)) { return m; } From 99463417fbac8d0cc14f665220b604fa13a82fe2 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Fri, 19 May 2017 18:28:43 -0700 Subject: [PATCH 21/35] Fix error checking. --- Python/import.c | 37 ++++++++++++++++--------------------- 1 file changed, 16 insertions(+), 21 deletions(-) diff --git a/Python/import.c b/Python/import.c index 63db3a44eeb101..bf8b1022803a56 100644 --- a/Python/import.c +++ b/Python/import.c @@ -339,12 +339,9 @@ _PyImport_GetModule(PyObject *name) PyObject *mod = PyObject_GetItem(modules, name); // For backward-comaptibility we copy the behavior of PyDict_GetItem(). - if (mod == NULL) { - if (PyErr_Occurred()) - PyErr_Clear(); - } else { - Py_DECREF(mod); - } + if (PyErr_Occurred()) + PyErr_Clear(); + Py_XDECREF(mod); return mod; } @@ -358,7 +355,7 @@ _PyImport_GetModuleWithError(PyObject *name) PyObject *mod = PyObject_GetItem(modules, name); // For backward-comaptibility we copy the behavior // of PyDict_GetItemWithError(). - if (mod == NULL && !PyMapping_HasKey(modules, name)) + if (PyErr_Occurred() && !PyMapping_HasKey(modules, name)) PyErr_Clear(); return mod; } @@ -373,12 +370,9 @@ _PyImport_GetModuleString(const char *name) PyObject *mod = PyMapping_GetItemString(modules, name); // For backward-comaptibility we copy the behavior // of PyDict_GetItemString(). - if (mod == NULL) { - if (PyErr_Occurred()) - PyErr_Clear(); - } else { - Py_DECREF(mod); - } + if (PyErr_Occurred()) + PyErr_Clear(); + Py_XDECREF(mod); return mod; } @@ -416,7 +410,7 @@ PyImport_GetModule(PyObject *name) return PyDict_GetItemWithError(modules, name); PyObject *m = PyObject_GetItem(modules, name); - if (m == NULL && !PyMapping_HasKey(modules, name)) + if (PyErr_Occurred() && !PyMapping_HasKey(modules, name)) PyErr_Clear(); return m; } @@ -799,15 +793,15 @@ _PyImport_AddModuleObject(PyObject *name, PyObject *modules) m = PyObject_GetItem(modules, name); // For backward-comaptibility we copy the behavior // of PyDict_GetItemWithError(). - if (m == NULL && !PyMapping_HasKey(modules, name)) + if (PyErr_Occurred() && !PyMapping_HasKey(modules, name)) PyErr_Clear(); } - if (m != NULL && PyModule_Check(m)) { - return m; - } if (PyErr_Occurred()) { return NULL; } + if (m != NULL && PyModule_Check(m)) { + return m; + } m = PyModule_NewObject(name); if (m == NULL) return NULL; @@ -838,11 +832,12 @@ static void remove_module(PyObject *name) { PyObject *modules = PyImport_GetModuleDict(); - if (!PyMapping_HasKey(modules, name)) - return; - if (PyMapping_DelItem(modules, name) < 0) + if (PyMapping_DelItem(modules, name) < 0) { + if (!PyMapping_HasKey(modules, name)) + return; Py_FatalError("import: deleting existing key in" "sys.modules failed"); + } } From 08e556e701a22e297abe3ec17e4434f14e37be68 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 22 May 2017 17:45:31 -0700 Subject: [PATCH 22/35] Use PyImport_GetModuleDict() in PyImport_GetModule(). --- Python/import.c | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/Python/import.c b/Python/import.c index bf8b1022803a56..00c8541492e858 100644 --- a/Python/import.c +++ b/Python/import.c @@ -400,18 +400,21 @@ _PyImport_SetModuleString(const char *name, PyObject *m) { PyObject * PyImport_GetModule(PyObject *name) { - _Py_IDENTIFIER(modules); - PyObject *modules = _PySys_GetObjectId(&PyId_modules); + PyObject *m; + PyObject *modules = PyImport_GetModuleDict(); if (modules == NULL) { PyErr_SetString(PyExc_RuntimeError, "unable to get sys.modules"); return NULL; } - if (PyDict_Check(modules)) - return PyDict_GetItemWithError(modules, name); - - PyObject *m = PyObject_GetItem(modules, name); - if (PyErr_Occurred() && !PyMapping_HasKey(modules, name)) - PyErr_Clear(); + Py_INCREF(modules); + if (PyDict_Check(modules)) { + m = PyDict_GetItemWithError(modules, name); + } else { + m = PyObject_GetItem(modules, name); + if (PyErr_Occurred() && !PyMapping_HasKey(modules, name)) + PyErr_Clear(); + } + Py_DECREF(modules); return m; } From f043e4775a098a39bdaa838626a86ac0e1cfddca Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Thu, 25 May 2017 11:17:42 -0700 Subject: [PATCH 23/35] Decref the module when done. --- Modules/_pickle.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/_pickle.c b/Modules/_pickle.c index d25dadf97b8465..2a3e73988d4c6e 100644 --- a/Modules/_pickle.c +++ b/Modules/_pickle.c @@ -6483,11 +6483,11 @@ _pickle_Unpickler_find_class_impl(UnpicklerObject *self, if (module == NULL) return NULL; global = getattribute(module, global_name, self->proto >= 4); - Py_DECREF(module); } else { global = getattribute(module, global_name, self->proto >= 4); } + Py_DECREF(module); return global; } From b2edd256b10a3c842661d69e546615231921aaec Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Thu, 25 May 2017 11:26:42 -0700 Subject: [PATCH 24/35] Add a missing incref. --- Python/import.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Python/import.c b/Python/import.c index 00c8541492e858..4e960da62427a2 100644 --- a/Python/import.c +++ b/Python/import.c @@ -408,7 +408,8 @@ PyImport_GetModule(PyObject *name) } Py_INCREF(modules); if (PyDict_Check(modules)) { - m = PyDict_GetItemWithError(modules, name); + m = PyDict_GetItemWithError(modules, name); /* borrowed */ + Py_XINCREF(m); } else { m = PyObject_GetItem(modules, name); if (PyErr_Occurred() && !PyMapping_HasKey(modules, name)) From 0eced28d91d12e21b39d6280751ecd5aebe3c567 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 4 Sep 2017 10:36:05 -0700 Subject: [PATCH 25/35] Fix post-rebase. --- Python/import.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Python/import.c b/Python/import.c index 4e960da62427a2..5bc1e50825a86a 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1894,7 +1894,7 @@ PyImport_Import(PyObject *module_name) Py_DECREF(r); r = _PyImport_GetModule(module_name); - if (r != NULL) + if (r != NULL) { Py_INCREF(r); } else if (!PyErr_Occurred()) { From a85a11f9878d2acce80eaf682f4fc55fd021ba58 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 4 Sep 2017 10:46:53 -0700 Subject: [PATCH 26/35] Add a Misc/NEWS entry. --- .../2017-09-04-10-46-09.bpo-28411.IU9rQL.rst | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2017-09-04-10-46-09.bpo-28411.IU9rQL.rst diff --git a/Misc/NEWS.d/next/Core and Builtins/2017-09-04-10-46-09.bpo-28411.IU9rQL.rst b/Misc/NEWS.d/next/Core and Builtins/2017-09-04-10-46-09.bpo-28411.IU9rQL.rst new file mode 100644 index 00000000000000..2417f781276c75 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2017-09-04-10-46-09.bpo-28411.IU9rQL.rst @@ -0,0 +1,4 @@ +``PyInterpreterState`` has a "modules" field that is copied into +``sys.modules`` during interpreter startup. This causes problems if a +program replaces ``sys.modules`` with something else. To solve this we +eliminate ``PyInterpreterState.modules``. From 9ba4c43b7b3a818c74a0201293d6605ffac34b06 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 4 Sep 2017 11:34:31 -0700 Subject: [PATCH 27/35] Make the docs for PyImport_GetModule() more clear. --- Doc/c-api/import.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Doc/c-api/import.rst b/Doc/c-api/import.rst index a79b4e6dc2dc64..8cdc256e7c9e0e 100644 --- a/Doc/c-api/import.rst +++ b/Doc/c-api/import.rst @@ -206,7 +206,9 @@ Importing Modules .. c:function:: PyObject* PyImport_GetModule(PyObject *name) - Return the already imported module with the given name. + Return the already imported module with the given name. If the + module has not been imported yet then returns NULL but does not set + an error. Returns NULL and sets an error if the lookup failed. .. versionadded:: 3.7 From f837a610334d942159482667baf9971061abbb28 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 4 Sep 2017 11:39:41 -0700 Subject: [PATCH 28/35] Fix style (bracket placement). --- Python/import.c | 39 ++++++++++++++++++++++++++------------- 1 file changed, 26 insertions(+), 13 deletions(-) diff --git a/Python/import.c b/Python/import.c index 5bc1e50825a86a..a8f2118cda86f2 100644 --- a/Python/import.c +++ b/Python/import.c @@ -304,13 +304,15 @@ PyObject * PyImport_GetModuleDict(void) { PyObject *sysdict = PyThreadState_GET()->interp->sysdict; - if (sysdict == NULL) + if (sysdict == NULL) { Py_FatalError("PyImport_GetModuleDict: no sys module!"); + } _Py_IDENTIFIER(modules); PyObject *modules = _PyDict_GetItemId(sysdict, &PyId_modules); - if (modules == NULL) + if (modules == NULL) { Py_FatalError("lost sys.modules"); + } return modules; } @@ -334,13 +336,15 @@ PyObject * _PyImport_GetModule(PyObject *name) { PyObject *modules = PyImport_GetModuleDict(); - if (PyDict_Check(modules)) + if (PyDict_Check(modules)) { return PyDict_GetItem(modules, name); + } PyObject *mod = PyObject_GetItem(modules, name); // For backward-comaptibility we copy the behavior of PyDict_GetItem(). - if (PyErr_Occurred()) + if (PyErr_Occurred()) { PyErr_Clear(); + } Py_XDECREF(mod); return mod; } @@ -349,14 +353,16 @@ PyObject * _PyImport_GetModuleWithError(PyObject *name) { PyObject *modules = PyImport_GetModuleDict(); - if (PyDict_Check(modules)) + if (PyDict_Check(modules)) { return PyDict_GetItemWithError(modules, name); + } PyObject *mod = PyObject_GetItem(modules, name); // For backward-comaptibility we copy the behavior // of PyDict_GetItemWithError(). - if (PyErr_Occurred() && !PyMapping_HasKey(modules, name)) + if (PyErr_Occurred() && !PyMapping_HasKey(modules, name)) { PyErr_Clear(); + } return mod; } @@ -364,14 +370,16 @@ PyObject * _PyImport_GetModuleString(const char *name) { PyObject *modules = PyImport_GetModuleDict(); - if (PyDict_Check(modules)) + if (PyDict_Check(modules)) { return PyDict_GetItemString(modules, name); + } PyObject *mod = PyMapping_GetItemString(modules, name); // For backward-comaptibility we copy the behavior // of PyDict_GetItemString(). - if (PyErr_Occurred()) + if (PyErr_Occurred()) { PyErr_Clear(); + } Py_XDECREF(mod); return mod; } @@ -380,19 +388,22 @@ PyObject * _PyImport_GetModuleId(struct _Py_Identifier *nameid) { PyObject *name = _PyUnicode_FromId(nameid); /* borrowed */ - if (name == NULL) + if (name == NULL) { return NULL; + } return _PyImport_GetModule(name); } int -_PyImport_SetModule(PyObject *name, PyObject *m) { +_PyImport_SetModule(PyObject *name, PyObject *m) +{ PyObject *modules = PyImport_GetModuleDict(); return PyObject_SetItem(modules, name, m); } int -_PyImport_SetModuleString(const char *name, PyObject *m) { +_PyImport_SetModuleString(const char *name, PyObject *m) +{ PyObject *modules = PyImport_GetModuleDict(); return PyMapping_SetItemString(modules, name, m); } @@ -410,7 +421,8 @@ PyImport_GetModule(PyObject *name) if (PyDict_Check(modules)) { m = PyDict_GetItemWithError(modules, name); /* borrowed */ Py_XINCREF(m); - } else { + } + else { m = PyObject_GetItem(modules, name); if (PyErr_Occurred() && !PyMapping_HasKey(modules, name)) PyErr_Clear(); @@ -793,7 +805,8 @@ _PyImport_AddModuleObject(PyObject *name, PyObject *modules) PyObject *m; if (PyDict_Check(modules)) { m = PyDict_GetItemWithError(modules, name); - } else { + } + else { m = PyObject_GetItem(modules, name); // For backward-comaptibility we copy the behavior // of PyDict_GetItemWithError(). From 101aa310cb07a6add9cecc20698b06871d2f2167 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 4 Sep 2017 12:13:41 -0700 Subject: [PATCH 29/35] Drop _PyImport_GetModuleString(). --- Include/import.h | 1 - Python/import.c | 21 ++------------------- Python/pylifecycle.c | 3 ++- 3 files changed, 4 insertions(+), 21 deletions(-) diff --git a/Include/import.h b/Include/import.h index d1ce030d29ebeb..27fc0014937dd1 100644 --- a/Include/import.h +++ b/Include/import.h @@ -45,7 +45,6 @@ PyAPI_FUNC(PyObject *) PyImport_GetModule(PyObject *name); #ifndef Py_LIMITED_API PyAPI_FUNC(PyObject *) _PyImport_GetModule(PyObject *name); PyAPI_FUNC(PyObject *) _PyImport_GetModuleWithError(PyObject *name); -PyAPI_FUNC(PyObject *) _PyImport_GetModuleString(const char *name); PyAPI_FUNC(PyObject *) _PyImport_GetModuleId(struct _Py_Identifier *name); PyAPI_FUNC(int) _PyImport_SetModule(PyObject *name, PyObject *module); PyAPI_FUNC(int) _PyImport_SetModuleString(const char *name, PyObject* module); diff --git a/Python/import.c b/Python/import.c index a8f2118cda86f2..0d689903fed6eb 100644 --- a/Python/import.c +++ b/Python/import.c @@ -366,24 +366,6 @@ _PyImport_GetModuleWithError(PyObject *name) return mod; } -PyObject * -_PyImport_GetModuleString(const char *name) -{ - PyObject *modules = PyImport_GetModuleDict(); - if (PyDict_Check(modules)) { - return PyDict_GetItemString(modules, name); - } - - PyObject *mod = PyMapping_GetItemString(modules, name); - // For backward-comaptibility we copy the behavior - // of PyDict_GetItemString(). - if (PyErr_Occurred()) { - PyErr_Clear(); - } - Py_XDECREF(mod); - return mod; -} - PyObject * _PyImport_GetModuleId(struct _Py_Identifier *nameid) { @@ -1815,9 +1797,10 @@ PyImport_ImportModuleLevel(const char *name, PyObject *globals, PyObject *locals PyObject * PyImport_ReloadModule(PyObject *m) { + _Py_IDENTIFIER(imp); _Py_IDENTIFIER(reload); PyObject *reloaded_module = NULL; - PyObject *imp = _PyImport_GetModuleString("imp"); + PyObject *imp = _PyImport_GetModuleId(&PyId_imp); if (imp == NULL) { imp = PyImport_ImportModule("imp"); if (imp == NULL) { diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 05ac297109fc55..662405bdeb312e 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -41,6 +41,7 @@ _Py_IDENTIFIER(name); _Py_IDENTIFIER(stdin); _Py_IDENTIFIER(stdout); _Py_IDENTIFIER(stderr); +_Py_IDENTIFIER(threading); #ifdef __cplusplus extern "C" { @@ -1880,7 +1881,7 @@ wait_for_thread_shutdown(void) #ifdef WITH_THREAD _Py_IDENTIFIER(_shutdown); PyObject *result; - PyObject *threading = _PyImport_GetModuleString("threading"); + PyObject *threading = _PyImport_GetModuleId(&PyId_threading); if (threading == NULL) { /* threading not imported */ PyErr_Clear(); From 8f8a7a2e0292ab59db0bc60bee59439739df39a3 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 4 Sep 2017 12:18:40 -0700 Subject: [PATCH 30/35] Use PyDict_CheckExact() for sys.modules in fast case. --- Python/import.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Python/import.c b/Python/import.c index 0d689903fed6eb..8b1d8580926290 100644 --- a/Python/import.c +++ b/Python/import.c @@ -336,7 +336,7 @@ PyObject * _PyImport_GetModule(PyObject *name) { PyObject *modules = PyImport_GetModuleDict(); - if (PyDict_Check(modules)) { + if (PyDict_CheckExact(modules)) { return PyDict_GetItem(modules, name); } @@ -353,7 +353,7 @@ PyObject * _PyImport_GetModuleWithError(PyObject *name) { PyObject *modules = PyImport_GetModuleDict(); - if (PyDict_Check(modules)) { + if (PyDict_CheckExact(modules)) { return PyDict_GetItemWithError(modules, name); } @@ -400,7 +400,7 @@ PyImport_GetModule(PyObject *name) return NULL; } Py_INCREF(modules); - if (PyDict_Check(modules)) { + if (PyDict_CheckExact(modules)) { m = PyDict_GetItemWithError(modules, name); /* borrowed */ Py_XINCREF(m); } @@ -785,7 +785,7 @@ PyObject * _PyImport_AddModuleObject(PyObject *name, PyObject *modules) { PyObject *m; - if (PyDict_Check(modules)) { + if (PyDict_CheckExact(modules)) { m = PyDict_GetItemWithError(modules, name); } else { From 3a56648724439986ccce02163c13e569ce3fd34d Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 4 Sep 2017 12:48:42 -0700 Subject: [PATCH 31/35] Do not use PyMapping_HasKey() with an error set. --- Python/import.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/Python/import.c b/Python/import.c index 8b1d8580926290..226fac930a3a6d 100644 --- a/Python/import.c +++ b/Python/import.c @@ -360,7 +360,7 @@ _PyImport_GetModuleWithError(PyObject *name) PyObject *mod = PyObject_GetItem(modules, name); // For backward-comaptibility we copy the behavior // of PyDict_GetItemWithError(). - if (PyErr_Occurred() && !PyMapping_HasKey(modules, name)) { + if (PyErr_ExceptionMatches(PyExc_KeyError)) { PyErr_Clear(); } return mod; @@ -406,8 +406,9 @@ PyImport_GetModule(PyObject *name) } else { m = PyObject_GetItem(modules, name); - if (PyErr_Occurred() && !PyMapping_HasKey(modules, name)) + if (PyErr_ExceptionMatches(PyExc_KeyError)) { PyErr_Clear(); + } } Py_DECREF(modules); return m; @@ -792,8 +793,9 @@ _PyImport_AddModuleObject(PyObject *name, PyObject *modules) m = PyObject_GetItem(modules, name); // For backward-comaptibility we copy the behavior // of PyDict_GetItemWithError(). - if (PyErr_Occurred() && !PyMapping_HasKey(modules, name)) + if (PyErr_ExceptionMatches(PyExc_KeyError)) { PyErr_Clear(); + } } if (PyErr_Occurred()) { return NULL; From 619b0a0fa72085e0a26a7b6f5f22c6eb8ba70d04 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 4 Sep 2017 13:00:34 -0700 Subject: [PATCH 32/35] Revert a code order change. --- Python/import.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Python/import.c b/Python/import.c index 226fac930a3a6d..649be417e76c9b 100644 --- a/Python/import.c +++ b/Python/import.c @@ -833,9 +833,10 @@ static void remove_module(PyObject *name) { PyObject *modules = PyImport_GetModuleDict(); + if (!PyMapping_HasKey(modules, name)) { + return; + } if (PyMapping_DelItem(modules, name) < 0) { - if (!PyMapping_HasKey(modules, name)) - return; Py_FatalError("import: deleting existing key in" "sys.modules failed"); } From 57eeb5b3c5dfbde602205013e89d6dd8fe82337d Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 4 Sep 2017 13:19:52 -0700 Subject: [PATCH 33/35] Drop _PyImport_FixupExtensionObjectEx(). --- Include/import.h | 5 ++--- Python/import.c | 17 +++++++---------- Python/importdl.c | 3 ++- 3 files changed, 11 insertions(+), 14 deletions(-) diff --git a/Include/import.h b/Include/import.h index 27fc0014937dd1..4ab14d72816693 100644 --- a/Include/import.h +++ b/Include/import.h @@ -122,9 +122,8 @@ PyAPI_FUNC(int) _PyImport_FixupBuiltin( const char *name, /* UTF-8 encoded string */ PyObject *modules ); -PyAPI_FUNC(int) _PyImport_FixupExtensionObject(PyObject*, PyObject *, PyObject *); -PyAPI_FUNC(int) _PyImport_FixupExtensionObjectEx(PyObject*, PyObject *, - PyObject *, PyObject *); +PyAPI_FUNC(int) _PyImport_FixupExtensionObject(PyObject*, PyObject *, + PyObject *, PyObject *); struct _inittab { const char *name; /* ASCII encoded string */ diff --git a/Python/import.c b/Python/import.c index 649be417e76c9b..e1340d3dbbcf2f 100644 --- a/Python/import.c +++ b/Python/import.c @@ -628,14 +628,6 @@ PyImport_GetMagicTag(void) int _PyImport_FixupExtensionObject(PyObject *mod, PyObject *name, - PyObject *filename) -{ - PyObject *modules = PyImport_GetModuleDict(); - return _PyImport_FixupExtensionObjectEx(mod, name, filename, modules); -} - -int -_PyImport_FixupExtensionObjectEx(PyObject *mod, PyObject *name, PyObject *filename, PyObject *modules) { PyObject *dict, *key; @@ -693,7 +685,7 @@ _PyImport_FixupBuiltin(PyObject *mod, const char *name, PyObject *modules) nameobj = PyUnicode_InternFromString(name); if (nameobj == NULL) return -1; - res = _PyImport_FixupExtensionObjectEx(mod, nameobj, nameobj, modules); + res = _PyImport_FixupExtensionObject(mod, nameobj, nameobj, modules); Py_DECREF(nameobj); return res; } @@ -1191,6 +1183,7 @@ _imp_create_builtin(PyObject *module, PyObject *spec) return NULL; } + PyObject *modules = NULL; for (p = PyImport_Inittab; p->name != NULL; p++) { PyModuleDef *def; if (_PyUnicode_EqualToASCIIString(name, p->name)) { @@ -1216,7 +1209,11 @@ _imp_create_builtin(PyObject *module, PyObject *spec) return NULL; } def->m_base.m_init = p->initfunc; - if (_PyImport_FixupExtensionObject(mod, name, name) < 0) { + if (modules == NULL) { + modules = PyImport_GetModuleDict(); + } + if (_PyImport_FixupExtensionObject(mod, name, name, + modules) < 0) { Py_DECREF(name); return NULL; } diff --git a/Python/importdl.c b/Python/importdl.c index d8656b943336ec..32fb7e1be212ca 100644 --- a/Python/importdl.c +++ b/Python/importdl.c @@ -215,7 +215,8 @@ _PyImport_LoadDynamicModuleWithSpec(PyObject *spec, FILE *fp) else Py_INCREF(path); - if (_PyImport_FixupExtensionObject(m, name_unicode, path) < 0) + PyObject *modules = PyImport_GetModuleDict(); + if (_PyImport_FixupExtensionObject(m, name_unicode, path, modules) < 0) goto error; Py_DECREF(name_unicode); From 6544fa28a60960c8390bd0c5cbe99e7b36c7eecf Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 4 Sep 2017 14:00:40 -0700 Subject: [PATCH 34/35] Switch the code order back. --- Python/import.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Python/import.c b/Python/import.c index e1340d3dbbcf2f..542a91b9e6c71d 100644 --- a/Python/import.c +++ b/Python/import.c @@ -825,10 +825,10 @@ static void remove_module(PyObject *name) { PyObject *modules = PyImport_GetModuleDict(); - if (!PyMapping_HasKey(modules, name)) { - return; - } if (PyMapping_DelItem(modules, name) < 0) { + if (!PyMapping_HasKey(modules, name)) { + return; + } Py_FatalError("import: deleting existing key in" "sys.modules failed"); } From 8e86c53e75c17ab5b5b608e96a2cac0d5f6fa9d4 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 4 Sep 2017 16:24:21 -0700 Subject: [PATCH 35/35] Add a whatsnew entry for porting. --- Doc/whatsnew/3.7.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Doc/whatsnew/3.7.rst b/Doc/whatsnew/3.7.rst index 8547660e1881a4..1e6464a3929262 100644 --- a/Doc/whatsnew/3.7.rst +++ b/Doc/whatsnew/3.7.rst @@ -449,6 +449,9 @@ Changes in the Python API and module are affected by this change. (Contributed by INADA Naoki and Eugene Toder in :issue:`29463`.) +* ``PyInterpreterState`` no longer has a ``modules`` field. Instead use + ``sys.modules``. + * The *mode* argument of :func:`os.makedirs` no longer affects the file permission bits of newly-created intermediate-level directories. To set their file permission bits you can set the umask before invoking