From a66423e37200b33334d73a70f28f47036b1b5dbe Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 16 Apr 2024 12:38:25 -0600 Subject: [PATCH 1/9] Move the audit entry to _imp_create_dynamic_impl(). --- Python/import.c | 8 ++++++++ Python/importdl.c | 5 ----- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/Python/import.c b/Python/import.c index b040c7d5c0f7f5..38214a73f64a8c 100644 --- a/Python/import.c +++ b/Python/import.c @@ -3745,6 +3745,14 @@ _imp_create_dynamic_impl(PyObject *module, PyObject *spec, PyObject *file) goto finally; } + if (PySys_Audit("import", "OOOOO", name, path, + Py_None, Py_None, Py_None) < 0) + { + goto finally; + } + + /* Is multi-phase init or this is the first time being loaded. */ + if (file != NULL) { fp = _Py_fopen_obj(path, "r"); if (fp == NULL) { diff --git a/Python/importdl.c b/Python/importdl.c index 7dfd301d77efb4..591956d9646feb 100644 --- a/Python/importdl.c +++ b/Python/importdl.c @@ -130,11 +130,6 @@ _PyImport_LoadDynamicModuleWithSpec(PyObject *spec, FILE *fp) if (path == NULL) goto error; - if (PySys_Audit("import", "OOOOO", name_unicode, path, - Py_None, Py_None, Py_None) < 0) { - goto error; - } - #ifdef MS_WINDOWS exportfunc = _PyImport_FindSharedFuncptrWindows(hook_prefix, name_buf, path, fp); From 47bc4caeb1e71390d7846752045e073c100f33c5 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 16 Apr 2024 16:31:28 -0600 Subject: [PATCH 2/9] Always close the file handle. --- Python/import.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Python/import.c b/Python/import.c index 38214a73f64a8c..88f19b7a6bb60c 100644 --- a/Python/import.c +++ b/Python/import.c @@ -3764,10 +3764,10 @@ _imp_create_dynamic_impl(PyObject *module, PyObject *spec, PyObject *file) mod = _PyImport_LoadDynamicModuleWithSpec(spec, fp); - if (fp) - fclose(fp); - finally: + if (fp) { + fclose(fp); + } Py_DECREF(name); Py_DECREF(path); return mod; From 48b10b3ba682329cbec8d1ac795a01ae431a6274 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 16 Apr 2024 16:31:28 -0600 Subject: [PATCH 3/9] Revert the fclose() change. --- Python/import.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Python/import.c b/Python/import.c index 88f19b7a6bb60c..d858ebaa2c5bd8 100644 --- a/Python/import.c +++ b/Python/import.c @@ -3764,10 +3764,12 @@ _imp_create_dynamic_impl(PyObject *module, PyObject *spec, PyObject *file) mod = _PyImport_LoadDynamicModuleWithSpec(spec, fp); -finally: + // XXX Shouldn't this happen in the error cases too. if (fp) { fclose(fp); } + +finally: Py_DECREF(name); Py_DECREF(path); return mod; From 56614ad345fff7cd7474378ed80d4c60597ab05b Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Wed, 17 Apr 2024 11:18:45 -0600 Subject: [PATCH 4/9] Move setting __file__ to _imp.create_dynamic(). --- Python/import.c | 9 ++++++++- Python/importdl.c | 5 ----- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/Python/import.c b/Python/import.c index d858ebaa2c5bd8..7394c2b49b14d6 100644 --- a/Python/import.c +++ b/Python/import.c @@ -3759,10 +3759,17 @@ _imp_create_dynamic_impl(PyObject *module, PyObject *spec, PyObject *file) goto finally; } } - else + else { fp = NULL; + } mod = _PyImport_LoadDynamicModuleWithSpec(spec, fp); + if (mod != NULL) { + /* Remember the filename as the __file__ attribute */ + if (PyModule_AddObjectRef(mod, "__file__", path) < 0) { + PyErr_Clear(); /* Not important enough to report */ + } + } // XXX Shouldn't this happen in the error cases too. if (fp) { diff --git a/Python/importdl.c b/Python/importdl.c index 591956d9646feb..bc15113af7332d 100644 --- a/Python/importdl.c +++ b/Python/importdl.c @@ -223,11 +223,6 @@ _PyImport_LoadDynamicModuleWithSpec(PyObject *spec, FILE *fp) } def->m_base.m_init = p0; - /* Remember the filename as the __file__ attribute */ - if (PyModule_AddObjectRef(m, "__file__", path) < 0) { - PyErr_Clear(); /* Not important enough to report */ - } - PyObject *modules = PyImport_GetModuleDict(); if (_PyImport_FixupExtensionObject(m, name_unicode, path, modules) < 0) goto error; From 6b6fdbe6e7d03e2772dfc227df72d41726dac2b2 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Wed, 17 Apr 2024 12:34:16 -0600 Subject: [PATCH 5/9] Add a note about _PyImport_FixupBuiltin(). --- Include/internal/pycore_import.h | 1 + 1 file changed, 1 insertion(+) diff --git a/Include/internal/pycore_import.h b/Include/internal/pycore_import.h index eb8a9a0db46c22..08af53258cde97 100644 --- a/Include/internal/pycore_import.h +++ b/Include/internal/pycore_import.h @@ -22,6 +22,7 @@ extern int _PyImport_SetModuleString(const char *name, PyObject* module); extern void _PyImport_AcquireLock(PyInterpreterState *interp); extern int _PyImport_ReleaseLock(PyInterpreterState *interp); +// This is used exclusively for the sys and builtins modules: extern int _PyImport_FixupBuiltin( PyObject *mod, const char *name, /* UTF-8 encoded string */ From 08eb45093c9c6181fbb2e282f2e8199bb463f5e0 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Wed, 17 Apr 2024 18:25:23 -0600 Subject: [PATCH 6/9] Clean up _PyImport_FixupBuiltin(). --- Python/import.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/Python/import.c b/Python/import.c index 7394c2b49b14d6..8a08a4d87029f3 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1336,11 +1336,20 @@ int _PyImport_FixupBuiltin(PyObject *mod, const char *name, PyObject *modules) { int res = -1; + assert(mod != NULL && PyModule_Check(mod)); + PyObject *nameobj; nameobj = PyUnicode_InternFromString(name); if (nameobj == NULL) { return -1; } + + PyModuleDef *def = PyModule_GetDef(mod); + if (def == NULL) { + PyErr_BadInternalCall(); + goto finally; + } + if (PyObject_SetItem(modules, nameobj, mod) < 0) { goto finally; } @@ -1348,6 +1357,7 @@ _PyImport_FixupBuiltin(PyObject *mod, const char *name, PyObject *modules) PyMapping_DelItem(modules, nameobj); goto finally; } + res = 0; finally: From d841ae8f618570e7b4dc79497c720e382500e61b Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 22 Apr 2024 13:38:59 -0600 Subject: [PATCH 7/9] Clean up create_builtin(). --- Python/import.c | 60 +++++++++++++++++++++++++++---------------------- 1 file changed, 33 insertions(+), 27 deletions(-) diff --git a/Python/import.c b/Python/import.c index 8a08a4d87029f3..384bc736b42651 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1392,39 +1392,45 @@ create_builtin(PyThreadState *tstate, PyObject *name, PyObject *spec) } PyObject *modules = MODULES(tstate->interp); + struct _inittab *found = NULL; for (struct _inittab *p = INITTAB; p->name != NULL; p++) { if (_PyUnicode_EqualToASCIIString(name, p->name)) { - if (p->initfunc == NULL) { - /* Cannot re-init internal module ("sys" or "builtins") */ - return import_add_module(tstate, name); - } - mod = (*p->initfunc)(); - if (mod == NULL) { - return NULL; - } + found = p; + } + } + if (found == NULL) { + // not found + Py_RETURN_NONE; + } - if (PyObject_TypeCheck(mod, &PyModuleDef_Type)) { - return PyModule_FromDefAndSpec((PyModuleDef*)mod, spec); - } - else { - /* Remember pointer to module init function. */ - PyModuleDef *def = PyModule_GetDef(mod); - if (def == NULL) { - return NULL; - } + PyModInitFunction p0 = (PyModInitFunction)found->initfunc; + if (p0 == NULL) { + /* Cannot re-init internal module ("sys" or "builtins") */ + assert(is_core_module(tstate->interp, name, name)); + return import_add_module(tstate, name); + } - def->m_base.m_init = p->initfunc; - if (_PyImport_FixupExtensionObject(mod, name, name, - modules) < 0) { - return NULL; - } - return mod; - } - } + mod = p0(); + if (mod == NULL) { + return NULL; } - // not found - Py_RETURN_NONE; + if (PyObject_TypeCheck(mod, &PyModuleDef_Type)) { + return PyModule_FromDefAndSpec((PyModuleDef*)mod, spec); + } + else { + /* Remember pointer to module init function. */ + PyModuleDef *def = PyModule_GetDef(mod); + if (def == NULL) { + return NULL; + } + + def->m_base.m_init = p0; + if (_PyImport_FixupExtensionObject(mod, name, name, modules) < 0) { + return NULL; + } + return mod; + } } From 88ff926b212b7bb6ce3301ed6696af3faea6398a Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Fri, 19 Apr 2024 14:20:48 -0600 Subject: [PATCH 8/9] Use "filename" and "path" variable names correctly. --- Python/import.c | 44 ++++++++++++++++++++++---------------------- Python/importdl.c | 37 ++++++++++++++++++++----------------- 2 files changed, 42 insertions(+), 39 deletions(-) diff --git a/Python/import.c b/Python/import.c index 384bc736b42651..d2e623f7469227 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1125,10 +1125,10 @@ _PyImport_CheckSubinterpIncompatibleExtensionAllowed(const char *name) static PyObject * get_core_module_dict(PyInterpreterState *interp, - PyObject *name, PyObject *filename) + PyObject *name, PyObject *path) { /* Only builtin modules are core. */ - if (filename == name) { + if (path == name) { assert(!PyErr_Occurred()); if (PyUnicode_CompareWithASCIIString(name, "sys") == 0) { return interp->sysdict_copy; @@ -1143,11 +1143,11 @@ get_core_module_dict(PyInterpreterState *interp, } static inline int -is_core_module(PyInterpreterState *interp, PyObject *name, PyObject *filename) +is_core_module(PyInterpreterState *interp, PyObject *name, PyObject *path) { /* This might be called before the core dict copies are in place, so we can't rely on get_core_module_dict() here. */ - if (filename == name) { + if (path == name) { if (PyUnicode_CompareWithASCIIString(name, "sys") == 0) { return 1; } @@ -1159,7 +1159,7 @@ is_core_module(PyInterpreterState *interp, PyObject *name, PyObject *filename) } static int -fix_up_extension(PyObject *mod, PyObject *name, PyObject *filename) +fix_up_extension(PyObject *mod, PyObject *name, PyObject *path) { if (mod == NULL || !PyModule_Check(mod)) { PyErr_BadInternalCall(); @@ -1180,7 +1180,7 @@ fix_up_extension(PyObject *mod, PyObject *name, PyObject *filename) // bpo-44050: Extensions and def->m_base.m_copy can be updated // when the extension module doesn't support sub-interpreters. if (def->m_size == -1) { - if (!is_core_module(tstate->interp, name, filename)) { + if (!is_core_module(tstate->interp, name, path)) { assert(PyUnicode_CompareWithASCIIString(name, "sys") != 0); assert(PyUnicode_CompareWithASCIIString(name, "builtins") != 0); if (def->m_base.m_copy) { @@ -1202,7 +1202,7 @@ fix_up_extension(PyObject *mod, PyObject *name, PyObject *filename) // XXX Why special-case the main interpreter? if (_Py_IsMainInterpreter(tstate->interp) || def->m_size == -1) { - if (_extensions_cache_set(filename, name, def) < 0) { + if (_extensions_cache_set(path, name, def) < 0) { return -1; } } @@ -1227,10 +1227,10 @@ _PyImport_FixupExtensionObject(PyObject *mod, PyObject *name, static PyObject * import_find_extension(PyThreadState *tstate, PyObject *name, - PyObject *filename) + PyObject *path) { /* Only single-phase init modules will be in the cache. */ - PyModuleDef *def = _extensions_cache_get(filename, name); + PyModuleDef *def = _extensions_cache_get(path, name); if (def == NULL) { return NULL; } @@ -1253,7 +1253,7 @@ import_find_extension(PyThreadState *tstate, PyObject *name, if (m_copy == NULL) { /* It might be a core module (e.g. sys & builtins), for which we don't set m_copy. */ - m_copy = get_core_module_dict(tstate->interp, name, filename); + m_copy = get_core_module_dict(tstate->interp, name, path); if (m_copy == NULL) { return NULL; } @@ -1292,16 +1292,16 @@ import_find_extension(PyThreadState *tstate, PyObject *name, int verbose = _PyInterpreterState_GetConfig(tstate->interp)->verbose; if (verbose) { PySys_FormatStderr("import %U # previously loaded (%R)\n", - name, filename); + name, path); } return mod; } static int clear_singlephase_extension(PyInterpreterState *interp, - PyObject *name, PyObject *filename) + PyObject *name, PyObject *path) { - PyModuleDef *def = _extensions_cache_get(filename, name); + PyModuleDef *def = _extensions_cache_get(path, name); if (def == NULL) { if (PyErr_Occurred()) { return -1; @@ -1322,7 +1322,7 @@ clear_singlephase_extension(PyInterpreterState *interp, } /* Clear the cached module def. */ - _extensions_cache_delete(filename, name); + _extensions_cache_delete(path, name); return 0; } @@ -3740,7 +3740,7 @@ static PyObject * _imp_create_dynamic_impl(PyObject *module, PyObject *spec, PyObject *file) /*[clinic end generated code: output=83249b827a4fde77 input=c31b954f4cf4e09d]*/ { - PyObject *mod, *name, *path; + PyObject *mod, *name, *filename; FILE *fp; name = PyObject_GetAttrString(spec, "name"); @@ -3748,20 +3748,20 @@ _imp_create_dynamic_impl(PyObject *module, PyObject *spec, PyObject *file) return NULL; } - path = PyObject_GetAttrString(spec, "origin"); - if (path == NULL) { + filename = PyObject_GetAttrString(spec, "origin"); + if (filename == NULL) { Py_DECREF(name); return NULL; } PyThreadState *tstate = _PyThreadState_GET(); - mod = import_find_extension(tstate, name, path); + mod = import_find_extension(tstate, name, filename); if (mod != NULL || _PyErr_Occurred(tstate)) { assert(mod == NULL || !_PyErr_Occurred(tstate)); goto finally; } - if (PySys_Audit("import", "OOOOO", name, path, + if (PySys_Audit("import", "OOOOO", name, filename, Py_None, Py_None, Py_None) < 0) { goto finally; @@ -3770,7 +3770,7 @@ _imp_create_dynamic_impl(PyObject *module, PyObject *spec, PyObject *file) /* Is multi-phase init or this is the first time being loaded. */ if (file != NULL) { - fp = _Py_fopen_obj(path, "r"); + fp = _Py_fopen_obj(filename, "r"); if (fp == NULL) { goto finally; } @@ -3782,7 +3782,7 @@ _imp_create_dynamic_impl(PyObject *module, PyObject *spec, PyObject *file) mod = _PyImport_LoadDynamicModuleWithSpec(spec, fp); if (mod != NULL) { /* Remember the filename as the __file__ attribute */ - if (PyModule_AddObjectRef(mod, "__file__", path) < 0) { + if (PyModule_AddObjectRef(mod, "__file__", filename) < 0) { PyErr_Clear(); /* Not important enough to report */ } } @@ -3794,7 +3794,7 @@ _imp_create_dynamic_impl(PyObject *module, PyObject *spec, PyObject *file) finally: Py_DECREF(name); - Py_DECREF(path); + Py_DECREF(filename); return mod; } diff --git a/Python/importdl.c b/Python/importdl.c index bc15113af7332d..7cf30bea3a861a 100644 --- a/Python/importdl.c +++ b/Python/importdl.c @@ -97,9 +97,10 @@ PyObject * _PyImport_LoadDynamicModuleWithSpec(PyObject *spec, FILE *fp) { #ifndef MS_WINDOWS - PyObject *pathbytes = NULL; + PyObject *filename_bytes = NULL; + const char *filename_buf; #endif - PyObject *name_unicode = NULL, *name = NULL, *path = NULL, *m = NULL; + PyObject *name_unicode = NULL, *name = NULL, *filename = NULL, *m = NULL; const char *name_buf, *hook_prefix; const char *oldcontext, *newcontext; dl_funcptr exportfunc; @@ -126,21 +127,23 @@ _PyImport_LoadDynamicModuleWithSpec(PyObject *spec, FILE *fp) } name_buf = PyBytes_AS_STRING(name); - path = PyObject_GetAttrString(spec, "origin"); - if (path == NULL) + filename = PyObject_GetAttrString(spec, "origin"); + if (filename == NULL) { goto error; + } #ifdef MS_WINDOWS - exportfunc = _PyImport_FindSharedFuncptrWindows(hook_prefix, name_buf, - path, fp); + exportfunc = _PyImport_FindSharedFuncptrWindows( + hook_prefix, name_buf, filename, fp); #else - pathbytes = PyUnicode_EncodeFSDefault(path); - if (pathbytes == NULL) + filename_bytes = PyUnicode_EncodeFSDefault(filename); + if (filename_bytes == NULL) { goto error; - exportfunc = _PyImport_FindSharedFuncptr(hook_prefix, name_buf, - PyBytes_AS_STRING(pathbytes), - fp); - Py_DECREF(pathbytes); + } + filename_buf = PyBytes_AS_STRING(filename_bytes); + exportfunc = _PyImport_FindSharedFuncptr( + hook_prefix, name_buf, filename_buf, fp); + Py_DECREF(filename_bytes); #endif if (exportfunc == NULL) { @@ -152,7 +155,7 @@ _PyImport_LoadDynamicModuleWithSpec(PyObject *spec, FILE *fp) hook_prefix, name_buf); if (msg == NULL) goto error; - PyErr_SetImportError(msg, name_unicode, path); + PyErr_SetImportError(msg, name_unicode, filename); Py_DECREF(msg); } goto error; @@ -194,7 +197,7 @@ _PyImport_LoadDynamicModuleWithSpec(PyObject *spec, FILE *fp) if (PyObject_TypeCheck(m, &PyModuleDef_Type)) { Py_DECREF(name_unicode); Py_DECREF(name); - Py_DECREF(path); + Py_DECREF(filename); return PyModule_FromDefAndSpec((PyModuleDef*)m, spec); } @@ -224,19 +227,19 @@ _PyImport_LoadDynamicModuleWithSpec(PyObject *spec, FILE *fp) def->m_base.m_init = p0; PyObject *modules = PyImport_GetModuleDict(); - if (_PyImport_FixupExtensionObject(m, name_unicode, path, modules) < 0) + if (_PyImport_FixupExtensionObject(m, name_unicode, filename, modules) < 0) goto error; Py_DECREF(name_unicode); Py_DECREF(name); - Py_DECREF(path); + Py_DECREF(filename); return m; error: Py_DECREF(name_unicode); Py_XDECREF(name); - Py_XDECREF(path); + Py_XDECREF(filename); Py_XDECREF(m); return NULL; } From be500203a50368f484d1f0bb8e76a4a0036b3205 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 16 Apr 2024 15:04:56 -0600 Subject: [PATCH 9/9] Add a comment. --- Python/import.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Python/import.c b/Python/import.c index d2e623f7469227..8cdc04f03dd201 100644 --- a/Python/import.c +++ b/Python/import.c @@ -3769,6 +3769,9 @@ _imp_create_dynamic_impl(PyObject *module, PyObject *spec, PyObject *file) /* Is multi-phase init or this is the first time being loaded. */ + /* We would move this (and the fclose() below) into + * _PyImport_GetModInitFunc(), but it isn't clear if the intervening + * code relies on fp still being open. */ if (file != NULL) { fp = _Py_fopen_obj(filename, "r"); if (fp == NULL) {